From 45f61d87d753b8b318862d0de0a4525f479222dd Mon Sep 17 00:00:00 2001 From: Charlotte Weaver Date: Wed, 18 Jul 2018 21:45:07 -0700 Subject: [PATCH 01/12] Initial CLI added basic cli for running --- server/app/app.py | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/server/app/app.py b/server/app/app.py index 4cd08ce4..d08446c9 100644 --- a/server/app/app.py +++ b/server/app/app.py @@ -1,4 +1,5 @@ import os +import argparse from flask import Flask from flask_compress import Compress @@ -35,9 +36,7 @@ app.config["PROFILE"] = True # Application Data data = None -if app.config["ENGINE"] == "scanpy": - from .scanpy_engine.scanpy_engine import ScanpyEngine - data = ScanpyEngine(app.config["DATA"], schema="data_schema.json") + # A list of swagger document objects docs = [] @@ -53,5 +52,28 @@ app.register_blueprint( app.add_url_rule("/", endpoint="index") -def main(): +def run(args): + global data + app.config.update( + ENGINE=args.engine, + DATA=args.data_directory + ) + if app.config["ENGINE"] == "scanpy": + from .scanpy_engine.scanpy_engine import ScanpyEngine + data = ScanpyEngine(app.config["DATA"], schema="data_schema.json") + app.run(host="0.0.0.0", debug=True, port=5005) + + +def main(): + parser = argparse.ArgumentParser(description="AAAAA") + subparsers = parser.add_subparsers(dest="cellxgene_command") + run_subparser = subparsers.add_parser('run', help="run cellxgene") + run_subparser.add_argument('engine', metavar='engine', help='Format that the backend uses for data') + run_subparser.add_argument('data_directory', metavar='dir', + help='Directory containing data and schema') + run_subparser.set_defaults(func=run) + args = parser.parse_args() + args.func(args) + + # app.run(host="0.0.0.0", debug=True, port=5005) From 2c7a11ab80f978e69687dd2f0db3c2839f9730a2 Mon Sep 17 00:00:00 2001 From: Charlotte Weaver Date: Thu, 19 Jul 2018 17:21:28 -0700 Subject: [PATCH 02/12] Added cli command for run --- server/app/app.py | 39 ++++++++++++++++----------------------- 1 file changed, 16 insertions(+), 23 deletions(-) diff --git a/server/app/app.py b/server/app/app.py index d08446c9..68d7b4b9 100644 --- a/server/app/app.py +++ b/server/app/app.py @@ -16,24 +16,12 @@ Compress(app) CORS(app) # Config -CXG_DIR = os.environ.get("CXG_DIRECTORY", default="example-dataset/") SECRET_KEY = os.environ.get("CXG_SECRET_KEY", default="SparkleAndShine") -ENGINE = os.environ.get("CXG_ENGINE", default="scanpy") -TITLE = os.environ.get("DATASET_TITLE", default="PBMC 3K") -# TODO remove the 2 when this is prod -CXG_API_BASE = os.environ.get("CXG_API_BASE2", default="http://0.0.0.0:5005/api/") app.config.update( SECRET_KEY=SECRET_KEY, - CXG_API_BASE=CXG_API_BASE, - ENGINE=ENGINE, - DATA=CXG_DIR, - DATASET_TITLE=TITLE ) -app.config["PROFILE"] = True -# app.wsgi_app = ProfilerMiddleware(app.wsgi_app, restrictions=[15]) - # Application Data data = None @@ -54,26 +42,31 @@ app.add_url_rule("/", endpoint="index") def run(args): global data + title = args.title + if not title: + title = os.path.basename(os.path.normpath(args.data_directory)) + api_base = f"http://0.0.0.0:{args.port}/api/" app.config.update( - ENGINE=args.engine, - DATA=args.data_directory + DATASET_TITLE=title, + CXG_API_BASE=api_base ) - if app.config["ENGINE"] == "scanpy": + if args.engine == "scanpy": from .scanpy_engine.scanpy_engine import ScanpyEngine - data = ScanpyEngine(app.config["DATA"], schema="data_schema.json") - - app.run(host="0.0.0.0", debug=True, port=5005) + data = ScanpyEngine(args.data_directory, schema="data_schema.json") + app.run(host="0.0.0.0", debug=True, port=args.port) def main(): - parser = argparse.ArgumentParser(description="AAAAA") + parser = argparse.ArgumentParser(description="Cellxgene is a tool for exploring single cell expression.") subparsers = parser.add_subparsers(dest="cellxgene_command") - run_subparser = subparsers.add_parser('run', help="run cellxgene") - run_subparser.add_argument('engine', metavar='engine', help='Format that the backend uses for data') - run_subparser.add_argument('data_directory', metavar='dir', - help='Directory containing data and schema') + run_subparser = subparsers.add_parser("run", help="run cellxgene") + run_subparser.add_argument("--title", "-t", help="Title to display (default = data directory name") + run_subparser.add_argument("--port", help="Port to run server on.", type=int, default=5005) + run_subparser.add_argument("engine", help="The underlying structure of the data") + run_subparser.add_argument("data_directory", metavar="dir", help="Directory containing data and schema") run_subparser.set_defaults(func=run) args = parser.parse_args() + args.func(args) # app.run(host="0.0.0.0", debug=True, port=5005) From 9c72f2c2d5c7fdf212fe52af1fea1c943e231e3f Mon Sep 17 00:00:00 2001 From: Charlotte Weaver Date: Thu, 19 Jul 2018 17:23:30 -0700 Subject: [PATCH 03/12] Updating travis file to match new cli --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6fb222c1..8a289a72 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,6 @@ script: - set -eo pipefail - flake8 server/app/ - pytest -s server/test/test_filter.py server/test/test_scanpy_engine.py - - cellxgene & + - cellxgene run scanpy example-dataset/ & - for i in {1..90}; do if http :5005/api/v0.1/initialize > /dev/null; then break; else echo "Waiting for server..."; sleep 1; fi; done - pytest server/test/test_api.py From de5e0f4ea7d55207c3d79482a848ee4981570f65 Mon Sep 17 00:00:00 2001 From: Charlotte Weaver Date: Fri, 20 Jul 2018 11:43:31 -0700 Subject: [PATCH 04/12] Address PR review issues --- server/app/app.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/server/app/app.py b/server/app/app.py index 68d7b4b9..d9e8e2ce 100644 --- a/server/app/app.py +++ b/server/app/app.py @@ -1,5 +1,5 @@ -import os import argparse +import os from flask import Flask from flask_compress import Compress @@ -40,7 +40,7 @@ app.register_blueprint( app.add_url_rule("/", endpoint="index") -def run(args): +def run_scanpy(args): global data title = args.title if not title: @@ -50,23 +50,20 @@ def run(args): DATASET_TITLE=title, CXG_API_BASE=api_base ) - if args.engine == "scanpy": - from .scanpy_engine.scanpy_engine import ScanpyEngine - data = ScanpyEngine(args.data_directory, schema="data_schema.json") + + from .scanpy_engine.scanpy_engine import ScanpyEngine + data = ScanpyEngine(args.data_directory, schema="data_schema.json") app.run(host="0.0.0.0", debug=True, port=args.port) def main(): parser = argparse.ArgumentParser(description="Cellxgene is a tool for exploring single cell expression.") + parser.add_argument("--title", "-t", help="Title to display -- if this is omitted the title will be the name " + "of the directory from the data_directory arg") + parser.add_argument("--port", help="Port to run server on.", type=int, default=5005) subparsers = parser.add_subparsers(dest="cellxgene_command") - run_subparser = subparsers.add_parser("run", help="run cellxgene") - run_subparser.add_argument("--title", "-t", help="Title to display (default = data directory name") - run_subparser.add_argument("--port", help="Port to run server on.", type=int, default=5005) - run_subparser.add_argument("engine", help="The underlying structure of the data") - run_subparser.add_argument("data_directory", metavar="dir", help="Directory containing data and schema") - run_subparser.set_defaults(func=run) + scanpy_subparser = subparsers.add_parser("scanpy", help="run cellxgene using the scanpy engine") + scanpy_subparser.add_argument("data_directory", metavar="dir", help="Directory containing data and schema file") + scanpy_subparser.set_defaults(func=run_scanpy) args = parser.parse_args() - args.func(args) - - # app.run(host="0.0.0.0", debug=True, port=5005) From 83e7685f570bc8047ff4dc42b86741f7b9e5a8ea Mon Sep 17 00:00:00 2001 From: Charlotte Weaver Date: Fri, 20 Jul 2018 11:47:42 -0700 Subject: [PATCH 05/12] Update readme for CLI --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 43c249ed..425004a8 100644 --- a/README.md +++ b/README.md @@ -30,9 +30,15 @@ A React + Redux web application for exploring large scale single cell RNA sequen python3 setup.py install +#### commandline help + + cellxgene --help + # For help with the scanpy engine + cellxgene scanpy --help + #### run (with demo data) - cellxgene + cellxgene --title PBMC3K scanpy example-dataset/ *Thanks to Alex Wolf his help with test data* From d77accc8586b811540eb445533562ad737e1de3a Mon Sep 17 00:00:00 2001 From: Charlotte Weaver Date: Fri, 20 Jul 2018 13:15:17 -0700 Subject: [PATCH 06/12] Update travis for CLI --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8a289a72..00b5150b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,6 @@ script: - set -eo pipefail - flake8 server/app/ - pytest -s server/test/test_filter.py server/test/test_scanpy_engine.py - - cellxgene run scanpy example-dataset/ & + - cellxgene scanpy example-dataset/ & - for i in {1..90}; do if http :5005/api/v0.1/initialize > /dev/null; then break; else echo "Waiting for server..."; sleep 1; fi; done - pytest server/test/test_api.py From 7642251d446b40c1966cb33fe2fa71a7663e3c3f Mon Sep 17 00:00:00 2001 From: Charlotte Weaver Date: Wed, 1 Aug 2018 13:32:04 -0700 Subject: [PATCH 07/12] Add simple cache to backend --- server/app/app.py | 2 ++ server/app/scanpy_engine/scanpy_engine.py | 7 +++++++ server/requirements.txt | 1 + 3 files changed, 10 insertions(+) diff --git a/server/app/app.py b/server/app/app.py index d9e8e2ce..397c71f5 100644 --- a/server/app/app.py +++ b/server/app/app.py @@ -2,6 +2,7 @@ import argparse import os from flask import Flask +from flask.ext.cache import Cache from flask_compress import Compress from flask_cors import CORS from flask_restful_swagger_2 import get_swagger_blueprint @@ -12,6 +13,7 @@ from .rest_api.rest import get_api_resources REACTIVE_LIMIT = 1_000_000 app = Flask(__name__) +cache = Cache(app, config={"CACHE_TYPE": "simple"}) Compress(app) CORS(app) diff --git a/server/app/scanpy_engine/scanpy_engine.py b/server/app/scanpy_engine/scanpy_engine.py index 405003f5..0b5ec9ca 100644 --- a/server/app/scanpy_engine/scanpy_engine.py +++ b/server/app/scanpy_engine/scanpy_engine.py @@ -6,6 +6,7 @@ from scipy import stats from ..util.schema_parse import parse_schema from ..driver.driver import CXGDriver +from server.app.app import cache class ScanpyEngine(CXGDriver): @@ -41,6 +42,7 @@ class ScanpyEngine(CXGDriver): def genes(self): return self.data.var.index.tolist() + # Can't seem to cache a view of a dataframe, need to investigate why def filter_cells(self, filter): """ Filter cells from data and return a subset of the data @@ -69,6 +71,7 @@ class ScanpyEngine(CXGDriver): cell_idx = np.logical_and(cell_idx, key_idx) return self.data[cell_idx, :] + @cache.memoize(86400000) def metadata_ranges(self, df=None): metadata_ranges = {} if not df: @@ -88,6 +91,7 @@ class ScanpyEngine(CXGDriver): } return metadata_ranges + @cache.memoize(86400000) def metadata(self, df, fields=None): """ Gets metadata key:value for each cells @@ -101,6 +105,7 @@ class ScanpyEngine(CXGDriver): metadata[idx]["CellName"] = metadata[idx].pop("cell_name", None) return metadata + @cache.memoize(86400000) def create_graph(self, df): """ Computes a n-d layout for cells through dimensionality reduction. @@ -112,6 +117,7 @@ class ScanpyEngine(CXGDriver): normalized_graph = (graph - graph.min()) / (graph.max() - graph.min()) return np.hstack((df.obs["cell_name"].values.reshape(len(df.obs.index), 1), normalized_graph)).tolist() + @cache.memoize(86400000) def diffexp(self, cell_list_1, cell_list_2, pval, num_genes): """ Computes the top differentially expressed genes between two clusters @@ -158,6 +164,7 @@ class ScanpyEngine(CXGDriver): }, } + @cache.memoize(86400000) def expression(self, cells=None, genes=None): """ Retrieves expression for each gene for cells in data frame diff --git a/server/requirements.txt b/server/requirements.txt index 7ab96abb..d12edc0a 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -6,6 +6,7 @@ click==6.7 cycler==0.10.0 decorator==4.3.0 Flask==0.12.4 +Flask-Cache==0.13.1 Flask-Compress==1.4.0 Flask-Cors==3.0.6 Flask-RESTful==0.3.6 From 69fca7ea8b2877c80b762b0167d1ca4460410732 Mon Sep 17 00:00:00 2001 From: Charlotte Weaver Date: Wed, 1 Aug 2018 15:46:04 -0700 Subject: [PATCH 08/12] Using flask_caching instead More recent module, same API --- server/app/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/app/app.py b/server/app/app.py index 397c71f5..1e41e04a 100644 --- a/server/app/app.py +++ b/server/app/app.py @@ -2,7 +2,7 @@ import argparse import os from flask import Flask -from flask.ext.cache import Cache +from flask_caching import Cache from flask_compress import Compress from flask_cors import CORS from flask_restful_swagger_2 import get_swagger_blueprint From 0a36fac14209205cde199a0a1c8b505fe8a4a1a2 Mon Sep 17 00:00:00 2001 From: Charlotte Weaver Date: Wed, 1 Aug 2018 15:48:56 -0700 Subject: [PATCH 09/12] 10 day cache (down from 1000) I can expect a scientist to leave this running over the weekend on their laptop and still expect fast results. If someone leaves it running for a few years the data can probably be safely recalculated. --- server/app/scanpy_engine/scanpy_engine.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/app/scanpy_engine/scanpy_engine.py b/server/app/scanpy_engine/scanpy_engine.py index 0b5ec9ca..54491480 100644 --- a/server/app/scanpy_engine/scanpy_engine.py +++ b/server/app/scanpy_engine/scanpy_engine.py @@ -71,7 +71,7 @@ class ScanpyEngine(CXGDriver): cell_idx = np.logical_and(cell_idx, key_idx) return self.data[cell_idx, :] - @cache.memoize(86400000) + @cache.memoize(864000) def metadata_ranges(self, df=None): metadata_ranges = {} if not df: @@ -91,7 +91,7 @@ class ScanpyEngine(CXGDriver): } return metadata_ranges - @cache.memoize(86400000) + @cache.memoize(864000) def metadata(self, df, fields=None): """ Gets metadata key:value for each cells @@ -105,7 +105,7 @@ class ScanpyEngine(CXGDriver): metadata[idx]["CellName"] = metadata[idx].pop("cell_name", None) return metadata - @cache.memoize(86400000) + @cache.memoize(864000) def create_graph(self, df): """ Computes a n-d layout for cells through dimensionality reduction. @@ -117,7 +117,7 @@ class ScanpyEngine(CXGDriver): normalized_graph = (graph - graph.min()) / (graph.max() - graph.min()) return np.hstack((df.obs["cell_name"].values.reshape(len(df.obs.index), 1), normalized_graph)).tolist() - @cache.memoize(86400000) + @cache.memoize(864000) def diffexp(self, cell_list_1, cell_list_2, pval, num_genes): """ Computes the top differentially expressed genes between two clusters @@ -164,7 +164,7 @@ class ScanpyEngine(CXGDriver): }, } - @cache.memoize(86400000) + @cache.memoize(864000) def expression(self, cells=None, genes=None): """ Retrieves expression for each gene for cells in data frame From 489a2dd5dbd2534fe96bf5b666de1f5a032da87a Mon Sep 17 00:00:00 2001 From: Charlotte Weaver Date: Wed, 1 Aug 2018 16:29:38 -0700 Subject: [PATCH 10/12] Add flask-caching to requirements --- server/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/requirements.txt b/server/requirements.txt index d12edc0a..a881e310 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -6,7 +6,7 @@ click==6.7 cycler==0.10.0 decorator==4.3.0 Flask==0.12.4 -Flask-Cache==0.13.1 +Flask-Caching==1.4.0 Flask-Compress==1.4.0 Flask-Cors==3.0.6 Flask-RESTful==0.3.6 From 116cef551ca98f071c50c09b99b3c3059470f32c Mon Sep 17 00:00:00 2001 From: Charlotte Weaver Date: Thu, 2 Aug 2018 11:01:38 -0700 Subject: [PATCH 11/12] Switch to default timeout for cache --- server/app/app.py | 2 +- server/app/scanpy_engine/scanpy_engine.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/server/app/app.py b/server/app/app.py index 1e41e04a..36c2f707 100644 --- a/server/app/app.py +++ b/server/app/app.py @@ -13,7 +13,7 @@ from .rest_api.rest import get_api_resources REACTIVE_LIMIT = 1_000_000 app = Flask(__name__) -cache = Cache(app, config={"CACHE_TYPE": "simple"}) +cache = Cache(app, config={"CACHE_TYPE": "simple", "CACHE_DEFAULT_TIMEOUT": 860000}) Compress(app) CORS(app) diff --git a/server/app/scanpy_engine/scanpy_engine.py b/server/app/scanpy_engine/scanpy_engine.py index 54491480..99334e87 100644 --- a/server/app/scanpy_engine/scanpy_engine.py +++ b/server/app/scanpy_engine/scanpy_engine.py @@ -71,7 +71,7 @@ class ScanpyEngine(CXGDriver): cell_idx = np.logical_and(cell_idx, key_idx) return self.data[cell_idx, :] - @cache.memoize(864000) + @cache.memoize() def metadata_ranges(self, df=None): metadata_ranges = {} if not df: @@ -91,7 +91,7 @@ class ScanpyEngine(CXGDriver): } return metadata_ranges - @cache.memoize(864000) + @cache.memoize() def metadata(self, df, fields=None): """ Gets metadata key:value for each cells @@ -105,7 +105,7 @@ class ScanpyEngine(CXGDriver): metadata[idx]["CellName"] = metadata[idx].pop("cell_name", None) return metadata - @cache.memoize(864000) + @cache.memoize() def create_graph(self, df): """ Computes a n-d layout for cells through dimensionality reduction. @@ -117,7 +117,7 @@ class ScanpyEngine(CXGDriver): normalized_graph = (graph - graph.min()) / (graph.max() - graph.min()) return np.hstack((df.obs["cell_name"].values.reshape(len(df.obs.index), 1), normalized_graph)).tolist() - @cache.memoize(864000) + @cache.memoize() def diffexp(self, cell_list_1, cell_list_2, pval, num_genes): """ Computes the top differentially expressed genes between two clusters @@ -164,7 +164,7 @@ class ScanpyEngine(CXGDriver): }, } - @cache.memoize(864000) + @cache.memoize() def expression(self, cells=None, genes=None): """ Retrieves expression for each gene for cells in data frame From e11b90583659fd0c44bf0c99df6e4b96a4d0d2be Mon Sep 17 00:00:00 2001 From: Charlotte Weaver Date: Thu, 2 Aug 2018 11:33:47 -0700 Subject: [PATCH 12/12] Cleaning up import statements --- server/app/app.py | 2 +- server/app/rest_api/rest.py | 4 ++-- server/app/scanpy_engine/scanpy_engine.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/app/app.py b/server/app/app.py index 36c2f707..547a9db5 100644 --- a/server/app/app.py +++ b/server/app/app.py @@ -7,8 +7,8 @@ from flask_compress import Compress from flask_cors import CORS from flask_restful_swagger_2 import get_swagger_blueprint -from .web import webapp from .rest_api.rest import get_api_resources +from .web import webapp REACTIVE_LIMIT = 1_000_000 diff --git a/server/app/rest_api/rest.py b/server/app/rest_api/rest.py index 40742dc3..f5892125 100644 --- a/server/app/rest_api/rest.py +++ b/server/app/rest_api/rest.py @@ -3,8 +3,8 @@ from flask import ( ) from flask_restful_swagger_2 import Api, swagger, Resource -from ..util.utils import make_payload -from ..util.filter import parse_filter +from server.app.util.utils import make_payload +from server.app.util.filter import parse_filter class InitializeAPI(Resource): diff --git a/server/app/scanpy_engine/scanpy_engine.py b/server/app/scanpy_engine/scanpy_engine.py index 99334e87..9ede3b45 100644 --- a/server/app/scanpy_engine/scanpy_engine.py +++ b/server/app/scanpy_engine/scanpy_engine.py @@ -4,9 +4,9 @@ import numpy as np import scanpy.api as sc from scipy import stats -from ..util.schema_parse import parse_schema -from ..driver.driver import CXGDriver from server.app.app import cache +from server.app.driver.driver import CXGDriver +from server.app.util.schema_parse import parse_schema class ScanpyEngine(CXGDriver):