Merge pull request #121 from chanzuckerberg/csweaver/cli

Csweaver/cli

.travis.yml

@@ -14,6 +14,6 @@ script:
   - set -eo pipefail
   - flake8 server/app/
   - pytest -s server/test/test_filter.py server/test/test_scanpy_engine.py
-  - cellxgene &
+  - cellxgene scanpy example-dataset/ &
   - for i in {1..90}; do if http :5005/api/v0.1/initialize > /dev/null; then break; else echo "Waiting for server..."; sleep 1; fi; done
   - pytest server/test/test_api.py

README.md

@@ -30,9 +30,15 @@ A React + Redux web application for exploring large scale single cell RNA sequen
 
     python3 setup.py install
 
+#### commandline help
+
+    cellxgene --help
+    # For help with the scanpy engine
+    cellxgene scanpy --help
+
 #### run (with demo data)
 
-    cellxgene
+    cellxgene --title PBMC3K scanpy example-dataset/
    
    
 *Thanks to Alex Wolf his help with test data*

server/app/app.py

@@ -1,43 +1,32 @@
+import argparse
 import os
 
 from flask import Flask
+from flask_caching import Cache
 from flask_compress import Compress
 from flask_cors import CORS
 from flask_restful_swagger_2 import get_swagger_blueprint
 
-from .web import webapp
 from .rest_api.rest import get_api_resources
+from .web import webapp
 
 REACTIVE_LIMIT = 1_000_000
 
 app = Flask(__name__)
+cache = Cache(app, config={"CACHE_TYPE": "simple", "CACHE_DEFAULT_TIMEOUT": 860000})
 Compress(app)
 CORS(app)
 
 # Config
-CXG_DIR = os.environ.get("CXG_DIRECTORY", default="example-dataset/")
 SECRET_KEY = os.environ.get("CXG_SECRET_KEY", default="SparkleAndShine")
-ENGINE = os.environ.get("CXG_ENGINE", default="scanpy")
-TITLE = os.environ.get("DATASET_TITLE", default="PBMC 3K")
-# TODO remove the 2 when this is prod
-CXG_API_BASE = os.environ.get("CXG_API_BASE2", default="http://0.0.0.0:5005/api/")
 
 app.config.update(
     SECRET_KEY=SECRET_KEY,
-    CXG_API_BASE=CXG_API_BASE,
-    ENGINE=ENGINE,
-    DATA=CXG_DIR,
-    DATASET_TITLE=TITLE
 )
 
-app.config["PROFILE"] = True
-# app.wsgi_app = ProfilerMiddleware(app.wsgi_app, restrictions=[15])
-
 # Application Data
 data = None
-if app.config["ENGINE"] == "scanpy":
+
-    from .scanpy_engine.scanpy_engine import ScanpyEngine
-    data = ScanpyEngine(app.config["DATA"], schema="data_schema.json")
 
 # A list of swagger document objects
 docs = []
@@ -53,5 +42,30 @@ app.register_blueprint(
 app.add_url_rule("/", endpoint="index")
 
 
+def run_scanpy(args):
+    global data
+    title = args.title
+    if not title:
+        title = os.path.basename(os.path.normpath(args.data_directory))
+    api_base = f"http://0.0.0.0:{args.port}/api/"
+    app.config.update(
+        DATASET_TITLE=title,
+        CXG_API_BASE=api_base
+    )
+
+    from .scanpy_engine.scanpy_engine import ScanpyEngine
+    data = ScanpyEngine(args.data_directory, schema="data_schema.json")
+    app.run(host="0.0.0.0", debug=True, port=args.port)
+
+
 def main():
-    app.run(host="0.0.0.0", debug=True, port=5005)
+    parser = argparse.ArgumentParser(description="Cellxgene is a tool for exploring single cell expression.")
+    parser.add_argument("--title", "-t", help="Title to display -- if this is omitted the title will be the name "
+                                              "of the directory from the data_directory arg")
+    parser.add_argument("--port", help="Port to run server on.", type=int, default=5005)
+    subparsers = parser.add_subparsers(dest="cellxgene_command")
+    scanpy_subparser = subparsers.add_parser("scanpy", help="run cellxgene using the scanpy engine")
+    scanpy_subparser.add_argument("data_directory", metavar="dir", help="Directory containing data and schema file")
+    scanpy_subparser.set_defaults(func=run_scanpy)
+    args = parser.parse_args()
+    args.func(args)

server/app/rest_api/rest.py

@@ -3,8 +3,8 @@ from flask import (
 )
 from flask_restful_swagger_2 import Api, swagger, Resource
 
-from ..util.utils import make_payload
+from server.app.util.utils import make_payload
-from ..util.filter import parse_filter
+from server.app.util.filter import parse_filter
 
 
 class InitializeAPI(Resource):

server/app/scanpy_engine/scanpy_engine.py

@@ -4,8 +4,9 @@ import numpy as np
 import scanpy.api as sc
 from scipy import stats
 
-from ..util.schema_parse import parse_schema
+from server.app.app import cache
-from ..driver.driver import CXGDriver
+from server.app.driver.driver import CXGDriver
+from server.app.util.schema_parse import parse_schema
 
 
 class ScanpyEngine(CXGDriver):
@@ -41,6 +42,7 @@ class ScanpyEngine(CXGDriver):
     def genes(self):
         return self.data.var.index.tolist()
 
+    # Can't seem to cache a view of a dataframe, need to investigate why
     def filter_cells(self, filter):
         """
         Filter cells from data and return a subset of the data
@@ -69,6 +71,7 @@ class ScanpyEngine(CXGDriver):
                     cell_idx = np.logical_and(cell_idx, key_idx)
         return self.data[cell_idx, :]
 
+    @cache.memoize()
     def metadata_ranges(self, df=None):
         metadata_ranges = {}
         if not df:
@@ -88,6 +91,7 @@ class ScanpyEngine(CXGDriver):
                 }
         return metadata_ranges
 
+    @cache.memoize()
     def metadata(self, df, fields=None):
         """
          Gets metadata key:value for each cells
@@ -101,6 +105,7 @@ class ScanpyEngine(CXGDriver):
             metadata[idx]["CellName"] = metadata[idx].pop("cell_name", None)
         return metadata
 
+    @cache.memoize()
     def create_graph(self, df):
         """
         Computes a n-d layout for cells through dimensionality reduction.
@@ -112,6 +117,7 @@ class ScanpyEngine(CXGDriver):
         normalized_graph = (graph - graph.min()) / (graph.max() - graph.min())
         return np.hstack((df.obs["cell_name"].values.reshape(len(df.obs.index), 1), normalized_graph)).tolist()
 
+    @cache.memoize()
     def diffexp(self, cell_list_1, cell_list_2, pval, num_genes):
         """
         Computes the top differentially expressed genes between two clusters
@@ -158,6 +164,7 @@ class ScanpyEngine(CXGDriver):
             },
         }
 
+    @cache.memoize()
     def expression(self, cells=None, genes=None):
         """
         Retrieves expression for each gene for cells in data frame

server/requirements.txt

@@ -6,6 +6,7 @@ click==6.7
 cycler==0.10.0
 decorator==4.3.0
 Flask==0.12.4
+Flask-Caching==1.4.0
 Flask-Compress==1.4.0
 Flask-Cors==3.0.6
 Flask-RESTful==0.3.6