From 7642251d446b40c1966cb33fe2fa71a7663e3c3f Mon Sep 17 00:00:00 2001 From: Charlotte Weaver Date: Wed, 1 Aug 2018 13:32:04 -0700 Subject: [PATCH 1/6] Add simple cache to backend --- server/app/app.py | 2 ++ server/app/scanpy_engine/scanpy_engine.py | 7 +++++++ server/requirements.txt | 1 + 3 files changed, 10 insertions(+) diff --git a/server/app/app.py b/server/app/app.py index d9e8e2ce..397c71f5 100644 --- a/server/app/app.py +++ b/server/app/app.py @@ -2,6 +2,7 @@ import argparse import os from flask import Flask +from flask.ext.cache import Cache from flask_compress import Compress from flask_cors import CORS from flask_restful_swagger_2 import get_swagger_blueprint @@ -12,6 +13,7 @@ from .rest_api.rest import get_api_resources REACTIVE_LIMIT = 1_000_000 app = Flask(__name__) +cache = Cache(app, config={"CACHE_TYPE": "simple"}) Compress(app) CORS(app) diff --git a/server/app/scanpy_engine/scanpy_engine.py b/server/app/scanpy_engine/scanpy_engine.py index 405003f5..0b5ec9ca 100644 --- a/server/app/scanpy_engine/scanpy_engine.py +++ b/server/app/scanpy_engine/scanpy_engine.py @@ -6,6 +6,7 @@ from scipy import stats from ..util.schema_parse import parse_schema from ..driver.driver import CXGDriver +from server.app.app import cache class ScanpyEngine(CXGDriver): @@ -41,6 +42,7 @@ class ScanpyEngine(CXGDriver): def genes(self): return self.data.var.index.tolist() + # Can't seem to cache a view of a dataframe, need to investigate why def filter_cells(self, filter): """ Filter cells from data and return a subset of the data @@ -69,6 +71,7 @@ class ScanpyEngine(CXGDriver): cell_idx = np.logical_and(cell_idx, key_idx) return self.data[cell_idx, :] + @cache.memoize(86400000) def metadata_ranges(self, df=None): metadata_ranges = {} if not df: @@ -88,6 +91,7 @@ class ScanpyEngine(CXGDriver): } return metadata_ranges + @cache.memoize(86400000) def metadata(self, df, fields=None): """ Gets metadata key:value for each cells @@ -101,6 +105,7 @@ class ScanpyEngine(CXGDriver): metadata[idx]["CellName"] = metadata[idx].pop("cell_name", None) return metadata + @cache.memoize(86400000) def create_graph(self, df): """ Computes a n-d layout for cells through dimensionality reduction. @@ -112,6 +117,7 @@ class ScanpyEngine(CXGDriver): normalized_graph = (graph - graph.min()) / (graph.max() - graph.min()) return np.hstack((df.obs["cell_name"].values.reshape(len(df.obs.index), 1), normalized_graph)).tolist() + @cache.memoize(86400000) def diffexp(self, cell_list_1, cell_list_2, pval, num_genes): """ Computes the top differentially expressed genes between two clusters @@ -158,6 +164,7 @@ class ScanpyEngine(CXGDriver): }, } + @cache.memoize(86400000) def expression(self, cells=None, genes=None): """ Retrieves expression for each gene for cells in data frame diff --git a/server/requirements.txt b/server/requirements.txt index 7ab96abb..d12edc0a 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -6,6 +6,7 @@ click==6.7 cycler==0.10.0 decorator==4.3.0 Flask==0.12.4 +Flask-Cache==0.13.1 Flask-Compress==1.4.0 Flask-Cors==3.0.6 Flask-RESTful==0.3.6 From 69fca7ea8b2877c80b762b0167d1ca4460410732 Mon Sep 17 00:00:00 2001 From: Charlotte Weaver Date: Wed, 1 Aug 2018 15:46:04 -0700 Subject: [PATCH 2/6] Using flask_caching instead More recent module, same API --- server/app/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/app/app.py b/server/app/app.py index 397c71f5..1e41e04a 100644 --- a/server/app/app.py +++ b/server/app/app.py @@ -2,7 +2,7 @@ import argparse import os from flask import Flask -from flask.ext.cache import Cache +from flask_caching import Cache from flask_compress import Compress from flask_cors import CORS from flask_restful_swagger_2 import get_swagger_blueprint From 0a36fac14209205cde199a0a1c8b505fe8a4a1a2 Mon Sep 17 00:00:00 2001 From: Charlotte Weaver Date: Wed, 1 Aug 2018 15:48:56 -0700 Subject: [PATCH 3/6] 10 day cache (down from 1000) I can expect a scientist to leave this running over the weekend on their laptop and still expect fast results. If someone leaves it running for a few years the data can probably be safely recalculated. --- server/app/scanpy_engine/scanpy_engine.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/app/scanpy_engine/scanpy_engine.py b/server/app/scanpy_engine/scanpy_engine.py index 0b5ec9ca..54491480 100644 --- a/server/app/scanpy_engine/scanpy_engine.py +++ b/server/app/scanpy_engine/scanpy_engine.py @@ -71,7 +71,7 @@ class ScanpyEngine(CXGDriver): cell_idx = np.logical_and(cell_idx, key_idx) return self.data[cell_idx, :] - @cache.memoize(86400000) + @cache.memoize(864000) def metadata_ranges(self, df=None): metadata_ranges = {} if not df: @@ -91,7 +91,7 @@ class ScanpyEngine(CXGDriver): } return metadata_ranges - @cache.memoize(86400000) + @cache.memoize(864000) def metadata(self, df, fields=None): """ Gets metadata key:value for each cells @@ -105,7 +105,7 @@ class ScanpyEngine(CXGDriver): metadata[idx]["CellName"] = metadata[idx].pop("cell_name", None) return metadata - @cache.memoize(86400000) + @cache.memoize(864000) def create_graph(self, df): """ Computes a n-d layout for cells through dimensionality reduction. @@ -117,7 +117,7 @@ class ScanpyEngine(CXGDriver): normalized_graph = (graph - graph.min()) / (graph.max() - graph.min()) return np.hstack((df.obs["cell_name"].values.reshape(len(df.obs.index), 1), normalized_graph)).tolist() - @cache.memoize(86400000) + @cache.memoize(864000) def diffexp(self, cell_list_1, cell_list_2, pval, num_genes): """ Computes the top differentially expressed genes between two clusters @@ -164,7 +164,7 @@ class ScanpyEngine(CXGDriver): }, } - @cache.memoize(86400000) + @cache.memoize(864000) def expression(self, cells=None, genes=None): """ Retrieves expression for each gene for cells in data frame From 489a2dd5dbd2534fe96bf5b666de1f5a032da87a Mon Sep 17 00:00:00 2001 From: Charlotte Weaver Date: Wed, 1 Aug 2018 16:29:38 -0700 Subject: [PATCH 4/6] Add flask-caching to requirements --- server/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/requirements.txt b/server/requirements.txt index d12edc0a..a881e310 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -6,7 +6,7 @@ click==6.7 cycler==0.10.0 decorator==4.3.0 Flask==0.12.4 -Flask-Cache==0.13.1 +Flask-Caching==1.4.0 Flask-Compress==1.4.0 Flask-Cors==3.0.6 Flask-RESTful==0.3.6 From 116cef551ca98f071c50c09b99b3c3059470f32c Mon Sep 17 00:00:00 2001 From: Charlotte Weaver Date: Thu, 2 Aug 2018 11:01:38 -0700 Subject: [PATCH 5/6] Switch to default timeout for cache --- server/app/app.py | 2 +- server/app/scanpy_engine/scanpy_engine.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/server/app/app.py b/server/app/app.py index 1e41e04a..36c2f707 100644 --- a/server/app/app.py +++ b/server/app/app.py @@ -13,7 +13,7 @@ from .rest_api.rest import get_api_resources REACTIVE_LIMIT = 1_000_000 app = Flask(__name__) -cache = Cache(app, config={"CACHE_TYPE": "simple"}) +cache = Cache(app, config={"CACHE_TYPE": "simple", "CACHE_DEFAULT_TIMEOUT": 860000}) Compress(app) CORS(app) diff --git a/server/app/scanpy_engine/scanpy_engine.py b/server/app/scanpy_engine/scanpy_engine.py index 54491480..99334e87 100644 --- a/server/app/scanpy_engine/scanpy_engine.py +++ b/server/app/scanpy_engine/scanpy_engine.py @@ -71,7 +71,7 @@ class ScanpyEngine(CXGDriver): cell_idx = np.logical_and(cell_idx, key_idx) return self.data[cell_idx, :] - @cache.memoize(864000) + @cache.memoize() def metadata_ranges(self, df=None): metadata_ranges = {} if not df: @@ -91,7 +91,7 @@ class ScanpyEngine(CXGDriver): } return metadata_ranges - @cache.memoize(864000) + @cache.memoize() def metadata(self, df, fields=None): """ Gets metadata key:value for each cells @@ -105,7 +105,7 @@ class ScanpyEngine(CXGDriver): metadata[idx]["CellName"] = metadata[idx].pop("cell_name", None) return metadata - @cache.memoize(864000) + @cache.memoize() def create_graph(self, df): """ Computes a n-d layout for cells through dimensionality reduction. @@ -117,7 +117,7 @@ class ScanpyEngine(CXGDriver): normalized_graph = (graph - graph.min()) / (graph.max() - graph.min()) return np.hstack((df.obs["cell_name"].values.reshape(len(df.obs.index), 1), normalized_graph)).tolist() - @cache.memoize(864000) + @cache.memoize() def diffexp(self, cell_list_1, cell_list_2, pval, num_genes): """ Computes the top differentially expressed genes between two clusters @@ -164,7 +164,7 @@ class ScanpyEngine(CXGDriver): }, } - @cache.memoize(864000) + @cache.memoize() def expression(self, cells=None, genes=None): """ Retrieves expression for each gene for cells in data frame From e11b90583659fd0c44bf0c99df6e4b96a4d0d2be Mon Sep 17 00:00:00 2001 From: Charlotte Weaver Date: Thu, 2 Aug 2018 11:33:47 -0700 Subject: [PATCH 6/6] Cleaning up import statements --- server/app/app.py | 2 +- server/app/rest_api/rest.py | 4 ++-- server/app/scanpy_engine/scanpy_engine.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/app/app.py b/server/app/app.py index 36c2f707..547a9db5 100644 --- a/server/app/app.py +++ b/server/app/app.py @@ -7,8 +7,8 @@ from flask_compress import Compress from flask_cors import CORS from flask_restful_swagger_2 import get_swagger_blueprint -from .web import webapp from .rest_api.rest import get_api_resources +from .web import webapp REACTIVE_LIMIT = 1_000_000 diff --git a/server/app/rest_api/rest.py b/server/app/rest_api/rest.py index 40742dc3..f5892125 100644 --- a/server/app/rest_api/rest.py +++ b/server/app/rest_api/rest.py @@ -3,8 +3,8 @@ from flask import ( ) from flask_restful_swagger_2 import Api, swagger, Resource -from ..util.utils import make_payload -from ..util.filter import parse_filter +from server.app.util.utils import make_payload +from server.app.util.filter import parse_filter class InitializeAPI(Resource): diff --git a/server/app/scanpy_engine/scanpy_engine.py b/server/app/scanpy_engine/scanpy_engine.py index 99334e87..9ede3b45 100644 --- a/server/app/scanpy_engine/scanpy_engine.py +++ b/server/app/scanpy_engine/scanpy_engine.py @@ -4,9 +4,9 @@ import numpy as np import scanpy.api as sc from scipy import stats -from ..util.schema_parse import parse_schema -from ..driver.driver import CXGDriver from server.app.app import cache +from server.app.driver.driver import CXGDriver +from server.app.util.schema_parse import parse_schema class ScanpyEngine(CXGDriver):