gene set summary progress (#2127)

* revert removal of cache control headers

* checkpoint work on revising summary route

* add summary query support to annoMatrix

* summarize route cleanup

* add mising file

* clean up summarize route

* add summary histogram

* update deps

* lint

* more lint

* lint

* manage crossfiler during gene set state changes

* remove obsolete debugging code

* correctly perform async watch in histogram

* better error handling

backend/server/app/app.py

@@ -206,11 +206,16 @@ class GenesetsAPI(Resource):
         return common_rest.genesets_put(request, data_adaptor)
 
 
-class GenesetSummaryAPI(Resource):
+class SummarizeVarAPI(Resource):
+    @rest_get_data_adaptor
+    @cache_control(public=True, max_age=ONE_WEEK)
+    def get(self, data_adaptor):
+        return common_rest.summarize_var_get(request, data_adaptor)
+
     @rest_get_data_adaptor
     @cache_control(no_store=True)
-    def get(self, data_adaptor):
-        return common_rest.geneset_summary_get(request, data_adaptor)
+    def post(self, data_adaptor):
+        return common_rest.summarize_var_post(request, data_adaptor)
 
 
 def get_api_base_resources(bp_base):
@@ -239,7 +244,7 @@ def get_api_dataroot_resources(bp_dataroot):
     add_resource(AnnotationsVarAPI, "/annotations/var")
     add_resource(DataVarAPI, "/data/var")
     add_resource(GenesetsAPI, "/genesets")
-    add_resource(GenesetSummaryAPI, "/geneset_summary")
+    add_resource(SummarizeVarAPI, "/summarize/var")
     # Display routes
     add_resource(ColorsAPI, "/colors")
     # Computation routes

backend/server/common/rest.py

@@ -3,6 +3,7 @@ import logging
 import sys
 from http import HTTPStatus
 import zlib
+import hashlib
 
 from flask import make_response, jsonify, current_app, abort
 from werkzeug.urls import url_unquote
@@ -383,31 +384,42 @@ def genesets_put(request, data_adaptor):
         return abort(HTTPStatus.NOT_FOUND, description=str(e))
 
 
-def geneset_summary_get(request, data_adaptor):
+def summarize_var_helper(request, data_adaptor, key, raw_query):
     preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
     if preferred_mimetype != "application/octet-stream":
         return abort(HTTPStatus.NOT_ACCEPTABLE)
 
-    geneset_name = request.args.get("geneset_name", default=None)
-    summary_method = request.args.get("method", default="mean")
-    request_tid = request.args.get("tid", default=None)
+    summary_method = request.values.get("method", default="mean")
+    query_hash = hashlib.sha1(raw_query).hexdigest()  # cache helper
+    if key and query_hash != key:
+        return abort(HTTPStatus.BAD_REQUEST, description="query key did not match")
 
-    try:
-        annotations = data_adaptor.dataset_config.user_annotations
-        (genesets, tid) = annotations.read_gene_sets(data_adaptor)
-
-        if request_tid is not None and int(request_tid) != tid:
-            return abort(HTTPStatus.NOT_FOUND, "Obsolete TID")
-        if geneset_name is None or geneset_name not in genesets:
-            return abort(HTTPStatus.BAD_REQUEST, "Gene set name not found.")
-        genes = [g["gene_symbol"] for g in genesets.get(geneset_name)["genes"]]
+    args_filter_only = request.values.copy()
+    args_filter_only.poplist("method")
+    args_filter_only.poplist("key")
 
+    try:    
+        filter = _query_parameter_to_filter(args_filter_only)
         return make_response(
-            data_adaptor.get_gene_set_summary(geneset_name, genes, summary_method),
+            data_adaptor.summarize_var(summary_method, filter, query_hash),
             HTTPStatus.OK,
             {"Content-Type": "application/octet-stream"},
         )
     except (ValueError) as e:
         return abort(HTTPStatus.NOT_FOUND, description=str(e))
-    except (UnsupportedSummaryMethod) as e:
+    except (UnsupportedSummaryMethod, FilterError) as e:
         return abort(HTTPStatus.BAD_REQUEST, description=str(e))
+
+
+def summarize_var_get(request, data_adaptor):
+    return summarize_var_helper(request, data_adaptor, None, request.query_string)
+
+
+def summarize_var_post(request, data_adaptor):
+    if not request.content_type or "application/x-www-form-urlencoded" not in request.content_type:
+        return abort(HTTPStatus.UNSUPPORTED_MEDIA_TYPE)
+    if request.content_length > 1_000_000:  # just a sanity check to avoid memory exhaustion
+        return abort(HTTPStatus.BAD_REQUEST)
+
+    key = request.args.get("key", default=None)
+    return summarize_var_helper(request, data_adaptor, key, request.get_data())

backend/server/data_anndata/anndata_adaptor.py

@@ -5,7 +5,6 @@ import anndata
 import numpy as np
 from packaging import version
 from pandas.core.dtypes.dtypes import CategoricalDtype
-import pandas as pd
 from scipy import sparse
 from server_timing import Timing as ServerTiming
 
@@ -13,7 +12,7 @@ import backend.server.compute.diffexp_generic as diffexp_generic
 from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
 from backend.common.constants import Axis, MAX_LAYOUTS
 from backend.server.common.corpora import corpora_get_props_from_anndata
-from backend.common.errors import PrepareError, DatasetAccessError, FilterError, UnsupportedSummaryMethod
+from backend.common.errors import PrepareError, DatasetAccessError, FilterError
 from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
 from backend.server.compute.scanpy import scanpy_umap
 from backend.server.data_common.data_adaptor import DataAdaptor
@@ -69,11 +68,11 @@ class AnndataAdaptor(DataAdaptor):
 
     @staticmethod
     def _create_unique_column_name(df, col_name_prefix):
-        """ given the columns of a dataframe, and a name prefix, return a column name which
-            does not exist in the dataframe, AND which is prefixed by `prefix`
+        """given the columns of a dataframe, and a name prefix, return a column name which
+        does not exist in the dataframe, AND which is prefixed by `prefix`
 
-            The approach is to append a numeric suffix, starting at zero and increasing by
-            one, until an unused name is found (eg, prefix_0, prefix_1, ...).
+        The approach is to append a numeric suffix, starting at zero and increasing by
+        one, until an unused name is found (eg, prefix_0, prefix_1, ...).
         """
         suffix = 0
         while f"{col_name_prefix}{suffix}" in df:
@@ -200,10 +199,10 @@ class AnndataAdaptor(DataAdaptor):
             self.parameters.update({"diffexp_may_be_slow": True})
 
     def _is_valid_layout(self, arr):
-        """ return True if this layout data is a valid array for front-end presentation:
-            * ndarray, dtype float/int/uint
-            * with shape (n_obs, >= 2)
-            * with all values finite or NaN (no +Inf or -Inf)
+        """return True if this layout data is a valid array for front-end presentation:
+        * ndarray, dtype float/int/uint
+        * with shape (n_obs, >= 2)
+        * with all values finite or NaN (no +Inf or -Inf)
         """
         is_valid = type(arr) == np.ndarray and arr.dtype.kind in "fiu"
         is_valid = is_valid and arr.shape[0] == self.data.n_obs and arr.shape[1] >= 2
@@ -368,29 +367,3 @@ class AnndataAdaptor(DataAdaptor):
     def get_var_keys(self):
         # return list of keys
         return self.data.var.keys().to_list()
-
-    def get_gene_set_summary(self, geneset_name, genes, method):
-        if method != "mean":
-            raise UnsupportedSummaryMethod("Unknown gene set summary method.")
-
-        var_index = self.parameters.get("var_names")
-        obs_selector, var_selector = self._filter_to_mask(
-            {
-                "var": {
-                    "annotation_value": [
-                        {
-                            "name": var_index,
-                            "values": genes,
-                        }
-                    ]
-                }
-            }
-        )
-
-        X = self.get_X_array(obs_selector, var_selector)
-        if sparse.issparse(X):
-            mean = X.mean(axis=1)
-        else:
-            mean = X.mean(axis=1, keepdims=True)
-        col_idx = pd.Index([geneset_name])
-        return encode_matrix_fbs(mean, col_idx=col_idx, row_idx=None)

backend/server/data_common/data_adaptor.py

@@ -1,14 +1,14 @@
 from abc import ABCMeta, abstractmethod
 from os.path import basename, splitext
 import re
-
 import numpy as np
 import pandas as pd
+from scipy import sparse
 from server_timing import Timing as ServerTiming
 
 from backend.server.common.config.app_config import AppConfig
 from backend.common.constants import Axis
-from backend.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError
+from backend.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError, UnsupportedSummaryMethod
 from backend.common.utils.utils import jsonify_numpy
 from backend.common.fbs.matrix import encode_matrix_fbs
 
@@ -482,6 +482,25 @@ class DataAdaptor(metaclass=ABCMeta):
             lastmod = None
         return lastmod
 
-    @abstractmethod
-    def get_gene_set_summary(self, geneset_name, genes, method):
-        pass
+    def summarize_var(self, method, filter, query_hash):
+        if method != "mean":
+            raise UnsupportedSummaryMethod("Unknown gene set summary method.")
+
+        obs_selector, var_selector = self._filter_to_mask(filter)
+        if obs_selector is not None:
+            raise FilterError("filtering on obs unsupported")
+
+        # if no filter, just return zeros.  We don't have a use case
+        # for summarizing the entire X without a filter, and it would
+        # potentially be quite compute / memory intensive.
+        if var_selector is None or np.count_nonzero(var_selector) == 0:
+            mean = np.zeros((self.get_shape()[0], 1), dtype=np.float32)
+        else:
+            X = self.get_X_array(obs_selector, var_selector)
+            if sparse.issparse(X):
+                mean = X.mean(axis=1)
+            else:
+                mean = X.mean(axis=1, keepdims=True)
+
+        col_idx = pd.Index([query_hash])
+        return encode_matrix_fbs(mean, col_idx=col_idx, row_idx=None)