gene sets summary route (#2099)

* gene sets summary route

* lint

* clarify return type

* style

local_server/common/annotations/annotations.py

@@ -110,7 +110,7 @@ class Annotations(metaclass=ABCMeta):
         from io import StringIO
         import csv
 
-        if type(genesets) == dict:
+        if isinstance(genesets, dict):
             genesets = genesets.values()
 
         with StringIO() as sio:

local_server/common/annotations/local_file_csv.py

@@ -26,10 +26,14 @@ class AnnotationsLocalFile(Annotations):
         self.label_lock = threading.RLock()
         self.gene_sets_lock = threading.RLock()
 
-        # cache the most recent annotations.
-        self.last_fname = None
+        # cache the most recent cell labels/annotations.
+        self.last_label_fname = None
         self.last_labels = None
 
+        # cache the most recent gene sets.
+        self.last_geneset_fname = None
+        self.last_geneset = None
+
         # txn ID - used to de-dup geneset writes
         self.last_geneset_tid = 0
 
@@ -63,14 +67,14 @@ class AnnotationsLocalFile(Annotations):
         with self.label_lock:
             if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0:
                 # returned the cached labels if possible, otherwise read them from the file
-                if fname == self.last_fname:
+                if fname == self.last_label_fname:
                     return self.last_labels
                 else:
                     labels = pd.read_csv(
                         fname, dtype="category", index_col=0, header=0, comment="#", keep_default_na=False
                     )
                     # update the cache
-                    self.last_fname = fname
+                    self.last_label_fname = fname
                     self.last_labels = labels
                     return labels
             else:
@@ -102,7 +106,7 @@ class AnnotationsLocalFile(Annotations):
                 open(fname, "w").close()
 
             # update the cache
-            self.last_fname = fname
+            self.last_label_fname = fname
             self.last_labels = df
 
     def read_gene_sets(self, data_adaptor, context=None):
@@ -116,8 +120,19 @@ class AnnotationsLocalFile(Annotations):
         with self.gene_sets_lock:
             tid = self.last_geneset_tid  # inside the critical section
             if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0:
-                with open(fname, newline="") as f:
-                    gene_sets = read_gene_set_tidycsv(f, context)
+                # return the cached genesets if possible, otherwise read from file and validate them
+                if fname == self.last_geneset_fname:
+                    gene_sets = self.last_geneset
+                else:
+                    with open(fname, newline="") as f:
+                        gene_sets = read_gene_set_tidycsv(f, context)
+
+                    # validate
+                    gene_sets = data_adaptor.check_new_gene_sets(gene_sets, context)
+
+                    # update cache
+                    self.last_geneset_fname = fname
+                    self.last_geneset = gene_sets
 
         return (gene_sets, tid)
 
@@ -127,6 +142,9 @@ class AnnotationsLocalFile(Annotations):
         if type(tid) != int or tid < 0:
             raise ValueError("tid must be a positive integer")
 
+        # may raise
+        gene_sets = data_adaptor.check_new_gene_sets(gene_sets)
+
         with self.gene_sets_lock:
             # skip if the request is stale
             if tid is not None:
@@ -149,6 +167,10 @@ class AnnotationsLocalFile(Annotations):
                 f.write(header)
                 f.write(self.gene_sets_to_csv(gene_sets))
 
+            # update the cache
+            self.last_geneset_fname = fname
+            self.last_geneset = gene_sets if type(gene_sets) == dict else {g["geneset_name"]: g for g in gene_sets}
+
     def _get_userdata_idhash(self, data_adaptor):
         """
         Return a short hash that weakly identifies the user and dataset.

local_server/common/config/dataset_config.py

@@ -33,9 +33,9 @@ class DatasetConfig(BaseConfig):
                 "obo_location"
             ]
             self.user_annotations__gene_sets__readonly = default_config["user_annotations"]["gene_sets"]["readonly"]
-            self.user_annotations__local_file_csv__gene_sets_file = default_config["user_annotations"]["local_file_csv"][
-                "gene_sets_file"
-            ]
+            self.user_annotations__local_file_csv__gene_sets_file = default_config["user_annotations"][
+                "local_file_csv"
+            ]["gene_sets_file"]
 
             self.embeddings__names = default_config["embeddings"]["names"]
             self.embeddings__enable_reembedding = default_config["embeddings"]["enable_reembedding"]
@@ -169,10 +169,10 @@ class DatasetConfig(BaseConfig):
         if server_config.single_dataset__datapath:
             data_adaptor = self.get_data_adaptor()
             if self.user_annotations__local_file_csv__file:
-                data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor))
+                self.user_annotations.read_labels(data_adaptor)
             if self.user_annotations__local_file_csv__gene_sets_file:
                 try:
-                    data_adaptor.check_new_gene_sets(self.user_annotations.read_gene_sets(data_adaptor, context), context)
+                    self.user_annotations.read_gene_sets(data_adaptor, context)
                 except (ValueError, AnnotationsError, KeyError) as e:
                     raise ConfigurationError(f"Unable to read genesets CSV file: {str(e)}") from e
 

local_server/common/errors.py

@@ -55,4 +55,5 @@ define_exception("OntologyLoadFailure", "Raised when reading the ontology file f
 define_exception("ConfigurationError", "Raised when checking configuration errors")
 define_exception("PrepareError", "Raised when data is misprepared")
 define_exception("SecretKeyRetrievalError", "Raised when get_secret_key from AWS fails")
-define_exception("ObsoleteRequest", "Raised when the request is no longer valid.")
+define_exception("ObsoleteRequest", "Raised when the request is no longer valid.")
+define_exception("UnsupportedSummaryMethod", "Raised when a gene set summary method is unknown or unsupported.")

local_server/common/rest.py

@@ -19,6 +19,7 @@ from local_server.common.errors import (
     ColorFormatException,
     AnnotationsError,
     ObsoleteRequest,
+    UnsupportedSummaryMethod,
 )
 
 import json
@@ -336,7 +337,7 @@ def genesets_get(request, data_adaptor):
 
     try:
         annotations = data_adaptor.dataset_config.user_annotations
-        (genesets, tid) = data_adaptor.check_new_gene_sets(annotations.read_gene_sets(data_adaptor))
+        (genesets, tid) = annotations.read_gene_sets(data_adaptor)
 
         if preferred_mimetype == "text/csv":
             return make_response(
@@ -373,10 +374,39 @@ def genesets_put(request, data_adaptor):
         if genesets is None:
             abort(HTTPStatus.BAD_REQUEST)
 
-        (gs, _) = data_adaptor.check_new_gene_sets((genesets, tid))
-        annotations.write_gene_sets(gs, tid, data_adaptor)
+        annotations.write_gene_sets(genesets, tid, data_adaptor)
         return make_response(jsonify({"status": "OK"}), HTTPStatus.OK)
     except (ValueError, DisabledFeatureError, KeyError) as e:
         return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
     except (ObsoleteRequest, TypeError) as e:
         return abort(HTTPStatus.NOT_FOUND, description=str(e))
+
+
+def geneset_summary_get(request, data_adaptor):
+    preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
+    if preferred_mimetype != "application/octet-stream":
+        return abort(HTTPStatus.NOT_ACCEPTABLE)
+
+    geneset_name = request.args.get("geneset_name", default=None)
+    summary_method = request.args.get("method", default="mean")
+    request_tid = request.args.get("tid", default=None)
+
+    try:
+        annotations = data_adaptor.dataset_config.user_annotations
+        (genesets, tid) = annotations.read_gene_sets(data_adaptor)
+
+        if request_tid is not None and int(request_tid) != tid:
+            return abort(HTTPStatus.NOT_FOUND, "Obsolete TID")
+        if geneset_name is None or geneset_name not in genesets:
+            return abort(HTTPStatus.BAD_REQUEST, "Gene set name not found.")
+        genes = [g["gene_symbol"] for g in genesets.get(geneset_name)["genes"]]
+
+        return make_response(
+            data_adaptor.get_gene_set_summary(geneset_name, genes, summary_method),
+            HTTPStatus.OK,
+            {"Content-Type": "application/octet-stream"},
+        )
+    except (ValueError) as e:
+        return abort(HTTPStatus.NOT_FOUND, description=str(e))
+    except (UnsupportedSummaryMethod) as e:
+        return abort(HTTPStatus.BAD_REQUEST, description=str(e))