genesets route for local server (#2079)

* first cut at GET /genesets route

* update existing tests to match code changes

* more GET /genesets and initial tests

* add missing test fixture

* geneset validation accepts OTA format

* genesets route: better error handling, more tests

* lint

local_server/app/app.py

@@ -144,6 +144,17 @@ class LayoutObsAPI(Resource):
         return common_rest.layout_obs_put(request, data_adaptor)
 
 
+class GenesetsAPI(Resource):
+    @rest_get_data_adaptor
+    def get(self, data_adaptor):
+        return common_rest.genesets_get(request, data_adaptor)
+
+    @requires_authentication
+    @rest_get_data_adaptor
+    def put(self, data_adaptor):
+        return common_rest.genesets_put(request, data_adaptor)
+
+
 def get_api_base_resources(bp_base):
     """Add resources that are accessed from the api url"""
     api = Api(bp_base)
@@ -169,6 +180,7 @@ def get_api_dataroot_resources(bp_dataroot):
     add_resource(AnnotationsObsAPI, "/annotations/obs")
     add_resource(AnnotationsVarAPI, "/annotations/var")
     add_resource(DataVarAPI, "/data/var")
+    add_resource(GenesetsAPI, "/genesets")
     # Display routes
     add_resource(ColorsAPI, "/colors")
     # Computation routes

local_server/cli/launch.py

@@ -32,16 +32,17 @@ def annotation_args(func):
         multiple=False,
         metavar="<path>",
         help="CSV file to initialize editing of existing annotations; will be altered in-place. "
-        "Incompatible with --annotations-dir.",
+        "Incompatible with --user-generated-data-dir.",
     )
     @click.option(
+        "--user-generated-data-dir",
         "--annotations-dir",
         default=DEFAULT_CONFIG.dataset_config.user_annotations__local_file_csv__directory,
         show_default=False,
         multiple=False,
         metavar="<directory path>",
         help="Directory of where to save output annotations; filename will be specified in the application. "
-        "Incompatible with --annotations-file.",
+        "Incompatible with --annotations-file and --genesets-file.",
     )
     @click.option(
         "--experimental-annotations-ontology",
@@ -57,6 +58,23 @@ def annotation_args(func):
         metavar="<path or url>",
         help="Location of OBO file defining cell annotation autosuggest terms.",
     )
+    @click.option(
+        "--disable-genesets-save",
+        is_flag=True,
+        default=DEFAULT_CONFIG.dataset_config.user_annotations__genesets__readonly,
+        show_default=False,
+        help="Disable saving gene sets. If disabled, users will be able to make changes to gene sets but all "
+        "changes will be lost on browser refresh.",
+    )
+    @click.option(
+        "--genesets-file",
+        default=DEFAULT_CONFIG.dataset_config.user_annotations__local_file_csv__genesets_file,
+        show_default=True,
+        multiple=False,
+        metavar="<path>",
+        help="CSV file to initialize editing of gene sets; will be altered in-place. Incompatible with "
+        "--user-generated-data-dir.",
+    )
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
         return func(*args, **kwargs)
@@ -315,7 +333,9 @@ def launch(
     about,
     disable_annotations,
     annotations_file,
-    annotations_dir,
+    user_generated_data_dir,
+    genesets_file,
+    disable_genesets_save,
     backed,
     disable_diffexp,
     experimental_annotations_ontology,
@@ -373,7 +393,9 @@ def launch(
             app__scripts=scripts,
             user_annotations__enable=not disable_annotations,
             user_annotations__local_file_csv__file=annotations_file,
-            user_annotations__local_file_csv__directory=annotations_dir,
+            user_annotations__local_file_csv__directory=user_generated_data_dir,
+            user_annotations__local_file_csv__genesets_file=genesets_file,
+            user_annotations__genesets__readonly=disable_genesets_save,
             user_annotations__ontology__enable=experimental_annotations_ontology,
             user_annotations__ontology__obo_location=experimental_annotations_ontology_obo,
             presentation__max_categories=max_category_items,

local_server/common/annotations/annotations.py

@@ -3,19 +3,34 @@ from abc import ABCMeta, abstractmethod
 import fastobo
 import fsspec
 
-from local_server.common.errors import OntologyLoadFailure
+from local_server.common.errors import OntologyLoadFailure, DisabledFeatureError
 from local_server.common.utils.type_conversion_utils import get_schema_type_hint_of_array
 
 
 class Annotations(metaclass=ABCMeta):
-    """ baseclass for annotations, including ontologies"""
+    """ baseclass for annotations, including ontologies and genesets"""
 
     """ our default ontology is the PURL for the Cell Ontology.
     See http://www.obofoundry.org/ontology/cl.html """
     DefaultOnotology = "http://purl.obolibrary.org/obo/cl.obo"
 
-    def __init__(self):
+    def __init__(self, config={}):
         self.ontology_data = None
+        self.config = config
+
+    def user_annotations_enabled(self):
+        return self.config.get("user-annotations", False)
+
+    def genesets_save_enabled(self):
+        return self.config.get("genesets-save", False)
+
+    def check_user_annotations_enabled(self):
+        if not self.user_annotations_enabled():
+            raise DisabledFeatureError("User annotations are disabled.")
+
+    def check_genesets_save_enabled(self):
+        if not self.genesets_save_enabled():
+            raise DisabledFeatureError("User genesets save is disabled.")
 
     def load_ontology(self, path):
         """Load and parse ontologies - currently support OBO files only."""
@@ -64,7 +79,66 @@ class Annotations(metaclass=ABCMeta):
         """Write the labels (df) to a persistent storage such that it can later be read"""
         pass
 
+    @abstractmethod
+    def read_genesets(self, data_adaptor):
+        """Return the genesets from persistent storage """
+        pass
+
+    @abstractmethod
+    def write_genesets(self, gs, data_adaptor):
+        """Write the genesets (gs) to a persistent storage such that it can later be read"""
+        pass
+
     @abstractmethod
     def update_parameters(self, parameters, data_adaptor):
         """Update configuration parameters that describe information about the annotations feature"""
         pass
+
+    Genesets_Header = [
+        "geneset_name",
+        "geneset_description",
+        "gene_symbol",
+        "gene_description",
+    ]
+
+    @staticmethod
+    def genesets_to_csv(genesets):
+        """
+        Convert the internal genesets format (returned by read_geneset) into
+        the simple Tidy CSV.
+        """
+        from io import StringIO
+        import csv
+
+        if type(genesets) == dict:
+            genesets = genesets.values()
+
+        with StringIO() as sio:
+            writer = csv.writer(sio, dialect='excel')
+            writer.writerow(Annotations.Genesets_Header)
+            for geneset in genesets:
+                # genes may be empty, in which case we skip the geneset entirely
+                genes = geneset["genes"]
+                if not genes:
+                    writer.writerow([geneset["geneset_name"], geneset.get("geneset_description", ""), "", ""])
+                else:
+                    writer.writerows(
+                        [
+                            [
+                                geneset["geneset_name"],
+                                geneset.get("geneset_description", ""),
+                                gene["gene_symbol"],
+                                gene.get("gene_description", ""),
+                            ]
+                            for gene in genes
+                        ]
+                    )
+            return sio.getvalue()
+
+    @staticmethod
+    def genesets_to_response(genesets):
+        """
+        Convert the internal genesets format (returned by read_geneset) into
+        the dict expected by the JSON REST API
+        """
+        return list(genesets.values())

local_server/common/annotations/local_file_csv.py

@@ -4,29 +4,35 @@ import re
 import threading
 from datetime import datetime
 from hashlib import blake2b
+import csv
 
 import pandas as pd
 from flask import session, has_request_context, current_app
 
 from local_server import __version__ as cellxgene_version
 from local_server.common.annotations.annotations import Annotations
-from local_server.common.errors import AnnotationsError
+from local_server.common.errors import AnnotationsError, ObsoleteRequest
 
 
 class AnnotationsLocalFile(Annotations):
     CXG_ANNO_COLLECTION = "cxg_anno_collection"
 
-    def __init__(self, output_dir, output_file):
+    def __init__(self, config, output_dir, label_output_file, genesets_output_file):
-        super().__init__()
+        super().__init__(config)
         self.output_dir = output_dir
-        self.output_file = output_file
+        self.label_output_file = label_output_file
+        self.genesets_output_file = genesets_output_file
         # lock used to protect label file write ops
         self.label_lock = threading.RLock()
+        self.genesets_lock = threading.RLock()
 
-        # cache the most recent annotations
+        # cache the most recent annotations.
         self.last_fname = None
         self.last_labels = None
 
+        # txn ID - used to de-dup geneset writes
+        self.last_geneset_tid = 0
+
     def is_safe_collection_name(self, name):
         """
         return true if this is a safe collection name
@@ -47,11 +53,13 @@ class AnnotationsLocalFile(Annotations):
         return session.get(self.CXG_ANNO_COLLECTION)
 
     def read_labels(self, data_adaptor):
+        self.check_user_annotations_enabled()  # raises
+
         if has_request_context():
             if not current_app.auth.is_user_authenticated():
                 return pd.DataFrame()
 
-        fname = self._get_filename(data_adaptor)
+        fname = self._get_celllabels_filename(data_adaptor)
         with self.label_lock:
             if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0:
                 # returned the cached labels if possible, otherwise read them from the file
@@ -69,6 +77,8 @@ class AnnotationsLocalFile(Annotations):
                 return pd.DataFrame()
 
     def write_labels(self, df, data_adaptor):
+        self.check_user_annotations_enabled()  # raises
+
         # update our internal state and save it.  Multi-threading often enabled,
         # so treat this as a critical section.
         with self.label_lock:
@@ -81,7 +91,7 @@ class AnnotationsLocalFile(Annotations):
                 f"which was last modified on {lastmodstr}\n"
             )
 
-            fname = self._get_filename(data_adaptor)
+            fname = self._get_celllabels_filename(data_adaptor)
             self._backup(fname)
             if not df.empty:
                 with open(fname, "w", newline="") as f:
@@ -95,12 +105,56 @@ class AnnotationsLocalFile(Annotations):
             self.last_fname = fname
             self.last_labels = df
 
+    def read_genesets(self, data_adaptor, context=None):
+        if has_request_context():
+            if not current_app.auth.is_user_authenticated():
+                return ([], None)
+
+        fname = self._get_genesets_filename(data_adaptor)
+        genesets = {}
+        tid = None
+        with self.genesets_lock:
+            tid = self.last_geneset_tid  # inside the critical section
+            if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0:
+                with open(fname, newline="") as f:
+                    genesets = read_geneset_tidycsv(f, context)
+
+        return (genesets, tid)
+
+    def write_genesets(self, genesets, tid, data_adaptor):
+        self.check_genesets_save_enabled()  # raises
+
+        if type(tid) != int or tid < 0:
+            raise ValueError("tid must be a positive integer")
+
+        with self.genesets_lock:
+            # skip if the request is stale
+            if tid is not None:
+                if tid <= self.last_geneset_tid:
+                    raise ObsoleteRequest("TID is stale.")
+                self.last_geneset_tid = tid
+
+            lastmod = data_adaptor.get_last_mod_time()
+            lastmodstr = "'unknown'" if lastmod is None else lastmod.isoformat(timespec="seconds")
+            header = (
+                f"# Geneset generated on {datetime.now().isoformat(timespec='seconds')} "
+                f"using cellxgene version {cellxgene_version}\n"
+                f"# Input data file was {data_adaptor.get_location()}, "
+                f"which was last modified on {lastmodstr}\n"
+            )
+
+            fname = self._get_genesets_filename(data_adaptor)
+            self._backup(fname)
+            with open(fname, "w", newline="") as f:
+                f.write(header)
+                f.write(self.genesets_to_csv(genesets))
+
     def _get_userdata_idhash(self, data_adaptor):
         """
         Return a short hash that weakly identifies the user and dataset.
         Used to create safe annotations output file names.
         """
-        uid = current_app.auth.get_user_id()
+        uid = current_app.auth.get_user_id() or ""
         id = (uid + data_adaptor.get_location()).encode()
         idhash = base64.b32encode(blake2b(id, digest_size=5).digest()).decode("utf-8")
         return idhash
@@ -109,16 +163,27 @@ class AnnotationsLocalFile(Annotations):
         if self.output_dir:
             return self.output_dir
 
-        if self.output_file:
+        output_file = self.label_output_file or self.genesets_output_file
-            return os.path.dirname(self.path.abspath(self.output_dir))
+        if output_file:
+            return os.path.dirname(self.path.abspath(output_file))
 
         return os.getcwd()
 
-    def _get_filename(self, data_adaptor):
+    def _get_celllabels_filename(self, data_adaptor):
         """ return the current annotation file name """
-        if self.output_file:
+        if self.label_output_file:
-            return self.output_file
+            return self.label_output_file
 
+        return self._get_filename(data_adaptor, "celllabels")
+
+    def _get_genesets_filename(self, data_adaptor):
+        """ return the current genesets file name """
+        if self.genesets_output_file:
+            return self.genesets_output_file
+
+        return self._get_filename(data_adaptor, "genesets")
+
+    def _get_filename(self, data_adaptor, anno_name):
         # we need to generate a file name, which we can only do if we have a UID and collection name
         if session is None:
             raise AnnotationsError("unable to determine file name for annotations")
@@ -131,7 +196,7 @@ class AnnotationsLocalFile(Annotations):
             raise AnnotationsError("unable to determine file name for annotations")
 
         idhash = self._get_userdata_idhash(data_adaptor)
-        return os.path.join(self._get_output_dir(), f"{collection}-{idhash}.csv")
+        return os.path.join(self._get_output_dir(), f"{collection}-{anno_name}-{idhash}.csv")
 
     def _backup(self, fname, max_backups=9):
         """
@@ -170,7 +235,8 @@ class AnnotationsLocalFile(Annotations):
 
     def update_parameters(self, parameters, data_adaptor):
         params = {}
-        params["annotations"] = True
+        params["annotations"] = self.user_annotations_enabled()
+        params["annotations_genesets_readonly"] = not self.genesets_save_enabled()
         params["user_annotation_collection_name_enabled"] = True
 
         if self.ontology_data:
@@ -179,18 +245,108 @@ class AnnotationsLocalFile(Annotations):
         else:
             params["annotations_cell_ontology_enabled"] = False
 
-        if self.output_file is not None:
+        if self.label_output_file is not None:
-            # user has hard-wired the name of the annotation data collection
+            # user has hard-wired the name of the annotation cell label data collection
-            fname = os.path.basename(self.output_file)
+            fname = os.path.basename(self.label_output_file)
             collection_fname = os.path.splitext(fname)[0]
             params["annotations-data-collection-is-read-only"] = True
             params["annotations-data-collection-name"] = collection_fname
 
         elif session is not None:
             collection = self.get_collection()
-            if current_app.auth.is_user_authenticated():
+            params["annotations-data-collection-is-read-only"] = False
-                params["annotations-user-data-idhash"] = self._get_userdata_idhash(data_adaptor)
+            params["annotations-data-collection-name"] = collection
-                params["annotations-data-collection-is-read-only"] = False
+
-                params["annotations-data-collection-name"] = collection
+        if current_app.auth.is_user_authenticated():
+            params["annotations-user-data-idhash"] = self._get_userdata_idhash(data_adaptor)
 
         parameters.update(params)
+
+
+def read_geneset_tidycsv(f, context=None):
+    """
+    Read & parse the Tidy CSV format, applying validation checks for mandatory
+    values, and de-duping rules.
+
+    Format is a four-column CSV, with a mandatory header row, and optional "#" prefixed
+    comments.  Format:
+
+        geneset_name, geneset_description, gene_symbol, gene_description
+
+    geneset_name and gene_symbol must be non-null; others are optional.
+
+    Returns: a dictionary of the shape (values in angle-brackets vary):
+
+        {
+            <string, a gene set name>: {
+                "geneset_name": <string, a gene set name>,
+                "geneset_description": <a string or None>,
+                "genes": [
+                    {
+                        "gene_symbol": <string, a gene symbol or name>,
+                        "gene_description": <a string or None>
+                    },
+                    ...
+                ]
+            },
+            ...
+        }
+    """
+
+    class myDialect(csv.excel):
+        skipinitialspace = True
+
+    def just(n, seq):
+        it = iter(seq)
+        for _ in range(n - 1):
+            yield next(it, "")
+        yield tuple(it)
+
+    messagefn = context["messagefn"] if context else (lambda x: None)
+
+    reader = csv.reader(f, dialect=myDialect())
+    genesets = {}
+    haveReadHeader = False
+    lineno = 0
+    for row in reader:
+        lineno += 1
+        # ignore empty rows
+        if len(row) == 0:
+            continue
+        # if row starts with '#' it is a comment
+        if row[0].startswith("#"):
+            continue
+        # if this is the first non-comment row, assume it is a header
+        if not haveReadHeader:
+            if row != Annotations.Genesets_Header:
+                raise AnnotationsError("Geneset CSV file missing the required column header.")
+            haveReadHeader = True
+            continue
+
+        geneset_name, geneset_description, gene_symbol, gene_description, _ = just(5, row)
+        if not geneset_name:
+            raise AnnotationsError(f"Geneset CSV missing required geneset or gene name on line {lineno}")
+        if (not gene_symbol) and gene_description:
+            messagefn(f"Warning: Missing gene name in geneset name {geneset_name} on line {lineno}.")
+
+        if geneset_name in genesets:
+            gs = genesets[geneset_name]
+        else:
+            gs = genesets[geneset_name] = {
+                "geneset_name": geneset_name,
+                "geneset_description": geneset_description,
+                "genes": [],
+            }
+        # Use first geneset_description with a value
+        if not gs["geneset_description"] and geneset_description:
+            gs["geneset_description"] = geneset_description
+        # add the gene if the gene_symbol is defined
+        if gene_symbol:
+            gs["genes"].append(
+                {
+                    "gene_symbol": gene_symbol,
+                    "gene_description": gene_description,
+                }
+            )
+
+    return genesets

local_server/common/config/client_config.py

@@ -44,6 +44,9 @@ def get_client_config(app_config, data_adaptor):
         "annotations": False,
         "annotations_file": None,
         "annotations_dir": None,
+        "annotations_genesets": True,  # feature flag
+        "annotations_genesets_readonly": dataset_config.user_annotations__genesets__readonly,
+        "annotations_genesets_summary_methods": ["mean"],
         "annotations_cell_ontology_enabled": False,
         "annotations_cell_ontology_obopath": None,
         "annotations_cell_ontology_terms": None,

local_server/common/config/dataset_config.py

@@ -3,7 +3,7 @@ from os.path import splitext, isdir
 
 from local_server.common.annotations.local_file_csv import AnnotationsLocalFile
 from local_server.common.config.base_config import BaseConfig
-from local_server.common.errors import ConfigurationError, OntologyLoadFailure
+from local_server.common.errors import ConfigurationError, OntologyLoadFailure, AnnotationsError
 from local_server.compute.scanpy import get_scanpy_module
 from local_server.data_common.matrix_loader import MatrixDataLoader
 
@@ -32,6 +32,10 @@ class DatasetConfig(BaseConfig):
             self.user_annotations__ontology__obo_location = default_config["user_annotations"]["ontology"][
                 "obo_location"
             ]
+            self.user_annotations__genesets__readonly = default_config["user_annotations"]["genesets"]["readonly"]
+            self.user_annotations__local_file_csv__genesets_file = default_config["user_annotations"]["local_file_csv"][
+                "genesets_file"
+            ]
 
             self.embeddings__names = default_config["embeddings"]["names"]
             self.embeddings__enable_reembedding = default_config["embeddings"]["enable_reembedding"]
@@ -56,9 +60,7 @@ class DatasetConfig(BaseConfig):
     def get_data_adaptor(self):
         server_config = self.app_config.server_config
         if not server_config.data_adaptor:
-            matrix_data_loader = MatrixDataLoader(
+            matrix_data_loader = MatrixDataLoader(server_config.single_dataset__datapath, app_config=self.app_config)
-                server_config.single_dataset__datapath, app_config=self.app_config
-            )
             server_config.data_adaptor = matrix_data_loader.open(self.app_config)
 
         return server_config.data_adaptor
@@ -96,11 +98,16 @@ class DatasetConfig(BaseConfig):
         self.validate_correct_type_of_configuration_attribute(
             "user_annotations__local_file_csv__file", (type(None), str)
         )
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__local_file_csv__genesets_file", (type(None), str)
+        )
         self.validate_correct_type_of_configuration_attribute("user_annotations__ontology__enable", bool)
         self.validate_correct_type_of_configuration_attribute(
             "user_annotations__ontology__obo_location", (type(None), str)
         )
-        if self.user_annotations__enable:
+        self.validate_correct_type_of_configuration_attribute("user_annotations__genesets__readonly", bool)
+
+        if self.user_annotations__enable or not self.user_annotations__genesets__readonly:
             server_config = self.app_config.server_config
             if not self.app__authentication_enable:
                 raise ConfigurationError("user annotations requires authentication to be enabled")
@@ -108,59 +115,84 @@ class DatasetConfig(BaseConfig):
                 auth_type = server_config.authentication__type
                 raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations")
 
-            if self.user_annotations__type == "local_file_csv":
+        # Must always have an annotations instance to support genesets. User annotation (cell labels) are optional
-                self.handle_local_file_csv_annotations()
+        # as are writable gene sets
-            else:
+        if self.user_annotations__type == "local_file_csv":
-                raise ConfigurationError('The only annotation type support is "local_file_csv"')
+            self.handle_local_file_csv_annotations(context)
+        else:
+            raise ConfigurationError('The only annotation type support is "local_file_csv"')
+
+        if self.user_annotations__enable:
             if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location:
                 try:
                     self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location)
                 except OntologyLoadFailure as e:
                     raise ConfigurationError("Unable to load ontology terms\n" + str(e))
-        else:
-            self.check_annotation_config_vars_not_set(context)
 
-    def handle_local_file_csv_annotations(self):
+        self.check_annotation_config_vars_not_set(context)
+
+    def handle_local_file_csv_annotations(self, context):
         dirname = self.user_annotations__local_file_csv__directory
         filename = self.user_annotations__local_file_csv__file
-        if filename is not None and dirname is not None:
+        genesets_filename = self.user_annotations__local_file_csv__genesets_file
-            raise ConfigurationError("'annotations-file' and 'annotations-dir' may not be used together.")
+
+        if dirname is not None and (filename is not None or genesets_filename is not None):
+            raise ConfigurationError(
+                "'user-generated-data-dir' may not be used with annotations-file' or 'genesets-file'."
+            )
 
         if filename is not None:
             lf_name, lf_ext = splitext(filename)
             if lf_ext and lf_ext != ".csv":
                 raise ConfigurationError(f"annotation file type must be .csv: {filename}")
 
+        if genesets_filename is not None:
+            lf_name, lf_ext = splitext(genesets_filename)
+            if lf_ext and lf_ext != ".csv":
+                raise ConfigurationError(f"genesets file type must be .csv: {genesets_filename}")
+
         if dirname is not None and not isdir(dirname):
             try:
                 os.mkdir(dirname)
             except OSError:
-                raise ConfigurationError("Unable to create directory specified by --annotations-dir")
+                raise ConfigurationError("Unable to create directory specified by --user-generated-data-dir")
 
-        self.user_annotations = AnnotationsLocalFile(dirname, filename)
+        anno_config = {
+            "user-annotations": self.user_annotations__enable,
+            "genesets-save": not self.user_annotations__genesets__readonly,
+        }
+        self.user_annotations = AnnotationsLocalFile(anno_config, dirname, filename, genesets_filename)
 
         # if the user has specified a fixed label file, go ahead and validate it
         # so that we can remove errors early in the process.
         server_config = self.app_config.server_config
-        if server_config.single_dataset__datapath and self.user_annotations__local_file_csv__file:
+        if server_config.single_dataset__datapath:
             data_adaptor = self.get_data_adaptor()
-            data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor))
+            if self.user_annotations__local_file_csv__file:
+                data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor))
+            if self.user_annotations__local_file_csv__genesets_file:
+                try:
+                    data_adaptor.check_new_genesets(self.user_annotations.read_genesets(data_adaptor, context), context)
+                except (ValueError, AnnotationsError, KeyError) as e:
+                    raise ConfigurationError(f"Unable to read genesets CSV file: {str(e)}") from e
 
     def check_annotation_config_vars_not_set(self, context):
         if self.user_annotations__type is not None:
             dirname = self.user_annotations__local_file_csv__directory
             filename = self.user_annotations__local_file_csv__file
-            if filename is not None:
+            if not self.user_annotations__enable:
-                context["messagefn"]("Warning: --annotations-file ignored as annotations are disabled.")
+                if filename is not None:
-            if dirname is not None:
+                    context["messagefn"]("Warning: --annotations-file ignored as annotations are disabled.")
-                context["messagefn"]("Warning: --annotations-dir ignored as annotations are disabled.")
+                if self.user_annotations__ontology__enable:
-
+                    context["messagefn"](
-        if self.user_annotations__ontology__enable:
+                        "Warning: --experimental-annotations-ontology ignored as annotations are disabled."
-            context["messagefn"]("Warning: --experimental-annotations-ontology ignored as annotations are disabled.")
+                    )
-        if self.user_annotations__ontology__obo_location is not None:
+                if self.user_annotations__ontology__obo_location is not None:
-            context["messagefn"](
+                    context["messagefn"](
-                "Warning: --experimental-annotations-ontology-obo ignored as annotations are disabled."
+                        "Warning: --experimental-annotations-ontology-obo ignored as annotations are disabled."
-            )
+                    )
+                if dirname is not None:
+                    context["messagefn"]("Warning: --user-generated-data-dir ignored as annotations are disabled.")
 
     def handle_embeddings(self):
         self.validate_correct_type_of_configuration_attribute("embeddings__names", list)
@@ -186,6 +218,5 @@ class DatasetConfig(BaseConfig):
         data_adaptor = self.get_data_adaptor()
         if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False):
             context["messagefn"](
-                "CAUTION: due to the size of your dataset, "
+                "CAUTION: due to the size of your dataset, " "running differential expression may take longer or fail."
-                "running differential expression may take longer or fail."
             )

local_server/common/errors.py

@@ -55,3 +55,4 @@ define_exception("OntologyLoadFailure", "Raised when reading the ontology file f
 define_exception("ConfigurationError", "Raised when checking configuration errors")
 define_exception("PrepareError", "Raised when data is misprepared")
 define_exception("SecretKeyRetrievalError", "Raised when get_secret_key from AWS fails")
+define_exception("ObsoleteRequest", "Raised when the request is no longer valid.")

local_server/common/rest.py

@@ -17,6 +17,8 @@ from local_server.common.errors import (
     ExceedsLimitError,
     DatasetAccessError,
     ColorFormatException,
+    AnnotationsError,
+    ObsoleteRequest,
 )
 
 import json
@@ -44,7 +46,7 @@ def _query_parameter_to_filter(args):
 
     Query param filters look like:  <axis>:name=value, where value
     may be one of:
-        - a range, min,max, where either may be an open range by using an asterisc, eg, 10,*
+        - a range, min,max, where either may be an open range by using an asterisk, eg, 10,*
         - a value
     Eg,
         ...?tissue=lung&obs:tissue=heart&obs:num_reads=1000,*
@@ -106,7 +108,7 @@ def schema_get_helper(data_adaptor):
 
     # add label obs annotations as needed
     annotations = data_adaptor.dataset_config.user_annotations
-    if annotations is not None:
+    if annotations.user_annotations_enabled():
         label_schema = annotations.get_schema(data_adaptor)
         schema["annotations"]["obs"]["columns"].extend(label_schema)
 
@@ -140,7 +142,7 @@ def annotations_obs_get(request, data_adaptor):
     try:
         labels = None
         annotations = data_adaptor.dataset_config.user_annotations
-        if annotations:
+        if annotations.user_annotations_enabled():
             labels = annotations.read_labels(data_adaptor)
         fbs = data_adaptor.annotation_to_fbs_matrix(Axis.OBS, fields, labels)
         return make_response(fbs, HTTPStatus.OK, {"Content-Type": "application/octet-stream"})
@@ -151,7 +153,7 @@ def annotations_obs_get(request, data_adaptor):
 def annotations_put_fbs_helper(data_adaptor, fbs):
     """helper function to write annotations from fbs"""
     annotations = data_adaptor.dataset_config.user_annotations
-    if annotations is None:
+    if not annotations.user_annotations_enabled():
         raise DisabledFeatureError("Writable annotations are not enabled")
 
     new_label_df = decode_matrix_fbs(fbs)
@@ -166,7 +168,7 @@ def inflate(data):
 
 def annotations_obs_put(request, data_adaptor):
     annotations = data_adaptor.dataset_config.user_annotations
-    if annotations is None:
+    if not annotations.user_annotations_enabled():
         return abort(HTTPStatus.NOT_IMPLEMENTED)
 
     anno_collection = request.args.get("annotation-collection-name", default=None)
@@ -196,9 +198,6 @@ def annotations_var_get(request, data_adaptor):
 
     try:
         labels = None
-        annotations = data_adaptor.dataset_config.user_annotations
-        if annotations is not None:
-            labels = annotations.read_labels(data_adaptor)
         return make_response(
             data_adaptor.annotation_to_fbs_matrix(Axis.VAR, fields, labels),
             HTTPStatus.OK,
@@ -328,3 +327,56 @@ def layout_obs_put(request, data_adaptor):
         return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, str(e))
     except (ValueError, DisabledFeatureError, FilterError) as e:
         return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+
+
+def genesets_get(request, data_adaptor):
+    preferred_mimetype = request.accept_mimetypes.best_match(["application/json", "text/csv"])
+    if preferred_mimetype not in ("application/json", "text/csv"):
+        return abort(HTTPStatus.NOT_ACCEPTABLE)
+
+    try:
+        annotations = data_adaptor.dataset_config.user_annotations
+        (genesets, tid) = data_adaptor.check_new_genesets(annotations.read_genesets(data_adaptor))
+
+        if preferred_mimetype == "text/csv":
+            return make_response(
+                annotations.genesets_to_csv(genesets),
+                HTTPStatus.OK,
+                {
+                    "Content-Type": "text/csv",
+                    "Content-Disposition": "attachment; filename=genesets.csv",
+                },
+            )
+        else:
+            return make_response(
+                jsonify({"genesets": annotations.genesets_to_response(genesets), "tid": tid}), HTTPStatus.OK
+            )
+    except (ValueError, KeyError, AnnotationsError) as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e))
+
+
+def genesets_put(request, data_adaptor):
+    annotations = data_adaptor.dataset_config.user_annotations
+    if not annotations.genesets_save_enabled():
+        return abort(HTTPStatus.NOT_IMPLEMENTED)
+
+    anno_collection = request.args.get("annotation-collection-name", default=None)
+    if anno_collection is not None:
+        if not annotations.is_safe_collection_name(anno_collection):
+            return abort(HTTPStatus.BAD_REQUEST, "Bad annotation collection name")
+        annotations.set_collection(anno_collection)
+
+    args = request.get_json()
+    try:
+        genesets = args.get("genesets", None)
+        tid = args.get("tid", None)
+        if genesets is None:
+            abort(HTTPStatus.BAD_REQUEST)
+
+        (gs, _) = data_adaptor.check_new_genesets((genesets, tid))
+        annotations.write_genesets(gs, tid, data_adaptor)
+        return make_response(jsonify({"status": "OK"}), HTTPStatus.OK)
+    except (ValueError, DisabledFeatureError, KeyError) as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+    except (ObsoleteRequest, TypeError) as e:
+        return abort(HTTPStatus.NOT_FOUND, description=str(e))

local_server/data_common/data_adaptor.py

@@ -1,5 +1,6 @@
 from abc import ABCMeta, abstractmethod
 from os.path import basename, splitext
+import re
 
 import numpy as np
 import pandas as pd
@@ -261,6 +262,95 @@ class DataAdaptor(metaclass=ABCMeta):
 
         return labels_df
 
+    def check_new_genesets(self, args, context=None):
+        """
+        Check validity of gene sets, return if correct, else raise error.
+        May also modify the gene set for conditions that should be resolved,
+        but which do not warrant a hard error.
+
+        Argument 'args' must be a tuple containing (genesets, tid).  Genesets
+        may be either the REST OTA format (list of dicts) or the internal format
+        (dict of dicts, keyed by the geneset name).
+
+        Rules:
+        0. all geneset names must be unique.
+        1. All geneset names must be legal, meaning:
+            * no leading or trailing white space
+            * no multi-space runs
+            * character set matches: [A-Z][a-z][0-9][ .()-]
+           Generates hard error.
+        2. Gene symbols must be part of the current var_index.  If symbol not in var_index,
+           will generate a warning and the symbol removed.
+        3. Duplicate gene symbols are silently de-duped.
+        """
+        (genesets, tid) = args
+        messagefn = context["messagefn"] if context else (lambda x: None)
+
+        # accept genesets args as either the internal (dict) or REST (list) format,
+        # as they are identical except for the dict being keyed by geneset_name.
+        if type(genesets) not in (dict, list):
+            raise ValueError("Genesets must be either dict or list.")
+        genesets = genesets if type(genesets) == list else genesets.values()
+
+        # 0. check for uniqueness of geneset names
+        geneset_names = [gs["geneset_name"] for gs in genesets]
+        if len(set(geneset_names)) != len(geneset_names):
+            raise KeyError("All geneset names must be unique.")
+
+        # 1. check gene set character set and format
+        legal_name = re.compile(r"^(\w|[ .()-])+$")
+        for name in geneset_names:
+            if type(name) != str or len(name) == 0:
+                raise KeyError("Geneset names must be non-null string.")
+            if name[0] in " \t\n\r" or name[-1] in " \t\n\r" or not legal_name.match(name) or "  " in name:
+                messagefn(
+                    "Error: "
+                    f"Geneset name {name} is not valid. Only alphanumeric and limited special characters (-_.) "
+                    "and space are allowed. Leading, trailing, and multiple spaces within a name are not allowed."
+                )
+                raise KeyError(
+                    "Geneset name is not valid, only alphanumeric and limited special characters (-_.) "
+                    "and space are allowed. Leading, trailing, and multiple spaces within a name are not allowed."
+                )
+
+        # 2. & 3. check for duplicate gene symbols, and those not present in the dataset. They will
+        # generate a warning and be removed.
+        var_names = set(self.query_var_array(self.parameters.get("var_names")))
+        for geneset in genesets:
+            if type(geneset) != dict:
+                raise ValueError("Each geneset must be a dict.")
+            geneset_name = geneset["geneset_name"]
+            genes = geneset["genes"]
+            if type(genes) != list:
+                raise ValueError("Geneset genes field must be a list")
+            gene_symbol_already_seen = set()
+            new_genes = []
+            for gene in genes:
+                gene_symbol = gene["gene_symbol"]
+                if type(gene_symbol) != str or len(gene_symbol) == 0:
+                    raise ValueError("Gene symbol must be non-null string.")
+                if gene_symbol in gene_symbol_already_seen:
+                    # duplicate check
+                    messagefn(
+                        f"Warning: a duplicate of gene {gene_symbol} was found in geneset {geneset_name}, "
+                        "and will be ignored."
+                    )
+                    continue
+
+                if gene_symbol not in var_names:
+                    messagefn(
+                        f"Warning: {gene_symbol}, used in geneset {geneset_name}, "
+                        "was not found in the dataset and will be ignored."
+                    )
+                    continue
+
+                gene_symbol_already_seen.add(gene_symbol)
+                new_genes.append(gene)
+
+            geneset["genes"] = new_genes
+
+        return args
+
     def data_frame_to_fbs_matrix(self, filter, axis):
         """
         Retrieves data 'X' and returns in a flatbuffer Matrix.
@@ -333,7 +423,7 @@ class DataAdaptor(metaclass=ABCMeta):
     @staticmethod
     def normalize_embedding(embedding):
         """Normalize embedding layout to meet client assumptions.
-           Embedding is an ndarray, shape (n_obs, n)., where n is normally 2
+        Embedding is an ndarray, shape (n_obs, n)., where n is normally 2
         """
 
         # scale isotropically

local_server/default_config.py

@@ -63,10 +63,13 @@ dataset:
     type: local_file_csv
     local_file_csv:
       directory: null
-      file: null
+      file: null            # annotations file name
+      genesets_file: null   # gene sets file name
     ontology:
       enable: false
       obo_location: null
+    genesets:
+      readonly: false      # genesets CRUD enabled/disabled
 
   embeddings:
     names : []

local_server/test/__init__.py

@@ -46,7 +46,11 @@ def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False):
 
     config.complete_config()
     data = MatrixDataLoader(data_locator.abspath()).open(config)
-    annotations = AnnotationsLocalFile(None, annotations_file)
+    anno_config = {
+        "user-annotations": True,
+        "genesets-save": False,
+    }
+    annotations = AnnotationsLocalFile(anno_config, None, annotations_file, None)
     return data, tmp_dir, annotations
 
 

local_server/test/fixtures/dataset_config_outline.py

@@ -16,9 +16,12 @@ dataset:
     local_file_csv:
       directory: {local_file_csv_directory}
       file: {local_file_csv_file}
+      genesets_file: {local_file_csv_genesets_file}
     ontology:
       enable: {ontology_enabled}
       obo_location: {obo_location}
+    genesets:
+      readonly: {genesets_readonly}
 
   embeddings:
     names: {embedding_names}

local_server/test/fixtures/pbmc3k-genesets.csv

@@ -0,0 +1,12 @@
+# Test fixture
+geneset_name, geneset_description, gene_symbol, gene_description
+first geneset name,,F5, a gene_description
+first geneset name,a description, NO_SUCH_GENE, non-existent gene
+first geneset name,a description, F5, duplicate gene
+first geneset name, a description, SUMO3,
+first geneset name,, SRM,
+second geneset,,RER1
+second geneset,,SIK1
+third geneset,,NO_SUCH_GENE
+fourth_geneset,fourth description,,gene intentionally missing
+fifth_dataset,,,

local_server/test/unit/auth/test_auth.py

@@ -14,7 +14,7 @@ class AuthTest(unittest.TestCase):
         app_config = AppConfig()
         app_config.update_server_config(app__flask_secret_key="secret")
         app_config.update_server_config(authentication__type=None, single_dataset__datapath=self.dataset_datapath)
-        app_config.update_dataset_config(user_annotations__enable=False)
+        app_config.update_dataset_config(user_annotations__enable=False, user_annotations__genesets__readonly=True)
 
         app_config.complete_config()
 

local_server/test/unit/common/config/__init__.py

@@ -92,8 +92,10 @@ class ConfigTests(unittest.TestCase):
         hosted_file_directory="null",
         local_file_csv_directory="null",
         local_file_csv_file="null",
+        local_file_csv_genesets_file="null",
         ontology_enabled="false",
         obo_location="null",
+        genesets_readonly="false",
         embedding_names=[],
         enable_reembedding="false",
         enable_difexp="true",
@@ -142,8 +144,10 @@ class ConfigTests(unittest.TestCase):
             hosted_file_directory=hosted_file_directory,
             local_file_csv_directory=local_file_csv_directory,
             local_file_csv_file=local_file_csv_file,
+            local_file_csv_genesets_file=local_file_csv_genesets_file,
             ontology_enabled=ontology_enabled,
             obo_location=obo_location,
+            genesets_readonly=genesets_readonly,
             embedding_names=embedding_names,
             enable_reembedding=enable_reembedding,
             enable_difexp=enable_difexp,
@@ -178,8 +182,10 @@ class ConfigTests(unittest.TestCase):
         hosted_file_directory="null",
         local_file_csv_directory="null",
         local_file_csv_file="null",
+        local_file_csv_genesets_file="null",
         ontology_enabled="false",
         obo_location="null",
+        genesets_readonly="false",
         embedding_names=[],
         enable_reembedding="false",
         enable_difexp="true",

local_server/test/unit/common/config/test_dataset_config.py

@@ -47,7 +47,7 @@ class TestDatasetConfig(ConfigTests):
         mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
         self.dataset_config.complete_config(self.context)
         self.assertIsNotNone(self.config.server_config.data_adaptor)
-        self.assertEqual(mock_check_attrs.call_count, 17)
+        self.assertEqual(mock_check_attrs.call_count, 19)
 
     def test_app_sets_script_vars(self):
         config = self.get_config(scripts=["path/to/script"])
@@ -102,7 +102,7 @@ class TestDatasetConfig(ConfigTests):
             enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv"
         )
         config.server_config.complete_config(self.context)
-        config.dataset_config.handle_local_file_csv_annotations()
+        config.dataset_config.handle_local_file_csv_annotations(self.context)
         self.assertIsInstance(config.dataset_config.user_annotations, AnnotationsLocalFile)
         cwd = os.getcwd()
         self.assertEqual(config.dataset_config.user_annotations._get_output_dir(), cwd)

local_server/test/unit/common/test_api.py

@@ -3,6 +3,8 @@ import time
 import unittest
 import zlib
 from http import HTTPStatus
+import tempfile
+from os import path
 
 import pandas as pd
 import requests
@@ -13,6 +15,7 @@ from local_server.test import (
     data_with_tmp_annotations,
     make_fbs,
     PROJECT_ROOT,
+    FIXTURES_ROOT,
     start_test_server,
     stop_test_server,
 )
@@ -26,6 +29,7 @@ BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
 
 class EndPoints(object):
     ANNOTATIONS_ENABLED = True
+    GENESETS_READONLY = False
 
     def test_initialize(self):
         endpoint = "schema"
@@ -49,6 +53,7 @@ class EndPoints(object):
         result_data = result.json()
         self.assertIn("library_versions", result_data["config"])
         self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k")
+        self.assertIsNotNone(result_data["config"]["parameters"])
 
     def test_get_layout_fbs(self):
         endpoint = "layout/obs"
@@ -286,6 +291,26 @@ class EndPoints(object):
         result = self.session.get(url)
         self.assertEqual(result.status_code, HTTPStatus.OK)
 
+    def test_genesets_config(self):
+        result = self.session.get(f"{self.URL_BASE}config")
+        config_data = result.json()
+        params = config_data["config"]["parameters"]
+        annotations_genesets = params["annotations_genesets"]
+        annotations_genesets_readonly = params["annotations_genesets_readonly"]
+        annotations_genesets_summary_methods = params["annotations_genesets_summary_methods"]
+        self.assertTrue(annotations_genesets)
+        self.assertEqual(annotations_genesets_readonly, self.GENESETS_READONLY)
+        self.assertEqual(annotations_genesets_summary_methods, ["mean"])
+
+    def test_get_genesets(self):
+        endpoint = "genesets"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.get(url, headers={"Accept": "application/json"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        self.assertIsNotNone(result_data["genesets"])
+
     def _setupClass(child_class, command_line):
         child_class.ps, child_class.server = start_test_server(command_line)
         child_class.URL_BASE = f"{child_class.server}/api/v0.2/"
@@ -304,7 +329,8 @@ class EndPointsAnnotations(EndPoints):
 
     def test_get_user_annotations_existing_obs_keys_fbs(self):
         self._test_get_user_annotations_obs_keys_fbs(
-            "cluster-test", {"unassigned", "one", "two", "three", "four", "five", "six", "seven"},
+            "cluster-test",
+            {"unassigned", "one", "two", "three", "four", "five", "six", "seven"},
         )
 
     def test_put_user_annotations_obs_fbs(self):
@@ -353,6 +379,7 @@ class EndPointsAnndata(unittest.TestCase, EndPoints):
     """Test Case for endpoints"""
 
     ANNOTATIONS_ENABLED = False
+    GENESETS_READONLY = True
 
     @classmethod
     def setUpClass(cls):
@@ -361,6 +388,7 @@ class EndPointsAnndata(unittest.TestCase, EndPoints):
             [
                 f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
                 "--disable-annotations",
+                "--disable-genesets-save",
                 "--experimental-enable-reembedding",
             ],
         )
@@ -408,15 +436,249 @@ class EndPointsAnndataAnnotations(unittest.TestCase, EndPointsAnnotations):
     """Test Case for endpoints"""
 
     ANNOTATIONS_ENABLED = True
+    GENESETS_READONLY = False
 
     @classmethod
     def setUpClass(cls):
         cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations(
             MatrixDataType.H5AD, annotations_fixture=True
         )
-        cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location()])
+        cls._setupClass(cls, ["--annotations-file", cls.annotations.label_output_file, cls.data.get_location()])
 
     @classmethod
     def tearDownClass(cls):
         shutil.rmtree(cls.tmp_dir)
         stop_test_server(cls.ps)
+
+
+class EndPointsAnnDataGenesets(unittest.TestCase, EndPoints):
+    ANNOTATIONS_ENABLED = False
+    GENESETS_READONLY = False
+
+    @classmethod
+    def setUpClass(cls):
+        cls.tmp_dir = tempfile.mkdtemp()
+        genesets_file = path.join(cls.tmp_dir, "test_genesets.csv")
+        shutil.copyfile(f"{FIXTURES_ROOT}/pbmc3k-genesets.csv", genesets_file)
+        cls._setupClass(
+            cls,
+            [
+                f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
+                "--disable-annotations",
+                "--genesets-file",
+                genesets_file,
+            ],
+        )
+
+    @classmethod
+    def tearDownClass(cls):
+        shutil.rmtree(cls.tmp_dir)
+        stop_test_server(cls.ps)
+
+    def test_get_genesets_json(self):
+        endpoint = "genesets"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.get(url, headers={"Accept": "application/json"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        self.assertIsNotNone(result_data["genesets"])
+        self.assertIsNotNone(result_data["tid"])
+
+        self.assertEqual(
+            result_data,
+            {
+                "genesets": [
+                    {
+                        "genes": [
+                            {"gene_description": "a gene_description", "gene_symbol": "F5"},
+                            {"gene_description": "", "gene_symbol": "SUMO3"},
+                            {"gene_description": "", "gene_symbol": "SRM"},
+                        ],
+                        "geneset_description": "a description",
+                        "geneset_name": "first geneset name",
+                    },
+                    {
+                        "genes": [
+                            {"gene_description": "", "gene_symbol": "RER1"},
+                            {"gene_description": "", "gene_symbol": "SIK1"},
+                        ],
+                        "geneset_description": "",
+                        "geneset_name": "second geneset",
+                    },
+                    {"genes": [], "geneset_description": "", "geneset_name": "third geneset"},
+                    {"genes": [], "geneset_description": "fourth description", "geneset_name": "fourth_geneset"},
+                    {"genes": [], "geneset_description": "", "geneset_name": "fifth_dataset"},
+                ],
+                "tid": 0,
+            },
+        )
+
+    def test_get_genesets_csv(self):
+        endpoint = "genesets"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.get(url, headers={"Accept": "text/csv"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "text/csv")
+        self.assertEqual(
+            result.text,
+            """geneset_name,geneset_description,gene_symbol,gene_description\r
+first geneset name,a description,F5,a gene_description\r
+first geneset name,a description,SUMO3,\r
+first geneset name,a description,SRM,\r
+second geneset,,RER1,\r
+second geneset,,SIK1,\r
+third geneset,,,\r
+fourth_geneset,fourth description,,\r
+fifth_dataset,,,\r
+""",
+        )
+
+    def test_put_genesets(self):
+        endpoint = "genesets"
+        url = f"{self.URL_BASE}{endpoint}"
+
+        # assume we start with TID 0
+        result = self.session.get(url, headers={"Accept": "application/json"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.json()["tid"], 0)
+
+        test1 = {"tid": 3, "genesets": []}
+        result = self.session.put(url, json=test1)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        result = self.session.get(url, headers={"Accept": "application/json"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.json(), test1)
+
+        # stale TID
+        result = self.session.put(url, json=test1)
+        self.assertEqual(result.status_code, HTTPStatus.NOT_FOUND)
+
+        test2 = {"tid": 4, "genesets": [{"geneset_name": "foobar", "genes": []}]}
+        test2_response = {"tid": 4, "genesets": [{"geneset_name": "foobar", "geneset_description": "", "genes": []}]}
+        result = self.session.put(url, json=test2)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        result = self.session.get(url, headers={"Accept": "application/json"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.json(), test2_response)
+
+        test3 = {
+            "tid": 5,
+            "genesets": [
+                {
+                    "geneset_name": "foobar",
+                    "geneset_description": "",
+                    "genes": [
+                        {
+                            "gene_symbol": "F5",
+                            "gene_description": "",
+                        }
+                    ],
+                }
+            ],
+        }
+        result = self.session.put(url, json=test3)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        result = self.session.get(url, headers={"Accept": "application/json"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.json(), test3)
+
+    def test_put_genesets_malformed(self):
+        """ test malformed submissions that we expect the backend to catch/tolerate """
+        endpoint = "genesets"
+        url = f"{self.URL_BASE}{endpoint}"
+
+        result = self.session.get(url, headers={"Accept": "application/json"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        original_data = result.json()
+        tid = original_data["tid"]
+
+        def test_case(test, expected_code, original_data):
+            """ check for expected error AND that no change was made to the original state """
+            result = self.session.put(url, json=test)
+            self.assertEqual(result.status_code, expected_code)
+            result = self.session.get(url, headers={"Accept": "application/json"})
+            self.assertEqual(result.status_code, HTTPStatus.OK)
+            self.assertEqual(result.json(), original_data)
+
+        # missing or malformed genesets
+        test_case(
+            {"tid": tid + 1},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+        test_case(
+            {"tid": tid + 1, "genesets": 99},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+
+        # illegal geneset_name
+        test_case(
+            {"tid": tid + 1, "genesets": [{"geneset_name": """, "genes": []}]},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+
+        # duplicate geneset_name
+        test_case(
+            {
+                "tid": tid + 1,
+                "genesets": [
+                    {"geneset_name": "foo", "genes": []},
+                    {"geneset_name": "foo", "genes": []},
+                ],
+            },
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+
+        # missing geneset_name
+        test_case(
+            {"tid": tid + 1, "genesets": [{"genes": []}]},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+
+        # non-numeric TID
+        test_case(
+            {"tid": [], "genesets": [{"geneset_name": "foo", "genes": []}]},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+        test_case(
+            {"tid": None, "genesets": [{"geneset_name": "foo", "genes": []}]},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+        test_case(
+            {"tid": "not a number", "genesets": [{"geneset_name": "foo", "genes": []}]},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+
+        # duplicate gene_symbol
+        test_case(
+            {
+                "tid": "not a number",
+                "genesets": [{"geneset_name": "foo", "genes": [{"gene_symbol": "SIK1"}, {"gene_symbol": "SIK1"}]}],
+            },
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+
+        # gene_symbol is not a string
+        test_case(
+            {
+                "tid": "not a number",
+                "genesets": [{"geneset_name": "foo", "genes": [{"gene_symbol": 99}]}],
+            },
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+
+    """
+    TODO once we have some code to support it:
+    1. GET genesets_summary
+    2. genesets_summary obeys tid
+    """

local_server/test/unit/common/test_writable_annotation.py

@@ -45,8 +45,8 @@ class WritableAnnotationTest(unittest.TestCase):
         )
         res = self.annotation_put_fbs(fbs)
         self.assertEqual(res, json.dumps({"status": "OK"}))
-        self.assertTrue(path.exists(self.annotations.output_file))
+        self.assertTrue(path.exists(self.annotations.label_output_file))
-        df = pd.read_csv(self.annotations.output_file, index_col=0, header=0, comment="#")
+        df = pd.read_csv(self.annotations.label_output_file, index_col=0, header=0, comment="#")
         self.assertEqual(df.shape, (n_rows, 2))
         self.assertEqual(set(df.columns), {"cat_A", "cat_B"})
         self.assertTrue(self.data.original_obs_index.equals(df.index))
@@ -62,14 +62,14 @@ class WritableAnnotationTest(unittest.TestCase):
         )
         res = self.annotation_put_fbs(fbs)
         self.assertEqual(res, json.dumps({"status": "OK"}))
-        self.assertTrue(path.exists(self.annotations.output_file))
+        self.assertTrue(path.exists(self.annotations.label_output_file))
-        df = pd.read_csv(self.annotations.output_file, index_col=0, header=0, comment="#")
+        df = pd.read_csv(self.annotations.label_output_file, index_col=0, header=0, comment="#")
         self.assertEqual(set(df.columns), {"cat_A", "cat_C"})
         self.assertTrue(np.all(df["cat_A"] == ["label_A1"] * n_rows))
         self.assertTrue(np.all(df["cat_C"] == ["label_C"] * n_rows))
 
         # rotation
-        name, ext = path.splitext(self.annotations.output_file)
+        name, ext = path.splitext(self.annotations.label_output_file)
         backup_dir = f"{name}-backups"
         self.assertTrue(path.isdir(backup_dir))
         found_files = listdir(backup_dir)
@@ -88,7 +88,7 @@ class WritableAnnotationTest(unittest.TestCase):
             res = self.annotation_put_fbs(fbs)
             self.assertEqual(res, json.dumps({"status": "OK"}))
 
-        name, ext = path.splitext(self.annotations.output_file)
+        name, ext = path.splitext(self.annotations.label_output_file)
         backup_dir = f"{name}-backups"
         self.assertTrue(path.isdir(backup_dir))
         found_files = listdir(backup_dir)