hosted gene sets routes, plus a few bug fixes (#2155)

* first cut at hosted gs routes

* lint

* update tests to match csv parser changes

* update tests to new API

* update gene set name validation rules to match requirements

* add path mapping from dataset to geneset

* add test cases for geneset GET route

* fix test assertion

* remove debugging code

* update gene set uri mapping function

* fix error message

* allow extra user-specified headers in gene set csv file

* clarify comment

backend/common/genesets.py

@@ -0,0 +1,239 @@
+"""
+Utility code for gene sets handling
+"""
+
+import re
+import csv
+import hashlib
+
+from .errors import AnnotationsError
+
+
+GENESETS_TIDYCSV_HEADER = [
+    "gene_set_name",
+    "gene_set_description",
+    "gene_symbol",
+    "gene_description",
+]
+
+
+def read_gene_sets_tidycsv(gs_locator, context=None):
+    """
+    Read & parse the Tidy CSV format, applying validation checks for mandatory
+    values, and de-duping rules.
+
+    Format is a four-column CSV, with a mandatory header row, and optional "#" prefixed
+    comments.  Format:
+
+        gene_set_name, gene_set_description, gene_symbol, gene_description
+
+    gene_set_name must be non-null; others are optional.
+
+    Returns: a dictionary of the shape (values in angle-brackets vary):
+
+        {
+            <string, a gene set name>: {
+                "geneset_name": <string, a gene set name>,
+                "geneset_description": <a string or None>,
+                "genes": [
+                    {
+                        "gene_symbol": <string, a gene symbol or name>,
+                        "gene_description": <a string or None>
+                    },
+                    ...
+                ]
+            },
+            ...
+        }
+    """
+
+    class myDialect(csv.excel):
+        skipinitialspace = False
+
+    def just(n, seq):
+        it = iter(seq)
+        for _ in range(n - 1):
+            yield next(it, "")
+        yield tuple(it)
+
+    messagefn = context["messagefn"] if context else (lambda x: None)
+
+    gene_sets = {}
+    with gs_locator.local_handle() as fname:
+        with open(fname, newline="") as f:
+            reader = csv.reader(f, dialect=myDialect())
+            haveReadHeader = False
+            lineno = 0
+            for row in reader:
+                lineno += 1
+                # ignore empty rows
+                if len(row) == 0:
+                    continue
+                # if row starts with '#' it is a comment
+                if row[0].startswith("#"):
+                    continue
+                # if this is the first non-comment row, assume it is a header and validate
+                # column names.  OK if the user has extra columns after our initial set.
+                if not haveReadHeader:
+                    if row[0:len(GENESETS_TIDYCSV_HEADER)] != GENESETS_TIDYCSV_HEADER:
+                        raise AnnotationsError("Gene set CSV file missing the required column header.")
+                    haveReadHeader = True
+                    continue
+
+                geneset_name, geneset_description, gene_symbol, gene_description, _ = just(5, row)
+                if not geneset_name:
+                    raise AnnotationsError(f"Gene set CSV missing required gene set name on line {lineno}")
+                if (not gene_symbol) and gene_description:
+                    messagefn(f"Warning: Missing gene name in gene set name {geneset_name} on line {lineno}.")
+
+                if geneset_name in gene_sets:
+                    gs = gene_sets[geneset_name]
+                else:
+                    gs = gene_sets[geneset_name] = {
+                        "geneset_name": geneset_name,
+                        "geneset_description": geneset_description,
+                        "genes": [],
+                    }
+                # Use first geneset_description with a value
+                if not gs["geneset_description"] and geneset_description:
+                    gs["geneset_description"] = geneset_description
+                # add the gene if the gene_symbol is defined
+                if gene_symbol:
+                    gs["genes"].append(
+                        {
+                            "gene_symbol": gene_symbol,
+                            "gene_description": gene_description,
+                        }
+                    )
+
+    return gene_sets
+
+
+def write_gene_sets_tidycsv(f, genesets):
+    """
+    Convert the internal gene sets format (returned by read_gene_set_tidycsv) into
+    the simple Tidy CSV.
+    """
+    writer = csv.writer(f, dialect="excel")
+    writer.writerow(GENESETS_TIDYCSV_HEADER)
+    for geneset in genesets:
+        # genes may be empty, in which case we skip the gene set entirely
+        genes = geneset["genes"]
+        if not genes:
+            writer.writerow([geneset["geneset_name"], geneset.get("geneset_description", ""), "", ""])
+        else:
+            writer.writerows(
+                [
+                    [
+                        geneset["geneset_name"],
+                        geneset.get("geneset_description", ""),
+                        gene["gene_symbol"],
+                        gene.get("gene_description", ""),
+                    ]
+                    for gene in genes
+                ]
+            )
+
+
+def summarizeQueryHash(raw_query):
+    """ generate a cache key (hash) from the raw query string """
+    return hashlib.sha1(raw_query).hexdigest()
+
+
+def validate_gene_sets(genesets, var_names, context=None):
+    """
+    Check validity of gene sets, return if correct, else raise error.
+    May also modify the gene set for conditions that should be resolved,
+    but which do not warrant a hard error.
+
+    Argument gene sets may be either the REST OTA format (list of dicts) or the internal
+    format (dict of dicts, keyed by the gene set name).
+
+    Will return a modified gene sets (eg, remove warnings) of the same type as the
+    provided argument. Ie, dict->dict, list->list
+
+    Rules:
+
+    0. All gene set names must be unique. [error]
+    1. Gene set names must conform to the following: [error]
+        * Names must be comprised of 1 or more ASCII characters 32-126
+        * No leading or trailing spaces (ASCII 32)
+        * No multi-space (ASCII 32) runs
+    2. Gene symbols must be part of the current var_index. [warning]
+       If gene symbol is not in the var_index, generate a warning and remove the symbol
+       from the gene sets.
+    3. Gene symbols must not be duplicated in a gene set.  [warning]
+       Duplications will be silently de-duped.
+
+    Items marked [error] will generate a hard error, causing the validation to fail.
+
+    Items marked [warning] will generate a warning, and will be resolved without failing
+    the validation (typically by removing the offending item from the gene sets).
+    """
+
+    messagefn = context["messagefn"] if context else (lambda x: None)
+
+    # accept genesets args as either the internal (dict) or REST (list) format,
+    # as they are identical except for the dict being keyed by geneset_name.
+    if not isinstance(genesets, dict) and not isinstance(genesets, list):
+        raise ValueError("Gene sets must be either dict or list.")
+    genesets_iterable = genesets if isinstance(genesets, list) else genesets.values()
+
+    # 0. check for uniqueness of geneset names
+    geneset_names = [gs["geneset_name"] for gs in genesets_iterable]
+    if len(set(geneset_names)) != len(geneset_names):
+        raise KeyError("All gene set names must be unique.")
+
+    # 1. check gene set character set and format
+    illegal_name = re.compile(r"^\s|  |[\u0000-\u001F\u007F-\uFFFF]|\s$")
+    for name in geneset_names:
+        if type(name) != str or len(name) == 0:
+            raise KeyError("Gene set names must be non-null string.")
+        if illegal_name.search(name):
+            messagefn(
+                "Error: "
+                f"Gene set name {name} "
+                "is not valid. Leading, trailing, and multiple spaces within a name are not allowed."
+            )
+            raise KeyError(
+                "Gene set name is not valid. Leading, trailing, and multiple spaces within a name are not allowed."
+            )
+
+    # 2. & 3. check for duplicate gene symbols, and those not present in the dataset. They will
+    # generate a warning and be removed.
+    for geneset in genesets_iterable:
+        if not isinstance(geneset, dict):
+            raise ValueError("Each gene set must be a dict.")
+        geneset_name = geneset["geneset_name"]
+        genes = geneset["genes"]
+        if not isinstance(genes, list):
+            raise ValueError("Gene set genes field must be a list")
+        geneset.setdefault("geneset_description", "")
+        gene_symbol_already_seen = set()
+        new_genes = []
+        for gene in genes:
+            gene_symbol = gene["gene_symbol"]
+            if not isinstance(gene_symbol, str) or len(gene_symbol) == 0:
+                raise ValueError("Gene symbol must be non-null string.")
+            if gene_symbol in gene_symbol_already_seen:
+                # duplicate check
+                messagefn(
+                    f"Warning: a duplicate of gene {gene_symbol} was found in gene set {geneset_name}, "
+                    "and will be ignored."
+                )
+                continue
+
+            if gene_symbol not in var_names:
+                messagefn(
+                    f"Warning: {gene_symbol}, used in gene set {geneset_name}, "
+                    "was not found in the dataset and will be ignored."
+                )
+                continue
+
+            gene_symbol_already_seen.add(gene_symbol)
+            gene.setdefault("gene_description", "")
+            new_genes.append(gene)
+
+        geneset["genes"] = new_genes
+
+    return genesets

backend/czi_hosted/app/app.py

@@ -318,6 +318,25 @@ class LayoutObsAPI(DatasetResource):
         return common_rest.layout_obs_put(request, data_adaptor)
 
 
+class GenesetsAPI(DatasetResource):
+    @cache_control(public=True, max_age=ONE_WEEK)
+    @rest_get_data_adaptor
+    def get(self, data_adaptor):
+        return common_rest.genesets_get(request, data_adaptor)
+
+
+class SummarizeVarAPI(DatasetResource):
+    @rest_get_data_adaptor
+    @cache_control(public=True, max_age=ONE_WEEK)
+    def get(self, data_adaptor):
+        return common_rest.summarize_var_get(request, data_adaptor)
+
+    @rest_get_data_adaptor
+    @cache_control(no_store=True)
+    def post(self, data_adaptor):
+        return common_rest.summarize_var_post(request, data_adaptor)
+
+
 def get_api_base_resources(bp_base):
     """Add resources that are accessed from the api_base_url"""
     api = Api(bp_base)
@@ -343,6 +362,8 @@ def get_api_dataroot_resources(bp_dataroot, url_dataroot=None):
     add_resource(AnnotationsObsAPI, "/annotations/obs")
     add_resource(AnnotationsVarAPI, "/annotations/var")
     add_resource(DataVarAPI, "/data/var")
+    add_resource(GenesetsAPI, "/genesets")
+    add_resource(SummarizeVarAPI, "/summarize/var")
     # Display routes
     add_resource(ColorsAPI, "/colors")
     # Computation routes

backend/czi_hosted/common/annotations/annotations.py

@@ -1,21 +1,33 @@
-from abc import ABCMeta, abstractmethod
-
 import fastobo
 import fsspec
+import os
 
-from backend.common.errors import OntologyLoadFailure
+from flask import current_app, has_request_context
+
+from backend.common.errors import OntologyLoadFailure, DisabledFeatureError
 from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
+from backend.common.genesets import write_gene_sets_tidycsv, read_gene_sets_tidycsv, validate_gene_sets
+from backend.common.utils.data_locator import DataLocator
+from backend.common.utils.utils import path_join
 
 
-class Annotations(metaclass=ABCMeta):
+class Annotations:
-    """ baseclass for annotations, including ontologies"""
+    """ baseclass for annotations, including ontologies and genesets """
 
     """ our default ontology is the PURL for the Cell Ontology.
     See http://www.obofoundry.org/ontology/cl.html """
     DefaultOnotology = "http://purl.obolibrary.org/obo/cl.obo"
 
-    def __init__(self):
+    def __init__(self, config={}):
         self.ontology_data = None
+        self.config = config
+
+    def user_annotations_enabled(self):
+        return self.config.get("user-annotations", False)
+
+    def check_user_annotations_enabled(self):
+        if not self.user_annotations_enabled():
+            raise DisabledFeatureError("User annotations are disabled.")
 
     def load_ontology(self, path):
         """Load and parse ontologies - currently support OBO files only."""
@@ -49,22 +61,78 @@ class Annotations(metaclass=ABCMeta):
 
         return schema
 
-    @abstractmethod
     def set_collection(self, name):
         """set or create a new annotation collection"""
-        pass
+        raise NotImplementedError
 
-    @abstractmethod
     def read_labels(self, data_adaptor):
         """Return the labels as a pandas.DataFrame"""
-        pass
+        raise NotImplementedError
 
-    @abstractmethod
     def write_labels(self, df, data_adaptor):
         """Write the labels (df) to a persistent storage such that it can later be read"""
-        pass
+        raise NotImplementedError
 
-    @abstractmethod
     def update_parameters(self, parameters, data_adaptor):
         """Update configuration parameters that describe information about the annotations feature"""
-        pass
+        params = {}
+        params["annotations_genesets_readonly"] = True
+        params["annotations_genesets_name_is_read_only"] = True
+        parameters.update(params)
+
+    @staticmethod
+    def gene_sets_to_csv(genesets):
+        """
+        Convert the internal genesets format (returned by read_gene_set) into
+        the simple Tidy CSV.
+        """
+        from io import StringIO
+
+        if isinstance(genesets, dict):
+            genesets = genesets.values()
+
+        with StringIO() as sio:
+            write_gene_sets_tidycsv(sio, genesets)
+            return sio.getvalue()
+
+    @staticmethod
+    def gene_sets_to_response(genesets):
+        """
+        Convert the internal genesets format (returned by read_gene_set) into
+        the dict expected by the JSON REST API
+        """
+        return list(genesets.values())
+
+    def read_gene_sets(self, data_adaptor, context=None):
+        if has_request_context():
+            if not current_app.auth.is_user_authenticated():
+                return ({}, 0)
+
+        gene_sets_uri_or_path = dataset_uri_to_geneset_uri(data_adaptor.data_locator.uri_or_path)
+
+        server_config = data_adaptor.server_config
+        region_name = None if server_config is None else server_config.data_locator__s3__region_name
+        gene_sets_locator = DataLocator(gene_sets_uri_or_path, region_name=region_name)
+        if not gene_sets_locator.exists():
+            return ({}, 0)
+
+        gene_sets = read_gene_sets_tidycsv(gene_sets_locator, context)
+        schema = data_adaptor.get_schema()
+        var_index = schema["annotations"]["var"].get("index", "index")
+        var_names = set(data_adaptor.query_var_array(var_index))
+
+        gene_sets = validate_gene_sets(gene_sets, var_names)
+        return (gene_sets, 0)
+
+
+def dataset_uri_to_geneset_uri(data_uri_or_path):
+    """ given a dataset URI, return the associated gene set URI """
+    data_basename = os.path.basename(data_uri_or_path)
+    base, ext = os.path.splitext(data_basename)
+    if ext is not None:  # strip extension, if any
+        data_basename = base
+
+    genesets_basename = f"{data_basename}-genesets.csv"
+    gene_sets_uri_or_path = path_join(data_uri_or_path, "..", genesets_basename)
+
+    return gene_sets_uri_or_path

backend/czi_hosted/common/annotations/hosted_tiledb.py

@@ -17,8 +17,8 @@ from backend.czi_hosted.db.cellxgene_orm import Annotation
 class AnnotationsHostedTileDB(Annotations):
     CXG_ANNO_COLLECTION = "cxg_anno_collection"
 
-    def __init__(self, directory_path, db):
+    def __init__(self, config, directory_path, db):
-        super().__init__()
+        super().__init__(config)
         self.db = db
         if directory_path[-1] == "/":
             self.directory_path = directory_path
@@ -158,6 +158,8 @@ class AnnotationsHostedTileDB(Annotations):
         self.db.session.commit()
 
     def update_parameters(self, parameters, data_adaptor):
+        super().update_parameters(parameters, data_adaptor)
+
         params = {}
         params["annotations"] = True
         params["user_annotation_collection_name_enabled"] = False

backend/czi_hosted/common/annotations/local_file_csv.py

@@ -16,8 +16,8 @@ from backend.common.errors import AnnotationsError
 class AnnotationsLocalFile(Annotations):
     CXG_ANNO_COLLECTION = "cxg_anno_collection"
 
-    def __init__(self, output_dir, output_file):
+    def __init__(self, config, output_dir, output_file):
-        super().__init__()
+        super().__init__(config)
         self.output_dir = output_dir
         self.output_file = output_file
         # lock used to protect label file write ops
@@ -169,6 +169,8 @@ class AnnotationsLocalFile(Annotations):
                 os.remove(os.path.join(backup_dir, bu))
 
     def update_parameters(self, parameters, data_adaptor):
+        super().update_parameters(parameters, data_adaptor)
+
         params = {}
         params["annotations"] = True
         params["user_annotation_collection_name_enabled"] = True
@@ -190,7 +192,7 @@ class AnnotationsLocalFile(Annotations):
             collection = self.get_collection()
             if current_app.auth.is_user_authenticated():
                 params["annotations-user-data-idhash"] = self._get_userdata_idhash(data_adaptor)
-                params["annotations-data-collection-is-read-only"] = False
+                params["annotations-data-collection-is-read-only"] = not self.user_annotations_enabled()
                 params["annotations-data-collection-name"] = collection
 
         parameters.update(params)

backend/czi_hosted/common/config/client_config.py

@@ -44,6 +44,9 @@ def get_client_config(app_config, data_adaptor):
         "annotations": False,
         "annotations_file": None,
         "annotations_dir": None,
+        "annotations_genesets": True,  # feature flag
+        "annotations_genesets_readonly": True,
+        "annotations_genesets_summary_methods": ["mean"],
         "annotations_cell_ontology_enabled": False,
         "annotations_cell_ontology_obopath": None,
         "annotations_cell_ontology_terms": None,

backend/czi_hosted/common/config/dataset_config.py

@@ -1,6 +1,7 @@
 import os
 from os.path import splitext, isdir
 
+from backend.czi_hosted.common.annotations.annotations import Annotations
 from backend.czi_hosted.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
 from backend.czi_hosted.common.annotations.local_file_csv import AnnotationsLocalFile
 from backend.czi_hosted.common.config.base_config import BaseConfig
@@ -53,8 +54,10 @@ class DatasetConfig(BaseConfig):
         except KeyError as e:
             raise ConfigurationError(f"Unexpected config: {str(e)}")
 
-        # The annotation object is created during complete_config and stored here.
+        # Create the default annotation, which supports gene set reading without
-        self.user_annotations = None
+        # further configuration. Depending on configuration options, `complete_config` 
+        # may create a more specialized annotation object and replace this default.
+        self.user_annotations = Annotations()
 
     def complete_config(self, context):
         self.handle_app()
@@ -147,7 +150,11 @@ class DatasetConfig(BaseConfig):
             except OSError:
                 raise ConfigurationError("Unable to create directory specified by --annotations-dir")
 
-        self.user_annotations = AnnotationsLocalFile(dirname, filename)
+        anno_config = {
+            "user-annotations": self.user_annotations__enable,
+            "genesets-save": False,
+        }
+        self.user_annotations = AnnotationsLocalFile(anno_config, dirname, filename)
 
         # if the user has specified a fixed label file, go ahead and validate it
         # so that we can remove errors early in the process.
@@ -163,7 +170,12 @@ class DatasetConfig(BaseConfig):
         self.validate_correct_type_of_configuration_attribute(
             "user_annotations__hosted_tiledb_array__hosted_file_directory", str
         )
+        anno_config = {
+            "user-annotations": self.user_annotations__enable,
+            "genesets-save": False,
+        }
         self.user_annotations = AnnotationsHostedTileDB(
+            anno_config,
             directory_path=self.user_annotations__hosted_tiledb_array__hosted_file_directory,
             db=DbUtils(self.user_annotations__hosted_tiledb_array__db_uri),
         )

backend/czi_hosted/common/rest.py

@@ -3,6 +3,7 @@ import logging
 import sys
 from http import HTTPStatus
 import zlib
+import json
 
 from flask import make_response, jsonify, current_app, abort
 from werkzeug.urls import url_unquote
@@ -17,9 +18,10 @@ from backend.common.errors import (
     ExceedsLimitError,
     DatasetAccessError,
     ColorFormatException,
+    AnnotationsError,
+    UnsupportedSummaryMethod,
 )
-
+from backend.common.genesets import summarizeQueryHash
-import json
 from backend.common.fbs.matrix import decode_matrix_fbs
 
 
@@ -106,7 +108,7 @@ def schema_get_helper(data_adaptor):
 
     # add label obs annotations as needed
     annotations = data_adaptor.dataset_config.user_annotations
-    if annotations is not None:
+    if annotations.user_annotations_enabled():
         label_schema = annotations.get_schema(data_adaptor)
         schema["annotations"]["obs"]["columns"].extend(label_schema)
 
@@ -140,7 +142,7 @@ def annotations_obs_get(request, data_adaptor):
     try:
         labels = None
         annotations = data_adaptor.dataset_config.user_annotations
-        if annotations:
+        if annotations.user_annotations_enabled():
             labels = annotations.read_labels(data_adaptor)
         fbs = data_adaptor.annotation_to_fbs_matrix(Axis.OBS, fields, labels)
         return make_response(fbs, HTTPStatus.OK, {"Content-Type": "application/octet-stream"})
@@ -151,7 +153,7 @@ def annotations_obs_get(request, data_adaptor):
 def annotations_put_fbs_helper(data_adaptor, fbs):
     """helper function to write annotations from fbs"""
     annotations = data_adaptor.dataset_config.user_annotations
-    if annotations is None:
+    if not annotations.user_annotations_enabled():
         raise DisabledFeatureError("Writable annotations are not enabled")
 
     new_label_df = decode_matrix_fbs(fbs)
@@ -166,7 +168,7 @@ def inflate(data):
 
 def annotations_obs_put(request, data_adaptor):
     annotations = data_adaptor.dataset_config.user_annotations
-    if annotations is None:
+    if not annotations.user_annotations_enabled():
         return abort(HTTPStatus.NOT_IMPLEMENTED)
 
     anno_collection = request.args.get("annotation-collection-name", default=None)
@@ -197,7 +199,7 @@ def annotations_var_get(request, data_adaptor):
     try:
         labels = None
         annotations = data_adaptor.dataset_config.user_annotations
-        if annotations is not None:
+        if annotations.user_annotations_enabled():
             labels = annotations.read_labels(data_adaptor)
         return make_response(
             data_adaptor.annotation_to_fbs_matrix(Axis.VAR, fields, labels),
@@ -328,3 +330,70 @@ def layout_obs_put(request, data_adaptor):
         return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, str(e))
     except (ValueError, DisabledFeatureError, FilterError) as e:
         return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+
+
+def genesets_get(request, data_adaptor):
+    preferred_mimetype = request.accept_mimetypes.best_match(["application/json", "text/csv"])
+    if preferred_mimetype not in ("application/json", "text/csv"):
+        return abort(HTTPStatus.NOT_ACCEPTABLE)
+
+    try:
+        annotations = data_adaptor.dataset_config.user_annotations
+        (genesets, tid) = annotations.read_gene_sets(data_adaptor)
+
+        if preferred_mimetype == "text/csv":
+            return make_response(
+                annotations.gene_sets_to_csv(genesets),
+                HTTPStatus.OK,
+                {
+                    "Content-Type": "text/csv",
+                    "Content-Disposition": "attachment; filename=genesets.csv",
+                },
+            )
+        else:
+            return make_response(
+                jsonify({"genesets": annotations.gene_sets_to_response(genesets), "tid": tid}), HTTPStatus.OK
+            )
+    except (ValueError, KeyError, AnnotationsError) as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e))
+
+
+def summarize_var_helper(request, data_adaptor, key, raw_query):
+    preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
+    if preferred_mimetype != "application/octet-stream":
+        return abort(HTTPStatus.NOT_ACCEPTABLE)
+
+    summary_method = request.values.get("method", default="mean")
+    query_hash = summarizeQueryHash(raw_query)
+    if key and query_hash != key:
+        return abort(HTTPStatus.BAD_REQUEST, description="query key did not match")
+
+    args_filter_only = request.values.copy()
+    args_filter_only.poplist("method")
+    args_filter_only.poplist("key")
+
+    try:
+        filter = _query_parameter_to_filter(args_filter_only)
+        return make_response(
+            data_adaptor.summarize_var(summary_method, filter, query_hash),
+            HTTPStatus.OK,
+            {"Content-Type": "application/octet-stream"},
+        )
+    except (ValueError) as e:
+        return abort(HTTPStatus.NOT_FOUND, description=str(e))
+    except (UnsupportedSummaryMethod, FilterError) as e:
+        return abort(HTTPStatus.BAD_REQUEST, description=str(e))
+
+
+def summarize_var_get(request, data_adaptor):
+    return summarize_var_helper(request, data_adaptor, None, request.query_string)
+
+
+def summarize_var_post(request, data_adaptor):
+    if not request.content_type or "application/x-www-form-urlencoded" not in request.content_type:
+        return abort(HTTPStatus.UNSUPPORTED_MEDIA_TYPE)
+    if request.content_length > 1_000_000:  # just a sanity check to avoid memory exhaustion
+        return abort(HTTPStatus.BAD_REQUEST)
+
+    key = request.args.get("key", default=None)
+    return summarize_var_helper(request, data_adaptor, key, request.get_data())

backend/czi_hosted/data_common/data_adaptor.py

@@ -3,11 +3,12 @@ from os.path import basename, splitext
 
 import numpy as np
 import pandas as pd
+from scipy import sparse
 from server_timing import Timing as ServerTiming
 
 from backend.czi_hosted.common.config.app_config import AppConfig
 from backend.common.constants import Axis
-from backend.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError
+from backend.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError, UnsupportedSummaryMethod
 from backend.common.utils.utils import jsonify_numpy
 from backend.common.fbs.matrix import encode_matrix_fbs
 
@@ -338,7 +339,7 @@ class DataAdaptor(metaclass=ABCMeta):
     @staticmethod
     def normalize_embedding(embedding):
         """Normalize embedding layout to meet client assumptions.
-           Embedding is an ndarray, shape (n_obs, n)., where n is normally 2
+        Embedding is an ndarray, shape (n_obs, n)., where n is normally 2
         """
 
         # scale isotropically
@@ -394,3 +395,26 @@ class DataAdaptor(metaclass=ABCMeta):
         except RuntimeError:
             lastmod = None
         return lastmod
+
+    def summarize_var(self, method, filter, query_hash):
+        if method != "mean":
+            raise UnsupportedSummaryMethod("Unknown gene set summary method.")
+
+        obs_selector, var_selector = self._filter_to_mask(filter)
+        if obs_selector is not None:
+            raise FilterError("filtering on obs unsupported")
+
+        # if no filter, just return zeros.  We don't have a use case
+        # for summarizing the entire X without a filter, and it would
+        # potentially be quite compute / memory intensive.
+        if var_selector is None or np.count_nonzero(var_selector) == 0:
+            mean = np.zeros((self.get_shape()[0], 1), dtype=np.float32)
+        else:
+            X = self.get_X_array(obs_selector, var_selector)
+            if sparse.issparse(X):
+                mean = X.mean(axis=1)
+            else:
+                mean = X.mean(axis=1, keepdims=True)
+
+        col_idx = pd.Index([query_hash])
+        return encode_matrix_fbs(mean, col_idx=col_idx, row_idx=None)

backend/server/common/annotations/annotations.py

@@ -5,10 +5,11 @@ import fsspec
 
 from backend.common.errors import OntologyLoadFailure, DisabledFeatureError
 from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
+from backend.common.genesets import write_gene_sets_tidycsv
 
 
 class Annotations(metaclass=ABCMeta):
-    """ baseclass for annotations, including ontologies and genesets"""
+    """ baseclass for annotations, including ontologies and gene sets"""
 
     """ our default ontology is the PURL for the Cell Ontology.
     See http://www.obofoundry.org/ontology/cl.html """
@@ -30,7 +31,7 @@ class Annotations(metaclass=ABCMeta):
 
     def check_gene_sets_save_enabled(self):
         if not self.gene_sets_save_enabled():
-            raise DisabledFeatureError("User genesets save is disabled.")
+            raise DisabledFeatureError("User gene sets save is disabled.")
 
     def load_ontology(self, path):
         """Load and parse ontologies - currently support OBO files only."""
@@ -81,12 +82,12 @@ class Annotations(metaclass=ABCMeta):
 
     @abstractmethod
     def read_gene_sets(self, data_adaptor):
-        """Return the genesets from persistent storage """
+        """Return the gene sets from persistent storage """
         pass
 
     @abstractmethod
     def write_gene_sets(self, gs, data_adaptor):
-        """Write the genesets (gs) to a persistent storage such that it can later be read"""
+        """Write the gene sets (gs) to a persistent storage such that it can later be read"""
         pass
 
     @abstractmethod
@@ -94,51 +95,25 @@ class Annotations(metaclass=ABCMeta):
         """Update configuration parameters that describe information about the annotations feature"""
         pass
 
-    Genesets_Header = [
-        "gene_set_name",
-        "gene_set_description",
-        "gene_symbol",
-        "gene_description",
-    ]
-
     @staticmethod
     def gene_sets_to_csv(genesets):
         """
-        Convert the internal genesets format (returned by read_gene_set) into
+        Convert the internal gene sets format (returned by read_gene_set) into
         the simple Tidy CSV.
         """
         from io import StringIO
-        import csv
 
         if isinstance(genesets, dict):
             genesets = genesets.values()
 
         with StringIO() as sio:
-            writer = csv.writer(sio, dialect='excel')
+            write_gene_sets_tidycsv(sio, genesets)
-            writer.writerow(Annotations.Genesets_Header)
-            for geneset in genesets:
-                # genes may be empty, in which case we skip the geneset entirely
-                genes = geneset["genes"]
-                if not genes:
-                    writer.writerow([geneset["geneset_name"], geneset.get("geneset_description", ""), "", ""])
-                else:
-                    writer.writerows(
-                        [
-                            [
-                                geneset["geneset_name"],
-                                geneset.get("geneset_description", ""),
-                                gene["gene_symbol"],
-                                gene.get("gene_description", ""),
-                            ]
-                            for gene in genes
-                        ]
-                    )
             return sio.getvalue()
 
     @staticmethod
     def gene_sets_to_response(genesets):
         """
-        Convert the internal genesets format (returned by read_gene_set) into
+        Convert the internal gene sets format (returned by read_gene_set) into
         the dict expected by the JSON REST API
         """
         return list(genesets.values())

backend/server/common/annotations/local_file_csv.py

@@ -4,14 +4,15 @@ import re
 import threading
 from datetime import datetime
 from hashlib import blake2b
-import csv
 
 import pandas as pd
 from flask import session, has_request_context, current_app
 
 from backend.server import __version__ as cellxgene_version
 from backend.server.common.annotations.annotations import Annotations
+from backend.common.genesets import read_gene_sets_tidycsv
 from backend.common.errors import AnnotationsError, ObsoleteRequest
+from backend.common.utils.data_locator import DataLocator
 
 
 class AnnotationsLocalFile(Annotations):
@@ -124,8 +125,8 @@ class AnnotationsLocalFile(Annotations):
                 if fname == self.last_geneset_fname:
                     gene_sets = self.last_geneset
                 else:
-                    with open(fname, newline="") as f:
+                    # read
-                        gene_sets = read_gene_set_tidycsv(f, context)
+                    gene_sets = read_gene_sets_tidycsv(DataLocator(fname), context)
 
                     # validate
                     gene_sets = data_adaptor.check_new_gene_sets(gene_sets, context)
@@ -259,6 +260,7 @@ class AnnotationsLocalFile(Annotations):
         params = {}
         params["annotations"] = self.user_annotations_enabled()
         params["annotations_genesets_readonly"] = not self.gene_sets_save_enabled()
+        params["annotations_genesets_name_is_read_only"] = self.gene_sets_output_file is not None
         params["user_annotation_collection_name_enabled"] = True
 
         if self.ontology_data:
@@ -276,99 +278,10 @@ class AnnotationsLocalFile(Annotations):
 
         elif session is not None:
             collection = self.get_collection()
-            params["annotations-data-collection-is-read-only"] = False
+            params["annotations-data-collection-is-read-only"] = not self.user_annotations_enabled()
             params["annotations-data-collection-name"] = collection
 
         if current_app.auth.is_user_authenticated():
             params["annotations-user-data-idhash"] = self._get_userdata_idhash(data_adaptor)
 
         parameters.update(params)
-
-
-def read_gene_set_tidycsv(f, context=None):
-    """
-    Read & parse the Tidy CSV format, applying validation checks for mandatory
-    values, and de-duping rules.
-
-    Format is a four-column CSV, with a mandatory header row, and optional "#" prefixed
-    comments.  Format:
-
-        gene_set_name, gene_set_description, gene_symbol, gene_description
-
-    gene_set_name must be non-null; others are optional.
-
-    Returns: a dictionary of the shape (values in angle-brackets vary):
-
-        {
-            <string, a gene set name>: {
-                "geneset_name": <string, a gene set name>,
-                "geneset_description": <a string or None>,
-                "genes": [
-                    {
-                        "gene_symbol": <string, a gene symbol or name>,
-                        "gene_description": <a string or None>
-                    },
-                    ...
-                ]
-            },
-            ...
-        }
-    """
-
-    class myDialect(csv.excel):
-        skipinitialspace = True
-
-    def just(n, seq):
-        it = iter(seq)
-        for _ in range(n - 1):
-            yield next(it, "")
-        yield tuple(it)
-
-    messagefn = context["messagefn"] if context else (lambda x: None)
-
-    reader = csv.reader(f, dialect=myDialect())
-    gene_sets = {}
-    haveReadHeader = False
-    lineno = 0
-    for row in reader:
-        lineno += 1
-        # ignore empty rows
-        if len(row) == 0:
-            continue
-        # if row starts with '#' it is a comment
-        if row[0].startswith("#"):
-            continue
-        # if this is the first non-comment row, assume it is a header
-        if not haveReadHeader:
-            if row != Annotations.Genesets_Header:
-                raise AnnotationsError("Geneset CSV file missing the required column header.")
-            haveReadHeader = True
-            continue
-
-        geneset_name, geneset_description, gene_symbol, gene_description, _ = just(5, row)
-        if not geneset_name:
-            raise AnnotationsError(f"Geneset CSV missing required geneset or gene name on line {lineno}")
-        if (not gene_symbol) and gene_description:
-            messagefn(f"Warning: Missing gene name in geneset name {geneset_name} on line {lineno}.")
-
-        if geneset_name in gene_sets:
-            gs = gene_sets[geneset_name]
-        else:
-            gs = gene_sets[geneset_name] = {
-                "geneset_name": geneset_name,
-                "geneset_description": geneset_description,
-                "genes": [],
-            }
-        # Use first geneset_description with a value
-        if not gs["geneset_description"] and geneset_description:
-            gs["geneset_description"] = geneset_description
-        # add the gene if the gene_symbol is defined
-        if gene_symbol:
-            gs["genes"].append(
-                {
-                    "gene_symbol": gene_symbol,
-                    "gene_description": gene_description,
-                }
-            )
-
-    return gene_sets

backend/server/common/config/dataset_config.py

@@ -138,7 +138,7 @@ class DatasetConfig(BaseConfig):
 
         if dirname is not None and (filename is not None or genesets_filename is not None):
             raise ConfigurationError(
-                "'user-generated-data-dir' may not be used with annotations-file' or 'genesets-file'."
+                "'user-generated-data-dir' may not be used with 'annotations-file' or 'gene-sets-file'."
             )
 
         if filename is not None:

backend/server/common/rest.py

@@ -3,7 +3,7 @@ import logging
 import sys
 from http import HTTPStatus
 import zlib
-import hashlib
+import json
 
 from flask import make_response, jsonify, current_app, abort
 from werkzeug.urls import url_unquote
@@ -22,9 +22,7 @@ from backend.common.errors import (
     ObsoleteRequest,
     UnsupportedSummaryMethod,
 )
-
+from backend.common.genesets import summarizeQueryHash
-import json
-
 from backend.common.fbs.matrix import decode_matrix_fbs
 
 
@@ -390,7 +388,7 @@ def summarize_var_helper(request, data_adaptor, key, raw_query):
         return abort(HTTPStatus.NOT_ACCEPTABLE)
 
     summary_method = request.values.get("method", default="mean")
-    query_hash = hashlib.sha1(raw_query).hexdigest()  # cache helper
+    query_hash = summarizeQueryHash(raw_query)
     if key and query_hash != key:
         return abort(HTTPStatus.BAD_REQUEST, description="query key did not match")
 
@@ -398,7 +396,7 @@ def summarize_var_helper(request, data_adaptor, key, raw_query):
     args_filter_only.poplist("method")
     args_filter_only.poplist("key")
 
-    try:    
+    try:
         filter = _query_parameter_to_filter(args_filter_only)
         return make_response(
             data_adaptor.summarize_var(summary_method, filter, query_hash),

backend/server/data_common/data_adaptor.py

@@ -1,6 +1,5 @@
 from abc import ABCMeta, abstractmethod
 from os.path import basename, splitext
-import re
 import numpy as np
 import pandas as pd
 from scipy import sparse
@@ -11,6 +10,7 @@ from backend.common.constants import Axis
 from backend.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError, UnsupportedSummaryMethod
 from backend.common.utils.utils import jsonify_numpy
 from backend.common.fbs.matrix import encode_matrix_fbs
+from backend.common.genesets import validate_gene_sets
 
 
 class DataAdaptor(metaclass=ABCMeta):
@@ -263,95 +263,8 @@ class DataAdaptor(metaclass=ABCMeta):
         return labels_df
 
     def check_new_gene_sets(self, genesets, context=None):
-        """
-        Check validity of gene sets, return if correct, else raise error.
-        May also modify the gene set for conditions that should be resolved,
-        but which do not warrant a hard error.
-
-        Argument genesets may be either the REST OTA format (list of dicts) or the internal
-        format (dict of dicts, keyed by the geneset name).
-
-        Will return a modified genesets (eg, remove dups) of the same type as the
-        provided argument. Ie, dict->dict, list->list
-
-        Rules:
-        0. all geneset names must be unique.
-        1. All geneset names must be comprised of legal characters, meaning:
-            * no leading or trailing white space
-            * no multi-space runs
-            * no tab, vertical tab, newline or return
-           Where "space" means ASCII 32. Generates hard error.
-        2. Gene symbols must be part of the current var_index.  If symbol not in var_index,
-           will generate a warning and the symbol removed.
-        3. Duplicate gene symbols are silently de-duped.
-        """
-        messagefn = context["messagefn"] if context else (lambda x: None)
-
-        # accept genesets args as either the internal (dict) or REST (list) format,
-        # as they are identical except for the dict being keyed by geneset_name.
-        if not isinstance(genesets, dict) and not isinstance(genesets, list):
-            raise ValueError("Genesets must be either dict or list.")
-        genesets_iterable = genesets if isinstance(genesets, list) else genesets.values()
-
-        # 0. check for uniqueness of geneset names
-        geneset_names = [gs["geneset_name"] for gs in genesets_iterable]
-        if len(set(geneset_names)) != len(geneset_names):
-            raise KeyError("All geneset names must be unique.")
-
-        # 1. check gene set character set and format
-        illegal_name = re.compile(r"^\s|  |[\v\t\r\n]|\s$")
-        for name in geneset_names:
-            if type(name) != str or len(name) == 0:
-                raise KeyError("Geneset names must be non-null string.")
-            if illegal_name.search(name):
-                messagefn(
-                    "Error: "
-                    f"Geneset name {name} "
-                    "is not valid. Leading, trailing, and multiple spaces within a name are not allowed."
-                )
-                raise KeyError(
-                    "Geneset name is not valid. Leading, trailing, and multiple spaces within a name are not allowed."
-                )
-
-        # 2. & 3. check for duplicate gene symbols, and those not present in the dataset. They will
-        # generate a warning and be removed.
         var_names = set(self.query_var_array(self.parameters.get("var_names")))
-        for geneset in genesets_iterable:
+        return validate_gene_sets(genesets, var_names)
-            if not isinstance(geneset, dict):
-                raise ValueError("Each geneset must be a dict.")
-            geneset_name = geneset["geneset_name"]
-            genes = geneset["genes"]
-            if not isinstance(genes, list):
-                raise ValueError("Geneset genes field must be a list")
-            geneset.setdefault("geneset_description", "")
-            gene_symbol_already_seen = set()
-            new_genes = []
-            for gene in genes:
-                gene_symbol = gene["gene_symbol"]
-                if not isinstance(gene_symbol, str) or len(gene_symbol) == 0:
-                    raise ValueError("Gene symbol must be non-null string.")
-                if gene_symbol in gene_symbol_already_seen:
-                    # duplicate check
-                    messagefn(
-                        f"Warning: a duplicate of gene {gene_symbol} was found in geneset {geneset_name}, "
-                        "and will be ignored."
-                    )
-                    continue
-
-                if gene_symbol not in var_names:
-                    messagefn(
-                        f"Warning: {gene_symbol}, used in geneset {geneset_name}, "
-                        "was not found in the dataset and will be ignored."
-                    )
-                    continue
-
-                gene_symbol_already_seen.add(gene_symbol)
-                gene.setdefault("gene_description", "")
-                new_genes.append(gene)
-
-            geneset["genes"] = new_genes
-
-        return genesets
 
     def data_frame_to_fbs_matrix(self, filter, axis):
         """

backend/test/fixtures/pbmc3k-genesets.csv

@@ -1,10 +1,10 @@
 # Test fixture
-gene_set_name, gene_set_description, gene_symbol, gene_description
+gene_set_name,gene_set_description,gene_symbol,gene_description
 first gene set name,,F5, a gene_description
-first gene set name,a description, NO_SUCH_GENE, non-existent gene
+first gene set name,a description,NO_SUCH_GENE, non-existent gene
-first gene set name,a description, F5, duplicate gene
+first gene set name,a description,F5, duplicate gene
-first gene set name, a description, SUMO3,
+first gene set name, a description,SUMO3,
-first gene set name,, SRM,
+first gene set name,,SRM,
 second gene set,,RER1
 second gene set,,SIK1
 third gene set,,NO_SUCH_GENE

backend/test/test_czi_hosted/unit/__init__.py

@@ -48,7 +48,14 @@ def data_with_tmp_tiledb_annotations(ext: MatrixDataType):
     config.complete_config()
 
     data = MatrixDataLoader(data_locator.abspath()).open(config)
-    annotations = AnnotationsHostedTileDB(tmp_dir, DbUtils("postgresql://postgres:test_pw@localhost:5432"),)
+    annotations = AnnotationsHostedTileDB(
+        {
+            "user-annotations": True,
+            "genesets-save": False,
+        },
+        tmp_dir,
+        DbUtils("postgresql://postgres:test_pw@localhost:5432"),
+    )
     return data, tmp_dir, annotations
 
 
@@ -70,12 +77,21 @@ def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False):
         single_dataset__datapath=data_locator.path,
     )
     config.update_default_dataset_config(
-        embeddings__names=["umap"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01,
+        embeddings__names=["umap"],
+        presentation__max_categories=100,
+        diffexp__lfc_cutoff=0.01,
     )
 
     config.complete_config()
     data = MatrixDataLoader(data_locator.abspath()).open(config)
-    annotations = AnnotationsLocalFile(None, annotations_file)
+    annotations = AnnotationsLocalFile(
+        {
+            "user-annotations": True,
+            "genesets-save": False,
+        },
+        None,
+        annotations_file,
+    )
     return data, tmp_dir, annotations
 
 

backend/test/test_czi_hosted/unit/common/test_api.py

@@ -3,6 +3,7 @@ import time
 import unittest
 import zlib
 from http import HTTPStatus
+import hashlib
 
 import pandas as pd
 import requests
@@ -315,6 +316,95 @@ class EndPoints(object):
         result = self.session.get(url)
         self.assertEqual(result.status_code, HTTPStatus.OK)
 
+    def test_genesets_config(self):
+        result = self.session.get(f"{self.URL_BASE}config")
+        config_data = result.json()
+        params = config_data["config"]["parameters"]
+        annotations_genesets = params["annotations_genesets"]
+        annotations_genesets_readonly = params["annotations_genesets_readonly"]
+        annotations_genesets_summary_methods = params["annotations_genesets_summary_methods"]
+        self.assertTrue(annotations_genesets)
+        self.assertTrue(annotations_genesets_readonly)
+        self.assertEqual(annotations_genesets_summary_methods, ["mean"])
+
+    def test_get_genesets(self):
+        endpoint = "genesets"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.get(url, headers={"Accept": "application/json"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        self.assertIsNotNone(result_data["genesets"])
+
+    def test_get_summaryvar(self):
+        index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
+        endpoint = "summarize/var"
+
+        # single column
+        filter = f"var:{index_col_name}=F5"
+        query = f"method=mean&{filter}"
+        query_hash = hashlib.sha1(query.encode()).hexdigest()
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 1)
+        self.assertEqual(df["col_idx"], [query_hash])
+        self.assertAlmostEqual(df["columns"][0][0], -0.110451095)
+
+        # multi-column
+        col_names = ["F5", "BEB3", "SIK1"]
+        filter = "&".join([f"var:{index_col_name}={name}" for name in col_names])
+        query = f"method=mean&{filter}"
+        query_hash = hashlib.sha1(query.encode()).hexdigest()
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 1)
+        self.assertEqual(df["col_idx"], [query_hash])
+        self.assertAlmostEqual(df["columns"][0][0], -0.16628358)
+
+    def test_post_summaryvar(self):
+        index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
+        endpoint = "summarize/var"
+        headers = {"Content-Type": "application/x-www-form-urlencoded", "Accept": "application/octet-stream"}
+
+        # single column
+        filter = f"var:{index_col_name}=F5"
+        query = f"method=mean&{filter}"
+        query_hash = hashlib.sha1(query.encode()).hexdigest()
+        url = f"{self.URL_BASE}{endpoint}?key={query_hash}"
+        result = self.session.post(url, headers=headers, data=query)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 1)
+        self.assertEqual(df["col_idx"], [query_hash])
+        self.assertAlmostEqual(df["columns"][0][0], -0.110451095)
+
+        # multi-column
+        col_names = ["F5", "BEB3", "SIK1"]
+        filter = "&".join([f"var:{index_col_name}={name}" for name in col_names])
+        query = f"method=mean&{filter}"
+        query_hash = hashlib.sha1(query.encode()).hexdigest()
+        url = f"{self.URL_BASE}{endpoint}?key={query_hash}"
+        result = self.session.post(url, headers=headers, data=query)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 1)
+        self.assertEqual(df["col_idx"], [query_hash])
+        self.assertAlmostEqual(df["columns"][0][0], -0.16628358)
+
     def _setupClass(child_class, command_line):
         child_class.ps, child_class.server = start_test_server(command_line)
         child_class.URL_BASE = f"{child_class.server}/api/v0.2/"
@@ -333,7 +423,8 @@ class EndPointsAnnotations(EndPoints):
 
     def test_get_user_annotations_existing_obs_keys_fbs(self):
         self._test_get_user_annotations_obs_keys_fbs(
-            "cluster-test", {"unassigned", "one", "two", "three", "four", "five", "six", "seven"},
+            "cluster-test",
+            {"unassigned", "one", "two", "three", "four", "five", "six", "seven"},
         )
 
     def test_put_user_annotations_obs_fbs(self):
@@ -416,6 +507,91 @@ class EndPointsCxg(unittest.TestCase, EndPoints):
     def tearDownClass(cls):
         stop_test_server(cls.ps)
 
+    def test_get_genesets_json(self):
+        endpoint = "genesets"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.get(url, headers={"Accept": "application/json"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        self.assertIsNotNone(result_data["genesets"])
+        self.assertIsNotNone(result_data["tid"])
+
+        self.assertEqual(
+            result_data,
+            {
+                "genesets": [
+                    {
+                        "genes": [
+                            {"gene_description": " a gene_description", "gene_symbol": "F5"},
+                            {"gene_description": "", "gene_symbol": "SUMO3"},
+                            {"gene_description": "", "gene_symbol": "SRM"},
+                        ],
+                        "geneset_description": "a description",
+                        "geneset_name": "first gene set name",
+                    },
+                    {
+                        "genes": [
+                            {"gene_description": "", "gene_symbol": "RER1"},
+                            {"gene_description": "", "gene_symbol": "SIK1"},
+                        ],
+                        "geneset_description": "",
+                        "geneset_name": "second gene set",
+                    },
+                    {"genes": [], "geneset_description": "", "geneset_name": "third gene set"},
+                    {"genes": [], "geneset_description": "fourth description", "geneset_name": "fourth_gene_set"},
+                    {"genes": [], "geneset_description": "", "geneset_name": "fifth_dataset"},
+                    {
+                        "genes": [
+                            {"gene_description": "", "gene_symbol": "ACD"},
+                            {"gene_description": "", "gene_symbol": "AATF"},
+                            {"gene_description": "", "gene_symbol": "F5"},
+                            {"gene_description": "", "gene_symbol": "PIGU"},
+                        ],
+                        "geneset_description": "",
+                        "geneset_name": "summary test",
+                    },
+                ],
+                "tid": 0,
+            },
+        )
+
+    def test_get_genesets_csv(self):
+        endpoint = "genesets"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.get(url, headers={"Accept": "text/csv"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "text/csv")
+        self.assertEqual(
+            result.text,
+            """gene_set_name,gene_set_description,gene_symbol,gene_description\r
+first gene set name,a description,F5, a gene_description\r
+first gene set name,a description,SUMO3,\r
+first gene set name,a description,SRM,\r
+second gene set,,RER1,\r
+second gene set,,SIK1,\r
+third gene set,,,\r
+fourth_gene_set,fourth description,,\r
+fifth_dataset,,,\r
+summary test,,ACD,\r
+summary test,,AATF,\r
+summary test,,F5,\r
+summary test,,PIGU,\r
+""",
+        )
+
+    def test_put_genesets(self):
+        endpoint = "genesets"
+        url = f"{self.URL_BASE}{endpoint}"
+
+        result = self.session.get(url, headers={"Accept": "application/json"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+
+        test1 = {"tid": 3, "genesets": []}
+        result = self.session.put(url, json=test1)
+
+        self.assertEqual(result.status_code, HTTPStatus.METHOD_NOT_ALLOWED)
+
 
 class EndPointsAnndataAnnotations(unittest.TestCase, EndPointsAnnotations):
     """Test Case for endpoints"""

backend/test/test_server/unit/common/test_api.py

@@ -560,7 +560,7 @@ class EndPointsAnnDataGenesets(unittest.TestCase, EndPoints):
                 "genesets": [
                     {
                         "genes": [
-                            {"gene_description": "a gene_description", "gene_symbol": "F5"},
+                            {"gene_description": " a gene_description", "gene_symbol": "F5"},
                             {"gene_description": "", "gene_symbol": "SUMO3"},
                             {"gene_description": "", "gene_symbol": "SRM"},
                         ],
@@ -602,7 +602,7 @@ class EndPointsAnnDataGenesets(unittest.TestCase, EndPoints):
         self.assertEqual(
             result.text,
             """gene_set_name,gene_set_description,gene_symbol,gene_description\r
-first gene set name,a description,F5,a gene_description\r
+first gene set name,a description,F5, a gene_description\r
 first gene set name,a description,SUMO3,\r
 first gene set name,a description,SRM,\r
 second gene set,,RER1,\r

client/src/reducers/annotations.js

@@ -35,9 +35,13 @@ const Annotations = (
       const dataCollectionName =
         action.config.parameters?.["annotations-data-collection-name"] ?? null;
       const dataCollectionNameIsReadOnly =
-        action.config.parameters?.[
+        (action.config.parameters?.[
-          "annotations-data-collection-name-is-read-only"
+          "annotations-data-collection-is-read-only"
-        ] ?? false;
+        ] ??
+          false) &&
+        (action.config.parameters?.annotations_genesets_name_is_read_only ??
+          true);
+
       const promptForFilename =
         action.config.parameters?.user_annotation_collection_name_enabled;
       return {