hosted gene sets routes, plus a few bug fixes (#2155)

* first cut at hosted gs routes

* lint

* update tests to match csv parser changes

* update tests to new API

* update gene set name validation rules to match requirements

* add path mapping from dataset to geneset

* add test cases for geneset GET route

* fix test assertion

* remove debugging code

* update gene set uri mapping function

* fix error message

* allow extra user-specified headers in gene set csv file

* clarify comment

backend/czi_hosted/common/annotations/annotations.py

@@ -1,21 +1,33 @@
-from abc import ABCMeta, abstractmethod
-
 import fastobo
 import fsspec
+import os
 
-from backend.common.errors import OntologyLoadFailure
+from flask import current_app, has_request_context
+
+from backend.common.errors import OntologyLoadFailure, DisabledFeatureError
 from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
+from backend.common.genesets import write_gene_sets_tidycsv, read_gene_sets_tidycsv, validate_gene_sets
+from backend.common.utils.data_locator import DataLocator
+from backend.common.utils.utils import path_join
 
 
-class Annotations(metaclass=ABCMeta):
-    """ baseclass for annotations, including ontologies"""
+class Annotations:
+    """ baseclass for annotations, including ontologies and genesets """
 
     """ our default ontology is the PURL for the Cell Ontology.
     See http://www.obofoundry.org/ontology/cl.html """
     DefaultOnotology = "http://purl.obolibrary.org/obo/cl.obo"
 
-    def __init__(self):
+    def __init__(self, config={}):
         self.ontology_data = None
+        self.config = config
+
+    def user_annotations_enabled(self):
+        return self.config.get("user-annotations", False)
+
+    def check_user_annotations_enabled(self):
+        if not self.user_annotations_enabled():
+            raise DisabledFeatureError("User annotations are disabled.")
 
     def load_ontology(self, path):
         """Load and parse ontologies - currently support OBO files only."""
@@ -49,22 +61,78 @@ class Annotations(metaclass=ABCMeta):
 
         return schema
 
-    @abstractmethod
     def set_collection(self, name):
         """set or create a new annotation collection"""
-        pass
+        raise NotImplementedError
 
-    @abstractmethod
     def read_labels(self, data_adaptor):
         """Return the labels as a pandas.DataFrame"""
-        pass
+        raise NotImplementedError
 
-    @abstractmethod
     def write_labels(self, df, data_adaptor):
         """Write the labels (df) to a persistent storage such that it can later be read"""
-        pass
+        raise NotImplementedError
 
-    @abstractmethod
     def update_parameters(self, parameters, data_adaptor):
         """Update configuration parameters that describe information about the annotations feature"""
-        pass
+        params = {}
+        params["annotations_genesets_readonly"] = True
+        params["annotations_genesets_name_is_read_only"] = True
+        parameters.update(params)
+
+    @staticmethod
+    def gene_sets_to_csv(genesets):
+        """
+        Convert the internal genesets format (returned by read_gene_set) into
+        the simple Tidy CSV.
+        """
+        from io import StringIO
+
+        if isinstance(genesets, dict):
+            genesets = genesets.values()
+
+        with StringIO() as sio:
+            write_gene_sets_tidycsv(sio, genesets)
+            return sio.getvalue()
+
+    @staticmethod
+    def gene_sets_to_response(genesets):
+        """
+        Convert the internal genesets format (returned by read_gene_set) into
+        the dict expected by the JSON REST API
+        """
+        return list(genesets.values())
+
+    def read_gene_sets(self, data_adaptor, context=None):
+        if has_request_context():
+            if not current_app.auth.is_user_authenticated():
+                return ({}, 0)
+
+        gene_sets_uri_or_path = dataset_uri_to_geneset_uri(data_adaptor.data_locator.uri_or_path)
+
+        server_config = data_adaptor.server_config
+        region_name = None if server_config is None else server_config.data_locator__s3__region_name
+        gene_sets_locator = DataLocator(gene_sets_uri_or_path, region_name=region_name)
+        if not gene_sets_locator.exists():
+            return ({}, 0)
+
+        gene_sets = read_gene_sets_tidycsv(gene_sets_locator, context)
+        schema = data_adaptor.get_schema()
+        var_index = schema["annotations"]["var"].get("index", "index")
+        var_names = set(data_adaptor.query_var_array(var_index))
+
+        gene_sets = validate_gene_sets(gene_sets, var_names)
+        return (gene_sets, 0)
+
+
+def dataset_uri_to_geneset_uri(data_uri_or_path):
+    """ given a dataset URI, return the associated gene set URI """
+    data_basename = os.path.basename(data_uri_or_path)
+    base, ext = os.path.splitext(data_basename)
+    if ext is not None:  # strip extension, if any
+        data_basename = base
+
+    genesets_basename = f"{data_basename}-genesets.csv"
+    gene_sets_uri_or_path = path_join(data_uri_or_path, "..", genesets_basename)
+
+    return gene_sets_uri_or_path

backend/czi_hosted/common/annotations/hosted_tiledb.py

@@ -17,8 +17,8 @@ from backend.czi_hosted.db.cellxgene_orm import Annotation
 class AnnotationsHostedTileDB(Annotations):
     CXG_ANNO_COLLECTION = "cxg_anno_collection"
 
-    def __init__(self, directory_path, db):
-        super().__init__()
+    def __init__(self, config, directory_path, db):
+        super().__init__(config)
         self.db = db
         if directory_path[-1] == "/":
             self.directory_path = directory_path
@@ -158,6 +158,8 @@ class AnnotationsHostedTileDB(Annotations):
         self.db.session.commit()
 
     def update_parameters(self, parameters, data_adaptor):
+        super().update_parameters(parameters, data_adaptor)
+
         params = {}
         params["annotations"] = True
         params["user_annotation_collection_name_enabled"] = False

backend/czi_hosted/common/annotations/local_file_csv.py

@@ -16,8 +16,8 @@ from backend.common.errors import AnnotationsError
 class AnnotationsLocalFile(Annotations):
     CXG_ANNO_COLLECTION = "cxg_anno_collection"
 
-    def __init__(self, output_dir, output_file):
-        super().__init__()
+    def __init__(self, config, output_dir, output_file):
+        super().__init__(config)
         self.output_dir = output_dir
         self.output_file = output_file
         # lock used to protect label file write ops
@@ -169,6 +169,8 @@ class AnnotationsLocalFile(Annotations):
                 os.remove(os.path.join(backup_dir, bu))
 
     def update_parameters(self, parameters, data_adaptor):
+        super().update_parameters(parameters, data_adaptor)
+
         params = {}
         params["annotations"] = True
         params["user_annotation_collection_name_enabled"] = True
@@ -190,7 +192,7 @@ class AnnotationsLocalFile(Annotations):
             collection = self.get_collection()
             if current_app.auth.is_user_authenticated():
                 params["annotations-user-data-idhash"] = self._get_userdata_idhash(data_adaptor)
-                params["annotations-data-collection-is-read-only"] = False
+                params["annotations-data-collection-is-read-only"] = not self.user_annotations_enabled()
                 params["annotations-data-collection-name"] = collection
 
         parameters.update(params)