diff --git a/local_server/app/app.py b/local_server/app/app.py index 84d37565..6bf53cd3 100644 --- a/local_server/app/app.py +++ b/local_server/app/app.py @@ -144,6 +144,17 @@ class LayoutObsAPI(Resource): return common_rest.layout_obs_put(request, data_adaptor) +class GenesetsAPI(Resource): + @rest_get_data_adaptor + def get(self, data_adaptor): + return common_rest.genesets_get(request, data_adaptor) + + @requires_authentication + @rest_get_data_adaptor + def put(self, data_adaptor): + return common_rest.genesets_put(request, data_adaptor) + + def get_api_base_resources(bp_base): """Add resources that are accessed from the api url""" api = Api(bp_base) @@ -169,6 +180,7 @@ def get_api_dataroot_resources(bp_dataroot): add_resource(AnnotationsObsAPI, "/annotations/obs") add_resource(AnnotationsVarAPI, "/annotations/var") add_resource(DataVarAPI, "/data/var") + add_resource(GenesetsAPI, "/genesets") # Display routes add_resource(ColorsAPI, "/colors") # Computation routes diff --git a/local_server/cli/launch.py b/local_server/cli/launch.py index 23963fde..5c652966 100644 --- a/local_server/cli/launch.py +++ b/local_server/cli/launch.py @@ -32,16 +32,17 @@ def annotation_args(func): multiple=False, metavar="", help="CSV file to initialize editing of existing annotations; will be altered in-place. " - "Incompatible with --annotations-dir.", + "Incompatible with --user-generated-data-dir.", ) @click.option( + "--user-generated-data-dir", "--annotations-dir", default=DEFAULT_CONFIG.dataset_config.user_annotations__local_file_csv__directory, show_default=False, multiple=False, metavar="", help="Directory of where to save output annotations; filename will be specified in the application. " - "Incompatible with --annotations-file.", + "Incompatible with --annotations-file and --genesets-file.", ) @click.option( "--experimental-annotations-ontology", @@ -57,6 +58,23 @@ def annotation_args(func): metavar="", help="Location of OBO file defining cell annotation autosuggest terms.", ) + @click.option( + "--disable-genesets-save", + is_flag=True, + default=DEFAULT_CONFIG.dataset_config.user_annotations__genesets__readonly, + show_default=False, + help="Disable saving gene sets. If disabled, users will be able to make changes to gene sets but all " + "changes will be lost on browser refresh.", + ) + @click.option( + "--genesets-file", + default=DEFAULT_CONFIG.dataset_config.user_annotations__local_file_csv__genesets_file, + show_default=True, + multiple=False, + metavar="", + help="CSV file to initialize editing of gene sets; will be altered in-place. Incompatible with " + "--user-generated-data-dir.", + ) @functools.wraps(func) def wrapper(*args, **kwargs): return func(*args, **kwargs) @@ -315,7 +333,9 @@ def launch( about, disable_annotations, annotations_file, - annotations_dir, + user_generated_data_dir, + genesets_file, + disable_genesets_save, backed, disable_diffexp, experimental_annotations_ontology, @@ -373,7 +393,9 @@ def launch( app__scripts=scripts, user_annotations__enable=not disable_annotations, user_annotations__local_file_csv__file=annotations_file, - user_annotations__local_file_csv__directory=annotations_dir, + user_annotations__local_file_csv__directory=user_generated_data_dir, + user_annotations__local_file_csv__genesets_file=genesets_file, + user_annotations__genesets__readonly=disable_genesets_save, user_annotations__ontology__enable=experimental_annotations_ontology, user_annotations__ontology__obo_location=experimental_annotations_ontology_obo, presentation__max_categories=max_category_items, diff --git a/local_server/common/annotations/annotations.py b/local_server/common/annotations/annotations.py index 232a354f..5339eab3 100644 --- a/local_server/common/annotations/annotations.py +++ b/local_server/common/annotations/annotations.py @@ -3,19 +3,34 @@ from abc import ABCMeta, abstractmethod import fastobo import fsspec -from local_server.common.errors import OntologyLoadFailure +from local_server.common.errors import OntologyLoadFailure, DisabledFeatureError from local_server.common.utils.type_conversion_utils import get_schema_type_hint_of_array class Annotations(metaclass=ABCMeta): - """ baseclass for annotations, including ontologies""" + """ baseclass for annotations, including ontologies and genesets""" """ our default ontology is the PURL for the Cell Ontology. See http://www.obofoundry.org/ontology/cl.html """ DefaultOnotology = "http://purl.obolibrary.org/obo/cl.obo" - def __init__(self): + def __init__(self, config={}): self.ontology_data = None + self.config = config + + def user_annotations_enabled(self): + return self.config.get("user-annotations", False) + + def genesets_save_enabled(self): + return self.config.get("genesets-save", False) + + def check_user_annotations_enabled(self): + if not self.user_annotations_enabled(): + raise DisabledFeatureError("User annotations are disabled.") + + def check_genesets_save_enabled(self): + if not self.genesets_save_enabled(): + raise DisabledFeatureError("User genesets save is disabled.") def load_ontology(self, path): """Load and parse ontologies - currently support OBO files only.""" @@ -64,7 +79,66 @@ class Annotations(metaclass=ABCMeta): """Write the labels (df) to a persistent storage such that it can later be read""" pass + @abstractmethod + def read_genesets(self, data_adaptor): + """Return the genesets from persistent storage """ + pass + + @abstractmethod + def write_genesets(self, gs, data_adaptor): + """Write the genesets (gs) to a persistent storage such that it can later be read""" + pass + @abstractmethod def update_parameters(self, parameters, data_adaptor): """Update configuration parameters that describe information about the annotations feature""" pass + + Genesets_Header = [ + "geneset_name", + "geneset_description", + "gene_symbol", + "gene_description", + ] + + @staticmethod + def genesets_to_csv(genesets): + """ + Convert the internal genesets format (returned by read_geneset) into + the simple Tidy CSV. + """ + from io import StringIO + import csv + + if type(genesets) == dict: + genesets = genesets.values() + + with StringIO() as sio: + writer = csv.writer(sio, dialect='excel') + writer.writerow(Annotations.Genesets_Header) + for geneset in genesets: + # genes may be empty, in which case we skip the geneset entirely + genes = geneset["genes"] + if not genes: + writer.writerow([geneset["geneset_name"], geneset.get("geneset_description", ""), "", ""]) + else: + writer.writerows( + [ + [ + geneset["geneset_name"], + geneset.get("geneset_description", ""), + gene["gene_symbol"], + gene.get("gene_description", ""), + ] + for gene in genes + ] + ) + return sio.getvalue() + + @staticmethod + def genesets_to_response(genesets): + """ + Convert the internal genesets format (returned by read_geneset) into + the dict expected by the JSON REST API + """ + return list(genesets.values()) diff --git a/local_server/common/annotations/local_file_csv.py b/local_server/common/annotations/local_file_csv.py index 639c1e0b..d23a3f0d 100644 --- a/local_server/common/annotations/local_file_csv.py +++ b/local_server/common/annotations/local_file_csv.py @@ -4,29 +4,35 @@ import re import threading from datetime import datetime from hashlib import blake2b +import csv import pandas as pd from flask import session, has_request_context, current_app from local_server import __version__ as cellxgene_version from local_server.common.annotations.annotations import Annotations -from local_server.common.errors import AnnotationsError +from local_server.common.errors import AnnotationsError, ObsoleteRequest class AnnotationsLocalFile(Annotations): CXG_ANNO_COLLECTION = "cxg_anno_collection" - def __init__(self, output_dir, output_file): - super().__init__() + def __init__(self, config, output_dir, label_output_file, genesets_output_file): + super().__init__(config) self.output_dir = output_dir - self.output_file = output_file + self.label_output_file = label_output_file + self.genesets_output_file = genesets_output_file # lock used to protect label file write ops self.label_lock = threading.RLock() + self.genesets_lock = threading.RLock() - # cache the most recent annotations + # cache the most recent annotations. self.last_fname = None self.last_labels = None + # txn ID - used to de-dup geneset writes + self.last_geneset_tid = 0 + def is_safe_collection_name(self, name): """ return true if this is a safe collection name @@ -47,11 +53,13 @@ class AnnotationsLocalFile(Annotations): return session.get(self.CXG_ANNO_COLLECTION) def read_labels(self, data_adaptor): + self.check_user_annotations_enabled() # raises + if has_request_context(): if not current_app.auth.is_user_authenticated(): return pd.DataFrame() - fname = self._get_filename(data_adaptor) + fname = self._get_celllabels_filename(data_adaptor) with self.label_lock: if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0: # returned the cached labels if possible, otherwise read them from the file @@ -69,6 +77,8 @@ class AnnotationsLocalFile(Annotations): return pd.DataFrame() def write_labels(self, df, data_adaptor): + self.check_user_annotations_enabled() # raises + # update our internal state and save it. Multi-threading often enabled, # so treat this as a critical section. with self.label_lock: @@ -81,7 +91,7 @@ class AnnotationsLocalFile(Annotations): f"which was last modified on {lastmodstr}\n" ) - fname = self._get_filename(data_adaptor) + fname = self._get_celllabels_filename(data_adaptor) self._backup(fname) if not df.empty: with open(fname, "w", newline="") as f: @@ -95,12 +105,56 @@ class AnnotationsLocalFile(Annotations): self.last_fname = fname self.last_labels = df + def read_genesets(self, data_adaptor, context=None): + if has_request_context(): + if not current_app.auth.is_user_authenticated(): + return ([], None) + + fname = self._get_genesets_filename(data_adaptor) + genesets = {} + tid = None + with self.genesets_lock: + tid = self.last_geneset_tid # inside the critical section + if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0: + with open(fname, newline="") as f: + genesets = read_geneset_tidycsv(f, context) + + return (genesets, tid) + + def write_genesets(self, genesets, tid, data_adaptor): + self.check_genesets_save_enabled() # raises + + if type(tid) != int or tid < 0: + raise ValueError("tid must be a positive integer") + + with self.genesets_lock: + # skip if the request is stale + if tid is not None: + if tid <= self.last_geneset_tid: + raise ObsoleteRequest("TID is stale.") + self.last_geneset_tid = tid + + lastmod = data_adaptor.get_last_mod_time() + lastmodstr = "'unknown'" if lastmod is None else lastmod.isoformat(timespec="seconds") + header = ( + f"# Geneset generated on {datetime.now().isoformat(timespec='seconds')} " + f"using cellxgene version {cellxgene_version}\n" + f"# Input data file was {data_adaptor.get_location()}, " + f"which was last modified on {lastmodstr}\n" + ) + + fname = self._get_genesets_filename(data_adaptor) + self._backup(fname) + with open(fname, "w", newline="") as f: + f.write(header) + f.write(self.genesets_to_csv(genesets)) + def _get_userdata_idhash(self, data_adaptor): """ Return a short hash that weakly identifies the user and dataset. Used to create safe annotations output file names. """ - uid = current_app.auth.get_user_id() + uid = current_app.auth.get_user_id() or "" id = (uid + data_adaptor.get_location()).encode() idhash = base64.b32encode(blake2b(id, digest_size=5).digest()).decode("utf-8") return idhash @@ -109,16 +163,27 @@ class AnnotationsLocalFile(Annotations): if self.output_dir: return self.output_dir - if self.output_file: - return os.path.dirname(self.path.abspath(self.output_dir)) + output_file = self.label_output_file or self.genesets_output_file + if output_file: + return os.path.dirname(self.path.abspath(output_file)) return os.getcwd() - def _get_filename(self, data_adaptor): + def _get_celllabels_filename(self, data_adaptor): """ return the current annotation file name """ - if self.output_file: - return self.output_file + if self.label_output_file: + return self.label_output_file + return self._get_filename(data_adaptor, "celllabels") + + def _get_genesets_filename(self, data_adaptor): + """ return the current genesets file name """ + if self.genesets_output_file: + return self.genesets_output_file + + return self._get_filename(data_adaptor, "genesets") + + def _get_filename(self, data_adaptor, anno_name): # we need to generate a file name, which we can only do if we have a UID and collection name if session is None: raise AnnotationsError("unable to determine file name for annotations") @@ -131,7 +196,7 @@ class AnnotationsLocalFile(Annotations): raise AnnotationsError("unable to determine file name for annotations") idhash = self._get_userdata_idhash(data_adaptor) - return os.path.join(self._get_output_dir(), f"{collection}-{idhash}.csv") + return os.path.join(self._get_output_dir(), f"{collection}-{anno_name}-{idhash}.csv") def _backup(self, fname, max_backups=9): """ @@ -170,7 +235,8 @@ class AnnotationsLocalFile(Annotations): def update_parameters(self, parameters, data_adaptor): params = {} - params["annotations"] = True + params["annotations"] = self.user_annotations_enabled() + params["annotations_genesets_readonly"] = not self.genesets_save_enabled() params["user_annotation_collection_name_enabled"] = True if self.ontology_data: @@ -179,18 +245,108 @@ class AnnotationsLocalFile(Annotations): else: params["annotations_cell_ontology_enabled"] = False - if self.output_file is not None: - # user has hard-wired the name of the annotation data collection - fname = os.path.basename(self.output_file) + if self.label_output_file is not None: + # user has hard-wired the name of the annotation cell label data collection + fname = os.path.basename(self.label_output_file) collection_fname = os.path.splitext(fname)[0] params["annotations-data-collection-is-read-only"] = True params["annotations-data-collection-name"] = collection_fname elif session is not None: collection = self.get_collection() - if current_app.auth.is_user_authenticated(): - params["annotations-user-data-idhash"] = self._get_userdata_idhash(data_adaptor) - params["annotations-data-collection-is-read-only"] = False - params["annotations-data-collection-name"] = collection + params["annotations-data-collection-is-read-only"] = False + params["annotations-data-collection-name"] = collection + + if current_app.auth.is_user_authenticated(): + params["annotations-user-data-idhash"] = self._get_userdata_idhash(data_adaptor) parameters.update(params) + + +def read_geneset_tidycsv(f, context=None): + """ + Read & parse the Tidy CSV format, applying validation checks for mandatory + values, and de-duping rules. + + Format is a four-column CSV, with a mandatory header row, and optional "#" prefixed + comments. Format: + + geneset_name, geneset_description, gene_symbol, gene_description + + geneset_name and gene_symbol must be non-null; others are optional. + + Returns: a dictionary of the shape (values in angle-brackets vary): + + { + : { + "geneset_name": , + "geneset_description": , + "genes": [ + { + "gene_symbol": , + "gene_description": + }, + ... + ] + }, + ... + } + """ + + class myDialect(csv.excel): + skipinitialspace = True + + def just(n, seq): + it = iter(seq) + for _ in range(n - 1): + yield next(it, "") + yield tuple(it) + + messagefn = context["messagefn"] if context else (lambda x: None) + + reader = csv.reader(f, dialect=myDialect()) + genesets = {} + haveReadHeader = False + lineno = 0 + for row in reader: + lineno += 1 + # ignore empty rows + if len(row) == 0: + continue + # if row starts with '#' it is a comment + if row[0].startswith("#"): + continue + # if this is the first non-comment row, assume it is a header + if not haveReadHeader: + if row != Annotations.Genesets_Header: + raise AnnotationsError("Geneset CSV file missing the required column header.") + haveReadHeader = True + continue + + geneset_name, geneset_description, gene_symbol, gene_description, _ = just(5, row) + if not geneset_name: + raise AnnotationsError(f"Geneset CSV missing required geneset or gene name on line {lineno}") + if (not gene_symbol) and gene_description: + messagefn(f"Warning: Missing gene name in geneset name {geneset_name} on line {lineno}.") + + if geneset_name in genesets: + gs = genesets[geneset_name] + else: + gs = genesets[geneset_name] = { + "geneset_name": geneset_name, + "geneset_description": geneset_description, + "genes": [], + } + # Use first geneset_description with a value + if not gs["geneset_description"] and geneset_description: + gs["geneset_description"] = geneset_description + # add the gene if the gene_symbol is defined + if gene_symbol: + gs["genes"].append( + { + "gene_symbol": gene_symbol, + "gene_description": gene_description, + } + ) + + return genesets diff --git a/local_server/common/config/client_config.py b/local_server/common/config/client_config.py index c6cde91c..872910c7 100644 --- a/local_server/common/config/client_config.py +++ b/local_server/common/config/client_config.py @@ -44,6 +44,9 @@ def get_client_config(app_config, data_adaptor): "annotations": False, "annotations_file": None, "annotations_dir": None, + "annotations_genesets": True, # feature flag + "annotations_genesets_readonly": dataset_config.user_annotations__genesets__readonly, + "annotations_genesets_summary_methods": ["mean"], "annotations_cell_ontology_enabled": False, "annotations_cell_ontology_obopath": None, "annotations_cell_ontology_terms": None, diff --git a/local_server/common/config/dataset_config.py b/local_server/common/config/dataset_config.py index 91993ae2..b5ed34b3 100644 --- a/local_server/common/config/dataset_config.py +++ b/local_server/common/config/dataset_config.py @@ -3,7 +3,7 @@ from os.path import splitext, isdir from local_server.common.annotations.local_file_csv import AnnotationsLocalFile from local_server.common.config.base_config import BaseConfig -from local_server.common.errors import ConfigurationError, OntologyLoadFailure +from local_server.common.errors import ConfigurationError, OntologyLoadFailure, AnnotationsError from local_server.compute.scanpy import get_scanpy_module from local_server.data_common.matrix_loader import MatrixDataLoader @@ -32,6 +32,10 @@ class DatasetConfig(BaseConfig): self.user_annotations__ontology__obo_location = default_config["user_annotations"]["ontology"][ "obo_location" ] + self.user_annotations__genesets__readonly = default_config["user_annotations"]["genesets"]["readonly"] + self.user_annotations__local_file_csv__genesets_file = default_config["user_annotations"]["local_file_csv"][ + "genesets_file" + ] self.embeddings__names = default_config["embeddings"]["names"] self.embeddings__enable_reembedding = default_config["embeddings"]["enable_reembedding"] @@ -56,9 +60,7 @@ class DatasetConfig(BaseConfig): def get_data_adaptor(self): server_config = self.app_config.server_config if not server_config.data_adaptor: - matrix_data_loader = MatrixDataLoader( - server_config.single_dataset__datapath, app_config=self.app_config - ) + matrix_data_loader = MatrixDataLoader(server_config.single_dataset__datapath, app_config=self.app_config) server_config.data_adaptor = matrix_data_loader.open(self.app_config) return server_config.data_adaptor @@ -96,11 +98,16 @@ class DatasetConfig(BaseConfig): self.validate_correct_type_of_configuration_attribute( "user_annotations__local_file_csv__file", (type(None), str) ) + self.validate_correct_type_of_configuration_attribute( + "user_annotations__local_file_csv__genesets_file", (type(None), str) + ) self.validate_correct_type_of_configuration_attribute("user_annotations__ontology__enable", bool) self.validate_correct_type_of_configuration_attribute( "user_annotations__ontology__obo_location", (type(None), str) ) - if self.user_annotations__enable: + self.validate_correct_type_of_configuration_attribute("user_annotations__genesets__readonly", bool) + + if self.user_annotations__enable or not self.user_annotations__genesets__readonly: server_config = self.app_config.server_config if not self.app__authentication_enable: raise ConfigurationError("user annotations requires authentication to be enabled") @@ -108,59 +115,84 @@ class DatasetConfig(BaseConfig): auth_type = server_config.authentication__type raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations") - if self.user_annotations__type == "local_file_csv": - self.handle_local_file_csv_annotations() - else: - raise ConfigurationError('The only annotation type support is "local_file_csv"') + # Must always have an annotations instance to support genesets. User annotation (cell labels) are optional + # as are writable gene sets + if self.user_annotations__type == "local_file_csv": + self.handle_local_file_csv_annotations(context) + else: + raise ConfigurationError('The only annotation type support is "local_file_csv"') + + if self.user_annotations__enable: if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location: try: self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location) except OntologyLoadFailure as e: raise ConfigurationError("Unable to load ontology terms\n" + str(e)) - else: - self.check_annotation_config_vars_not_set(context) - def handle_local_file_csv_annotations(self): + self.check_annotation_config_vars_not_set(context) + + def handle_local_file_csv_annotations(self, context): dirname = self.user_annotations__local_file_csv__directory filename = self.user_annotations__local_file_csv__file - if filename is not None and dirname is not None: - raise ConfigurationError("'annotations-file' and 'annotations-dir' may not be used together.") + genesets_filename = self.user_annotations__local_file_csv__genesets_file + + if dirname is not None and (filename is not None or genesets_filename is not None): + raise ConfigurationError( + "'user-generated-data-dir' may not be used with annotations-file' or 'genesets-file'." + ) if filename is not None: lf_name, lf_ext = splitext(filename) if lf_ext and lf_ext != ".csv": raise ConfigurationError(f"annotation file type must be .csv: {filename}") + if genesets_filename is not None: + lf_name, lf_ext = splitext(genesets_filename) + if lf_ext and lf_ext != ".csv": + raise ConfigurationError(f"genesets file type must be .csv: {genesets_filename}") + if dirname is not None and not isdir(dirname): try: os.mkdir(dirname) except OSError: - raise ConfigurationError("Unable to create directory specified by --annotations-dir") + raise ConfigurationError("Unable to create directory specified by --user-generated-data-dir") - self.user_annotations = AnnotationsLocalFile(dirname, filename) + anno_config = { + "user-annotations": self.user_annotations__enable, + "genesets-save": not self.user_annotations__genesets__readonly, + } + self.user_annotations = AnnotationsLocalFile(anno_config, dirname, filename, genesets_filename) # if the user has specified a fixed label file, go ahead and validate it # so that we can remove errors early in the process. server_config = self.app_config.server_config - if server_config.single_dataset__datapath and self.user_annotations__local_file_csv__file: + if server_config.single_dataset__datapath: data_adaptor = self.get_data_adaptor() - data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor)) + if self.user_annotations__local_file_csv__file: + data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor)) + if self.user_annotations__local_file_csv__genesets_file: + try: + data_adaptor.check_new_genesets(self.user_annotations.read_genesets(data_adaptor, context), context) + except (ValueError, AnnotationsError, KeyError) as e: + raise ConfigurationError(f"Unable to read genesets CSV file: {str(e)}") from e def check_annotation_config_vars_not_set(self, context): if self.user_annotations__type is not None: dirname = self.user_annotations__local_file_csv__directory filename = self.user_annotations__local_file_csv__file - if filename is not None: - context["messagefn"]("Warning: --annotations-file ignored as annotations are disabled.") - if dirname is not None: - context["messagefn"]("Warning: --annotations-dir ignored as annotations are disabled.") - - if self.user_annotations__ontology__enable: - context["messagefn"]("Warning: --experimental-annotations-ontology ignored as annotations are disabled.") - if self.user_annotations__ontology__obo_location is not None: - context["messagefn"]( - "Warning: --experimental-annotations-ontology-obo ignored as annotations are disabled." - ) + if not self.user_annotations__enable: + if filename is not None: + context["messagefn"]("Warning: --annotations-file ignored as annotations are disabled.") + if self.user_annotations__ontology__enable: + context["messagefn"]( + "Warning: --experimental-annotations-ontology ignored as annotations are disabled." + ) + if self.user_annotations__ontology__obo_location is not None: + context["messagefn"]( + "Warning: --experimental-annotations-ontology-obo ignored as annotations are disabled." + ) + if dirname is not None: + context["messagefn"]("Warning: --user-generated-data-dir ignored as annotations are disabled.") def handle_embeddings(self): self.validate_correct_type_of_configuration_attribute("embeddings__names", list) @@ -186,6 +218,5 @@ class DatasetConfig(BaseConfig): data_adaptor = self.get_data_adaptor() if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False): context["messagefn"]( - "CAUTION: due to the size of your dataset, " - "running differential expression may take longer or fail." + "CAUTION: due to the size of your dataset, " "running differential expression may take longer or fail." ) diff --git a/local_server/common/errors.py b/local_server/common/errors.py index ca339bee..b582a43f 100644 --- a/local_server/common/errors.py +++ b/local_server/common/errors.py @@ -55,3 +55,4 @@ define_exception("OntologyLoadFailure", "Raised when reading the ontology file f define_exception("ConfigurationError", "Raised when checking configuration errors") define_exception("PrepareError", "Raised when data is misprepared") define_exception("SecretKeyRetrievalError", "Raised when get_secret_key from AWS fails") +define_exception("ObsoleteRequest", "Raised when the request is no longer valid.") \ No newline at end of file diff --git a/local_server/common/rest.py b/local_server/common/rest.py index 8e12bd2a..1d634548 100644 --- a/local_server/common/rest.py +++ b/local_server/common/rest.py @@ -17,6 +17,8 @@ from local_server.common.errors import ( ExceedsLimitError, DatasetAccessError, ColorFormatException, + AnnotationsError, + ObsoleteRequest, ) import json @@ -44,7 +46,7 @@ def _query_parameter_to_filter(args): Query param filters look like: :name=value, where value may be one of: - - a range, min,max, where either may be an open range by using an asterisc, eg, 10,* + - a range, min,max, where either may be an open range by using an asterisk, eg, 10,* - a value Eg, ...?tissue=lung&obs:tissue=heart&obs:num_reads=1000,* @@ -106,7 +108,7 @@ def schema_get_helper(data_adaptor): # add label obs annotations as needed annotations = data_adaptor.dataset_config.user_annotations - if annotations is not None: + if annotations.user_annotations_enabled(): label_schema = annotations.get_schema(data_adaptor) schema["annotations"]["obs"]["columns"].extend(label_schema) @@ -140,7 +142,7 @@ def annotations_obs_get(request, data_adaptor): try: labels = None annotations = data_adaptor.dataset_config.user_annotations - if annotations: + if annotations.user_annotations_enabled(): labels = annotations.read_labels(data_adaptor) fbs = data_adaptor.annotation_to_fbs_matrix(Axis.OBS, fields, labels) return make_response(fbs, HTTPStatus.OK, {"Content-Type": "application/octet-stream"}) @@ -151,7 +153,7 @@ def annotations_obs_get(request, data_adaptor): def annotations_put_fbs_helper(data_adaptor, fbs): """helper function to write annotations from fbs""" annotations = data_adaptor.dataset_config.user_annotations - if annotations is None: + if not annotations.user_annotations_enabled(): raise DisabledFeatureError("Writable annotations are not enabled") new_label_df = decode_matrix_fbs(fbs) @@ -166,7 +168,7 @@ def inflate(data): def annotations_obs_put(request, data_adaptor): annotations = data_adaptor.dataset_config.user_annotations - if annotations is None: + if not annotations.user_annotations_enabled(): return abort(HTTPStatus.NOT_IMPLEMENTED) anno_collection = request.args.get("annotation-collection-name", default=None) @@ -196,9 +198,6 @@ def annotations_var_get(request, data_adaptor): try: labels = None - annotations = data_adaptor.dataset_config.user_annotations - if annotations is not None: - labels = annotations.read_labels(data_adaptor) return make_response( data_adaptor.annotation_to_fbs_matrix(Axis.VAR, fields, labels), HTTPStatus.OK, @@ -328,3 +327,56 @@ def layout_obs_put(request, data_adaptor): return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, str(e)) except (ValueError, DisabledFeatureError, FilterError) as e: return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True) + + +def genesets_get(request, data_adaptor): + preferred_mimetype = request.accept_mimetypes.best_match(["application/json", "text/csv"]) + if preferred_mimetype not in ("application/json", "text/csv"): + return abort(HTTPStatus.NOT_ACCEPTABLE) + + try: + annotations = data_adaptor.dataset_config.user_annotations + (genesets, tid) = data_adaptor.check_new_genesets(annotations.read_genesets(data_adaptor)) + + if preferred_mimetype == "text/csv": + return make_response( + annotations.genesets_to_csv(genesets), + HTTPStatus.OK, + { + "Content-Type": "text/csv", + "Content-Disposition": "attachment; filename=genesets.csv", + }, + ) + else: + return make_response( + jsonify({"genesets": annotations.genesets_to_response(genesets), "tid": tid}), HTTPStatus.OK + ) + except (ValueError, KeyError, AnnotationsError) as e: + return abort_and_log(HTTPStatus.BAD_REQUEST, str(e)) + + +def genesets_put(request, data_adaptor): + annotations = data_adaptor.dataset_config.user_annotations + if not annotations.genesets_save_enabled(): + return abort(HTTPStatus.NOT_IMPLEMENTED) + + anno_collection = request.args.get("annotation-collection-name", default=None) + if anno_collection is not None: + if not annotations.is_safe_collection_name(anno_collection): + return abort(HTTPStatus.BAD_REQUEST, "Bad annotation collection name") + annotations.set_collection(anno_collection) + + args = request.get_json() + try: + genesets = args.get("genesets", None) + tid = args.get("tid", None) + if genesets is None: + abort(HTTPStatus.BAD_REQUEST) + + (gs, _) = data_adaptor.check_new_genesets((genesets, tid)) + annotations.write_genesets(gs, tid, data_adaptor) + return make_response(jsonify({"status": "OK"}), HTTPStatus.OK) + except (ValueError, DisabledFeatureError, KeyError) as e: + return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True) + except (ObsoleteRequest, TypeError) as e: + return abort(HTTPStatus.NOT_FOUND, description=str(e)) diff --git a/local_server/data_common/data_adaptor.py b/local_server/data_common/data_adaptor.py index c9f6d079..5dd743cf 100644 --- a/local_server/data_common/data_adaptor.py +++ b/local_server/data_common/data_adaptor.py @@ -1,5 +1,6 @@ from abc import ABCMeta, abstractmethod from os.path import basename, splitext +import re import numpy as np import pandas as pd @@ -261,6 +262,95 @@ class DataAdaptor(metaclass=ABCMeta): return labels_df + def check_new_genesets(self, args, context=None): + """ + Check validity of gene sets, return if correct, else raise error. + May also modify the gene set for conditions that should be resolved, + but which do not warrant a hard error. + + Argument 'args' must be a tuple containing (genesets, tid). Genesets + may be either the REST OTA format (list of dicts) or the internal format + (dict of dicts, keyed by the geneset name). + + Rules: + 0. all geneset names must be unique. + 1. All geneset names must be legal, meaning: + * no leading or trailing white space + * no multi-space runs + * character set matches: [A-Z][a-z][0-9][ .()-] + Generates hard error. + 2. Gene symbols must be part of the current var_index. If symbol not in var_index, + will generate a warning and the symbol removed. + 3. Duplicate gene symbols are silently de-duped. + """ + (genesets, tid) = args + messagefn = context["messagefn"] if context else (lambda x: None) + + # accept genesets args as either the internal (dict) or REST (list) format, + # as they are identical except for the dict being keyed by geneset_name. + if type(genesets) not in (dict, list): + raise ValueError("Genesets must be either dict or list.") + genesets = genesets if type(genesets) == list else genesets.values() + + # 0. check for uniqueness of geneset names + geneset_names = [gs["geneset_name"] for gs in genesets] + if len(set(geneset_names)) != len(geneset_names): + raise KeyError("All geneset names must be unique.") + + # 1. check gene set character set and format + legal_name = re.compile(r"^(\w|[ .()-])+$") + for name in geneset_names: + if type(name) != str or len(name) == 0: + raise KeyError("Geneset names must be non-null string.") + if name[0] in " \t\n\r" or name[-1] in " \t\n\r" or not legal_name.match(name) or " " in name: + messagefn( + "Error: " + f"Geneset name {name} is not valid. Only alphanumeric and limited special characters (-_.) " + "and space are allowed. Leading, trailing, and multiple spaces within a name are not allowed." + ) + raise KeyError( + "Geneset name is not valid, only alphanumeric and limited special characters (-_.) " + "and space are allowed. Leading, trailing, and multiple spaces within a name are not allowed." + ) + + # 2. & 3. check for duplicate gene symbols, and those not present in the dataset. They will + # generate a warning and be removed. + var_names = set(self.query_var_array(self.parameters.get("var_names"))) + for geneset in genesets: + if type(geneset) != dict: + raise ValueError("Each geneset must be a dict.") + geneset_name = geneset["geneset_name"] + genes = geneset["genes"] + if type(genes) != list: + raise ValueError("Geneset genes field must be a list") + gene_symbol_already_seen = set() + new_genes = [] + for gene in genes: + gene_symbol = gene["gene_symbol"] + if type(gene_symbol) != str or len(gene_symbol) == 0: + raise ValueError("Gene symbol must be non-null string.") + if gene_symbol in gene_symbol_already_seen: + # duplicate check + messagefn( + f"Warning: a duplicate of gene {gene_symbol} was found in geneset {geneset_name}, " + "and will be ignored." + ) + continue + + if gene_symbol not in var_names: + messagefn( + f"Warning: {gene_symbol}, used in geneset {geneset_name}, " + "was not found in the dataset and will be ignored." + ) + continue + + gene_symbol_already_seen.add(gene_symbol) + new_genes.append(gene) + + geneset["genes"] = new_genes + + return args + def data_frame_to_fbs_matrix(self, filter, axis): """ Retrieves data 'X' and returns in a flatbuffer Matrix. @@ -333,7 +423,7 @@ class DataAdaptor(metaclass=ABCMeta): @staticmethod def normalize_embedding(embedding): """Normalize embedding layout to meet client assumptions. - Embedding is an ndarray, shape (n_obs, n)., where n is normally 2 + Embedding is an ndarray, shape (n_obs, n)., where n is normally 2 """ # scale isotropically diff --git a/local_server/default_config.py b/local_server/default_config.py index e722a8d5..ac457746 100644 --- a/local_server/default_config.py +++ b/local_server/default_config.py @@ -63,10 +63,13 @@ dataset: type: local_file_csv local_file_csv: directory: null - file: null + file: null # annotations file name + genesets_file: null # gene sets file name ontology: enable: false obo_location: null + genesets: + readonly: false # genesets CRUD enabled/disabled embeddings: names : [] diff --git a/local_server/test/__init__.py b/local_server/test/__init__.py index c18cafd0..7dbd481e 100644 --- a/local_server/test/__init__.py +++ b/local_server/test/__init__.py @@ -46,7 +46,11 @@ def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False): config.complete_config() data = MatrixDataLoader(data_locator.abspath()).open(config) - annotations = AnnotationsLocalFile(None, annotations_file) + anno_config = { + "user-annotations": True, + "genesets-save": False, + } + annotations = AnnotationsLocalFile(anno_config, None, annotations_file, None) return data, tmp_dir, annotations diff --git a/local_server/test/fixtures/dataset_config_outline.py b/local_server/test/fixtures/dataset_config_outline.py index c41a0980..95564861 100644 --- a/local_server/test/fixtures/dataset_config_outline.py +++ b/local_server/test/fixtures/dataset_config_outline.py @@ -16,9 +16,12 @@ dataset: local_file_csv: directory: {local_file_csv_directory} file: {local_file_csv_file} + genesets_file: {local_file_csv_genesets_file} ontology: enable: {ontology_enabled} obo_location: {obo_location} + genesets: + readonly: {genesets_readonly} embeddings: names: {embedding_names} diff --git a/local_server/test/fixtures/pbmc3k-genesets.csv b/local_server/test/fixtures/pbmc3k-genesets.csv new file mode 100644 index 00000000..3b7e92c9 --- /dev/null +++ b/local_server/test/fixtures/pbmc3k-genesets.csv @@ -0,0 +1,12 @@ +# Test fixture +geneset_name, geneset_description, gene_symbol, gene_description +first geneset name,,F5, a gene_description +first geneset name,a description, NO_SUCH_GENE, non-existent gene +first geneset name,a description, F5, duplicate gene +first geneset name, a description, SUMO3, +first geneset name,, SRM, +second geneset,,RER1 +second geneset,,SIK1 +third geneset,,NO_SUCH_GENE +fourth_geneset,fourth description,,gene intentionally missing +fifth_dataset,,, diff --git a/local_server/test/unit/auth/test_auth.py b/local_server/test/unit/auth/test_auth.py index dc44e30f..5e806019 100644 --- a/local_server/test/unit/auth/test_auth.py +++ b/local_server/test/unit/auth/test_auth.py @@ -14,7 +14,7 @@ class AuthTest(unittest.TestCase): app_config = AppConfig() app_config.update_server_config(app__flask_secret_key="secret") app_config.update_server_config(authentication__type=None, single_dataset__datapath=self.dataset_datapath) - app_config.update_dataset_config(user_annotations__enable=False) + app_config.update_dataset_config(user_annotations__enable=False, user_annotations__genesets__readonly=True) app_config.complete_config() diff --git a/local_server/test/unit/common/config/__init__.py b/local_server/test/unit/common/config/__init__.py index e2f57bab..3a2f6c53 100644 --- a/local_server/test/unit/common/config/__init__.py +++ b/local_server/test/unit/common/config/__init__.py @@ -92,8 +92,10 @@ class ConfigTests(unittest.TestCase): hosted_file_directory="null", local_file_csv_directory="null", local_file_csv_file="null", + local_file_csv_genesets_file="null", ontology_enabled="false", obo_location="null", + genesets_readonly="false", embedding_names=[], enable_reembedding="false", enable_difexp="true", @@ -142,8 +144,10 @@ class ConfigTests(unittest.TestCase): hosted_file_directory=hosted_file_directory, local_file_csv_directory=local_file_csv_directory, local_file_csv_file=local_file_csv_file, + local_file_csv_genesets_file=local_file_csv_genesets_file, ontology_enabled=ontology_enabled, obo_location=obo_location, + genesets_readonly=genesets_readonly, embedding_names=embedding_names, enable_reembedding=enable_reembedding, enable_difexp=enable_difexp, @@ -178,8 +182,10 @@ class ConfigTests(unittest.TestCase): hosted_file_directory="null", local_file_csv_directory="null", local_file_csv_file="null", + local_file_csv_genesets_file="null", ontology_enabled="false", obo_location="null", + genesets_readonly="false", embedding_names=[], enable_reembedding="false", enable_difexp="true", diff --git a/local_server/test/unit/common/config/test_dataset_config.py b/local_server/test/unit/common/config/test_dataset_config.py index a5f47cba..4a66a80a 100644 --- a/local_server/test/unit/common/config/test_dataset_config.py +++ b/local_server/test/unit/common/config/test_dataset_config.py @@ -47,7 +47,7 @@ class TestDatasetConfig(ConfigTests): mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute() self.dataset_config.complete_config(self.context) self.assertIsNotNone(self.config.server_config.data_adaptor) - self.assertEqual(mock_check_attrs.call_count, 17) + self.assertEqual(mock_check_attrs.call_count, 19) def test_app_sets_script_vars(self): config = self.get_config(scripts=["path/to/script"]) @@ -102,7 +102,7 @@ class TestDatasetConfig(ConfigTests): enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv" ) config.server_config.complete_config(self.context) - config.dataset_config.handle_local_file_csv_annotations() + config.dataset_config.handle_local_file_csv_annotations(self.context) self.assertIsInstance(config.dataset_config.user_annotations, AnnotationsLocalFile) cwd = os.getcwd() self.assertEqual(config.dataset_config.user_annotations._get_output_dir(), cwd) diff --git a/local_server/test/unit/common/test_api.py b/local_server/test/unit/common/test_api.py index a1db48d7..f587c60f 100644 --- a/local_server/test/unit/common/test_api.py +++ b/local_server/test/unit/common/test_api.py @@ -3,6 +3,8 @@ import time import unittest import zlib from http import HTTPStatus +import tempfile +from os import path import pandas as pd import requests @@ -13,6 +15,7 @@ from local_server.test import ( data_with_tmp_annotations, make_fbs, PROJECT_ROOT, + FIXTURES_ROOT, start_test_server, stop_test_server, ) @@ -26,6 +29,7 @@ BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}} class EndPoints(object): ANNOTATIONS_ENABLED = True + GENESETS_READONLY = False def test_initialize(self): endpoint = "schema" @@ -49,6 +53,7 @@ class EndPoints(object): result_data = result.json() self.assertIn("library_versions", result_data["config"]) self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k") + self.assertIsNotNone(result_data["config"]["parameters"]) def test_get_layout_fbs(self): endpoint = "layout/obs" @@ -286,6 +291,26 @@ class EndPoints(object): result = self.session.get(url) self.assertEqual(result.status_code, HTTPStatus.OK) + def test_genesets_config(self): + result = self.session.get(f"{self.URL_BASE}config") + config_data = result.json() + params = config_data["config"]["parameters"] + annotations_genesets = params["annotations_genesets"] + annotations_genesets_readonly = params["annotations_genesets_readonly"] + annotations_genesets_summary_methods = params["annotations_genesets_summary_methods"] + self.assertTrue(annotations_genesets) + self.assertEqual(annotations_genesets_readonly, self.GENESETS_READONLY) + self.assertEqual(annotations_genesets_summary_methods, ["mean"]) + + def test_get_genesets(self): + endpoint = "genesets" + url = f"{self.URL_BASE}{endpoint}" + result = self.session.get(url, headers={"Accept": "application/json"}) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/json") + result_data = result.json() + self.assertIsNotNone(result_data["genesets"]) + def _setupClass(child_class, command_line): child_class.ps, child_class.server = start_test_server(command_line) child_class.URL_BASE = f"{child_class.server}/api/v0.2/" @@ -304,7 +329,8 @@ class EndPointsAnnotations(EndPoints): def test_get_user_annotations_existing_obs_keys_fbs(self): self._test_get_user_annotations_obs_keys_fbs( - "cluster-test", {"unassigned", "one", "two", "three", "four", "five", "six", "seven"}, + "cluster-test", + {"unassigned", "one", "two", "three", "four", "five", "six", "seven"}, ) def test_put_user_annotations_obs_fbs(self): @@ -353,6 +379,7 @@ class EndPointsAnndata(unittest.TestCase, EndPoints): """Test Case for endpoints""" ANNOTATIONS_ENABLED = False + GENESETS_READONLY = True @classmethod def setUpClass(cls): @@ -361,6 +388,7 @@ class EndPointsAnndata(unittest.TestCase, EndPoints): [ f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", "--disable-annotations", + "--disable-genesets-save", "--experimental-enable-reembedding", ], ) @@ -408,15 +436,249 @@ class EndPointsAnndataAnnotations(unittest.TestCase, EndPointsAnnotations): """Test Case for endpoints""" ANNOTATIONS_ENABLED = True + GENESETS_READONLY = False @classmethod def setUpClass(cls): cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations( MatrixDataType.H5AD, annotations_fixture=True ) - cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location()]) + cls._setupClass(cls, ["--annotations-file", cls.annotations.label_output_file, cls.data.get_location()]) @classmethod def tearDownClass(cls): shutil.rmtree(cls.tmp_dir) stop_test_server(cls.ps) + + +class EndPointsAnnDataGenesets(unittest.TestCase, EndPoints): + ANNOTATIONS_ENABLED = False + GENESETS_READONLY = False + + @classmethod + def setUpClass(cls): + cls.tmp_dir = tempfile.mkdtemp() + genesets_file = path.join(cls.tmp_dir, "test_genesets.csv") + shutil.copyfile(f"{FIXTURES_ROOT}/pbmc3k-genesets.csv", genesets_file) + cls._setupClass( + cls, + [ + f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", + "--disable-annotations", + "--genesets-file", + genesets_file, + ], + ) + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.tmp_dir) + stop_test_server(cls.ps) + + def test_get_genesets_json(self): + endpoint = "genesets" + url = f"{self.URL_BASE}{endpoint}" + result = self.session.get(url, headers={"Accept": "application/json"}) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/json") + result_data = result.json() + self.assertIsNotNone(result_data["genesets"]) + self.assertIsNotNone(result_data["tid"]) + + self.assertEqual( + result_data, + { + "genesets": [ + { + "genes": [ + {"gene_description": "a gene_description", "gene_symbol": "F5"}, + {"gene_description": "", "gene_symbol": "SUMO3"}, + {"gene_description": "", "gene_symbol": "SRM"}, + ], + "geneset_description": "a description", + "geneset_name": "first geneset name", + }, + { + "genes": [ + {"gene_description": "", "gene_symbol": "RER1"}, + {"gene_description": "", "gene_symbol": "SIK1"}, + ], + "geneset_description": "", + "geneset_name": "second geneset", + }, + {"genes": [], "geneset_description": "", "geneset_name": "third geneset"}, + {"genes": [], "geneset_description": "fourth description", "geneset_name": "fourth_geneset"}, + {"genes": [], "geneset_description": "", "geneset_name": "fifth_dataset"}, + ], + "tid": 0, + }, + ) + + def test_get_genesets_csv(self): + endpoint = "genesets" + url = f"{self.URL_BASE}{endpoint}" + result = self.session.get(url, headers={"Accept": "text/csv"}) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "text/csv") + self.assertEqual( + result.text, + """geneset_name,geneset_description,gene_symbol,gene_description\r +first geneset name,a description,F5,a gene_description\r +first geneset name,a description,SUMO3,\r +first geneset name,a description,SRM,\r +second geneset,,RER1,\r +second geneset,,SIK1,\r +third geneset,,,\r +fourth_geneset,fourth description,,\r +fifth_dataset,,,\r +""", + ) + + def test_put_genesets(self): + endpoint = "genesets" + url = f"{self.URL_BASE}{endpoint}" + + # assume we start with TID 0 + result = self.session.get(url, headers={"Accept": "application/json"}) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.json()["tid"], 0) + + test1 = {"tid": 3, "genesets": []} + result = self.session.put(url, json=test1) + self.assertEqual(result.status_code, HTTPStatus.OK) + result = self.session.get(url, headers={"Accept": "application/json"}) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.json(), test1) + + # stale TID + result = self.session.put(url, json=test1) + self.assertEqual(result.status_code, HTTPStatus.NOT_FOUND) + + test2 = {"tid": 4, "genesets": [{"geneset_name": "foobar", "genes": []}]} + test2_response = {"tid": 4, "genesets": [{"geneset_name": "foobar", "geneset_description": "", "genes": []}]} + result = self.session.put(url, json=test2) + self.assertEqual(result.status_code, HTTPStatus.OK) + result = self.session.get(url, headers={"Accept": "application/json"}) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.json(), test2_response) + + test3 = { + "tid": 5, + "genesets": [ + { + "geneset_name": "foobar", + "geneset_description": "", + "genes": [ + { + "gene_symbol": "F5", + "gene_description": "", + } + ], + } + ], + } + result = self.session.put(url, json=test3) + self.assertEqual(result.status_code, HTTPStatus.OK) + result = self.session.get(url, headers={"Accept": "application/json"}) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.json(), test3) + + def test_put_genesets_malformed(self): + """ test malformed submissions that we expect the backend to catch/tolerate """ + endpoint = "genesets" + url = f"{self.URL_BASE}{endpoint}" + + result = self.session.get(url, headers={"Accept": "application/json"}) + self.assertEqual(result.status_code, HTTPStatus.OK) + original_data = result.json() + tid = original_data["tid"] + + def test_case(test, expected_code, original_data): + """ check for expected error AND that no change was made to the original state """ + result = self.session.put(url, json=test) + self.assertEqual(result.status_code, expected_code) + result = self.session.get(url, headers={"Accept": "application/json"}) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.json(), original_data) + + # missing or malformed genesets + test_case( + {"tid": tid + 1}, + HTTPStatus.BAD_REQUEST, + original_data, + ) + test_case( + {"tid": tid + 1, "genesets": 99}, + HTTPStatus.BAD_REQUEST, + original_data, + ) + + # illegal geneset_name + test_case( + {"tid": tid + 1, "genesets": [{"geneset_name": """, "genes": []}]}, + HTTPStatus.BAD_REQUEST, + original_data, + ) + + # duplicate geneset_name + test_case( + { + "tid": tid + 1, + "genesets": [ + {"geneset_name": "foo", "genes": []}, + {"geneset_name": "foo", "genes": []}, + ], + }, + HTTPStatus.BAD_REQUEST, + original_data, + ) + + # missing geneset_name + test_case( + {"tid": tid + 1, "genesets": [{"genes": []}]}, + HTTPStatus.BAD_REQUEST, + original_data, + ) + + # non-numeric TID + test_case( + {"tid": [], "genesets": [{"geneset_name": "foo", "genes": []}]}, + HTTPStatus.BAD_REQUEST, + original_data, + ) + test_case( + {"tid": None, "genesets": [{"geneset_name": "foo", "genes": []}]}, + HTTPStatus.BAD_REQUEST, + original_data, + ) + test_case( + {"tid": "not a number", "genesets": [{"geneset_name": "foo", "genes": []}]}, + HTTPStatus.BAD_REQUEST, + original_data, + ) + + # duplicate gene_symbol + test_case( + { + "tid": "not a number", + "genesets": [{"geneset_name": "foo", "genes": [{"gene_symbol": "SIK1"}, {"gene_symbol": "SIK1"}]}], + }, + HTTPStatus.BAD_REQUEST, + original_data, + ) + + # gene_symbol is not a string + test_case( + { + "tid": "not a number", + "genesets": [{"geneset_name": "foo", "genes": [{"gene_symbol": 99}]}], + }, + HTTPStatus.BAD_REQUEST, + original_data, + ) + + """ + TODO once we have some code to support it: + 1. GET genesets_summary + 2. genesets_summary obeys tid + """ diff --git a/local_server/test/unit/common/test_writable_annotation.py b/local_server/test/unit/common/test_writable_annotation.py index 74170b8c..2d26e630 100644 --- a/local_server/test/unit/common/test_writable_annotation.py +++ b/local_server/test/unit/common/test_writable_annotation.py @@ -45,8 +45,8 @@ class WritableAnnotationTest(unittest.TestCase): ) res = self.annotation_put_fbs(fbs) self.assertEqual(res, json.dumps({"status": "OK"})) - self.assertTrue(path.exists(self.annotations.output_file)) - df = pd.read_csv(self.annotations.output_file, index_col=0, header=0, comment="#") + self.assertTrue(path.exists(self.annotations.label_output_file)) + df = pd.read_csv(self.annotations.label_output_file, index_col=0, header=0, comment="#") self.assertEqual(df.shape, (n_rows, 2)) self.assertEqual(set(df.columns), {"cat_A", "cat_B"}) self.assertTrue(self.data.original_obs_index.equals(df.index)) @@ -62,14 +62,14 @@ class WritableAnnotationTest(unittest.TestCase): ) res = self.annotation_put_fbs(fbs) self.assertEqual(res, json.dumps({"status": "OK"})) - self.assertTrue(path.exists(self.annotations.output_file)) - df = pd.read_csv(self.annotations.output_file, index_col=0, header=0, comment="#") + self.assertTrue(path.exists(self.annotations.label_output_file)) + df = pd.read_csv(self.annotations.label_output_file, index_col=0, header=0, comment="#") self.assertEqual(set(df.columns), {"cat_A", "cat_C"}) self.assertTrue(np.all(df["cat_A"] == ["label_A1"] * n_rows)) self.assertTrue(np.all(df["cat_C"] == ["label_C"] * n_rows)) # rotation - name, ext = path.splitext(self.annotations.output_file) + name, ext = path.splitext(self.annotations.label_output_file) backup_dir = f"{name}-backups" self.assertTrue(path.isdir(backup_dir)) found_files = listdir(backup_dir) @@ -88,7 +88,7 @@ class WritableAnnotationTest(unittest.TestCase): res = self.annotation_put_fbs(fbs) self.assertEqual(res, json.dumps({"status": "OK"})) - name, ext = path.splitext(self.annotations.output_file) + name, ext = path.splitext(self.annotations.label_output_file) backup_dir = f"{name}-backups" self.assertTrue(path.isdir(backup_dir)) found_files = listdir(backup_dir)