diff --git a/backend/common/genesets.py b/backend/common/genesets.py new file mode 100644 index 00000000..cd22b792 --- /dev/null +++ b/backend/common/genesets.py @@ -0,0 +1,239 @@ +""" +Utility code for gene sets handling +""" + +import re +import csv +import hashlib + +from .errors import AnnotationsError + + +GENESETS_TIDYCSV_HEADER = [ + "gene_set_name", + "gene_set_description", + "gene_symbol", + "gene_description", +] + + +def read_gene_sets_tidycsv(gs_locator, context=None): + """ + Read & parse the Tidy CSV format, applying validation checks for mandatory + values, and de-duping rules. + + Format is a four-column CSV, with a mandatory header row, and optional "#" prefixed + comments. Format: + + gene_set_name, gene_set_description, gene_symbol, gene_description + + gene_set_name must be non-null; others are optional. + + Returns: a dictionary of the shape (values in angle-brackets vary): + + { + : { + "geneset_name": , + "geneset_description": , + "genes": [ + { + "gene_symbol": , + "gene_description": + }, + ... + ] + }, + ... + } + """ + + class myDialect(csv.excel): + skipinitialspace = False + + def just(n, seq): + it = iter(seq) + for _ in range(n - 1): + yield next(it, "") + yield tuple(it) + + messagefn = context["messagefn"] if context else (lambda x: None) + + gene_sets = {} + with gs_locator.local_handle() as fname: + with open(fname, newline="") as f: + reader = csv.reader(f, dialect=myDialect()) + haveReadHeader = False + lineno = 0 + for row in reader: + lineno += 1 + # ignore empty rows + if len(row) == 0: + continue + # if row starts with '#' it is a comment + if row[0].startswith("#"): + continue + # if this is the first non-comment row, assume it is a header and validate + # column names. OK if the user has extra columns after our initial set. + if not haveReadHeader: + if row[0:len(GENESETS_TIDYCSV_HEADER)] != GENESETS_TIDYCSV_HEADER: + raise AnnotationsError("Gene set CSV file missing the required column header.") + haveReadHeader = True + continue + + geneset_name, geneset_description, gene_symbol, gene_description, _ = just(5, row) + if not geneset_name: + raise AnnotationsError(f"Gene set CSV missing required gene set name on line {lineno}") + if (not gene_symbol) and gene_description: + messagefn(f"Warning: Missing gene name in gene set name {geneset_name} on line {lineno}.") + + if geneset_name in gene_sets: + gs = gene_sets[geneset_name] + else: + gs = gene_sets[geneset_name] = { + "geneset_name": geneset_name, + "geneset_description": geneset_description, + "genes": [], + } + # Use first geneset_description with a value + if not gs["geneset_description"] and geneset_description: + gs["geneset_description"] = geneset_description + # add the gene if the gene_symbol is defined + if gene_symbol: + gs["genes"].append( + { + "gene_symbol": gene_symbol, + "gene_description": gene_description, + } + ) + + return gene_sets + + +def write_gene_sets_tidycsv(f, genesets): + """ + Convert the internal gene sets format (returned by read_gene_set_tidycsv) into + the simple Tidy CSV. + """ + writer = csv.writer(f, dialect="excel") + writer.writerow(GENESETS_TIDYCSV_HEADER) + for geneset in genesets: + # genes may be empty, in which case we skip the gene set entirely + genes = geneset["genes"] + if not genes: + writer.writerow([geneset["geneset_name"], geneset.get("geneset_description", ""), "", ""]) + else: + writer.writerows( + [ + [ + geneset["geneset_name"], + geneset.get("geneset_description", ""), + gene["gene_symbol"], + gene.get("gene_description", ""), + ] + for gene in genes + ] + ) + + +def summarizeQueryHash(raw_query): + """ generate a cache key (hash) from the raw query string """ + return hashlib.sha1(raw_query).hexdigest() + + +def validate_gene_sets(genesets, var_names, context=None): + """ + Check validity of gene sets, return if correct, else raise error. + May also modify the gene set for conditions that should be resolved, + but which do not warrant a hard error. + + Argument gene sets may be either the REST OTA format (list of dicts) or the internal + format (dict of dicts, keyed by the gene set name). + + Will return a modified gene sets (eg, remove warnings) of the same type as the + provided argument. Ie, dict->dict, list->list + + Rules: + + 0. All gene set names must be unique. [error] + 1. Gene set names must conform to the following: [error] + * Names must be comprised of 1 or more ASCII characters 32-126 + * No leading or trailing spaces (ASCII 32) + * No multi-space (ASCII 32) runs + 2. Gene symbols must be part of the current var_index. [warning] + If gene symbol is not in the var_index, generate a warning and remove the symbol + from the gene sets. + 3. Gene symbols must not be duplicated in a gene set. [warning] + Duplications will be silently de-duped. + + Items marked [error] will generate a hard error, causing the validation to fail. + + Items marked [warning] will generate a warning, and will be resolved without failing + the validation (typically by removing the offending item from the gene sets). + """ + + messagefn = context["messagefn"] if context else (lambda x: None) + + # accept genesets args as either the internal (dict) or REST (list) format, + # as they are identical except for the dict being keyed by geneset_name. + if not isinstance(genesets, dict) and not isinstance(genesets, list): + raise ValueError("Gene sets must be either dict or list.") + genesets_iterable = genesets if isinstance(genesets, list) else genesets.values() + + # 0. check for uniqueness of geneset names + geneset_names = [gs["geneset_name"] for gs in genesets_iterable] + if len(set(geneset_names)) != len(geneset_names): + raise KeyError("All gene set names must be unique.") + + # 1. check gene set character set and format + illegal_name = re.compile(r"^\s| |[\u0000-\u001F\u007F-\uFFFF]|\s$") + for name in geneset_names: + if type(name) != str or len(name) == 0: + raise KeyError("Gene set names must be non-null string.") + if illegal_name.search(name): + messagefn( + "Error: " + f"Gene set name {name} " + "is not valid. Leading, trailing, and multiple spaces within a name are not allowed." + ) + raise KeyError( + "Gene set name is not valid. Leading, trailing, and multiple spaces within a name are not allowed." + ) + + # 2. & 3. check for duplicate gene symbols, and those not present in the dataset. They will + # generate a warning and be removed. + for geneset in genesets_iterable: + if not isinstance(geneset, dict): + raise ValueError("Each gene set must be a dict.") + geneset_name = geneset["geneset_name"] + genes = geneset["genes"] + if not isinstance(genes, list): + raise ValueError("Gene set genes field must be a list") + geneset.setdefault("geneset_description", "") + gene_symbol_already_seen = set() + new_genes = [] + for gene in genes: + gene_symbol = gene["gene_symbol"] + if not isinstance(gene_symbol, str) or len(gene_symbol) == 0: + raise ValueError("Gene symbol must be non-null string.") + if gene_symbol in gene_symbol_already_seen: + # duplicate check + messagefn( + f"Warning: a duplicate of gene {gene_symbol} was found in gene set {geneset_name}, " + "and will be ignored." + ) + continue + + if gene_symbol not in var_names: + messagefn( + f"Warning: {gene_symbol}, used in gene set {geneset_name}, " + "was not found in the dataset and will be ignored." + ) + continue + + gene_symbol_already_seen.add(gene_symbol) + gene.setdefault("gene_description", "") + new_genes.append(gene) + + geneset["genes"] = new_genes + + return genesets diff --git a/backend/czi_hosted/app/app.py b/backend/czi_hosted/app/app.py index 523b377e..d95af29a 100644 --- a/backend/czi_hosted/app/app.py +++ b/backend/czi_hosted/app/app.py @@ -318,6 +318,25 @@ class LayoutObsAPI(DatasetResource): return common_rest.layout_obs_put(request, data_adaptor) +class GenesetsAPI(DatasetResource): + @cache_control(public=True, max_age=ONE_WEEK) + @rest_get_data_adaptor + def get(self, data_adaptor): + return common_rest.genesets_get(request, data_adaptor) + + +class SummarizeVarAPI(DatasetResource): + @rest_get_data_adaptor + @cache_control(public=True, max_age=ONE_WEEK) + def get(self, data_adaptor): + return common_rest.summarize_var_get(request, data_adaptor) + + @rest_get_data_adaptor + @cache_control(no_store=True) + def post(self, data_adaptor): + return common_rest.summarize_var_post(request, data_adaptor) + + def get_api_base_resources(bp_base): """Add resources that are accessed from the api_base_url""" api = Api(bp_base) @@ -343,6 +362,8 @@ def get_api_dataroot_resources(bp_dataroot, url_dataroot=None): add_resource(AnnotationsObsAPI, "/annotations/obs") add_resource(AnnotationsVarAPI, "/annotations/var") add_resource(DataVarAPI, "/data/var") + add_resource(GenesetsAPI, "/genesets") + add_resource(SummarizeVarAPI, "/summarize/var") # Display routes add_resource(ColorsAPI, "/colors") # Computation routes diff --git a/backend/czi_hosted/common/annotations/annotations.py b/backend/czi_hosted/common/annotations/annotations.py index 75cce9b3..530d9964 100644 --- a/backend/czi_hosted/common/annotations/annotations.py +++ b/backend/czi_hosted/common/annotations/annotations.py @@ -1,21 +1,33 @@ -from abc import ABCMeta, abstractmethod - import fastobo import fsspec +import os -from backend.common.errors import OntologyLoadFailure +from flask import current_app, has_request_context + +from backend.common.errors import OntologyLoadFailure, DisabledFeatureError from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array +from backend.common.genesets import write_gene_sets_tidycsv, read_gene_sets_tidycsv, validate_gene_sets +from backend.common.utils.data_locator import DataLocator +from backend.common.utils.utils import path_join -class Annotations(metaclass=ABCMeta): - """ baseclass for annotations, including ontologies""" +class Annotations: + """ baseclass for annotations, including ontologies and genesets """ """ our default ontology is the PURL for the Cell Ontology. See http://www.obofoundry.org/ontology/cl.html """ DefaultOnotology = "http://purl.obolibrary.org/obo/cl.obo" - def __init__(self): + def __init__(self, config={}): self.ontology_data = None + self.config = config + + def user_annotations_enabled(self): + return self.config.get("user-annotations", False) + + def check_user_annotations_enabled(self): + if not self.user_annotations_enabled(): + raise DisabledFeatureError("User annotations are disabled.") def load_ontology(self, path): """Load and parse ontologies - currently support OBO files only.""" @@ -49,22 +61,78 @@ class Annotations(metaclass=ABCMeta): return schema - @abstractmethod def set_collection(self, name): """set or create a new annotation collection""" - pass + raise NotImplementedError - @abstractmethod def read_labels(self, data_adaptor): """Return the labels as a pandas.DataFrame""" - pass + raise NotImplementedError - @abstractmethod def write_labels(self, df, data_adaptor): """Write the labels (df) to a persistent storage such that it can later be read""" - pass + raise NotImplementedError - @abstractmethod def update_parameters(self, parameters, data_adaptor): """Update configuration parameters that describe information about the annotations feature""" - pass + params = {} + params["annotations_genesets_readonly"] = True + params["annotations_genesets_name_is_read_only"] = True + parameters.update(params) + + @staticmethod + def gene_sets_to_csv(genesets): + """ + Convert the internal genesets format (returned by read_gene_set) into + the simple Tidy CSV. + """ + from io import StringIO + + if isinstance(genesets, dict): + genesets = genesets.values() + + with StringIO() as sio: + write_gene_sets_tidycsv(sio, genesets) + return sio.getvalue() + + @staticmethod + def gene_sets_to_response(genesets): + """ + Convert the internal genesets format (returned by read_gene_set) into + the dict expected by the JSON REST API + """ + return list(genesets.values()) + + def read_gene_sets(self, data_adaptor, context=None): + if has_request_context(): + if not current_app.auth.is_user_authenticated(): + return ({}, 0) + + gene_sets_uri_or_path = dataset_uri_to_geneset_uri(data_adaptor.data_locator.uri_or_path) + + server_config = data_adaptor.server_config + region_name = None if server_config is None else server_config.data_locator__s3__region_name + gene_sets_locator = DataLocator(gene_sets_uri_or_path, region_name=region_name) + if not gene_sets_locator.exists(): + return ({}, 0) + + gene_sets = read_gene_sets_tidycsv(gene_sets_locator, context) + schema = data_adaptor.get_schema() + var_index = schema["annotations"]["var"].get("index", "index") + var_names = set(data_adaptor.query_var_array(var_index)) + + gene_sets = validate_gene_sets(gene_sets, var_names) + return (gene_sets, 0) + + +def dataset_uri_to_geneset_uri(data_uri_or_path): + """ given a dataset URI, return the associated gene set URI """ + data_basename = os.path.basename(data_uri_or_path) + base, ext = os.path.splitext(data_basename) + if ext is not None: # strip extension, if any + data_basename = base + + genesets_basename = f"{data_basename}-genesets.csv" + gene_sets_uri_or_path = path_join(data_uri_or_path, "..", genesets_basename) + + return gene_sets_uri_or_path diff --git a/backend/czi_hosted/common/annotations/hosted_tiledb.py b/backend/czi_hosted/common/annotations/hosted_tiledb.py index b215405a..ba957720 100644 --- a/backend/czi_hosted/common/annotations/hosted_tiledb.py +++ b/backend/czi_hosted/common/annotations/hosted_tiledb.py @@ -17,8 +17,8 @@ from backend.czi_hosted.db.cellxgene_orm import Annotation class AnnotationsHostedTileDB(Annotations): CXG_ANNO_COLLECTION = "cxg_anno_collection" - def __init__(self, directory_path, db): - super().__init__() + def __init__(self, config, directory_path, db): + super().__init__(config) self.db = db if directory_path[-1] == "/": self.directory_path = directory_path @@ -158,6 +158,8 @@ class AnnotationsHostedTileDB(Annotations): self.db.session.commit() def update_parameters(self, parameters, data_adaptor): + super().update_parameters(parameters, data_adaptor) + params = {} params["annotations"] = True params["user_annotation_collection_name_enabled"] = False diff --git a/backend/czi_hosted/common/annotations/local_file_csv.py b/backend/czi_hosted/common/annotations/local_file_csv.py index c5124ab8..468d8cd0 100644 --- a/backend/czi_hosted/common/annotations/local_file_csv.py +++ b/backend/czi_hosted/common/annotations/local_file_csv.py @@ -16,8 +16,8 @@ from backend.common.errors import AnnotationsError class AnnotationsLocalFile(Annotations): CXG_ANNO_COLLECTION = "cxg_anno_collection" - def __init__(self, output_dir, output_file): - super().__init__() + def __init__(self, config, output_dir, output_file): + super().__init__(config) self.output_dir = output_dir self.output_file = output_file # lock used to protect label file write ops @@ -169,6 +169,8 @@ class AnnotationsLocalFile(Annotations): os.remove(os.path.join(backup_dir, bu)) def update_parameters(self, parameters, data_adaptor): + super().update_parameters(parameters, data_adaptor) + params = {} params["annotations"] = True params["user_annotation_collection_name_enabled"] = True @@ -190,7 +192,7 @@ class AnnotationsLocalFile(Annotations): collection = self.get_collection() if current_app.auth.is_user_authenticated(): params["annotations-user-data-idhash"] = self._get_userdata_idhash(data_adaptor) - params["annotations-data-collection-is-read-only"] = False + params["annotations-data-collection-is-read-only"] = not self.user_annotations_enabled() params["annotations-data-collection-name"] = collection parameters.update(params) diff --git a/backend/czi_hosted/common/config/client_config.py b/backend/czi_hosted/common/config/client_config.py index 5214e488..dbb727c5 100644 --- a/backend/czi_hosted/common/config/client_config.py +++ b/backend/czi_hosted/common/config/client_config.py @@ -44,6 +44,9 @@ def get_client_config(app_config, data_adaptor): "annotations": False, "annotations_file": None, "annotations_dir": None, + "annotations_genesets": True, # feature flag + "annotations_genesets_readonly": True, + "annotations_genesets_summary_methods": ["mean"], "annotations_cell_ontology_enabled": False, "annotations_cell_ontology_obopath": None, "annotations_cell_ontology_terms": None, diff --git a/backend/czi_hosted/common/config/dataset_config.py b/backend/czi_hosted/common/config/dataset_config.py index 4f9f7eef..161341a5 100644 --- a/backend/czi_hosted/common/config/dataset_config.py +++ b/backend/czi_hosted/common/config/dataset_config.py @@ -1,6 +1,7 @@ import os from os.path import splitext, isdir +from backend.czi_hosted.common.annotations.annotations import Annotations from backend.czi_hosted.common.annotations.hosted_tiledb import AnnotationsHostedTileDB from backend.czi_hosted.common.annotations.local_file_csv import AnnotationsLocalFile from backend.czi_hosted.common.config.base_config import BaseConfig @@ -53,8 +54,10 @@ class DatasetConfig(BaseConfig): except KeyError as e: raise ConfigurationError(f"Unexpected config: {str(e)}") - # The annotation object is created during complete_config and stored here. - self.user_annotations = None + # Create the default annotation, which supports gene set reading without + # further configuration. Depending on configuration options, `complete_config` + # may create a more specialized annotation object and replace this default. + self.user_annotations = Annotations() def complete_config(self, context): self.handle_app() @@ -147,7 +150,11 @@ class DatasetConfig(BaseConfig): except OSError: raise ConfigurationError("Unable to create directory specified by --annotations-dir") - self.user_annotations = AnnotationsLocalFile(dirname, filename) + anno_config = { + "user-annotations": self.user_annotations__enable, + "genesets-save": False, + } + self.user_annotations = AnnotationsLocalFile(anno_config, dirname, filename) # if the user has specified a fixed label file, go ahead and validate it # so that we can remove errors early in the process. @@ -163,7 +170,12 @@ class DatasetConfig(BaseConfig): self.validate_correct_type_of_configuration_attribute( "user_annotations__hosted_tiledb_array__hosted_file_directory", str ) + anno_config = { + "user-annotations": self.user_annotations__enable, + "genesets-save": False, + } self.user_annotations = AnnotationsHostedTileDB( + anno_config, directory_path=self.user_annotations__hosted_tiledb_array__hosted_file_directory, db=DbUtils(self.user_annotations__hosted_tiledb_array__db_uri), ) diff --git a/backend/czi_hosted/common/rest.py b/backend/czi_hosted/common/rest.py index 4d95a885..f9980744 100644 --- a/backend/czi_hosted/common/rest.py +++ b/backend/czi_hosted/common/rest.py @@ -3,6 +3,7 @@ import logging import sys from http import HTTPStatus import zlib +import json from flask import make_response, jsonify, current_app, abort from werkzeug.urls import url_unquote @@ -17,9 +18,10 @@ from backend.common.errors import ( ExceedsLimitError, DatasetAccessError, ColorFormatException, + AnnotationsError, + UnsupportedSummaryMethod, ) - -import json +from backend.common.genesets import summarizeQueryHash from backend.common.fbs.matrix import decode_matrix_fbs @@ -106,7 +108,7 @@ def schema_get_helper(data_adaptor): # add label obs annotations as needed annotations = data_adaptor.dataset_config.user_annotations - if annotations is not None: + if annotations.user_annotations_enabled(): label_schema = annotations.get_schema(data_adaptor) schema["annotations"]["obs"]["columns"].extend(label_schema) @@ -140,7 +142,7 @@ def annotations_obs_get(request, data_adaptor): try: labels = None annotations = data_adaptor.dataset_config.user_annotations - if annotations: + if annotations.user_annotations_enabled(): labels = annotations.read_labels(data_adaptor) fbs = data_adaptor.annotation_to_fbs_matrix(Axis.OBS, fields, labels) return make_response(fbs, HTTPStatus.OK, {"Content-Type": "application/octet-stream"}) @@ -151,7 +153,7 @@ def annotations_obs_get(request, data_adaptor): def annotations_put_fbs_helper(data_adaptor, fbs): """helper function to write annotations from fbs""" annotations = data_adaptor.dataset_config.user_annotations - if annotations is None: + if not annotations.user_annotations_enabled(): raise DisabledFeatureError("Writable annotations are not enabled") new_label_df = decode_matrix_fbs(fbs) @@ -166,7 +168,7 @@ def inflate(data): def annotations_obs_put(request, data_adaptor): annotations = data_adaptor.dataset_config.user_annotations - if annotations is None: + if not annotations.user_annotations_enabled(): return abort(HTTPStatus.NOT_IMPLEMENTED) anno_collection = request.args.get("annotation-collection-name", default=None) @@ -197,7 +199,7 @@ def annotations_var_get(request, data_adaptor): try: labels = None annotations = data_adaptor.dataset_config.user_annotations - if annotations is not None: + if annotations.user_annotations_enabled(): labels = annotations.read_labels(data_adaptor) return make_response( data_adaptor.annotation_to_fbs_matrix(Axis.VAR, fields, labels), @@ -328,3 +330,70 @@ def layout_obs_put(request, data_adaptor): return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, str(e)) except (ValueError, DisabledFeatureError, FilterError) as e: return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True) + + +def genesets_get(request, data_adaptor): + preferred_mimetype = request.accept_mimetypes.best_match(["application/json", "text/csv"]) + if preferred_mimetype not in ("application/json", "text/csv"): + return abort(HTTPStatus.NOT_ACCEPTABLE) + + try: + annotations = data_adaptor.dataset_config.user_annotations + (genesets, tid) = annotations.read_gene_sets(data_adaptor) + + if preferred_mimetype == "text/csv": + return make_response( + annotations.gene_sets_to_csv(genesets), + HTTPStatus.OK, + { + "Content-Type": "text/csv", + "Content-Disposition": "attachment; filename=genesets.csv", + }, + ) + else: + return make_response( + jsonify({"genesets": annotations.gene_sets_to_response(genesets), "tid": tid}), HTTPStatus.OK + ) + except (ValueError, KeyError, AnnotationsError) as e: + return abort_and_log(HTTPStatus.BAD_REQUEST, str(e)) + + +def summarize_var_helper(request, data_adaptor, key, raw_query): + preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"]) + if preferred_mimetype != "application/octet-stream": + return abort(HTTPStatus.NOT_ACCEPTABLE) + + summary_method = request.values.get("method", default="mean") + query_hash = summarizeQueryHash(raw_query) + if key and query_hash != key: + return abort(HTTPStatus.BAD_REQUEST, description="query key did not match") + + args_filter_only = request.values.copy() + args_filter_only.poplist("method") + args_filter_only.poplist("key") + + try: + filter = _query_parameter_to_filter(args_filter_only) + return make_response( + data_adaptor.summarize_var(summary_method, filter, query_hash), + HTTPStatus.OK, + {"Content-Type": "application/octet-stream"}, + ) + except (ValueError) as e: + return abort(HTTPStatus.NOT_FOUND, description=str(e)) + except (UnsupportedSummaryMethod, FilterError) as e: + return abort(HTTPStatus.BAD_REQUEST, description=str(e)) + + +def summarize_var_get(request, data_adaptor): + return summarize_var_helper(request, data_adaptor, None, request.query_string) + + +def summarize_var_post(request, data_adaptor): + if not request.content_type or "application/x-www-form-urlencoded" not in request.content_type: + return abort(HTTPStatus.UNSUPPORTED_MEDIA_TYPE) + if request.content_length > 1_000_000: # just a sanity check to avoid memory exhaustion + return abort(HTTPStatus.BAD_REQUEST) + + key = request.args.get("key", default=None) + return summarize_var_helper(request, data_adaptor, key, request.get_data()) diff --git a/backend/czi_hosted/data_common/data_adaptor.py b/backend/czi_hosted/data_common/data_adaptor.py index eb1e9e5c..9b80289c 100644 --- a/backend/czi_hosted/data_common/data_adaptor.py +++ b/backend/czi_hosted/data_common/data_adaptor.py @@ -3,11 +3,12 @@ from os.path import basename, splitext import numpy as np import pandas as pd +from scipy import sparse from server_timing import Timing as ServerTiming from backend.czi_hosted.common.config.app_config import AppConfig from backend.common.constants import Axis -from backend.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError +from backend.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError, UnsupportedSummaryMethod from backend.common.utils.utils import jsonify_numpy from backend.common.fbs.matrix import encode_matrix_fbs @@ -338,7 +339,7 @@ class DataAdaptor(metaclass=ABCMeta): @staticmethod def normalize_embedding(embedding): """Normalize embedding layout to meet client assumptions. - Embedding is an ndarray, shape (n_obs, n)., where n is normally 2 + Embedding is an ndarray, shape (n_obs, n)., where n is normally 2 """ # scale isotropically @@ -394,3 +395,26 @@ class DataAdaptor(metaclass=ABCMeta): except RuntimeError: lastmod = None return lastmod + + def summarize_var(self, method, filter, query_hash): + if method != "mean": + raise UnsupportedSummaryMethod("Unknown gene set summary method.") + + obs_selector, var_selector = self._filter_to_mask(filter) + if obs_selector is not None: + raise FilterError("filtering on obs unsupported") + + # if no filter, just return zeros. We don't have a use case + # for summarizing the entire X without a filter, and it would + # potentially be quite compute / memory intensive. + if var_selector is None or np.count_nonzero(var_selector) == 0: + mean = np.zeros((self.get_shape()[0], 1), dtype=np.float32) + else: + X = self.get_X_array(obs_selector, var_selector) + if sparse.issparse(X): + mean = X.mean(axis=1) + else: + mean = X.mean(axis=1, keepdims=True) + + col_idx = pd.Index([query_hash]) + return encode_matrix_fbs(mean, col_idx=col_idx, row_idx=None) diff --git a/backend/server/common/annotations/annotations.py b/backend/server/common/annotations/annotations.py index 49ee07e3..1f75b9db 100644 --- a/backend/server/common/annotations/annotations.py +++ b/backend/server/common/annotations/annotations.py @@ -5,10 +5,11 @@ import fsspec from backend.common.errors import OntologyLoadFailure, DisabledFeatureError from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array +from backend.common.genesets import write_gene_sets_tidycsv class Annotations(metaclass=ABCMeta): - """ baseclass for annotations, including ontologies and genesets""" + """ baseclass for annotations, including ontologies and gene sets""" """ our default ontology is the PURL for the Cell Ontology. See http://www.obofoundry.org/ontology/cl.html """ @@ -30,7 +31,7 @@ class Annotations(metaclass=ABCMeta): def check_gene_sets_save_enabled(self): if not self.gene_sets_save_enabled(): - raise DisabledFeatureError("User genesets save is disabled.") + raise DisabledFeatureError("User gene sets save is disabled.") def load_ontology(self, path): """Load and parse ontologies - currently support OBO files only.""" @@ -81,12 +82,12 @@ class Annotations(metaclass=ABCMeta): @abstractmethod def read_gene_sets(self, data_adaptor): - """Return the genesets from persistent storage """ + """Return the gene sets from persistent storage """ pass @abstractmethod def write_gene_sets(self, gs, data_adaptor): - """Write the genesets (gs) to a persistent storage such that it can later be read""" + """Write the gene sets (gs) to a persistent storage such that it can later be read""" pass @abstractmethod @@ -94,51 +95,25 @@ class Annotations(metaclass=ABCMeta): """Update configuration parameters that describe information about the annotations feature""" pass - Genesets_Header = [ - "gene_set_name", - "gene_set_description", - "gene_symbol", - "gene_description", - ] - @staticmethod def gene_sets_to_csv(genesets): """ - Convert the internal genesets format (returned by read_gene_set) into + Convert the internal gene sets format (returned by read_gene_set) into the simple Tidy CSV. """ from io import StringIO - import csv if isinstance(genesets, dict): genesets = genesets.values() with StringIO() as sio: - writer = csv.writer(sio, dialect='excel') - writer.writerow(Annotations.Genesets_Header) - for geneset in genesets: - # genes may be empty, in which case we skip the geneset entirely - genes = geneset["genes"] - if not genes: - writer.writerow([geneset["geneset_name"], geneset.get("geneset_description", ""), "", ""]) - else: - writer.writerows( - [ - [ - geneset["geneset_name"], - geneset.get("geneset_description", ""), - gene["gene_symbol"], - gene.get("gene_description", ""), - ] - for gene in genes - ] - ) + write_gene_sets_tidycsv(sio, genesets) return sio.getvalue() @staticmethod def gene_sets_to_response(genesets): """ - Convert the internal genesets format (returned by read_gene_set) into + Convert the internal gene sets format (returned by read_gene_set) into the dict expected by the JSON REST API """ return list(genesets.values()) diff --git a/backend/server/common/annotations/local_file_csv.py b/backend/server/common/annotations/local_file_csv.py index e18da0f3..1f61a1e7 100644 --- a/backend/server/common/annotations/local_file_csv.py +++ b/backend/server/common/annotations/local_file_csv.py @@ -4,14 +4,15 @@ import re import threading from datetime import datetime from hashlib import blake2b -import csv import pandas as pd from flask import session, has_request_context, current_app from backend.server import __version__ as cellxgene_version from backend.server.common.annotations.annotations import Annotations +from backend.common.genesets import read_gene_sets_tidycsv from backend.common.errors import AnnotationsError, ObsoleteRequest +from backend.common.utils.data_locator import DataLocator class AnnotationsLocalFile(Annotations): @@ -124,8 +125,8 @@ class AnnotationsLocalFile(Annotations): if fname == self.last_geneset_fname: gene_sets = self.last_geneset else: - with open(fname, newline="") as f: - gene_sets = read_gene_set_tidycsv(f, context) + # read + gene_sets = read_gene_sets_tidycsv(DataLocator(fname), context) # validate gene_sets = data_adaptor.check_new_gene_sets(gene_sets, context) @@ -259,6 +260,7 @@ class AnnotationsLocalFile(Annotations): params = {} params["annotations"] = self.user_annotations_enabled() params["annotations_genesets_readonly"] = not self.gene_sets_save_enabled() + params["annotations_genesets_name_is_read_only"] = self.gene_sets_output_file is not None params["user_annotation_collection_name_enabled"] = True if self.ontology_data: @@ -276,99 +278,10 @@ class AnnotationsLocalFile(Annotations): elif session is not None: collection = self.get_collection() - params["annotations-data-collection-is-read-only"] = False + params["annotations-data-collection-is-read-only"] = not self.user_annotations_enabled() params["annotations-data-collection-name"] = collection if current_app.auth.is_user_authenticated(): params["annotations-user-data-idhash"] = self._get_userdata_idhash(data_adaptor) parameters.update(params) - - -def read_gene_set_tidycsv(f, context=None): - """ - Read & parse the Tidy CSV format, applying validation checks for mandatory - values, and de-duping rules. - - Format is a four-column CSV, with a mandatory header row, and optional "#" prefixed - comments. Format: - - gene_set_name, gene_set_description, gene_symbol, gene_description - - gene_set_name must be non-null; others are optional. - - Returns: a dictionary of the shape (values in angle-brackets vary): - - { - : { - "geneset_name": , - "geneset_description": , - "genes": [ - { - "gene_symbol": , - "gene_description": - }, - ... - ] - }, - ... - } - """ - - class myDialect(csv.excel): - skipinitialspace = True - - def just(n, seq): - it = iter(seq) - for _ in range(n - 1): - yield next(it, "") - yield tuple(it) - - messagefn = context["messagefn"] if context else (lambda x: None) - - reader = csv.reader(f, dialect=myDialect()) - gene_sets = {} - haveReadHeader = False - lineno = 0 - for row in reader: - lineno += 1 - # ignore empty rows - if len(row) == 0: - continue - # if row starts with '#' it is a comment - if row[0].startswith("#"): - continue - # if this is the first non-comment row, assume it is a header - if not haveReadHeader: - if row != Annotations.Genesets_Header: - raise AnnotationsError("Geneset CSV file missing the required column header.") - haveReadHeader = True - continue - - geneset_name, geneset_description, gene_symbol, gene_description, _ = just(5, row) - if not geneset_name: - raise AnnotationsError(f"Geneset CSV missing required geneset or gene name on line {lineno}") - if (not gene_symbol) and gene_description: - messagefn(f"Warning: Missing gene name in geneset name {geneset_name} on line {lineno}.") - - if geneset_name in gene_sets: - gs = gene_sets[geneset_name] - else: - gs = gene_sets[geneset_name] = { - "geneset_name": geneset_name, - "geneset_description": geneset_description, - "genes": [], - } - # Use first geneset_description with a value - if not gs["geneset_description"] and geneset_description: - gs["geneset_description"] = geneset_description - # add the gene if the gene_symbol is defined - if gene_symbol: - gs["genes"].append( - { - "gene_symbol": gene_symbol, - "gene_description": gene_description, - } - ) - - return gene_sets diff --git a/backend/server/common/config/dataset_config.py b/backend/server/common/config/dataset_config.py index b9594c66..9f805b75 100644 --- a/backend/server/common/config/dataset_config.py +++ b/backend/server/common/config/dataset_config.py @@ -138,7 +138,7 @@ class DatasetConfig(BaseConfig): if dirname is not None and (filename is not None or genesets_filename is not None): raise ConfigurationError( - "'user-generated-data-dir' may not be used with annotations-file' or 'genesets-file'." + "'user-generated-data-dir' may not be used with 'annotations-file' or 'gene-sets-file'." ) if filename is not None: diff --git a/backend/server/common/rest.py b/backend/server/common/rest.py index d7c2e5af..a5a7bb8d 100644 --- a/backend/server/common/rest.py +++ b/backend/server/common/rest.py @@ -3,7 +3,7 @@ import logging import sys from http import HTTPStatus import zlib -import hashlib +import json from flask import make_response, jsonify, current_app, abort from werkzeug.urls import url_unquote @@ -22,9 +22,7 @@ from backend.common.errors import ( ObsoleteRequest, UnsupportedSummaryMethod, ) - -import json - +from backend.common.genesets import summarizeQueryHash from backend.common.fbs.matrix import decode_matrix_fbs @@ -390,7 +388,7 @@ def summarize_var_helper(request, data_adaptor, key, raw_query): return abort(HTTPStatus.NOT_ACCEPTABLE) summary_method = request.values.get("method", default="mean") - query_hash = hashlib.sha1(raw_query).hexdigest() # cache helper + query_hash = summarizeQueryHash(raw_query) if key and query_hash != key: return abort(HTTPStatus.BAD_REQUEST, description="query key did not match") @@ -398,7 +396,7 @@ def summarize_var_helper(request, data_adaptor, key, raw_query): args_filter_only.poplist("method") args_filter_only.poplist("key") - try: + try: filter = _query_parameter_to_filter(args_filter_only) return make_response( data_adaptor.summarize_var(summary_method, filter, query_hash), diff --git a/backend/server/data_common/data_adaptor.py b/backend/server/data_common/data_adaptor.py index b1299234..310ead61 100644 --- a/backend/server/data_common/data_adaptor.py +++ b/backend/server/data_common/data_adaptor.py @@ -1,6 +1,5 @@ from abc import ABCMeta, abstractmethod from os.path import basename, splitext -import re import numpy as np import pandas as pd from scipy import sparse @@ -11,6 +10,7 @@ from backend.common.constants import Axis from backend.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError, UnsupportedSummaryMethod from backend.common.utils.utils import jsonify_numpy from backend.common.fbs.matrix import encode_matrix_fbs +from backend.common.genesets import validate_gene_sets class DataAdaptor(metaclass=ABCMeta): @@ -263,95 +263,8 @@ class DataAdaptor(metaclass=ABCMeta): return labels_df def check_new_gene_sets(self, genesets, context=None): - """ - Check validity of gene sets, return if correct, else raise error. - May also modify the gene set for conditions that should be resolved, - but which do not warrant a hard error. - - Argument genesets may be either the REST OTA format (list of dicts) or the internal - format (dict of dicts, keyed by the geneset name). - - Will return a modified genesets (eg, remove dups) of the same type as the - provided argument. Ie, dict->dict, list->list - - Rules: - 0. all geneset names must be unique. - 1. All geneset names must be comprised of legal characters, meaning: - * no leading or trailing white space - * no multi-space runs - * no tab, vertical tab, newline or return - Where "space" means ASCII 32. Generates hard error. - 2. Gene symbols must be part of the current var_index. If symbol not in var_index, - will generate a warning and the symbol removed. - 3. Duplicate gene symbols are silently de-duped. - """ - messagefn = context["messagefn"] if context else (lambda x: None) - - # accept genesets args as either the internal (dict) or REST (list) format, - # as they are identical except for the dict being keyed by geneset_name. - if not isinstance(genesets, dict) and not isinstance(genesets, list): - raise ValueError("Genesets must be either dict or list.") - genesets_iterable = genesets if isinstance(genesets, list) else genesets.values() - - # 0. check for uniqueness of geneset names - geneset_names = [gs["geneset_name"] for gs in genesets_iterable] - if len(set(geneset_names)) != len(geneset_names): - raise KeyError("All geneset names must be unique.") - - # 1. check gene set character set and format - illegal_name = re.compile(r"^\s| |[\v\t\r\n]|\s$") - for name in geneset_names: - if type(name) != str or len(name) == 0: - raise KeyError("Geneset names must be non-null string.") - if illegal_name.search(name): - messagefn( - "Error: " - f"Geneset name {name} " - "is not valid. Leading, trailing, and multiple spaces within a name are not allowed." - ) - raise KeyError( - "Geneset name is not valid. Leading, trailing, and multiple spaces within a name are not allowed." - ) - - # 2. & 3. check for duplicate gene symbols, and those not present in the dataset. They will - # generate a warning and be removed. var_names = set(self.query_var_array(self.parameters.get("var_names"))) - for geneset in genesets_iterable: - if not isinstance(geneset, dict): - raise ValueError("Each geneset must be a dict.") - geneset_name = geneset["geneset_name"] - genes = geneset["genes"] - if not isinstance(genes, list): - raise ValueError("Geneset genes field must be a list") - geneset.setdefault("geneset_description", "") - gene_symbol_already_seen = set() - new_genes = [] - for gene in genes: - gene_symbol = gene["gene_symbol"] - if not isinstance(gene_symbol, str) or len(gene_symbol) == 0: - raise ValueError("Gene symbol must be non-null string.") - if gene_symbol in gene_symbol_already_seen: - # duplicate check - messagefn( - f"Warning: a duplicate of gene {gene_symbol} was found in geneset {geneset_name}, " - "and will be ignored." - ) - continue - - if gene_symbol not in var_names: - messagefn( - f"Warning: {gene_symbol}, used in geneset {geneset_name}, " - "was not found in the dataset and will be ignored." - ) - continue - - gene_symbol_already_seen.add(gene_symbol) - gene.setdefault("gene_description", "") - new_genes.append(gene) - - geneset["genes"] = new_genes - - return genesets + return validate_gene_sets(genesets, var_names) def data_frame_to_fbs_matrix(self, filter, axis): """ diff --git a/backend/test/fixtures/pbmc3k-genesets.csv b/backend/test/fixtures/pbmc3k-genesets.csv index 952e40ef..da62da69 100644 --- a/backend/test/fixtures/pbmc3k-genesets.csv +++ b/backend/test/fixtures/pbmc3k-genesets.csv @@ -1,10 +1,10 @@ # Test fixture -gene_set_name, gene_set_description, gene_symbol, gene_description +gene_set_name,gene_set_description,gene_symbol,gene_description first gene set name,,F5, a gene_description -first gene set name,a description, NO_SUCH_GENE, non-existent gene -first gene set name,a description, F5, duplicate gene -first gene set name, a description, SUMO3, -first gene set name,, SRM, +first gene set name,a description,NO_SUCH_GENE, non-existent gene +first gene set name,a description,F5, duplicate gene +first gene set name, a description,SUMO3, +first gene set name,,SRM, second gene set,,RER1 second gene set,,SIK1 third gene set,,NO_SUCH_GENE diff --git a/backend/test/test_czi_hosted/unit/__init__.py b/backend/test/test_czi_hosted/unit/__init__.py index 7d3e3799..108f6835 100644 --- a/backend/test/test_czi_hosted/unit/__init__.py +++ b/backend/test/test_czi_hosted/unit/__init__.py @@ -48,7 +48,14 @@ def data_with_tmp_tiledb_annotations(ext: MatrixDataType): config.complete_config() data = MatrixDataLoader(data_locator.abspath()).open(config) - annotations = AnnotationsHostedTileDB(tmp_dir, DbUtils("postgresql://postgres:test_pw@localhost:5432"),) + annotations = AnnotationsHostedTileDB( + { + "user-annotations": True, + "genesets-save": False, + }, + tmp_dir, + DbUtils("postgresql://postgres:test_pw@localhost:5432"), + ) return data, tmp_dir, annotations @@ -70,12 +77,21 @@ def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False): single_dataset__datapath=data_locator.path, ) config.update_default_dataset_config( - embeddings__names=["umap"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01, + embeddings__names=["umap"], + presentation__max_categories=100, + diffexp__lfc_cutoff=0.01, ) config.complete_config() data = MatrixDataLoader(data_locator.abspath()).open(config) - annotations = AnnotationsLocalFile(None, annotations_file) + annotations = AnnotationsLocalFile( + { + "user-annotations": True, + "genesets-save": False, + }, + None, + annotations_file, + ) return data, tmp_dir, annotations diff --git a/backend/test/test_czi_hosted/unit/common/test_api.py b/backend/test/test_czi_hosted/unit/common/test_api.py index dea2693e..00bce5b2 100644 --- a/backend/test/test_czi_hosted/unit/common/test_api.py +++ b/backend/test/test_czi_hosted/unit/common/test_api.py @@ -3,6 +3,7 @@ import time import unittest import zlib from http import HTTPStatus +import hashlib import pandas as pd import requests @@ -315,6 +316,95 @@ class EndPoints(object): result = self.session.get(url) self.assertEqual(result.status_code, HTTPStatus.OK) + def test_genesets_config(self): + result = self.session.get(f"{self.URL_BASE}config") + config_data = result.json() + params = config_data["config"]["parameters"] + annotations_genesets = params["annotations_genesets"] + annotations_genesets_readonly = params["annotations_genesets_readonly"] + annotations_genesets_summary_methods = params["annotations_genesets_summary_methods"] + self.assertTrue(annotations_genesets) + self.assertTrue(annotations_genesets_readonly) + self.assertEqual(annotations_genesets_summary_methods, ["mean"]) + + def test_get_genesets(self): + endpoint = "genesets" + url = f"{self.URL_BASE}{endpoint}" + result = self.session.get(url, headers={"Accept": "application/json"}) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/json") + result_data = result.json() + self.assertIsNotNone(result_data["genesets"]) + + def test_get_summaryvar(self): + index_col_name = self.schema["schema"]["annotations"]["var"]["index"] + endpoint = "summarize/var" + + # single column + filter = f"var:{index_col_name}=F5" + query = f"method=mean&{filter}" + query_hash = hashlib.sha1(query.encode()).hexdigest() + url = f"{self.URL_BASE}{endpoint}?{query}" + header = {"Accept": "application/octet-stream"} + result = self.session.get(url, headers=header) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertEqual(df["n_rows"], 2638) + self.assertEqual(df["n_cols"], 1) + self.assertEqual(df["col_idx"], [query_hash]) + self.assertAlmostEqual(df["columns"][0][0], -0.110451095) + + # multi-column + col_names = ["F5", "BEB3", "SIK1"] + filter = "&".join([f"var:{index_col_name}={name}" for name in col_names]) + query = f"method=mean&{filter}" + query_hash = hashlib.sha1(query.encode()).hexdigest() + url = f"{self.URL_BASE}{endpoint}?{query}" + header = {"Accept": "application/octet-stream"} + result = self.session.get(url, headers=header) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertEqual(df["n_rows"], 2638) + self.assertEqual(df["n_cols"], 1) + self.assertEqual(df["col_idx"], [query_hash]) + self.assertAlmostEqual(df["columns"][0][0], -0.16628358) + + def test_post_summaryvar(self): + index_col_name = self.schema["schema"]["annotations"]["var"]["index"] + endpoint = "summarize/var" + headers = {"Content-Type": "application/x-www-form-urlencoded", "Accept": "application/octet-stream"} + + # single column + filter = f"var:{index_col_name}=F5" + query = f"method=mean&{filter}" + query_hash = hashlib.sha1(query.encode()).hexdigest() + url = f"{self.URL_BASE}{endpoint}?key={query_hash}" + result = self.session.post(url, headers=headers, data=query) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertEqual(df["n_rows"], 2638) + self.assertEqual(df["n_cols"], 1) + self.assertEqual(df["col_idx"], [query_hash]) + self.assertAlmostEqual(df["columns"][0][0], -0.110451095) + + # multi-column + col_names = ["F5", "BEB3", "SIK1"] + filter = "&".join([f"var:{index_col_name}={name}" for name in col_names]) + query = f"method=mean&{filter}" + query_hash = hashlib.sha1(query.encode()).hexdigest() + url = f"{self.URL_BASE}{endpoint}?key={query_hash}" + result = self.session.post(url, headers=headers, data=query) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertEqual(df["n_rows"], 2638) + self.assertEqual(df["n_cols"], 1) + self.assertEqual(df["col_idx"], [query_hash]) + self.assertAlmostEqual(df["columns"][0][0], -0.16628358) + def _setupClass(child_class, command_line): child_class.ps, child_class.server = start_test_server(command_line) child_class.URL_BASE = f"{child_class.server}/api/v0.2/" @@ -333,7 +423,8 @@ class EndPointsAnnotations(EndPoints): def test_get_user_annotations_existing_obs_keys_fbs(self): self._test_get_user_annotations_obs_keys_fbs( - "cluster-test", {"unassigned", "one", "two", "three", "four", "five", "six", "seven"}, + "cluster-test", + {"unassigned", "one", "two", "three", "four", "five", "six", "seven"}, ) def test_put_user_annotations_obs_fbs(self): @@ -416,6 +507,91 @@ class EndPointsCxg(unittest.TestCase, EndPoints): def tearDownClass(cls): stop_test_server(cls.ps) + def test_get_genesets_json(self): + endpoint = "genesets" + url = f"{self.URL_BASE}{endpoint}" + result = self.session.get(url, headers={"Accept": "application/json"}) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/json") + result_data = result.json() + self.assertIsNotNone(result_data["genesets"]) + self.assertIsNotNone(result_data["tid"]) + + self.assertEqual( + result_data, + { + "genesets": [ + { + "genes": [ + {"gene_description": " a gene_description", "gene_symbol": "F5"}, + {"gene_description": "", "gene_symbol": "SUMO3"}, + {"gene_description": "", "gene_symbol": "SRM"}, + ], + "geneset_description": "a description", + "geneset_name": "first gene set name", + }, + { + "genes": [ + {"gene_description": "", "gene_symbol": "RER1"}, + {"gene_description": "", "gene_symbol": "SIK1"}, + ], + "geneset_description": "", + "geneset_name": "second gene set", + }, + {"genes": [], "geneset_description": "", "geneset_name": "third gene set"}, + {"genes": [], "geneset_description": "fourth description", "geneset_name": "fourth_gene_set"}, + {"genes": [], "geneset_description": "", "geneset_name": "fifth_dataset"}, + { + "genes": [ + {"gene_description": "", "gene_symbol": "ACD"}, + {"gene_description": "", "gene_symbol": "AATF"}, + {"gene_description": "", "gene_symbol": "F5"}, + {"gene_description": "", "gene_symbol": "PIGU"}, + ], + "geneset_description": "", + "geneset_name": "summary test", + }, + ], + "tid": 0, + }, + ) + + def test_get_genesets_csv(self): + endpoint = "genesets" + url = f"{self.URL_BASE}{endpoint}" + result = self.session.get(url, headers={"Accept": "text/csv"}) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "text/csv") + self.assertEqual( + result.text, + """gene_set_name,gene_set_description,gene_symbol,gene_description\r +first gene set name,a description,F5, a gene_description\r +first gene set name,a description,SUMO3,\r +first gene set name,a description,SRM,\r +second gene set,,RER1,\r +second gene set,,SIK1,\r +third gene set,,,\r +fourth_gene_set,fourth description,,\r +fifth_dataset,,,\r +summary test,,ACD,\r +summary test,,AATF,\r +summary test,,F5,\r +summary test,,PIGU,\r +""", + ) + + def test_put_genesets(self): + endpoint = "genesets" + url = f"{self.URL_BASE}{endpoint}" + + result = self.session.get(url, headers={"Accept": "application/json"}) + self.assertEqual(result.status_code, HTTPStatus.OK) + + test1 = {"tid": 3, "genesets": []} + result = self.session.put(url, json=test1) + + self.assertEqual(result.status_code, HTTPStatus.METHOD_NOT_ALLOWED) + class EndPointsAnndataAnnotations(unittest.TestCase, EndPointsAnnotations): """Test Case for endpoints""" diff --git a/backend/test/test_server/unit/common/test_api.py b/backend/test/test_server/unit/common/test_api.py index 376706f3..19ac7873 100644 --- a/backend/test/test_server/unit/common/test_api.py +++ b/backend/test/test_server/unit/common/test_api.py @@ -560,7 +560,7 @@ class EndPointsAnnDataGenesets(unittest.TestCase, EndPoints): "genesets": [ { "genes": [ - {"gene_description": "a gene_description", "gene_symbol": "F5"}, + {"gene_description": " a gene_description", "gene_symbol": "F5"}, {"gene_description": "", "gene_symbol": "SUMO3"}, {"gene_description": "", "gene_symbol": "SRM"}, ], @@ -602,7 +602,7 @@ class EndPointsAnnDataGenesets(unittest.TestCase, EndPoints): self.assertEqual( result.text, """gene_set_name,gene_set_description,gene_symbol,gene_description\r -first gene set name,a description,F5,a gene_description\r +first gene set name,a description,F5, a gene_description\r first gene set name,a description,SUMO3,\r first gene set name,a description,SRM,\r second gene set,,RER1,\r diff --git a/client/src/reducers/annotations.js b/client/src/reducers/annotations.js index ed27386f..5178d0cf 100644 --- a/client/src/reducers/annotations.js +++ b/client/src/reducers/annotations.js @@ -35,9 +35,13 @@ const Annotations = ( const dataCollectionName = action.config.parameters?.["annotations-data-collection-name"] ?? null; const dataCollectionNameIsReadOnly = - action.config.parameters?.[ - "annotations-data-collection-name-is-read-only" - ] ?? false; + (action.config.parameters?.[ + "annotations-data-collection-is-read-only" + ] ?? + false) && + (action.config.parameters?.annotations_genesets_name_is_read_only ?? + true); + const promptForFilename = action.config.parameters?.user_annotation_collection_name_enabled; return {