fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-29 08:18:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

genesets route for local server (#2079)

Browse Source 
* first cut at GET /genesets route

* update existing tests to match code changes

* more GET /genesets and initial tests

* add missing test fixture

* geneset validation accepts OTA format

* genesets route: better error handling, more tests

* lint
This commit is contained in:
Bruce Martin
2021-02-26 17:53:07 -08:00
committed by GitHub
parent 09466a5c32
commit f3a3820ffa
18 changed files with 813 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
local_server/common/annotations/annotations.py
+77 -3
View File
@@ -3,19 +3,34 @@ from abc import ABCMeta, abstractmethod
import fastobo
import fsspec
from local_server.common.errors import OntologyLoadFailure
from local_server.common.errors import OntologyLoadFailure, DisabledFeatureError
from local_server.common.utils.type_conversion_utils import get_schema_type_hint_of_array
class Annotations(metaclass=ABCMeta):
""" baseclass for annotations, including ontologies"""
""" baseclass for annotations, including ontologies and genesets"""
""" our default ontology is the PURL for the Cell Ontology.
See http://www.obofoundry.org/ontology/cl.html """
DefaultOnotology = "http://purl.obolibrary.org/obo/cl.obo"
def __init__(self):
def __init__(self, config={}):
self.ontology_data = None
self.config = config
def user_annotations_enabled(self):
return self.config.get("user-annotations", False)
def genesets_save_enabled(self):
return self.config.get("genesets-save", False)
def check_user_annotations_enabled(self):
if not self.user_annotations_enabled():
raise DisabledFeatureError("User annotations are disabled.")
def check_genesets_save_enabled(self):
if not self.genesets_save_enabled():
raise DisabledFeatureError("User genesets save is disabled.")
def load_ontology(self, path):
"""Load and parse ontologies - currently support OBO files only."""
@@ -64,7 +79,66 @@ class Annotations(metaclass=ABCMeta):
"""Write the labels (df) to a persistent storage such that it can later be read"""
pass
@abstractmethod
def read_genesets(self, data_adaptor):
"""Return the genesets from persistent storage """
pass
@abstractmethod
def write_genesets(self, gs, data_adaptor):
"""Write the genesets (gs) to a persistent storage such that it can later be read"""
pass
@abstractmethod
def update_parameters(self, parameters, data_adaptor):
"""Update configuration parameters that describe information about the annotations feature"""
pass
Genesets_Header = [
"geneset_name",
"geneset_description",
"gene_symbol",
"gene_description",
]
@staticmethod
def genesets_to_csv(genesets):
"""
Convert the internal genesets format (returned by read_geneset) into
the simple Tidy CSV.
"""
from io import StringIO
import csv
if type(genesets) == dict:
genesets = genesets.values()
with StringIO() as sio:
writer = csv.writer(sio, dialect='excel')
writer.writerow(Annotations.Genesets_Header)
for geneset in genesets:
# genes may be empty, in which case we skip the geneset entirely
genes = geneset["genes"]
if not genes:
writer.writerow([geneset["geneset_name"], geneset.get("geneset_description", ""), "", ""])
else:
writer.writerows(
[
[
geneset["geneset_name"],
geneset.get("geneset_description", ""),
gene["gene_symbol"],
gene.get("gene_description", ""),
]
for gene in genes
]
)
return sio.getvalue()
@staticmethod
def genesets_to_response(genesets):
"""
Convert the internal genesets format (returned by read_geneset) into
the dict expected by the JSON REST API
"""
return list(genesets.values())
local_server/common/annotations/local_file_csv.py
+178 -22
View File
@@ -4,29 +4,35 @@ import re
import threading
from datetime import datetime
from hashlib import blake2b
import csv
import pandas as pd
from flask import session, has_request_context, current_app
from local_server import __version__ as cellxgene_version
from local_server.common.annotations.annotations import Annotations
from local_server.common.errors import AnnotationsError
from local_server.common.errors import AnnotationsError, ObsoleteRequest
class AnnotationsLocalFile(Annotations):
CXG_ANNO_COLLECTION = "cxg_anno_collection"
def __init__(self, output_dir, output_file):
super().__init__()
def __init__(self, config, output_dir, label_output_file, genesets_output_file):
super().__init__(config)
self.output_dir = output_dir
self.output_file = output_file
self.label_output_file = label_output_file
self.genesets_output_file = genesets_output_file
# lock used to protect label file write ops
self.label_lock = threading.RLock()
self.genesets_lock = threading.RLock()
# cache the most recent annotations
# cache the most recent annotations.
self.last_fname = None
self.last_labels = None
# txn ID - used to de-dup geneset writes
self.last_geneset_tid = 0
def is_safe_collection_name(self, name):
"""
return true if this is a safe collection name
@@ -47,11 +53,13 @@ class AnnotationsLocalFile(Annotations):
return session.get(self.CXG_ANNO_COLLECTION)
def read_labels(self, data_adaptor):
self.check_user_annotations_enabled() # raises
if has_request_context():
if not current_app.auth.is_user_authenticated():
return pd.DataFrame()
fname = self._get_filename(data_adaptor)
fname = self._get_celllabels_filename(data_adaptor)
with self.label_lock:
if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0:
# returned the cached labels if possible, otherwise read them from the file
@@ -69,6 +77,8 @@ class AnnotationsLocalFile(Annotations):
return pd.DataFrame()
def write_labels(self, df, data_adaptor):
self.check_user_annotations_enabled() # raises
# update our internal state and save it. Multi-threading often enabled,
# so treat this as a critical section.
with self.label_lock:
@@ -81,7 +91,7 @@ class AnnotationsLocalFile(Annotations):
f"which was last modified on {lastmodstr}\n"
)
fname = self._get_filename(data_adaptor)
fname = self._get_celllabels_filename(data_adaptor)
self._backup(fname)
if not df.empty:
with open(fname, "w", newline="") as f:
@@ -95,12 +105,56 @@ class AnnotationsLocalFile(Annotations):
self.last_fname = fname
self.last_labels = df
def read_genesets(self, data_adaptor, context=None):
if has_request_context():
if not current_app.auth.is_user_authenticated():
return ([], None)
fname = self._get_genesets_filename(data_adaptor)
genesets = {}
tid = None
with self.genesets_lock:
tid = self.last_geneset_tid # inside the critical section
if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0:
with open(fname, newline="") as f:
genesets = read_geneset_tidycsv(f, context)
return (genesets, tid)
def write_genesets(self, genesets, tid, data_adaptor):
self.check_genesets_save_enabled() # raises
if type(tid) != int or tid < 0:
raise ValueError("tid must be a positive integer")
with self.genesets_lock:
# skip if the request is stale
if tid is not None:
if tid <= self.last_geneset_tid:
raise ObsoleteRequest("TID is stale.")
self.last_geneset_tid = tid
lastmod = data_adaptor.get_last_mod_time()
lastmodstr = "'unknown'" if lastmod is None else lastmod.isoformat(timespec="seconds")
header = (
f"# Geneset generated on {datetime.now().isoformat(timespec='seconds')} "
f"using cellxgene version {cellxgene_version}\n"
f"# Input data file was {data_adaptor.get_location()}, "
f"which was last modified on {lastmodstr}\n"
)
fname = self._get_genesets_filename(data_adaptor)
self._backup(fname)
with open(fname, "w", newline="") as f:
f.write(header)
f.write(self.genesets_to_csv(genesets))
def _get_userdata_idhash(self, data_adaptor):
"""
Return a short hash that weakly identifies the user and dataset.
Used to create safe annotations output file names.
"""
uid = current_app.auth.get_user_id()
uid = current_app.auth.get_user_id() or ""
id = (uid + data_adaptor.get_location()).encode()
idhash = base64.b32encode(blake2b(id, digest_size=5).digest()).decode("utf-8")
return idhash
@@ -109,16 +163,27 @@ class AnnotationsLocalFile(Annotations):
if self.output_dir:
return self.output_dir
if self.output_file:
return os.path.dirname(self.path.abspath(self.output_dir))
output_file = self.label_output_file or self.genesets_output_file
if output_file:
return os.path.dirname(self.path.abspath(output_file))
return os.getcwd()
def _get_filename(self, data_adaptor):
def _get_celllabels_filename(self, data_adaptor):
""" return the current annotation file name """
if self.output_file:
return self.output_file
if self.label_output_file:
return self.label_output_file
return self._get_filename(data_adaptor, "celllabels")
def _get_genesets_filename(self, data_adaptor):
""" return the current genesets file name """
if self.genesets_output_file:
return self.genesets_output_file
return self._get_filename(data_adaptor, "genesets")
def _get_filename(self, data_adaptor, anno_name):
# we need to generate a file name, which we can only do if we have a UID and collection name
if session is None:
raise AnnotationsError("unable to determine file name for annotations")
@@ -131,7 +196,7 @@ class AnnotationsLocalFile(Annotations):
raise AnnotationsError("unable to determine file name for annotations")
idhash = self._get_userdata_idhash(data_adaptor)
return os.path.join(self._get_output_dir(), f"{collection}-{idhash}.csv")
return os.path.join(self._get_output_dir(), f"{collection}-{anno_name}-{idhash}.csv")
def _backup(self, fname, max_backups=9):
"""
@@ -170,7 +235,8 @@ class AnnotationsLocalFile(Annotations):
def update_parameters(self, parameters, data_adaptor):
params = {}
params["annotations"] = True
params["annotations"] = self.user_annotations_enabled()
params["annotations_genesets_readonly"] = not self.genesets_save_enabled()
params["user_annotation_collection_name_enabled"] = True
if self.ontology_data:
@@ -179,18 +245,108 @@ class AnnotationsLocalFile(Annotations):
else:
params["annotations_cell_ontology_enabled"] = False
if self.output_file is not None:
# user has hard-wired the name of the annotation data collection
fname = os.path.basename(self.output_file)
if self.label_output_file is not None:
# user has hard-wired the name of the annotation cell label data collection
fname = os.path.basename(self.label_output_file)
collection_fname = os.path.splitext(fname)[0]
params["annotations-data-collection-is-read-only"] = True
params["annotations-data-collection-name"] = collection_fname
elif session is not None:
collection = self.get_collection()
if current_app.auth.is_user_authenticated():
params["annotations-user-data-idhash"] = self._get_userdata_idhash(data_adaptor)
params["annotations-data-collection-is-read-only"] = False
params["annotations-data-collection-name"] = collection
params["annotations-data-collection-is-read-only"] = False
params["annotations-data-collection-name"] = collection
if current_app.auth.is_user_authenticated():
params["annotations-user-data-idhash"] = self._get_userdata_idhash(data_adaptor)
parameters.update(params)
def read_geneset_tidycsv(f, context=None):
"""
Read & parse the Tidy CSV format, applying validation checks for mandatory
values, and de-duping rules.
Format is a four-column CSV, with a mandatory header row, and optional "#" prefixed
comments. Format:
geneset_name, geneset_description, gene_symbol, gene_description
geneset_name and gene_symbol must be non-null; others are optional.
Returns: a dictionary of the shape (values in angle-brackets vary):
{
<string, a gene set name>: {
"geneset_name": <string, a gene set name>,
"geneset_description": <a string or None>,
"genes": [
{
"gene_symbol": <string, a gene symbol or name>,
"gene_description": <a string or None>
},
...
]
},
...
}
"""
class myDialect(csv.excel):
skipinitialspace = True
def just(n, seq):
it = iter(seq)
for _ in range(n - 1):
yield next(it, "")
yield tuple(it)
messagefn = context["messagefn"] if context else (lambda x: None)
reader = csv.reader(f, dialect=myDialect())
genesets = {}
haveReadHeader = False
lineno = 0
for row in reader:
lineno += 1
# ignore empty rows
if len(row) == 0:
continue
# if row starts with '#' it is a comment
if row[0].startswith("#"):
continue
# if this is the first non-comment row, assume it is a header
if not haveReadHeader:
if row != Annotations.Genesets_Header:
raise AnnotationsError("Geneset CSV file missing the required column header.")
haveReadHeader = True
continue
geneset_name, geneset_description, gene_symbol, gene_description, _ = just(5, row)
if not geneset_name:
raise AnnotationsError(f"Geneset CSV missing required geneset or gene name on line {lineno}")
if (not gene_symbol) and gene_description:
messagefn(f"Warning: Missing gene name in geneset name {geneset_name} on line {lineno}.")
if geneset_name in genesets:
gs = genesets[geneset_name]
else:
gs = genesets[geneset_name] = {
"geneset_name": geneset_name,
"geneset_description": geneset_description,
"genes": [],
}
# Use first geneset_description with a value
if not gs["geneset_description"] and geneset_description:
gs["geneset_description"] = geneset_description
# add the gene if the gene_symbol is defined
if gene_symbol:
gs["genes"].append(
{
"gene_symbol": gene_symbol,
"gene_description": gene_description,
}
)
return genesets