fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-27 08:58:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

annotations CLI and file UX rework (#1049)

Browse Source 
* rename config param label-file

* annotations rework - CLI params, file naming and backups

* lint

* improve cli option error checks

* enable session cookies

* enable session cookies

* add session id

* name annotations file in multi-dataset and multi-user safe manner

* pass data user hash to front-end

* add annotation collection name support to front-end

* add constant for annotation data collection name

* parameterize annotation collection name; make it sticky in the session

* clarify comments

* hard wire a temporary data collection name for testing

* prettier

* test comment

* package command

* set annotations  filename dialog

* name  and hash are visible

* wire up data collection capture
This commit is contained in:
Bruce Martin
2019-11-25 15:28:28 -08:00
committed by GitHub
parent 575f71aaee
commit a593e95ab3
18 changed files with 551 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/app/scanpy_engine/scanpy_engine.py
+114 -34
View File
@@ -2,6 +2,9 @@ import warnings
import copy
import threading
from datetime import datetime
import os.path
from hashlib import blake2b
import base64
import numpy as np
import pandas
@@ -37,7 +40,7 @@ class ScanpyEngine(CXGDriver):
def __init__(self, data_locator=None, args={}):
super().__init__(data_locator, args)
# lock used to protect label file write ops
self.label_lock = threading.Lock()
self.label_lock = threading.RLock()
if self.data:
self._validate_and_initialize()
@@ -54,12 +57,41 @@ class ScanpyEngine(CXGDriver):
"obs_names": None,
"var_names": None,
"diffexp_lfc_cutoff": 0.01,
"label_file": None,
"annotations": False,
"annotations_file": None,
"annotations_output_dir": None,
"backed": False,
"disable_diffexp": False,
"diffexp_may_be_slow": False
}
def get_config_parameters(self, uid=None, collection=None):
params = {
"max-category-items": self.config["max_category_items"],
"disable-diffexp": self.config["disable_diffexp"],
"diffexp-may-be-slow": self.config["diffexp_may_be_slow"],
"annotations": self.config["annotations"]
}
if self.config["annotations"]:
if uid is not None:
params.update({
"annotations-user-data-idhash": self.get_userdata_idhash(uid)
})
if self.config['annotations_file'] is not None:
# user has hard-wired the name of the annotation data collection
fname = os.path.basename(self.config['annotations_file'])
collection_fname = os.path.splitext(fname)[0]
params.update({
'annotations-data-collection-is-read-only': True,
'annotations-data-collection-name': collection_fname
})
elif collection is not None:
params.update({
'annotations-data-collection-is-read-only': False,
'annotations-data-collection-name': collection
})
return params
@staticmethod
def _create_unique_column_name(df, col_name_prefix):
""" given the columns of a dataframe, and a name prefix, return a column name which
@@ -197,20 +229,66 @@ class ScanpyEngine(CXGDriver):
self.schema["layout"]["obs"].append(layout_schema)
@requires_data
def get_schema(self):
def get_schema(self, uid=None, collection=None):
schema = self.schema # base schema
# add label obs annotations as needed
if self.labels is not None:
labels = read_labels(self.get_anno_fname(uid, collection))
if labels is not None and not labels.empty:
schema = copy.deepcopy(schema)
for col in self.labels.columns:
for col in labels.columns:
col_schema = {
"name": col,
"writable": True,
}
col_schema.update(self._get_col_type(self.labels[col]))
col_schema.update(self._get_col_type(labels[col]))
schema["annotations"]["obs"]["columns"].append(col_schema)
return schema
def get_userdata_idhash(self, uid):
"""
Return a short hash that weakly identifies the user and dataset.
Used to create safe annotations output file names.
"""
id = (uid + self.data_locator.abspath()).encode()
idhash = base64.b32encode(blake2b(id, digest_size=5).digest()).decode('utf-8')
return idhash
def get_anno_fname(self, uid=None, collection=None):
""" return the current annotation file name """
if not self.config["annotations"]:
return None
if self.config["annotations_file"] is not None:
return self.config["annotations_file"]
# we need to generate a file name, which we can only do if we have a UID and collection name
if uid is None or collection is None:
return None
idhash = self.get_userdata_idhash(uid)
return os.path.join(self.get_anno_output_dir(), f"{collection}-{idhash}.csv")
def get_anno_output_dir(self):
""" return the current annotation output directory """
if not self.config["annotations"]:
return None
if self.config['annotations_output_dir']:
return self.config['annotations_output_dir']
if self.config['annotations_file']:
return os.path.dirname(os.path.abspath(self.config['annotations_file']))
return os.getcwd()
def get_anno_backup_dir(self, uid, collection=None):
""" return the current annotation backup directory """
if not self.config["annotations"]:
return None
fname = self.get_anno_fname(uid, collection)
root, ext = os.path.splitext(fname)
return f"{root}-backups"
def _load_data(self, data_locator):
# as of AnnData 0.6.19, backed mode performs initial load fast, but at the
# cost of significantly slower access to X data.
@@ -240,17 +318,6 @@ class ScanpyEngine(CXGDriver):
f"Please check your input and try again."
)
if self.config["label_file"]:
try:
self.labels = read_labels(self.config["label_file"])
except Exception as e:
raise ScanpyFileError(
f"Error while loading label file: {e}, File must be in the .csv format, please check "
f"your input and try again."
)
else:
self.labels = None
@requires_data
def _validate_and_initialize(self):
# var and obs column names must be unique
@@ -262,9 +329,13 @@ class ScanpyEngine(CXGDriver):
self.cell_count = self.data.shape[0]
self.gene_count = self.data.shape[1]
self._default_and_validate_layouts()
self._validate_label_data()
self._create_schema()
# if the user has specified a fixed label file, go ahead and validate it
# so that we can remove errors early in the process.
if self.config["annotations_file"]:
self._validate_label_data(read_labels(self.get_anno_fname()))
# heuristic
n_values = self.data.shape[0] * self.data.shape[1]
if (n_values > 1e8 and self.config['backed'] is True) or (n_values > 5e8):
@@ -352,24 +423,20 @@ class ScanpyEngine(CXGDriver):
)
@requires_data
def _validate_label_data(self, labels=None):
def _validate_label_data(self, labels):
"""
labels is None if disabled, empty if enabled by no data
"""
if labels is None:
labels = self.labels
if labels is None or labels.empty:
return
# all lables must have a name, which must be unique and not used in obs column names
if not labels.columns.is_unique:
raise KeyError(f"All column names specified in {self.config['label_file']} must be unique.")
raise KeyError(f"All column names specified in user annotations must be unique.")
# the label index must be unique, and must have same values the anndata obs index
if not labels.index.is_unique:
raise KeyError(f"All row index values specified in the label file "
f"`{self.config['label_file']}` must be unique.")
raise KeyError(f"All row index values specified in user annotations must be unique.")
if not labels.index.equals(self.original_obs_index):
raise KeyError("Label file row index does not match H5AD file index. "
@@ -450,10 +517,21 @@ class ScanpyEngine(CXGDriver):
return obs_selector, var_selector
@requires_data
def annotation_to_fbs_matrix(self, axis, fields=None):
def annotation_to_fbs_matrix(self, axis, fields=None, uid=None, collection=None):
if axis == Axis.OBS:
if self.labels is not None and not self.labels.empty:
df = self.data.obs.join(self.labels, self.config['obs_names'])
if self.config["annotations"]:
try:
labels = read_labels(self.get_anno_fname(uid, collection))
except Exception as e:
raise ScanpyFileError(
f"Error while loading label file: {e}, File must be in the .csv format, please check "
f"your input and try again."
)
else:
labels = None
if labels is not None and not labels.empty:
df = self.data.obs.join(labels, self.config['obs_names'])
else:
df = self.data.obs
else:
@@ -463,18 +541,21 @@ class ScanpyEngine(CXGDriver):
return encode_matrix_fbs(df, col_idx=df.columns)
@requires_data
def annotation_put_fbs(self, axis, fbs):
fname = self.config["label_file"]
if not fname or self.labels is None:
def annotation_put_fbs(self, axis, fbs, uid=None, collection=None):
if not self.config["annotations"]:
raise DisabledFeatureError("Writable annotations are not enabled")
fname = self.get_anno_fname(uid, collection)
if not fname:
raise ScanpyFileError("Writable annotations - unable to determine file name for annotations")
if axis != Axis.OBS:
raise ValueError("Only OBS dimension access is supported")
new_label_df = decode_matrix_fbs(fbs)
if not new_label_df.empty:
new_label_df.index = self.original_obs_index
self._validate_label_data(labels=new_label_df) # paranoia
self._validate_label_data(new_label_df) # paranoia
# if any of the new column labels overlap with our existing labels, raise error
duplicate_columns = list(set(new_label_df.columns) & set(self.data.obs.columns))
@@ -485,14 +566,13 @@ class ScanpyEngine(CXGDriver):
# update our internal state and save it. Multi-threading often enabled,
# so treat this as a critical section.
with self.label_lock:
self.labels = new_label_df
lastmod = self.data_locator.lastmodtime()
lastmodstr = "'unknown'" if lastmod is None else lastmod.isoformat(timespec="seconds")
header = f"# Annotations generated on {datetime.now().isoformat(timespec='seconds')} " \
f"using cellxgene version {cellxgene_version}\n" \
f"# Input data file was {self.data_locator.uri_or_path}, " \
f"which was last modified on {lastmodstr}\n"
write_labels(fname, self.labels, header)
write_labels(fname, new_label_df, header, backup_dir=self.get_anno_backup_dir(uid, collection))
return jsonify_scanpy({"status": "OK"})