fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-30 02:08:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

Launch validation (#414)

Browse Source 
* WIP

* Cleanup

* Validation

* typo

* Max category limit 100 -> 500
This commit is contained in:
Charlotte Weaver
2018-11-07 13:01:13 -08:00
committed by GitHub
parent a9189017c7
commit 7d5c054b90
10 changed files with 204 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/app/scanpy_engine/scanpy_engine.py
+52 -18
View File
@@ -2,12 +2,13 @@ import warnings
import numpy as np
from pandas import DataFrame
from pandas.core.dtypes.dtypes import CategoricalDtype
import scanpy.api as sc
from scipy import stats, sparse
from server.app.driver.driver import CXGDriver
from server.app.util.constants import Axis, DEFAULT_TOP_N, DiffExpMode
from server.app.util.utils import FilterError, InteractiveError, PrepareError
from server.app.util.errors import FilterError, InteractiveError, PrepareError, ScanpyFileError
"""
Sort order for methods
@@ -23,9 +24,10 @@ class ScanpyEngine(CXGDriver):
def __init__(self, data, args):
super().__init__(data, args)
self._alias_annotation_names(Axis.OBS, args['obs_names'])
self._alias_annotation_names(Axis.VAR, args['var_names'])
self._alias_annotation_names(Axis.OBS, args["obs_names"])
self._alias_annotation_names(Axis.VAR, args["var_names"])
self._validate_data_types()
self._validate_data_calculations()
self.cell_count = self.data.shape[0]
self.gene_count = self.data.shape[1]
self.layout_options = ["umap", "tsne"]
@@ -39,27 +41,28 @@ class ScanpyEngine(CXGDriver):
As a *critical* side-effect, ensure the indices are simple number ranges
(accomplished by calling pandas.DataFrame.reset_index())
"""
if name == 'name':
if name == "name":
# a noop, so skip it
return
ax_name = str(axis)
df_axis = getattr(self.data, ax_name)
if name is None:
# reset index to simple range; alias 'name' to point at the
# reset index to simple range; alias "name" to point at the
# previously specified index.
df_axis = df_axis.reset_index().rename(columns={'index': 'name'})
df_axis.reset_index(inplace=True)
df_axis.rename(inplace=True, columns={"index": "name"})
elif name in df_axis.columns:
if name not in df_axis.columns:
raise KeyError(f"Annotation name {name}, specified in --{ax_name}-name does not exist.")
if not df_axis[name].is_unique:
raise KeyError(f"Values in -{ax_name}-name must be unique. "
"Please prepare data to contain unique values.")
# reset index to simple range; alias user-specified annotation to 'name'
df_axis = df_axis.reset_index(drop=True).rename(columns={name: 'name'})
# reset index to simple range; alias user-specified annotation to "name"
df_axis.reset_index(drop=True, inplace=True)
df_axis.rename(inplace=True, columns={name: "name"})
else:
raise KeyError(f"Annotation name {name}, specified in --{ax_name}_name does not exist.")
setattr(self.data, ax_name, df_axis)
def _create_schema(self):
self.schema = {
@@ -78,9 +81,9 @@ class ScanpyEngine(CXGDriver):
for ann in curr_axis:
ann_schema = {"name": ann}
data_kind = curr_axis[ann].dtype.kind
if data_kind == 'f':
if data_kind == "f":
ann_schema["type"] = "float32"
elif data_kind in ['i', 'u']:
elif data_kind in ["i", "u"]:
ann_schema["type"] = "int32"
elif data_kind == "?":
ann_schema["type"] = "boolean"
@@ -98,7 +101,18 @@ class ScanpyEngine(CXGDriver):
# Based on benchmarking, cache=True has no impact on perf.
# Note: as of current scanpy/anndata release, setting backed='r' will
# result in an error. https://github.com/theislab/anndata/issues/79
return sc.read(data, cache=False)
try:
result = sc.read(data, cache=True)
except ValueError:
raise ScanpyFileError("File must be in the .h5ad format. Please read "
"https://github.com/theislab/scanpy_usage/blob/master/170505_seurat/info_h5ad.md to "
"learn more about this format. You may be able to convert your file into this format "
"using `cellxgene prepare`, please run `cellxgene prepare --help` for more "
"information.")
except Exception as e:
raise ScanpyFileError(f"Error while loading file: {e}, File must be in the .h5ad format, please check "
f"that your input and try again.")
return result
@staticmethod
def _top_sort(values, sort_order, top_n=None):
@@ -137,14 +151,34 @@ class ScanpyEngine(CXGDriver):
curr_axis = getattr(self.data, str(ax))
for ann in curr_axis:
datatype = curr_axis[ann].dtype
downcast_map = {'int64': 'int32',
'uint32': 'int32',
'uint64': 'int32',
'float64': 'float32',
downcast_map = {"int64": "int32",
"uint32": "int32",
"uint64": "int32",
"float64": "float32",
}
if datatype in downcast_map:
warnings.warn(f"Scanpy annotation {ax}:{ann} is in unsupported format: {datatype}. "
f"Data will be downcast to {downcast_map[datatype]}.")
if isinstance(datatype, CategoricalDtype):
category_num = len(curr_axis[ann].dtype.categories)
if category_num > 500 and category_num > self.max_category_items:
warnings.warn(
f"{str(ax).title()} annotation '{ann}' has {category_num} categories, this may be "
f"cumbersome or slow to display. We recommend setting the "
f"--max-category-items option to 500, this will hide categorical "
f"annotations with more than 500 categories in the UI")
def _validate_data_calculations(self):
layout_key = f"X_{self.layout_method}"
try:
assert layout_key in self.data.obsm_keys()
except AssertionError:
raise PrepareError(
f"Cannot find a field with coordinates for the {self.layout_method} layout requested. A different"
f" layout may have been computed. The requested layout must be pre-calculated and saved "
f"back in the h5ad file. You can run "
f"`cellxgene prepare --layout {self.layout_method} <datafile>` "
f"to solve this problem. ")
def filter_dataframe(self, filter, include_uns=False):
"""
@@ -232,8 +266,8 @@ class ScanpyEngine(CXGDriver):
https://docs.scipy.org/doc/scipy/reference/sparse.html
"""
prefer_row_access = sparse.isspmatrix_csr(data._X) or \
sparse.isspmatrix_lil(data._X) or sparse.isspmatrix_bsr(data._X)
prefer_row_access = sparse.isspmatrix_csr(data._X) or sparse.isspmatrix_lil(data._X) \
or sparse.isspmatrix_bsr(data._X)
if prefer_row_access:
# Row-major slicing
if obs_selector is not None: