fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-15 12:47:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

Apply yapf to python files

Browse Source
This commit is contained in:
Matt Weiden
2019-12-18 19:18:31 -08:00
parent 42a8d45bd7
commit cdca128a01
43 changed files with 1143 additions and 678 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
server/.flake8 Normal file
View File

@@ -0,0 +1,12 @@
[flake8]
ignore =
# split before binary operator
W504,
# visually indented line with same indent as next logical line,
E129,
# whitespacek before ':'
E203,
# unexpected spaces around keyword / parameter equals
E251
max-line-length=120

2
server/Makefile
View File

@@ -1,6 +1,6 @@
.PHONY: fmt
fmt:
yapf -ir .
yapf -ipr .
.PHONY: lint
lint:

4
server/__main__.py
View File

@@ -5,11 +5,11 @@ if __package__ is None:
PKG_PATH = Path(__file__).parent
sys.path.insert(0, str(PKG_PATH.parent))
import server # noqa F401
import server # noqa F401
__package__ = PKG_PATH.name
# Main thing
from .cli.cli import cli # noqa F402
from .cli.cli import cli # noqa F402
cli()

6
server/app/app.py
View File

@@ -12,9 +12,13 @@ from server.app.web import webapp
class Server:
def __init__(self):
self.data = None
self.cache = Cache(config={"CACHE_TYPE": "simple", "CACHE_DEFAULT_TIMEOUT": 860_000})
self.cache = Cache(config={
"CACHE_TYPE": "simple",
"CACHE_DEFAULT_TIMEOUT": 860_000
})
self.app = None
def create_app(self):

31
server/app/driver/driver.py
View File

@@ -1,5 +1,4 @@
from abc import ABCMeta, abstractmethod
"""
Sort order for methods
1. Initialize
@@ -11,6 +10,7 @@ Sort order for methods
class CXGDriver(metaclass=ABCMeta):
def __init__(self, data_locator=None, args={}):
self.config = self._get_default_config()
self.config.update(args)
@@ -49,14 +49,29 @@ class CXGDriver(metaclass=ABCMeta):
@property
def features(self):
features = {
"cluster": {"available": False},
"layout": {"obs": {"available": False}, "var": {"available": False}},
"diffexp": {"available": True, "interactiveLimit": 50000}
"cluster": {
"available": False
},
"layout": {
"obs": {
"available": False
},
"var": {
"available": False
}
},
"diffexp": {
"available": True,
"interactiveLimit": 50000
}
}
# TODO - Interactive limit should be generated from the actual available methods see GH issue #94
if self.config["layout"]:
# TODO handle "var" when gene layout becomes available
features["layout"]["obs"] = {"available": True, "interactiveLimit": 50000}
features["layout"]["obs"] = {
"available": True,
"interactiveLimit": 50000
}
return features
@abstractmethod
@@ -92,7 +107,11 @@ class CXGDriver(metaclass=ABCMeta):
pass
@abstractmethod
def diffexp_topN(self, obsFilter1, obsFilter2, top_n=None, interactive_limit=None):
def diffexp_topN(self,
obsFilter1,
obsFilter2,
top_n=None,
interactive_limit=None):
"""
Computes the top N differentially expressed variables between two observation sets. If mode
is "TOP_N", then stats for the top N

145
server/app/rest_api/rest.py
View File

@@ -8,13 +8,9 @@ from flask_restful import Api, Resource
from server import __version__ as cellxgene_version
from anndata import __version__ as anndata_version
from server.app.util.constants import (
Axis,
DiffExpMode,
JSON_NaN_to_num_warning_msg,
CXGUID,
CXG_ANNO_COLLECTION
)
from server.app.util.constants import (Axis, DiffExpMode,
JSON_NaN_to_num_warning_msg, CXGUID,
CXG_ANNO_COLLECTION)
from server.app.util.errors import (
FilterError,
InteractiveError,
@@ -25,15 +21,20 @@ from server.app.util.errors import (
class SchemaAPI(Resource):
def get(self):
cxguid = get_userid(session)
anno_collection = get_anno_collection(session)
return make_response(
jsonify({"schema": current_app.data.get_schema(uid=cxguid, collection=anno_collection)}), HTTPStatus.OK
)
jsonify({
"schema":
current_app.data.get_schema(uid=cxguid,
collection=anno_collection)
}), HTTPStatus.OK)
class ConfigAPI(Resource):
def get(self):
cxguid = get_userid(session)
anno_collection = get_anno_collection(session)
@@ -69,7 +70,8 @@ class ConfigAPI(Resource):
"about-dataset": current_app.config["ABOUT_DATASET"]
},
"parameters": {
**current_app.data.get_config_parameters(uid=cxguid, collection=anno_collection)
**current_app.data.get_config_parameters(uid=cxguid,
collection=anno_collection)
},
"library_versions": {
"cellxgene": cellxgene_version,
@@ -82,42 +84,48 @@ class ConfigAPI(Resource):
class AnnotationsObsAPI(Resource):
def get(self):
fields = request.args.getlist("annotation-name", None)
preferred_mimetype = request.accept_mimetypes.best_match(
["application/octet-stream"]
)
["application/octet-stream"])
cxguid = get_userid(session)
anno_collection = get_anno_collection(session)
try:
if preferred_mimetype == "application/octet-stream":
fbs = current_app.data.annotation_to_fbs_matrix("obs", fields, uid=cxguid, collection=anno_collection)
return make_response(fbs,
HTTPStatus.OK,
{"Content-Type": "application/octet-stream"})
fbs = current_app.data.annotation_to_fbs_matrix(
"obs", fields, uid=cxguid, collection=anno_collection)
return make_response(
fbs, HTTPStatus.OK,
{"Content-Type": "application/octet-stream"})
else:
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
return make_response(
f"Unsupported MIME type '{request.accept_mimetypes}'",
HTTPStatus.NOT_ACCEPTABLE)
except KeyError:
return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST)
return make_response(f"Error bad key in {fields}",
HTTPStatus.BAD_REQUEST)
except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
def put(self):
cxguid = get_userid(session)
anno_collection = request.args.get("annotation-collection-name", default=None)
anno_collection = request.args.get("annotation-collection-name",
default=None)
if anno_collection is not None:
if not is_safe_collection_name(anno_collection):
return make_response(f"Error, bad annotation collection name", HTTPStatus.BAD_REQUEST)
return make_response(f"Error, bad annotation collection name",
HTTPStatus.BAD_REQUEST)
set_anno_collection(session, anno_collection)
else:
anno_collection = get_anno_collection(session)
try:
fbs = request.get_data()
res = current_app.data.annotation_put_fbs("obs", fbs, uid=cxguid, collection=anno_collection)
return make_response(
res, HTTPStatus.OK, {"Content-Type": "application/json"}
)
res = current_app.data.annotation_put_fbs(
"obs", fbs, uid=cxguid, collection=anno_collection)
return make_response(res, HTTPStatus.OK,
{"Content-Type": "application/json"})
except (ValueError, DisabledFeatureError, KeyError) as e:
return make_response(str(e), HTTPStatus.BAD_REQUEST)
except Exception as e:
@@ -125,41 +133,44 @@ class AnnotationsObsAPI(Resource):
class AnnotationsVarAPI(Resource):
def get(self):
fields = request.args.getlist("annotation-name", None)
preferred_mimetype = request.accept_mimetypes.best_match(
["application/octet-stream"]
)
["application/octet-stream"])
try:
if preferred_mimetype == "application/octet-stream":
return make_response(current_app.data.annotation_to_fbs_matrix("var", fields),
HTTPStatus.OK,
{"Content-Type": "application/octet-stream"})
return make_response(
current_app.data.annotation_to_fbs_matrix("var", fields),
HTTPStatus.OK, {"Content-Type": "application/octet-stream"})
else:
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
return make_response(
f"Unsupported MIME type '{request.accept_mimetypes}'",
HTTPStatus.NOT_ACCEPTABLE)
except KeyError:
return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST)
return make_response(f"Error bad key in {fields}",
HTTPStatus.BAD_REQUEST)
except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
class DataVarAPI(Resource):
def put(self):
preferred_mimetype = request.accept_mimetypes.best_match(
["application/octet-stream"]
)
["application/octet-stream"])
try:
if preferred_mimetype == "application/octet-stream":
filter_json = request.get_json()
filter = filter_json["filter"] if filter_json else None
return make_response(
current_app.data.data_frame_to_fbs_matrix(
filter, axis=Axis.VAR
),
HTTPStatus.OK,
{"Content-Type": "application/octet-stream"})
current_app.data.data_frame_to_fbs_matrix(filter,
axis=Axis.VAR),
HTTPStatus.OK, {"Content-Type": "application/octet-stream"})
else:
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
return make_response(
f"Unsupported MIME type '{request.accept_mimetypes}'",
HTTPStatus.NOT_ACCEPTABLE)
except FilterError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
except ValueError as e:
@@ -167,43 +178,39 @@ class DataVarAPI(Resource):
class DiffExpObsAPI(Resource):
def post(self):
args = request.get_json()
# confirm mode is present and legal
try:
mode = DiffExpMode(args["mode"])
except KeyError:
return make_response("Error: mode is required", HTTPStatus.BAD_REQUEST)
return make_response("Error: mode is required",
HTTPStatus.BAD_REQUEST)
except ValueError:
return make_response(
f"Error: invalid mode option {args['mode']}", HTTPStatus.BAD_REQUEST
)
return make_response(f"Error: invalid mode option {args['mode']}",
HTTPStatus.BAD_REQUEST)
# Validate filters
if mode == DiffExpMode.VAR_FILTER or "varFilter" in args:
# not NOT_IMPLEMENTED
return make_response(
"mode=varfilter not implemented", HTTPStatus.NOT_IMPLEMENTED
)
return make_response("mode=varfilter not implemented",
HTTPStatus.NOT_IMPLEMENTED)
if mode == DiffExpMode.TOP_N and "count" not in args:
return make_response(
"mode=topN requires a count parameter", HTTPStatus.BAD_REQUEST
)
return make_response("mode=topN requires a count parameter",
HTTPStatus.BAD_REQUEST)
if "set1" not in args:
return make_response("set1 is required.", HTTPStatus.BAD_REQUEST)
if Axis.VAR in args["set1"]["filter"]:
return make_response(
"Var filter not allowed for set1", HTTPStatus.BAD_REQUEST
)
return make_response("Var filter not allowed for set1",
HTTPStatus.BAD_REQUEST)
# set2
if "set2" not in args:
return make_response(
"Set2 as inverse of set1 is not implemented", HTTPStatus.NOT_IMPLEMENTED
)
return make_response("Set2 as inverse of set1 is not implemented",
HTTPStatus.NOT_IMPLEMENTED)
if Axis.VAR in args["set2"]["filter"]:
return make_response(
"Var filter not allowed for set2", HTTPStatus.BAD_REQUEST
)
return make_response("Var filter not allowed for set2",
HTTPStatus.BAD_REQUEST)
set1_filter = args["set1"]["filter"]
set2_filter = args.get("set2", {"filter": {}})["filter"]
@@ -219,13 +226,13 @@ class DiffExpObsAPI(Resource):
count,
current_app.data.features["diffexp"]["interactiveLimit"],
)
return make_response(
diffexp, HTTPStatus.OK, {"Content-Type": "application/json"}
)
return make_response(diffexp, HTTPStatus.OK,
{"Content-Type": "application/json"})
except (ValueError, FilterError) as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
except InteractiveError:
return make_response("Non-interactive request", HTTPStatus.FORBIDDEN)
return make_response("Non-interactive request",
HTTPStatus.FORBIDDEN)
except JSONEncodingValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
@@ -235,17 +242,19 @@ class DiffExpObsAPI(Resource):
class LayoutObsAPI(Resource):
def get(self):
preferred_mimetype = request.accept_mimetypes.best_match(
["application/octet-stream"]
)
["application/octet-stream"])
try:
if preferred_mimetype == "application/octet-stream":
return make_response(current_app.data.layout_to_fbs_matrix(),
HTTPStatus.OK,
{"Content-Type": "application/octet-stream"})
return make_response(
current_app.data.layout_to_fbs_matrix(), HTTPStatus.OK,
{"Content-Type": "application/octet-stream"})
else:
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
return make_response(
f"Unsupported MIME type '{request.accept_mimetypes}'",
HTTPStatus.NOT_ACCEPTABLE)
except PrepareError as e:
return make_response(e.message, HTTPStatus.INTERNAL_SERVER_ERROR)
except ValueError as e:

17
server/app/scanpy_engine/diffexp.py
View File

@@ -32,8 +32,8 @@ def _mean_var_n(X):
v = sumsq / (n - 1)
if fp_err_occurred:
mean[np.isfinite(mean) == False] = 0 # noqa: E712
v[np.isfinite(v) == False] = 0 # noqa: E712
mean[np.isfinite(mean) == False] = 0 # noqa: E712
v[np.isfinite(v) == False] = 0 # noqa: E712
return mean, v, n
@@ -76,7 +76,7 @@ def diffexp_ttest(adata, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
# degrees of freedom for Welch's t-test
with np.errstate(divide="ignore", invalid="ignore"):
dof = sum_vn ** 2 / (vnA ** 2 / (nA - 1) + vnB ** 2 / (nB - 1))
dof = sum_vn**2 / (vnA**2 / (nA - 1) + vnB**2 / (nB - 1))
dof[np.isnan(dof)] = 1
# Welch's t-test score calculation
@@ -93,13 +93,15 @@ def diffexp_ttest(adata, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
logfoldchanges = np.log2(np.abs((meanA + 1e-9) / (meanB + 1e-9)))
# find all with lfc > cutoff
lfc_above_cutoff_idx = np.nonzero(np.abs(logfoldchanges) > diffexp_lfc_cutoff)[0]
lfc_above_cutoff_idx = np.nonzero(
np.abs(logfoldchanges) > diffexp_lfc_cutoff)[0]
stats_to_sort = np.abs(tscores)
# derive sort order
if lfc_above_cutoff_idx.shape[0] > top_n:
# partition top N
rel_t_partition = np.argpartition(stats_to_sort[lfc_above_cutoff_idx], -top_n)[-top_n:]
rel_t_partition = np.argpartition(stats_to_sort[lfc_above_cutoff_idx],
-top_n)[-top_n:]
t_partition = lfc_above_cutoff_idx[rel_t_partition]
# sort the top N partition
rel_sort_order = np.argsort(stats_to_sort[t_partition])[::-1]
@@ -117,5 +119,8 @@ def diffexp_ttest(adata, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
pvals_adj_top_n = pvals_adj[sort_order]
# varIndex, logfoldchange, pval, pval_adj
result = [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in range(top_n)]
result = [[
sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i],
pvals_adj_top_n[i]
] for i in range(top_n)]
return result

15
server/app/scanpy_engine/labels.py
View File

@@ -8,8 +8,13 @@ import pandas as pd
def read_labels(fname):
if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0:
return pd.read_csv(fname, dtype='category', index_col=0, header=0, comment='#')
if fname is not None and os.path.exists(
fname) and os.path.getsize(fname) > 0:
return pd.read_csv(fname,
dtype='category',
index_col=0,
header=0,
comment='#')
else:
return pd.DataFrame()
@@ -47,13 +52,15 @@ def backup(fname, backup_dir, max_backups=9):
fname_base_root, fname_base_ext = os.path.splitext(fname_base)
# don't use ISO standard time format, as it contains characters illegal on some filesytems.
nowish = datetime.now().strftime('%Y-%m-%dT%H-%M-%S')
backup_fname = os.path.join(backup_dir, f"{fname_base_root}-{nowish}{fname_base_ext}")
backup_fname = os.path.join(backup_dir,
f"{fname_base_root}-{nowish}{fname_base_ext}")
if os.path.exists(backup_fname):
os.remove(backup_fname)
os.rename(fname, backup_fname)
# prune the backup_dir to max number of backup files, keeping the most recent backups
backups = list(filter(lambda s: s.startswith(fname_base_root), os.listdir(backup_dir)))
backups = list(
filter(lambda s: s.startswith(fname_base_root), os.listdir(backup_dir)))
excess_count = len(backups) - max_backups
if excess_count > 0:
backups.sort()

4
server/app/scanpy_engine/matrix_proxy.py
View File

@@ -1,6 +1,4 @@
from server.app.util.matrix_proxy import MatrixProxyView, ArrayProxyView
"""
AnnData/h5py are inconsistent in the API supported by various types of
X matrices. Sometimes you get a fully ndarray, sometims a Scipy sparse
@@ -17,6 +15,7 @@ class ArrayProxyView_anndata_h5py(ArrayProxyView):
override to handle sparse getitem semantics, which differ
from numpy.
"""
def toarray(self):
""" sadly, sparse indexing doesn't drop dimensions like numpy! """
arr = self.m[self._index[0], self._index[1]]
@@ -30,6 +29,7 @@ class MatrixProxy_anndata_h5py(MatrixProxyView):
AnnData sparse array stored in H5AD, or proxies for backed data.
None of these handle indexing very well, so we plop a proxy on top.
"""
@classmethod
def __supports__(cls):
return ("anndata.h5py.h5sparse.SparseDataset",

192
server/app/scanpy_engine/scanpy_engine.py
View File

@@ -37,6 +37,7 @@ def has_method(o, name):
class ScanpyEngine(CXGDriver):
def __init__(self, data_locator=None, args={}):
super().__init__(data_locator, args)
# lock used to protect label file write ops
@@ -75,7 +76,8 @@ class ScanpyEngine(CXGDriver):
if self.config["annotations"]:
if uid is not None:
params.update({
"annotations-user-data-idhash": self.get_userdata_idhash(uid)
"annotations-user-data-idhash":
self.get_userdata_idhash(uid)
})
if self.config['annotations_file'] is not None:
# user has hard-wired the name of the annotation data collection
@@ -119,7 +121,8 @@ class ScanpyEngine(CXGDriver):
"""
self.original_obs_index = self.data.obs.index
for (ax_name, config_name) in ((Axis.OBS, "obs_names"), (Axis.VAR, "var_names")):
for (ax_name, config_name) in ((Axis.OBS, "obs_names"), (Axis.VAR,
"var_names")):
name = self.config[config_name]
df_axis = getattr(self.data, str(ax_name))
if name is None:
@@ -128,8 +131,7 @@ class ScanpyEngine(CXGDriver):
raise KeyError(
f"Values in {ax_name}.index must be unique. "
"Please prepare data to contain unique index values, or specify an "
"alternative with --{ax_name}-name."
)
"alternative with --{ax_name}-name.")
name = self._create_unique_column_name(df_axis.columns, "name_")
self.config[config_name] = name
# reset index to simple range; alias name to point at the
@@ -141,8 +143,7 @@ class ScanpyEngine(CXGDriver):
if not df_axis[name].is_unique:
raise KeyError(
f"Values in {ax_name}.{name} must be unique. "
"Please prepare data to contain unique values."
)
"Please prepare data to contain unique values.")
df_axis.reset_index(drop=True, inplace=True)
else:
# user specified a non-existent column name
@@ -189,8 +190,7 @@ class ScanpyEngine(CXGDriver):
schema["categories"] = dtype.categories.tolist()
else:
raise TypeError(
f"Annotations of type {dtype} are unsupported by cellxgene."
)
f"Annotations of type {dtype} are unsupported by cellxgene.")
return schema
@requires_data
@@ -211,7 +211,9 @@ class ScanpyEngine(CXGDriver):
"columns": []
}
},
"layout": {"obs": []}
"layout": {
"obs": []
}
}
for ax in Axis:
curr_axis = getattr(self.data, str(ax))
@@ -250,7 +252,8 @@ class ScanpyEngine(CXGDriver):
Used to create safe annotations output file names.
"""
id = (uid + self.data_locator.abspath()).encode()
idhash = base64.b32encode(blake2b(id, digest_size=5).digest()).decode('utf-8')
idhash = base64.b32encode(blake2b(
id, digest_size=5).digest()).decode('utf-8')
return idhash
def get_anno_fname(self, uid=None, collection=None):
@@ -265,7 +268,8 @@ class ScanpyEngine(CXGDriver):
if uid is None or collection is None:
return None
idhash = self.get_userdata_idhash(uid)
return os.path.join(self.get_anno_output_dir(), f"{collection}-{idhash}.csv")
return os.path.join(self.get_anno_output_dir(),
f"{collection}-{idhash}.csv")
def get_anno_output_dir(self):
""" return the current annotation output directory """
@@ -276,7 +280,8 @@ class ScanpyEngine(CXGDriver):
return self.config['annotations_output_dir']
if self.config['annotations_file']:
return os.path.dirname(os.path.abspath(self.config['annotations_file']))
return os.path.dirname(
os.path.abspath(self.config['annotations_file']))
return os.getcwd()
@@ -308,15 +313,14 @@ class ScanpyEngine(CXGDriver):
"https://github.com/theislab/scanpy_usage/blob/master/170505_seurat/info_h5ad.md to "
"learn more about this format. You may be able to convert your file into this format "
"using `cellxgene prepare`, please run `cellxgene prepare --help` for more "
"information."
)
"information.")
except MemoryError:
raise ScanpyFileError("Out of memory - file is too large for available memory.")
raise ScanpyFileError(
"Out of memory - file is too large for available memory.")
except Exception as e:
raise ScanpyFileError(
f"{e} - file not found or is inaccessible. File must be an .h5ad object. "
f"Please check your input and try again."
)
f"Please check your input and try again.")
@requires_data
def _validate_and_initialize(self):
@@ -338,7 +342,8 @@ class ScanpyEngine(CXGDriver):
# heuristic
n_values = self.data.shape[0] * self.data.shape[1]
if (n_values > 1e8 and self.config['backed'] is True) or (n_values > 5e8):
if (n_values > 1e8 and
self.config['backed'] is True) or (n_values > 5e8):
self.config.update({"diffexp_may_be_slow": True})
@requires_data
@@ -352,9 +357,15 @@ class ScanpyEngine(CXGDriver):
# handle default
if layouts is None or len(layouts) == 0:
# load default layouts from the data.
layouts = [key[2:] for key in self.data.obsm_keys() if type(key) == str and key.startswith("X_")]
layouts = [
key[2:]
for key in self.data.obsm_keys()
if type(key) == str and key.startswith("X_")
]
if len(layouts) == 0:
raise PrepareError(f"Unable to find any precomputed layouts within the dataset.")
raise PrepareError(
f"Unable to find any precomputed layouts within the dataset."
)
# remove invalid layouts
valid_layouts = []
@@ -364,7 +375,9 @@ class ScanpyEngine(CXGDriver):
if layout_name not in obsm_keys:
warnings.warn(f"Ignoring unknown layout name: {layout}.")
elif not self._is_valid_layout(self.data.obsm[layout_name]):
warnings.warn(f"Ignoring layout due to malformed shape or data type: {layout}")
warnings.warn(
f"Ignoring layout due to malformed shape or data type: {layout}"
)
else:
valid_layouts.append(layout)
@@ -381,22 +394,22 @@ class ScanpyEngine(CXGDriver):
* contains only finite values
"""
is_valid = type(arr) == np.ndarray and arr.dtype.kind in "fiu"
is_valid = is_valid and arr.shape[0] == self.data.n_obs and arr.shape[1] >= 2
is_valid = is_valid and arr.shape[
0] == self.data.n_obs and arr.shape[1] >= 2
is_valid = is_valid and np.all(np.isfinite(arr))
return is_valid
@requires_data
def _validate_data_types(self):
if sparse.isspmatrix(self.data.X) and not sparse.isspmatrix_csc(self.data.X):
if sparse.isspmatrix(
self.data.X) and not sparse.isspmatrix_csc(self.data.X):
warnings.warn(
f"Scanpy data matrix is sparse, but not a CSC (columnar) matrix. "
f"Performance may be improved by using CSC."
)
f"Performance may be improved by using CSC.")
if self.data.X.dtype != "float32":
warnings.warn(
f"Scanpy data matrix is in {self.data.X.dtype} format not float32. "
f"Precision may be truncated."
)
f"Precision may be truncated.")
for ax in Axis:
curr_axis = getattr(self.data, str(ax))
for ann in curr_axis:
@@ -410,11 +423,11 @@ class ScanpyEngine(CXGDriver):
if datatype in downcast_map:
warnings.warn(
f"Scanpy annotation {ax}:{ann} is in unsupported format: {datatype}. "
f"Data will be downcast to {downcast_map[datatype]}."
)
f"Data will be downcast to {downcast_map[datatype]}.")
if isinstance(datatype, CategoricalDtype):
category_num = len(curr_axis[ann].dtype.categories)
if category_num > 500 and category_num > self.config['max_category_items']:
if category_num > 500 and category_num > self.config[
'max_category_items']:
warnings.warn(
f"{str(ax).title()} annotation '{ann}' has {category_num} categories, this may be "
f"cumbersome or slow to display. We recommend setting the "
@@ -432,31 +445,41 @@ class ScanpyEngine(CXGDriver):
# all lables must have a name, which must be unique and not used in obs column names
if not labels.columns.is_unique:
raise KeyError(f"All column names specified in user annotations must be unique.")
raise KeyError(
f"All column names specified in user annotations must be unique."
)
# the label index must be unique, and must have same values the anndata obs index
if not labels.index.is_unique:
raise KeyError(f"All row index values specified in user annotations must be unique.")
raise KeyError(
f"All row index values specified in user annotations must be unique."
)
if not labels.index.equals(self.original_obs_index):
raise KeyError("Label file row index does not match H5AD file index. "
"Please ensure that column zero (0) in the label file contain the same "
"index values as the H5AD file.")
raise KeyError(
"Label file row index does not match H5AD file index. "
"Please ensure that column zero (0) in the label file contain the same "
"index values as the H5AD file.")
duplicate_columns = list(set(labels.columns) & set(self.data.obs.columns))
duplicate_columns = list(
set(labels.columns) & set(self.data.obs.columns))
if len(duplicate_columns) > 0:
raise KeyError(f"Labels file may not contain column names which overlap "
f"with h5ad obs columns {duplicate_columns}")
raise KeyError(
f"Labels file may not contain column names which overlap "
f"with h5ad obs columns {duplicate_columns}")
# labels must have same count as obs annotations
if labels.shape[0] != self.data.obs.shape[0]:
raise ValueError("Labels file must have same number of rows as h5ad file.")
raise ValueError(
"Labels file must have same number of rows as h5ad file.")
@staticmethod
def _annotation_filter_to_mask(filter, d_axis, count):
mask = np.ones((count,), dtype=bool)
for v in filter:
if d_axis[v["name"]].dtype.name in ["boolean", "category", "object"]:
if d_axis[v["name"]].dtype.name in [
"boolean", "category", "object"
]:
key_idx = np.in1d(getattr(d_axis, v["name"]), v["values"])
mask = np.logical_and(mask, key_idx)
else:
@@ -475,7 +498,7 @@ class ScanpyEngine(CXGDriver):
mask = np.zeros((count,), dtype=bool)
for i in filter:
if type(i) == list:
mask[i[0]: i[1]] = True
mask[i[0]:i[1]] = True
else:
mask[i] = True
return mask
@@ -485,14 +508,13 @@ class ScanpyEngine(CXGDriver):
mask = np.ones((count,), dtype=bool)
if "index" in filter:
mask = np.logical_and(
mask, ScanpyEngine._index_filter_to_mask(filter["index"], count)
)
mask,
ScanpyEngine._index_filter_to_mask(filter["index"], count))
if "annotation_value" in filter:
mask = np.logical_and(
mask,
ScanpyEngine._annotation_filter_to_mask(
filter["annotation_value"], d_axis, count
),
filter["annotation_value"], d_axis, count),
)
return mask
@@ -508,16 +530,18 @@ class ScanpyEngine(CXGDriver):
if filter is not None:
if Axis.OBS in filter:
obs_selector = self._axis_filter_to_mask(
filter["obs"], self.data.obs, self.data.n_obs
)
filter["obs"], self.data.obs, self.data.n_obs)
if Axis.VAR in filter:
var_selector = self._axis_filter_to_mask(
filter["var"], self.data.var, self.data.n_vars
)
filter["var"], self.data.var, self.data.n_vars)
return obs_selector, var_selector
@requires_data
def annotation_to_fbs_matrix(self, axis, fields=None, uid=None, collection=None):
def annotation_to_fbs_matrix(self,
axis,
fields=None,
uid=None,
collection=None):
if axis == Axis.OBS:
if self.config["annotations"]:
try:
@@ -525,8 +549,7 @@ class ScanpyEngine(CXGDriver):
except Exception as e:
raise ScanpyFileError(
f"Error while loading label file: {e}, File must be in the .csv format, please check "
f"your input and try again."
)
f"your input and try again.")
else:
labels = None
@@ -547,7 +570,9 @@ class ScanpyEngine(CXGDriver):
fname = self.get_anno_fname(uid, collection)
if not fname:
raise ScanpyFileError("Writable annotations - unable to determine file name for annotations")
raise ScanpyFileError(
"Writable annotations - unable to determine file name for annotations"
)
if axis != Axis.OBS:
raise ValueError("Only OBS dimension access is supported")
@@ -558,21 +583,27 @@ class ScanpyEngine(CXGDriver):
self._validate_label_data(new_label_df) # paranoia
# if any of the new column labels overlap with our existing labels, raise error
duplicate_columns = list(set(new_label_df.columns) & set(self.data.obs.columns))
duplicate_columns = list(
set(new_label_df.columns) & set(self.data.obs.columns))
if not new_label_df.columns.is_unique or len(duplicate_columns) > 0:
raise KeyError(f"Labels file may not contain column names which overlap "
f"with h5ad obs columns {duplicate_columns}")
raise KeyError(
f"Labels file may not contain column names which overlap "
f"with h5ad obs columns {duplicate_columns}")
# update our internal state and save it. Multi-threading often enabled,
# so treat this as a critical section.
with self.label_lock:
lastmod = self.data_locator.lastmodtime()
lastmodstr = "'unknown'" if lastmod is None else lastmod.isoformat(timespec="seconds")
lastmodstr = "'unknown'" if lastmod is None else lastmod.isoformat(
timespec="seconds")
header = f"# Annotations generated on {datetime.now().isoformat(timespec='seconds')} " \
f"using cellxgene version {cellxgene_version}\n" \
f"# Input data file was {self.data_locator.uri_or_path}, " \
f"which was last modified on {lastmodstr}\n"
write_labels(fname, new_label_df, header, backup_dir=self.get_anno_backup_dir(uid, collection))
write_labels(fname,
new_label_df,
header,
backup_dir=self.get_anno_backup_dir(uid, collection))
return jsonify_scanpy({"status": "OK"})
@@ -591,41 +622,48 @@ class ScanpyEngine(CXGDriver):
if axis != Axis.VAR:
raise ValueError("Only VAR dimension access is supported")
try:
obs_selector, var_selector = self._filter_to_mask(filter, use_slices=False)
obs_selector, var_selector = self._filter_to_mask(filter,
use_slices=False)
except (KeyError, IndexError, TypeError) as e:
raise FilterError(f"Error parsing filter: {e}") from e
if obs_selector is not None:
raise FilterError("filtering on obs unsupported")
# Currently only handles VAR dimension
X = MatrixProxy.create(self.data.X if var_selector is None
else self.data.X[:, var_selector])
return encode_matrix_fbs(X, col_idx=np.nonzero(var_selector)[0], row_idx=None)
X = MatrixProxy.create(
self.data.X if var_selector is None else self.data.X[:,
var_selector])
return encode_matrix_fbs(X,
col_idx=np.nonzero(var_selector)[0],
row_idx=None)
@requires_data
def diffexp_topN(self, obsFilterA, obsFilterB, top_n=None, interactive_limit=None):
def diffexp_topN(self,
obsFilterA,
obsFilterB,
top_n=None,
interactive_limit=None):
if Axis.VAR in obsFilterA or Axis.VAR in obsFilterB:
raise FilterError("Observation filters may not contain vaiable conditions")
raise FilterError(
"Observation filters may not contain vaiable conditions")
try:
obs_mask_A = self._axis_filter_to_mask(
obsFilterA["obs"], self.data.obs, self.data.n_obs
)
obs_mask_B = self._axis_filter_to_mask(
obsFilterB["obs"], self.data.obs, self.data.n_obs
)
obs_mask_A = self._axis_filter_to_mask(obsFilterA["obs"],
self.data.obs,
self.data.n_obs)
obs_mask_B = self._axis_filter_to_mask(obsFilterB["obs"],
self.data.obs,
self.data.n_obs)
except (KeyError, IndexError) as e:
raise FilterError(f"Error parsing filter: {e}") from e
if top_n is None:
top_n = DEFAULT_TOP_N
result = diffexp_ttest(
self.data, obs_mask_A, obs_mask_B, top_n, self.config['diffexp_lfc_cutoff']
)
result = diffexp_ttest(self.data, obs_mask_A, obs_mask_B, top_n,
self.config['diffexp_lfc_cutoff'])
try:
return jsonify_scanpy(result)
except ValueError:
raise JSONEncodingValueError(
"Error encoding differential expression to JSON"
)
"Error encoding differential expression to JSON")
@requires_data
def layout_to_fbs_matrix(self):
@@ -656,7 +694,9 @@ class ScanpyEngine(CXGDriver):
normalized_layout = normalized_layout + translate
normalized_layout = normalized_layout.astype(dtype=np.float32)
layout_data.append(pandas.DataFrame(normalized_layout, columns=[f"{layout}_0", f"{layout}_1"]))
layout_data.append(
pandas.DataFrame(normalized_layout,
columns=[f"{layout}_0", f"{layout}_1"]))
except ValueError as e:
raise PrepareError(

2
server/app/util/constants.py
View File

@@ -1,10 +1,10 @@
from enum import Enum
DEFAULT_TOP_N = 10
class AugmentedEnum(Enum):
def __hash__(self):
return self.value.__hash__()

7
server/app/util/data_locator.py
View File

@@ -27,7 +27,8 @@ class DataLocator():
def __init__(self, uri_or_path):
self.uri_or_path = uri_or_path
self.protocol, self.path = DataLocator._get_protocol_and_path(uri_or_path)
self.protocol, self.path = DataLocator._get_protocol_and_path(
uri_or_path)
# work-around for LocalFileSystem not treating file: and None as the same scheme/protocol
self.cname = self.path if self.protocol == 'file' else self.uri_or_path
# will throw RuntimeError if the protocol is unsupported
@@ -82,7 +83,8 @@ class DataLocator():
# if not local, create a tmp file system object to contain the data,
# and clean it up when done.
with self.open() as src, tempfile.NamedTemporaryFile(prefix="cellxgene_", delete=False) as tmp:
with self.open() as src, tempfile.NamedTemporaryFile(
prefix="cellxgene_", delete=False) as tmp:
tmp.write(src.read())
tmp.close()
src.close()
@@ -91,6 +93,7 @@ class DataLocator():
class LocalFilePath():
def __init__(self, tmp_path, delete=False):
self.tmp_path = tmp_path
self.delete = delete

24
server/app/util/fbs/NetEncoding/Column.py
View File

@@ -4,6 +4,7 @@
import flatbuffers
class Column(object):
__slots__ = ['_tab']
@@ -22,7 +23,8 @@ class Column(object):
def UType(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos)
return self._tab.Get(flatbuffers.number_types.Uint8Flags,
o + self._tab.Pos)
return 0
# Column
@@ -35,7 +37,19 @@ class Column(object):
return obj
return None
def ColumnStart(builder): builder.StartObject(2)
def ColumnAddUType(builder, uType): builder.PrependUint8Slot(0, uType, 0)
def ColumnAddU(builder, u): builder.PrependUOffsetTRelativeSlot(1, flatbuffers.number_types.UOffsetTFlags.py_type(u), 0)
def ColumnEnd(builder): return builder.EndObject()
def ColumnStart(builder):
builder.StartObject(2)
def ColumnAddUType(builder, uType):
builder.PrependUint8Slot(0, uType, 0)
def ColumnAddU(builder, u):
builder.PrependUOffsetTRelativeSlot(
1, flatbuffers.number_types.UOffsetTFlags.py_type(u), 0)
def ColumnEnd(builder):
return builder.EndObject()

28
server/app/util/fbs/NetEncoding/Float32Array.py
View File

@@ -4,6 +4,7 @@
import flatbuffers
class Float32Array(object):
__slots__ = ['_tab']
@@ -23,14 +24,17 @@ class Float32Array(object):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
a = self._tab.Vector(o)
return self._tab.Get(flatbuffers.number_types.Float32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
return self._tab.Get(
flatbuffers.number_types.Float32Flags,
a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
return 0
# Float32Array
def DataAsNumpy(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Float32Flags, o)
return self._tab.GetVectorAsNumpy(
flatbuffers.number_types.Float32Flags, o)
return 0
# Float32Array
@@ -40,7 +44,19 @@ class Float32Array(object):
return self._tab.VectorLen(o)
return 0
def Float32ArrayStart(builder): builder.StartObject(1)
def Float32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
def Float32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4)
def Float32ArrayEnd(builder): return builder.EndObject()
def Float32ArrayStart(builder):
builder.StartObject(1)
def Float32ArrayAddData(builder, data):
builder.PrependUOffsetTRelativeSlot(
0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
def Float32ArrayStartDataVector(builder, numElems):
return builder.StartVector(4, numElems, 4)
def Float32ArrayEnd(builder):
return builder.EndObject()

28
server/app/util/fbs/NetEncoding/Float64Array.py
View File

@@ -4,6 +4,7 @@
import flatbuffers
class Float64Array(object):
__slots__ = ['_tab']
@@ -23,14 +24,17 @@ class Float64Array(object):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
a = self._tab.Vector(o)
return self._tab.Get(flatbuffers.number_types.Float64Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 8))
return self._tab.Get(
flatbuffers.number_types.Float64Flags,
a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 8))
return 0
# Float64Array
def DataAsNumpy(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Float64Flags, o)
return self._tab.GetVectorAsNumpy(
flatbuffers.number_types.Float64Flags, o)
return 0
# Float64Array
@@ -40,7 +44,19 @@ class Float64Array(object):
return self._tab.VectorLen(o)
return 0
def Float64ArrayStart(builder): builder.StartObject(1)
def Float64ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
def Float64ArrayStartDataVector(builder, numElems): return builder.StartVector(8, numElems, 8)
def Float64ArrayEnd(builder): return builder.EndObject()
def Float64ArrayStart(builder):
builder.StartObject(1)
def Float64ArrayAddData(builder, data):
builder.PrependUOffsetTRelativeSlot(
0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
def Float64ArrayStartDataVector(builder, numElems):
return builder.StartVector(8, numElems, 8)
def Float64ArrayEnd(builder):
return builder.EndObject()

28
server/app/util/fbs/NetEncoding/Int32Array.py
View File

@@ -4,6 +4,7 @@
import flatbuffers
class Int32Array(object):
__slots__ = ['_tab']
@@ -23,14 +24,17 @@ class Int32Array(object):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
a = self._tab.Vector(o)
return self._tab.Get(flatbuffers.number_types.Int32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
return self._tab.Get(
flatbuffers.number_types.Int32Flags,
a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
return 0
# Int32Array
def DataAsNumpy(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Int32Flags, o)
return self._tab.GetVectorAsNumpy(
flatbuffers.number_types.Int32Flags, o)
return 0
# Int32Array
@@ -40,7 +44,19 @@ class Int32Array(object):
return self._tab.VectorLen(o)
return 0
def Int32ArrayStart(builder): builder.StartObject(1)
def Int32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
def Int32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4)
def Int32ArrayEnd(builder): return builder.EndObject()
def Int32ArrayStart(builder):
builder.StartObject(1)
def Int32ArrayAddData(builder, data):
builder.PrependUOffsetTRelativeSlot(
0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
def Int32ArrayStartDataVector(builder, numElems):
return builder.StartVector(4, numElems, 4)
def Int32ArrayEnd(builder):
return builder.EndObject()

28
server/app/util/fbs/NetEncoding/JSONEncodedArray.py
View File

@@ -4,6 +4,7 @@
import flatbuffers
class JSONEncodedArray(object):
__slots__ = ['_tab']
@@ -23,14 +24,17 @@ class JSONEncodedArray(object):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
a = self._tab.Vector(o)
return self._tab.Get(flatbuffers.number_types.Uint8Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 1))
return self._tab.Get(
flatbuffers.number_types.Uint8Flags,
a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 1))
return 0
# JSONEncodedArray
def DataAsNumpy(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Uint8Flags, o)
return self._tab.GetVectorAsNumpy(
flatbuffers.number_types.Uint8Flags, o)
return 0
# JSONEncodedArray
@@ -40,7 +44,19 @@ class JSONEncodedArray(object):
return self._tab.VectorLen(o)
return 0
def JSONEncodedArrayStart(builder): builder.StartObject(1)
def JSONEncodedArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
def JSONEncodedArrayStartDataVector(builder, numElems): return builder.StartVector(1, numElems, 1)
def JSONEncodedArrayEnd(builder): return builder.EndObject()
def JSONEncodedArrayStart(builder):
builder.StartObject(1)
def JSONEncodedArrayAddData(builder, data):
builder.PrependUOffsetTRelativeSlot(
0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
def JSONEncodedArrayStartDataVector(builder, numElems):
return builder.StartVector(1, numElems, 1)
def JSONEncodedArrayEnd(builder):
return builder.EndObject()

65
server/app/util/fbs/NetEncoding/Matrix.py
View File

@@ -4,6 +4,7 @@
import flatbuffers
class Matrix(object):
__slots__ = ['_tab']
@@ -22,14 +23,16 @@ class Matrix(object):
def NRows(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
return self._tab.Get(flatbuffers.number_types.Uint32Flags, o + self._tab.Pos)
return self._tab.Get(flatbuffers.number_types.Uint32Flags,
o + self._tab.Pos)
return 0
# Matrix
def NCols(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6))
if o != 0:
return self._tab.Get(flatbuffers.number_types.Uint32Flags, o + self._tab.Pos)
return self._tab.Get(flatbuffers.number_types.Uint32Flags,
o + self._tab.Pos)
return 0
# Matrix
@@ -56,7 +59,8 @@ class Matrix(object):
def ColIndexType(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10))
if o != 0:
return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos)
return self._tab.Get(flatbuffers.number_types.Uint8Flags,
o + self._tab.Pos)
return 0
# Matrix
@@ -73,7 +77,8 @@ class Matrix(object):
def RowIndexType(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14))
if o != 0:
return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos)
return self._tab.Get(flatbuffers.number_types.Uint8Flags,
o + self._tab.Pos)
return 0
# Matrix
@@ -86,13 +91,45 @@ class Matrix(object):
return obj
return None
def MatrixStart(builder): builder.StartObject(7)
def MatrixAddNRows(builder, nRows): builder.PrependUint32Slot(0, nRows, 0)
def MatrixAddNCols(builder, nCols): builder.PrependUint32Slot(1, nCols, 0)
def MatrixAddColumns(builder, columns): builder.PrependUOffsetTRelativeSlot(2, flatbuffers.number_types.UOffsetTFlags.py_type(columns), 0)
def MatrixStartColumnsVector(builder, numElems): return builder.StartVector(4, numElems, 4)
def MatrixAddColIndexType(builder, colIndexType): builder.PrependUint8Slot(3, colIndexType, 0)
def MatrixAddColIndex(builder, colIndex): builder.PrependUOffsetTRelativeSlot(4, flatbuffers.number_types.UOffsetTFlags.py_type(colIndex), 0)
def MatrixAddRowIndexType(builder, rowIndexType): builder.PrependUint8Slot(5, rowIndexType, 0)
def MatrixAddRowIndex(builder, rowIndex): builder.PrependUOffsetTRelativeSlot(6, flatbuffers.number_types.UOffsetTFlags.py_type(rowIndex), 0)
def MatrixEnd(builder): return builder.EndObject()
def MatrixStart(builder):
builder.StartObject(7)
def MatrixAddNRows(builder, nRows):
builder.PrependUint32Slot(0, nRows, 0)
def MatrixAddNCols(builder, nCols):
builder.PrependUint32Slot(1, nCols, 0)
def MatrixAddColumns(builder, columns):
builder.PrependUOffsetTRelativeSlot(
2, flatbuffers.number_types.UOffsetTFlags.py_type(columns), 0)
def MatrixStartColumnsVector(builder, numElems):
return builder.StartVector(4, numElems, 4)
def MatrixAddColIndexType(builder, colIndexType):
builder.PrependUint8Slot(3, colIndexType, 0)
def MatrixAddColIndex(builder, colIndex):
builder.PrependUOffsetTRelativeSlot(
4, flatbuffers.number_types.UOffsetTFlags.py_type(colIndex), 0)
def MatrixAddRowIndexType(builder, rowIndexType):
builder.PrependUint8Slot(5, rowIndexType, 0)
def MatrixAddRowIndex(builder, rowIndex):
builder.PrependUOffsetTRelativeSlot(
6, flatbuffers.number_types.UOffsetTFlags.py_type(rowIndex), 0)
def MatrixEnd(builder):
return builder.EndObject()

2
server/app/util/fbs/NetEncoding/TypedArray.py
View File

@@ -2,6 +2,7 @@
# namespace: NetEncoding
class TypedArray(object):
NONE = 0
Float32Array = 1
@@ -9,4 +10,3 @@ class TypedArray(object):
Uint32Array = 3
Float64Array = 4
JSONEncodedArray = 5

28
server/app/util/fbs/NetEncoding/Uint32Array.py
View File

@@ -4,6 +4,7 @@
import flatbuffers
class Uint32Array(object):
__slots__ = ['_tab']
@@ -23,14 +24,17 @@ class Uint32Array(object):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
a = self._tab.Vector(o)
return self._tab.Get(flatbuffers.number_types.Uint32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
return self._tab.Get(
flatbuffers.number_types.Uint32Flags,
a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
return 0
# Uint32Array
def DataAsNumpy(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Uint32Flags, o)
return self._tab.GetVectorAsNumpy(
flatbuffers.number_types.Uint32Flags, o)
return 0
# Uint32Array
@@ -40,7 +44,19 @@ class Uint32Array(object):
return self._tab.VectorLen(o)
return 0
def Uint32ArrayStart(builder): builder.StartObject(1)
def Uint32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
def Uint32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4)
def Uint32ArrayEnd(builder): return builder.EndObject()
def Uint32ArrayStart(builder):
builder.StartObject(1)
def Uint32ArrayAddData(builder, data):
builder.PrependUOffsetTRelativeSlot(
0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
def Uint32ArrayStartDataVector(builder, numElems):
return builder.StartVector(4, numElems, 4)
def Uint32ArrayEnd(builder):
return builder.EndObject()

40
server/app/util/fbs/matrix.py
View File

@@ -25,7 +25,8 @@ def CreateNumpyVector(builder, x):
"""CreateNumpyVector writes a numpy array into the buffer."""
if not isinstance(x, np.ndarray):
raise TypeError(f"non-numpy-ndarray passed to CreateNumpyVector ({type(x)}")
raise TypeError(
f"non-numpy-ndarray passed to CreateNumpyVector ({type(x)}")
if x.dtype.kind not in ['b', 'i', 'u', 'f']:
raise TypeError("numpy-ndarray holds elements of unsupported datatype")
@@ -46,7 +47,8 @@ def CreateNumpyVector(builder, x):
builder.head = int(builder.Head() - len)
# tobytes ensures c_contiguous ordering
builder.Bytes[builder.Head():builder.Head() + len] = x_little_endian.tobytes(order='C')
builder.Bytes[builder.Head():builder.Head() +
len] = x_little_endian.tobytes(order='C')
return builder.EndVector(x.size)
@@ -119,12 +121,10 @@ column_encoding_type_map = {
np.dtype(np.float64).str: (TypedArray.TypedArray.Float32Array, np.float32),
np.dtype(np.float32).str: (TypedArray.TypedArray.Float32Array, np.float32),
np.dtype(np.float16).str: (TypedArray.TypedArray.Float32Array, np.float32),
np.dtype(np.int8).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.int16).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.uint8).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.dtype(np.uint16).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
@@ -141,7 +141,6 @@ index_encoding_type_map = {
# array protocol string: ( array_type, as_type )
np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32)
}
@@ -192,7 +191,8 @@ def encode_matrix_fbs(matrix, row_idx=None, col_idx=None):
columns = []
for cidx in range(n_cols - 1, -1, -1):
# serialize the typed array
col = matrix.iloc[:, cidx] if isinstance(matrix, pd.DataFrame) else matrix[:, cidx]
col = matrix.iloc[:, cidx] if isinstance(
matrix, pd.DataFrame) else matrix[:, cidx]
typed_arr = serialize_typed_array(builder, col, column_encoding)
# serialize the Column union
@@ -218,12 +218,18 @@ def encode_matrix_fbs(matrix, row_idx=None, col_idx=None):
def deserialize_typed_array(tarr):
type_map = {
TypedArray.TypedArray.NONE: None,
TypedArray.TypedArray.Uint32Array: Uint32Array.Uint32Array,
TypedArray.TypedArray.Int32Array: Int32Array.Int32Array,
TypedArray.TypedArray.Float32Array: Float32Array.Float32Array,
TypedArray.TypedArray.Float64Array: Float64Array.Float64Array,
TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray
TypedArray.TypedArray.NONE:
None,
TypedArray.TypedArray.Uint32Array:
Uint32Array.Uint32Array,
TypedArray.TypedArray.Int32Array:
Int32Array.Int32Array,
TypedArray.TypedArray.Float32Array:
Float32Array.Float32Array,
TypedArray.TypedArray.Float64Array:
Float64Array.Float64Array,
TypedArray.TypedArray.JSONEncodedArray:
JSONEncodedArray.JSONEncodedArray
}
(u_type, u) = tarr
if u_type is TypedArray.TypedArray.NONE:
@@ -257,13 +263,16 @@ def decode_matrix_fbs(fbs):
columns_length = matrix.ColumnsLength()
columns_index = deserialize_typed_array((matrix.ColIndexType(), matrix.ColIndex()))
columns_index = deserialize_typed_array(
(matrix.ColIndexType(), matrix.ColIndex()))
if columns_index is None:
columns_index = range(0, n_cols)
# sanity checks
if len(columns_index) != n_cols or columns_length != n_cols:
raise ValueError("FBS column count does not match number of columns in underlying matrix")
raise ValueError(
"FBS column count does not match number of columns in underlying matrix"
)
columns_data = {}
columns_type = {}
@@ -277,7 +286,8 @@ def decode_matrix_fbs(fbs):
if col.UType() is TypedArray.TypedArray.JSONEncodedArray:
columns_type[columns_index[col_idx]] = "category"
df = pd.DataFrame.from_dict(data=columns_data).astype(columns_type, copy=False)
df = pd.DataFrame.from_dict(data=columns_data).astype(columns_type,
copy=False)
# more sanity checks
if not df.columns.is_unique or len(df.columns) != n_cols:

71
server/app/util/matrix_proxy.py
View File

@@ -2,7 +2,6 @@ import abc
from itertools import zip_longest
from copy import copy
import numpy as np
"""
cellxgene deals with a variety of matrix data types, many of which do
not support a consistent API. This framework allows proxies to be created
@@ -18,6 +17,7 @@ class _ArrayProxyBase(abc.ABC):
Private base class for array or matrix proxy. This summarizes
the interface used by the rest of cellxgene.
"""
@property
@abc.abstractmethod
def dtype(self):
@@ -59,7 +59,6 @@ class MatrixProxy(_ArrayProxyBase):
This class primarily provides the factory method and related support.
All other functionality is delegated to subclasses.
"""
"""
Registry of types to proxy class, where values are:
* None: unsupported
@@ -128,21 +127,25 @@ class MatrixProxyView(MatrixProxy):
"""
2D matrix view to a 2D matrix
"""
def __init__(self, arg1, shape=None, index=(),
transposed=False, copy=False):
def __init__(self,
arg1,
shape=None,
index=(),
transposed=False,
copy=False):
if not copy:
m = arg1
super().__init__(m)
if shape is None:
shape = m.shape
assert(len(shape) == 2)
assert (len(shape) == 2)
index = tuple(
map(lambda s_i:
slice(0, s_i[0], 1) if s_i[1] is None else s_i[1],
zip_longest(shape, index))
)
map(
lambda s_i: slice(0, s_i[0], 1)
if s_i[1] is None else s_i[1], zip_longest(shape, index)))
self._shape = shape
self._index = index
@@ -234,21 +237,29 @@ class MatrixProxyView(MatrixProxy):
NOTE: these follow the numpy rules for dimensionality reduction
when an integer index is specified.
"""
def _getitem_intXint(self, row, col):
return self.m[row, col]
def _getitem_intXslice(self, row, col):
shape = (_slice_length(col, self.m.shape[1]), )
return self.__class__.create_array(self.m, shape=shape, index=(row, col))
shape = (_slice_length(col, self.m.shape[1]),)
return self.__class__.create_array(self.m,
shape=shape,
index=(row, col))
def _getitem_sliceXint(self, row, col):
shape = (_slice_length(row, self.m.shape[0]), )
return self.__class__.create_array(self.m, shape=shape, index=(row, col))
shape = (_slice_length(row, self.m.shape[0]),)
return self.__class__.create_array(self.m,
shape=shape,
index=(row, col))
def _getitem_sliceXslice(self, row, col):
shape = (_slice_length(row, self.m.shape[0]),
_slice_length(col, self.m.shape[1]))
return self.__class__(self.m, shape=shape, index=(row, col), transposed=self.transposed)
return self.__class__(self.m,
shape=shape,
index=(row, col),
transposed=self.transposed)
def toarray(self):
arr = self.m[self._index]
@@ -261,22 +272,24 @@ class ArrayProxyView(_ArrayProxyBase):
"""
1D array view to a 2D matrix
"""
def __init__(self, arg1, shape=None, index=None, copy=False):
super().__init__()
if not copy:
m = arg1
# one index MUST be an integer and the other MUST be a slice
assert(len(index) == 2)
assert(all(isinstance(idx, INT_TYPES + (slice, )) for idx in index))
assert(isinstance(index[0], INT_TYPES) != isinstance(index[1], INT_TYPES))
assert (len(index) == 2)
assert (all(isinstance(idx, INT_TYPES + (slice,)) for idx in index))
assert (isinstance(index[0], INT_TYPES) != isinstance(
index[1], INT_TYPES))
if shape is None:
if isinstance(index[0], INT_TYPES):
shape = (m.shape[0], )
shape = (m.shape[0],)
else:
shape = (m.shape[1], )
assert(len(shape) == 1)
shape = (m.shape[1],)
assert (len(shape) == 1)
self._shape = shape
self.m = m
@@ -336,7 +349,7 @@ class ArrayProxyView(_ArrayProxyBase):
elif isinstance(col, slice):
return self._getitem_intXslice(row, col)
elif isinstance(row, slice):
assert(isinstance(col, INT_TYPES))
assert (isinstance(col, INT_TYPES))
return self._getitem_sliceXint(row, col)
raise IndexError("unsupported column index types")
@@ -345,11 +358,11 @@ class ArrayProxyView(_ArrayProxyBase):
return self.m[row, col]
def _getitem_intXslice(self, row, col):
shape = (_slice_length(col, self.m.shape[1]), )
shape = (_slice_length(col, self.m.shape[1]),)
return self.__class__(self.m, shape=shape, index=(row, col))
def _getitem_sliceXint(self, row, col):
shape = (_slice_length(row, self.m.shape[0]), )
shape = (_slice_length(row, self.m.shape[0]),)
return self.__class__(self.m, shape=shape, index=(row, col))
def toarray(self):
@@ -358,7 +371,7 @@ class ArrayProxyView(_ArrayProxyBase):
def _unpack_index(index, shape):
if not isinstance(index, tuple):
index = (index, )
index = (index,)
if len(shape) < len(index):
raise IndexError("invalid index dimensionality - must be 2")
@@ -366,7 +379,7 @@ def _unpack_index(index, shape):
for shp, idx in zip_longest(shape, index):
idx = slice(None) if idx is None else idx
idx = _slice_defaults(idx, shp) if isinstance(idx, slice) else idx
unpacked += (idx, )
unpacked += (idx,)
return unpacked
@@ -376,7 +389,7 @@ def _slice_slice(outer, outer_len, inner, inner_len):
slice a slice - we take advantage of Python 3 range's support
for indexing.
"""
assert(outer_len >= inner_len)
assert (outer_len >= inner_len)
outer_rng = range(*outer.indices(outer_len))
rng = outer_rng[inner]
start, stop, step = rng.start, rng.stop, rng.step
@@ -387,8 +400,8 @@ def _slice_slice(outer, outer_len, inner, inner_len):
def _range_length(start, stop, step):
""" return length of range """
assert(step != 0)
assert(start is not None and stop is not None and step is not None)
assert (step != 0)
assert (start is not None and stop is not None and step is not None)
if step > 0 and start < stop:
return 1 + (stop - 1 - start) // step
elif step < 0 and start > stop:
@@ -404,7 +417,7 @@ def _slice_length(s, length):
def _slice_defaults(s, length):
""" apply slice defaulting conventions """
assert(length >= 0)
assert (length >= 0)
step = 1 if s.step is None else s.step

6
server/app/util/utils.py
View File

@@ -7,6 +7,7 @@ from server.app.util.errors import DriverError
class Float32JSONEncoder(json.JSONEncoder):
def __init__(self, *args, **kwargs):
"""
NaN/Infinities are illegal in standard JSON. Python extends JSON with
@@ -35,9 +36,12 @@ def jsonify_scanpy(data):
def requires_data(func):
@wraps(func)
def wrapped_function(self, *args, **kwargs):
if self.data is None:
raise DriverError(f"error data must be loaded before you call {func.__name__}")
raise DriverError(
f"error data must be loaded before you call {func.__name__}")
return func(self, *args, **kwargs)
return wrapped_function

8
server/app/web/webapp.py
View File

@@ -1,7 +1,6 @@
import os
from flask import Blueprint, render_template, send_from_directory, current_app
bp = Blueprint("webapp", __name__, template_folder="templates")
@@ -9,9 +8,12 @@ bp = Blueprint("webapp", __name__, template_folder="templates")
def index():
dataset_title = current_app.config["DATASET_TITLE"]
scripts = current_app.config["SCRIPTS"]
return render_template("index.html", datasetTitle=dataset_title, SCRIPTS=scripts)
return render_template("index.html",
datasetTitle=dataset_title,
SCRIPTS=scripts)
@bp.route("/favicon.png")
def favicon():
return send_from_directory(os.path.join(bp.root_path, "static/img/"), "favicon.png")
return send_from_directory(os.path.join(bp.root_path, "static/img/"),
"favicon.png")

9
server/cli/cli.py
View File

@@ -10,11 +10,10 @@ from .prepare import prepare
context_settings=dict(max_content_width=85,
help_option_names=['-h', '--help']))
@click.help_option("--help", "-h", help="Show this message and exit.")
@click.version_option(
version="0.13.0",
prog_name="cellxgene",
message="[%(prog)s] Version %(version)s",
help="Show the software version and exit.")
@click.version_option(version="0.13.0",
prog_name="cellxgene",
message="[%(prog)s] Version %(version)s",
help="Show the software version and exit.")
def cli():
pass

214
server/cli/launch.py
View File

@@ -17,7 +17,7 @@ from server.utils.utils import find_available_port, is_port_available, sort_opti
from server.app.util.data_locator import DataLocator
# anything bigger than this will generate a special message
BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20 # 100MB
BIG_FILE_SIZE_THRESHOLD = 100 * 2**20 # 100MB
def common_args(func):
@@ -25,16 +25,14 @@ def common_args(func):
Decorator to contain CLI args that will be common to both CLI and GUI: title and engine args.
"""
@click.option(
"--title",
"-t",
metavar="<text>",
help="Title to display. If omitted will use file name.")
@click.option(
"--about",
metavar="<URL>",
help="URL providing more information about the dataset "
"(hint: must be a fully specified absolute URL).")
@click.option("--title",
"-t",
metavar="<text>",
help="Title to display. If omitted will use file name.")
@click.option("--about",
metavar="<URL>",
help="URL providing more information about the dataset "
"(hint: must be a fully specified absolute URL).")
@click.option(
"--embedding",
"-e",
@@ -42,69 +40,80 @@ def common_args(func):
multiple=True,
show_default=False,
metavar="<text>",
help="Embedding name, eg, 'umap'. Repeat option for multiple embeddings. Defaults to all."
help=
"Embedding name, eg, 'umap'. Repeat option for multiple embeddings. Defaults to all."
)
@click.option(
"--obs-names",
"-obs",
default=None,
metavar="<text>",
help="Name of annotation field to use for observations. If not specified cellxgene will use the the obs index.")
help=
"Name of annotation field to use for observations. If not specified cellxgene will use the the obs index."
)
@click.option(
"--var-names",
"-var",
default=None,
metavar="<text>",
help="Name of annotation to use for variables. If not specified cellxgene will use the the var index.")
help=
"Name of annotation to use for variables. If not specified cellxgene will use the the var index."
)
@click.option(
"--max-category-items",
default=1000,
metavar="<integer>",
show_default=True,
help="Will not display categories with more distinct values than specified.",)
help=
"Will not display categories with more distinct values than specified.",
)
@click.option(
"--diffexp-lfc-cutoff",
"-de",
default=0.01,
show_default=True,
metavar="<float>",
help="Minimum log fold change threshold for differential expression.",)
@click.option(
"--experimental-annotations",
is_flag=True,
default=False,
show_default=True,
help="Enable user annotation of data."
help="Minimum log fold change threshold for differential expression.",
)
@click.option("--experimental-annotations",
is_flag=True,
default=False,
show_default=True,
help="Enable user annotation of data.")
@click.option(
"--experimental-annotations-file",
default=None,
show_default=True,
multiple=False,
metavar="<path>",
help="CSV file to initialize editing of existing annotations; will be altered in-place. "
"Incompatible with --annotations-output-dir.",)
help=
"CSV file to initialize editing of existing annotations; will be altered in-place. "
"Incompatible with --annotations-output-dir.",
)
@click.option(
"--experimental-annotations-output-dir",
default=None,
show_default=False,
multiple=False,
metavar="<directory path>",
help="Directory of where to save output annotations; filename will be specified in the application. "
"Incompatible with --annotations-input-file.",)
help=
"Directory of where to save output annotations; filename will be specified in the application. "
"Incompatible with --annotations-input-file.",
)
@click.option(
"--backed",
"-b",
is_flag=True,
default=False,
show_default=False,
help="Load data in file-backed mode. This may save memory, but may result in slower overall performance.")
@click.option(
"--disable-diffexp",
is_flag=True,
default=False,
show_default=False,
help="Disable on-demand differential expression.")
help=
"Load data in file-backed mode. This may save memory, but may result in slower overall performance."
)
@click.option("--disable-diffexp",
is_flag=True,
default=False,
show_default=False,
help="Disable on-demand differential expression.")
@functools.wraps(func)
def wrapper(*args, **kwargs):
return func(*args, **kwargs)
@@ -112,9 +121,11 @@ def common_args(func):
return wrapper
def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffexp_lfc_cutoff,
experimental_annotations, experimental_annotations_file,
experimental_annotations_output_dir, backed, disable_diffexp):
def parse_engine_args(embedding, obs_names, var_names, max_category_items,
diffexp_lfc_cutoff, experimental_annotations,
experimental_annotations_file,
experimental_annotations_output_dir, backed,
disable_diffexp):
annotations_file = experimental_annotations_file if experimental_annotations else None
annotations_output_dir = experimental_annotations_output_dir if experimental_annotations else None
return {
@@ -132,9 +143,11 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffe
@sort_options
@click.command(short_help="Launch the cellxgene data viewer. "
"Run `cellxgene launch --help` for more information.",
options_metavar="<options>",)
@click.command(
short_help="Launch the cellxgene data viewer. "
"Run `cellxgene launch --help` for more information.",
options_metavar="<options>",
)
@click.argument("data", nargs=1, metavar="<path to data file>", required=True)
@click.option(
"--verbose",
@@ -142,7 +155,8 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffe
is_flag=True,
default=False,
show_default=True,
help="Provide verbose output, including warnings and all server requests.",)
help="Provide verbose output, including warnings and all server requests.",
)
@click.option(
"--debug",
"-d",
@@ -150,7 +164,8 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffe
default=False,
show_default=True,
help="Run in debug mode. This is helpful for cellxgene developers, "
"or when you want more information about an error condition.",)
"or when you want more information about an error condition.",
)
@click.option(
"--open",
"-o",
@@ -158,19 +173,24 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffe
is_flag=True,
default=False,
show_default=True,
help="Open web browser after launch.",)
help="Open web browser after launch.",
)
@click.option(
"--port",
"-p",
metavar="<port>",
show_default=True,
help="Port to run server on. If not specified cellxgene will find an available port.",)
help=
"Port to run server on. If not specified cellxgene will find an available port.",
)
@click.option(
"--host",
metavar="<IP address>",
default="127.0.0.1",
show_default=False,
help="Host IP address. By default cellxgene will use localhost (e.g. 127.0.0.1).")
help=
"Host IP address. By default cellxgene will use localhost (e.g. 127.0.0.1)."
)
@click.option(
"--scripts",
"-s",
@@ -178,31 +198,15 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffe
multiple=True,
metavar="<text>",
help="Additional script files to include in HTML page. If not specified, "
"no additional script files will be included.",
show_default=False,)
"no additional script files will be included.",
show_default=False,
)
@click.help_option("--help", "-h", help="Show this message and exit.")
@common_args
def launch(
data,
verbose,
debug,
open_browser,
port,
host,
embedding,
obs_names,
var_names,
max_category_items,
diffexp_lfc_cutoff,
title,
scripts,
about,
experimental_annotations,
experimental_annotations_file,
experimental_annotations_output_dir,
backed,
disable_diffexp
):
def launch(data, verbose, debug, open_browser, port, host, embedding, obs_names,
var_names, max_category_items, diffexp_lfc_cutoff, title, scripts,
about, experimental_annotations, experimental_annotations_file,
experimental_annotations_output_dir, backed, disable_diffexp):
"""Launch the cellxgene data viewer.
This web app lets you explore single-cell expression data.
Data must be in a format that cellxgene expects.
@@ -217,17 +221,17 @@ def launch(
> cellxgene launch <url>"""
e_args = parse_engine_args(embedding, obs_names, var_names, max_category_items,
diffexp_lfc_cutoff,
e_args = parse_engine_args(embedding, obs_names, var_names,
max_category_items, diffexp_lfc_cutoff,
experimental_annotations,
experimental_annotations_file,
experimental_annotations_output_dir,
backed,
experimental_annotations_output_dir, backed,
disable_diffexp)
try:
data_locator = DataLocator(data)
except RuntimeError as re:
raise click.ClickException(f"Unable to access data at {data}. {str(re)}")
raise click.ClickException(
f"Unable to access data at {data}. {str(re)}")
# Startup message
click.echo("[cellxgene] Starting the CLI...")
@@ -244,7 +248,8 @@ def launch(
raise click.FileError(data, hint="data is not a file")
name, extension = splitext(data)
if extension != ".h5ad":
raise click.FileError(basename(data), hint="file type must be .h5ad")
raise click.FileError(basename(data),
hint="file type must be .h5ad")
if debug:
verbose = True
@@ -266,7 +271,9 @@ def launch(
security risk by including the --scripts flag. Make sure you trust the scripts that you are including.
""")
scripts_pretty = ", ".join(scripts)
click.confirm(f"Are you sure you want to inject these scripts: {scripts_pretty}?", abort=True)
click.confirm(
f"Are you sure you want to inject these scripts: {scripts_pretty}?",
abort=True)
if not title:
file_parts = splitext(basename(data))
@@ -274,7 +281,9 @@ def launch(
if port:
if debug:
raise click.ClickException("--port and --debug may not be used together (try --verbose for error logging).")
raise click.ClickException(
"--port and --debug may not be used together (try --verbose for error logging)."
)
if not is_port_available(host, int(port)):
raise click.ClickException(
f"The port selected {port} is in use, please specify an open port using the --port flag."
@@ -284,27 +293,36 @@ def launch(
if not experimental_annotations:
if experimental_annotations_file is not None:
click.echo("Warning: --experimental-annotations-file ignored as --annotations not enabled.")
click.echo(
"Warning: --experimental-annotations-file ignored as --annotations not enabled."
)
if experimental_annotations_output_dir is not None:
click.echo("Warning: --experimental-annotations-output-dir ignored as --annotations not enabled.")
click.echo(
"Warning: --experimental-annotations-output-dir ignored as --annotations not enabled."
)
else:
if experimental_annotations_file is not None and experimental_annotations_output_dir is not None:
raise click.ClickException("--experimental-annotations-file and --experimental-annotations-output-dir "
"may not be used together.")
raise click.ClickException(
"--experimental-annotations-file and --experimental-annotations-output-dir "
"may not be used together.")
if experimental_annotations_file is not None:
lf_name, lf_ext = splitext(experimental_annotations_file)
if lf_ext and lf_ext != ".csv":
raise click.FileError(basename(experimental_annotations_file), hint="annotation file type must be .csv")
raise click.FileError(basename(experimental_annotations_file),
hint="annotation file type must be .csv")
if experimental_annotations_output_dir is not None and not isdir(experimental_annotations_output_dir):
if experimental_annotations_output_dir is not None and not isdir(
experimental_annotations_output_dir):
try:
mkdir(experimental_annotations_output_dir)
except OSError:
raise click.ClickException("Unable to create directory specified by "
"--experimental-annotations-output-dir")
raise click.ClickException(
"Unable to create directory specified by "
"--experimental-annotations-output-dir")
if about:
def url_check(url):
try:
result = urlparse(url)
@@ -316,7 +334,9 @@ def launch(
return False
if not url_check(about):
raise click.ClickException("Must provide an absolute URL for --about. (Example format: http://example.com)")
raise click.ClickException(
"Must provide an absolute URL for --about. (Example format: http://example.com)"
)
# Setup app
cellxgene_url = f"http://{host}:{port}"
@@ -335,14 +355,18 @@ def launch(
# if a big file, let the user know it may take a while to load.
if file_size > BIG_FILE_SIZE_THRESHOLD:
click.echo(f"[cellxgene] Loading data from {basename(data)}, this may take a while...")
click.echo(
f"[cellxgene] Loading data from {basename(data)}, this may take a while..."
)
else:
click.echo(f"[cellxgene] Loading data from {basename(data)}.")
from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
try:
server.attach_data(ScanpyEngine(data_locator, e_args), title=title, about=about)
server.attach_data(ScanpyEngine(data_locator, e_args),
title=title,
about=about)
except ScanpyFileError as e:
raise click.ClickException(f"{e}")
@@ -351,10 +375,14 @@ def launch(
f"running differential expression may take longer or fail.")
if open_browser:
click.echo(f"[cellxgene] Launching! Opening your browser to {cellxgene_url} now.")
click.echo(
f"[cellxgene] Launching! Opening your browser to {cellxgene_url} now."
)
webbrowser.open(cellxgene_url)
else:
click.echo(f"[cellxgene] Launching! Please go to {cellxgene_url} in your browser.")
click.echo(
f"[cellxgene] Launching! Please go to {cellxgene_url} in your browser."
)
click.echo("[cellxgene] Type CTRL-C at any time to exit.")
@@ -363,8 +391,14 @@ def launch(
sys.stdout = f
try:
server.app.run(host=host, debug=debug, port=port, threaded=False if debug else True, use_debugger=False)
server.app.run(host=host,
debug=debug,
port=port,
threaded=False if debug else True,
use_debugger=False)
except OSError as e:
if e.errno == errno.EADDRINUSE:
raise click.ClickException("Port is in use, please specify an open port using the --port flag.") from e
raise click.ClickException(
"Port is in use, please specify an open port using the --port flag."
) from e
raise

137
server/cli/prepare.py
View File

@@ -8,9 +8,11 @@ from server.utils.utils import sort_options
@sort_options
@click.command(short_help="Preprocess data for use with cellxgene. "
"Run `cellxgene prepare --help` for more information.",
options_metavar="<options>",)
@click.command(
short_help="Preprocess data for use with cellxgene. "
"Run `cellxgene prepare --help` for more information.",
options_metavar="<options>",
)
@click.argument("data", nargs=1, metavar="<path to data file>", required=True)
@click.option(
"--embedding",
@@ -29,43 +31,66 @@ from server.utils.utils import sort_options
help="Preprocessing to run.",
show_default=True,
)
@click.option("--output", "-o", default="", help="Save a new file to filename.", metavar="<filename>")
@click.option("--plotting", "-p", default=False, is_flag=True, help="Generate plots.", show_default=True)
@click.option("--sparse", default=False, is_flag=True, help="Force sparsity.", show_default=True)
@click.option("--overwrite", default=False, is_flag=True, help="Allow file overwriting.", show_default=True)
@click.option("--set-obs-names", default="", help="Named field to set as index for obs.", metavar="<name>")
@click.option("--set-var-names", default="", help="Named field to set as index for var.", metavar="<name>")
@click.option("--skip-qc", default=False, is_flag=True,
help="Do not run quality control metrics. By default cellxgene runs them "
"(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).")
@click.option("--output",
"-o",
default="",
help="Save a new file to filename.",
metavar="<filename>")
@click.option("--plotting",
"-p",
default=False,
is_flag=True,
help="Generate plots.",
show_default=True)
@click.option("--sparse",
default=False,
is_flag=True,
help="Force sparsity.",
show_default=True)
@click.option("--overwrite",
default=False,
is_flag=True,
help="Allow file overwriting.",
show_default=True)
@click.option("--set-obs-names",
default="",
help="Named field to set as index for obs.",
metavar="<name>")
@click.option("--set-var-names",
default="",
help="Named field to set as index for var.",
metavar="<name>")
@click.option(
"--make-obs-names-unique",
default=True,
"--skip-qc",
default=False,
is_flag=True,
help="Ensure obs index is unique.",
show_default=True
)
@click.option(
"--make-var-names-unique",
default=True,
is_flag=True,
help="Ensure var index is unique.",
show_default=True
help="Do not run quality control metrics. By default cellxgene runs them "
"(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details)."
)
@click.option("--make-obs-names-unique",
default=True,
is_flag=True,
help="Ensure obs index is unique.",
show_default=True)
@click.option("--make-var-names-unique",
default=True,
is_flag=True,
help="Ensure var index is unique.",
show_default=True)
@click.help_option("--help", "-h", help="Show this message and exit.")
def prepare(
data,
embedding,
recipe,
output,
plotting,
sparse,
overwrite,
set_obs_names,
set_var_names,
skip_qc,
make_obs_names_unique,
make_var_names_unique,
data,
embedding,
recipe,
output,
plotting,
sparse,
overwrite,
set_obs_names,
set_var_names,
skip_qc,
make_obs_names_unique,
make_var_names_unique,
):
"""
Preprocess data for use with cellxgene.
@@ -86,8 +111,7 @@ def prepare(
except ImportError:
raise click.ClickException(
"[cellxgene] cellxgene prepare has not been installed. Please run `pip install cellxgene[prepare]` "
"to install the necessary requirements."
)
"to install the necessary requirements.")
# scanpy settings
sc.settings.verbosity = 0
@@ -102,10 +126,11 @@ def prepare(
if not output:
click.echo(
"Warning: No file will be saved, to save the results of cellxgene prepare include "
"--output <filename> to save output to a new file"
)
"--output <filename> to save output to a new file")
if isfile(output) and not overwrite:
raise click.UsageError(f"Cannot overwrite existing file {output}, try using the flag --overwrite")
raise click.UsageError(
f"Cannot overwrite existing file {output}, try using the flag --overwrite"
)
def load_data(data):
if isfile(data):
@@ -115,7 +140,9 @@ def prepare(
elif extension == ".loom":
adata = sc.read_loom(data)
else:
raise click.FileError(data, hint="does not have a valid extension [.h5ad | .loom]")
raise click.FileError(
data,
hint="does not have a valid extension [.h5ad | .loom]")
elif isdir(data):
if not data.endswith(sep):
data += sep
@@ -125,11 +152,15 @@ def prepare(
if not set_obs_names == "":
if set_obs_names not in adata.obs_keys():
raise click.UsageError(f"obs {set_obs_names} not found, options are: {adata.obs_keys()}")
raise click.UsageError(
f"obs {set_obs_names} not found, options are: {adata.obs_keys()}"
)
adata.obs_names = adata.obs[set_obs_names]
if not set_var_names == "":
if set_var_names not in adata.var_keys():
raise click.UsageError(f"var {set_var_names} not found, options are: {adata.var_keys()}")
raise click.UsageError(
f"var {set_var_names} not found, options are: {adata.var_keys()}"
)
adata.var_names = adata.var[set_var_names]
if make_obs_names_unique:
adata.obs_names_make_unique()
@@ -184,12 +215,18 @@ def prepare(
if "umap" in embedding:
sc.tl.umap(adata)
if plotting:
sc.pl.umap(adata, color="louvain", palette=palette, save="_louvain")
sc.pl.umap(adata,
color="louvain",
palette=palette,
save="_louvain")
if "tsne" in embedding:
sc.tl.tsne(adata)
if plotting:
sc.pl.tsne(adata, color="louvain", palette=palette, save="_louvain")
sc.pl.tsne(adata,
color="louvain",
palette=palette,
save="_louvain")
def show_step(item):
if not skip_qc:
@@ -208,13 +245,19 @@ def prepare(
if item is not None:
return names[item.__name__]
steps = [calculate_qc_metrics, make_sparse, run_recipe, run_pca, run_neighbors, run_louvain, run_embedding]
steps = [
calculate_qc_metrics, make_sparse, run_recipe, run_pca, run_neighbors,
run_louvain, run_embedding
]
click.echo(f"[cellxgene] Loading data from {data}, please wait...")
adata = load_data(data)
click.echo("[cellxgene] Beginning preprocessing...")
with click.progressbar(steps, label="[cellxgene] Progress", show_eta=False, item_show_func=show_step) as bar:
with click.progressbar(steps,
label="[cellxgene] Progress",
show_eta=False,
item_show_func=show_step) as bar:
for step in bar:
step(adata)

11
server/gui/browser.py
View File

@@ -12,7 +12,9 @@ WindowUtils = cef.WindowUtils()
# noinspection PyUnresolvedReferences
CefWidgetParent = QWidget
class CefWidget(CefWidgetParent):
def __init__(self, parent=None):
super(CefWidget, self).__init__(parent)
self.parent = parent
@@ -58,8 +60,8 @@ class CefWidget(CefWidgetParent):
if WINDOWS:
WindowUtils.OnSize(self.getHandle(), 0, 0, 0)
elif LINUX:
self.browser.SetBounds(self.x, self.y,
self.width(), self.height())
self.browser.SetBounds(self.x, self.y, self.width(),
self.height())
self.browser.NotifyMoveOrResizeStarted()
def resizeEvent(self, event):
@@ -68,12 +70,13 @@ class CefWidget(CefWidgetParent):
if WINDOWS:
WindowUtils.OnSize(self.getHandle(), 0, 0, 0)
elif LINUX:
self.browser.SetBounds(self.x, self.y,
size.width(), size.height())
self.browser.SetBounds(self.x, self.y, size.width(),
size.height())
self.browser.NotifyMoveOrResizeStarted()
class CefApplication(QApplication):
def __init__(self, args):
super(CefApplication, self).__init__(args)
if not cef.GetAppSetting("external_message_pump"):

9
server/gui/cellxgene_rc.py
View File

@@ -431,10 +431,15 @@ qt_resource_struct = b"\
\x00\x00\x00,\x00\x00\x00\x00\x00\x01\x00\x00\x11>\
"
def qInitResources():
QtCore.qRegisterResourceData(0x01, qt_resource_struct, qt_resource_name, qt_resource_data)
QtCore.qRegisterResourceData(0x01, qt_resource_struct, qt_resource_name,
qt_resource_data)
def qCleanupResources():
QtCore.qUnregisterResourceData(0x01, qt_resource_struct, qt_resource_name, qt_resource_data)
QtCore.qUnregisterResourceData(0x01, qt_resource_struct, qt_resource_name,
qt_resource_data)
qInitResources()

20
server/gui/hook-cefpython3.py
View File

@@ -50,22 +50,22 @@ def check_pyinstaller_version():
version = PyInstaller.__version__
match = re.search(r"^\d+\.\d+(\.\d+)?", version)
if not (match.group(0) >= PYINSTALLER_MIN_VERSION):
raise SystemExit("Error: pyinstaller %s or higher is required"
% PYINSTALLER_MIN_VERSION)
raise SystemExit("Error: pyinstaller %s or higher is required" %
PYINSTALLER_MIN_VERSION)
def check_cefpython3_version():
if not is_module_satisfies("cefpython3 >= %s" % CEFPYTHON_MIN_VERSION):
raise SystemExit("Error: cefpython3 %s or higher is required"
% CEFPYTHON_MIN_VERSION)
raise SystemExit("Error: cefpython3 %s or higher is required" %
CEFPYTHON_MIN_VERSION)
def get_cefpython_modules():
"""Get all cefpython Cython modules in the cefpython3 package.
It returns a list of names without file extension. Eg.
'cefpython_py27'. """
pyds = glob.glob(os.path.join(CEFPYTHON3_DIR,
"cefpython_py*" + CYTHON_MODULE_EXT))
pyds = glob.glob(
os.path.join(CEFPYTHON3_DIR, "cefpython_py*" + CYTHON_MODULE_EXT))
assert len(pyds) > 1, "Missing cefpython3 Cython modules"
modules = []
for path in pyds:
@@ -155,7 +155,8 @@ def get_cefpython3_datas():
absolute_file_path = os.path.join(path, file)
dest_path = os.path.relpath(path, CEFPYTHON3_DIR)
ret.append((absolute_file_path, dest_path))
logger.info("Include cefpython3 data: {}/{}".format(dest_path, file))
logger.info("Include cefpython3 data: {}/{}".format(
dest_path, file))
elif is_win or is_linux:
# The .pak files in cefpython3/locales/ directory
locales_dir = os.path.join(CEFPYTHON3_DIR, "locales")
@@ -164,8 +165,9 @@ def get_cefpython3_datas():
for filename in os.listdir(locales_dir):
logger.info("Include cefpython3 data: {}/{}".format(
os.path.basename(locales_dir), filename))
ret.append((os.path.join(locales_dir, filename),
os.path.join(cefdatadir, "locales")))
ret.append(
(os.path.join(locales_dir,
filename), os.path.join(cefdatadir, "locales")))
# Optional .so/.dll files in cefpython3/swiftshader/ directory
swiftshader_dir = os.path.join(CEFPYTHON3_DIR, "swiftshader")

44
server/gui/main.py
View File

@@ -34,6 +34,7 @@ LOAD_INDEX = 1
class MainWindow(QMainWindow):
def __init__(self):
super(MainWindow, self).__init__(None)
self.cef_widget = None
@@ -59,14 +60,17 @@ class MainWindow(QMainWindow):
# close emitter on error/finished
self.parent_conn, self.child_conn = Pipe()
self.load_emitter = Emitter(self.parent_conn, WorkerSignals)
self.emitter_thread = threading.Thread(target=self.load_emitter.run, daemon=True)
self.emitter_thread = threading.Thread(target=self.load_emitter.run,
daemon=True)
self.emitter_thread.start()
# send to load with error message?
def setupLayout(self):
self.resize(WIDTH, HEIGHT)
self.cef_widget = CefWidget(self)
self.cef_widget.setSizePolicy(QSizePolicy(QSizePolicy.MinimumExpanding, QSizePolicy.MinimumExpanding))
self.cef_widget.setSizePolicy(
QSizePolicy(QSizePolicy.MinimumExpanding,
QSizePolicy.MinimumExpanding))
self.data_widget = LoadWidget(self)
self.stacked_layout = QStackedLayout()
self.stacked_layout.addWidget(self.cef_widget)
@@ -104,7 +108,8 @@ class MainWindow(QMainWindow):
# close emitter on error/finished
self.parent_conn, self.child_conn = Pipe()
self.load_emitter = Emitter(self.parent_conn, WorkerSignals)
self.emitter_thread = threading.Thread(target=self.load_emitter.run, daemon=True)
self.emitter_thread = threading.Thread(target=self.load_emitter.run,
daemon=True)
self.emitter_thread.start()
# send to load with error message?
@@ -144,6 +149,7 @@ class MainWindow(QMainWindow):
class LoadWidget(QFrame):
def __init__(self, parent):
super(LoadWidget, self).__init__(parent=parent)
# Init layout
@@ -240,11 +246,18 @@ class LoadWidget(QFrame):
def createScanpyEngine(self, file_name):
title = splitext(basename(file_name))[0]
self.window().setupServer()
worker = Worker(self.window().parent_conn, self.window().child_conn, file_name, host="127.0.0.1",
port=GUI_PORT, title=title, engine_options={})
worker = Worker(self.window().parent_conn,
self.window().child_conn,
file_name,
host="127.0.0.1",
port=GUI_PORT,
title=title,
engine_options={})
self.window().load_emitter.signals.ready.connect(self.onDataReady)
self.window().load_emitter.signals.engine_error.connect(self.onServerError)
self.window().load_emitter.signals.server_error.connect(self.onServerError)
self.window().load_emitter.signals.engine_error.connect(
self.onServerError)
self.window().load_emitter.signals.server_error.connect(
self.onServerError)
# Error is generic error from emitter
self.window().load_emitter.signals.error.connect(self.onServerError)
self.window().worker = Process(target=worker.run, daemon=True)
@@ -266,7 +279,8 @@ class LoadWidget(QFrame):
self.site_ready_worker.signals.ready.connect(self.onServerReady)
self.site_ready_worker.signals.error.connect(self.onServerError)
srw_thread = threading.Thread(target=self.site_ready_worker.run, daemon=True)
srw_thread = threading.Thread(target=self.site_ready_worker.run,
daemon=True)
srw_thread.start()
def onServerReady(self):
@@ -289,7 +303,9 @@ class LoadWidget(QFrame):
onServerError = partialmethod(onError, server_error=True)
class FilePath(QObject):
def __init__(self):
super(FilePath, self).__init__()
self.value = ""
@@ -301,6 +317,7 @@ class FilePath(QObject):
class FileArea(QFrame):
def __init__(self, parent):
super(FileArea, self).__init__()
self.setFrameShape(QFrame.Box)
@@ -309,10 +326,12 @@ class FileArea(QFrame):
self.setAcceptDrops(True)
self.instructions = QLabel(self)
self.instructions.setText("Drag & Drop a h5ad file to load or open")
self.instructions.setGeometry(10, 10, MAX_CONTENT_WIDTH, self.instructions.height())
self.instructions.setGeometry(10, 10, MAX_CONTENT_WIDTH,
self.instructions.height())
self.loadButton = QPushButton("Open...", parent=self)
x_pos = (MAX_CONTENT_WIDTH - self.loadButton.width()) / 2
self.loadButton.setGeometry(x_pos, 50, self.loadButton.width(), self.loadButton.height())
self.loadButton.setGeometry(x_pos, 50, self.loadButton.width(),
self.loadButton.height())
self.loadButton.clicked.connect(self.fileBrowse)
self.label = QLabel(self)
self.label.setGeometry(10, 75, MAX_CONTENT_WIDTH, self.label.height())
@@ -321,7 +340,10 @@ class FileArea(QFrame):
options = QFileDialog.Options()
# options |= QFileDialog.DontUseNativeDialog
file_name, _ = QFileDialog.getOpenFileName(self,
"Open H5AD File", "", "H5AD Files (*.h5ad)", options=options)
"Open H5AD File",
"",
"H5AD Files (*.h5ad)",
options=options)
if file_name:
self.parent().file_name.updateValue(file_name)
self.parent().onLoad()

1
server/gui/utils.py
View File

@@ -49,6 +49,7 @@ class FileChanged(QObject):
class Emitter:
def __init__(self, transport, signals):
self.transport = transport
self.signals = signals()

11
server/gui/workers.py
View File

@@ -7,6 +7,7 @@ from server.gui.utils import SiteReadySignals
class EmittingProcess(Process):
def __init__(self, parent_conn, child_conn, *arg, **kwargs):
super(EmittingProcess, self).__init__()
self.parent_conn = parent_conn
@@ -21,7 +22,9 @@ class EmittingProcess(Process):
class Worker(EmittingProcess):
def __init__(self, parent_conn, child_conn, data_file, host, port, title, engine_options, *args, **kwargs):
def __init__(self, parent_conn, child_conn, data_file, host, port, title,
engine_options, *args, **kwargs):
super(Worker, self).__init__(parent_conn, child_conn)
self.data_file = data_file
self.host = host
@@ -62,7 +65,10 @@ class Worker(EmittingProcess):
return
# launch server
try:
server.app.run(host=self.host, debug=False, port=self.port, threaded=True)
server.app.run(host=self.host,
debug=False,
port=self.port,
threaded=True)
except Exception as e:
self.emit("server_error", str(e))
finally:
@@ -70,6 +76,7 @@ class Worker(EmittingProcess):
class SiteReadyWorker:
def __init__(self, location):
super(SiteReadyWorker, self).__init__()
self.signals = SiteReadySignals()

18
server/test/decode_fbs.py
View File

@@ -1,4 +1,3 @@
"""
Code to decode, for testing purposes, the flatbuffer encoded blobs.
This code will need to be updated if fbs/matrix.fbs changes.
@@ -18,18 +17,23 @@ import server.app.util.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray
def decode_typed_array(tarr):
type_map = {
TypedArray.TypedArray.Uint32Array: Uint32Array.Uint32Array,
TypedArray.TypedArray.Int32Array: Int32Array.Int32Array,
TypedArray.TypedArray.Float32Array: Float32Array.Float32Array,
TypedArray.TypedArray.Float64Array: Float64Array.Float64Array,
TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray
TypedArray.TypedArray.Uint32Array:
Uint32Array.Uint32Array,
TypedArray.TypedArray.Int32Array:
Int32Array.Int32Array,
TypedArray.TypedArray.Float32Array:
Float32Array.Float32Array,
TypedArray.TypedArray.Float64Array:
Float64Array.Float64Array,
TypedArray.TypedArray.JSONEncodedArray:
JSONEncodedArray.JSONEncodedArray
}
(u_type, u) = tarr
if u_type == TypedArray.TypedArray.NONE:
return None
TarType = type_map.get(u_type, None)
assert(TarType is not None)
assert (TarType is not None)
arr = TarType()
arr.Init(u.Bytes, u.Pos)

108
server/test/test_api.py
View File

@@ -19,7 +19,10 @@ class EndPoints(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.ps = Popen(["cellxgene", "launch", "../example-dataset/pbmc3k.h5ad", "--verbose", "--port", "5005"])
cls.ps = Popen([
"cellxgene", "launch", "../example-dataset/pbmc3k.h5ad",
"--verbose", "--port", "5005"
])
session = requests.Session()
for i in range(90):
try:
@@ -47,7 +50,8 @@ class EndPoints(unittest.TestCase):
result_data = result.json()
self.assertEqual(result_data["schema"]["dataframe"]["nObs"], 2638)
self.assertEqual(len(result_data["schema"]["annotations"]["obs"]), 2)
self.assertEqual(len(result_data["schema"]["annotations"]["obs"]["columns"]), 5)
self.assertEqual(
len(result_data["schema"]["annotations"]["obs"]["columns"]), 5)
def test_config(self):
endpoint = "config"
@@ -57,7 +61,8 @@ class EndPoints(unittest.TestCase):
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertIn("library_versions", result_data["config"])
self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k")
self.assertEqual(result_data["config"]["displayNames"]["dataset"],
"pbmc3k")
self.assertEqual(len(result_data["config"]["features"]), 4)
def test_get_layout_fbs(self):
@@ -66,13 +71,15 @@ class EndPoints(unittest.TestCase):
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
self.assertEqual(result.headers["Content-Type"],
"application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638)
self.assertEqual(df['n_cols'], 8)
self.assertIsNotNone(df['columns'])
self.assertListEqual(df['col_idx'], [
'pca_0', 'pca_1', 'tsne_0', 'tsne_1', 'umap_0', 'umap_1', 'draw_graph_fr_0', 'draw_graph_fr_1'
'pca_0', 'pca_1', 'tsne_0', 'tsne_1', 'umap_0', 'umap_1',
'draw_graph_fr_0', 'draw_graph_fr_1'
])
self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols'])
@@ -89,7 +96,8 @@ class EndPoints(unittest.TestCase):
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
self.assertEqual(result.headers["Content-Type"],
"application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638)
self.assertEqual(df['n_cols'], 5)
@@ -97,8 +105,11 @@ class EndPoints(unittest.TestCase):
self.assertIsNotNone(df['col_idx'])
self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols'])
obs_index_col_name = self.schema["schema"]["annotations"]["obs"]["index"]
self.assertListEqual(df['col_idx'], [obs_index_col_name, 'n_genes', 'percent_mito', 'n_counts', 'louvain'])
obs_index_col_name = self.schema["schema"]["annotations"]["obs"][
"index"]
self.assertListEqual(df['col_idx'], [
obs_index_col_name, 'n_genes', 'percent_mito', 'n_counts', 'louvain'
])
def test_get_annotations_obs_keys_fbs(self):
endpoint = "annotations/obs"
@@ -107,7 +118,8 @@ class EndPoints(unittest.TestCase):
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
self.assertEqual(result.headers["Content-Type"],
"application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638)
self.assertEqual(df['n_cols'], 2)
@@ -129,8 +141,26 @@ class EndPoints(unittest.TestCase):
url = f"{URL_BASE}{endpoint}"
params = {
"mode": "topN",
"set1": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["NK cells"]}]}}},
"set2": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["CD8 T cells"]}]}}},
"set1": {
"filter": {
"obs": {
"annotation_value": [{
"name": "louvain",
"values": ["NK cells"]
}]
}
}
},
"set2": {
"filter": {
"obs": {
"annotation_value": [{
"name": "louvain",
"values": ["CD8 T cells"]
}]
}
}
},
"count": 7,
}
result = self.session.post(url, json=params)
@@ -145,8 +175,20 @@ class EndPoints(unittest.TestCase):
params = {
"mode": "topN",
"count": 10,
"set1": {"filter": {"obs": {"index": [[0, 500]]}}},
"set2": {"filter": {"obs": {"index": [[500, 1000]]}}},
"set1": {
"filter": {
"obs": {
"index": [[0, 500]]
}
}
},
"set2": {
"filter": {
"obs": {
"index": [[500, 1000]]
}
}
},
}
result = self.session.post(url, json=params)
self.assertEqual(result.status_code, HTTPStatus.OK)
@@ -160,7 +202,8 @@ class EndPoints(unittest.TestCase):
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
self.assertEqual(result.headers["Content-Type"],
"application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 1838)
self.assertEqual(df['n_cols'], 2)
@@ -168,7 +211,8 @@ class EndPoints(unittest.TestCase):
self.assertIsNotNone(df['col_idx'])
self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols'])
var_index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
var_index_col_name = self.schema["schema"]["annotations"]["var"][
"index"]
self.assertListEqual(df['col_idx'], [var_index_col_name, 'n_cells'])
def test_get_annotations_var_keys_fbs(self):
@@ -178,7 +222,8 @@ class EndPoints(unittest.TestCase):
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
self.assertEqual(result.headers["Content-Type"],
"application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 1838)
self.assertEqual(df['n_cols'], 1)
@@ -207,7 +252,8 @@ class EndPoints(unittest.TestCase):
url = f"{URL_BASE}{endpoint}"
result = self.session.put(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
self.assertEqual(result.headers["Content-Type"],
"application/octet-stream")
def test_data_put_fbs(self):
endpoint = f"data/var"
@@ -215,7 +261,8 @@ class EndPoints(unittest.TestCase):
header = {"Accept": "application/octet-stream"}
result = self.session.put(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
self.assertEqual(result.headers["Content-Type"],
"application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638)
self.assertEqual(df['n_cols'], 1838)
@@ -228,16 +275,11 @@ class EndPoints(unittest.TestCase):
endpoint = f"data/var"
url = f"{URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
filter = {
"filter": {
"var": {
"index": [0, 1, 4]
}
}
}
filter = {"filter": {"var": {"index": [0, 1, 4]}}}
result = self.session.put(url, headers=header, json=filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
self.assertEqual(result.headers["Content-Type"],
"application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638)
self.assertEqual(df['n_cols'], 3)
@@ -252,10 +294,20 @@ class EndPoints(unittest.TestCase):
url = f"{URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
var_filter = {"filter": {"var": {"annotation_value": [{"name": index_col_name, "values": ["RER1"]}]}}}
var_filter = {
"filter": {
"var": {
"annotation_value": [{
"name": index_col_name,
"values": ["RER1"]
}]
}
}
}
result = self.session.put(url, headers=header, json=var_filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
self.assertEqual(result.headers["Content-Type"],
"application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 1)

53
server/test/test_fbs.py
View File

@@ -40,49 +40,52 @@ class FbsTests(unittest.TestCase):
def test_encode_DataFrame(self):
df = pd.DataFrame(
data={
'a': np.zeros((10,), dtype=np.float32),
'b': np.ones((10,), dtype=np.int64),
'c': np.array([i for i in range(0, 10)], dtype=np.uint16),
'd': pd.Series(['x', 'y', 'z', 'x', 'y', 'z', 'a', 'x', 'y', 'z'], dtype='category')
'a':
np.zeros((10,), dtype=np.float32),
'b':
np.ones((10,), dtype=np.int64),
'c':
np.array([i for i in range(0, 10)], dtype=np.uint16),
'd':
pd.Series(
['x', 'y', 'z', 'x', 'y', 'z', 'a', 'x', 'y', 'z'],
dtype='category')
})
expected_types = (
(np.ndarray, np.float32),
(np.ndarray, np.int32),
(np.ndarray, np.uint32),
(list, None)
)
expected_types = ((np.ndarray, np.float32), (np.ndarray, np.int32),
(np.ndarray, np.uint32), (list, None))
fbs = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
self.fbs_checks(fbs, (10, 4), expected_types, ['a', 'b', 'c', 'd'])
def test_encode_ndarray(self):
arr = np.zeros((3, 2), dtype=np.float32)
expected_types = (
(np.ndarray, np.float32),
(np.ndarray, np.float32),
(np.ndarray, np.float32)
)
expected_types = ((np.ndarray, np.float32), (np.ndarray, np.float32),
(np.ndarray, np.float32))
fbs = encode_matrix_fbs(matrix=arr, row_idx=None, col_idx=None)
self.fbs_checks(fbs, (3, 2), expected_types, None)
def test_encode_sparse(self):
csc = sparse.csc_matrix(np.array([[0, 1, 2], [3, 0, 4]]))
expected_types = (
(np.ndarray, np.int32),
(np.ndarray, np.int32),
(np.ndarray, np.int32)
)
expected_types = ((np.ndarray, np.int32), (np.ndarray, np.int32),
(np.ndarray, np.int32))
fbs = encode_matrix_fbs(matrix=csc, row_idx=None, col_idx=None)
self.fbs_checks(fbs, (2, 3), expected_types, None)
def test_roundtrip(self):
dfSrc = pd.DataFrame(
data={
'a': np.zeros((10,), dtype=np.float32),
'b': np.ones((10,), dtype=np.int64),
'c': np.array([i for i in range(0, 10)], dtype=np.uint16),
'd': pd.Series(['x', 'y', 'z', 'x', 'y', 'z', 'a', 'x', 'y', 'z'], dtype='category')
'a':
np.zeros((10,), dtype=np.float32),
'b':
np.ones((10,), dtype=np.int64),
'c':
np.array([i for i in range(0, 10)], dtype=np.uint16),
'd':
pd.Series(
['x', 'y', 'z', 'x', 'y', 'z', 'a', 'x', 'y', 'z'],
dtype='category')
})
dfDst = decode_matrix_fbs(encode_matrix_fbs(matrix=dfSrc, col_idx=dfSrc.columns))
dfDst = decode_matrix_fbs(
encode_matrix_fbs(matrix=dfSrc, col_idx=dfSrc.columns))
self.assertEqual(dfSrc.shape, dfDst.shape)
self.assertEqual(set(dfSrc.columns), set(dfDst.columns))
for c in dfSrc.columns:

138
server/test/test_matrix_proxy.py
View File

@@ -7,12 +7,14 @@ class NdArrayProxyView(MatrixProxyView):
"""
Fake test class for matrix proxy - wraps ndarray
"""
@classmethod
def __supports__(cls):
return ('numpy.ndarray', )
return ('numpy.ndarray',)
class MatrixProxyViewTest(unittest.TestCase):
def test_ismatrixproxy(self):
n = np.zeros((2, 4))
mp = MatrixProxy.create(n)
@@ -41,18 +43,13 @@ class MatrixProxyViewTest(unittest.TestCase):
def test_toarray(self):
n = np.arange(15, dtype=np.float32).reshape((3, 5))
mp = MatrixProxy.create(n)
self.assertTrue(np.all(mp.toarray() == [
[0., 1., 2., 3., 4.],
[5., 6., 7., 8., 9.],
[10., 11., 12., 13., 14.]
]))
self.assertTrue(np.all(mp.T.toarray() == [
[0., 5., 10.],
[1., 6., 11.],
[2., 7., 12.],
[3., 8., 13.],
[4., 9., 14.]
]))
self.assertTrue(
np.all(mp.toarray() == [[0., 1., 2., 3., 4.], [5., 6., 7., 8., 9.],
[10., 11., 12., 13., 14.]]))
self.assertTrue(
np.all(
mp.T.toarray() == [[0., 5., 10.], [1., 6., 11.], [2., 7., 12.],
[3., 8., 13.], [4., 9., 14.]]))
def test_indexing(self):
"""
@@ -95,47 +92,25 @@ class MatrixProxyViewTest(unittest.TestCase):
# slice, slice
self.assertTrue(np.all(mp[1:3, 2:4].toarray() == [
[7, 8],
[12, 13]
]))
self.assertTrue(np.all(mp[:3, :4].toarray() == [
[0., 1., 2., 3.],
[5., 6., 7., 8.],
[10., 11., 12., 13.]
]))
self.assertTrue(np.all(mp[::-1, ::-1].toarray() == [
[14, 13, 12, 11, 10],
[9, 8, 7, 6, 5],
[4, 3, 2, 1, 0]
]))
self.assertTrue(np.all(mp[::-2, ::-2].toarray() == [
[14, 12, 10],
[4, 2, 0]
]))
self.assertTrue(np.all(mp[1:3, 2:4].toarray() == [[7, 8], [12, 13]]))
self.assertTrue(
np.all(mp[:3, :4].toarray() == [[0., 1., 2., 3.], [5., 6., 7., 8.],
[10., 11., 12., 13.]]))
self.assertTrue(
np.all(mp[::-1, ::-1].toarray() ==
[[14, 13, 12, 11, 10], [9, 8, 7, 6, 5], [4, 3, 2, 1, 0]]))
self.assertTrue(
np.all(mp[::-2, ::-2].toarray() == [[14, 12, 10], [4, 2, 0]]))
self.assertTrue(np.all(mp.T[2:4, 1:3].toarray() == [
[7, 12],
[8, 13]
]))
self.assertTrue(np.all(mp.T[:4, :3].toarray() == [
[0, 5, 10],
[1, 6, 11],
[2, 7, 12],
[3, 8, 13]
]))
self.assertTrue(np.all(mp.T[::-1, ::-1].toarray() == [
[14, 9, 4],
[13, 8, 3],
[12, 7, 2],
[11, 6, 1],
[10, 5, 0]
]))
self.assertTrue(np.all(mp.T[::-2, ::-2].toarray() == [
[14, 4],
[12, 2],
[10, 0]
]))
self.assertTrue(np.all(mp.T[2:4, 1:3].toarray() == [[7, 12], [8, 13]]))
self.assertTrue(
np.all(mp.T[:4, :3].toarray() == [[0, 5, 10], [1, 6, 11],
[2, 7, 12], [3, 8, 13]]))
self.assertTrue(
np.all(mp.T[::-1, ::-1].toarray(
) == [[14, 9, 4], [13, 8, 3], [12, 7, 2], [11, 6, 1], [10, 5, 0]]))
self.assertTrue(
np.all(mp.T[::-2, ::-2].toarray() == [[14, 4], [12, 2], [10, 0]]))
def test_repeated_indexing(self):
"""
@@ -149,31 +124,16 @@ class MatrixProxyViewTest(unittest.TestCase):
self.assertEqual(mp[0][1], 1)
self.assertEqual(mp.T[0][1], 5)
self.assertTrue(np.all(mp[0::-1, ::-1][0, 2:4].toarray() == [
2, 1
]))
self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 1:3:1].toarray() == [
2, 3
]))
self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 2:0:-1].toarray() == [
3, 2
]))
self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, 1:3:1].toarray() == [
3, 2
]))
self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, 2:0:-1].toarray() == [
2, 3
]))
self.assertTrue(np.all(mp[0::-1, ::-1][0, 2:4].toarray() == [2, 1]))
self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 1:3:1].toarray() == [2, 3]))
self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 2:0:-1].toarray() == [3, 2]))
self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, 1:3:1].toarray() == [3, 2]))
self.assertTrue(np.all(mp[0::-1, 5:1:-1][0,
2:0:-1].toarray() == [2, 3]))
self.assertTrue(np.all(mp.T[::-1, 0::-1][2:4, 0].toarray() == [
2, 1
]))
self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 1:3:1].toarray() == [
10
]))
self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 2:0:-1].toarray() == [
10
]))
self.assertTrue(np.all(mp.T[::-1, 0::-1][2:4, 0].toarray() == [2, 1]))
self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 1:3:1].toarray() == [10]))
self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 2:0:-1].toarray() == [10]))
self.assertTrue(np.all(mp.T[0::-1, 5:1:-1][0, 1:3:1].toarray() == []))
self.assertTrue(np.all(mp.T[0::-1, 5:1:-1][0, 2:0:-1].toarray() == []))
@@ -190,20 +150,12 @@ class MatrixProxyViewTest(unittest.TestCase):
self.assertEqual(mp[0, 0], 0)
# drop 1 dimension, to an array
self.assertTrue(np.all(mp[0, :].toarray() == [
0, 1, 2, 3, 4
]))
self.assertTrue(np.all(mp[:, 0].toarray() == [
0, 5, 10
]))
self.assertTrue(np.all(mp[0, :].toarray() == [0, 1, 2, 3, 4]))
self.assertTrue(np.all(mp[:, 0].toarray() == [0, 5, 10]))
# with .T
self.assertTrue(np.all(mp[0:2].T[-1:].toarray() == [
[4, 9]
]))
self.assertTrue(np.all(mp[0:2].T[-1].toarray() == [
4, 9
]))
self.assertTrue(np.all(mp[0:2].T[-1:].toarray() == [[4, 9]]))
self.assertTrue(np.all(mp[0:2].T[-1].toarray() == [4, 9]))
def test_iter(self):
"""
@@ -214,17 +166,13 @@ class MatrixProxyViewTest(unittest.TestCase):
rows = [r for r in mp]
self.assertEqual(len(rows), 3)
self.assertTrue(np.all(rows[0].toarray() == [
0, 1, 2, 3, 4
]))
self.assertTrue(np.all(rows[0].toarray() == [0, 1, 2, 3, 4]))
for i, r in enumerate(rows):
self.assertTrue(np.all(mp[i].toarray() == r.toarray()))
cols = [c for c in mp.T]
self.assertEqual(len(cols), 5)
self.assertTrue(np.all(cols[0].toarray() == [
0, 5, 10
]))
self.assertTrue(np.all(cols[0].toarray() == [0, 5, 10]))
for i, c in enumerate(cols):
self.assertTrue(np.all(mp.T[i].toarray() == c.toarray()))

16
server/test/test_nan_rest.py
View File

@@ -20,9 +20,10 @@ class WithNaNs(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.ps = Popen(
["cellxgene", "launch", "test/test_datasets/nan.h5ad", "--verbose", "--port", "5006"]
)
cls.ps = Popen([
"cellxgene", "launch", "test/test_datasets/nan.h5ad", "--verbose",
"--port", "5006"
])
session = requests.Session()
for i in range(90):
try:
@@ -51,7 +52,8 @@ class WithNaNs(unittest.TestCase):
url = f"{URL_BASE}{endpoint}"
result = self.session.put(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
self.assertEqual(result.headers["Content-Type"],
"application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertTrue(math.isnan(df["columns"][3][3]))
@@ -60,7 +62,8 @@ class WithNaNs(unittest.TestCase):
url = f"{URL_BASE}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
self.assertEqual(result.headers["Content-Type"],
"application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertTrue(math.isnan(df["columns"][2][0]))
@@ -69,6 +72,7 @@ class WithNaNs(unittest.TestCase):
url = f"{URL_BASE}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
self.assertEqual(result.headers["Content-Type"],
"application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertTrue(math.isnan(df["columns"][2][0]))

32
server/test/test_nan_scanpy_engine.py
View File

@@ -11,6 +11,7 @@ from server.app.util.data_locator import DataLocator
class NaNTest(unittest.TestCase):
def setUp(self):
self.args = {
"layout": ["umap"],
@@ -21,7 +22,8 @@ class NaNTest(unittest.TestCase):
}
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UserWarning)
self.data = ScanpyEngine(DataLocator("test/test_datasets/nan.h5ad"), self.args)
self.data = ScanpyEngine(DataLocator("test/test_datasets/nan.h5ad"),
self.args)
self.data._create_schema()
def test_load(self):
@@ -35,7 +37,8 @@ class NaNTest(unittest.TestCase):
self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
def test_dataframe(self):
data_frame_var = decode_fbs.decode_matrix_FBS(self.data.data_frame_to_fbs_matrix(None, "var"))
data_frame_var = decode_fbs.decode_matrix_FBS(
self.data.data_frame_to_fbs_matrix(None, "var"))
self.assertIsNotNone(data_frame_var)
self.assertEqual(data_frame_var["n_rows"], 100)
self.assertEqual(data_frame_var["n_cols"], 100)
@@ -44,30 +47,29 @@ class NaNTest(unittest.TestCase):
with pytest.raises(FilterError):
self.data.data_frame_to_fbs_matrix("an erroneous filter", "var")
with pytest.raises(FilterError):
filter_ = {
"filter": {
"obs": {"index": [1, 99, [200, 300]]}
}
}
filter_ = {"filter": {"obs": {"index": [1, 99, [200, 300]]}}}
self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
def test_dataframe_obs_not_implemented(self):
with self.assertRaises(ValueError) as cm:
decode_fbs.decode_matrix_FBS(self.data.data_frame_to_fbs_matrix(None, "obs"))
decode_fbs.decode_matrix_FBS(
self.data.data_frame_to_fbs_matrix(None, "obs"))
self.assertIsNotNone(cm.exception)
def test_annotation(self):
annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("obs"))
annotations = decode_fbs.decode_matrix_FBS(
self.data.annotation_to_fbs_matrix("obs"))
obs_index_col_name = self.data.schema["annotations"]["obs"]["index"]
self.assertEqual(
annotations["col_idx"],
[obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"]
)
self.assertEqual(annotations["col_idx"], [
obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"
])
self.assertEqual(annotations["n_rows"], 100)
self.assertTrue(math.isnan(annotations["columns"][2][0]))
annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("var"))
annotations = decode_fbs.decode_matrix_FBS(
self.data.annotation_to_fbs_matrix("var"))
var_index_col_name = self.data.schema["annotations"]["var"]["index"]
self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells", "var_with_nans"])
self.assertEqual(annotations["col_idx"],
[var_index_col_name, "n_cells", "var_with_nans"])
self.assertEqual(annotations["n_rows"], 100)
self.assertTrue(math.isnan(annotations["columns"][2][0]))

90
server/test/test_scanpy_engine.py
View File

@@ -12,7 +12,6 @@ import pandas as pd
from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
from server.app.util.errors import FilterError, DisabledFeatureError
from server.app.util.data_locator import DataLocator
"""
Test the scanpy engine using the pbmc3k data set.
"""
@@ -22,12 +21,12 @@ Test the scanpy engine using the pbmc3k data set.
("../example-dataset/pbmc3k.h5ad", False),
("test/test_datasets/pbmc3k-CSC-gz.h5ad", False),
("test/test_datasets/pbmc3k-CSR-gz.h5ad", False),
("../example-dataset/pbmc3k.h5ad", True),
("test/test_datasets/pbmc3k-CSC-gz.h5ad", True),
("test/test_datasets/pbmc3k-CSR-gz.h5ad", True),
])
class EngineTest(unittest.TestCase):
def setUp(self):
args = {
"layout": ["umap"],
@@ -47,10 +46,12 @@ class EngineTest(unittest.TestCase):
self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
def test_mandatory_annotations(self):
obs_index_col_name = self.data.get_schema()["annotations"]["obs"]["index"]
obs_index_col_name = self.data.get_schema(
)["annotations"]["obs"]["index"]
self.assertIn(obs_index_col_name, self.data.data.obs)
self.assertEqual(list(self.data.data.obs.index), list(range(2638)))
var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
var_index_col_name = self.data.get_schema(
)["annotations"]["var"]["index"]
self.assertIn(var_index_col_name, self.data.data.var)
self.assertEqual(list(self.data.data.var.index), list(range(1838)))
@@ -64,11 +65,7 @@ class EngineTest(unittest.TestCase):
self.data._validate_data_types()
def test_filter_idx(self):
filter_ = {
"filter": {
"var": {"index": [1, 99, [200, 300]]}
}
}
filter_ = {"filter": {"var": {"index": [1, 99, [200, 300]]}}}
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
data = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(data["n_rows"], 2638)
@@ -78,9 +75,10 @@ class EngineTest(unittest.TestCase):
filter_ = {
"filter": {
"var": {
"annotation_value": [
{"name": "n_cells", "min": 10}
],
"annotation_value": [{
"name": "n_cells",
"min": 10
}],
"index": [1, 99, [200, 300]]
}
}
@@ -91,8 +89,12 @@ class EngineTest(unittest.TestCase):
self.assertEqual(data["n_cols"], 91)
def test_obs_and_var_names(self):
self.assertEqual(np.sum(self.data.data.var[self.data.get_schema()["annotations"]["var"]["index"]].isna()), 0)
self.assertEqual(np.sum(self.data.data.obs[self.data.get_schema()["annotations"]["obs"]["index"]].isna()), 0)
self.assertEqual(
np.sum(self.data.data.var[self.data.get_schema()["annotations"]
["var"]["index"]].isna()), 0)
self.assertEqual(
np.sum(self.data.data.obs[self.data.get_schema()["annotations"]
["obs"]["index"]].isna()), 0)
def test_get_schema(self):
with open(path.join(path.dirname(__file__), "schema.json")) as fh:
@@ -110,7 +112,10 @@ class EngineTest(unittest.TestCase):
def test_config(self):
self.assertEqual(
self.data.features["layout"]["obs"],
{"available": True, "interactiveLimit": 50000},
{
"available": True,
"interactiveLimit": 50000
},
)
def test_layout(self):
@@ -129,18 +134,24 @@ class EngineTest(unittest.TestCase):
annotations = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(annotations["n_rows"], 2638)
self.assertEqual(annotations["n_cols"], 5)
obs_index_col_name = self.data.get_schema()["annotations"]["obs"]["index"]
obs_index_col_name = self.data.get_schema(
)["annotations"]["obs"]["index"]
self.assertEqual(
annotations["col_idx"],
[obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"],
[
obs_index_col_name, "n_genes", "percent_mito", "n_counts",
"louvain"
],
)
fbs = self.data.annotation_to_fbs_matrix("var")
annotations = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(annotations['n_rows'], 1838)
self.assertEqual(annotations['n_cols'], 2)
var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells"])
var_index_col_name = self.data.get_schema(
)["annotations"]["var"]["index"]
self.assertEqual(annotations["col_idx"],
[var_index_col_name, "n_cells"])
def test_annotation_fields(self):
fbs = self.data.annotation_to_fbs_matrix("obs", ["n_genes", "n_counts"])
@@ -148,7 +159,8 @@ class EngineTest(unittest.TestCase):
self.assertEqual(annotations["n_rows"], 2638)
self.assertEqual(annotations['n_cols'], 2)
var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
var_index_col_name = self.data.get_schema(
)["annotations"]["var"]["index"]
fbs = self.data.annotation_to_fbs_matrix("var", [var_index_col_name])
annotations = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(annotations['n_rows'], 1838)
@@ -163,7 +175,8 @@ class EngineTest(unittest.TestCase):
f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
self.assertEqual(len(result), 10)
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
result = json.loads(
self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
self.assertEqual(len(result), 20)
def test_data_frame(self):
@@ -178,7 +191,14 @@ class EngineTest(unittest.TestCase):
def test_filtered_data_frame(self):
filter_ = {
"filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 100}]}}
"filter": {
"var": {
"annotation_value": [{
"name": "n_cells",
"min": 100
}]
}
}
}
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
data = decode_fbs.decode_matrix_FBS(fbs)
@@ -186,16 +206,29 @@ class EngineTest(unittest.TestCase):
self.assertEqual(data["n_cols"], 1040)
filter_ = {
"filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}
"filter": {
"obs": {
"annotation_value": [{
"name": "n_counts",
"min": 3000
}]
}
}
}
with self.assertRaises(FilterError):
self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
def test_data_named_gene(self):
var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
var_index_col_name = self.data.get_schema(
)["annotations"]["var"]["index"]
filter_ = {
"filter": {
"var": {"annotation_value": [{"name": var_index_col_name, "values": ["RER1"]}]}
"var": {
"annotation_value": [{
"name": var_index_col_name,
"values": ["RER1"]
}]
}
}
}
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
@@ -206,7 +239,12 @@ class EngineTest(unittest.TestCase):
filter_ = {
"filter": {
"var": {"annotation_value": [{"name": var_index_col_name, "values": ["SPEN", "TYMP", "PRMT2"]}]}
"var": {
"annotation_value": [{
"name": var_index_col_name,
"values": ["SPEN", "TYMP", "PRMT2"]
}]
}
}
}
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")

5
server/test/test_scanpy_engine_data_load.py
View File

@@ -10,6 +10,7 @@ class DataLoadEngineTest(unittest.TestCase):
"""
Test file loading, including deferred loading/update.
"""
def setUp(self):
self.data_file = DataLocator("../example-dataset/pbmc3k.h5ad")
self.data = ScanpyEngine()
@@ -52,7 +53,8 @@ class DataLoadEngineTest(unittest.TestCase):
f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
self.assertEqual(len(result), 10)
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
result = json.loads(
self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
self.assertEqual(len(result), 20)
@@ -60,6 +62,7 @@ class DataLocatorEngineTest(unittest.TestCase):
"""
Test various types of data locators we expect to consume
"""
def setUp(self):
self.args = {
"layout": ["umap"],

105
server/test/test_writable_annotation.py
View File

@@ -14,6 +14,7 @@ from server.app.util.data_locator import DataLocator
class WritableAnnotationTest(unittest.TestCase):
def setUp(self):
self.tmpDir = tempfile.mkdtemp()
self.annotations_file = path.join(self.tmpDir, "test_annotations.csv")
@@ -27,7 +28,8 @@ class WritableAnnotationTest(unittest.TestCase):
"annotations_file": self.annotations_file,
"annotations_output_dir": None
}
self.data = ScanpyEngine(DataLocator("../example-dataset/pbmc3k.h5ad"), args)
self.data = ScanpyEngine(DataLocator("../example-dataset/pbmc3k.h5ad"),
args)
def tearDown(self):
shutil.rmtree(self.tmpDir)
@@ -41,7 +43,9 @@ class WritableAnnotationTest(unittest.TestCase):
n_rows = self.data.data.obs.shape[0]
fbs_bad = self.make_fbs({
'louvain': pd.Series(['undefined' for l in range(0, n_rows)], dtype='category')
'louvain':
pd.Series(['undefined' for l in range(0, n_rows)],
dtype='category')
})
# ensure attempt to change VAR annotation
@@ -56,31 +60,49 @@ class WritableAnnotationTest(unittest.TestCase):
# verify the file is written as expected
n_rows = self.data.data.obs.shape[0]
fbs = self.make_fbs({
'cat_A': pd.Series(['label_A' for l in range(0, n_rows)], dtype='category'),
'cat_B': pd.Series(['label_B' for l in range(0, n_rows)], dtype='category')
'cat_A':
pd.Series(['label_A' for l in range(0, n_rows)],
dtype='category'),
'cat_B':
pd.Series(['label_B' for l in range(0, n_rows)],
dtype='category')
})
res = self.data.annotation_put_fbs("obs", fbs)
self.assertEqual(res, json.dumps({"status": "OK"}))
self.assertTrue(path.exists(self.annotations_file))
df = pd.read_csv(self.annotations_file, index_col=0, header=0, comment='#')
df = pd.read_csv(self.annotations_file,
index_col=0,
header=0,
comment='#')
self.assertEqual(df.shape, (n_rows, 2))
self.assertEqual(set(df.columns), set(['cat_A', 'cat_B']))
self.assertTrue(self.data.original_obs_index.equals(df.index))
self.assertTrue(np.all(df['cat_A'] == ['label_A' for l in range(0, n_rows)]))
self.assertTrue(np.all(df['cat_B'] == ['label_B' for l in range(0, n_rows)]))
self.assertTrue(
np.all(df['cat_A'] == ['label_A' for l in range(0, n_rows)]))
self.assertTrue(
np.all(df['cat_B'] == ['label_B' for l in range(0, n_rows)]))
# verify complete overwrite on second attempt, AND rotation occurs
fbs = self.make_fbs({
'cat_A': pd.Series(['label_A1' for l in range(0, n_rows)], dtype='category'),
'cat_C': pd.Series(['label_C' for l in range(0, n_rows)], dtype='category')
'cat_A':
pd.Series(['label_A1' for l in range(0, n_rows)],
dtype='category'),
'cat_C':
pd.Series(['label_C' for l in range(0, n_rows)],
dtype='category')
})
res = self.data.annotation_put_fbs("obs", fbs)
self.assertEqual(res, json.dumps({"status": "OK"}))
self.assertTrue(path.exists(self.annotations_file))
df = pd.read_csv(self.annotations_file, index_col=0, header=0, comment='#')
df = pd.read_csv(self.annotations_file,
index_col=0,
header=0,
comment='#')
self.assertEqual(set(df.columns), set(['cat_A', 'cat_C']))
self.assertTrue(np.all(df['cat_A'] == ['label_A1' for l in range(0, n_rows)]))
self.assertTrue(np.all(df['cat_C'] == ['label_C' for l in range(0, n_rows)]))
self.assertTrue(
np.all(df['cat_A'] == ['label_A1' for l in range(0, n_rows)]))
self.assertTrue(
np.all(df['cat_C'] == ['label_C' for l in range(0, n_rows)]))
# rotation
name, ext = path.splitext(self.annotations_file)
@@ -93,8 +115,12 @@ class WritableAnnotationTest(unittest.TestCase):
# verify we stop rotation at 9
n_rows = self.data.data.obs.shape[0]
fbs = self.make_fbs({
'cat_A': pd.Series(['label_A' for l in range(0, n_rows)], dtype='category'),
'cat_B': pd.Series(['label_B' for l in range(0, n_rows)], dtype='category')
'cat_A':
pd.Series(['label_A' for l in range(0, n_rows)],
dtype='category'),
'cat_B':
pd.Series(['label_B' for l in range(0, n_rows)],
dtype='category')
})
for i in range(0, 11):
res = self.data.annotation_put_fbs("obs", fbs)
@@ -112,8 +138,12 @@ class WritableAnnotationTest(unittest.TestCase):
n_rows = self.data.data.obs.shape[0]
fbs = self.make_fbs({
'cat_A': pd.Series(['label_A' for l in range(0, n_rows)], dtype='category'),
'cat_B': pd.Series(['label_B' for l in range(0, n_rows)], dtype='category')
'cat_A':
pd.Series(['label_A' for l in range(0, n_rows)],
dtype='category'),
'cat_B':
pd.Series(['label_B' for l in range(0, n_rows)],
dtype='category')
})
# put
@@ -129,27 +159,30 @@ class WritableAnnotationTest(unittest.TestCase):
self.assertEqual(annotations["n_cols"], 7)
self.assertIsNone(annotations["row_idx"])
self.assertEqual(annotations["col_idx"], [
obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain", "cat_A", "cat_B"
obs_index_col_name, "n_genes", "percent_mito", "n_counts",
"louvain", "cat_A", "cat_B"
])
col_idx = annotations["col_idx"]
self.assertEqual(annotations["columns"][col_idx.index('cat_A')], [
'label_A' for l in range(0, n_rows)
])
self.assertEqual(annotations["columns"][col_idx.index('cat_B')], [
'label_B' for l in range(0, n_rows)
])
self.assertEqual(annotations["columns"][col_idx.index('cat_A')],
['label_A' for l in range(0, n_rows)])
self.assertEqual(annotations["columns"][col_idx.index('cat_B')],
['label_B' for l in range(0, n_rows)])
# verify the schema was updated
all_col_schema = {c["name"]: c for c in schema["annotations"]["obs"]["columns"]}
self.assertEqual(all_col_schema["cat_A"], {
"name": "cat_A",
"type": "categorical",
"categories": ["label_A"],
"writable": True
})
self.assertEqual(all_col_schema["cat_B"], {
"name": "cat_B",
"type": "categorical",
"categories": ["label_B"],
"writable": True
})
all_col_schema = {
c["name"]: c for c in schema["annotations"]["obs"]["columns"]
}
self.assertEqual(
all_col_schema["cat_A"], {
"name": "cat_A",
"type": "categorical",
"categories": ["label_A"],
"writable": True
})
self.assertEqual(
all_col_schema["cat_B"], {
"name": "cat_B",
"type": "categorical",
"categories": ["label_B"],
"writable": True
})

7
server/utils/utils.py
View File

@@ -12,12 +12,15 @@ def find_available_port(host, port=5005):
for port_to_try in range(port, port + num_ports_to_try):
if is_port_available(host, port_to_try):
return port_to_try
raise socket.error(errno.EADDRINUSE, f"No port in range {port} - {port + num_ports_to_try - 1} available.")
raise socket.error(
errno.EADDRINUSE,
f"No port in range {port} - {port + num_ports_to_try - 1} available.")
def is_port_available(host, port):
is_available = False
with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
with contextlib.closing(socket.socket(socket.AF_INET,
socket.SOCK_STREAM)) as s:
try:
s.bind((host, port))
is_available = True