fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-10-04 21:38:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Apply yapf to python files

Browse Source
This commit is contained in:
Matt Weiden
2019-12-19 15:20:41 -08:00
parent 42a8d45bd7
commit cdca128a01
43 changed files with 1143 additions and 678 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/.flake8
+12
View File
@@ -0,0 +1,12 @@
[flake8]
ignore =
# split before binary operator
W504,
# visually indented line with same indent as next logical line,
E129,
# whitespacek before ':'
E203,
# unexpected spaces around keyword / parameter equals
E251
max-line-length=120
server/Makefile
+1 -1
View File
@@ -1,6 +1,6 @@
.PHONY: fmt .PHONY: fmt
fmt: fmt:
yapf -ir . yapf -ipr .
.PHONY: lint .PHONY: lint
lint: lint:
server/__main__.py
+2 -2
View File
@@ -5,11 +5,11 @@ if __package__ is None:
PKG_PATH = Path(__file__).parent PKG_PATH = Path(__file__).parent
sys.path.insert(0, str(PKG_PATH.parent)) sys.path.insert(0, str(PKG_PATH.parent))
import server # noqa F401 import server # noqa F401
__package__ = PKG_PATH.name __package__ = PKG_PATH.name
# Main thing # Main thing
from .cli.cli import cli # noqa F402 from .cli.cli import cli # noqa F402
cli() cli()
server/app/app.py
+5 -1
View File
@@ -12,9 +12,13 @@ from server.app.web import webapp
class Server: class Server:
def __init__(self): def __init__(self):
self.data = None self.data = None
self.cache = Cache(config={"CACHE_TYPE": "simple", "CACHE_DEFAULT_TIMEOUT": 860_000}) self.cache = Cache(config={
"CACHE_TYPE": "simple",
"CACHE_DEFAULT_TIMEOUT": 860_000
})
self.app = None self.app = None
def create_app(self): def create_app(self):
server/app/driver/driver.py
+25 -6
View File
@@ -1,5 +1,4 @@
from abc import ABCMeta, abstractmethod from abc import ABCMeta, abstractmethod
""" """
Sort order for methods Sort order for methods
1. Initialize 1. Initialize
@@ -11,6 +10,7 @@ Sort order for methods
class CXGDriver(metaclass=ABCMeta): class CXGDriver(metaclass=ABCMeta):
def __init__(self, data_locator=None, args={}): def __init__(self, data_locator=None, args={}):
self.config = self._get_default_config() self.config = self._get_default_config()
self.config.update(args) self.config.update(args)
@@ -49,14 +49,29 @@ class CXGDriver(metaclass=ABCMeta):
@property @property
def features(self): def features(self):
features = { features = {
"cluster": {"available": False}, "cluster": {
"layout": {"obs": {"available": False}, "var": {"available": False}}, "available": False
"diffexp": {"available": True, "interactiveLimit": 50000} },
"layout": {
"obs": {
"available": False
},
"var": {
"available": False
}
},
"diffexp": {
"available": True,
"interactiveLimit": 50000
}
} }
# TODO - Interactive limit should be generated from the actual available methods see GH issue #94 # TODO - Interactive limit should be generated from the actual available methods see GH issue #94
if self.config["layout"]: if self.config["layout"]:
# TODO handle "var" when gene layout becomes available # TODO handle "var" when gene layout becomes available
features["layout"]["obs"] = {"available": True, "interactiveLimit": 50000} features["layout"]["obs"] = {
"available": True,
"interactiveLimit": 50000
}
return features return features
@abstractmethod @abstractmethod
@@ -92,7 +107,11 @@ class CXGDriver(metaclass=ABCMeta):
pass pass
@abstractmethod @abstractmethod
def diffexp_topN(self, obsFilter1, obsFilter2, top_n=None, interactive_limit=None): def diffexp_topN(self,
obsFilter1,
obsFilter2,
top_n=None,
interactive_limit=None):
""" """
Computes the top N differentially expressed variables between two observation sets. If mode Computes the top N differentially expressed variables between two observation sets. If mode
is "TOP_N", then stats for the top N is "TOP_N", then stats for the top N
server/app/rest_api/rest.py
+77 -68
View File
@@ -8,13 +8,9 @@ from flask_restful import Api, Resource
from server import __version__ as cellxgene_version from server import __version__ as cellxgene_version
from anndata import __version__ as anndata_version from anndata import __version__ as anndata_version
from server.app.util.constants import ( from server.app.util.constants import (Axis, DiffExpMode,
Axis, JSON_NaN_to_num_warning_msg, CXGUID,
DiffExpMode, CXG_ANNO_COLLECTION)
JSON_NaN_to_num_warning_msg,
CXGUID,
CXG_ANNO_COLLECTION
)
from server.app.util.errors import ( from server.app.util.errors import (
FilterError, FilterError,
InteractiveError, InteractiveError,
@@ -25,15 +21,20 @@ from server.app.util.errors import (
class SchemaAPI(Resource): class SchemaAPI(Resource):
def get(self): def get(self):
cxguid = get_userid(session) cxguid = get_userid(session)
anno_collection = get_anno_collection(session) anno_collection = get_anno_collection(session)
return make_response( return make_response(
jsonify({"schema": current_app.data.get_schema(uid=cxguid, collection=anno_collection)}), HTTPStatus.OK jsonify({
) "schema":
current_app.data.get_schema(uid=cxguid,
collection=anno_collection)
}), HTTPStatus.OK)
class ConfigAPI(Resource): class ConfigAPI(Resource):
def get(self): def get(self):
cxguid = get_userid(session) cxguid = get_userid(session)
anno_collection = get_anno_collection(session) anno_collection = get_anno_collection(session)
@@ -69,7 +70,8 @@ class ConfigAPI(Resource):
"about-dataset": current_app.config["ABOUT_DATASET"] "about-dataset": current_app.config["ABOUT_DATASET"]
}, },
"parameters": { "parameters": {
**current_app.data.get_config_parameters(uid=cxguid, collection=anno_collection) **current_app.data.get_config_parameters(uid=cxguid,
collection=anno_collection)
}, },
"library_versions": { "library_versions": {
"cellxgene": cellxgene_version, "cellxgene": cellxgene_version,
@@ -82,42 +84,48 @@ class ConfigAPI(Resource):
class AnnotationsObsAPI(Resource): class AnnotationsObsAPI(Resource):
def get(self): def get(self):
fields = request.args.getlist("annotation-name", None) fields = request.args.getlist("annotation-name", None)
preferred_mimetype = request.accept_mimetypes.best_match( preferred_mimetype = request.accept_mimetypes.best_match(
["application/octet-stream"] ["application/octet-stream"])
)
cxguid = get_userid(session) cxguid = get_userid(session)
anno_collection = get_anno_collection(session) anno_collection = get_anno_collection(session)
try: try:
if preferred_mimetype == "application/octet-stream": if preferred_mimetype == "application/octet-stream":
fbs = current_app.data.annotation_to_fbs_matrix("obs", fields, uid=cxguid, collection=anno_collection) fbs = current_app.data.annotation_to_fbs_matrix(
return make_response(fbs, "obs", fields, uid=cxguid, collection=anno_collection)
HTTPStatus.OK, return make_response(
{"Content-Type": "application/octet-stream"}) fbs, HTTPStatus.OK,
{"Content-Type": "application/octet-stream"})
else: else:
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) return make_response(
f"Unsupported MIME type '{request.accept_mimetypes}'",
HTTPStatus.NOT_ACCEPTABLE)
except KeyError: except KeyError:
return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST) return make_response(f"Error bad key in {fields}",
HTTPStatus.BAD_REQUEST)
except ValueError as e: except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR) return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
def put(self): def put(self):
cxguid = get_userid(session) cxguid = get_userid(session)
anno_collection = request.args.get("annotation-collection-name", default=None) anno_collection = request.args.get("annotation-collection-name",
default=None)
if anno_collection is not None: if anno_collection is not None:
if not is_safe_collection_name(anno_collection): if not is_safe_collection_name(anno_collection):
return make_response(f"Error, bad annotation collection name", HTTPStatus.BAD_REQUEST) return make_response(f"Error, bad annotation collection name",
HTTPStatus.BAD_REQUEST)
set_anno_collection(session, anno_collection) set_anno_collection(session, anno_collection)
else: else:
anno_collection = get_anno_collection(session) anno_collection = get_anno_collection(session)
try: try:
fbs = request.get_data() fbs = request.get_data()
res = current_app.data.annotation_put_fbs("obs", fbs, uid=cxguid, collection=anno_collection) res = current_app.data.annotation_put_fbs(
return make_response( "obs", fbs, uid=cxguid, collection=anno_collection)
res, HTTPStatus.OK, {"Content-Type": "application/json"} return make_response(res, HTTPStatus.OK,
) {"Content-Type": "application/json"})
except (ValueError, DisabledFeatureError, KeyError) as e: except (ValueError, DisabledFeatureError, KeyError) as e:
return make_response(str(e), HTTPStatus.BAD_REQUEST) return make_response(str(e), HTTPStatus.BAD_REQUEST)
except Exception as e: except Exception as e:
@@ -125,41 +133,44 @@ class AnnotationsObsAPI(Resource):
class AnnotationsVarAPI(Resource): class AnnotationsVarAPI(Resource):
def get(self): def get(self):
fields = request.args.getlist("annotation-name", None) fields = request.args.getlist("annotation-name", None)
preferred_mimetype = request.accept_mimetypes.best_match( preferred_mimetype = request.accept_mimetypes.best_match(
["application/octet-stream"] ["application/octet-stream"])
)
try: try:
if preferred_mimetype == "application/octet-stream": if preferred_mimetype == "application/octet-stream":
return make_response(current_app.data.annotation_to_fbs_matrix("var", fields), return make_response(
HTTPStatus.OK, current_app.data.annotation_to_fbs_matrix("var", fields),
{"Content-Type": "application/octet-stream"}) HTTPStatus.OK, {"Content-Type": "application/octet-stream"})
else: else:
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) return make_response(
f"Unsupported MIME type '{request.accept_mimetypes}'",
HTTPStatus.NOT_ACCEPTABLE)
except KeyError: except KeyError:
return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST) return make_response(f"Error bad key in {fields}",
HTTPStatus.BAD_REQUEST)
except ValueError as e: except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR) return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
class DataVarAPI(Resource): class DataVarAPI(Resource):
def put(self): def put(self):
preferred_mimetype = request.accept_mimetypes.best_match( preferred_mimetype = request.accept_mimetypes.best_match(
["application/octet-stream"] ["application/octet-stream"])
)
try: try:
if preferred_mimetype == "application/octet-stream": if preferred_mimetype == "application/octet-stream":
filter_json = request.get_json() filter_json = request.get_json()
filter = filter_json["filter"] if filter_json else None filter = filter_json["filter"] if filter_json else None
return make_response( return make_response(
current_app.data.data_frame_to_fbs_matrix( current_app.data.data_frame_to_fbs_matrix(filter,
filter, axis=Axis.VAR axis=Axis.VAR),
), HTTPStatus.OK, {"Content-Type": "application/octet-stream"})
HTTPStatus.OK,
{"Content-Type": "application/octet-stream"})
else: else:
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) return make_response(
f"Unsupported MIME type '{request.accept_mimetypes}'",
HTTPStatus.NOT_ACCEPTABLE)
except FilterError as e: except FilterError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST) return make_response(e.message, HTTPStatus.BAD_REQUEST)
except ValueError as e: except ValueError as e:
@@ -167,43 +178,39 @@ class DataVarAPI(Resource):
class DiffExpObsAPI(Resource): class DiffExpObsAPI(Resource):
def post(self): def post(self):
args = request.get_json() args = request.get_json()
# confirm mode is present and legal # confirm mode is present and legal
try: try:
mode = DiffExpMode(args["mode"]) mode = DiffExpMode(args["mode"])
except KeyError: except KeyError:
return make_response("Error: mode is required", HTTPStatus.BAD_REQUEST) return make_response("Error: mode is required",
HTTPStatus.BAD_REQUEST)
except ValueError: except ValueError:
return make_response( return make_response(f"Error: invalid mode option {args['mode']}",
f"Error: invalid mode option {args['mode']}", HTTPStatus.BAD_REQUEST HTTPStatus.BAD_REQUEST)
)
# Validate filters # Validate filters
if mode == DiffExpMode.VAR_FILTER or "varFilter" in args: if mode == DiffExpMode.VAR_FILTER or "varFilter" in args:
# not NOT_IMPLEMENTED # not NOT_IMPLEMENTED
return make_response( return make_response("mode=varfilter not implemented",
"mode=varfilter not implemented", HTTPStatus.NOT_IMPLEMENTED HTTPStatus.NOT_IMPLEMENTED)
)
if mode == DiffExpMode.TOP_N and "count" not in args: if mode == DiffExpMode.TOP_N and "count" not in args:
return make_response( return make_response("mode=topN requires a count parameter",
"mode=topN requires a count parameter", HTTPStatus.BAD_REQUEST HTTPStatus.BAD_REQUEST)
)
if "set1" not in args: if "set1" not in args:
return make_response("set1 is required.", HTTPStatus.BAD_REQUEST) return make_response("set1 is required.", HTTPStatus.BAD_REQUEST)
if Axis.VAR in args["set1"]["filter"]: if Axis.VAR in args["set1"]["filter"]:
return make_response( return make_response("Var filter not allowed for set1",
"Var filter not allowed for set1", HTTPStatus.BAD_REQUEST HTTPStatus.BAD_REQUEST)
)
# set2 # set2
if "set2" not in args: if "set2" not in args:
return make_response( return make_response("Set2 as inverse of set1 is not implemented",
"Set2 as inverse of set1 is not implemented", HTTPStatus.NOT_IMPLEMENTED HTTPStatus.NOT_IMPLEMENTED)
)
if Axis.VAR in args["set2"]["filter"]: if Axis.VAR in args["set2"]["filter"]:
return make_response( return make_response("Var filter not allowed for set2",
"Var filter not allowed for set2", HTTPStatus.BAD_REQUEST HTTPStatus.BAD_REQUEST)
)
set1_filter = args["set1"]["filter"] set1_filter = args["set1"]["filter"]
set2_filter = args.get("set2", {"filter": {}})["filter"] set2_filter = args.get("set2", {"filter": {}})["filter"]
@@ -219,13 +226,13 @@ class DiffExpObsAPI(Resource):
count, count,
current_app.data.features["diffexp"]["interactiveLimit"], current_app.data.features["diffexp"]["interactiveLimit"],
) )
return make_response( return make_response(diffexp, HTTPStatus.OK,
diffexp, HTTPStatus.OK, {"Content-Type": "application/json"} {"Content-Type": "application/json"})
)
except (ValueError, FilterError) as e: except (ValueError, FilterError) as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST) return make_response(e.message, HTTPStatus.BAD_REQUEST)
except InteractiveError: except InteractiveError:
return make_response("Non-interactive request", HTTPStatus.FORBIDDEN) return make_response("Non-interactive request",
HTTPStatus.FORBIDDEN)
except JSONEncodingValueError as e: except JSONEncodingValueError as e:
# JSON encoding failure, usually due to bad data # JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg) warnings.warn(JSON_NaN_to_num_warning_msg)
@@ -235,17 +242,19 @@ class DiffExpObsAPI(Resource):
class LayoutObsAPI(Resource): class LayoutObsAPI(Resource):
def get(self): def get(self):
preferred_mimetype = request.accept_mimetypes.best_match( preferred_mimetype = request.accept_mimetypes.best_match(
["application/octet-stream"] ["application/octet-stream"])
)
try: try:
if preferred_mimetype == "application/octet-stream": if preferred_mimetype == "application/octet-stream":
return make_response(current_app.data.layout_to_fbs_matrix(), return make_response(
HTTPStatus.OK, current_app.data.layout_to_fbs_matrix(), HTTPStatus.OK,
{"Content-Type": "application/octet-stream"}) {"Content-Type": "application/octet-stream"})
else: else:
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) return make_response(
f"Unsupported MIME type '{request.accept_mimetypes}'",
HTTPStatus.NOT_ACCEPTABLE)
except PrepareError as e: except PrepareError as e:
return make_response(e.message, HTTPStatus.INTERNAL_SERVER_ERROR) return make_response(e.message, HTTPStatus.INTERNAL_SERVER_ERROR)
except ValueError as e: except ValueError as e:
server/app/scanpy_engine/diffexp.py
+11 -6
View File
@@ -32,8 +32,8 @@ def _mean_var_n(X):
v = sumsq / (n - 1) v = sumsq / (n - 1)
if fp_err_occurred: if fp_err_occurred:
mean[np.isfinite(mean) == False] = 0 # noqa: E712 mean[np.isfinite(mean) == False] = 0 # noqa: E712
v[np.isfinite(v) == False] = 0 # noqa: E712 v[np.isfinite(v) == False] = 0 # noqa: E712
return mean, v, n return mean, v, n
@@ -76,7 +76,7 @@ def diffexp_ttest(adata, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
# degrees of freedom for Welch's t-test # degrees of freedom for Welch's t-test
with np.errstate(divide="ignore", invalid="ignore"): with np.errstate(divide="ignore", invalid="ignore"):
dof = sum_vn ** 2 / (vnA ** 2 / (nA - 1) + vnB ** 2 / (nB - 1)) dof = sum_vn**2 / (vnA**2 / (nA - 1) + vnB**2 / (nB - 1))
dof[np.isnan(dof)] = 1 dof[np.isnan(dof)] = 1
# Welch's t-test score calculation # Welch's t-test score calculation
@@ -93,13 +93,15 @@ def diffexp_ttest(adata, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
logfoldchanges = np.log2(np.abs((meanA + 1e-9) / (meanB + 1e-9))) logfoldchanges = np.log2(np.abs((meanA + 1e-9) / (meanB + 1e-9)))
# find all with lfc > cutoff # find all with lfc > cutoff
lfc_above_cutoff_idx = np.nonzero(np.abs(logfoldchanges) > diffexp_lfc_cutoff)[0] lfc_above_cutoff_idx = np.nonzero(
np.abs(logfoldchanges) > diffexp_lfc_cutoff)[0]
stats_to_sort = np.abs(tscores) stats_to_sort = np.abs(tscores)
# derive sort order # derive sort order
if lfc_above_cutoff_idx.shape[0] > top_n: if lfc_above_cutoff_idx.shape[0] > top_n:
# partition top N # partition top N
rel_t_partition = np.argpartition(stats_to_sort[lfc_above_cutoff_idx], -top_n)[-top_n:] rel_t_partition = np.argpartition(stats_to_sort[lfc_above_cutoff_idx],
-top_n)[-top_n:]
t_partition = lfc_above_cutoff_idx[rel_t_partition] t_partition = lfc_above_cutoff_idx[rel_t_partition]
# sort the top N partition # sort the top N partition
rel_sort_order = np.argsort(stats_to_sort[t_partition])[::-1] rel_sort_order = np.argsort(stats_to_sort[t_partition])[::-1]
@@ -117,5 +119,8 @@ def diffexp_ttest(adata, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
pvals_adj_top_n = pvals_adj[sort_order] pvals_adj_top_n = pvals_adj[sort_order]
# varIndex, logfoldchange, pval, pval_adj # varIndex, logfoldchange, pval, pval_adj
result = [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in range(top_n)] result = [[
sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i],
pvals_adj_top_n[i]
] for i in range(top_n)]
return result return result
server/app/scanpy_engine/labels.py
+11 -4
View File
@@ -8,8 +8,13 @@ import pandas as pd
def read_labels(fname): def read_labels(fname):
if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0: if fname is not None and os.path.exists(
return pd.read_csv(fname, dtype='category', index_col=0, header=0, comment='#') fname) and os.path.getsize(fname) > 0:
return pd.read_csv(fname,
dtype='category',
index_col=0,
header=0,
comment='#')
else: else:
return pd.DataFrame() return pd.DataFrame()
@@ -47,13 +52,15 @@ def backup(fname, backup_dir, max_backups=9):
fname_base_root, fname_base_ext = os.path.splitext(fname_base) fname_base_root, fname_base_ext = os.path.splitext(fname_base)
# don't use ISO standard time format, as it contains characters illegal on some filesytems. # don't use ISO standard time format, as it contains characters illegal on some filesytems.
nowish = datetime.now().strftime('%Y-%m-%dT%H-%M-%S') nowish = datetime.now().strftime('%Y-%m-%dT%H-%M-%S')
backup_fname = os.path.join(backup_dir, f"{fname_base_root}-{nowish}{fname_base_ext}") backup_fname = os.path.join(backup_dir,
f"{fname_base_root}-{nowish}{fname_base_ext}")
if os.path.exists(backup_fname): if os.path.exists(backup_fname):
os.remove(backup_fname) os.remove(backup_fname)
os.rename(fname, backup_fname) os.rename(fname, backup_fname)
# prune the backup_dir to max number of backup files, keeping the most recent backups # prune the backup_dir to max number of backup files, keeping the most recent backups
backups = list(filter(lambda s: s.startswith(fname_base_root), os.listdir(backup_dir))) backups = list(
filter(lambda s: s.startswith(fname_base_root), os.listdir(backup_dir)))
excess_count = len(backups) - max_backups excess_count = len(backups) - max_backups
if excess_count > 0: if excess_count > 0:
backups.sort() backups.sort()
server/app/scanpy_engine/matrix_proxy.py
+2 -2
View File
@@ -1,6 +1,4 @@
from server.app.util.matrix_proxy import MatrixProxyView, ArrayProxyView from server.app.util.matrix_proxy import MatrixProxyView, ArrayProxyView
""" """
AnnData/h5py are inconsistent in the API supported by various types of AnnData/h5py are inconsistent in the API supported by various types of
X matrices. Sometimes you get a fully ndarray, sometims a Scipy sparse X matrices. Sometimes you get a fully ndarray, sometims a Scipy sparse
@@ -17,6 +15,7 @@ class ArrayProxyView_anndata_h5py(ArrayProxyView):
override to handle sparse getitem semantics, which differ override to handle sparse getitem semantics, which differ
from numpy. from numpy.
""" """
def toarray(self): def toarray(self):
""" sadly, sparse indexing doesn't drop dimensions like numpy! """ """ sadly, sparse indexing doesn't drop dimensions like numpy! """
arr = self.m[self._index[0], self._index[1]] arr = self.m[self._index[0], self._index[1]]
@@ -30,6 +29,7 @@ class MatrixProxy_anndata_h5py(MatrixProxyView):
AnnData sparse array stored in H5AD, or proxies for backed data. AnnData sparse array stored in H5AD, or proxies for backed data.
None of these handle indexing very well, so we plop a proxy on top. None of these handle indexing very well, so we plop a proxy on top.
""" """
@classmethod @classmethod
def __supports__(cls): def __supports__(cls):
return ("anndata.h5py.h5sparse.SparseDataset", return ("anndata.h5py.h5sparse.SparseDataset",
server/app/scanpy_engine/scanpy_engine.py
+116 -76
View File
@@ -37,6 +37,7 @@ def has_method(o, name):
class ScanpyEngine(CXGDriver): class ScanpyEngine(CXGDriver):
def __init__(self, data_locator=None, args={}): def __init__(self, data_locator=None, args={}):
super().__init__(data_locator, args) super().__init__(data_locator, args)
# lock used to protect label file write ops # lock used to protect label file write ops
@@ -75,7 +76,8 @@ class ScanpyEngine(CXGDriver):
if self.config["annotations"]: if self.config["annotations"]:
if uid is not None: if uid is not None:
params.update({ params.update({
"annotations-user-data-idhash": self.get_userdata_idhash(uid) "annotations-user-data-idhash":
self.get_userdata_idhash(uid)
}) })
if self.config['annotations_file'] is not None: if self.config['annotations_file'] is not None:
# user has hard-wired the name of the annotation data collection # user has hard-wired the name of the annotation data collection
@@ -119,7 +121,8 @@ class ScanpyEngine(CXGDriver):
""" """
self.original_obs_index = self.data.obs.index self.original_obs_index = self.data.obs.index
for (ax_name, config_name) in ((Axis.OBS, "obs_names"), (Axis.VAR, "var_names")): for (ax_name, config_name) in ((Axis.OBS, "obs_names"), (Axis.VAR,
"var_names")):
name = self.config[config_name] name = self.config[config_name]
df_axis = getattr(self.data, str(ax_name)) df_axis = getattr(self.data, str(ax_name))
if name is None: if name is None:
@@ -128,8 +131,7 @@ class ScanpyEngine(CXGDriver):
raise KeyError( raise KeyError(
f"Values in {ax_name}.index must be unique. " f"Values in {ax_name}.index must be unique. "
"Please prepare data to contain unique index values, or specify an " "Please prepare data to contain unique index values, or specify an "
"alternative with --{ax_name}-name." "alternative with --{ax_name}-name.")
)
name = self._create_unique_column_name(df_axis.columns, "name_") name = self._create_unique_column_name(df_axis.columns, "name_")
self.config[config_name] = name self.config[config_name] = name
# reset index to simple range; alias name to point at the # reset index to simple range; alias name to point at the
@@ -141,8 +143,7 @@ class ScanpyEngine(CXGDriver):
if not df_axis[name].is_unique: if not df_axis[name].is_unique:
raise KeyError( raise KeyError(
f"Values in {ax_name}.{name} must be unique. " f"Values in {ax_name}.{name} must be unique. "
"Please prepare data to contain unique values." "Please prepare data to contain unique values.")
)
df_axis.reset_index(drop=True, inplace=True) df_axis.reset_index(drop=True, inplace=True)
else: else:
# user specified a non-existent column name # user specified a non-existent column name
@@ -189,8 +190,7 @@ class ScanpyEngine(CXGDriver):
schema["categories"] = dtype.categories.tolist() schema["categories"] = dtype.categories.tolist()
else: else:
raise TypeError( raise TypeError(
f"Annotations of type {dtype} are unsupported by cellxgene." f"Annotations of type {dtype} are unsupported by cellxgene.")
)
return schema return schema
@requires_data @requires_data
@@ -211,7 +211,9 @@ class ScanpyEngine(CXGDriver):
"columns": [] "columns": []
} }
}, },
"layout": {"obs": []} "layout": {
"obs": []
}
} }
for ax in Axis: for ax in Axis:
curr_axis = getattr(self.data, str(ax)) curr_axis = getattr(self.data, str(ax))
@@ -250,7 +252,8 @@ class ScanpyEngine(CXGDriver):
Used to create safe annotations output file names. Used to create safe annotations output file names.
""" """
id = (uid + self.data_locator.abspath()).encode() id = (uid + self.data_locator.abspath()).encode()
idhash = base64.b32encode(blake2b(id, digest_size=5).digest()).decode('utf-8') idhash = base64.b32encode(blake2b(
id, digest_size=5).digest()).decode('utf-8')
return idhash return idhash
def get_anno_fname(self, uid=None, collection=None): def get_anno_fname(self, uid=None, collection=None):
@@ -265,7 +268,8 @@ class ScanpyEngine(CXGDriver):
if uid is None or collection is None: if uid is None or collection is None:
return None return None
idhash = self.get_userdata_idhash(uid) idhash = self.get_userdata_idhash(uid)
return os.path.join(self.get_anno_output_dir(), f"{collection}-{idhash}.csv") return os.path.join(self.get_anno_output_dir(),
f"{collection}-{idhash}.csv")
def get_anno_output_dir(self): def get_anno_output_dir(self):
""" return the current annotation output directory """ """ return the current annotation output directory """
@@ -276,7 +280,8 @@ class ScanpyEngine(CXGDriver):
return self.config['annotations_output_dir'] return self.config['annotations_output_dir']
if self.config['annotations_file']: if self.config['annotations_file']:
return os.path.dirname(os.path.abspath(self.config['annotations_file'])) return os.path.dirname(
os.path.abspath(self.config['annotations_file']))
return os.getcwd() return os.getcwd()
@@ -308,15 +313,14 @@ class ScanpyEngine(CXGDriver):
"https://github.com/theislab/scanpy_usage/blob/master/170505_seurat/info_h5ad.md to " "https://github.com/theislab/scanpy_usage/blob/master/170505_seurat/info_h5ad.md to "
"learn more about this format. You may be able to convert your file into this format " "learn more about this format. You may be able to convert your file into this format "
"using `cellxgene prepare`, please run `cellxgene prepare --help` for more " "using `cellxgene prepare`, please run `cellxgene prepare --help` for more "
"information." "information.")
)
except MemoryError: except MemoryError:
raise ScanpyFileError("Out of memory - file is too large for available memory.") raise ScanpyFileError(
"Out of memory - file is too large for available memory.")
except Exception as e: except Exception as e:
raise ScanpyFileError( raise ScanpyFileError(
f"{e} - file not found or is inaccessible. File must be an .h5ad object. " f"{e} - file not found or is inaccessible. File must be an .h5ad object. "
f"Please check your input and try again." f"Please check your input and try again.")
)
@requires_data @requires_data
def _validate_and_initialize(self): def _validate_and_initialize(self):
@@ -338,7 +342,8 @@ class ScanpyEngine(CXGDriver):
# heuristic # heuristic
n_values = self.data.shape[0] * self.data.shape[1] n_values = self.data.shape[0] * self.data.shape[1]
if (n_values > 1e8 and self.config['backed'] is True) or (n_values > 5e8): if (n_values > 1e8 and
self.config['backed'] is True) or (n_values > 5e8):
self.config.update({"diffexp_may_be_slow": True}) self.config.update({"diffexp_may_be_slow": True})
@requires_data @requires_data
@@ -352,9 +357,15 @@ class ScanpyEngine(CXGDriver):
# handle default # handle default
if layouts is None or len(layouts) == 0: if layouts is None or len(layouts) == 0:
# load default layouts from the data. # load default layouts from the data.
layouts = [key[2:] for key in self.data.obsm_keys() if type(key) == str and key.startswith("X_")] layouts = [
key[2:]
for key in self.data.obsm_keys()
if type(key) == str and key.startswith("X_")
]
if len(layouts) == 0: if len(layouts) == 0:
raise PrepareError(f"Unable to find any precomputed layouts within the dataset.") raise PrepareError(
f"Unable to find any precomputed layouts within the dataset."
)
# remove invalid layouts # remove invalid layouts
valid_layouts = [] valid_layouts = []
@@ -364,7 +375,9 @@ class ScanpyEngine(CXGDriver):
if layout_name not in obsm_keys: if layout_name not in obsm_keys:
warnings.warn(f"Ignoring unknown layout name: {layout}.") warnings.warn(f"Ignoring unknown layout name: {layout}.")
elif not self._is_valid_layout(self.data.obsm[layout_name]): elif not self._is_valid_layout(self.data.obsm[layout_name]):
warnings.warn(f"Ignoring layout due to malformed shape or data type: {layout}") warnings.warn(
f"Ignoring layout due to malformed shape or data type: {layout}"
)
else: else:
valid_layouts.append(layout) valid_layouts.append(layout)
@@ -381,22 +394,22 @@ class ScanpyEngine(CXGDriver):
* contains only finite values * contains only finite values
""" """
is_valid = type(arr) == np.ndarray and arr.dtype.kind in "fiu" is_valid = type(arr) == np.ndarray and arr.dtype.kind in "fiu"
is_valid = is_valid and arr.shape[0] == self.data.n_obs and arr.shape[1] >= 2 is_valid = is_valid and arr.shape[
0] == self.data.n_obs and arr.shape[1] >= 2
is_valid = is_valid and np.all(np.isfinite(arr)) is_valid = is_valid and np.all(np.isfinite(arr))
return is_valid return is_valid
@requires_data @requires_data
def _validate_data_types(self): def _validate_data_types(self):
if sparse.isspmatrix(self.data.X) and not sparse.isspmatrix_csc(self.data.X): if sparse.isspmatrix(
self.data.X) and not sparse.isspmatrix_csc(self.data.X):
warnings.warn( warnings.warn(
f"Scanpy data matrix is sparse, but not a CSC (columnar) matrix. " f"Scanpy data matrix is sparse, but not a CSC (columnar) matrix. "
f"Performance may be improved by using CSC." f"Performance may be improved by using CSC.")
)
if self.data.X.dtype != "float32": if self.data.X.dtype != "float32":
warnings.warn( warnings.warn(
f"Scanpy data matrix is in {self.data.X.dtype} format not float32. " f"Scanpy data matrix is in {self.data.X.dtype} format not float32. "
f"Precision may be truncated." f"Precision may be truncated.")
)
for ax in Axis: for ax in Axis:
curr_axis = getattr(self.data, str(ax)) curr_axis = getattr(self.data, str(ax))
for ann in curr_axis: for ann in curr_axis:
@@ -410,11 +423,11 @@ class ScanpyEngine(CXGDriver):
if datatype in downcast_map: if datatype in downcast_map:
warnings.warn( warnings.warn(
f"Scanpy annotation {ax}:{ann} is in unsupported format: {datatype}. " f"Scanpy annotation {ax}:{ann} is in unsupported format: {datatype}. "
f"Data will be downcast to {downcast_map[datatype]}." f"Data will be downcast to {downcast_map[datatype]}.")
)
if isinstance(datatype, CategoricalDtype): if isinstance(datatype, CategoricalDtype):
category_num = len(curr_axis[ann].dtype.categories) category_num = len(curr_axis[ann].dtype.categories)
if category_num > 500 and category_num > self.config['max_category_items']: if category_num > 500 and category_num > self.config[
'max_category_items']:
warnings.warn( warnings.warn(
f"{str(ax).title()} annotation '{ann}' has {category_num} categories, this may be " f"{str(ax).title()} annotation '{ann}' has {category_num} categories, this may be "
f"cumbersome or slow to display. We recommend setting the " f"cumbersome or slow to display. We recommend setting the "
@@ -432,31 +445,41 @@ class ScanpyEngine(CXGDriver):
# all lables must have a name, which must be unique and not used in obs column names # all lables must have a name, which must be unique and not used in obs column names
if not labels.columns.is_unique: if not labels.columns.is_unique:
raise KeyError(f"All column names specified in user annotations must be unique.") raise KeyError(
f"All column names specified in user annotations must be unique."
)
# the label index must be unique, and must have same values the anndata obs index # the label index must be unique, and must have same values the anndata obs index
if not labels.index.is_unique: if not labels.index.is_unique:
raise KeyError(f"All row index values specified in user annotations must be unique.") raise KeyError(
f"All row index values specified in user annotations must be unique."
)
if not labels.index.equals(self.original_obs_index): if not labels.index.equals(self.original_obs_index):
raise KeyError("Label file row index does not match H5AD file index. " raise KeyError(
"Please ensure that column zero (0) in the label file contain the same " "Label file row index does not match H5AD file index. "
"index values as the H5AD file.") "Please ensure that column zero (0) in the label file contain the same "
"index values as the H5AD file.")
duplicate_columns = list(set(labels.columns) & set(self.data.obs.columns)) duplicate_columns = list(
set(labels.columns) & set(self.data.obs.columns))
if len(duplicate_columns) > 0: if len(duplicate_columns) > 0:
raise KeyError(f"Labels file may not contain column names which overlap " raise KeyError(
f"with h5ad obs columns {duplicate_columns}") f"Labels file may not contain column names which overlap "
f"with h5ad obs columns {duplicate_columns}")
# labels must have same count as obs annotations # labels must have same count as obs annotations
if labels.shape[0] != self.data.obs.shape[0]: if labels.shape[0] != self.data.obs.shape[0]:
raise ValueError("Labels file must have same number of rows as h5ad file.") raise ValueError(
"Labels file must have same number of rows as h5ad file.")
@staticmethod @staticmethod
def _annotation_filter_to_mask(filter, d_axis, count): def _annotation_filter_to_mask(filter, d_axis, count):
mask = np.ones((count,), dtype=bool) mask = np.ones((count,), dtype=bool)
for v in filter: for v in filter:
if d_axis[v["name"]].dtype.name in ["boolean", "category", "object"]: if d_axis[v["name"]].dtype.name in [
"boolean", "category", "object"
]:
key_idx = np.in1d(getattr(d_axis, v["name"]), v["values"]) key_idx = np.in1d(getattr(d_axis, v["name"]), v["values"])
mask = np.logical_and(mask, key_idx) mask = np.logical_and(mask, key_idx)
else: else:
@@ -475,7 +498,7 @@ class ScanpyEngine(CXGDriver):
mask = np.zeros((count,), dtype=bool) mask = np.zeros((count,), dtype=bool)
for i in filter: for i in filter:
if type(i) == list: if type(i) == list:
mask[i[0]: i[1]] = True mask[i[0]:i[1]] = True
else: else:
mask[i] = True mask[i] = True
return mask return mask
@@ -485,14 +508,13 @@ class ScanpyEngine(CXGDriver):
mask = np.ones((count,), dtype=bool) mask = np.ones((count,), dtype=bool)
if "index" in filter: if "index" in filter:
mask = np.logical_and( mask = np.logical_and(
mask, ScanpyEngine._index_filter_to_mask(filter["index"], count) mask,
) ScanpyEngine._index_filter_to_mask(filter["index"], count))
if "annotation_value" in filter: if "annotation_value" in filter:
mask = np.logical_and( mask = np.logical_and(
mask, mask,
ScanpyEngine._annotation_filter_to_mask( ScanpyEngine._annotation_filter_to_mask(
filter["annotation_value"], d_axis, count filter["annotation_value"], d_axis, count),
),
) )
return mask return mask
@@ -508,16 +530,18 @@ class ScanpyEngine(CXGDriver):
if filter is not None: if filter is not None:
if Axis.OBS in filter: if Axis.OBS in filter:
obs_selector = self._axis_filter_to_mask( obs_selector = self._axis_filter_to_mask(
filter["obs"], self.data.obs, self.data.n_obs filter["obs"], self.data.obs, self.data.n_obs)
)
if Axis.VAR in filter: if Axis.VAR in filter:
var_selector = self._axis_filter_to_mask( var_selector = self._axis_filter_to_mask(
filter["var"], self.data.var, self.data.n_vars filter["var"], self.data.var, self.data.n_vars)
)
return obs_selector, var_selector return obs_selector, var_selector
@requires_data @requires_data
def annotation_to_fbs_matrix(self, axis, fields=None, uid=None, collection=None): def annotation_to_fbs_matrix(self,
axis,
fields=None,
uid=None,
collection=None):
if axis == Axis.OBS: if axis == Axis.OBS:
if self.config["annotations"]: if self.config["annotations"]:
try: try:
@@ -525,8 +549,7 @@ class ScanpyEngine(CXGDriver):
except Exception as e: except Exception as e:
raise ScanpyFileError( raise ScanpyFileError(
f"Error while loading label file: {e}, File must be in the .csv format, please check " f"Error while loading label file: {e}, File must be in the .csv format, please check "
f"your input and try again." f"your input and try again.")
)
else: else:
labels = None labels = None
@@ -547,7 +570,9 @@ class ScanpyEngine(CXGDriver):
fname = self.get_anno_fname(uid, collection) fname = self.get_anno_fname(uid, collection)
if not fname: if not fname:
raise ScanpyFileError("Writable annotations - unable to determine file name for annotations") raise ScanpyFileError(
"Writable annotations - unable to determine file name for annotations"
)
if axis != Axis.OBS: if axis != Axis.OBS:
raise ValueError("Only OBS dimension access is supported") raise ValueError("Only OBS dimension access is supported")
@@ -558,21 +583,27 @@ class ScanpyEngine(CXGDriver):
self._validate_label_data(new_label_df) # paranoia self._validate_label_data(new_label_df) # paranoia
# if any of the new column labels overlap with our existing labels, raise error # if any of the new column labels overlap with our existing labels, raise error
duplicate_columns = list(set(new_label_df.columns) & set(self.data.obs.columns)) duplicate_columns = list(
set(new_label_df.columns) & set(self.data.obs.columns))
if not new_label_df.columns.is_unique or len(duplicate_columns) > 0: if not new_label_df.columns.is_unique or len(duplicate_columns) > 0:
raise KeyError(f"Labels file may not contain column names which overlap " raise KeyError(
f"with h5ad obs columns {duplicate_columns}") f"Labels file may not contain column names which overlap "
f"with h5ad obs columns {duplicate_columns}")
# update our internal state and save it. Multi-threading often enabled, # update our internal state and save it. Multi-threading often enabled,
# so treat this as a critical section. # so treat this as a critical section.
with self.label_lock: with self.label_lock:
lastmod = self.data_locator.lastmodtime() lastmod = self.data_locator.lastmodtime()
lastmodstr = "'unknown'" if lastmod is None else lastmod.isoformat(timespec="seconds") lastmodstr = "'unknown'" if lastmod is None else lastmod.isoformat(
timespec="seconds")
header = f"# Annotations generated on {datetime.now().isoformat(timespec='seconds')} " \ header = f"# Annotations generated on {datetime.now().isoformat(timespec='seconds')} " \
f"using cellxgene version {cellxgene_version}\n" \ f"using cellxgene version {cellxgene_version}\n" \
f"# Input data file was {self.data_locator.uri_or_path}, " \ f"# Input data file was {self.data_locator.uri_or_path}, " \
f"which was last modified on {lastmodstr}\n" f"which was last modified on {lastmodstr}\n"
write_labels(fname, new_label_df, header, backup_dir=self.get_anno_backup_dir(uid, collection)) write_labels(fname,
new_label_df,
header,
backup_dir=self.get_anno_backup_dir(uid, collection))
return jsonify_scanpy({"status": "OK"}) return jsonify_scanpy({"status": "OK"})
@@ -591,41 +622,48 @@ class ScanpyEngine(CXGDriver):
if axis != Axis.VAR: if axis != Axis.VAR:
raise ValueError("Only VAR dimension access is supported") raise ValueError("Only VAR dimension access is supported")
try: try:
obs_selector, var_selector = self._filter_to_mask(filter, use_slices=False) obs_selector, var_selector = self._filter_to_mask(filter,
use_slices=False)
except (KeyError, IndexError, TypeError) as e: except (KeyError, IndexError, TypeError) as e:
raise FilterError(f"Error parsing filter: {e}") from e raise FilterError(f"Error parsing filter: {e}") from e
if obs_selector is not None: if obs_selector is not None:
raise FilterError("filtering on obs unsupported") raise FilterError("filtering on obs unsupported")
# Currently only handles VAR dimension # Currently only handles VAR dimension
X = MatrixProxy.create(self.data.X if var_selector is None X = MatrixProxy.create(
else self.data.X[:, var_selector]) self.data.X if var_selector is None else self.data.X[:,
return encode_matrix_fbs(X, col_idx=np.nonzero(var_selector)[0], row_idx=None) var_selector])
return encode_matrix_fbs(X,
col_idx=np.nonzero(var_selector)[0],
row_idx=None)
@requires_data @requires_data
def diffexp_topN(self, obsFilterA, obsFilterB, top_n=None, interactive_limit=None): def diffexp_topN(self,
obsFilterA,
obsFilterB,
top_n=None,
interactive_limit=None):
if Axis.VAR in obsFilterA or Axis.VAR in obsFilterB: if Axis.VAR in obsFilterA or Axis.VAR in obsFilterB:
raise FilterError("Observation filters may not contain vaiable conditions") raise FilterError(
"Observation filters may not contain vaiable conditions")
try: try:
obs_mask_A = self._axis_filter_to_mask( obs_mask_A = self._axis_filter_to_mask(obsFilterA["obs"],
obsFilterA["obs"], self.data.obs, self.data.n_obs self.data.obs,
) self.data.n_obs)
obs_mask_B = self._axis_filter_to_mask( obs_mask_B = self._axis_filter_to_mask(obsFilterB["obs"],
obsFilterB["obs"], self.data.obs, self.data.n_obs self.data.obs,
) self.data.n_obs)
except (KeyError, IndexError) as e: except (KeyError, IndexError) as e:
raise FilterError(f"Error parsing filter: {e}") from e raise FilterError(f"Error parsing filter: {e}") from e
if top_n is None: if top_n is None:
top_n = DEFAULT_TOP_N top_n = DEFAULT_TOP_N
result = diffexp_ttest( result = diffexp_ttest(self.data, obs_mask_A, obs_mask_B, top_n,
self.data, obs_mask_A, obs_mask_B, top_n, self.config['diffexp_lfc_cutoff'] self.config['diffexp_lfc_cutoff'])
)
try: try:
return jsonify_scanpy(result) return jsonify_scanpy(result)
except ValueError: except ValueError:
raise JSONEncodingValueError( raise JSONEncodingValueError(
"Error encoding differential expression to JSON" "Error encoding differential expression to JSON")
)
@requires_data @requires_data
def layout_to_fbs_matrix(self): def layout_to_fbs_matrix(self):
@@ -656,7 +694,9 @@ class ScanpyEngine(CXGDriver):
normalized_layout = normalized_layout + translate normalized_layout = normalized_layout + translate
normalized_layout = normalized_layout.astype(dtype=np.float32) normalized_layout = normalized_layout.astype(dtype=np.float32)
layout_data.append(pandas.DataFrame(normalized_layout, columns=[f"{layout}_0", f"{layout}_1"])) layout_data.append(
pandas.DataFrame(normalized_layout,
columns=[f"{layout}_0", f"{layout}_1"]))
except ValueError as e: except ValueError as e:
raise PrepareError( raise PrepareError(
server/app/util/constants.py
+1 -1
View File
@@ -1,10 +1,10 @@
from enum import Enum from enum import Enum
DEFAULT_TOP_N = 10 DEFAULT_TOP_N = 10
class AugmentedEnum(Enum): class AugmentedEnum(Enum):
def __hash__(self): def __hash__(self):
return self.value.__hash__() return self.value.__hash__()
server/app/util/data_locator.py
+5 -2
View File
@@ -27,7 +27,8 @@ class DataLocator():
def __init__(self, uri_or_path): def __init__(self, uri_or_path):
self.uri_or_path = uri_or_path self.uri_or_path = uri_or_path
self.protocol, self.path = DataLocator._get_protocol_and_path(uri_or_path) self.protocol, self.path = DataLocator._get_protocol_and_path(
uri_or_path)
# work-around for LocalFileSystem not treating file: and None as the same scheme/protocol # work-around for LocalFileSystem not treating file: and None as the same scheme/protocol
self.cname = self.path if self.protocol == 'file' else self.uri_or_path self.cname = self.path if self.protocol == 'file' else self.uri_or_path
# will throw RuntimeError if the protocol is unsupported # will throw RuntimeError if the protocol is unsupported
@@ -82,7 +83,8 @@ class DataLocator():
# if not local, create a tmp file system object to contain the data, # if not local, create a tmp file system object to contain the data,
# and clean it up when done. # and clean it up when done.
with self.open() as src, tempfile.NamedTemporaryFile(prefix="cellxgene_", delete=False) as tmp: with self.open() as src, tempfile.NamedTemporaryFile(
prefix="cellxgene_", delete=False) as tmp:
tmp.write(src.read()) tmp.write(src.read())
tmp.close() tmp.close()
src.close() src.close()
@@ -91,6 +93,7 @@ class DataLocator():
class LocalFilePath(): class LocalFilePath():
def __init__(self, tmp_path, delete=False): def __init__(self, tmp_path, delete=False):
self.tmp_path = tmp_path self.tmp_path = tmp_path
self.delete = delete self.delete = delete
server/app/util/fbs/NetEncoding/Column.py
+19 -5
View File
@@ -4,6 +4,7 @@
import flatbuffers import flatbuffers
class Column(object): class Column(object):
__slots__ = ['_tab'] __slots__ = ['_tab']
@@ -22,7 +23,8 @@ class Column(object):
def UType(self): def UType(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0: if o != 0:
return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos) return self._tab.Get(flatbuffers.number_types.Uint8Flags,
o + self._tab.Pos)
return 0 return 0
# Column # Column
@@ -35,7 +37,19 @@ class Column(object):
return obj return obj
return None return None
def ColumnStart(builder): builder.StartObject(2)
def ColumnAddUType(builder, uType): builder.PrependUint8Slot(0, uType, 0) def ColumnStart(builder):
def ColumnAddU(builder, u): builder.PrependUOffsetTRelativeSlot(1, flatbuffers.number_types.UOffsetTFlags.py_type(u), 0) builder.StartObject(2)
def ColumnEnd(builder): return builder.EndObject()
def ColumnAddUType(builder, uType):
builder.PrependUint8Slot(0, uType, 0)
def ColumnAddU(builder, u):
builder.PrependUOffsetTRelativeSlot(
1, flatbuffers.number_types.UOffsetTFlags.py_type(u), 0)
def ColumnEnd(builder):
return builder.EndObject()
server/app/util/fbs/NetEncoding/Float32Array.py
+22 -6
View File
@@ -4,6 +4,7 @@
import flatbuffers import flatbuffers
class Float32Array(object): class Float32Array(object):
__slots__ = ['_tab'] __slots__ = ['_tab']
@@ -23,14 +24,17 @@ class Float32Array(object):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0: if o != 0:
a = self._tab.Vector(o) a = self._tab.Vector(o)
return self._tab.Get(flatbuffers.number_types.Float32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4)) return self._tab.Get(
flatbuffers.number_types.Float32Flags,
a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
return 0 return 0
# Float32Array # Float32Array
def DataAsNumpy(self): def DataAsNumpy(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0: if o != 0:
return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Float32Flags, o) return self._tab.GetVectorAsNumpy(
flatbuffers.number_types.Float32Flags, o)
return 0 return 0
# Float32Array # Float32Array
@@ -40,7 +44,19 @@ class Float32Array(object):
return self._tab.VectorLen(o) return self._tab.VectorLen(o)
return 0 return 0
def Float32ArrayStart(builder): builder.StartObject(1)
def Float32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0) def Float32ArrayStart(builder):
def Float32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4) builder.StartObject(1)
def Float32ArrayEnd(builder): return builder.EndObject()
def Float32ArrayAddData(builder, data):
builder.PrependUOffsetTRelativeSlot(
0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
def Float32ArrayStartDataVector(builder, numElems):
return builder.StartVector(4, numElems, 4)
def Float32ArrayEnd(builder):
return builder.EndObject()
server/app/util/fbs/NetEncoding/Float64Array.py
+22 -6
View File
@@ -4,6 +4,7 @@
import flatbuffers import flatbuffers
class Float64Array(object): class Float64Array(object):
__slots__ = ['_tab'] __slots__ = ['_tab']
@@ -23,14 +24,17 @@ class Float64Array(object):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0: if o != 0:
a = self._tab.Vector(o) a = self._tab.Vector(o)
return self._tab.Get(flatbuffers.number_types.Float64Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 8)) return self._tab.Get(
flatbuffers.number_types.Float64Flags,
a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 8))
return 0 return 0
# Float64Array # Float64Array
def DataAsNumpy(self): def DataAsNumpy(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0: if o != 0:
return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Float64Flags, o) return self._tab.GetVectorAsNumpy(
flatbuffers.number_types.Float64Flags, o)
return 0 return 0
# Float64Array # Float64Array
@@ -40,7 +44,19 @@ class Float64Array(object):
return self._tab.VectorLen(o) return self._tab.VectorLen(o)
return 0 return 0
def Float64ArrayStart(builder): builder.StartObject(1)
def Float64ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0) def Float64ArrayStart(builder):
def Float64ArrayStartDataVector(builder, numElems): return builder.StartVector(8, numElems, 8) builder.StartObject(1)
def Float64ArrayEnd(builder): return builder.EndObject()
def Float64ArrayAddData(builder, data):
builder.PrependUOffsetTRelativeSlot(
0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
def Float64ArrayStartDataVector(builder, numElems):
return builder.StartVector(8, numElems, 8)
def Float64ArrayEnd(builder):
return builder.EndObject()
server/app/util/fbs/NetEncoding/Int32Array.py
+22 -6
View File
@@ -4,6 +4,7 @@
import flatbuffers import flatbuffers
class Int32Array(object): class Int32Array(object):
__slots__ = ['_tab'] __slots__ = ['_tab']
@@ -23,14 +24,17 @@ class Int32Array(object):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0: if o != 0:
a = self._tab.Vector(o) a = self._tab.Vector(o)
return self._tab.Get(flatbuffers.number_types.Int32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4)) return self._tab.Get(
flatbuffers.number_types.Int32Flags,
a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
return 0 return 0
# Int32Array # Int32Array
def DataAsNumpy(self): def DataAsNumpy(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0: if o != 0:
return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Int32Flags, o) return self._tab.GetVectorAsNumpy(
flatbuffers.number_types.Int32Flags, o)
return 0 return 0
# Int32Array # Int32Array
@@ -40,7 +44,19 @@ class Int32Array(object):
return self._tab.VectorLen(o) return self._tab.VectorLen(o)
return 0 return 0
def Int32ArrayStart(builder): builder.StartObject(1)
def Int32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0) def Int32ArrayStart(builder):
def Int32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4) builder.StartObject(1)
def Int32ArrayEnd(builder): return builder.EndObject()
def Int32ArrayAddData(builder, data):
builder.PrependUOffsetTRelativeSlot(
0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
def Int32ArrayStartDataVector(builder, numElems):
return builder.StartVector(4, numElems, 4)
def Int32ArrayEnd(builder):
return builder.EndObject()
server/app/util/fbs/NetEncoding/JSONEncodedArray.py
+22 -6
View File
@@ -4,6 +4,7 @@
import flatbuffers import flatbuffers
class JSONEncodedArray(object): class JSONEncodedArray(object):
__slots__ = ['_tab'] __slots__ = ['_tab']
@@ -23,14 +24,17 @@ class JSONEncodedArray(object):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0: if o != 0:
a = self._tab.Vector(o) a = self._tab.Vector(o)
return self._tab.Get(flatbuffers.number_types.Uint8Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 1)) return self._tab.Get(
flatbuffers.number_types.Uint8Flags,
a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 1))
return 0 return 0
# JSONEncodedArray # JSONEncodedArray
def DataAsNumpy(self): def DataAsNumpy(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0: if o != 0:
return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Uint8Flags, o) return self._tab.GetVectorAsNumpy(
flatbuffers.number_types.Uint8Flags, o)
return 0 return 0
# JSONEncodedArray # JSONEncodedArray
@@ -40,7 +44,19 @@ class JSONEncodedArray(object):
return self._tab.VectorLen(o) return self._tab.VectorLen(o)
return 0 return 0
def JSONEncodedArrayStart(builder): builder.StartObject(1)
def JSONEncodedArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0) def JSONEncodedArrayStart(builder):
def JSONEncodedArrayStartDataVector(builder, numElems): return builder.StartVector(1, numElems, 1) builder.StartObject(1)
def JSONEncodedArrayEnd(builder): return builder.EndObject()
def JSONEncodedArrayAddData(builder, data):
builder.PrependUOffsetTRelativeSlot(
0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
def JSONEncodedArrayStartDataVector(builder, numElems):
return builder.StartVector(1, numElems, 1)
def JSONEncodedArrayEnd(builder):
return builder.EndObject()
server/app/util/fbs/NetEncoding/Matrix.py
+51 -14
View File
@@ -4,6 +4,7 @@
import flatbuffers import flatbuffers
class Matrix(object): class Matrix(object):
__slots__ = ['_tab'] __slots__ = ['_tab']
@@ -22,14 +23,16 @@ class Matrix(object):
def NRows(self): def NRows(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0: if o != 0:
return self._tab.Get(flatbuffers.number_types.Uint32Flags, o + self._tab.Pos) return self._tab.Get(flatbuffers.number_types.Uint32Flags,
o + self._tab.Pos)
return 0 return 0
# Matrix # Matrix
def NCols(self): def NCols(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6))
if o != 0: if o != 0:
return self._tab.Get(flatbuffers.number_types.Uint32Flags, o + self._tab.Pos) return self._tab.Get(flatbuffers.number_types.Uint32Flags,
o + self._tab.Pos)
return 0 return 0
# Matrix # Matrix
@@ -56,7 +59,8 @@ class Matrix(object):
def ColIndexType(self): def ColIndexType(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10)) o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10))
if o != 0: if o != 0:
return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos) return self._tab.Get(flatbuffers.number_types.Uint8Flags,
o + self._tab.Pos)
return 0 return 0
# Matrix # Matrix
@@ -73,7 +77,8 @@ class Matrix(object):
def RowIndexType(self): def RowIndexType(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14)) o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14))
if o != 0: if o != 0:
return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos) return self._tab.Get(flatbuffers.number_types.Uint8Flags,
o + self._tab.Pos)
return 0 return 0
# Matrix # Matrix
@@ -86,13 +91,45 @@ class Matrix(object):
return obj return obj
return None return None
def MatrixStart(builder): builder.StartObject(7)
def MatrixAddNRows(builder, nRows): builder.PrependUint32Slot(0, nRows, 0) def MatrixStart(builder):
def MatrixAddNCols(builder, nCols): builder.PrependUint32Slot(1, nCols, 0) builder.StartObject(7)
def MatrixAddColumns(builder, columns): builder.PrependUOffsetTRelativeSlot(2, flatbuffers.number_types.UOffsetTFlags.py_type(columns), 0)
def MatrixStartColumnsVector(builder, numElems): return builder.StartVector(4, numElems, 4)
def MatrixAddColIndexType(builder, colIndexType): builder.PrependUint8Slot(3, colIndexType, 0) def MatrixAddNRows(builder, nRows):
def MatrixAddColIndex(builder, colIndex): builder.PrependUOffsetTRelativeSlot(4, flatbuffers.number_types.UOffsetTFlags.py_type(colIndex), 0) builder.PrependUint32Slot(0, nRows, 0)
def MatrixAddRowIndexType(builder, rowIndexType): builder.PrependUint8Slot(5, rowIndexType, 0)
def MatrixAddRowIndex(builder, rowIndex): builder.PrependUOffsetTRelativeSlot(6, flatbuffers.number_types.UOffsetTFlags.py_type(rowIndex), 0)
def MatrixEnd(builder): return builder.EndObject() def MatrixAddNCols(builder, nCols):
builder.PrependUint32Slot(1, nCols, 0)
def MatrixAddColumns(builder, columns):
builder.PrependUOffsetTRelativeSlot(
2, flatbuffers.number_types.UOffsetTFlags.py_type(columns), 0)
def MatrixStartColumnsVector(builder, numElems):
return builder.StartVector(4, numElems, 4)
def MatrixAddColIndexType(builder, colIndexType):
builder.PrependUint8Slot(3, colIndexType, 0)
def MatrixAddColIndex(builder, colIndex):
builder.PrependUOffsetTRelativeSlot(
4, flatbuffers.number_types.UOffsetTFlags.py_type(colIndex), 0)
def MatrixAddRowIndexType(builder, rowIndexType):
builder.PrependUint8Slot(5, rowIndexType, 0)
def MatrixAddRowIndex(builder, rowIndex):
builder.PrependUOffsetTRelativeSlot(
6, flatbuffers.number_types.UOffsetTFlags.py_type(rowIndex), 0)
def MatrixEnd(builder):
return builder.EndObject()
server/app/util/fbs/NetEncoding/TypedArray.py
+1 -1
View File
@@ -2,6 +2,7 @@
# namespace: NetEncoding # namespace: NetEncoding
class TypedArray(object): class TypedArray(object):
NONE = 0 NONE = 0
Float32Array = 1 Float32Array = 1
@@ -9,4 +10,3 @@ class TypedArray(object):
Uint32Array = 3 Uint32Array = 3
Float64Array = 4 Float64Array = 4
JSONEncodedArray = 5 JSONEncodedArray = 5
server/app/util/fbs/NetEncoding/Uint32Array.py
+22 -6
View File
@@ -4,6 +4,7 @@
import flatbuffers import flatbuffers
class Uint32Array(object): class Uint32Array(object):
__slots__ = ['_tab'] __slots__ = ['_tab']
@@ -23,14 +24,17 @@ class Uint32Array(object):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0: if o != 0:
a = self._tab.Vector(o) a = self._tab.Vector(o)
return self._tab.Get(flatbuffers.number_types.Uint32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4)) return self._tab.Get(
flatbuffers.number_types.Uint32Flags,
a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
return 0 return 0
# Uint32Array # Uint32Array
def DataAsNumpy(self): def DataAsNumpy(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0: if o != 0:
return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Uint32Flags, o) return self._tab.GetVectorAsNumpy(
flatbuffers.number_types.Uint32Flags, o)
return 0 return 0
# Uint32Array # Uint32Array
@@ -40,7 +44,19 @@ class Uint32Array(object):
return self._tab.VectorLen(o) return self._tab.VectorLen(o)
return 0 return 0
def Uint32ArrayStart(builder): builder.StartObject(1)
def Uint32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0) def Uint32ArrayStart(builder):
def Uint32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4) builder.StartObject(1)
def Uint32ArrayEnd(builder): return builder.EndObject()
def Uint32ArrayAddData(builder, data):
builder.PrependUOffsetTRelativeSlot(
0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
def Uint32ArrayStartDataVector(builder, numElems):
return builder.StartVector(4, numElems, 4)
def Uint32ArrayEnd(builder):
return builder.EndObject()
server/app/util/fbs/matrix.py
+25 -15
View File
@@ -25,7 +25,8 @@ def CreateNumpyVector(builder, x):
"""CreateNumpyVector writes a numpy array into the buffer.""" """CreateNumpyVector writes a numpy array into the buffer."""
if not isinstance(x, np.ndarray): if not isinstance(x, np.ndarray):
raise TypeError(f"non-numpy-ndarray passed to CreateNumpyVector ({type(x)}") raise TypeError(
f"non-numpy-ndarray passed to CreateNumpyVector ({type(x)}")
if x.dtype.kind not in ['b', 'i', 'u', 'f']: if x.dtype.kind not in ['b', 'i', 'u', 'f']:
raise TypeError("numpy-ndarray holds elements of unsupported datatype") raise TypeError("numpy-ndarray holds elements of unsupported datatype")
@@ -46,7 +47,8 @@ def CreateNumpyVector(builder, x):
builder.head = int(builder.Head() - len) builder.head = int(builder.Head() - len)
# tobytes ensures c_contiguous ordering # tobytes ensures c_contiguous ordering
builder.Bytes[builder.Head():builder.Head() + len] = x_little_endian.tobytes(order='C') builder.Bytes[builder.Head():builder.Head() +
len] = x_little_endian.tobytes(order='C')
return builder.EndVector(x.size) return builder.EndVector(x.size)
@@ -119,12 +121,10 @@ column_encoding_type_map = {
np.dtype(np.float64).str: (TypedArray.TypedArray.Float32Array, np.float32), np.dtype(np.float64).str: (TypedArray.TypedArray.Float32Array, np.float32),
np.dtype(np.float32).str: (TypedArray.TypedArray.Float32Array, np.float32), np.dtype(np.float32).str: (TypedArray.TypedArray.Float32Array, np.float32),
np.dtype(np.float16).str: (TypedArray.TypedArray.Float32Array, np.float32), np.dtype(np.float16).str: (TypedArray.TypedArray.Float32Array, np.float32),
np.dtype(np.int8).str: (TypedArray.TypedArray.Int32Array, np.int32), np.dtype(np.int8).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.int16).str: (TypedArray.TypedArray.Int32Array, np.int32), np.dtype(np.int16).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32), np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32), np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.uint8).str: (TypedArray.TypedArray.Uint32Array, np.uint32), np.dtype(np.uint8).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.dtype(np.uint16).str: (TypedArray.TypedArray.Uint32Array, np.uint32), np.dtype(np.uint16).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32), np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
@@ -141,7 +141,6 @@ index_encoding_type_map = {
# array protocol string: ( array_type, as_type ) # array protocol string: ( array_type, as_type )
np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32), np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32), np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32), np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32) np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32)
} }
@@ -192,7 +191,8 @@ def encode_matrix_fbs(matrix, row_idx=None, col_idx=None):
columns = [] columns = []
for cidx in range(n_cols - 1, -1, -1): for cidx in range(n_cols - 1, -1, -1):
# serialize the typed array # serialize the typed array
col = matrix.iloc[:, cidx] if isinstance(matrix, pd.DataFrame) else matrix[:, cidx] col = matrix.iloc[:, cidx] if isinstance(
matrix, pd.DataFrame) else matrix[:, cidx]
typed_arr = serialize_typed_array(builder, col, column_encoding) typed_arr = serialize_typed_array(builder, col, column_encoding)
# serialize the Column union # serialize the Column union
@@ -218,12 +218,18 @@ def encode_matrix_fbs(matrix, row_idx=None, col_idx=None):
def deserialize_typed_array(tarr): def deserialize_typed_array(tarr):
type_map = { type_map = {
TypedArray.TypedArray.NONE: None, TypedArray.TypedArray.NONE:
TypedArray.TypedArray.Uint32Array: Uint32Array.Uint32Array, None,
TypedArray.TypedArray.Int32Array: Int32Array.Int32Array, TypedArray.TypedArray.Uint32Array:
TypedArray.TypedArray.Float32Array: Float32Array.Float32Array, Uint32Array.Uint32Array,
TypedArray.TypedArray.Float64Array: Float64Array.Float64Array, TypedArray.TypedArray.Int32Array:
TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray Int32Array.Int32Array,
TypedArray.TypedArray.Float32Array:
Float32Array.Float32Array,
TypedArray.TypedArray.Float64Array:
Float64Array.Float64Array,
TypedArray.TypedArray.JSONEncodedArray:
JSONEncodedArray.JSONEncodedArray
} }
(u_type, u) = tarr (u_type, u) = tarr
if u_type is TypedArray.TypedArray.NONE: if u_type is TypedArray.TypedArray.NONE:
@@ -257,13 +263,16 @@ def decode_matrix_fbs(fbs):
columns_length = matrix.ColumnsLength() columns_length = matrix.ColumnsLength()
columns_index = deserialize_typed_array((matrix.ColIndexType(), matrix.ColIndex())) columns_index = deserialize_typed_array(
(matrix.ColIndexType(), matrix.ColIndex()))
if columns_index is None: if columns_index is None:
columns_index = range(0, n_cols) columns_index = range(0, n_cols)
# sanity checks # sanity checks
if len(columns_index) != n_cols or columns_length != n_cols: if len(columns_index) != n_cols or columns_length != n_cols:
raise ValueError("FBS column count does not match number of columns in underlying matrix") raise ValueError(
"FBS column count does not match number of columns in underlying matrix"
)
columns_data = {} columns_data = {}
columns_type = {} columns_type = {}
@@ -277,7 +286,8 @@ def decode_matrix_fbs(fbs):
if col.UType() is TypedArray.TypedArray.JSONEncodedArray: if col.UType() is TypedArray.TypedArray.JSONEncodedArray:
columns_type[columns_index[col_idx]] = "category" columns_type[columns_index[col_idx]] = "category"
df = pd.DataFrame.from_dict(data=columns_data).astype(columns_type, copy=False) df = pd.DataFrame.from_dict(data=columns_data).astype(columns_type,
copy=False)
# more sanity checks # more sanity checks
if not df.columns.is_unique or len(df.columns) != n_cols: if not df.columns.is_unique or len(df.columns) != n_cols:
server/app/util/matrix_proxy.py
+42 -29
View File
@@ -2,7 +2,6 @@ import abc
from itertools import zip_longest from itertools import zip_longest
from copy import copy from copy import copy
import numpy as np import numpy as np
""" """
cellxgene deals with a variety of matrix data types, many of which do cellxgene deals with a variety of matrix data types, many of which do
not support a consistent API. This framework allows proxies to be created not support a consistent API. This framework allows proxies to be created
@@ -18,6 +17,7 @@ class _ArrayProxyBase(abc.ABC):
Private base class for array or matrix proxy. This summarizes Private base class for array or matrix proxy. This summarizes
the interface used by the rest of cellxgene. the interface used by the rest of cellxgene.
""" """
@property @property
@abc.abstractmethod @abc.abstractmethod
def dtype(self): def dtype(self):
@@ -59,7 +59,6 @@ class MatrixProxy(_ArrayProxyBase):
This class primarily provides the factory method and related support. This class primarily provides the factory method and related support.
All other functionality is delegated to subclasses. All other functionality is delegated to subclasses.
""" """
""" """
Registry of types to proxy class, where values are: Registry of types to proxy class, where values are:
* None: unsupported * None: unsupported
@@ -128,21 +127,25 @@ class MatrixProxyView(MatrixProxy):
""" """
2D matrix view to a 2D matrix 2D matrix view to a 2D matrix
""" """
def __init__(self, arg1, shape=None, index=(),
transposed=False, copy=False): def __init__(self,
arg1,
shape=None,
index=(),
transposed=False,
copy=False):
if not copy: if not copy:
m = arg1 m = arg1
super().__init__(m) super().__init__(m)
if shape is None: if shape is None:
shape = m.shape shape = m.shape
assert(len(shape) == 2) assert (len(shape) == 2)
index = tuple( index = tuple(
map(lambda s_i: map(
slice(0, s_i[0], 1) if s_i[1] is None else s_i[1], lambda s_i: slice(0, s_i[0], 1)
zip_longest(shape, index)) if s_i[1] is None else s_i[1], zip_longest(shape, index)))
)
self._shape = shape self._shape = shape
self._index = index self._index = index
@@ -234,21 +237,29 @@ class MatrixProxyView(MatrixProxy):
NOTE: these follow the numpy rules for dimensionality reduction NOTE: these follow the numpy rules for dimensionality reduction
when an integer index is specified. when an integer index is specified.
""" """
def _getitem_intXint(self, row, col): def _getitem_intXint(self, row, col):
return self.m[row, col] return self.m[row, col]
def _getitem_intXslice(self, row, col): def _getitem_intXslice(self, row, col):
shape = (_slice_length(col, self.m.shape[1]), ) shape = (_slice_length(col, self.m.shape[1]),)
return self.__class__.create_array(self.m, shape=shape, index=(row, col)) return self.__class__.create_array(self.m,
shape=shape,
index=(row, col))
def _getitem_sliceXint(self, row, col): def _getitem_sliceXint(self, row, col):
shape = (_slice_length(row, self.m.shape[0]), ) shape = (_slice_length(row, self.m.shape[0]),)
return self.__class__.create_array(self.m, shape=shape, index=(row, col)) return self.__class__.create_array(self.m,
shape=shape,
index=(row, col))
def _getitem_sliceXslice(self, row, col): def _getitem_sliceXslice(self, row, col):
shape = (_slice_length(row, self.m.shape[0]), shape = (_slice_length(row, self.m.shape[0]),
_slice_length(col, self.m.shape[1])) _slice_length(col, self.m.shape[1]))
return self.__class__(self.m, shape=shape, index=(row, col), transposed=self.transposed) return self.__class__(self.m,
shape=shape,
index=(row, col),
transposed=self.transposed)
def toarray(self): def toarray(self):
arr = self.m[self._index] arr = self.m[self._index]
@@ -261,22 +272,24 @@ class ArrayProxyView(_ArrayProxyBase):
""" """
1D array view to a 2D matrix 1D array view to a 2D matrix
""" """
def __init__(self, arg1, shape=None, index=None, copy=False): def __init__(self, arg1, shape=None, index=None, copy=False):
super().__init__() super().__init__()
if not copy: if not copy:
m = arg1 m = arg1
# one index MUST be an integer and the other MUST be a slice # one index MUST be an integer and the other MUST be a slice
assert(len(index) == 2) assert (len(index) == 2)
assert(all(isinstance(idx, INT_TYPES + (slice, )) for idx in index)) assert (all(isinstance(idx, INT_TYPES + (slice,)) for idx in index))
assert(isinstance(index[0], INT_TYPES) != isinstance(index[1], INT_TYPES)) assert (isinstance(index[0], INT_TYPES) != isinstance(
index[1], INT_TYPES))
if shape is None: if shape is None:
if isinstance(index[0], INT_TYPES): if isinstance(index[0], INT_TYPES):
shape = (m.shape[0], ) shape = (m.shape[0],)
else: else:
shape = (m.shape[1], ) shape = (m.shape[1],)
assert(len(shape) == 1) assert (len(shape) == 1)
self._shape = shape self._shape = shape
self.m = m self.m = m
@@ -336,7 +349,7 @@ class ArrayProxyView(_ArrayProxyBase):
elif isinstance(col, slice): elif isinstance(col, slice):
return self._getitem_intXslice(row, col) return self._getitem_intXslice(row, col)
elif isinstance(row, slice): elif isinstance(row, slice):
assert(isinstance(col, INT_TYPES)) assert (isinstance(col, INT_TYPES))
return self._getitem_sliceXint(row, col) return self._getitem_sliceXint(row, col)
raise IndexError("unsupported column index types") raise IndexError("unsupported column index types")
@@ -345,11 +358,11 @@ class ArrayProxyView(_ArrayProxyBase):
return self.m[row, col] return self.m[row, col]
def _getitem_intXslice(self, row, col): def _getitem_intXslice(self, row, col):
shape = (_slice_length(col, self.m.shape[1]), ) shape = (_slice_length(col, self.m.shape[1]),)
return self.__class__(self.m, shape=shape, index=(row, col)) return self.__class__(self.m, shape=shape, index=(row, col))
def _getitem_sliceXint(self, row, col): def _getitem_sliceXint(self, row, col):
shape = (_slice_length(row, self.m.shape[0]), ) shape = (_slice_length(row, self.m.shape[0]),)
return self.__class__(self.m, shape=shape, index=(row, col)) return self.__class__(self.m, shape=shape, index=(row, col))
def toarray(self): def toarray(self):
@@ -358,7 +371,7 @@ class ArrayProxyView(_ArrayProxyBase):
def _unpack_index(index, shape): def _unpack_index(index, shape):
if not isinstance(index, tuple): if not isinstance(index, tuple):
index = (index, ) index = (index,)
if len(shape) < len(index): if len(shape) < len(index):
raise IndexError("invalid index dimensionality - must be 2") raise IndexError("invalid index dimensionality - must be 2")
@@ -366,7 +379,7 @@ def _unpack_index(index, shape):
for shp, idx in zip_longest(shape, index): for shp, idx in zip_longest(shape, index):
idx = slice(None) if idx is None else idx idx = slice(None) if idx is None else idx
idx = _slice_defaults(idx, shp) if isinstance(idx, slice) else idx idx = _slice_defaults(idx, shp) if isinstance(idx, slice) else idx
unpacked += (idx, ) unpacked += (idx,)
return unpacked return unpacked
@@ -376,7 +389,7 @@ def _slice_slice(outer, outer_len, inner, inner_len):
slice a slice - we take advantage of Python 3 range's support slice a slice - we take advantage of Python 3 range's support
for indexing. for indexing.
""" """
assert(outer_len >= inner_len) assert (outer_len >= inner_len)
outer_rng = range(*outer.indices(outer_len)) outer_rng = range(*outer.indices(outer_len))
rng = outer_rng[inner] rng = outer_rng[inner]
start, stop, step = rng.start, rng.stop, rng.step start, stop, step = rng.start, rng.stop, rng.step
@@ -387,8 +400,8 @@ def _slice_slice(outer, outer_len, inner, inner_len):
def _range_length(start, stop, step): def _range_length(start, stop, step):
""" return length of range """ """ return length of range """
assert(step != 0) assert (step != 0)
assert(start is not None and stop is not None and step is not None) assert (start is not None and stop is not None and step is not None)
if step > 0 and start < stop: if step > 0 and start < stop:
return 1 + (stop - 1 - start) // step return 1 + (stop - 1 - start) // step
elif step < 0 and start > stop: elif step < 0 and start > stop:
@@ -404,7 +417,7 @@ def _slice_length(s, length):
def _slice_defaults(s, length): def _slice_defaults(s, length):
""" apply slice defaulting conventions """ """ apply slice defaulting conventions """
assert(length >= 0) assert (length >= 0)
step = 1 if s.step is None else s.step step = 1 if s.step is None else s.step
server/app/util/utils.py
+5 -1
View File
@@ -7,6 +7,7 @@ from server.app.util.errors import DriverError
class Float32JSONEncoder(json.JSONEncoder): class Float32JSONEncoder(json.JSONEncoder):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
""" """
NaN/Infinities are illegal in standard JSON. Python extends JSON with NaN/Infinities are illegal in standard JSON. Python extends JSON with
@@ -35,9 +36,12 @@ def jsonify_scanpy(data):
def requires_data(func): def requires_data(func):
@wraps(func) @wraps(func)
def wrapped_function(self, *args, **kwargs): def wrapped_function(self, *args, **kwargs):
if self.data is None: if self.data is None:
raise DriverError(f"error data must be loaded before you call {func.__name__}") raise DriverError(
f"error data must be loaded before you call {func.__name__}")
return func(self, *args, **kwargs) return func(self, *args, **kwargs)
return wrapped_function return wrapped_function
server/app/web/webapp.py
+5 -3
View File
@@ -1,7 +1,6 @@
import os import os
from flask import Blueprint, render_template, send_from_directory, current_app from flask import Blueprint, render_template, send_from_directory, current_app
bp = Blueprint("webapp", __name__, template_folder="templates") bp = Blueprint("webapp", __name__, template_folder="templates")
@@ -9,9 +8,12 @@ bp = Blueprint("webapp", __name__, template_folder="templates")
def index(): def index():
dataset_title = current_app.config["DATASET_TITLE"] dataset_title = current_app.config["DATASET_TITLE"]
scripts = current_app.config["SCRIPTS"] scripts = current_app.config["SCRIPTS"]
return render_template("index.html", datasetTitle=dataset_title, SCRIPTS=scripts) return render_template("index.html",
datasetTitle=dataset_title,
SCRIPTS=scripts)
@bp.route("/favicon.png") @bp.route("/favicon.png")
def favicon(): def favicon():
return send_from_directory(os.path.join(bp.root_path, "static/img/"), "favicon.png") return send_from_directory(os.path.join(bp.root_path, "static/img/"),
"favicon.png")
server/cli/cli.py
+4 -5
View File
@@ -10,11 +10,10 @@ from .prepare import prepare
context_settings=dict(max_content_width=85, context_settings=dict(max_content_width=85,
help_option_names=['-h', '--help'])) help_option_names=['-h', '--help']))
@click.help_option("--help", "-h", help="Show this message and exit.") @click.help_option("--help", "-h", help="Show this message and exit.")
@click.version_option( @click.version_option(version="0.13.0",
version="0.13.0", prog_name="cellxgene",
prog_name="cellxgene", message="[%(prog)s] Version %(version)s",
message="[%(prog)s] Version %(version)s", help="Show the software version and exit.")
help="Show the software version and exit.")
def cli(): def cli():
pass pass
server/cli/launch.py
+124 -90
View File
@@ -17,7 +17,7 @@ from server.utils.utils import find_available_port, is_port_available, sort_opti
from server.app.util.data_locator import DataLocator from server.app.util.data_locator import DataLocator
# anything bigger than this will generate a special message # anything bigger than this will generate a special message
BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20 # 100MB BIG_FILE_SIZE_THRESHOLD = 100 * 2**20 # 100MB
def common_args(func): def common_args(func):
@@ -25,16 +25,14 @@ def common_args(func):
Decorator to contain CLI args that will be common to both CLI and GUI: title and engine args. Decorator to contain CLI args that will be common to both CLI and GUI: title and engine args.
""" """
@click.option( @click.option("--title",
"--title", "-t",
"-t", metavar="<text>",
metavar="<text>", help="Title to display. If omitted will use file name.")
help="Title to display. If omitted will use file name.") @click.option("--about",
@click.option( metavar="<URL>",
"--about", help="URL providing more information about the dataset "
metavar="<URL>", "(hint: must be a fully specified absolute URL).")
help="URL providing more information about the dataset "
"(hint: must be a fully specified absolute URL).")
@click.option( @click.option(
"--embedding", "--embedding",
"-e", "-e",
@@ -42,69 +40,80 @@ def common_args(func):
multiple=True, multiple=True,
show_default=False, show_default=False,
metavar="<text>", metavar="<text>",
help="Embedding name, eg, 'umap'. Repeat option for multiple embeddings. Defaults to all." help=
"Embedding name, eg, 'umap'. Repeat option for multiple embeddings. Defaults to all."
) )
@click.option( @click.option(
"--obs-names", "--obs-names",
"-obs", "-obs",
default=None, default=None,
metavar="<text>", metavar="<text>",
help="Name of annotation field to use for observations. If not specified cellxgene will use the the obs index.") help=
"Name of annotation field to use for observations. If not specified cellxgene will use the the obs index."
)
@click.option( @click.option(
"--var-names", "--var-names",
"-var", "-var",
default=None, default=None,
metavar="<text>", metavar="<text>",
help="Name of annotation to use for variables. If not specified cellxgene will use the the var index.") help=
"Name of annotation to use for variables. If not specified cellxgene will use the the var index."
)
@click.option( @click.option(
"--max-category-items", "--max-category-items",
default=1000, default=1000,
metavar="<integer>", metavar="<integer>",
show_default=True, show_default=True,
help="Will not display categories with more distinct values than specified.",) help=
"Will not display categories with more distinct values than specified.",
)
@click.option( @click.option(
"--diffexp-lfc-cutoff", "--diffexp-lfc-cutoff",
"-de", "-de",
default=0.01, default=0.01,
show_default=True, show_default=True,
metavar="<float>", metavar="<float>",
help="Minimum log fold change threshold for differential expression.",) help="Minimum log fold change threshold for differential expression.",
@click.option(
"--experimental-annotations",
is_flag=True,
default=False,
show_default=True,
help="Enable user annotation of data."
) )
@click.option("--experimental-annotations",
is_flag=True,
default=False,
show_default=True,
help="Enable user annotation of data.")
@click.option( @click.option(
"--experimental-annotations-file", "--experimental-annotations-file",
default=None, default=None,
show_default=True, show_default=True,
multiple=False, multiple=False,
metavar="<path>", metavar="<path>",
help="CSV file to initialize editing of existing annotations; will be altered in-place. " help=
"Incompatible with --annotations-output-dir.",) "CSV file to initialize editing of existing annotations; will be altered in-place. "
"Incompatible with --annotations-output-dir.",
)
@click.option( @click.option(
"--experimental-annotations-output-dir", "--experimental-annotations-output-dir",
default=None, default=None,
show_default=False, show_default=False,
multiple=False, multiple=False,
metavar="<directory path>", metavar="<directory path>",
help="Directory of where to save output annotations; filename will be specified in the application. " help=
"Incompatible with --annotations-input-file.",) "Directory of where to save output annotations; filename will be specified in the application. "
"Incompatible with --annotations-input-file.",
)
@click.option( @click.option(
"--backed", "--backed",
"-b", "-b",
is_flag=True, is_flag=True,
default=False, default=False,
show_default=False, show_default=False,
help="Load data in file-backed mode. This may save memory, but may result in slower overall performance.") help=
@click.option( "Load data in file-backed mode. This may save memory, but may result in slower overall performance."
"--disable-diffexp", )
is_flag=True, @click.option("--disable-diffexp",
default=False, is_flag=True,
show_default=False, default=False,
help="Disable on-demand differential expression.") show_default=False,
help="Disable on-demand differential expression.")
@functools.wraps(func) @functools.wraps(func)
def wrapper(*args, **kwargs): def wrapper(*args, **kwargs):
return func(*args, **kwargs) return func(*args, **kwargs)
@@ -112,9 +121,11 @@ def common_args(func):
return wrapper return wrapper
def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffexp_lfc_cutoff, def parse_engine_args(embedding, obs_names, var_names, max_category_items,
experimental_annotations, experimental_annotations_file, diffexp_lfc_cutoff, experimental_annotations,
experimental_annotations_output_dir, backed, disable_diffexp): experimental_annotations_file,
experimental_annotations_output_dir, backed,
disable_diffexp):
annotations_file = experimental_annotations_file if experimental_annotations else None annotations_file = experimental_annotations_file if experimental_annotations else None
annotations_output_dir = experimental_annotations_output_dir if experimental_annotations else None annotations_output_dir = experimental_annotations_output_dir if experimental_annotations else None
return { return {
@@ -132,9 +143,11 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffe
@sort_options @sort_options
@click.command(short_help="Launch the cellxgene data viewer. " @click.command(
"Run `cellxgene launch --help` for more information.", short_help="Launch the cellxgene data viewer. "
options_metavar="<options>",) "Run `cellxgene launch --help` for more information.",
options_metavar="<options>",
)
@click.argument("data", nargs=1, metavar="<path to data file>", required=True) @click.argument("data", nargs=1, metavar="<path to data file>", required=True)
@click.option( @click.option(
"--verbose", "--verbose",
@@ -142,7 +155,8 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffe
is_flag=True, is_flag=True,
default=False, default=False,
show_default=True, show_default=True,
help="Provide verbose output, including warnings and all server requests.",) help="Provide verbose output, including warnings and all server requests.",
)
@click.option( @click.option(
"--debug", "--debug",
"-d", "-d",
@@ -150,7 +164,8 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffe
default=False, default=False,
show_default=True, show_default=True,
help="Run in debug mode. This is helpful for cellxgene developers, " help="Run in debug mode. This is helpful for cellxgene developers, "
"or when you want more information about an error condition.",) "or when you want more information about an error condition.",
)
@click.option( @click.option(
"--open", "--open",
"-o", "-o",
@@ -158,19 +173,24 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffe
is_flag=True, is_flag=True,
default=False, default=False,
show_default=True, show_default=True,
help="Open web browser after launch.",) help="Open web browser after launch.",
)
@click.option( @click.option(
"--port", "--port",
"-p", "-p",
metavar="<port>", metavar="<port>",
show_default=True, show_default=True,
help="Port to run server on. If not specified cellxgene will find an available port.",) help=
"Port to run server on. If not specified cellxgene will find an available port.",
)
@click.option( @click.option(
"--host", "--host",
metavar="<IP address>", metavar="<IP address>",
default="127.0.0.1", default="127.0.0.1",
show_default=False, show_default=False,
help="Host IP address. By default cellxgene will use localhost (e.g. 127.0.0.1).") help=
"Host IP address. By default cellxgene will use localhost (e.g. 127.0.0.1)."
)
@click.option( @click.option(
"--scripts", "--scripts",
"-s", "-s",
@@ -178,31 +198,15 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffe
multiple=True, multiple=True,
metavar="<text>", metavar="<text>",
help="Additional script files to include in HTML page. If not specified, " help="Additional script files to include in HTML page. If not specified, "
"no additional script files will be included.", "no additional script files will be included.",
show_default=False,) show_default=False,
)
@click.help_option("--help", "-h", help="Show this message and exit.") @click.help_option("--help", "-h", help="Show this message and exit.")
@common_args @common_args
def launch( def launch(data, verbose, debug, open_browser, port, host, embedding, obs_names,
data, var_names, max_category_items, diffexp_lfc_cutoff, title, scripts,
verbose, about, experimental_annotations, experimental_annotations_file,
debug, experimental_annotations_output_dir, backed, disable_diffexp):
open_browser,
port,
host,
embedding,
obs_names,
var_names,
max_category_items,
diffexp_lfc_cutoff,
title,
scripts,
about,
experimental_annotations,
experimental_annotations_file,
experimental_annotations_output_dir,
backed,
disable_diffexp
):
"""Launch the cellxgene data viewer. """Launch the cellxgene data viewer.
This web app lets you explore single-cell expression data. This web app lets you explore single-cell expression data.
Data must be in a format that cellxgene expects. Data must be in a format that cellxgene expects.
@@ -217,17 +221,17 @@ def launch(
> cellxgene launch <url>""" > cellxgene launch <url>"""
e_args = parse_engine_args(embedding, obs_names, var_names, max_category_items, e_args = parse_engine_args(embedding, obs_names, var_names,
diffexp_lfc_cutoff, max_category_items, diffexp_lfc_cutoff,
experimental_annotations, experimental_annotations,
experimental_annotations_file, experimental_annotations_file,
experimental_annotations_output_dir, experimental_annotations_output_dir, backed,
backed,
disable_diffexp) disable_diffexp)
try: try:
data_locator = DataLocator(data) data_locator = DataLocator(data)
except RuntimeError as re: except RuntimeError as re:
raise click.ClickException(f"Unable to access data at {data}. {str(re)}") raise click.ClickException(
f"Unable to access data at {data}. {str(re)}")
# Startup message # Startup message
click.echo("[cellxgene] Starting the CLI...") click.echo("[cellxgene] Starting the CLI...")
@@ -244,7 +248,8 @@ def launch(
raise click.FileError(data, hint="data is not a file") raise click.FileError(data, hint="data is not a file")
name, extension = splitext(data) name, extension = splitext(data)
if extension != ".h5ad": if extension != ".h5ad":
raise click.FileError(basename(data), hint="file type must be .h5ad") raise click.FileError(basename(data),
hint="file type must be .h5ad")
if debug: if debug:
verbose = True verbose = True
@@ -266,7 +271,9 @@ def launch(
security risk by including the --scripts flag. Make sure you trust the scripts that you are including. security risk by including the --scripts flag. Make sure you trust the scripts that you are including.
""") """)
scripts_pretty = ", ".join(scripts) scripts_pretty = ", ".join(scripts)
click.confirm(f"Are you sure you want to inject these scripts: {scripts_pretty}?", abort=True) click.confirm(
f"Are you sure you want to inject these scripts: {scripts_pretty}?",
abort=True)
if not title: if not title:
file_parts = splitext(basename(data)) file_parts = splitext(basename(data))
@@ -274,7 +281,9 @@ def launch(
if port: if port:
if debug: if debug:
raise click.ClickException("--port and --debug may not be used together (try --verbose for error logging).") raise click.ClickException(
"--port and --debug may not be used together (try --verbose for error logging)."
)
if not is_port_available(host, int(port)): if not is_port_available(host, int(port)):
raise click.ClickException( raise click.ClickException(
f"The port selected {port} is in use, please specify an open port using the --port flag." f"The port selected {port} is in use, please specify an open port using the --port flag."
@@ -284,27 +293,36 @@ def launch(
if not experimental_annotations: if not experimental_annotations:
if experimental_annotations_file is not None: if experimental_annotations_file is not None:
click.echo("Warning: --experimental-annotations-file ignored as --annotations not enabled.") click.echo(
"Warning: --experimental-annotations-file ignored as --annotations not enabled."
)
if experimental_annotations_output_dir is not None: if experimental_annotations_output_dir is not None:
click.echo("Warning: --experimental-annotations-output-dir ignored as --annotations not enabled.") click.echo(
"Warning: --experimental-annotations-output-dir ignored as --annotations not enabled."
)
else: else:
if experimental_annotations_file is not None and experimental_annotations_output_dir is not None: if experimental_annotations_file is not None and experimental_annotations_output_dir is not None:
raise click.ClickException("--experimental-annotations-file and --experimental-annotations-output-dir " raise click.ClickException(
"may not be used together.") "--experimental-annotations-file and --experimental-annotations-output-dir "
"may not be used together.")
if experimental_annotations_file is not None: if experimental_annotations_file is not None:
lf_name, lf_ext = splitext(experimental_annotations_file) lf_name, lf_ext = splitext(experimental_annotations_file)
if lf_ext and lf_ext != ".csv": if lf_ext and lf_ext != ".csv":
raise click.FileError(basename(experimental_annotations_file), hint="annotation file type must be .csv") raise click.FileError(basename(experimental_annotations_file),
hint="annotation file type must be .csv")
if experimental_annotations_output_dir is not None and not isdir(experimental_annotations_output_dir): if experimental_annotations_output_dir is not None and not isdir(
experimental_annotations_output_dir):
try: try:
mkdir(experimental_annotations_output_dir) mkdir(experimental_annotations_output_dir)
except OSError: except OSError:
raise click.ClickException("Unable to create directory specified by " raise click.ClickException(
"--experimental-annotations-output-dir") "Unable to create directory specified by "
"--experimental-annotations-output-dir")
if about: if about:
def url_check(url): def url_check(url):
try: try:
result = urlparse(url) result = urlparse(url)
@@ -316,7 +334,9 @@ def launch(
return False return False
if not url_check(about): if not url_check(about):
raise click.ClickException("Must provide an absolute URL for --about. (Example format: http://example.com)") raise click.ClickException(
"Must provide an absolute URL for --about. (Example format: http://example.com)"
)
# Setup app # Setup app
cellxgene_url = f"http://{host}:{port}" cellxgene_url = f"http://{host}:{port}"
@@ -335,14 +355,18 @@ def launch(
# if a big file, let the user know it may take a while to load. # if a big file, let the user know it may take a while to load.
if file_size > BIG_FILE_SIZE_THRESHOLD: if file_size > BIG_FILE_SIZE_THRESHOLD:
click.echo(f"[cellxgene] Loading data from {basename(data)}, this may take a while...") click.echo(
f"[cellxgene] Loading data from {basename(data)}, this may take a while..."
)
else: else:
click.echo(f"[cellxgene] Loading data from {basename(data)}.") click.echo(f"[cellxgene] Loading data from {basename(data)}.")
from server.app.scanpy_engine.scanpy_engine import ScanpyEngine from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
try: try:
server.attach_data(ScanpyEngine(data_locator, e_args), title=title, about=about) server.attach_data(ScanpyEngine(data_locator, e_args),
title=title,
about=about)
except ScanpyFileError as e: except ScanpyFileError as e:
raise click.ClickException(f"{e}") raise click.ClickException(f"{e}")
@@ -351,10 +375,14 @@ def launch(
f"running differential expression may take longer or fail.") f"running differential expression may take longer or fail.")
if open_browser: if open_browser:
click.echo(f"[cellxgene] Launching! Opening your browser to {cellxgene_url} now.") click.echo(
f"[cellxgene] Launching! Opening your browser to {cellxgene_url} now."
)
webbrowser.open(cellxgene_url) webbrowser.open(cellxgene_url)
else: else:
click.echo(f"[cellxgene] Launching! Please go to {cellxgene_url} in your browser.") click.echo(
f"[cellxgene] Launching! Please go to {cellxgene_url} in your browser."
)
click.echo("[cellxgene] Type CTRL-C at any time to exit.") click.echo("[cellxgene] Type CTRL-C at any time to exit.")
@@ -363,8 +391,14 @@ def launch(
sys.stdout = f sys.stdout = f
try: try:
server.app.run(host=host, debug=debug, port=port, threaded=False if debug else True, use_debugger=False) server.app.run(host=host,
debug=debug,
port=port,
threaded=False if debug else True,
use_debugger=False)
except OSError as e: except OSError as e:
if e.errno == errno.EADDRINUSE: if e.errno == errno.EADDRINUSE:
raise click.ClickException("Port is in use, please specify an open port using the --port flag.") from e raise click.ClickException(
"Port is in use, please specify an open port using the --port flag."
) from e
raise raise
server/cli/prepare.py
+90 -47
View File
@@ -8,9 +8,11 @@ from server.utils.utils import sort_options
@sort_options @sort_options
@click.command(short_help="Preprocess data for use with cellxgene. " @click.command(
"Run `cellxgene prepare --help` for more information.", short_help="Preprocess data for use with cellxgene. "
options_metavar="<options>",) "Run `cellxgene prepare --help` for more information.",
options_metavar="<options>",
)
@click.argument("data", nargs=1, metavar="<path to data file>", required=True) @click.argument("data", nargs=1, metavar="<path to data file>", required=True)
@click.option( @click.option(
"--embedding", "--embedding",
@@ -29,43 +31,66 @@ from server.utils.utils import sort_options
help="Preprocessing to run.", help="Preprocessing to run.",
show_default=True, show_default=True,
) )
@click.option("--output", "-o", default="", help="Save a new file to filename.", metavar="<filename>") @click.option("--output",
@click.option("--plotting", "-p", default=False, is_flag=True, help="Generate plots.", show_default=True) "-o",
@click.option("--sparse", default=False, is_flag=True, help="Force sparsity.", show_default=True) default="",
@click.option("--overwrite", default=False, is_flag=True, help="Allow file overwriting.", show_default=True) help="Save a new file to filename.",
@click.option("--set-obs-names", default="", help="Named field to set as index for obs.", metavar="<name>") metavar="<filename>")
@click.option("--set-var-names", default="", help="Named field to set as index for var.", metavar="<name>") @click.option("--plotting",
@click.option("--skip-qc", default=False, is_flag=True, "-p",
help="Do not run quality control metrics. By default cellxgene runs them " default=False,
"(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).") is_flag=True,
help="Generate plots.",
show_default=True)
@click.option("--sparse",
default=False,
is_flag=True,
help="Force sparsity.",
show_default=True)
@click.option("--overwrite",
default=False,
is_flag=True,
help="Allow file overwriting.",
show_default=True)
@click.option("--set-obs-names",
default="",
help="Named field to set as index for obs.",
metavar="<name>")
@click.option("--set-var-names",
default="",
help="Named field to set as index for var.",
metavar="<name>")
@click.option( @click.option(
"--make-obs-names-unique", "--skip-qc",
default=True, default=False,
is_flag=True, is_flag=True,
help="Ensure obs index is unique.", help="Do not run quality control metrics. By default cellxgene runs them "
show_default=True "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details)."
)
@click.option(
"--make-var-names-unique",
default=True,
is_flag=True,
help="Ensure var index is unique.",
show_default=True
) )
@click.option("--make-obs-names-unique",
default=True,
is_flag=True,
help="Ensure obs index is unique.",
show_default=True)
@click.option("--make-var-names-unique",
default=True,
is_flag=True,
help="Ensure var index is unique.",
show_default=True)
@click.help_option("--help", "-h", help="Show this message and exit.") @click.help_option("--help", "-h", help="Show this message and exit.")
def prepare( def prepare(
data, data,
embedding, embedding,
recipe, recipe,
output, output,
plotting, plotting,
sparse, sparse,
overwrite, overwrite,
set_obs_names, set_obs_names,
set_var_names, set_var_names,
skip_qc, skip_qc,
make_obs_names_unique, make_obs_names_unique,
make_var_names_unique, make_var_names_unique,
): ):
""" """
Preprocess data for use with cellxgene. Preprocess data for use with cellxgene.
@@ -86,8 +111,7 @@ def prepare(
except ImportError: except ImportError:
raise click.ClickException( raise click.ClickException(
"[cellxgene] cellxgene prepare has not been installed. Please run `pip install cellxgene[prepare]` " "[cellxgene] cellxgene prepare has not been installed. Please run `pip install cellxgene[prepare]` "
"to install the necessary requirements." "to install the necessary requirements.")
)
# scanpy settings # scanpy settings
sc.settings.verbosity = 0 sc.settings.verbosity = 0
@@ -102,10 +126,11 @@ def prepare(
if not output: if not output:
click.echo( click.echo(
"Warning: No file will be saved, to save the results of cellxgene prepare include " "Warning: No file will be saved, to save the results of cellxgene prepare include "
"--output <filename> to save output to a new file" "--output <filename> to save output to a new file")
)
if isfile(output) and not overwrite: if isfile(output) and not overwrite:
raise click.UsageError(f"Cannot overwrite existing file {output}, try using the flag --overwrite") raise click.UsageError(
f"Cannot overwrite existing file {output}, try using the flag --overwrite"
)
def load_data(data): def load_data(data):
if isfile(data): if isfile(data):
@@ -115,7 +140,9 @@ def prepare(
elif extension == ".loom": elif extension == ".loom":
adata = sc.read_loom(data) adata = sc.read_loom(data)
else: else:
raise click.FileError(data, hint="does not have a valid extension [.h5ad | .loom]") raise click.FileError(
data,
hint="does not have a valid extension [.h5ad | .loom]")
elif isdir(data): elif isdir(data):
if not data.endswith(sep): if not data.endswith(sep):
data += sep data += sep
@@ -125,11 +152,15 @@ def prepare(
if not set_obs_names == "": if not set_obs_names == "":
if set_obs_names not in adata.obs_keys(): if set_obs_names not in adata.obs_keys():
raise click.UsageError(f"obs {set_obs_names} not found, options are: {adata.obs_keys()}") raise click.UsageError(
f"obs {set_obs_names} not found, options are: {adata.obs_keys()}"
)
adata.obs_names = adata.obs[set_obs_names] adata.obs_names = adata.obs[set_obs_names]
if not set_var_names == "": if not set_var_names == "":
if set_var_names not in adata.var_keys(): if set_var_names not in adata.var_keys():
raise click.UsageError(f"var {set_var_names} not found, options are: {adata.var_keys()}") raise click.UsageError(
f"var {set_var_names} not found, options are: {adata.var_keys()}"
)
adata.var_names = adata.var[set_var_names] adata.var_names = adata.var[set_var_names]
if make_obs_names_unique: if make_obs_names_unique:
adata.obs_names_make_unique() adata.obs_names_make_unique()
@@ -184,12 +215,18 @@ def prepare(
if "umap" in embedding: if "umap" in embedding:
sc.tl.umap(adata) sc.tl.umap(adata)
if plotting: if plotting:
sc.pl.umap(adata, color="louvain", palette=palette, save="_louvain") sc.pl.umap(adata,
color="louvain",
palette=palette,
save="_louvain")
if "tsne" in embedding: if "tsne" in embedding:
sc.tl.tsne(adata) sc.tl.tsne(adata)
if plotting: if plotting:
sc.pl.tsne(adata, color="louvain", palette=palette, save="_louvain") sc.pl.tsne(adata,
color="louvain",
palette=palette,
save="_louvain")
def show_step(item): def show_step(item):
if not skip_qc: if not skip_qc:
@@ -208,13 +245,19 @@ def prepare(
if item is not None: if item is not None:
return names[item.__name__] return names[item.__name__]
steps = [calculate_qc_metrics, make_sparse, run_recipe, run_pca, run_neighbors, run_louvain, run_embedding] steps = [
calculate_qc_metrics, make_sparse, run_recipe, run_pca, run_neighbors,
run_louvain, run_embedding
]
click.echo(f"[cellxgene] Loading data from {data}, please wait...") click.echo(f"[cellxgene] Loading data from {data}, please wait...")
adata = load_data(data) adata = load_data(data)
click.echo("[cellxgene] Beginning preprocessing...") click.echo("[cellxgene] Beginning preprocessing...")
with click.progressbar(steps, label="[cellxgene] Progress", show_eta=False, item_show_func=show_step) as bar: with click.progressbar(steps,
label="[cellxgene] Progress",
show_eta=False,
item_show_func=show_step) as bar:
for step in bar: for step in bar:
step(adata) step(adata)
server/gui/browser.py
+7 -4
View File
@@ -12,7 +12,9 @@ WindowUtils = cef.WindowUtils()
# noinspection PyUnresolvedReferences # noinspection PyUnresolvedReferences
CefWidgetParent = QWidget CefWidgetParent = QWidget
class CefWidget(CefWidgetParent): class CefWidget(CefWidgetParent):
def __init__(self, parent=None): def __init__(self, parent=None):
super(CefWidget, self).__init__(parent) super(CefWidget, self).__init__(parent)
self.parent = parent self.parent = parent
@@ -58,8 +60,8 @@ class CefWidget(CefWidgetParent):
if WINDOWS: if WINDOWS:
WindowUtils.OnSize(self.getHandle(), 0, 0, 0) WindowUtils.OnSize(self.getHandle(), 0, 0, 0)
elif LINUX: elif LINUX:
self.browser.SetBounds(self.x, self.y, self.browser.SetBounds(self.x, self.y, self.width(),
self.width(), self.height()) self.height())
self.browser.NotifyMoveOrResizeStarted() self.browser.NotifyMoveOrResizeStarted()
def resizeEvent(self, event): def resizeEvent(self, event):
@@ -68,12 +70,13 @@ class CefWidget(CefWidgetParent):
if WINDOWS: if WINDOWS:
WindowUtils.OnSize(self.getHandle(), 0, 0, 0) WindowUtils.OnSize(self.getHandle(), 0, 0, 0)
elif LINUX: elif LINUX:
self.browser.SetBounds(self.x, self.y, self.browser.SetBounds(self.x, self.y, size.width(),
size.width(), size.height()) size.height())
self.browser.NotifyMoveOrResizeStarted() self.browser.NotifyMoveOrResizeStarted()
class CefApplication(QApplication): class CefApplication(QApplication):
def __init__(self, args): def __init__(self, args):
super(CefApplication, self).__init__(args) super(CefApplication, self).__init__(args)
if not cef.GetAppSetting("external_message_pump"): if not cef.GetAppSetting("external_message_pump"):
server/gui/cellxgene_rc.py
+7 -2
View File
@@ -431,10 +431,15 @@ qt_resource_struct = b"\
\x00\x00\x00,\x00\x00\x00\x00\x00\x01\x00\x00\x11>\ \x00\x00\x00,\x00\x00\x00\x00\x00\x01\x00\x00\x11>\
" "
def qInitResources(): def qInitResources():
QtCore.qRegisterResourceData(0x01, qt_resource_struct, qt_resource_name, qt_resource_data) QtCore.qRegisterResourceData(0x01, qt_resource_struct, qt_resource_name,
qt_resource_data)
def qCleanupResources(): def qCleanupResources():
QtCore.qUnregisterResourceData(0x01, qt_resource_struct, qt_resource_name, qt_resource_data) QtCore.qUnregisterResourceData(0x01, qt_resource_struct, qt_resource_name,
qt_resource_data)
qInitResources() qInitResources()
server/gui/hook-cefpython3.py
+11 -9
View File
@@ -50,22 +50,22 @@ def check_pyinstaller_version():
version = PyInstaller.__version__ version = PyInstaller.__version__
match = re.search(r"^\d+\.\d+(\.\d+)?", version) match = re.search(r"^\d+\.\d+(\.\d+)?", version)
if not (match.group(0) >= PYINSTALLER_MIN_VERSION): if not (match.group(0) >= PYINSTALLER_MIN_VERSION):
raise SystemExit("Error: pyinstaller %s or higher is required" raise SystemExit("Error: pyinstaller %s or higher is required" %
% PYINSTALLER_MIN_VERSION) PYINSTALLER_MIN_VERSION)
def check_cefpython3_version(): def check_cefpython3_version():
if not is_module_satisfies("cefpython3 >= %s" % CEFPYTHON_MIN_VERSION): if not is_module_satisfies("cefpython3 >= %s" % CEFPYTHON_MIN_VERSION):
raise SystemExit("Error: cefpython3 %s or higher is required" raise SystemExit("Error: cefpython3 %s or higher is required" %
% CEFPYTHON_MIN_VERSION) CEFPYTHON_MIN_VERSION)
def get_cefpython_modules(): def get_cefpython_modules():
"""Get all cefpython Cython modules in the cefpython3 package. """Get all cefpython Cython modules in the cefpython3 package.
It returns a list of names without file extension. Eg. It returns a list of names without file extension. Eg.
'cefpython_py27'. """ 'cefpython_py27'. """
pyds = glob.glob(os.path.join(CEFPYTHON3_DIR, pyds = glob.glob(
"cefpython_py*" + CYTHON_MODULE_EXT)) os.path.join(CEFPYTHON3_DIR, "cefpython_py*" + CYTHON_MODULE_EXT))
assert len(pyds) > 1, "Missing cefpython3 Cython modules" assert len(pyds) > 1, "Missing cefpython3 Cython modules"
modules = [] modules = []
for path in pyds: for path in pyds:
@@ -155,7 +155,8 @@ def get_cefpython3_datas():
absolute_file_path = os.path.join(path, file) absolute_file_path = os.path.join(path, file)
dest_path = os.path.relpath(path, CEFPYTHON3_DIR) dest_path = os.path.relpath(path, CEFPYTHON3_DIR)
ret.append((absolute_file_path, dest_path)) ret.append((absolute_file_path, dest_path))
logger.info("Include cefpython3 data: {}/{}".format(dest_path, file)) logger.info("Include cefpython3 data: {}/{}".format(
dest_path, file))
elif is_win or is_linux: elif is_win or is_linux:
# The .pak files in cefpython3/locales/ directory # The .pak files in cefpython3/locales/ directory
locales_dir = os.path.join(CEFPYTHON3_DIR, "locales") locales_dir = os.path.join(CEFPYTHON3_DIR, "locales")
@@ -164,8 +165,9 @@ def get_cefpython3_datas():
for filename in os.listdir(locales_dir): for filename in os.listdir(locales_dir):
logger.info("Include cefpython3 data: {}/{}".format( logger.info("Include cefpython3 data: {}/{}".format(
os.path.basename(locales_dir), filename)) os.path.basename(locales_dir), filename))
ret.append((os.path.join(locales_dir, filename), ret.append(
os.path.join(cefdatadir, "locales"))) (os.path.join(locales_dir,
filename), os.path.join(cefdatadir, "locales")))
# Optional .so/.dll files in cefpython3/swiftshader/ directory # Optional .so/.dll files in cefpython3/swiftshader/ directory
swiftshader_dir = os.path.join(CEFPYTHON3_DIR, "swiftshader") swiftshader_dir = os.path.join(CEFPYTHON3_DIR, "swiftshader")
server/gui/main.py
+33 -11
View File
@@ -34,6 +34,7 @@ LOAD_INDEX = 1
class MainWindow(QMainWindow): class MainWindow(QMainWindow):
def __init__(self): def __init__(self):
super(MainWindow, self).__init__(None) super(MainWindow, self).__init__(None)
self.cef_widget = None self.cef_widget = None
@@ -59,14 +60,17 @@ class MainWindow(QMainWindow):
# close emitter on error/finished # close emitter on error/finished
self.parent_conn, self.child_conn = Pipe() self.parent_conn, self.child_conn = Pipe()
self.load_emitter = Emitter(self.parent_conn, WorkerSignals) self.load_emitter = Emitter(self.parent_conn, WorkerSignals)
self.emitter_thread = threading.Thread(target=self.load_emitter.run, daemon=True) self.emitter_thread = threading.Thread(target=self.load_emitter.run,
daemon=True)
self.emitter_thread.start() self.emitter_thread.start()
# send to load with error message? # send to load with error message?
def setupLayout(self): def setupLayout(self):
self.resize(WIDTH, HEIGHT) self.resize(WIDTH, HEIGHT)
self.cef_widget = CefWidget(self) self.cef_widget = CefWidget(self)
self.cef_widget.setSizePolicy(QSizePolicy(QSizePolicy.MinimumExpanding, QSizePolicy.MinimumExpanding)) self.cef_widget.setSizePolicy(
QSizePolicy(QSizePolicy.MinimumExpanding,
QSizePolicy.MinimumExpanding))
self.data_widget = LoadWidget(self) self.data_widget = LoadWidget(self)
self.stacked_layout = QStackedLayout() self.stacked_layout = QStackedLayout()
self.stacked_layout.addWidget(self.cef_widget) self.stacked_layout.addWidget(self.cef_widget)
@@ -104,7 +108,8 @@ class MainWindow(QMainWindow):
# close emitter on error/finished # close emitter on error/finished
self.parent_conn, self.child_conn = Pipe() self.parent_conn, self.child_conn = Pipe()
self.load_emitter = Emitter(self.parent_conn, WorkerSignals) self.load_emitter = Emitter(self.parent_conn, WorkerSignals)
self.emitter_thread = threading.Thread(target=self.load_emitter.run, daemon=True) self.emitter_thread = threading.Thread(target=self.load_emitter.run,
daemon=True)
self.emitter_thread.start() self.emitter_thread.start()
# send to load with error message? # send to load with error message?
@@ -144,6 +149,7 @@ class MainWindow(QMainWindow):
class LoadWidget(QFrame): class LoadWidget(QFrame):
def __init__(self, parent): def __init__(self, parent):
super(LoadWidget, self).__init__(parent=parent) super(LoadWidget, self).__init__(parent=parent)
# Init layout # Init layout
@@ -240,11 +246,18 @@ class LoadWidget(QFrame):
def createScanpyEngine(self, file_name): def createScanpyEngine(self, file_name):
title = splitext(basename(file_name))[0] title = splitext(basename(file_name))[0]
self.window().setupServer() self.window().setupServer()
worker = Worker(self.window().parent_conn, self.window().child_conn, file_name, host="127.0.0.1", worker = Worker(self.window().parent_conn,
port=GUI_PORT, title=title, engine_options={}) self.window().child_conn,
file_name,
host="127.0.0.1",
port=GUI_PORT,
title=title,
engine_options={})
self.window().load_emitter.signals.ready.connect(self.onDataReady) self.window().load_emitter.signals.ready.connect(self.onDataReady)
self.window().load_emitter.signals.engine_error.connect(self.onServerError) self.window().load_emitter.signals.engine_error.connect(
self.window().load_emitter.signals.server_error.connect(self.onServerError) self.onServerError)
self.window().load_emitter.signals.server_error.connect(
self.onServerError)
# Error is generic error from emitter # Error is generic error from emitter
self.window().load_emitter.signals.error.connect(self.onServerError) self.window().load_emitter.signals.error.connect(self.onServerError)
self.window().worker = Process(target=worker.run, daemon=True) self.window().worker = Process(target=worker.run, daemon=True)
@@ -266,7 +279,8 @@ class LoadWidget(QFrame):
self.site_ready_worker.signals.ready.connect(self.onServerReady) self.site_ready_worker.signals.ready.connect(self.onServerReady)
self.site_ready_worker.signals.error.connect(self.onServerError) self.site_ready_worker.signals.error.connect(self.onServerError)
srw_thread = threading.Thread(target=self.site_ready_worker.run, daemon=True) srw_thread = threading.Thread(target=self.site_ready_worker.run,
daemon=True)
srw_thread.start() srw_thread.start()
def onServerReady(self): def onServerReady(self):
@@ -289,7 +303,9 @@ class LoadWidget(QFrame):
onServerError = partialmethod(onError, server_error=True) onServerError = partialmethod(onError, server_error=True)
class FilePath(QObject): class FilePath(QObject):
def __init__(self): def __init__(self):
super(FilePath, self).__init__() super(FilePath, self).__init__()
self.value = "" self.value = ""
@@ -301,6 +317,7 @@ class FilePath(QObject):
class FileArea(QFrame): class FileArea(QFrame):
def __init__(self, parent): def __init__(self, parent):
super(FileArea, self).__init__() super(FileArea, self).__init__()
self.setFrameShape(QFrame.Box) self.setFrameShape(QFrame.Box)
@@ -309,10 +326,12 @@ class FileArea(QFrame):
self.setAcceptDrops(True) self.setAcceptDrops(True)
self.instructions = QLabel(self) self.instructions = QLabel(self)
self.instructions.setText("Drag & Drop a h5ad file to load or open") self.instructions.setText("Drag & Drop a h5ad file to load or open")
self.instructions.setGeometry(10, 10, MAX_CONTENT_WIDTH, self.instructions.height()) self.instructions.setGeometry(10, 10, MAX_CONTENT_WIDTH,
self.instructions.height())
self.loadButton = QPushButton("Open...", parent=self) self.loadButton = QPushButton("Open...", parent=self)
x_pos = (MAX_CONTENT_WIDTH - self.loadButton.width()) / 2 x_pos = (MAX_CONTENT_WIDTH - self.loadButton.width()) / 2
self.loadButton.setGeometry(x_pos, 50, self.loadButton.width(), self.loadButton.height()) self.loadButton.setGeometry(x_pos, 50, self.loadButton.width(),
self.loadButton.height())
self.loadButton.clicked.connect(self.fileBrowse) self.loadButton.clicked.connect(self.fileBrowse)
self.label = QLabel(self) self.label = QLabel(self)
self.label.setGeometry(10, 75, MAX_CONTENT_WIDTH, self.label.height()) self.label.setGeometry(10, 75, MAX_CONTENT_WIDTH, self.label.height())
@@ -321,7 +340,10 @@ class FileArea(QFrame):
options = QFileDialog.Options() options = QFileDialog.Options()
# options |= QFileDialog.DontUseNativeDialog # options |= QFileDialog.DontUseNativeDialog
file_name, _ = QFileDialog.getOpenFileName(self, file_name, _ = QFileDialog.getOpenFileName(self,
"Open H5AD File", "", "H5AD Files (*.h5ad)", options=options) "Open H5AD File",
"",
"H5AD Files (*.h5ad)",
options=options)
if file_name: if file_name:
self.parent().file_name.updateValue(file_name) self.parent().file_name.updateValue(file_name)
self.parent().onLoad() self.parent().onLoad()
server/gui/utils.py
+1
View File
@@ -49,6 +49,7 @@ class FileChanged(QObject):
class Emitter: class Emitter:
def __init__(self, transport, signals): def __init__(self, transport, signals):
self.transport = transport self.transport = transport
self.signals = signals() self.signals = signals()
server/gui/workers.py
+9 -2
View File
@@ -7,6 +7,7 @@ from server.gui.utils import SiteReadySignals
class EmittingProcess(Process): class EmittingProcess(Process):
def __init__(self, parent_conn, child_conn, *arg, **kwargs): def __init__(self, parent_conn, child_conn, *arg, **kwargs):
super(EmittingProcess, self).__init__() super(EmittingProcess, self).__init__()
self.parent_conn = parent_conn self.parent_conn = parent_conn
@@ -21,7 +22,9 @@ class EmittingProcess(Process):
class Worker(EmittingProcess): class Worker(EmittingProcess):
def __init__(self, parent_conn, child_conn, data_file, host, port, title, engine_options, *args, **kwargs):
def __init__(self, parent_conn, child_conn, data_file, host, port, title,
engine_options, *args, **kwargs):
super(Worker, self).__init__(parent_conn, child_conn) super(Worker, self).__init__(parent_conn, child_conn)
self.data_file = data_file self.data_file = data_file
self.host = host self.host = host
@@ -62,7 +65,10 @@ class Worker(EmittingProcess):
return return
# launch server # launch server
try: try:
server.app.run(host=self.host, debug=False, port=self.port, threaded=True) server.app.run(host=self.host,
debug=False,
port=self.port,
threaded=True)
except Exception as e: except Exception as e:
self.emit("server_error", str(e)) self.emit("server_error", str(e))
finally: finally:
@@ -70,6 +76,7 @@ class Worker(EmittingProcess):
class SiteReadyWorker: class SiteReadyWorker:
def __init__(self, location): def __init__(self, location):
super(SiteReadyWorker, self).__init__() super(SiteReadyWorker, self).__init__()
self.signals = SiteReadySignals() self.signals = SiteReadySignals()
server/test/decode_fbs.py
+11 -7
View File
@@ -1,4 +1,3 @@
""" """
Code to decode, for testing purposes, the flatbuffer encoded blobs. Code to decode, for testing purposes, the flatbuffer encoded blobs.
This code will need to be updated if fbs/matrix.fbs changes. This code will need to be updated if fbs/matrix.fbs changes.
@@ -18,18 +17,23 @@ import server.app.util.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray
def decode_typed_array(tarr): def decode_typed_array(tarr):
type_map = { type_map = {
TypedArray.TypedArray.Uint32Array: Uint32Array.Uint32Array, TypedArray.TypedArray.Uint32Array:
TypedArray.TypedArray.Int32Array: Int32Array.Int32Array, Uint32Array.Uint32Array,
TypedArray.TypedArray.Float32Array: Float32Array.Float32Array, TypedArray.TypedArray.Int32Array:
TypedArray.TypedArray.Float64Array: Float64Array.Float64Array, Int32Array.Int32Array,
TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray TypedArray.TypedArray.Float32Array:
Float32Array.Float32Array,
TypedArray.TypedArray.Float64Array:
Float64Array.Float64Array,
TypedArray.TypedArray.JSONEncodedArray:
JSONEncodedArray.JSONEncodedArray
} }
(u_type, u) = tarr (u_type, u) = tarr
if u_type == TypedArray.TypedArray.NONE: if u_type == TypedArray.TypedArray.NONE:
return None return None
TarType = type_map.get(u_type, None) TarType = type_map.get(u_type, None)
assert(TarType is not None) assert (TarType is not None)
arr = TarType() arr = TarType()
arr.Init(u.Bytes, u.Pos) arr.Init(u.Bytes, u.Pos)
server/test/test_api.py
+80 -28
View File
@@ -19,7 +19,10 @@ class EndPoints(unittest.TestCase):
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
cls.ps = Popen(["cellxgene", "launch", "../example-dataset/pbmc3k.h5ad", "--verbose", "--port", "5005"]) cls.ps = Popen([
"cellxgene", "launch", "../example-dataset/pbmc3k.h5ad",
"--verbose", "--port", "5005"
])
session = requests.Session() session = requests.Session()
for i in range(90): for i in range(90):
try: try:
@@ -47,7 +50,8 @@ class EndPoints(unittest.TestCase):
result_data = result.json() result_data = result.json()
self.assertEqual(result_data["schema"]["dataframe"]["nObs"], 2638) self.assertEqual(result_data["schema"]["dataframe"]["nObs"], 2638)
self.assertEqual(len(result_data["schema"]["annotations"]["obs"]), 2) self.assertEqual(len(result_data["schema"]["annotations"]["obs"]), 2)
self.assertEqual(len(result_data["schema"]["annotations"]["obs"]["columns"]), 5) self.assertEqual(
len(result_data["schema"]["annotations"]["obs"]["columns"]), 5)
def test_config(self): def test_config(self):
endpoint = "config" endpoint = "config"
@@ -57,7 +61,8 @@ class EndPoints(unittest.TestCase):
self.assertEqual(result.headers["Content-Type"], "application/json") self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json() result_data = result.json()
self.assertIn("library_versions", result_data["config"]) self.assertIn("library_versions", result_data["config"])
self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k") self.assertEqual(result_data["config"]["displayNames"]["dataset"],
"pbmc3k")
self.assertEqual(len(result_data["config"]["features"]), 4) self.assertEqual(len(result_data["config"]["features"]), 4)
def test_get_layout_fbs(self): def test_get_layout_fbs(self):
@@ -66,13 +71,15 @@ class EndPoints(unittest.TestCase):
header = {"Accept": "application/octet-stream"} header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header) result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream") self.assertEqual(result.headers["Content-Type"],
"application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content) df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638) self.assertEqual(df['n_rows'], 2638)
self.assertEqual(df['n_cols'], 8) self.assertEqual(df['n_cols'], 8)
self.assertIsNotNone(df['columns']) self.assertIsNotNone(df['columns'])
self.assertListEqual(df['col_idx'], [ self.assertListEqual(df['col_idx'], [
'pca_0', 'pca_1', 'tsne_0', 'tsne_1', 'umap_0', 'umap_1', 'draw_graph_fr_0', 'draw_graph_fr_1' 'pca_0', 'pca_1', 'tsne_0', 'tsne_1', 'umap_0', 'umap_1',
'draw_graph_fr_0', 'draw_graph_fr_1'
]) ])
self.assertIsNone(df['row_idx']) self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols']) self.assertEqual(len(df['columns']), df['n_cols'])
@@ -89,7 +96,8 @@ class EndPoints(unittest.TestCase):
header = {"Accept": "application/octet-stream"} header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header) result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream") self.assertEqual(result.headers["Content-Type"],
"application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content) df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638) self.assertEqual(df['n_rows'], 2638)
self.assertEqual(df['n_cols'], 5) self.assertEqual(df['n_cols'], 5)
@@ -97,8 +105,11 @@ class EndPoints(unittest.TestCase):
self.assertIsNotNone(df['col_idx']) self.assertIsNotNone(df['col_idx'])
self.assertIsNone(df['row_idx']) self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols']) self.assertEqual(len(df['columns']), df['n_cols'])
obs_index_col_name = self.schema["schema"]["annotations"]["obs"]["index"] obs_index_col_name = self.schema["schema"]["annotations"]["obs"][
self.assertListEqual(df['col_idx'], [obs_index_col_name, 'n_genes', 'percent_mito', 'n_counts', 'louvain']) "index"]
self.assertListEqual(df['col_idx'], [
obs_index_col_name, 'n_genes', 'percent_mito', 'n_counts', 'louvain'
])
def test_get_annotations_obs_keys_fbs(self): def test_get_annotations_obs_keys_fbs(self):
endpoint = "annotations/obs" endpoint = "annotations/obs"
@@ -107,7 +118,8 @@ class EndPoints(unittest.TestCase):
header = {"Accept": "application/octet-stream"} header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header) result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream") self.assertEqual(result.headers["Content-Type"],
"application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content) df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638) self.assertEqual(df['n_rows'], 2638)
self.assertEqual(df['n_cols'], 2) self.assertEqual(df['n_cols'], 2)
@@ -129,8 +141,26 @@ class EndPoints(unittest.TestCase):
url = f"{URL_BASE}{endpoint}" url = f"{URL_BASE}{endpoint}"
params = { params = {
"mode": "topN", "mode": "topN",
"set1": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["NK cells"]}]}}}, "set1": {
"set2": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["CD8 T cells"]}]}}}, "filter": {
"obs": {
"annotation_value": [{
"name": "louvain",
"values": ["NK cells"]
}]
}
}
},
"set2": {
"filter": {
"obs": {
"annotation_value": [{
"name": "louvain",
"values": ["CD8 T cells"]
}]
}
}
},
"count": 7, "count": 7,
} }
result = self.session.post(url, json=params) result = self.session.post(url, json=params)
@@ -145,8 +175,20 @@ class EndPoints(unittest.TestCase):
params = { params = {
"mode": "topN", "mode": "topN",
"count": 10, "count": 10,
"set1": {"filter": {"obs": {"index": [[0, 500]]}}}, "set1": {
"set2": {"filter": {"obs": {"index": [[500, 1000]]}}}, "filter": {
"obs": {
"index": [[0, 500]]
}
}
},
"set2": {
"filter": {
"obs": {
"index": [[500, 1000]]
}
}
},
} }
result = self.session.post(url, json=params) result = self.session.post(url, json=params)
self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.status_code, HTTPStatus.OK)
@@ -160,7 +202,8 @@ class EndPoints(unittest.TestCase):
header = {"Accept": "application/octet-stream"} header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header) result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream") self.assertEqual(result.headers["Content-Type"],
"application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content) df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 1838) self.assertEqual(df['n_rows'], 1838)
self.assertEqual(df['n_cols'], 2) self.assertEqual(df['n_cols'], 2)
@@ -168,7 +211,8 @@ class EndPoints(unittest.TestCase):
self.assertIsNotNone(df['col_idx']) self.assertIsNotNone(df['col_idx'])
self.assertIsNone(df['row_idx']) self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols']) self.assertEqual(len(df['columns']), df['n_cols'])
var_index_col_name = self.schema["schema"]["annotations"]["var"]["index"] var_index_col_name = self.schema["schema"]["annotations"]["var"][
"index"]
self.assertListEqual(df['col_idx'], [var_index_col_name, 'n_cells']) self.assertListEqual(df['col_idx'], [var_index_col_name, 'n_cells'])
def test_get_annotations_var_keys_fbs(self): def test_get_annotations_var_keys_fbs(self):
@@ -178,7 +222,8 @@ class EndPoints(unittest.TestCase):
header = {"Accept": "application/octet-stream"} header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header) result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream") self.assertEqual(result.headers["Content-Type"],
"application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content) df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 1838) self.assertEqual(df['n_rows'], 1838)
self.assertEqual(df['n_cols'], 1) self.assertEqual(df['n_cols'], 1)
@@ -207,7 +252,8 @@ class EndPoints(unittest.TestCase):
url = f"{URL_BASE}{endpoint}" url = f"{URL_BASE}{endpoint}"
result = self.session.put(url) result = self.session.put(url)
self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream") self.assertEqual(result.headers["Content-Type"],
"application/octet-stream")
def test_data_put_fbs(self): def test_data_put_fbs(self):
endpoint = f"data/var" endpoint = f"data/var"
@@ -215,7 +261,8 @@ class EndPoints(unittest.TestCase):
header = {"Accept": "application/octet-stream"} header = {"Accept": "application/octet-stream"}
result = self.session.put(url, headers=header) result = self.session.put(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream") self.assertEqual(result.headers["Content-Type"],
"application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content) df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638) self.assertEqual(df['n_rows'], 2638)
self.assertEqual(df['n_cols'], 1838) self.assertEqual(df['n_cols'], 1838)
@@ -228,16 +275,11 @@ class EndPoints(unittest.TestCase):
endpoint = f"data/var" endpoint = f"data/var"
url = f"{URL_BASE}{endpoint}" url = f"{URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"} header = {"Accept": "application/octet-stream"}
filter = { filter = {"filter": {"var": {"index": [0, 1, 4]}}}
"filter": {
"var": {
"index": [0, 1, 4]
}
}
}
result = self.session.put(url, headers=header, json=filter) result = self.session.put(url, headers=header, json=filter)
self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream") self.assertEqual(result.headers["Content-Type"],
"application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content) df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638) self.assertEqual(df['n_rows'], 2638)
self.assertEqual(df['n_cols'], 3) self.assertEqual(df['n_cols'], 3)
@@ -252,10 +294,20 @@ class EndPoints(unittest.TestCase):
url = f"{URL_BASE}{endpoint}" url = f"{URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"} header = {"Accept": "application/octet-stream"}
index_col_name = self.schema["schema"]["annotations"]["var"]["index"] index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
var_filter = {"filter": {"var": {"annotation_value": [{"name": index_col_name, "values": ["RER1"]}]}}} var_filter = {
"filter": {
"var": {
"annotation_value": [{
"name": index_col_name,
"values": ["RER1"]
}]
}
}
}
result = self.session.put(url, headers=header, json=var_filter) result = self.session.put(url, headers=header, json=var_filter)
self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream") self.assertEqual(result.headers["Content-Type"],
"application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content) df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df["n_rows"], 2638) self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 1) self.assertEqual(df["n_cols"], 1)
server/test/test_fbs.py
+28 -25
View File
@@ -40,49 +40,52 @@ class FbsTests(unittest.TestCase):
def test_encode_DataFrame(self): def test_encode_DataFrame(self):
df = pd.DataFrame( df = pd.DataFrame(
data={ data={
'a': np.zeros((10,), dtype=np.float32), 'a':
'b': np.ones((10,), dtype=np.int64), np.zeros((10,), dtype=np.float32),
'c': np.array([i for i in range(0, 10)], dtype=np.uint16), 'b':
'd': pd.Series(['x', 'y', 'z', 'x', 'y', 'z', 'a', 'x', 'y', 'z'], dtype='category') np.ones((10,), dtype=np.int64),
'c':
np.array([i for i in range(0, 10)], dtype=np.uint16),
'd':
pd.Series(
['x', 'y', 'z', 'x', 'y', 'z', 'a', 'x', 'y', 'z'],
dtype='category')
}) })
expected_types = ( expected_types = ((np.ndarray, np.float32), (np.ndarray, np.int32),
(np.ndarray, np.float32), (np.ndarray, np.uint32), (list, None))
(np.ndarray, np.int32),
(np.ndarray, np.uint32),
(list, None)
)
fbs = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns) fbs = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
self.fbs_checks(fbs, (10, 4), expected_types, ['a', 'b', 'c', 'd']) self.fbs_checks(fbs, (10, 4), expected_types, ['a', 'b', 'c', 'd'])
def test_encode_ndarray(self): def test_encode_ndarray(self):
arr = np.zeros((3, 2), dtype=np.float32) arr = np.zeros((3, 2), dtype=np.float32)
expected_types = ( expected_types = ((np.ndarray, np.float32), (np.ndarray, np.float32),
(np.ndarray, np.float32), (np.ndarray, np.float32))
(np.ndarray, np.float32),
(np.ndarray, np.float32)
)
fbs = encode_matrix_fbs(matrix=arr, row_idx=None, col_idx=None) fbs = encode_matrix_fbs(matrix=arr, row_idx=None, col_idx=None)
self.fbs_checks(fbs, (3, 2), expected_types, None) self.fbs_checks(fbs, (3, 2), expected_types, None)
def test_encode_sparse(self): def test_encode_sparse(self):
csc = sparse.csc_matrix(np.array([[0, 1, 2], [3, 0, 4]])) csc = sparse.csc_matrix(np.array([[0, 1, 2], [3, 0, 4]]))
expected_types = ( expected_types = ((np.ndarray, np.int32), (np.ndarray, np.int32),
(np.ndarray, np.int32), (np.ndarray, np.int32))
(np.ndarray, np.int32),
(np.ndarray, np.int32)
)
fbs = encode_matrix_fbs(matrix=csc, row_idx=None, col_idx=None) fbs = encode_matrix_fbs(matrix=csc, row_idx=None, col_idx=None)
self.fbs_checks(fbs, (2, 3), expected_types, None) self.fbs_checks(fbs, (2, 3), expected_types, None)
def test_roundtrip(self): def test_roundtrip(self):
dfSrc = pd.DataFrame( dfSrc = pd.DataFrame(
data={ data={
'a': np.zeros((10,), dtype=np.float32), 'a':
'b': np.ones((10,), dtype=np.int64), np.zeros((10,), dtype=np.float32),
'c': np.array([i for i in range(0, 10)], dtype=np.uint16), 'b':
'd': pd.Series(['x', 'y', 'z', 'x', 'y', 'z', 'a', 'x', 'y', 'z'], dtype='category') np.ones((10,), dtype=np.int64),
'c':
np.array([i for i in range(0, 10)], dtype=np.uint16),
'd':
pd.Series(
['x', 'y', 'z', 'x', 'y', 'z', 'a', 'x', 'y', 'z'],
dtype='category')
}) })
dfDst = decode_matrix_fbs(encode_matrix_fbs(matrix=dfSrc, col_idx=dfSrc.columns)) dfDst = decode_matrix_fbs(
encode_matrix_fbs(matrix=dfSrc, col_idx=dfSrc.columns))
self.assertEqual(dfSrc.shape, dfDst.shape) self.assertEqual(dfSrc.shape, dfDst.shape)
self.assertEqual(set(dfSrc.columns), set(dfDst.columns)) self.assertEqual(set(dfSrc.columns), set(dfDst.columns))
for c in dfSrc.columns: for c in dfSrc.columns:
server/test/test_matrix_proxy.py
+43 -95
View File
@@ -7,12 +7,14 @@ class NdArrayProxyView(MatrixProxyView):
""" """
Fake test class for matrix proxy - wraps ndarray Fake test class for matrix proxy - wraps ndarray
""" """
@classmethod @classmethod
def __supports__(cls): def __supports__(cls):
return ('numpy.ndarray', ) return ('numpy.ndarray',)
class MatrixProxyViewTest(unittest.TestCase): class MatrixProxyViewTest(unittest.TestCase):
def test_ismatrixproxy(self): def test_ismatrixproxy(self):
n = np.zeros((2, 4)) n = np.zeros((2, 4))
mp = MatrixProxy.create(n) mp = MatrixProxy.create(n)
@@ -41,18 +43,13 @@ class MatrixProxyViewTest(unittest.TestCase):
def test_toarray(self): def test_toarray(self):
n = np.arange(15, dtype=np.float32).reshape((3, 5)) n = np.arange(15, dtype=np.float32).reshape((3, 5))
mp = MatrixProxy.create(n) mp = MatrixProxy.create(n)
self.assertTrue(np.all(mp.toarray() == [ self.assertTrue(
[0., 1., 2., 3., 4.], np.all(mp.toarray() == [[0., 1., 2., 3., 4.], [5., 6., 7., 8., 9.],
[5., 6., 7., 8., 9.], [10., 11., 12., 13., 14.]]))
[10., 11., 12., 13., 14.] self.assertTrue(
])) np.all(
self.assertTrue(np.all(mp.T.toarray() == [ mp.T.toarray() == [[0., 5., 10.], [1., 6., 11.], [2., 7., 12.],
[0., 5., 10.], [3., 8., 13.], [4., 9., 14.]]))
[1., 6., 11.],
[2., 7., 12.],
[3., 8., 13.],
[4., 9., 14.]
]))
def test_indexing(self): def test_indexing(self):
""" """
@@ -95,47 +92,25 @@ class MatrixProxyViewTest(unittest.TestCase):
# slice, slice # slice, slice
self.assertTrue(np.all(mp[1:3, 2:4].toarray() == [ self.assertTrue(np.all(mp[1:3, 2:4].toarray() == [[7, 8], [12, 13]]))
[7, 8], self.assertTrue(
[12, 13] np.all(mp[:3, :4].toarray() == [[0., 1., 2., 3.], [5., 6., 7., 8.],
])) [10., 11., 12., 13.]]))
self.assertTrue(np.all(mp[:3, :4].toarray() == [ self.assertTrue(
[0., 1., 2., 3.], np.all(mp[::-1, ::-1].toarray() ==
[5., 6., 7., 8.], [[14, 13, 12, 11, 10], [9, 8, 7, 6, 5], [4, 3, 2, 1, 0]]))
[10., 11., 12., 13.] self.assertTrue(
])) np.all(mp[::-2, ::-2].toarray() == [[14, 12, 10], [4, 2, 0]]))
self.assertTrue(np.all(mp[::-1, ::-1].toarray() == [
[14, 13, 12, 11, 10],
[9, 8, 7, 6, 5],
[4, 3, 2, 1, 0]
]))
self.assertTrue(np.all(mp[::-2, ::-2].toarray() == [
[14, 12, 10],
[4, 2, 0]
]))
self.assertTrue(np.all(mp.T[2:4, 1:3].toarray() == [ self.assertTrue(np.all(mp.T[2:4, 1:3].toarray() == [[7, 12], [8, 13]]))
[7, 12], self.assertTrue(
[8, 13] np.all(mp.T[:4, :3].toarray() == [[0, 5, 10], [1, 6, 11],
])) [2, 7, 12], [3, 8, 13]]))
self.assertTrue(np.all(mp.T[:4, :3].toarray() == [ self.assertTrue(
[0, 5, 10], np.all(mp.T[::-1, ::-1].toarray(
[1, 6, 11], ) == [[14, 9, 4], [13, 8, 3], [12, 7, 2], [11, 6, 1], [10, 5, 0]]))
[2, 7, 12], self.assertTrue(
[3, 8, 13] np.all(mp.T[::-2, ::-2].toarray() == [[14, 4], [12, 2], [10, 0]]))
]))
self.assertTrue(np.all(mp.T[::-1, ::-1].toarray() == [
[14, 9, 4],
[13, 8, 3],
[12, 7, 2],
[11, 6, 1],
[10, 5, 0]
]))
self.assertTrue(np.all(mp.T[::-2, ::-2].toarray() == [
[14, 4],
[12, 2],
[10, 0]
]))
def test_repeated_indexing(self): def test_repeated_indexing(self):
""" """
@@ -149,31 +124,16 @@ class MatrixProxyViewTest(unittest.TestCase):
self.assertEqual(mp[0][1], 1) self.assertEqual(mp[0][1], 1)
self.assertEqual(mp.T[0][1], 5) self.assertEqual(mp.T[0][1], 5)
self.assertTrue(np.all(mp[0::-1, ::-1][0, 2:4].toarray() == [ self.assertTrue(np.all(mp[0::-1, ::-1][0, 2:4].toarray() == [2, 1]))
2, 1 self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 1:3:1].toarray() == [2, 3]))
])) self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 2:0:-1].toarray() == [3, 2]))
self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 1:3:1].toarray() == [ self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, 1:3:1].toarray() == [3, 2]))
2, 3 self.assertTrue(np.all(mp[0::-1, 5:1:-1][0,
])) 2:0:-1].toarray() == [2, 3]))
self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 2:0:-1].toarray() == [
3, 2
]))
self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, 1:3:1].toarray() == [
3, 2
]))
self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, 2:0:-1].toarray() == [
2, 3
]))
self.assertTrue(np.all(mp.T[::-1, 0::-1][2:4, 0].toarray() == [ self.assertTrue(np.all(mp.T[::-1, 0::-1][2:4, 0].toarray() == [2, 1]))
2, 1 self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 1:3:1].toarray() == [10]))
])) self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 2:0:-1].toarray() == [10]))
self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 1:3:1].toarray() == [
10
]))
self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 2:0:-1].toarray() == [
10
]))
self.assertTrue(np.all(mp.T[0::-1, 5:1:-1][0, 1:3:1].toarray() == [])) self.assertTrue(np.all(mp.T[0::-1, 5:1:-1][0, 1:3:1].toarray() == []))
self.assertTrue(np.all(mp.T[0::-1, 5:1:-1][0, 2:0:-1].toarray() == [])) self.assertTrue(np.all(mp.T[0::-1, 5:1:-1][0, 2:0:-1].toarray() == []))
@@ -190,20 +150,12 @@ class MatrixProxyViewTest(unittest.TestCase):
self.assertEqual(mp[0, 0], 0) self.assertEqual(mp[0, 0], 0)
# drop 1 dimension, to an array # drop 1 dimension, to an array
self.assertTrue(np.all(mp[0, :].toarray() == [ self.assertTrue(np.all(mp[0, :].toarray() == [0, 1, 2, 3, 4]))
0, 1, 2, 3, 4 self.assertTrue(np.all(mp[:, 0].toarray() == [0, 5, 10]))
]))
self.assertTrue(np.all(mp[:, 0].toarray() == [
0, 5, 10
]))
# with .T # with .T
self.assertTrue(np.all(mp[0:2].T[-1:].toarray() == [ self.assertTrue(np.all(mp[0:2].T[-1:].toarray() == [[4, 9]]))
[4, 9] self.assertTrue(np.all(mp[0:2].T[-1].toarray() == [4, 9]))
]))
self.assertTrue(np.all(mp[0:2].T[-1].toarray() == [
4, 9
]))
def test_iter(self): def test_iter(self):
""" """
@@ -214,17 +166,13 @@ class MatrixProxyViewTest(unittest.TestCase):
rows = [r for r in mp] rows = [r for r in mp]
self.assertEqual(len(rows), 3) self.assertEqual(len(rows), 3)
self.assertTrue(np.all(rows[0].toarray() == [ self.assertTrue(np.all(rows[0].toarray() == [0, 1, 2, 3, 4]))
0, 1, 2, 3, 4
]))
for i, r in enumerate(rows): for i, r in enumerate(rows):
self.assertTrue(np.all(mp[i].toarray() == r.toarray())) self.assertTrue(np.all(mp[i].toarray() == r.toarray()))
cols = [c for c in mp.T] cols = [c for c in mp.T]
self.assertEqual(len(cols), 5) self.assertEqual(len(cols), 5)
self.assertTrue(np.all(cols[0].toarray() == [ self.assertTrue(np.all(cols[0].toarray() == [0, 5, 10]))
0, 5, 10
]))
for i, c in enumerate(cols): for i, c in enumerate(cols):
self.assertTrue(np.all(mp.T[i].toarray() == c.toarray())) self.assertTrue(np.all(mp.T[i].toarray() == c.toarray()))
server/test/test_nan_rest.py
+10 -6
View File
@@ -20,9 +20,10 @@ class WithNaNs(unittest.TestCase):
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
cls.ps = Popen( cls.ps = Popen([
["cellxgene", "launch", "test/test_datasets/nan.h5ad", "--verbose", "--port", "5006"] "cellxgene", "launch", "test/test_datasets/nan.h5ad", "--verbose",
) "--port", "5006"
])
session = requests.Session() session = requests.Session()
for i in range(90): for i in range(90):
try: try:
@@ -51,7 +52,8 @@ class WithNaNs(unittest.TestCase):
url = f"{URL_BASE}{endpoint}" url = f"{URL_BASE}{endpoint}"
result = self.session.put(url) result = self.session.put(url)
self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream") self.assertEqual(result.headers["Content-Type"],
"application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content) df = decode_fbs.decode_matrix_FBS(result.content)
self.assertTrue(math.isnan(df["columns"][3][3])) self.assertTrue(math.isnan(df["columns"][3][3]))
@@ -60,7 +62,8 @@ class WithNaNs(unittest.TestCase):
url = f"{URL_BASE}{endpoint}" url = f"{URL_BASE}{endpoint}"
result = self.session.get(url) result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream") self.assertEqual(result.headers["Content-Type"],
"application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content) df = decode_fbs.decode_matrix_FBS(result.content)
self.assertTrue(math.isnan(df["columns"][2][0])) self.assertTrue(math.isnan(df["columns"][2][0]))
@@ -69,6 +72,7 @@ class WithNaNs(unittest.TestCase):
url = f"{URL_BASE}{endpoint}" url = f"{URL_BASE}{endpoint}"
result = self.session.get(url) result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream") self.assertEqual(result.headers["Content-Type"],
"application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content) df = decode_fbs.decode_matrix_FBS(result.content)
self.assertTrue(math.isnan(df["columns"][2][0])) self.assertTrue(math.isnan(df["columns"][2][0]))
server/test/test_nan_scanpy_engine.py
+17 -15
View File
@@ -11,6 +11,7 @@ from server.app.util.data_locator import DataLocator
class NaNTest(unittest.TestCase): class NaNTest(unittest.TestCase):
def setUp(self): def setUp(self):
self.args = { self.args = {
"layout": ["umap"], "layout": ["umap"],
@@ -21,7 +22,8 @@ class NaNTest(unittest.TestCase):
} }
with warnings.catch_warnings(): with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UserWarning) warnings.simplefilter("ignore", category=UserWarning)
self.data = ScanpyEngine(DataLocator("test/test_datasets/nan.h5ad"), self.args) self.data = ScanpyEngine(DataLocator("test/test_datasets/nan.h5ad"),
self.args)
self.data._create_schema() self.data._create_schema()
def test_load(self): def test_load(self):
@@ -35,7 +37,8 @@ class NaNTest(unittest.TestCase):
self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon) self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
def test_dataframe(self): def test_dataframe(self):
data_frame_var = decode_fbs.decode_matrix_FBS(self.data.data_frame_to_fbs_matrix(None, "var")) data_frame_var = decode_fbs.decode_matrix_FBS(
self.data.data_frame_to_fbs_matrix(None, "var"))
self.assertIsNotNone(data_frame_var) self.assertIsNotNone(data_frame_var)
self.assertEqual(data_frame_var["n_rows"], 100) self.assertEqual(data_frame_var["n_rows"], 100)
self.assertEqual(data_frame_var["n_cols"], 100) self.assertEqual(data_frame_var["n_cols"], 100)
@@ -44,30 +47,29 @@ class NaNTest(unittest.TestCase):
with pytest.raises(FilterError): with pytest.raises(FilterError):
self.data.data_frame_to_fbs_matrix("an erroneous filter", "var") self.data.data_frame_to_fbs_matrix("an erroneous filter", "var")
with pytest.raises(FilterError): with pytest.raises(FilterError):
filter_ = { filter_ = {"filter": {"obs": {"index": [1, 99, [200, 300]]}}}
"filter": {
"obs": {"index": [1, 99, [200, 300]]}
}
}
self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
def test_dataframe_obs_not_implemented(self): def test_dataframe_obs_not_implemented(self):
with self.assertRaises(ValueError) as cm: with self.assertRaises(ValueError) as cm:
decode_fbs.decode_matrix_FBS(self.data.data_frame_to_fbs_matrix(None, "obs")) decode_fbs.decode_matrix_FBS(
self.data.data_frame_to_fbs_matrix(None, "obs"))
self.assertIsNotNone(cm.exception) self.assertIsNotNone(cm.exception)
def test_annotation(self): def test_annotation(self):
annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("obs")) annotations = decode_fbs.decode_matrix_FBS(
self.data.annotation_to_fbs_matrix("obs"))
obs_index_col_name = self.data.schema["annotations"]["obs"]["index"] obs_index_col_name = self.data.schema["annotations"]["obs"]["index"]
self.assertEqual( self.assertEqual(annotations["col_idx"], [
annotations["col_idx"], obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"
[obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"] ])
)
self.assertEqual(annotations["n_rows"], 100) self.assertEqual(annotations["n_rows"], 100)
self.assertTrue(math.isnan(annotations["columns"][2][0])) self.assertTrue(math.isnan(annotations["columns"][2][0]))
annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("var")) annotations = decode_fbs.decode_matrix_FBS(
self.data.annotation_to_fbs_matrix("var"))
var_index_col_name = self.data.schema["annotations"]["var"]["index"] var_index_col_name = self.data.schema["annotations"]["var"]["index"]
self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells", "var_with_nans"]) self.assertEqual(annotations["col_idx"],
[var_index_col_name, "n_cells", "var_with_nans"])
self.assertEqual(annotations["n_rows"], 100) self.assertEqual(annotations["n_rows"], 100)
self.assertTrue(math.isnan(annotations["columns"][2][0])) self.assertTrue(math.isnan(annotations["columns"][2][0]))
server/test/test_scanpy_engine.py
+64 -26
View File
@@ -12,7 +12,6 @@ import pandas as pd
from server.app.scanpy_engine.scanpy_engine import ScanpyEngine from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
from server.app.util.errors import FilterError, DisabledFeatureError from server.app.util.errors import FilterError, DisabledFeatureError
from server.app.util.data_locator import DataLocator from server.app.util.data_locator import DataLocator
""" """
Test the scanpy engine using the pbmc3k data set. Test the scanpy engine using the pbmc3k data set.
""" """
@@ -22,12 +21,12 @@ Test the scanpy engine using the pbmc3k data set.
("../example-dataset/pbmc3k.h5ad", False), ("../example-dataset/pbmc3k.h5ad", False),
("test/test_datasets/pbmc3k-CSC-gz.h5ad", False), ("test/test_datasets/pbmc3k-CSC-gz.h5ad", False),
("test/test_datasets/pbmc3k-CSR-gz.h5ad", False), ("test/test_datasets/pbmc3k-CSR-gz.h5ad", False),
("../example-dataset/pbmc3k.h5ad", True), ("../example-dataset/pbmc3k.h5ad", True),
("test/test_datasets/pbmc3k-CSC-gz.h5ad", True), ("test/test_datasets/pbmc3k-CSC-gz.h5ad", True),
("test/test_datasets/pbmc3k-CSR-gz.h5ad", True), ("test/test_datasets/pbmc3k-CSR-gz.h5ad", True),
]) ])
class EngineTest(unittest.TestCase): class EngineTest(unittest.TestCase):
def setUp(self): def setUp(self):
args = { args = {
"layout": ["umap"], "layout": ["umap"],
@@ -47,10 +46,12 @@ class EngineTest(unittest.TestCase):
self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon) self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
def test_mandatory_annotations(self): def test_mandatory_annotations(self):
obs_index_col_name = self.data.get_schema()["annotations"]["obs"]["index"] obs_index_col_name = self.data.get_schema(
)["annotations"]["obs"]["index"]
self.assertIn(obs_index_col_name, self.data.data.obs) self.assertIn(obs_index_col_name, self.data.data.obs)
self.assertEqual(list(self.data.data.obs.index), list(range(2638))) self.assertEqual(list(self.data.data.obs.index), list(range(2638)))
var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"] var_index_col_name = self.data.get_schema(
)["annotations"]["var"]["index"]
self.assertIn(var_index_col_name, self.data.data.var) self.assertIn(var_index_col_name, self.data.data.var)
self.assertEqual(list(self.data.data.var.index), list(range(1838))) self.assertEqual(list(self.data.data.var.index), list(range(1838)))
@@ -64,11 +65,7 @@ class EngineTest(unittest.TestCase):
self.data._validate_data_types() self.data._validate_data_types()
def test_filter_idx(self): def test_filter_idx(self):
filter_ = { filter_ = {"filter": {"var": {"index": [1, 99, [200, 300]]}}}
"filter": {
"var": {"index": [1, 99, [200, 300]]}
}
}
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
data = decode_fbs.decode_matrix_FBS(fbs) data = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(data["n_rows"], 2638) self.assertEqual(data["n_rows"], 2638)
@@ -78,9 +75,10 @@ class EngineTest(unittest.TestCase):
filter_ = { filter_ = {
"filter": { "filter": {
"var": { "var": {
"annotation_value": [ "annotation_value": [{
{"name": "n_cells", "min": 10} "name": "n_cells",
], "min": 10
}],
"index": [1, 99, [200, 300]] "index": [1, 99, [200, 300]]
} }
} }
@@ -91,8 +89,12 @@ class EngineTest(unittest.TestCase):
self.assertEqual(data["n_cols"], 91) self.assertEqual(data["n_cols"], 91)
def test_obs_and_var_names(self): def test_obs_and_var_names(self):
self.assertEqual(np.sum(self.data.data.var[self.data.get_schema()["annotations"]["var"]["index"]].isna()), 0) self.assertEqual(
self.assertEqual(np.sum(self.data.data.obs[self.data.get_schema()["annotations"]["obs"]["index"]].isna()), 0) np.sum(self.data.data.var[self.data.get_schema()["annotations"]
["var"]["index"]].isna()), 0)
self.assertEqual(
np.sum(self.data.data.obs[self.data.get_schema()["annotations"]
["obs"]["index"]].isna()), 0)
def test_get_schema(self): def test_get_schema(self):
with open(path.join(path.dirname(__file__), "schema.json")) as fh: with open(path.join(path.dirname(__file__), "schema.json")) as fh:
@@ -110,7 +112,10 @@ class EngineTest(unittest.TestCase):
def test_config(self): def test_config(self):
self.assertEqual( self.assertEqual(
self.data.features["layout"]["obs"], self.data.features["layout"]["obs"],
{"available": True, "interactiveLimit": 50000}, {
"available": True,
"interactiveLimit": 50000
},
) )
def test_layout(self): def test_layout(self):
@@ -129,18 +134,24 @@ class EngineTest(unittest.TestCase):
annotations = decode_fbs.decode_matrix_FBS(fbs) annotations = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(annotations["n_rows"], 2638) self.assertEqual(annotations["n_rows"], 2638)
self.assertEqual(annotations["n_cols"], 5) self.assertEqual(annotations["n_cols"], 5)
obs_index_col_name = self.data.get_schema()["annotations"]["obs"]["index"] obs_index_col_name = self.data.get_schema(
)["annotations"]["obs"]["index"]
self.assertEqual( self.assertEqual(
annotations["col_idx"], annotations["col_idx"],
[obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"], [
obs_index_col_name, "n_genes", "percent_mito", "n_counts",
"louvain"
],
) )
fbs = self.data.annotation_to_fbs_matrix("var") fbs = self.data.annotation_to_fbs_matrix("var")
annotations = decode_fbs.decode_matrix_FBS(fbs) annotations = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(annotations['n_rows'], 1838) self.assertEqual(annotations['n_rows'], 1838)
self.assertEqual(annotations['n_cols'], 2) self.assertEqual(annotations['n_cols'], 2)
var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"] var_index_col_name = self.data.get_schema(
self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells"]) )["annotations"]["var"]["index"]
self.assertEqual(annotations["col_idx"],
[var_index_col_name, "n_cells"])
def test_annotation_fields(self): def test_annotation_fields(self):
fbs = self.data.annotation_to_fbs_matrix("obs", ["n_genes", "n_counts"]) fbs = self.data.annotation_to_fbs_matrix("obs", ["n_genes", "n_counts"])
@@ -148,7 +159,8 @@ class EngineTest(unittest.TestCase):
self.assertEqual(annotations["n_rows"], 2638) self.assertEqual(annotations["n_rows"], 2638)
self.assertEqual(annotations['n_cols'], 2) self.assertEqual(annotations['n_cols'], 2)
var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"] var_index_col_name = self.data.get_schema(
)["annotations"]["var"]["index"]
fbs = self.data.annotation_to_fbs_matrix("var", [var_index_col_name]) fbs = self.data.annotation_to_fbs_matrix("var", [var_index_col_name])
annotations = decode_fbs.decode_matrix_FBS(fbs) annotations = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(annotations['n_rows'], 1838) self.assertEqual(annotations['n_rows'], 1838)
@@ -163,7 +175,8 @@ class EngineTest(unittest.TestCase):
f2 = {"filter": {"obs": {"index": [[500, 1000]]}}} f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"])) result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
self.assertEqual(len(result), 10) self.assertEqual(len(result), 10)
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20)) result = json.loads(
self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
self.assertEqual(len(result), 20) self.assertEqual(len(result), 20)
def test_data_frame(self): def test_data_frame(self):
@@ -178,7 +191,14 @@ class EngineTest(unittest.TestCase):
def test_filtered_data_frame(self): def test_filtered_data_frame(self):
filter_ = { filter_ = {
"filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 100}]}} "filter": {
"var": {
"annotation_value": [{
"name": "n_cells",
"min": 100
}]
}
}
} }
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
data = decode_fbs.decode_matrix_FBS(fbs) data = decode_fbs.decode_matrix_FBS(fbs)
@@ -186,16 +206,29 @@ class EngineTest(unittest.TestCase):
self.assertEqual(data["n_cols"], 1040) self.assertEqual(data["n_cols"], 1040)
filter_ = { filter_ = {
"filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}} "filter": {
"obs": {
"annotation_value": [{
"name": "n_counts",
"min": 3000
}]
}
}
} }
with self.assertRaises(FilterError): with self.assertRaises(FilterError):
self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
def test_data_named_gene(self): def test_data_named_gene(self):
var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"] var_index_col_name = self.data.get_schema(
)["annotations"]["var"]["index"]
filter_ = { filter_ = {
"filter": { "filter": {
"var": {"annotation_value": [{"name": var_index_col_name, "values": ["RER1"]}]} "var": {
"annotation_value": [{
"name": var_index_col_name,
"values": ["RER1"]
}]
}
} }
} }
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
@@ -206,7 +239,12 @@ class EngineTest(unittest.TestCase):
filter_ = { filter_ = {
"filter": { "filter": {
"var": {"annotation_value": [{"name": var_index_col_name, "values": ["SPEN", "TYMP", "PRMT2"]}]} "var": {
"annotation_value": [{
"name": var_index_col_name,
"values": ["SPEN", "TYMP", "PRMT2"]
}]
}
} }
} }
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
server/test/test_scanpy_engine_data_load.py
+4 -1
View File
@@ -10,6 +10,7 @@ class DataLoadEngineTest(unittest.TestCase):
""" """
Test file loading, including deferred loading/update. Test file loading, including deferred loading/update.
""" """
def setUp(self): def setUp(self):
self.data_file = DataLocator("../example-dataset/pbmc3k.h5ad") self.data_file = DataLocator("../example-dataset/pbmc3k.h5ad")
self.data = ScanpyEngine() self.data = ScanpyEngine()
@@ -52,7 +53,8 @@ class DataLoadEngineTest(unittest.TestCase):
f2 = {"filter": {"obs": {"index": [[500, 1000]]}}} f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"])) result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
self.assertEqual(len(result), 10) self.assertEqual(len(result), 10)
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20)) result = json.loads(
self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
self.assertEqual(len(result), 20) self.assertEqual(len(result), 20)
@@ -60,6 +62,7 @@ class DataLocatorEngineTest(unittest.TestCase):
""" """
Test various types of data locators we expect to consume Test various types of data locators we expect to consume
""" """
def setUp(self): def setUp(self):
self.args = { self.args = {
"layout": ["umap"], "layout": ["umap"],
server/test/test_writable_annotation.py
+69 -36
View File
@@ -14,6 +14,7 @@ from server.app.util.data_locator import DataLocator
class WritableAnnotationTest(unittest.TestCase): class WritableAnnotationTest(unittest.TestCase):
def setUp(self): def setUp(self):
self.tmpDir = tempfile.mkdtemp() self.tmpDir = tempfile.mkdtemp()
self.annotations_file = path.join(self.tmpDir, "test_annotations.csv") self.annotations_file = path.join(self.tmpDir, "test_annotations.csv")
@@ -27,7 +28,8 @@ class WritableAnnotationTest(unittest.TestCase):
"annotations_file": self.annotations_file, "annotations_file": self.annotations_file,
"annotations_output_dir": None "annotations_output_dir": None
} }
self.data = ScanpyEngine(DataLocator("../example-dataset/pbmc3k.h5ad"), args) self.data = ScanpyEngine(DataLocator("../example-dataset/pbmc3k.h5ad"),
args)
def tearDown(self): def tearDown(self):
shutil.rmtree(self.tmpDir) shutil.rmtree(self.tmpDir)
@@ -41,7 +43,9 @@ class WritableAnnotationTest(unittest.TestCase):
n_rows = self.data.data.obs.shape[0] n_rows = self.data.data.obs.shape[0]
fbs_bad = self.make_fbs({ fbs_bad = self.make_fbs({
'louvain': pd.Series(['undefined' for l in range(0, n_rows)], dtype='category') 'louvain':
pd.Series(['undefined' for l in range(0, n_rows)],
dtype='category')
}) })
# ensure attempt to change VAR annotation # ensure attempt to change VAR annotation
@@ -56,31 +60,49 @@ class WritableAnnotationTest(unittest.TestCase):
# verify the file is written as expected # verify the file is written as expected
n_rows = self.data.data.obs.shape[0] n_rows = self.data.data.obs.shape[0]
fbs = self.make_fbs({ fbs = self.make_fbs({
'cat_A': pd.Series(['label_A' for l in range(0, n_rows)], dtype='category'), 'cat_A':
'cat_B': pd.Series(['label_B' for l in range(0, n_rows)], dtype='category') pd.Series(['label_A' for l in range(0, n_rows)],
dtype='category'),
'cat_B':
pd.Series(['label_B' for l in range(0, n_rows)],
dtype='category')
}) })
res = self.data.annotation_put_fbs("obs", fbs) res = self.data.annotation_put_fbs("obs", fbs)
self.assertEqual(res, json.dumps({"status": "OK"})) self.assertEqual(res, json.dumps({"status": "OK"}))
self.assertTrue(path.exists(self.annotations_file)) self.assertTrue(path.exists(self.annotations_file))
df = pd.read_csv(self.annotations_file, index_col=0, header=0, comment='#') df = pd.read_csv(self.annotations_file,
index_col=0,
header=0,
comment='#')
self.assertEqual(df.shape, (n_rows, 2)) self.assertEqual(df.shape, (n_rows, 2))
self.assertEqual(set(df.columns), set(['cat_A', 'cat_B'])) self.assertEqual(set(df.columns), set(['cat_A', 'cat_B']))
self.assertTrue(self.data.original_obs_index.equals(df.index)) self.assertTrue(self.data.original_obs_index.equals(df.index))
self.assertTrue(np.all(df['cat_A'] == ['label_A' for l in range(0, n_rows)])) self.assertTrue(
self.assertTrue(np.all(df['cat_B'] == ['label_B' for l in range(0, n_rows)])) np.all(df['cat_A'] == ['label_A' for l in range(0, n_rows)]))
self.assertTrue(
np.all(df['cat_B'] == ['label_B' for l in range(0, n_rows)]))
# verify complete overwrite on second attempt, AND rotation occurs # verify complete overwrite on second attempt, AND rotation occurs
fbs = self.make_fbs({ fbs = self.make_fbs({
'cat_A': pd.Series(['label_A1' for l in range(0, n_rows)], dtype='category'), 'cat_A':
'cat_C': pd.Series(['label_C' for l in range(0, n_rows)], dtype='category') pd.Series(['label_A1' for l in range(0, n_rows)],
dtype='category'),
'cat_C':
pd.Series(['label_C' for l in range(0, n_rows)],
dtype='category')
}) })
res = self.data.annotation_put_fbs("obs", fbs) res = self.data.annotation_put_fbs("obs", fbs)
self.assertEqual(res, json.dumps({"status": "OK"})) self.assertEqual(res, json.dumps({"status": "OK"}))
self.assertTrue(path.exists(self.annotations_file)) self.assertTrue(path.exists(self.annotations_file))
df = pd.read_csv(self.annotations_file, index_col=0, header=0, comment='#') df = pd.read_csv(self.annotations_file,
index_col=0,
header=0,
comment='#')
self.assertEqual(set(df.columns), set(['cat_A', 'cat_C'])) self.assertEqual(set(df.columns), set(['cat_A', 'cat_C']))
self.assertTrue(np.all(df['cat_A'] == ['label_A1' for l in range(0, n_rows)])) self.assertTrue(
self.assertTrue(np.all(df['cat_C'] == ['label_C' for l in range(0, n_rows)])) np.all(df['cat_A'] == ['label_A1' for l in range(0, n_rows)]))
self.assertTrue(
np.all(df['cat_C'] == ['label_C' for l in range(0, n_rows)]))
# rotation # rotation
name, ext = path.splitext(self.annotations_file) name, ext = path.splitext(self.annotations_file)
@@ -93,8 +115,12 @@ class WritableAnnotationTest(unittest.TestCase):
# verify we stop rotation at 9 # verify we stop rotation at 9
n_rows = self.data.data.obs.shape[0] n_rows = self.data.data.obs.shape[0]
fbs = self.make_fbs({ fbs = self.make_fbs({
'cat_A': pd.Series(['label_A' for l in range(0, n_rows)], dtype='category'), 'cat_A':
'cat_B': pd.Series(['label_B' for l in range(0, n_rows)], dtype='category') pd.Series(['label_A' for l in range(0, n_rows)],
dtype='category'),
'cat_B':
pd.Series(['label_B' for l in range(0, n_rows)],
dtype='category')
}) })
for i in range(0, 11): for i in range(0, 11):
res = self.data.annotation_put_fbs("obs", fbs) res = self.data.annotation_put_fbs("obs", fbs)
@@ -112,8 +138,12 @@ class WritableAnnotationTest(unittest.TestCase):
n_rows = self.data.data.obs.shape[0] n_rows = self.data.data.obs.shape[0]
fbs = self.make_fbs({ fbs = self.make_fbs({
'cat_A': pd.Series(['label_A' for l in range(0, n_rows)], dtype='category'), 'cat_A':
'cat_B': pd.Series(['label_B' for l in range(0, n_rows)], dtype='category') pd.Series(['label_A' for l in range(0, n_rows)],
dtype='category'),
'cat_B':
pd.Series(['label_B' for l in range(0, n_rows)],
dtype='category')
}) })
# put # put
@@ -129,27 +159,30 @@ class WritableAnnotationTest(unittest.TestCase):
self.assertEqual(annotations["n_cols"], 7) self.assertEqual(annotations["n_cols"], 7)
self.assertIsNone(annotations["row_idx"]) self.assertIsNone(annotations["row_idx"])
self.assertEqual(annotations["col_idx"], [ self.assertEqual(annotations["col_idx"], [
obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain", "cat_A", "cat_B" obs_index_col_name, "n_genes", "percent_mito", "n_counts",
"louvain", "cat_A", "cat_B"
]) ])
col_idx = annotations["col_idx"] col_idx = annotations["col_idx"]
self.assertEqual(annotations["columns"][col_idx.index('cat_A')], [ self.assertEqual(annotations["columns"][col_idx.index('cat_A')],
'label_A' for l in range(0, n_rows) ['label_A' for l in range(0, n_rows)])
]) self.assertEqual(annotations["columns"][col_idx.index('cat_B')],
self.assertEqual(annotations["columns"][col_idx.index('cat_B')], [ ['label_B' for l in range(0, n_rows)])
'label_B' for l in range(0, n_rows)
])
# verify the schema was updated # verify the schema was updated
all_col_schema = {c["name"]: c for c in schema["annotations"]["obs"]["columns"]} all_col_schema = {
self.assertEqual(all_col_schema["cat_A"], { c["name"]: c for c in schema["annotations"]["obs"]["columns"]
"name": "cat_A", }
"type": "categorical", self.assertEqual(
"categories": ["label_A"], all_col_schema["cat_A"], {
"writable": True "name": "cat_A",
}) "type": "categorical",
self.assertEqual(all_col_schema["cat_B"], { "categories": ["label_A"],
"name": "cat_B", "writable": True
"type": "categorical", })
"categories": ["label_B"], self.assertEqual(
"writable": True all_col_schema["cat_B"], {
}) "name": "cat_B",
"type": "categorical",
"categories": ["label_B"],
"writable": True
})
server/utils/utils.py
+5 -2
View File
@@ -12,12 +12,15 @@ def find_available_port(host, port=5005):
for port_to_try in range(port, port + num_ports_to_try): for port_to_try in range(port, port + num_ports_to_try):
if is_port_available(host, port_to_try): if is_port_available(host, port_to_try):
return port_to_try return port_to_try
raise socket.error(errno.EADDRINUSE, f"No port in range {port} - {port + num_ports_to_try - 1} available.") raise socket.error(
errno.EADDRINUSE,
f"No port in range {port} - {port + num_ports_to_try - 1} available.")
def is_port_available(host, port): def is_port_available(host, port):
is_available = False is_available = False
with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: with contextlib.closing(socket.socket(socket.AF_INET,
socket.SOCK_STREAM)) as s:
try: try:
s.bind((host, port)) s.bind((host, port))
is_available = True is_available = True