fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-29 18:48:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

Apply yapf to python files

Browse Source
This commit is contained in:
Matt Weiden
2019-12-19 15:20:41 -08:00
parent 42a8d45bd7
commit cdca128a01
43 changed files with 1143 additions and 678 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/app/app.py
+5 -1
View File
@@ -12,9 +12,13 @@ from server.app.web import webapp
class Server:
def __init__(self):
self.data = None
self.cache = Cache(config={"CACHE_TYPE": "simple", "CACHE_DEFAULT_TIMEOUT": 860_000})
self.cache = Cache(config={
"CACHE_TYPE": "simple",
"CACHE_DEFAULT_TIMEOUT": 860_000
})
self.app = None
def create_app(self):
server/app/driver/driver.py
+25 -6
View File
@@ -1,5 +1,4 @@
from abc import ABCMeta, abstractmethod
"""
Sort order for methods
1. Initialize
@@ -11,6 +10,7 @@ Sort order for methods
class CXGDriver(metaclass=ABCMeta):
def __init__(self, data_locator=None, args={}):
self.config = self._get_default_config()
self.config.update(args)
@@ -49,14 +49,29 @@ class CXGDriver(metaclass=ABCMeta):
@property
def features(self):
features = {
"cluster": {"available": False},
"layout": {"obs": {"available": False}, "var": {"available": False}},
"diffexp": {"available": True, "interactiveLimit": 50000}
"cluster": {
"available": False
},
"layout": {
"obs": {
"available": False
},
"var": {
"available": False
}
},
"diffexp": {
"available": True,
"interactiveLimit": 50000
}
}
# TODO - Interactive limit should be generated from the actual available methods see GH issue #94
if self.config["layout"]:
# TODO handle "var" when gene layout becomes available
features["layout"]["obs"] = {"available": True, "interactiveLimit": 50000}
features["layout"]["obs"] = {
"available": True,
"interactiveLimit": 50000
}
return features
@abstractmethod
@@ -92,7 +107,11 @@ class CXGDriver(metaclass=ABCMeta):
pass
@abstractmethod
def diffexp_topN(self, obsFilter1, obsFilter2, top_n=None, interactive_limit=None):
def diffexp_topN(self,
obsFilter1,
obsFilter2,
top_n=None,
interactive_limit=None):
"""
Computes the top N differentially expressed variables between two observation sets. If mode
is "TOP_N", then stats for the top N
server/app/rest_api/rest.py
+77 -68
View File
@@ -8,13 +8,9 @@ from flask_restful import Api, Resource
from server import __version__ as cellxgene_version
from anndata import __version__ as anndata_version
from server.app.util.constants import (
Axis,
DiffExpMode,
JSON_NaN_to_num_warning_msg,
CXGUID,
CXG_ANNO_COLLECTION
)
from server.app.util.constants import (Axis, DiffExpMode,
JSON_NaN_to_num_warning_msg, CXGUID,
CXG_ANNO_COLLECTION)
from server.app.util.errors import (
FilterError,
InteractiveError,
@@ -25,15 +21,20 @@ from server.app.util.errors import (
class SchemaAPI(Resource):
def get(self):
cxguid = get_userid(session)
anno_collection = get_anno_collection(session)
return make_response(
jsonify({"schema": current_app.data.get_schema(uid=cxguid, collection=anno_collection)}), HTTPStatus.OK
)
jsonify({
"schema":
current_app.data.get_schema(uid=cxguid,
collection=anno_collection)
}), HTTPStatus.OK)
class ConfigAPI(Resource):
def get(self):
cxguid = get_userid(session)
anno_collection = get_anno_collection(session)
@@ -69,7 +70,8 @@ class ConfigAPI(Resource):
"about-dataset": current_app.config["ABOUT_DATASET"]
},
"parameters": {
**current_app.data.get_config_parameters(uid=cxguid, collection=anno_collection)
**current_app.data.get_config_parameters(uid=cxguid,
collection=anno_collection)
},
"library_versions": {
"cellxgene": cellxgene_version,
@@ -82,42 +84,48 @@ class ConfigAPI(Resource):
class AnnotationsObsAPI(Resource):
def get(self):
fields = request.args.getlist("annotation-name", None)
preferred_mimetype = request.accept_mimetypes.best_match(
["application/octet-stream"]
)
["application/octet-stream"])
cxguid = get_userid(session)
anno_collection = get_anno_collection(session)
try:
if preferred_mimetype == "application/octet-stream":
fbs = current_app.data.annotation_to_fbs_matrix("obs", fields, uid=cxguid, collection=anno_collection)
return make_response(fbs,
HTTPStatus.OK,
{"Content-Type": "application/octet-stream"})
fbs = current_app.data.annotation_to_fbs_matrix(
"obs", fields, uid=cxguid, collection=anno_collection)
return make_response(
fbs, HTTPStatus.OK,
{"Content-Type": "application/octet-stream"})
else:
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
return make_response(
f"Unsupported MIME type '{request.accept_mimetypes}'",
HTTPStatus.NOT_ACCEPTABLE)
except KeyError:
return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST)
return make_response(f"Error bad key in {fields}",
HTTPStatus.BAD_REQUEST)
except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
def put(self):
cxguid = get_userid(session)
anno_collection = request.args.get("annotation-collection-name", default=None)
anno_collection = request.args.get("annotation-collection-name",
default=None)
if anno_collection is not None:
if not is_safe_collection_name(anno_collection):
return make_response(f"Error, bad annotation collection name", HTTPStatus.BAD_REQUEST)
return make_response(f"Error, bad annotation collection name",
HTTPStatus.BAD_REQUEST)
set_anno_collection(session, anno_collection)
else:
anno_collection = get_anno_collection(session)
try:
fbs = request.get_data()
res = current_app.data.annotation_put_fbs("obs", fbs, uid=cxguid, collection=anno_collection)
return make_response(
res, HTTPStatus.OK, {"Content-Type": "application/json"}
)
res = current_app.data.annotation_put_fbs(
"obs", fbs, uid=cxguid, collection=anno_collection)
return make_response(res, HTTPStatus.OK,
{"Content-Type": "application/json"})
except (ValueError, DisabledFeatureError, KeyError) as e:
return make_response(str(e), HTTPStatus.BAD_REQUEST)
except Exception as e:
@@ -125,41 +133,44 @@ class AnnotationsObsAPI(Resource):
class AnnotationsVarAPI(Resource):
def get(self):
fields = request.args.getlist("annotation-name", None)
preferred_mimetype = request.accept_mimetypes.best_match(
["application/octet-stream"]
)
["application/octet-stream"])
try:
if preferred_mimetype == "application/octet-stream":
return make_response(current_app.data.annotation_to_fbs_matrix("var", fields),
HTTPStatus.OK,
{"Content-Type": "application/octet-stream"})
return make_response(
current_app.data.annotation_to_fbs_matrix("var", fields),
HTTPStatus.OK, {"Content-Type": "application/octet-stream"})
else:
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
return make_response(
f"Unsupported MIME type '{request.accept_mimetypes}'",
HTTPStatus.NOT_ACCEPTABLE)
except KeyError:
return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST)
return make_response(f"Error bad key in {fields}",
HTTPStatus.BAD_REQUEST)
except ValueError as e:
return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR)
class DataVarAPI(Resource):
def put(self):
preferred_mimetype = request.accept_mimetypes.best_match(
["application/octet-stream"]
)
["application/octet-stream"])
try:
if preferred_mimetype == "application/octet-stream":
filter_json = request.get_json()
filter = filter_json["filter"] if filter_json else None
return make_response(
current_app.data.data_frame_to_fbs_matrix(
filter, axis=Axis.VAR
),
HTTPStatus.OK,
{"Content-Type": "application/octet-stream"})
current_app.data.data_frame_to_fbs_matrix(filter,
axis=Axis.VAR),
HTTPStatus.OK, {"Content-Type": "application/octet-stream"})
else:
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
return make_response(
f"Unsupported MIME type '{request.accept_mimetypes}'",
HTTPStatus.NOT_ACCEPTABLE)
except FilterError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
except ValueError as e:
@@ -167,43 +178,39 @@ class DataVarAPI(Resource):
class DiffExpObsAPI(Resource):
def post(self):
args = request.get_json()
# confirm mode is present and legal
try:
mode = DiffExpMode(args["mode"])
except KeyError:
return make_response("Error: mode is required", HTTPStatus.BAD_REQUEST)
return make_response("Error: mode is required",
HTTPStatus.BAD_REQUEST)
except ValueError:
return make_response(
f"Error: invalid mode option {args['mode']}", HTTPStatus.BAD_REQUEST
)
return make_response(f"Error: invalid mode option {args['mode']}",
HTTPStatus.BAD_REQUEST)
# Validate filters
if mode == DiffExpMode.VAR_FILTER or "varFilter" in args:
# not NOT_IMPLEMENTED
return make_response(
"mode=varfilter not implemented", HTTPStatus.NOT_IMPLEMENTED
)
return make_response("mode=varfilter not implemented",
HTTPStatus.NOT_IMPLEMENTED)
if mode == DiffExpMode.TOP_N and "count" not in args:
return make_response(
"mode=topN requires a count parameter", HTTPStatus.BAD_REQUEST
)
return make_response("mode=topN requires a count parameter",
HTTPStatus.BAD_REQUEST)
if "set1" not in args:
return make_response("set1 is required.", HTTPStatus.BAD_REQUEST)
if Axis.VAR in args["set1"]["filter"]:
return make_response(
"Var filter not allowed for set1", HTTPStatus.BAD_REQUEST
)
return make_response("Var filter not allowed for set1",
HTTPStatus.BAD_REQUEST)
# set2
if "set2" not in args:
return make_response(
"Set2 as inverse of set1 is not implemented", HTTPStatus.NOT_IMPLEMENTED
)
return make_response("Set2 as inverse of set1 is not implemented",
HTTPStatus.NOT_IMPLEMENTED)
if Axis.VAR in args["set2"]["filter"]:
return make_response(
"Var filter not allowed for set2", HTTPStatus.BAD_REQUEST
)
return make_response("Var filter not allowed for set2",
HTTPStatus.BAD_REQUEST)
set1_filter = args["set1"]["filter"]
set2_filter = args.get("set2", {"filter": {}})["filter"]
@@ -219,13 +226,13 @@ class DiffExpObsAPI(Resource):
count,
current_app.data.features["diffexp"]["interactiveLimit"],
)
return make_response(
diffexp, HTTPStatus.OK, {"Content-Type": "application/json"}
)
return make_response(diffexp, HTTPStatus.OK,
{"Content-Type": "application/json"})
except (ValueError, FilterError) as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
except InteractiveError:
return make_response("Non-interactive request", HTTPStatus.FORBIDDEN)
return make_response("Non-interactive request",
HTTPStatus.FORBIDDEN)
except JSONEncodingValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)
@@ -235,17 +242,19 @@ class DiffExpObsAPI(Resource):
class LayoutObsAPI(Resource):
def get(self):
preferred_mimetype = request.accept_mimetypes.best_match(
["application/octet-stream"]
)
["application/octet-stream"])
try:
if preferred_mimetype == "application/octet-stream":
return make_response(current_app.data.layout_to_fbs_matrix(),
HTTPStatus.OK,
{"Content-Type": "application/octet-stream"})
return make_response(
current_app.data.layout_to_fbs_matrix(), HTTPStatus.OK,
{"Content-Type": "application/octet-stream"})
else:
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
return make_response(
f"Unsupported MIME type '{request.accept_mimetypes}'",
HTTPStatus.NOT_ACCEPTABLE)
except PrepareError as e:
return make_response(e.message, HTTPStatus.INTERNAL_SERVER_ERROR)
except ValueError as e:
server/app/scanpy_engine/diffexp.py
+11 -6
View File
@@ -32,8 +32,8 @@ def _mean_var_n(X):
v = sumsq / (n - 1)
if fp_err_occurred:
mean[np.isfinite(mean) == False] = 0 # noqa: E712
v[np.isfinite(v) == False] = 0 # noqa: E712
mean[np.isfinite(mean) == False] = 0 # noqa: E712
v[np.isfinite(v) == False] = 0 # noqa: E712
return mean, v, n
@@ -76,7 +76,7 @@ def diffexp_ttest(adata, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
# degrees of freedom for Welch's t-test
with np.errstate(divide="ignore", invalid="ignore"):
dof = sum_vn ** 2 / (vnA ** 2 / (nA - 1) + vnB ** 2 / (nB - 1))
dof = sum_vn**2 / (vnA**2 / (nA - 1) + vnB**2 / (nB - 1))
dof[np.isnan(dof)] = 1
# Welch's t-test score calculation
@@ -93,13 +93,15 @@ def diffexp_ttest(adata, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
logfoldchanges = np.log2(np.abs((meanA + 1e-9) / (meanB + 1e-9)))
# find all with lfc > cutoff
lfc_above_cutoff_idx = np.nonzero(np.abs(logfoldchanges) > diffexp_lfc_cutoff)[0]
lfc_above_cutoff_idx = np.nonzero(
np.abs(logfoldchanges) > diffexp_lfc_cutoff)[0]
stats_to_sort = np.abs(tscores)
# derive sort order
if lfc_above_cutoff_idx.shape[0] > top_n:
# partition top N
rel_t_partition = np.argpartition(stats_to_sort[lfc_above_cutoff_idx], -top_n)[-top_n:]
rel_t_partition = np.argpartition(stats_to_sort[lfc_above_cutoff_idx],
-top_n)[-top_n:]
t_partition = lfc_above_cutoff_idx[rel_t_partition]
# sort the top N partition
rel_sort_order = np.argsort(stats_to_sort[t_partition])[::-1]
@@ -117,5 +119,8 @@ def diffexp_ttest(adata, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
pvals_adj_top_n = pvals_adj[sort_order]
# varIndex, logfoldchange, pval, pval_adj
result = [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in range(top_n)]
result = [[
sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i],
pvals_adj_top_n[i]
] for i in range(top_n)]
return result
server/app/scanpy_engine/labels.py
+11 -4
View File
@@ -8,8 +8,13 @@ import pandas as pd
def read_labels(fname):
if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0:
return pd.read_csv(fname, dtype='category', index_col=0, header=0, comment='#')
if fname is not None and os.path.exists(
fname) and os.path.getsize(fname) > 0:
return pd.read_csv(fname,
dtype='category',
index_col=0,
header=0,
comment='#')
else:
return pd.DataFrame()
@@ -47,13 +52,15 @@ def backup(fname, backup_dir, max_backups=9):
fname_base_root, fname_base_ext = os.path.splitext(fname_base)
# don't use ISO standard time format, as it contains characters illegal on some filesytems.
nowish = datetime.now().strftime('%Y-%m-%dT%H-%M-%S')
backup_fname = os.path.join(backup_dir, f"{fname_base_root}-{nowish}{fname_base_ext}")
backup_fname = os.path.join(backup_dir,
f"{fname_base_root}-{nowish}{fname_base_ext}")
if os.path.exists(backup_fname):
os.remove(backup_fname)
os.rename(fname, backup_fname)
# prune the backup_dir to max number of backup files, keeping the most recent backups
backups = list(filter(lambda s: s.startswith(fname_base_root), os.listdir(backup_dir)))
backups = list(
filter(lambda s: s.startswith(fname_base_root), os.listdir(backup_dir)))
excess_count = len(backups) - max_backups
if excess_count > 0:
backups.sort()
server/app/scanpy_engine/matrix_proxy.py
+2 -2
View File
@@ -1,6 +1,4 @@
from server.app.util.matrix_proxy import MatrixProxyView, ArrayProxyView
"""
AnnData/h5py are inconsistent in the API supported by various types of
X matrices. Sometimes you get a fully ndarray, sometims a Scipy sparse
@@ -17,6 +15,7 @@ class ArrayProxyView_anndata_h5py(ArrayProxyView):
override to handle sparse getitem semantics, which differ
from numpy.
"""
def toarray(self):
""" sadly, sparse indexing doesn't drop dimensions like numpy! """
arr = self.m[self._index[0], self._index[1]]
@@ -30,6 +29,7 @@ class MatrixProxy_anndata_h5py(MatrixProxyView):
AnnData sparse array stored in H5AD, or proxies for backed data.
None of these handle indexing very well, so we plop a proxy on top.
"""
@classmethod
def __supports__(cls):
return ("anndata.h5py.h5sparse.SparseDataset",
server/app/scanpy_engine/scanpy_engine.py
+116 -76
View File
@@ -37,6 +37,7 @@ def has_method(o, name):
class ScanpyEngine(CXGDriver):
def __init__(self, data_locator=None, args={}):
super().__init__(data_locator, args)
# lock used to protect label file write ops
@@ -75,7 +76,8 @@ class ScanpyEngine(CXGDriver):
if self.config["annotations"]:
if uid is not None:
params.update({
"annotations-user-data-idhash": self.get_userdata_idhash(uid)
"annotations-user-data-idhash":
self.get_userdata_idhash(uid)
})
if self.config['annotations_file'] is not None:
# user has hard-wired the name of the annotation data collection
@@ -119,7 +121,8 @@ class ScanpyEngine(CXGDriver):
"""
self.original_obs_index = self.data.obs.index
for (ax_name, config_name) in ((Axis.OBS, "obs_names"), (Axis.VAR, "var_names")):
for (ax_name, config_name) in ((Axis.OBS, "obs_names"), (Axis.VAR,
"var_names")):
name = self.config[config_name]
df_axis = getattr(self.data, str(ax_name))
if name is None:
@@ -128,8 +131,7 @@ class ScanpyEngine(CXGDriver):
raise KeyError(
f"Values in {ax_name}.index must be unique. "
"Please prepare data to contain unique index values, or specify an "
"alternative with --{ax_name}-name."
)
"alternative with --{ax_name}-name.")
name = self._create_unique_column_name(df_axis.columns, "name_")
self.config[config_name] = name
# reset index to simple range; alias name to point at the
@@ -141,8 +143,7 @@ class ScanpyEngine(CXGDriver):
if not df_axis[name].is_unique:
raise KeyError(
f"Values in {ax_name}.{name} must be unique. "
"Please prepare data to contain unique values."
)
"Please prepare data to contain unique values.")
df_axis.reset_index(drop=True, inplace=True)
else:
# user specified a non-existent column name
@@ -189,8 +190,7 @@ class ScanpyEngine(CXGDriver):
schema["categories"] = dtype.categories.tolist()
else:
raise TypeError(
f"Annotations of type {dtype} are unsupported by cellxgene."
)
f"Annotations of type {dtype} are unsupported by cellxgene.")
return schema
@requires_data
@@ -211,7 +211,9 @@ class ScanpyEngine(CXGDriver):
"columns": []
}
},
"layout": {"obs": []}
"layout": {
"obs": []
}
}
for ax in Axis:
curr_axis = getattr(self.data, str(ax))
@@ -250,7 +252,8 @@ class ScanpyEngine(CXGDriver):
Used to create safe annotations output file names.
"""
id = (uid + self.data_locator.abspath()).encode()
idhash = base64.b32encode(blake2b(id, digest_size=5).digest()).decode('utf-8')
idhash = base64.b32encode(blake2b(
id, digest_size=5).digest()).decode('utf-8')
return idhash
def get_anno_fname(self, uid=None, collection=None):
@@ -265,7 +268,8 @@ class ScanpyEngine(CXGDriver):
if uid is None or collection is None:
return None
idhash = self.get_userdata_idhash(uid)
return os.path.join(self.get_anno_output_dir(), f"{collection}-{idhash}.csv")
return os.path.join(self.get_anno_output_dir(),
f"{collection}-{idhash}.csv")
def get_anno_output_dir(self):
""" return the current annotation output directory """
@@ -276,7 +280,8 @@ class ScanpyEngine(CXGDriver):
return self.config['annotations_output_dir']
if self.config['annotations_file']:
return os.path.dirname(os.path.abspath(self.config['annotations_file']))
return os.path.dirname(
os.path.abspath(self.config['annotations_file']))
return os.getcwd()
@@ -308,15 +313,14 @@ class ScanpyEngine(CXGDriver):
"https://github.com/theislab/scanpy_usage/blob/master/170505_seurat/info_h5ad.md to "
"learn more about this format. You may be able to convert your file into this format "
"using `cellxgene prepare`, please run `cellxgene prepare --help` for more "
"information."
)
"information.")
except MemoryError:
raise ScanpyFileError("Out of memory - file is too large for available memory.")
raise ScanpyFileError(
"Out of memory - file is too large for available memory.")
except Exception as e:
raise ScanpyFileError(
f"{e} - file not found or is inaccessible. File must be an .h5ad object. "
f"Please check your input and try again."
)
f"Please check your input and try again.")
@requires_data
def _validate_and_initialize(self):
@@ -338,7 +342,8 @@ class ScanpyEngine(CXGDriver):
# heuristic
n_values = self.data.shape[0] * self.data.shape[1]
if (n_values > 1e8 and self.config['backed'] is True) or (n_values > 5e8):
if (n_values > 1e8 and
self.config['backed'] is True) or (n_values > 5e8):
self.config.update({"diffexp_may_be_slow": True})
@requires_data
@@ -352,9 +357,15 @@ class ScanpyEngine(CXGDriver):
# handle default
if layouts is None or len(layouts) == 0:
# load default layouts from the data.
layouts = [key[2:] for key in self.data.obsm_keys() if type(key) == str and key.startswith("X_")]
layouts = [
key[2:]
for key in self.data.obsm_keys()
if type(key) == str and key.startswith("X_")
]
if len(layouts) == 0:
raise PrepareError(f"Unable to find any precomputed layouts within the dataset.")
raise PrepareError(
f"Unable to find any precomputed layouts within the dataset."
)
# remove invalid layouts
valid_layouts = []
@@ -364,7 +375,9 @@ class ScanpyEngine(CXGDriver):
if layout_name not in obsm_keys:
warnings.warn(f"Ignoring unknown layout name: {layout}.")
elif not self._is_valid_layout(self.data.obsm[layout_name]):
warnings.warn(f"Ignoring layout due to malformed shape or data type: {layout}")
warnings.warn(
f"Ignoring layout due to malformed shape or data type: {layout}"
)
else:
valid_layouts.append(layout)
@@ -381,22 +394,22 @@ class ScanpyEngine(CXGDriver):
* contains only finite values
"""
is_valid = type(arr) == np.ndarray and arr.dtype.kind in "fiu"
is_valid = is_valid and arr.shape[0] == self.data.n_obs and arr.shape[1] >= 2
is_valid = is_valid and arr.shape[
0] == self.data.n_obs and arr.shape[1] >= 2
is_valid = is_valid and np.all(np.isfinite(arr))
return is_valid
@requires_data
def _validate_data_types(self):
if sparse.isspmatrix(self.data.X) and not sparse.isspmatrix_csc(self.data.X):
if sparse.isspmatrix(
self.data.X) and not sparse.isspmatrix_csc(self.data.X):
warnings.warn(
f"Scanpy data matrix is sparse, but not a CSC (columnar) matrix. "
f"Performance may be improved by using CSC."
)
f"Performance may be improved by using CSC.")
if self.data.X.dtype != "float32":
warnings.warn(
f"Scanpy data matrix is in {self.data.X.dtype} format not float32. "
f"Precision may be truncated."
)
f"Precision may be truncated.")
for ax in Axis:
curr_axis = getattr(self.data, str(ax))
for ann in curr_axis:
@@ -410,11 +423,11 @@ class ScanpyEngine(CXGDriver):
if datatype in downcast_map:
warnings.warn(
f"Scanpy annotation {ax}:{ann} is in unsupported format: {datatype}. "
f"Data will be downcast to {downcast_map[datatype]}."
)
f"Data will be downcast to {downcast_map[datatype]}.")
if isinstance(datatype, CategoricalDtype):
category_num = len(curr_axis[ann].dtype.categories)
if category_num > 500 and category_num > self.config['max_category_items']:
if category_num > 500 and category_num > self.config[
'max_category_items']:
warnings.warn(
f"{str(ax).title()} annotation '{ann}' has {category_num} categories, this may be "
f"cumbersome or slow to display. We recommend setting the "
@@ -432,31 +445,41 @@ class ScanpyEngine(CXGDriver):
# all lables must have a name, which must be unique and not used in obs column names
if not labels.columns.is_unique:
raise KeyError(f"All column names specified in user annotations must be unique.")
raise KeyError(
f"All column names specified in user annotations must be unique."
)
# the label index must be unique, and must have same values the anndata obs index
if not labels.index.is_unique:
raise KeyError(f"All row index values specified in user annotations must be unique.")
raise KeyError(
f"All row index values specified in user annotations must be unique."
)
if not labels.index.equals(self.original_obs_index):
raise KeyError("Label file row index does not match H5AD file index. "
"Please ensure that column zero (0) in the label file contain the same "
"index values as the H5AD file.")
raise KeyError(
"Label file row index does not match H5AD file index. "
"Please ensure that column zero (0) in the label file contain the same "
"index values as the H5AD file.")
duplicate_columns = list(set(labels.columns) & set(self.data.obs.columns))
duplicate_columns = list(
set(labels.columns) & set(self.data.obs.columns))
if len(duplicate_columns) > 0:
raise KeyError(f"Labels file may not contain column names which overlap "
f"with h5ad obs columns {duplicate_columns}")
raise KeyError(
f"Labels file may not contain column names which overlap "
f"with h5ad obs columns {duplicate_columns}")
# labels must have same count as obs annotations
if labels.shape[0] != self.data.obs.shape[0]:
raise ValueError("Labels file must have same number of rows as h5ad file.")
raise ValueError(
"Labels file must have same number of rows as h5ad file.")
@staticmethod
def _annotation_filter_to_mask(filter, d_axis, count):
mask = np.ones((count,), dtype=bool)
for v in filter:
if d_axis[v["name"]].dtype.name in ["boolean", "category", "object"]:
if d_axis[v["name"]].dtype.name in [
"boolean", "category", "object"
]:
key_idx = np.in1d(getattr(d_axis, v["name"]), v["values"])
mask = np.logical_and(mask, key_idx)
else:
@@ -475,7 +498,7 @@ class ScanpyEngine(CXGDriver):
mask = np.zeros((count,), dtype=bool)
for i in filter:
if type(i) == list:
mask[i[0]: i[1]] = True
mask[i[0]:i[1]] = True
else:
mask[i] = True
return mask
@@ -485,14 +508,13 @@ class ScanpyEngine(CXGDriver):
mask = np.ones((count,), dtype=bool)
if "index" in filter:
mask = np.logical_and(
mask, ScanpyEngine._index_filter_to_mask(filter["index"], count)
)
mask,
ScanpyEngine._index_filter_to_mask(filter["index"], count))
if "annotation_value" in filter:
mask = np.logical_and(
mask,
ScanpyEngine._annotation_filter_to_mask(
filter["annotation_value"], d_axis, count
),
filter["annotation_value"], d_axis, count),
)
return mask
@@ -508,16 +530,18 @@ class ScanpyEngine(CXGDriver):
if filter is not None:
if Axis.OBS in filter:
obs_selector = self._axis_filter_to_mask(
filter["obs"], self.data.obs, self.data.n_obs
)
filter["obs"], self.data.obs, self.data.n_obs)
if Axis.VAR in filter:
var_selector = self._axis_filter_to_mask(
filter["var"], self.data.var, self.data.n_vars
)
filter["var"], self.data.var, self.data.n_vars)
return obs_selector, var_selector
@requires_data
def annotation_to_fbs_matrix(self, axis, fields=None, uid=None, collection=None):
def annotation_to_fbs_matrix(self,
axis,
fields=None,
uid=None,
collection=None):
if axis == Axis.OBS:
if self.config["annotations"]:
try:
@@ -525,8 +549,7 @@ class ScanpyEngine(CXGDriver):
except Exception as e:
raise ScanpyFileError(
f"Error while loading label file: {e}, File must be in the .csv format, please check "
f"your input and try again."
)
f"your input and try again.")
else:
labels = None
@@ -547,7 +570,9 @@ class ScanpyEngine(CXGDriver):
fname = self.get_anno_fname(uid, collection)
if not fname:
raise ScanpyFileError("Writable annotations - unable to determine file name for annotations")
raise ScanpyFileError(
"Writable annotations - unable to determine file name for annotations"
)
if axis != Axis.OBS:
raise ValueError("Only OBS dimension access is supported")
@@ -558,21 +583,27 @@ class ScanpyEngine(CXGDriver):
self._validate_label_data(new_label_df) # paranoia
# if any of the new column labels overlap with our existing labels, raise error
duplicate_columns = list(set(new_label_df.columns) & set(self.data.obs.columns))
duplicate_columns = list(
set(new_label_df.columns) & set(self.data.obs.columns))
if not new_label_df.columns.is_unique or len(duplicate_columns) > 0:
raise KeyError(f"Labels file may not contain column names which overlap "
f"with h5ad obs columns {duplicate_columns}")
raise KeyError(
f"Labels file may not contain column names which overlap "
f"with h5ad obs columns {duplicate_columns}")
# update our internal state and save it. Multi-threading often enabled,
# so treat this as a critical section.
with self.label_lock:
lastmod = self.data_locator.lastmodtime()
lastmodstr = "'unknown'" if lastmod is None else lastmod.isoformat(timespec="seconds")
lastmodstr = "'unknown'" if lastmod is None else lastmod.isoformat(
timespec="seconds")
header = f"# Annotations generated on {datetime.now().isoformat(timespec='seconds')} " \
f"using cellxgene version {cellxgene_version}\n" \
f"# Input data file was {self.data_locator.uri_or_path}, " \
f"which was last modified on {lastmodstr}\n"
write_labels(fname, new_label_df, header, backup_dir=self.get_anno_backup_dir(uid, collection))
write_labels(fname,
new_label_df,
header,
backup_dir=self.get_anno_backup_dir(uid, collection))
return jsonify_scanpy({"status": "OK"})
@@ -591,41 +622,48 @@ class ScanpyEngine(CXGDriver):
if axis != Axis.VAR:
raise ValueError("Only VAR dimension access is supported")
try:
obs_selector, var_selector = self._filter_to_mask(filter, use_slices=False)
obs_selector, var_selector = self._filter_to_mask(filter,
use_slices=False)
except (KeyError, IndexError, TypeError) as e:
raise FilterError(f"Error parsing filter: {e}") from e
if obs_selector is not None:
raise FilterError("filtering on obs unsupported")
# Currently only handles VAR dimension
X = MatrixProxy.create(self.data.X if var_selector is None
else self.data.X[:, var_selector])
return encode_matrix_fbs(X, col_idx=np.nonzero(var_selector)[0], row_idx=None)
X = MatrixProxy.create(
self.data.X if var_selector is None else self.data.X[:,
var_selector])
return encode_matrix_fbs(X,
col_idx=np.nonzero(var_selector)[0],
row_idx=None)
@requires_data
def diffexp_topN(self, obsFilterA, obsFilterB, top_n=None, interactive_limit=None):
def diffexp_topN(self,
obsFilterA,
obsFilterB,
top_n=None,
interactive_limit=None):
if Axis.VAR in obsFilterA or Axis.VAR in obsFilterB:
raise FilterError("Observation filters may not contain vaiable conditions")
raise FilterError(
"Observation filters may not contain vaiable conditions")
try:
obs_mask_A = self._axis_filter_to_mask(
obsFilterA["obs"], self.data.obs, self.data.n_obs
)
obs_mask_B = self._axis_filter_to_mask(
obsFilterB["obs"], self.data.obs, self.data.n_obs
)
obs_mask_A = self._axis_filter_to_mask(obsFilterA["obs"],
self.data.obs,
self.data.n_obs)
obs_mask_B = self._axis_filter_to_mask(obsFilterB["obs"],
self.data.obs,
self.data.n_obs)
except (KeyError, IndexError) as e:
raise FilterError(f"Error parsing filter: {e}") from e
if top_n is None:
top_n = DEFAULT_TOP_N
result = diffexp_ttest(
self.data, obs_mask_A, obs_mask_B, top_n, self.config['diffexp_lfc_cutoff']
)
result = diffexp_ttest(self.data, obs_mask_A, obs_mask_B, top_n,
self.config['diffexp_lfc_cutoff'])
try:
return jsonify_scanpy(result)
except ValueError:
raise JSONEncodingValueError(
"Error encoding differential expression to JSON"
)
"Error encoding differential expression to JSON")
@requires_data
def layout_to_fbs_matrix(self):
@@ -656,7 +694,9 @@ class ScanpyEngine(CXGDriver):
normalized_layout = normalized_layout + translate
normalized_layout = normalized_layout.astype(dtype=np.float32)
layout_data.append(pandas.DataFrame(normalized_layout, columns=[f"{layout}_0", f"{layout}_1"]))
layout_data.append(
pandas.DataFrame(normalized_layout,
columns=[f"{layout}_0", f"{layout}_1"]))
except ValueError as e:
raise PrepareError(
server/app/util/constants.py
+1 -1
View File
@@ -1,10 +1,10 @@
from enum import Enum
DEFAULT_TOP_N = 10
class AugmentedEnum(Enum):
def __hash__(self):
return self.value.__hash__()
server/app/util/data_locator.py
+5 -2
View File
@@ -27,7 +27,8 @@ class DataLocator():
def __init__(self, uri_or_path):
self.uri_or_path = uri_or_path
self.protocol, self.path = DataLocator._get_protocol_and_path(uri_or_path)
self.protocol, self.path = DataLocator._get_protocol_and_path(
uri_or_path)
# work-around for LocalFileSystem not treating file: and None as the same scheme/protocol
self.cname = self.path if self.protocol == 'file' else self.uri_or_path
# will throw RuntimeError if the protocol is unsupported
@@ -82,7 +83,8 @@ class DataLocator():
# if not local, create a tmp file system object to contain the data,
# and clean it up when done.
with self.open() as src, tempfile.NamedTemporaryFile(prefix="cellxgene_", delete=False) as tmp:
with self.open() as src, tempfile.NamedTemporaryFile(
prefix="cellxgene_", delete=False) as tmp:
tmp.write(src.read())
tmp.close()
src.close()
@@ -91,6 +93,7 @@ class DataLocator():
class LocalFilePath():
def __init__(self, tmp_path, delete=False):
self.tmp_path = tmp_path
self.delete = delete
server/app/util/fbs/NetEncoding/Column.py
+19 -5
View File
@@ -4,6 +4,7 @@
import flatbuffers
class Column(object):
__slots__ = ['_tab']
@@ -22,7 +23,8 @@ class Column(object):
def UType(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos)
return self._tab.Get(flatbuffers.number_types.Uint8Flags,
o + self._tab.Pos)
return 0
# Column
@@ -35,7 +37,19 @@ class Column(object):
return obj
return None
def ColumnStart(builder): builder.StartObject(2)
def ColumnAddUType(builder, uType): builder.PrependUint8Slot(0, uType, 0)
def ColumnAddU(builder, u): builder.PrependUOffsetTRelativeSlot(1, flatbuffers.number_types.UOffsetTFlags.py_type(u), 0)
def ColumnEnd(builder): return builder.EndObject()
def ColumnStart(builder):
builder.StartObject(2)
def ColumnAddUType(builder, uType):
builder.PrependUint8Slot(0, uType, 0)
def ColumnAddU(builder, u):
builder.PrependUOffsetTRelativeSlot(
1, flatbuffers.number_types.UOffsetTFlags.py_type(u), 0)
def ColumnEnd(builder):
return builder.EndObject()
server/app/util/fbs/NetEncoding/Float32Array.py
+22 -6
View File
@@ -4,6 +4,7 @@
import flatbuffers
class Float32Array(object):
__slots__ = ['_tab']
@@ -23,14 +24,17 @@ class Float32Array(object):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
a = self._tab.Vector(o)
return self._tab.Get(flatbuffers.number_types.Float32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
return self._tab.Get(
flatbuffers.number_types.Float32Flags,
a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
return 0
# Float32Array
def DataAsNumpy(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Float32Flags, o)
return self._tab.GetVectorAsNumpy(
flatbuffers.number_types.Float32Flags, o)
return 0
# Float32Array
@@ -40,7 +44,19 @@ class Float32Array(object):
return self._tab.VectorLen(o)
return 0
def Float32ArrayStart(builder): builder.StartObject(1)
def Float32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
def Float32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4)
def Float32ArrayEnd(builder): return builder.EndObject()
def Float32ArrayStart(builder):
builder.StartObject(1)
def Float32ArrayAddData(builder, data):
builder.PrependUOffsetTRelativeSlot(
0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
def Float32ArrayStartDataVector(builder, numElems):
return builder.StartVector(4, numElems, 4)
def Float32ArrayEnd(builder):
return builder.EndObject()
server/app/util/fbs/NetEncoding/Float64Array.py
+22 -6
View File
@@ -4,6 +4,7 @@
import flatbuffers
class Float64Array(object):
__slots__ = ['_tab']
@@ -23,14 +24,17 @@ class Float64Array(object):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
a = self._tab.Vector(o)
return self._tab.Get(flatbuffers.number_types.Float64Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 8))
return self._tab.Get(
flatbuffers.number_types.Float64Flags,
a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 8))
return 0
# Float64Array
def DataAsNumpy(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Float64Flags, o)
return self._tab.GetVectorAsNumpy(
flatbuffers.number_types.Float64Flags, o)
return 0
# Float64Array
@@ -40,7 +44,19 @@ class Float64Array(object):
return self._tab.VectorLen(o)
return 0
def Float64ArrayStart(builder): builder.StartObject(1)
def Float64ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
def Float64ArrayStartDataVector(builder, numElems): return builder.StartVector(8, numElems, 8)
def Float64ArrayEnd(builder): return builder.EndObject()
def Float64ArrayStart(builder):
builder.StartObject(1)
def Float64ArrayAddData(builder, data):
builder.PrependUOffsetTRelativeSlot(
0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
def Float64ArrayStartDataVector(builder, numElems):
return builder.StartVector(8, numElems, 8)
def Float64ArrayEnd(builder):
return builder.EndObject()
server/app/util/fbs/NetEncoding/Int32Array.py
+22 -6
View File
@@ -4,6 +4,7 @@
import flatbuffers
class Int32Array(object):
__slots__ = ['_tab']
@@ -23,14 +24,17 @@ class Int32Array(object):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
a = self._tab.Vector(o)
return self._tab.Get(flatbuffers.number_types.Int32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
return self._tab.Get(
flatbuffers.number_types.Int32Flags,
a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
return 0
# Int32Array
def DataAsNumpy(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Int32Flags, o)
return self._tab.GetVectorAsNumpy(
flatbuffers.number_types.Int32Flags, o)
return 0
# Int32Array
@@ -40,7 +44,19 @@ class Int32Array(object):
return self._tab.VectorLen(o)
return 0
def Int32ArrayStart(builder): builder.StartObject(1)
def Int32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
def Int32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4)
def Int32ArrayEnd(builder): return builder.EndObject()
def Int32ArrayStart(builder):
builder.StartObject(1)
def Int32ArrayAddData(builder, data):
builder.PrependUOffsetTRelativeSlot(
0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
def Int32ArrayStartDataVector(builder, numElems):
return builder.StartVector(4, numElems, 4)
def Int32ArrayEnd(builder):
return builder.EndObject()
server/app/util/fbs/NetEncoding/JSONEncodedArray.py
+22 -6
View File
@@ -4,6 +4,7 @@
import flatbuffers
class JSONEncodedArray(object):
__slots__ = ['_tab']
@@ -23,14 +24,17 @@ class JSONEncodedArray(object):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
a = self._tab.Vector(o)
return self._tab.Get(flatbuffers.number_types.Uint8Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 1))
return self._tab.Get(
flatbuffers.number_types.Uint8Flags,
a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 1))
return 0
# JSONEncodedArray
def DataAsNumpy(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Uint8Flags, o)
return self._tab.GetVectorAsNumpy(
flatbuffers.number_types.Uint8Flags, o)
return 0
# JSONEncodedArray
@@ -40,7 +44,19 @@ class JSONEncodedArray(object):
return self._tab.VectorLen(o)
return 0
def JSONEncodedArrayStart(builder): builder.StartObject(1)
def JSONEncodedArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
def JSONEncodedArrayStartDataVector(builder, numElems): return builder.StartVector(1, numElems, 1)
def JSONEncodedArrayEnd(builder): return builder.EndObject()
def JSONEncodedArrayStart(builder):
builder.StartObject(1)
def JSONEncodedArrayAddData(builder, data):
builder.PrependUOffsetTRelativeSlot(
0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
def JSONEncodedArrayStartDataVector(builder, numElems):
return builder.StartVector(1, numElems, 1)
def JSONEncodedArrayEnd(builder):
return builder.EndObject()
server/app/util/fbs/NetEncoding/Matrix.py
+51 -14
View File
@@ -4,6 +4,7 @@
import flatbuffers
class Matrix(object):
__slots__ = ['_tab']
@@ -22,14 +23,16 @@ class Matrix(object):
def NRows(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
return self._tab.Get(flatbuffers.number_types.Uint32Flags, o + self._tab.Pos)
return self._tab.Get(flatbuffers.number_types.Uint32Flags,
o + self._tab.Pos)
return 0
# Matrix
def NCols(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6))
if o != 0:
return self._tab.Get(flatbuffers.number_types.Uint32Flags, o + self._tab.Pos)
return self._tab.Get(flatbuffers.number_types.Uint32Flags,
o + self._tab.Pos)
return 0
# Matrix
@@ -56,7 +59,8 @@ class Matrix(object):
def ColIndexType(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10))
if o != 0:
return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos)
return self._tab.Get(flatbuffers.number_types.Uint8Flags,
o + self._tab.Pos)
return 0
# Matrix
@@ -73,7 +77,8 @@ class Matrix(object):
def RowIndexType(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14))
if o != 0:
return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos)
return self._tab.Get(flatbuffers.number_types.Uint8Flags,
o + self._tab.Pos)
return 0
# Matrix
@@ -86,13 +91,45 @@ class Matrix(object):
return obj
return None
def MatrixStart(builder): builder.StartObject(7)
def MatrixAddNRows(builder, nRows): builder.PrependUint32Slot(0, nRows, 0)
def MatrixAddNCols(builder, nCols): builder.PrependUint32Slot(1, nCols, 0)
def MatrixAddColumns(builder, columns): builder.PrependUOffsetTRelativeSlot(2, flatbuffers.number_types.UOffsetTFlags.py_type(columns), 0)
def MatrixStartColumnsVector(builder, numElems): return builder.StartVector(4, numElems, 4)
def MatrixAddColIndexType(builder, colIndexType): builder.PrependUint8Slot(3, colIndexType, 0)
def MatrixAddColIndex(builder, colIndex): builder.PrependUOffsetTRelativeSlot(4, flatbuffers.number_types.UOffsetTFlags.py_type(colIndex), 0)
def MatrixAddRowIndexType(builder, rowIndexType): builder.PrependUint8Slot(5, rowIndexType, 0)
def MatrixAddRowIndex(builder, rowIndex): builder.PrependUOffsetTRelativeSlot(6, flatbuffers.number_types.UOffsetTFlags.py_type(rowIndex), 0)
def MatrixEnd(builder): return builder.EndObject()
def MatrixStart(builder):
builder.StartObject(7)
def MatrixAddNRows(builder, nRows):
builder.PrependUint32Slot(0, nRows, 0)
def MatrixAddNCols(builder, nCols):
builder.PrependUint32Slot(1, nCols, 0)
def MatrixAddColumns(builder, columns):
builder.PrependUOffsetTRelativeSlot(
2, flatbuffers.number_types.UOffsetTFlags.py_type(columns), 0)
def MatrixStartColumnsVector(builder, numElems):
return builder.StartVector(4, numElems, 4)
def MatrixAddColIndexType(builder, colIndexType):
builder.PrependUint8Slot(3, colIndexType, 0)
def MatrixAddColIndex(builder, colIndex):
builder.PrependUOffsetTRelativeSlot(
4, flatbuffers.number_types.UOffsetTFlags.py_type(colIndex), 0)
def MatrixAddRowIndexType(builder, rowIndexType):
builder.PrependUint8Slot(5, rowIndexType, 0)
def MatrixAddRowIndex(builder, rowIndex):
builder.PrependUOffsetTRelativeSlot(
6, flatbuffers.number_types.UOffsetTFlags.py_type(rowIndex), 0)
def MatrixEnd(builder):
return builder.EndObject()
server/app/util/fbs/NetEncoding/TypedArray.py
+1 -1
View File
@@ -2,6 +2,7 @@
# namespace: NetEncoding
class TypedArray(object):
NONE = 0
Float32Array = 1
@@ -9,4 +10,3 @@ class TypedArray(object):
Uint32Array = 3
Float64Array = 4
JSONEncodedArray = 5
server/app/util/fbs/NetEncoding/Uint32Array.py
+22 -6
View File
@@ -4,6 +4,7 @@
import flatbuffers
class Uint32Array(object):
__slots__ = ['_tab']
@@ -23,14 +24,17 @@ class Uint32Array(object):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
a = self._tab.Vector(o)
return self._tab.Get(flatbuffers.number_types.Uint32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
return self._tab.Get(
flatbuffers.number_types.Uint32Flags,
a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
return 0
# Uint32Array
def DataAsNumpy(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Uint32Flags, o)
return self._tab.GetVectorAsNumpy(
flatbuffers.number_types.Uint32Flags, o)
return 0
# Uint32Array
@@ -40,7 +44,19 @@ class Uint32Array(object):
return self._tab.VectorLen(o)
return 0
def Uint32ArrayStart(builder): builder.StartObject(1)
def Uint32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
def Uint32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4)
def Uint32ArrayEnd(builder): return builder.EndObject()
def Uint32ArrayStart(builder):
builder.StartObject(1)
def Uint32ArrayAddData(builder, data):
builder.PrependUOffsetTRelativeSlot(
0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
def Uint32ArrayStartDataVector(builder, numElems):
return builder.StartVector(4, numElems, 4)
def Uint32ArrayEnd(builder):
return builder.EndObject()
server/app/util/fbs/matrix.py
+25 -15
View File
@@ -25,7 +25,8 @@ def CreateNumpyVector(builder, x):
"""CreateNumpyVector writes a numpy array into the buffer."""
if not isinstance(x, np.ndarray):
raise TypeError(f"non-numpy-ndarray passed to CreateNumpyVector ({type(x)}")
raise TypeError(
f"non-numpy-ndarray passed to CreateNumpyVector ({type(x)}")
if x.dtype.kind not in ['b', 'i', 'u', 'f']:
raise TypeError("numpy-ndarray holds elements of unsupported datatype")
@@ -46,7 +47,8 @@ def CreateNumpyVector(builder, x):
builder.head = int(builder.Head() - len)
# tobytes ensures c_contiguous ordering
builder.Bytes[builder.Head():builder.Head() + len] = x_little_endian.tobytes(order='C')
builder.Bytes[builder.Head():builder.Head() +
len] = x_little_endian.tobytes(order='C')
return builder.EndVector(x.size)
@@ -119,12 +121,10 @@ column_encoding_type_map = {
np.dtype(np.float64).str: (TypedArray.TypedArray.Float32Array, np.float32),
np.dtype(np.float32).str: (TypedArray.TypedArray.Float32Array, np.float32),
np.dtype(np.float16).str: (TypedArray.TypedArray.Float32Array, np.float32),
np.dtype(np.int8).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.int16).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.uint8).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.dtype(np.uint16).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
@@ -141,7 +141,6 @@ index_encoding_type_map = {
# array protocol string: ( array_type, as_type )
np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32)
}
@@ -192,7 +191,8 @@ def encode_matrix_fbs(matrix, row_idx=None, col_idx=None):
columns = []
for cidx in range(n_cols - 1, -1, -1):
# serialize the typed array
col = matrix.iloc[:, cidx] if isinstance(matrix, pd.DataFrame) else matrix[:, cidx]
col = matrix.iloc[:, cidx] if isinstance(
matrix, pd.DataFrame) else matrix[:, cidx]
typed_arr = serialize_typed_array(builder, col, column_encoding)
# serialize the Column union
@@ -218,12 +218,18 @@ def encode_matrix_fbs(matrix, row_idx=None, col_idx=None):
def deserialize_typed_array(tarr):
type_map = {
TypedArray.TypedArray.NONE: None,
TypedArray.TypedArray.Uint32Array: Uint32Array.Uint32Array,
TypedArray.TypedArray.Int32Array: Int32Array.Int32Array,
TypedArray.TypedArray.Float32Array: Float32Array.Float32Array,
TypedArray.TypedArray.Float64Array: Float64Array.Float64Array,
TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray
TypedArray.TypedArray.NONE:
None,
TypedArray.TypedArray.Uint32Array:
Uint32Array.Uint32Array,
TypedArray.TypedArray.Int32Array:
Int32Array.Int32Array,
TypedArray.TypedArray.Float32Array:
Float32Array.Float32Array,
TypedArray.TypedArray.Float64Array:
Float64Array.Float64Array,
TypedArray.TypedArray.JSONEncodedArray:
JSONEncodedArray.JSONEncodedArray
}
(u_type, u) = tarr
if u_type is TypedArray.TypedArray.NONE:
@@ -257,13 +263,16 @@ def decode_matrix_fbs(fbs):
columns_length = matrix.ColumnsLength()
columns_index = deserialize_typed_array((matrix.ColIndexType(), matrix.ColIndex()))
columns_index = deserialize_typed_array(
(matrix.ColIndexType(), matrix.ColIndex()))
if columns_index is None:
columns_index = range(0, n_cols)
# sanity checks
if len(columns_index) != n_cols or columns_length != n_cols:
raise ValueError("FBS column count does not match number of columns in underlying matrix")
raise ValueError(
"FBS column count does not match number of columns in underlying matrix"
)
columns_data = {}
columns_type = {}
@@ -277,7 +286,8 @@ def decode_matrix_fbs(fbs):
if col.UType() is TypedArray.TypedArray.JSONEncodedArray:
columns_type[columns_index[col_idx]] = "category"
df = pd.DataFrame.from_dict(data=columns_data).astype(columns_type, copy=False)
df = pd.DataFrame.from_dict(data=columns_data).astype(columns_type,
copy=False)
# more sanity checks
if not df.columns.is_unique or len(df.columns) != n_cols:
server/app/util/matrix_proxy.py
+42 -29
View File
@@ -2,7 +2,6 @@ import abc
from itertools import zip_longest
from copy import copy
import numpy as np
"""
cellxgene deals with a variety of matrix data types, many of which do
not support a consistent API. This framework allows proxies to be created
@@ -18,6 +17,7 @@ class _ArrayProxyBase(abc.ABC):
Private base class for array or matrix proxy. This summarizes
the interface used by the rest of cellxgene.
"""
@property
@abc.abstractmethod
def dtype(self):
@@ -59,7 +59,6 @@ class MatrixProxy(_ArrayProxyBase):
This class primarily provides the factory method and related support.
All other functionality is delegated to subclasses.
"""
"""
Registry of types to proxy class, where values are:
* None: unsupported
@@ -128,21 +127,25 @@ class MatrixProxyView(MatrixProxy):
"""
2D matrix view to a 2D matrix
"""
def __init__(self, arg1, shape=None, index=(),
transposed=False, copy=False):
def __init__(self,
arg1,
shape=None,
index=(),
transposed=False,
copy=False):
if not copy:
m = arg1
super().__init__(m)
if shape is None:
shape = m.shape
assert(len(shape) == 2)
assert (len(shape) == 2)
index = tuple(
map(lambda s_i:
slice(0, s_i[0], 1) if s_i[1] is None else s_i[1],
zip_longest(shape, index))
)
map(
lambda s_i: slice(0, s_i[0], 1)
if s_i[1] is None else s_i[1], zip_longest(shape, index)))
self._shape = shape
self._index = index
@@ -234,21 +237,29 @@ class MatrixProxyView(MatrixProxy):
NOTE: these follow the numpy rules for dimensionality reduction
when an integer index is specified.
"""
def _getitem_intXint(self, row, col):
return self.m[row, col]
def _getitem_intXslice(self, row, col):
shape = (_slice_length(col, self.m.shape[1]), )
return self.__class__.create_array(self.m, shape=shape, index=(row, col))
shape = (_slice_length(col, self.m.shape[1]),)
return self.__class__.create_array(self.m,
shape=shape,
index=(row, col))
def _getitem_sliceXint(self, row, col):
shape = (_slice_length(row, self.m.shape[0]), )
return self.__class__.create_array(self.m, shape=shape, index=(row, col))
shape = (_slice_length(row, self.m.shape[0]),)
return self.__class__.create_array(self.m,
shape=shape,
index=(row, col))
def _getitem_sliceXslice(self, row, col):
shape = (_slice_length(row, self.m.shape[0]),
_slice_length(col, self.m.shape[1]))
return self.__class__(self.m, shape=shape, index=(row, col), transposed=self.transposed)
return self.__class__(self.m,
shape=shape,
index=(row, col),
transposed=self.transposed)
def toarray(self):
arr = self.m[self._index]
@@ -261,22 +272,24 @@ class ArrayProxyView(_ArrayProxyBase):
"""
1D array view to a 2D matrix
"""
def __init__(self, arg1, shape=None, index=None, copy=False):
super().__init__()
if not copy:
m = arg1
# one index MUST be an integer and the other MUST be a slice
assert(len(index) == 2)
assert(all(isinstance(idx, INT_TYPES + (slice, )) for idx in index))
assert(isinstance(index[0], INT_TYPES) != isinstance(index[1], INT_TYPES))
assert (len(index) == 2)
assert (all(isinstance(idx, INT_TYPES + (slice,)) for idx in index))
assert (isinstance(index[0], INT_TYPES) != isinstance(
index[1], INT_TYPES))
if shape is None:
if isinstance(index[0], INT_TYPES):
shape = (m.shape[0], )
shape = (m.shape[0],)
else:
shape = (m.shape[1], )
assert(len(shape) == 1)
shape = (m.shape[1],)
assert (len(shape) == 1)
self._shape = shape
self.m = m
@@ -336,7 +349,7 @@ class ArrayProxyView(_ArrayProxyBase):
elif isinstance(col, slice):
return self._getitem_intXslice(row, col)
elif isinstance(row, slice):
assert(isinstance(col, INT_TYPES))
assert (isinstance(col, INT_TYPES))
return self._getitem_sliceXint(row, col)
raise IndexError("unsupported column index types")
@@ -345,11 +358,11 @@ class ArrayProxyView(_ArrayProxyBase):
return self.m[row, col]
def _getitem_intXslice(self, row, col):
shape = (_slice_length(col, self.m.shape[1]), )
shape = (_slice_length(col, self.m.shape[1]),)
return self.__class__(self.m, shape=shape, index=(row, col))
def _getitem_sliceXint(self, row, col):
shape = (_slice_length(row, self.m.shape[0]), )
shape = (_slice_length(row, self.m.shape[0]),)
return self.__class__(self.m, shape=shape, index=(row, col))
def toarray(self):
@@ -358,7 +371,7 @@ class ArrayProxyView(_ArrayProxyBase):
def _unpack_index(index, shape):
if not isinstance(index, tuple):
index = (index, )
index = (index,)
if len(shape) < len(index):
raise IndexError("invalid index dimensionality - must be 2")
@@ -366,7 +379,7 @@ def _unpack_index(index, shape):
for shp, idx in zip_longest(shape, index):
idx = slice(None) if idx is None else idx
idx = _slice_defaults(idx, shp) if isinstance(idx, slice) else idx
unpacked += (idx, )
unpacked += (idx,)
return unpacked
@@ -376,7 +389,7 @@ def _slice_slice(outer, outer_len, inner, inner_len):
slice a slice - we take advantage of Python 3 range's support
for indexing.
"""
assert(outer_len >= inner_len)
assert (outer_len >= inner_len)
outer_rng = range(*outer.indices(outer_len))
rng = outer_rng[inner]
start, stop, step = rng.start, rng.stop, rng.step
@@ -387,8 +400,8 @@ def _slice_slice(outer, outer_len, inner, inner_len):
def _range_length(start, stop, step):
""" return length of range """
assert(step != 0)
assert(start is not None and stop is not None and step is not None)
assert (step != 0)
assert (start is not None and stop is not None and step is not None)
if step > 0 and start < stop:
return 1 + (stop - 1 - start) // step
elif step < 0 and start > stop:
@@ -404,7 +417,7 @@ def _slice_length(s, length):
def _slice_defaults(s, length):
""" apply slice defaulting conventions """
assert(length >= 0)
assert (length >= 0)
step = 1 if s.step is None else s.step
server/app/util/utils.py
+5 -1
View File
@@ -7,6 +7,7 @@ from server.app.util.errors import DriverError
class Float32JSONEncoder(json.JSONEncoder):
def __init__(self, *args, **kwargs):
"""
NaN/Infinities are illegal in standard JSON. Python extends JSON with
@@ -35,9 +36,12 @@ def jsonify_scanpy(data):
def requires_data(func):
@wraps(func)
def wrapped_function(self, *args, **kwargs):
if self.data is None:
raise DriverError(f"error data must be loaded before you call {func.__name__}")
raise DriverError(
f"error data must be loaded before you call {func.__name__}")
return func(self, *args, **kwargs)
return wrapped_function
server/app/web/webapp.py
+5 -3
View File
@@ -1,7 +1,6 @@
import os
from flask import Blueprint, render_template, send_from_directory, current_app
bp = Blueprint("webapp", __name__, template_folder="templates")
@@ -9,9 +8,12 @@ bp = Blueprint("webapp", __name__, template_folder="templates")
def index():
dataset_title = current_app.config["DATASET_TITLE"]
scripts = current_app.config["SCRIPTS"]
return render_template("index.html", datasetTitle=dataset_title, SCRIPTS=scripts)
return render_template("index.html",
datasetTitle=dataset_title,
SCRIPTS=scripts)
@bp.route("/favicon.png")
def favicon():
return send_from_directory(os.path.join(bp.root_path, "static/img/"), "favicon.png")
return send_from_directory(os.path.join(bp.root_path, "static/img/"),
"favicon.png")