From cdca128a01f484c28c3264f8d9fa411103220c2c Mon Sep 17 00:00:00 2001 From: Matt Weiden <538456+mweiden@users.noreply.github.com> Date: Wed, 18 Dec 2019 19:18:31 -0800 Subject: [PATCH] Apply yapf to python files --- server/.flake8 | 12 + server/Makefile | 2 +- server/__main__.py | 4 +- server/app/app.py | 6 +- server/app/driver/driver.py | 31 ++- server/app/rest_api/rest.py | 145 ++++++------ server/app/scanpy_engine/diffexp.py | 17 +- server/app/scanpy_engine/labels.py | 15 +- server/app/scanpy_engine/matrix_proxy.py | 4 +- server/app/scanpy_engine/scanpy_engine.py | 192 +++++++++------- server/app/util/constants.py | 2 +- server/app/util/data_locator.py | 7 +- server/app/util/fbs/NetEncoding/Column.py | 24 +- .../app/util/fbs/NetEncoding/Float32Array.py | 28 ++- .../app/util/fbs/NetEncoding/Float64Array.py | 28 ++- server/app/util/fbs/NetEncoding/Int32Array.py | 28 ++- .../util/fbs/NetEncoding/JSONEncodedArray.py | 28 ++- server/app/util/fbs/NetEncoding/Matrix.py | 65 ++++-- server/app/util/fbs/NetEncoding/TypedArray.py | 2 +- .../app/util/fbs/NetEncoding/Uint32Array.py | 28 ++- server/app/util/fbs/matrix.py | 40 ++-- server/app/util/matrix_proxy.py | 71 +++--- server/app/util/utils.py | 6 +- server/app/web/webapp.py | 8 +- server/cli/cli.py | 9 +- server/cli/launch.py | 214 ++++++++++-------- server/cli/prepare.py | 137 +++++++---- server/gui/browser.py | 11 +- server/gui/cellxgene_rc.py | 9 +- server/gui/hook-cefpython3.py | 20 +- server/gui/main.py | 44 +++- server/gui/utils.py | 1 + server/gui/workers.py | 11 +- server/test/decode_fbs.py | 18 +- server/test/test_api.py | 108 ++++++--- server/test/test_fbs.py | 53 +++-- server/test/test_matrix_proxy.py | 138 ++++------- server/test/test_nan_rest.py | 16 +- server/test/test_nan_scanpy_engine.py | 32 +-- server/test/test_scanpy_engine.py | 90 +++++--- server/test/test_scanpy_engine_data_load.py | 5 +- server/test/test_writable_annotation.py | 105 ++++++--- server/utils/utils.py | 7 +- 43 files changed, 1143 insertions(+), 678 deletions(-) create mode 100644 server/.flake8 diff --git a/server/.flake8 b/server/.flake8 new file mode 100644 index 00000000..f84ff7a9 --- /dev/null +++ b/server/.flake8 @@ -0,0 +1,12 @@ +[flake8] +ignore = + # split before binary operator + W504, + # visually indented line with same indent as next logical line, + E129, + # whitespacek before ':' + E203, + # unexpected spaces around keyword / parameter equals + E251 + +max-line-length=120 diff --git a/server/Makefile b/server/Makefile index 67d7b1d8..a8ed5951 100644 --- a/server/Makefile +++ b/server/Makefile @@ -1,6 +1,6 @@ .PHONY: fmt fmt: - yapf -ir . + yapf -ipr . .PHONY: lint lint: diff --git a/server/__main__.py b/server/__main__.py index 1f0b8730..b582e892 100644 --- a/server/__main__.py +++ b/server/__main__.py @@ -5,11 +5,11 @@ if __package__ is None: PKG_PATH = Path(__file__).parent sys.path.insert(0, str(PKG_PATH.parent)) - import server # noqa F401 + import server # noqa F401 __package__ = PKG_PATH.name # Main thing -from .cli.cli import cli # noqa F402 +from .cli.cli import cli # noqa F402 cli() diff --git a/server/app/app.py b/server/app/app.py index 466cc822..02bc082a 100644 --- a/server/app/app.py +++ b/server/app/app.py @@ -12,9 +12,13 @@ from server.app.web import webapp class Server: + def __init__(self): self.data = None - self.cache = Cache(config={"CACHE_TYPE": "simple", "CACHE_DEFAULT_TIMEOUT": 860_000}) + self.cache = Cache(config={ + "CACHE_TYPE": "simple", + "CACHE_DEFAULT_TIMEOUT": 860_000 + }) self.app = None def create_app(self): diff --git a/server/app/driver/driver.py b/server/app/driver/driver.py index 30aa8f06..84eb898e 100644 --- a/server/app/driver/driver.py +++ b/server/app/driver/driver.py @@ -1,5 +1,4 @@ from abc import ABCMeta, abstractmethod - """ Sort order for methods 1. Initialize @@ -11,6 +10,7 @@ Sort order for methods class CXGDriver(metaclass=ABCMeta): + def __init__(self, data_locator=None, args={}): self.config = self._get_default_config() self.config.update(args) @@ -49,14 +49,29 @@ class CXGDriver(metaclass=ABCMeta): @property def features(self): features = { - "cluster": {"available": False}, - "layout": {"obs": {"available": False}, "var": {"available": False}}, - "diffexp": {"available": True, "interactiveLimit": 50000} + "cluster": { + "available": False + }, + "layout": { + "obs": { + "available": False + }, + "var": { + "available": False + } + }, + "diffexp": { + "available": True, + "interactiveLimit": 50000 + } } # TODO - Interactive limit should be generated from the actual available methods see GH issue #94 if self.config["layout"]: # TODO handle "var" when gene layout becomes available - features["layout"]["obs"] = {"available": True, "interactiveLimit": 50000} + features["layout"]["obs"] = { + "available": True, + "interactiveLimit": 50000 + } return features @abstractmethod @@ -92,7 +107,11 @@ class CXGDriver(metaclass=ABCMeta): pass @abstractmethod - def diffexp_topN(self, obsFilter1, obsFilter2, top_n=None, interactive_limit=None): + def diffexp_topN(self, + obsFilter1, + obsFilter2, + top_n=None, + interactive_limit=None): """ Computes the top N differentially expressed variables between two observation sets. If mode is "TOP_N", then stats for the top N diff --git a/server/app/rest_api/rest.py b/server/app/rest_api/rest.py index 780dd38f..0b37df37 100644 --- a/server/app/rest_api/rest.py +++ b/server/app/rest_api/rest.py @@ -8,13 +8,9 @@ from flask_restful import Api, Resource from server import __version__ as cellxgene_version from anndata import __version__ as anndata_version -from server.app.util.constants import ( - Axis, - DiffExpMode, - JSON_NaN_to_num_warning_msg, - CXGUID, - CXG_ANNO_COLLECTION -) +from server.app.util.constants import (Axis, DiffExpMode, + JSON_NaN_to_num_warning_msg, CXGUID, + CXG_ANNO_COLLECTION) from server.app.util.errors import ( FilterError, InteractiveError, @@ -25,15 +21,20 @@ from server.app.util.errors import ( class SchemaAPI(Resource): + def get(self): cxguid = get_userid(session) anno_collection = get_anno_collection(session) return make_response( - jsonify({"schema": current_app.data.get_schema(uid=cxguid, collection=anno_collection)}), HTTPStatus.OK - ) + jsonify({ + "schema": + current_app.data.get_schema(uid=cxguid, + collection=anno_collection) + }), HTTPStatus.OK) class ConfigAPI(Resource): + def get(self): cxguid = get_userid(session) anno_collection = get_anno_collection(session) @@ -69,7 +70,8 @@ class ConfigAPI(Resource): "about-dataset": current_app.config["ABOUT_DATASET"] }, "parameters": { - **current_app.data.get_config_parameters(uid=cxguid, collection=anno_collection) + **current_app.data.get_config_parameters(uid=cxguid, + collection=anno_collection) }, "library_versions": { "cellxgene": cellxgene_version, @@ -82,42 +84,48 @@ class ConfigAPI(Resource): class AnnotationsObsAPI(Resource): + def get(self): fields = request.args.getlist("annotation-name", None) preferred_mimetype = request.accept_mimetypes.best_match( - ["application/octet-stream"] - ) + ["application/octet-stream"]) cxguid = get_userid(session) anno_collection = get_anno_collection(session) try: if preferred_mimetype == "application/octet-stream": - fbs = current_app.data.annotation_to_fbs_matrix("obs", fields, uid=cxguid, collection=anno_collection) - return make_response(fbs, - HTTPStatus.OK, - {"Content-Type": "application/octet-stream"}) + fbs = current_app.data.annotation_to_fbs_matrix( + "obs", fields, uid=cxguid, collection=anno_collection) + return make_response( + fbs, HTTPStatus.OK, + {"Content-Type": "application/octet-stream"}) else: - return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) + return make_response( + f"Unsupported MIME type '{request.accept_mimetypes}'", + HTTPStatus.NOT_ACCEPTABLE) except KeyError: - return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST) + return make_response(f"Error bad key in {fields}", + HTTPStatus.BAD_REQUEST) except ValueError as e: return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR) def put(self): cxguid = get_userid(session) - anno_collection = request.args.get("annotation-collection-name", default=None) + anno_collection = request.args.get("annotation-collection-name", + default=None) if anno_collection is not None: if not is_safe_collection_name(anno_collection): - return make_response(f"Error, bad annotation collection name", HTTPStatus.BAD_REQUEST) + return make_response(f"Error, bad annotation collection name", + HTTPStatus.BAD_REQUEST) set_anno_collection(session, anno_collection) else: anno_collection = get_anno_collection(session) try: fbs = request.get_data() - res = current_app.data.annotation_put_fbs("obs", fbs, uid=cxguid, collection=anno_collection) - return make_response( - res, HTTPStatus.OK, {"Content-Type": "application/json"} - ) + res = current_app.data.annotation_put_fbs( + "obs", fbs, uid=cxguid, collection=anno_collection) + return make_response(res, HTTPStatus.OK, + {"Content-Type": "application/json"}) except (ValueError, DisabledFeatureError, KeyError) as e: return make_response(str(e), HTTPStatus.BAD_REQUEST) except Exception as e: @@ -125,41 +133,44 @@ class AnnotationsObsAPI(Resource): class AnnotationsVarAPI(Resource): + def get(self): fields = request.args.getlist("annotation-name", None) preferred_mimetype = request.accept_mimetypes.best_match( - ["application/octet-stream"] - ) + ["application/octet-stream"]) try: if preferred_mimetype == "application/octet-stream": - return make_response(current_app.data.annotation_to_fbs_matrix("var", fields), - HTTPStatus.OK, - {"Content-Type": "application/octet-stream"}) + return make_response( + current_app.data.annotation_to_fbs_matrix("var", fields), + HTTPStatus.OK, {"Content-Type": "application/octet-stream"}) else: - return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) + return make_response( + f"Unsupported MIME type '{request.accept_mimetypes}'", + HTTPStatus.NOT_ACCEPTABLE) except KeyError: - return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST) + return make_response(f"Error bad key in {fields}", + HTTPStatus.BAD_REQUEST) except ValueError as e: return make_response(str(e), HTTPStatus.INTERNAL_SERVER_ERROR) class DataVarAPI(Resource): + def put(self): preferred_mimetype = request.accept_mimetypes.best_match( - ["application/octet-stream"] - ) + ["application/octet-stream"]) try: if preferred_mimetype == "application/octet-stream": filter_json = request.get_json() filter = filter_json["filter"] if filter_json else None return make_response( - current_app.data.data_frame_to_fbs_matrix( - filter, axis=Axis.VAR - ), - HTTPStatus.OK, - {"Content-Type": "application/octet-stream"}) + current_app.data.data_frame_to_fbs_matrix(filter, + axis=Axis.VAR), + HTTPStatus.OK, {"Content-Type": "application/octet-stream"}) else: - return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) + return make_response( + f"Unsupported MIME type '{request.accept_mimetypes}'", + HTTPStatus.NOT_ACCEPTABLE) except FilterError as e: return make_response(e.message, HTTPStatus.BAD_REQUEST) except ValueError as e: @@ -167,43 +178,39 @@ class DataVarAPI(Resource): class DiffExpObsAPI(Resource): + def post(self): args = request.get_json() # confirm mode is present and legal try: mode = DiffExpMode(args["mode"]) except KeyError: - return make_response("Error: mode is required", HTTPStatus.BAD_REQUEST) + return make_response("Error: mode is required", + HTTPStatus.BAD_REQUEST) except ValueError: - return make_response( - f"Error: invalid mode option {args['mode']}", HTTPStatus.BAD_REQUEST - ) + return make_response(f"Error: invalid mode option {args['mode']}", + HTTPStatus.BAD_REQUEST) # Validate filters if mode == DiffExpMode.VAR_FILTER or "varFilter" in args: # not NOT_IMPLEMENTED - return make_response( - "mode=varfilter not implemented", HTTPStatus.NOT_IMPLEMENTED - ) + return make_response("mode=varfilter not implemented", + HTTPStatus.NOT_IMPLEMENTED) if mode == DiffExpMode.TOP_N and "count" not in args: - return make_response( - "mode=topN requires a count parameter", HTTPStatus.BAD_REQUEST - ) + return make_response("mode=topN requires a count parameter", + HTTPStatus.BAD_REQUEST) if "set1" not in args: return make_response("set1 is required.", HTTPStatus.BAD_REQUEST) if Axis.VAR in args["set1"]["filter"]: - return make_response( - "Var filter not allowed for set1", HTTPStatus.BAD_REQUEST - ) + return make_response("Var filter not allowed for set1", + HTTPStatus.BAD_REQUEST) # set2 if "set2" not in args: - return make_response( - "Set2 as inverse of set1 is not implemented", HTTPStatus.NOT_IMPLEMENTED - ) + return make_response("Set2 as inverse of set1 is not implemented", + HTTPStatus.NOT_IMPLEMENTED) if Axis.VAR in args["set2"]["filter"]: - return make_response( - "Var filter not allowed for set2", HTTPStatus.BAD_REQUEST - ) + return make_response("Var filter not allowed for set2", + HTTPStatus.BAD_REQUEST) set1_filter = args["set1"]["filter"] set2_filter = args.get("set2", {"filter": {}})["filter"] @@ -219,13 +226,13 @@ class DiffExpObsAPI(Resource): count, current_app.data.features["diffexp"]["interactiveLimit"], ) - return make_response( - diffexp, HTTPStatus.OK, {"Content-Type": "application/json"} - ) + return make_response(diffexp, HTTPStatus.OK, + {"Content-Type": "application/json"}) except (ValueError, FilterError) as e: return make_response(e.message, HTTPStatus.BAD_REQUEST) except InteractiveError: - return make_response("Non-interactive request", HTTPStatus.FORBIDDEN) + return make_response("Non-interactive request", + HTTPStatus.FORBIDDEN) except JSONEncodingValueError as e: # JSON encoding failure, usually due to bad data warnings.warn(JSON_NaN_to_num_warning_msg) @@ -235,17 +242,19 @@ class DiffExpObsAPI(Resource): class LayoutObsAPI(Resource): + def get(self): preferred_mimetype = request.accept_mimetypes.best_match( - ["application/octet-stream"] - ) + ["application/octet-stream"]) try: if preferred_mimetype == "application/octet-stream": - return make_response(current_app.data.layout_to_fbs_matrix(), - HTTPStatus.OK, - {"Content-Type": "application/octet-stream"}) + return make_response( + current_app.data.layout_to_fbs_matrix(), HTTPStatus.OK, + {"Content-Type": "application/octet-stream"}) else: - return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) + return make_response( + f"Unsupported MIME type '{request.accept_mimetypes}'", + HTTPStatus.NOT_ACCEPTABLE) except PrepareError as e: return make_response(e.message, HTTPStatus.INTERNAL_SERVER_ERROR) except ValueError as e: diff --git a/server/app/scanpy_engine/diffexp.py b/server/app/scanpy_engine/diffexp.py index d991f1c6..6defdde7 100644 --- a/server/app/scanpy_engine/diffexp.py +++ b/server/app/scanpy_engine/diffexp.py @@ -32,8 +32,8 @@ def _mean_var_n(X): v = sumsq / (n - 1) if fp_err_occurred: - mean[np.isfinite(mean) == False] = 0 # noqa: E712 - v[np.isfinite(v) == False] = 0 # noqa: E712 + mean[np.isfinite(mean) == False] = 0 # noqa: E712 + v[np.isfinite(v) == False] = 0 # noqa: E712 return mean, v, n @@ -76,7 +76,7 @@ def diffexp_ttest(adata, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01): # degrees of freedom for Welch's t-test with np.errstate(divide="ignore", invalid="ignore"): - dof = sum_vn ** 2 / (vnA ** 2 / (nA - 1) + vnB ** 2 / (nB - 1)) + dof = sum_vn**2 / (vnA**2 / (nA - 1) + vnB**2 / (nB - 1)) dof[np.isnan(dof)] = 1 # Welch's t-test score calculation @@ -93,13 +93,15 @@ def diffexp_ttest(adata, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01): logfoldchanges = np.log2(np.abs((meanA + 1e-9) / (meanB + 1e-9))) # find all with lfc > cutoff - lfc_above_cutoff_idx = np.nonzero(np.abs(logfoldchanges) > diffexp_lfc_cutoff)[0] + lfc_above_cutoff_idx = np.nonzero( + np.abs(logfoldchanges) > diffexp_lfc_cutoff)[0] stats_to_sort = np.abs(tscores) # derive sort order if lfc_above_cutoff_idx.shape[0] > top_n: # partition top N - rel_t_partition = np.argpartition(stats_to_sort[lfc_above_cutoff_idx], -top_n)[-top_n:] + rel_t_partition = np.argpartition(stats_to_sort[lfc_above_cutoff_idx], + -top_n)[-top_n:] t_partition = lfc_above_cutoff_idx[rel_t_partition] # sort the top N partition rel_sort_order = np.argsort(stats_to_sort[t_partition])[::-1] @@ -117,5 +119,8 @@ def diffexp_ttest(adata, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01): pvals_adj_top_n = pvals_adj[sort_order] # varIndex, logfoldchange, pval, pval_adj - result = [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in range(top_n)] + result = [[ + sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], + pvals_adj_top_n[i] + ] for i in range(top_n)] return result diff --git a/server/app/scanpy_engine/labels.py b/server/app/scanpy_engine/labels.py index 3a5f7d08..e2cb4ebe 100644 --- a/server/app/scanpy_engine/labels.py +++ b/server/app/scanpy_engine/labels.py @@ -8,8 +8,13 @@ import pandas as pd def read_labels(fname): - if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0: - return pd.read_csv(fname, dtype='category', index_col=0, header=0, comment='#') + if fname is not None and os.path.exists( + fname) and os.path.getsize(fname) > 0: + return pd.read_csv(fname, + dtype='category', + index_col=0, + header=0, + comment='#') else: return pd.DataFrame() @@ -47,13 +52,15 @@ def backup(fname, backup_dir, max_backups=9): fname_base_root, fname_base_ext = os.path.splitext(fname_base) # don't use ISO standard time format, as it contains characters illegal on some filesytems. nowish = datetime.now().strftime('%Y-%m-%dT%H-%M-%S') - backup_fname = os.path.join(backup_dir, f"{fname_base_root}-{nowish}{fname_base_ext}") + backup_fname = os.path.join(backup_dir, + f"{fname_base_root}-{nowish}{fname_base_ext}") if os.path.exists(backup_fname): os.remove(backup_fname) os.rename(fname, backup_fname) # prune the backup_dir to max number of backup files, keeping the most recent backups - backups = list(filter(lambda s: s.startswith(fname_base_root), os.listdir(backup_dir))) + backups = list( + filter(lambda s: s.startswith(fname_base_root), os.listdir(backup_dir))) excess_count = len(backups) - max_backups if excess_count > 0: backups.sort() diff --git a/server/app/scanpy_engine/matrix_proxy.py b/server/app/scanpy_engine/matrix_proxy.py index 137c676b..80800cf1 100644 --- a/server/app/scanpy_engine/matrix_proxy.py +++ b/server/app/scanpy_engine/matrix_proxy.py @@ -1,6 +1,4 @@ - from server.app.util.matrix_proxy import MatrixProxyView, ArrayProxyView - """ AnnData/h5py are inconsistent in the API supported by various types of X matrices. Sometimes you get a fully ndarray, sometims a Scipy sparse @@ -17,6 +15,7 @@ class ArrayProxyView_anndata_h5py(ArrayProxyView): override to handle sparse getitem semantics, which differ from numpy. """ + def toarray(self): """ sadly, sparse indexing doesn't drop dimensions like numpy! """ arr = self.m[self._index[0], self._index[1]] @@ -30,6 +29,7 @@ class MatrixProxy_anndata_h5py(MatrixProxyView): AnnData sparse array stored in H5AD, or proxies for backed data. None of these handle indexing very well, so we plop a proxy on top. """ + @classmethod def __supports__(cls): return ("anndata.h5py.h5sparse.SparseDataset", diff --git a/server/app/scanpy_engine/scanpy_engine.py b/server/app/scanpy_engine/scanpy_engine.py index a1a2b32a..226d89b3 100644 --- a/server/app/scanpy_engine/scanpy_engine.py +++ b/server/app/scanpy_engine/scanpy_engine.py @@ -37,6 +37,7 @@ def has_method(o, name): class ScanpyEngine(CXGDriver): + def __init__(self, data_locator=None, args={}): super().__init__(data_locator, args) # lock used to protect label file write ops @@ -75,7 +76,8 @@ class ScanpyEngine(CXGDriver): if self.config["annotations"]: if uid is not None: params.update({ - "annotations-user-data-idhash": self.get_userdata_idhash(uid) + "annotations-user-data-idhash": + self.get_userdata_idhash(uid) }) if self.config['annotations_file'] is not None: # user has hard-wired the name of the annotation data collection @@ -119,7 +121,8 @@ class ScanpyEngine(CXGDriver): """ self.original_obs_index = self.data.obs.index - for (ax_name, config_name) in ((Axis.OBS, "obs_names"), (Axis.VAR, "var_names")): + for (ax_name, config_name) in ((Axis.OBS, "obs_names"), (Axis.VAR, + "var_names")): name = self.config[config_name] df_axis = getattr(self.data, str(ax_name)) if name is None: @@ -128,8 +131,7 @@ class ScanpyEngine(CXGDriver): raise KeyError( f"Values in {ax_name}.index must be unique. " "Please prepare data to contain unique index values, or specify an " - "alternative with --{ax_name}-name." - ) + "alternative with --{ax_name}-name.") name = self._create_unique_column_name(df_axis.columns, "name_") self.config[config_name] = name # reset index to simple range; alias name to point at the @@ -141,8 +143,7 @@ class ScanpyEngine(CXGDriver): if not df_axis[name].is_unique: raise KeyError( f"Values in {ax_name}.{name} must be unique. " - "Please prepare data to contain unique values." - ) + "Please prepare data to contain unique values.") df_axis.reset_index(drop=True, inplace=True) else: # user specified a non-existent column name @@ -189,8 +190,7 @@ class ScanpyEngine(CXGDriver): schema["categories"] = dtype.categories.tolist() else: raise TypeError( - f"Annotations of type {dtype} are unsupported by cellxgene." - ) + f"Annotations of type {dtype} are unsupported by cellxgene.") return schema @requires_data @@ -211,7 +211,9 @@ class ScanpyEngine(CXGDriver): "columns": [] } }, - "layout": {"obs": []} + "layout": { + "obs": [] + } } for ax in Axis: curr_axis = getattr(self.data, str(ax)) @@ -250,7 +252,8 @@ class ScanpyEngine(CXGDriver): Used to create safe annotations output file names. """ id = (uid + self.data_locator.abspath()).encode() - idhash = base64.b32encode(blake2b(id, digest_size=5).digest()).decode('utf-8') + idhash = base64.b32encode(blake2b( + id, digest_size=5).digest()).decode('utf-8') return idhash def get_anno_fname(self, uid=None, collection=None): @@ -265,7 +268,8 @@ class ScanpyEngine(CXGDriver): if uid is None or collection is None: return None idhash = self.get_userdata_idhash(uid) - return os.path.join(self.get_anno_output_dir(), f"{collection}-{idhash}.csv") + return os.path.join(self.get_anno_output_dir(), + f"{collection}-{idhash}.csv") def get_anno_output_dir(self): """ return the current annotation output directory """ @@ -276,7 +280,8 @@ class ScanpyEngine(CXGDriver): return self.config['annotations_output_dir'] if self.config['annotations_file']: - return os.path.dirname(os.path.abspath(self.config['annotations_file'])) + return os.path.dirname( + os.path.abspath(self.config['annotations_file'])) return os.getcwd() @@ -308,15 +313,14 @@ class ScanpyEngine(CXGDriver): "https://github.com/theislab/scanpy_usage/blob/master/170505_seurat/info_h5ad.md to " "learn more about this format. You may be able to convert your file into this format " "using `cellxgene prepare`, please run `cellxgene prepare --help` for more " - "information." - ) + "information.") except MemoryError: - raise ScanpyFileError("Out of memory - file is too large for available memory.") + raise ScanpyFileError( + "Out of memory - file is too large for available memory.") except Exception as e: raise ScanpyFileError( f"{e} - file not found or is inaccessible. File must be an .h5ad object. " - f"Please check your input and try again." - ) + f"Please check your input and try again.") @requires_data def _validate_and_initialize(self): @@ -338,7 +342,8 @@ class ScanpyEngine(CXGDriver): # heuristic n_values = self.data.shape[0] * self.data.shape[1] - if (n_values > 1e8 and self.config['backed'] is True) or (n_values > 5e8): + if (n_values > 1e8 and + self.config['backed'] is True) or (n_values > 5e8): self.config.update({"diffexp_may_be_slow": True}) @requires_data @@ -352,9 +357,15 @@ class ScanpyEngine(CXGDriver): # handle default if layouts is None or len(layouts) == 0: # load default layouts from the data. - layouts = [key[2:] for key in self.data.obsm_keys() if type(key) == str and key.startswith("X_")] + layouts = [ + key[2:] + for key in self.data.obsm_keys() + if type(key) == str and key.startswith("X_") + ] if len(layouts) == 0: - raise PrepareError(f"Unable to find any precomputed layouts within the dataset.") + raise PrepareError( + f"Unable to find any precomputed layouts within the dataset." + ) # remove invalid layouts valid_layouts = [] @@ -364,7 +375,9 @@ class ScanpyEngine(CXGDriver): if layout_name not in obsm_keys: warnings.warn(f"Ignoring unknown layout name: {layout}.") elif not self._is_valid_layout(self.data.obsm[layout_name]): - warnings.warn(f"Ignoring layout due to malformed shape or data type: {layout}") + warnings.warn( + f"Ignoring layout due to malformed shape or data type: {layout}" + ) else: valid_layouts.append(layout) @@ -381,22 +394,22 @@ class ScanpyEngine(CXGDriver): * contains only finite values """ is_valid = type(arr) == np.ndarray and arr.dtype.kind in "fiu" - is_valid = is_valid and arr.shape[0] == self.data.n_obs and arr.shape[1] >= 2 + is_valid = is_valid and arr.shape[ + 0] == self.data.n_obs and arr.shape[1] >= 2 is_valid = is_valid and np.all(np.isfinite(arr)) return is_valid @requires_data def _validate_data_types(self): - if sparse.isspmatrix(self.data.X) and not sparse.isspmatrix_csc(self.data.X): + if sparse.isspmatrix( + self.data.X) and not sparse.isspmatrix_csc(self.data.X): warnings.warn( f"Scanpy data matrix is sparse, but not a CSC (columnar) matrix. " - f"Performance may be improved by using CSC." - ) + f"Performance may be improved by using CSC.") if self.data.X.dtype != "float32": warnings.warn( f"Scanpy data matrix is in {self.data.X.dtype} format not float32. " - f"Precision may be truncated." - ) + f"Precision may be truncated.") for ax in Axis: curr_axis = getattr(self.data, str(ax)) for ann in curr_axis: @@ -410,11 +423,11 @@ class ScanpyEngine(CXGDriver): if datatype in downcast_map: warnings.warn( f"Scanpy annotation {ax}:{ann} is in unsupported format: {datatype}. " - f"Data will be downcast to {downcast_map[datatype]}." - ) + f"Data will be downcast to {downcast_map[datatype]}.") if isinstance(datatype, CategoricalDtype): category_num = len(curr_axis[ann].dtype.categories) - if category_num > 500 and category_num > self.config['max_category_items']: + if category_num > 500 and category_num > self.config[ + 'max_category_items']: warnings.warn( f"{str(ax).title()} annotation '{ann}' has {category_num} categories, this may be " f"cumbersome or slow to display. We recommend setting the " @@ -432,31 +445,41 @@ class ScanpyEngine(CXGDriver): # all lables must have a name, which must be unique and not used in obs column names if not labels.columns.is_unique: - raise KeyError(f"All column names specified in user annotations must be unique.") + raise KeyError( + f"All column names specified in user annotations must be unique." + ) # the label index must be unique, and must have same values the anndata obs index if not labels.index.is_unique: - raise KeyError(f"All row index values specified in user annotations must be unique.") + raise KeyError( + f"All row index values specified in user annotations must be unique." + ) if not labels.index.equals(self.original_obs_index): - raise KeyError("Label file row index does not match H5AD file index. " - "Please ensure that column zero (0) in the label file contain the same " - "index values as the H5AD file.") + raise KeyError( + "Label file row index does not match H5AD file index. " + "Please ensure that column zero (0) in the label file contain the same " + "index values as the H5AD file.") - duplicate_columns = list(set(labels.columns) & set(self.data.obs.columns)) + duplicate_columns = list( + set(labels.columns) & set(self.data.obs.columns)) if len(duplicate_columns) > 0: - raise KeyError(f"Labels file may not contain column names which overlap " - f"with h5ad obs columns {duplicate_columns}") + raise KeyError( + f"Labels file may not contain column names which overlap " + f"with h5ad obs columns {duplicate_columns}") # labels must have same count as obs annotations if labels.shape[0] != self.data.obs.shape[0]: - raise ValueError("Labels file must have same number of rows as h5ad file.") + raise ValueError( + "Labels file must have same number of rows as h5ad file.") @staticmethod def _annotation_filter_to_mask(filter, d_axis, count): mask = np.ones((count,), dtype=bool) for v in filter: - if d_axis[v["name"]].dtype.name in ["boolean", "category", "object"]: + if d_axis[v["name"]].dtype.name in [ + "boolean", "category", "object" + ]: key_idx = np.in1d(getattr(d_axis, v["name"]), v["values"]) mask = np.logical_and(mask, key_idx) else: @@ -475,7 +498,7 @@ class ScanpyEngine(CXGDriver): mask = np.zeros((count,), dtype=bool) for i in filter: if type(i) == list: - mask[i[0]: i[1]] = True + mask[i[0]:i[1]] = True else: mask[i] = True return mask @@ -485,14 +508,13 @@ class ScanpyEngine(CXGDriver): mask = np.ones((count,), dtype=bool) if "index" in filter: mask = np.logical_and( - mask, ScanpyEngine._index_filter_to_mask(filter["index"], count) - ) + mask, + ScanpyEngine._index_filter_to_mask(filter["index"], count)) if "annotation_value" in filter: mask = np.logical_and( mask, ScanpyEngine._annotation_filter_to_mask( - filter["annotation_value"], d_axis, count - ), + filter["annotation_value"], d_axis, count), ) return mask @@ -508,16 +530,18 @@ class ScanpyEngine(CXGDriver): if filter is not None: if Axis.OBS in filter: obs_selector = self._axis_filter_to_mask( - filter["obs"], self.data.obs, self.data.n_obs - ) + filter["obs"], self.data.obs, self.data.n_obs) if Axis.VAR in filter: var_selector = self._axis_filter_to_mask( - filter["var"], self.data.var, self.data.n_vars - ) + filter["var"], self.data.var, self.data.n_vars) return obs_selector, var_selector @requires_data - def annotation_to_fbs_matrix(self, axis, fields=None, uid=None, collection=None): + def annotation_to_fbs_matrix(self, + axis, + fields=None, + uid=None, + collection=None): if axis == Axis.OBS: if self.config["annotations"]: try: @@ -525,8 +549,7 @@ class ScanpyEngine(CXGDriver): except Exception as e: raise ScanpyFileError( f"Error while loading label file: {e}, File must be in the .csv format, please check " - f"your input and try again." - ) + f"your input and try again.") else: labels = None @@ -547,7 +570,9 @@ class ScanpyEngine(CXGDriver): fname = self.get_anno_fname(uid, collection) if not fname: - raise ScanpyFileError("Writable annotations - unable to determine file name for annotations") + raise ScanpyFileError( + "Writable annotations - unable to determine file name for annotations" + ) if axis != Axis.OBS: raise ValueError("Only OBS dimension access is supported") @@ -558,21 +583,27 @@ class ScanpyEngine(CXGDriver): self._validate_label_data(new_label_df) # paranoia # if any of the new column labels overlap with our existing labels, raise error - duplicate_columns = list(set(new_label_df.columns) & set(self.data.obs.columns)) + duplicate_columns = list( + set(new_label_df.columns) & set(self.data.obs.columns)) if not new_label_df.columns.is_unique or len(duplicate_columns) > 0: - raise KeyError(f"Labels file may not contain column names which overlap " - f"with h5ad obs columns {duplicate_columns}") + raise KeyError( + f"Labels file may not contain column names which overlap " + f"with h5ad obs columns {duplicate_columns}") # update our internal state and save it. Multi-threading often enabled, # so treat this as a critical section. with self.label_lock: lastmod = self.data_locator.lastmodtime() - lastmodstr = "'unknown'" if lastmod is None else lastmod.isoformat(timespec="seconds") + lastmodstr = "'unknown'" if lastmod is None else lastmod.isoformat( + timespec="seconds") header = f"# Annotations generated on {datetime.now().isoformat(timespec='seconds')} " \ f"using cellxgene version {cellxgene_version}\n" \ f"# Input data file was {self.data_locator.uri_or_path}, " \ f"which was last modified on {lastmodstr}\n" - write_labels(fname, new_label_df, header, backup_dir=self.get_anno_backup_dir(uid, collection)) + write_labels(fname, + new_label_df, + header, + backup_dir=self.get_anno_backup_dir(uid, collection)) return jsonify_scanpy({"status": "OK"}) @@ -591,41 +622,48 @@ class ScanpyEngine(CXGDriver): if axis != Axis.VAR: raise ValueError("Only VAR dimension access is supported") try: - obs_selector, var_selector = self._filter_to_mask(filter, use_slices=False) + obs_selector, var_selector = self._filter_to_mask(filter, + use_slices=False) except (KeyError, IndexError, TypeError) as e: raise FilterError(f"Error parsing filter: {e}") from e if obs_selector is not None: raise FilterError("filtering on obs unsupported") # Currently only handles VAR dimension - X = MatrixProxy.create(self.data.X if var_selector is None - else self.data.X[:, var_selector]) - return encode_matrix_fbs(X, col_idx=np.nonzero(var_selector)[0], row_idx=None) + X = MatrixProxy.create( + self.data.X if var_selector is None else self.data.X[:, + var_selector]) + return encode_matrix_fbs(X, + col_idx=np.nonzero(var_selector)[0], + row_idx=None) @requires_data - def diffexp_topN(self, obsFilterA, obsFilterB, top_n=None, interactive_limit=None): + def diffexp_topN(self, + obsFilterA, + obsFilterB, + top_n=None, + interactive_limit=None): if Axis.VAR in obsFilterA or Axis.VAR in obsFilterB: - raise FilterError("Observation filters may not contain vaiable conditions") + raise FilterError( + "Observation filters may not contain vaiable conditions") try: - obs_mask_A = self._axis_filter_to_mask( - obsFilterA["obs"], self.data.obs, self.data.n_obs - ) - obs_mask_B = self._axis_filter_to_mask( - obsFilterB["obs"], self.data.obs, self.data.n_obs - ) + obs_mask_A = self._axis_filter_to_mask(obsFilterA["obs"], + self.data.obs, + self.data.n_obs) + obs_mask_B = self._axis_filter_to_mask(obsFilterB["obs"], + self.data.obs, + self.data.n_obs) except (KeyError, IndexError) as e: raise FilterError(f"Error parsing filter: {e}") from e if top_n is None: top_n = DEFAULT_TOP_N - result = diffexp_ttest( - self.data, obs_mask_A, obs_mask_B, top_n, self.config['diffexp_lfc_cutoff'] - ) + result = diffexp_ttest(self.data, obs_mask_A, obs_mask_B, top_n, + self.config['diffexp_lfc_cutoff']) try: return jsonify_scanpy(result) except ValueError: raise JSONEncodingValueError( - "Error encoding differential expression to JSON" - ) + "Error encoding differential expression to JSON") @requires_data def layout_to_fbs_matrix(self): @@ -656,7 +694,9 @@ class ScanpyEngine(CXGDriver): normalized_layout = normalized_layout + translate normalized_layout = normalized_layout.astype(dtype=np.float32) - layout_data.append(pandas.DataFrame(normalized_layout, columns=[f"{layout}_0", f"{layout}_1"])) + layout_data.append( + pandas.DataFrame(normalized_layout, + columns=[f"{layout}_0", f"{layout}_1"])) except ValueError as e: raise PrepareError( diff --git a/server/app/util/constants.py b/server/app/util/constants.py index b53d476c..aa710058 100644 --- a/server/app/util/constants.py +++ b/server/app/util/constants.py @@ -1,10 +1,10 @@ from enum import Enum - DEFAULT_TOP_N = 10 class AugmentedEnum(Enum): + def __hash__(self): return self.value.__hash__() diff --git a/server/app/util/data_locator.py b/server/app/util/data_locator.py index 8a44f06d..7f5f0047 100644 --- a/server/app/util/data_locator.py +++ b/server/app/util/data_locator.py @@ -27,7 +27,8 @@ class DataLocator(): def __init__(self, uri_or_path): self.uri_or_path = uri_or_path - self.protocol, self.path = DataLocator._get_protocol_and_path(uri_or_path) + self.protocol, self.path = DataLocator._get_protocol_and_path( + uri_or_path) # work-around for LocalFileSystem not treating file: and None as the same scheme/protocol self.cname = self.path if self.protocol == 'file' else self.uri_or_path # will throw RuntimeError if the protocol is unsupported @@ -82,7 +83,8 @@ class DataLocator(): # if not local, create a tmp file system object to contain the data, # and clean it up when done. - with self.open() as src, tempfile.NamedTemporaryFile(prefix="cellxgene_", delete=False) as tmp: + with self.open() as src, tempfile.NamedTemporaryFile( + prefix="cellxgene_", delete=False) as tmp: tmp.write(src.read()) tmp.close() src.close() @@ -91,6 +93,7 @@ class DataLocator(): class LocalFilePath(): + def __init__(self, tmp_path, delete=False): self.tmp_path = tmp_path self.delete = delete diff --git a/server/app/util/fbs/NetEncoding/Column.py b/server/app/util/fbs/NetEncoding/Column.py index 89b786d6..367c8ef6 100644 --- a/server/app/util/fbs/NetEncoding/Column.py +++ b/server/app/util/fbs/NetEncoding/Column.py @@ -4,6 +4,7 @@ import flatbuffers + class Column(object): __slots__ = ['_tab'] @@ -22,7 +23,8 @@ class Column(object): def UType(self): o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) if o != 0: - return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos) + return self._tab.Get(flatbuffers.number_types.Uint8Flags, + o + self._tab.Pos) return 0 # Column @@ -35,7 +37,19 @@ class Column(object): return obj return None -def ColumnStart(builder): builder.StartObject(2) -def ColumnAddUType(builder, uType): builder.PrependUint8Slot(0, uType, 0) -def ColumnAddU(builder, u): builder.PrependUOffsetTRelativeSlot(1, flatbuffers.number_types.UOffsetTFlags.py_type(u), 0) -def ColumnEnd(builder): return builder.EndObject() + +def ColumnStart(builder): + builder.StartObject(2) + + +def ColumnAddUType(builder, uType): + builder.PrependUint8Slot(0, uType, 0) + + +def ColumnAddU(builder, u): + builder.PrependUOffsetTRelativeSlot( + 1, flatbuffers.number_types.UOffsetTFlags.py_type(u), 0) + + +def ColumnEnd(builder): + return builder.EndObject() diff --git a/server/app/util/fbs/NetEncoding/Float32Array.py b/server/app/util/fbs/NetEncoding/Float32Array.py index 1acc426c..d74eac33 100644 --- a/server/app/util/fbs/NetEncoding/Float32Array.py +++ b/server/app/util/fbs/NetEncoding/Float32Array.py @@ -4,6 +4,7 @@ import flatbuffers + class Float32Array(object): __slots__ = ['_tab'] @@ -23,14 +24,17 @@ class Float32Array(object): o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) if o != 0: a = self._tab.Vector(o) - return self._tab.Get(flatbuffers.number_types.Float32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4)) + return self._tab.Get( + flatbuffers.number_types.Float32Flags, + a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4)) return 0 # Float32Array def DataAsNumpy(self): o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) if o != 0: - return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Float32Flags, o) + return self._tab.GetVectorAsNumpy( + flatbuffers.number_types.Float32Flags, o) return 0 # Float32Array @@ -40,7 +44,19 @@ class Float32Array(object): return self._tab.VectorLen(o) return 0 -def Float32ArrayStart(builder): builder.StartObject(1) -def Float32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0) -def Float32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4) -def Float32ArrayEnd(builder): return builder.EndObject() + +def Float32ArrayStart(builder): + builder.StartObject(1) + + +def Float32ArrayAddData(builder, data): + builder.PrependUOffsetTRelativeSlot( + 0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0) + + +def Float32ArrayStartDataVector(builder, numElems): + return builder.StartVector(4, numElems, 4) + + +def Float32ArrayEnd(builder): + return builder.EndObject() diff --git a/server/app/util/fbs/NetEncoding/Float64Array.py b/server/app/util/fbs/NetEncoding/Float64Array.py index 2ec343a2..2153f2bf 100644 --- a/server/app/util/fbs/NetEncoding/Float64Array.py +++ b/server/app/util/fbs/NetEncoding/Float64Array.py @@ -4,6 +4,7 @@ import flatbuffers + class Float64Array(object): __slots__ = ['_tab'] @@ -23,14 +24,17 @@ class Float64Array(object): o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) if o != 0: a = self._tab.Vector(o) - return self._tab.Get(flatbuffers.number_types.Float64Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 8)) + return self._tab.Get( + flatbuffers.number_types.Float64Flags, + a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 8)) return 0 # Float64Array def DataAsNumpy(self): o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) if o != 0: - return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Float64Flags, o) + return self._tab.GetVectorAsNumpy( + flatbuffers.number_types.Float64Flags, o) return 0 # Float64Array @@ -40,7 +44,19 @@ class Float64Array(object): return self._tab.VectorLen(o) return 0 -def Float64ArrayStart(builder): builder.StartObject(1) -def Float64ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0) -def Float64ArrayStartDataVector(builder, numElems): return builder.StartVector(8, numElems, 8) -def Float64ArrayEnd(builder): return builder.EndObject() + +def Float64ArrayStart(builder): + builder.StartObject(1) + + +def Float64ArrayAddData(builder, data): + builder.PrependUOffsetTRelativeSlot( + 0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0) + + +def Float64ArrayStartDataVector(builder, numElems): + return builder.StartVector(8, numElems, 8) + + +def Float64ArrayEnd(builder): + return builder.EndObject() diff --git a/server/app/util/fbs/NetEncoding/Int32Array.py b/server/app/util/fbs/NetEncoding/Int32Array.py index f3f8156f..f1114a11 100644 --- a/server/app/util/fbs/NetEncoding/Int32Array.py +++ b/server/app/util/fbs/NetEncoding/Int32Array.py @@ -4,6 +4,7 @@ import flatbuffers + class Int32Array(object): __slots__ = ['_tab'] @@ -23,14 +24,17 @@ class Int32Array(object): o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) if o != 0: a = self._tab.Vector(o) - return self._tab.Get(flatbuffers.number_types.Int32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4)) + return self._tab.Get( + flatbuffers.number_types.Int32Flags, + a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4)) return 0 # Int32Array def DataAsNumpy(self): o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) if o != 0: - return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Int32Flags, o) + return self._tab.GetVectorAsNumpy( + flatbuffers.number_types.Int32Flags, o) return 0 # Int32Array @@ -40,7 +44,19 @@ class Int32Array(object): return self._tab.VectorLen(o) return 0 -def Int32ArrayStart(builder): builder.StartObject(1) -def Int32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0) -def Int32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4) -def Int32ArrayEnd(builder): return builder.EndObject() + +def Int32ArrayStart(builder): + builder.StartObject(1) + + +def Int32ArrayAddData(builder, data): + builder.PrependUOffsetTRelativeSlot( + 0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0) + + +def Int32ArrayStartDataVector(builder, numElems): + return builder.StartVector(4, numElems, 4) + + +def Int32ArrayEnd(builder): + return builder.EndObject() diff --git a/server/app/util/fbs/NetEncoding/JSONEncodedArray.py b/server/app/util/fbs/NetEncoding/JSONEncodedArray.py index 366ebadd..a75df72f 100644 --- a/server/app/util/fbs/NetEncoding/JSONEncodedArray.py +++ b/server/app/util/fbs/NetEncoding/JSONEncodedArray.py @@ -4,6 +4,7 @@ import flatbuffers + class JSONEncodedArray(object): __slots__ = ['_tab'] @@ -23,14 +24,17 @@ class JSONEncodedArray(object): o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) if o != 0: a = self._tab.Vector(o) - return self._tab.Get(flatbuffers.number_types.Uint8Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 1)) + return self._tab.Get( + flatbuffers.number_types.Uint8Flags, + a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 1)) return 0 # JSONEncodedArray def DataAsNumpy(self): o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) if o != 0: - return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Uint8Flags, o) + return self._tab.GetVectorAsNumpy( + flatbuffers.number_types.Uint8Flags, o) return 0 # JSONEncodedArray @@ -40,7 +44,19 @@ class JSONEncodedArray(object): return self._tab.VectorLen(o) return 0 -def JSONEncodedArrayStart(builder): builder.StartObject(1) -def JSONEncodedArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0) -def JSONEncodedArrayStartDataVector(builder, numElems): return builder.StartVector(1, numElems, 1) -def JSONEncodedArrayEnd(builder): return builder.EndObject() + +def JSONEncodedArrayStart(builder): + builder.StartObject(1) + + +def JSONEncodedArrayAddData(builder, data): + builder.PrependUOffsetTRelativeSlot( + 0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0) + + +def JSONEncodedArrayStartDataVector(builder, numElems): + return builder.StartVector(1, numElems, 1) + + +def JSONEncodedArrayEnd(builder): + return builder.EndObject() diff --git a/server/app/util/fbs/NetEncoding/Matrix.py b/server/app/util/fbs/NetEncoding/Matrix.py index 8c9d0eb8..62f5d439 100644 --- a/server/app/util/fbs/NetEncoding/Matrix.py +++ b/server/app/util/fbs/NetEncoding/Matrix.py @@ -4,6 +4,7 @@ import flatbuffers + class Matrix(object): __slots__ = ['_tab'] @@ -22,14 +23,16 @@ class Matrix(object): def NRows(self): o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) if o != 0: - return self._tab.Get(flatbuffers.number_types.Uint32Flags, o + self._tab.Pos) + return self._tab.Get(flatbuffers.number_types.Uint32Flags, + o + self._tab.Pos) return 0 # Matrix def NCols(self): o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) if o != 0: - return self._tab.Get(flatbuffers.number_types.Uint32Flags, o + self._tab.Pos) + return self._tab.Get(flatbuffers.number_types.Uint32Flags, + o + self._tab.Pos) return 0 # Matrix @@ -56,7 +59,8 @@ class Matrix(object): def ColIndexType(self): o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10)) if o != 0: - return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos) + return self._tab.Get(flatbuffers.number_types.Uint8Flags, + o + self._tab.Pos) return 0 # Matrix @@ -73,7 +77,8 @@ class Matrix(object): def RowIndexType(self): o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14)) if o != 0: - return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos) + return self._tab.Get(flatbuffers.number_types.Uint8Flags, + o + self._tab.Pos) return 0 # Matrix @@ -86,13 +91,45 @@ class Matrix(object): return obj return None -def MatrixStart(builder): builder.StartObject(7) -def MatrixAddNRows(builder, nRows): builder.PrependUint32Slot(0, nRows, 0) -def MatrixAddNCols(builder, nCols): builder.PrependUint32Slot(1, nCols, 0) -def MatrixAddColumns(builder, columns): builder.PrependUOffsetTRelativeSlot(2, flatbuffers.number_types.UOffsetTFlags.py_type(columns), 0) -def MatrixStartColumnsVector(builder, numElems): return builder.StartVector(4, numElems, 4) -def MatrixAddColIndexType(builder, colIndexType): builder.PrependUint8Slot(3, colIndexType, 0) -def MatrixAddColIndex(builder, colIndex): builder.PrependUOffsetTRelativeSlot(4, flatbuffers.number_types.UOffsetTFlags.py_type(colIndex), 0) -def MatrixAddRowIndexType(builder, rowIndexType): builder.PrependUint8Slot(5, rowIndexType, 0) -def MatrixAddRowIndex(builder, rowIndex): builder.PrependUOffsetTRelativeSlot(6, flatbuffers.number_types.UOffsetTFlags.py_type(rowIndex), 0) -def MatrixEnd(builder): return builder.EndObject() + +def MatrixStart(builder): + builder.StartObject(7) + + +def MatrixAddNRows(builder, nRows): + builder.PrependUint32Slot(0, nRows, 0) + + +def MatrixAddNCols(builder, nCols): + builder.PrependUint32Slot(1, nCols, 0) + + +def MatrixAddColumns(builder, columns): + builder.PrependUOffsetTRelativeSlot( + 2, flatbuffers.number_types.UOffsetTFlags.py_type(columns), 0) + + +def MatrixStartColumnsVector(builder, numElems): + return builder.StartVector(4, numElems, 4) + + +def MatrixAddColIndexType(builder, colIndexType): + builder.PrependUint8Slot(3, colIndexType, 0) + + +def MatrixAddColIndex(builder, colIndex): + builder.PrependUOffsetTRelativeSlot( + 4, flatbuffers.number_types.UOffsetTFlags.py_type(colIndex), 0) + + +def MatrixAddRowIndexType(builder, rowIndexType): + builder.PrependUint8Slot(5, rowIndexType, 0) + + +def MatrixAddRowIndex(builder, rowIndex): + builder.PrependUOffsetTRelativeSlot( + 6, flatbuffers.number_types.UOffsetTFlags.py_type(rowIndex), 0) + + +def MatrixEnd(builder): + return builder.EndObject() diff --git a/server/app/util/fbs/NetEncoding/TypedArray.py b/server/app/util/fbs/NetEncoding/TypedArray.py index e36c4f1b..5da1b1d9 100644 --- a/server/app/util/fbs/NetEncoding/TypedArray.py +++ b/server/app/util/fbs/NetEncoding/TypedArray.py @@ -2,6 +2,7 @@ # namespace: NetEncoding + class TypedArray(object): NONE = 0 Float32Array = 1 @@ -9,4 +10,3 @@ class TypedArray(object): Uint32Array = 3 Float64Array = 4 JSONEncodedArray = 5 - diff --git a/server/app/util/fbs/NetEncoding/Uint32Array.py b/server/app/util/fbs/NetEncoding/Uint32Array.py index 01c7dfdd..35beadb1 100644 --- a/server/app/util/fbs/NetEncoding/Uint32Array.py +++ b/server/app/util/fbs/NetEncoding/Uint32Array.py @@ -4,6 +4,7 @@ import flatbuffers + class Uint32Array(object): __slots__ = ['_tab'] @@ -23,14 +24,17 @@ class Uint32Array(object): o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) if o != 0: a = self._tab.Vector(o) - return self._tab.Get(flatbuffers.number_types.Uint32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4)) + return self._tab.Get( + flatbuffers.number_types.Uint32Flags, + a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4)) return 0 # Uint32Array def DataAsNumpy(self): o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) if o != 0: - return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Uint32Flags, o) + return self._tab.GetVectorAsNumpy( + flatbuffers.number_types.Uint32Flags, o) return 0 # Uint32Array @@ -40,7 +44,19 @@ class Uint32Array(object): return self._tab.VectorLen(o) return 0 -def Uint32ArrayStart(builder): builder.StartObject(1) -def Uint32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0) -def Uint32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4) -def Uint32ArrayEnd(builder): return builder.EndObject() + +def Uint32ArrayStart(builder): + builder.StartObject(1) + + +def Uint32ArrayAddData(builder, data): + builder.PrependUOffsetTRelativeSlot( + 0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0) + + +def Uint32ArrayStartDataVector(builder, numElems): + return builder.StartVector(4, numElems, 4) + + +def Uint32ArrayEnd(builder): + return builder.EndObject() diff --git a/server/app/util/fbs/matrix.py b/server/app/util/fbs/matrix.py index b29f4eb4..1e44896d 100644 --- a/server/app/util/fbs/matrix.py +++ b/server/app/util/fbs/matrix.py @@ -25,7 +25,8 @@ def CreateNumpyVector(builder, x): """CreateNumpyVector writes a numpy array into the buffer.""" if not isinstance(x, np.ndarray): - raise TypeError(f"non-numpy-ndarray passed to CreateNumpyVector ({type(x)}") + raise TypeError( + f"non-numpy-ndarray passed to CreateNumpyVector ({type(x)}") if x.dtype.kind not in ['b', 'i', 'u', 'f']: raise TypeError("numpy-ndarray holds elements of unsupported datatype") @@ -46,7 +47,8 @@ def CreateNumpyVector(builder, x): builder.head = int(builder.Head() - len) # tobytes ensures c_contiguous ordering - builder.Bytes[builder.Head():builder.Head() + len] = x_little_endian.tobytes(order='C') + builder.Bytes[builder.Head():builder.Head() + + len] = x_little_endian.tobytes(order='C') return builder.EndVector(x.size) @@ -119,12 +121,10 @@ column_encoding_type_map = { np.dtype(np.float64).str: (TypedArray.TypedArray.Float32Array, np.float32), np.dtype(np.float32).str: (TypedArray.TypedArray.Float32Array, np.float32), np.dtype(np.float16).str: (TypedArray.TypedArray.Float32Array, np.float32), - np.dtype(np.int8).str: (TypedArray.TypedArray.Int32Array, np.int32), np.dtype(np.int16).str: (TypedArray.TypedArray.Int32Array, np.int32), np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32), np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32), - np.dtype(np.uint8).str: (TypedArray.TypedArray.Uint32Array, np.uint32), np.dtype(np.uint16).str: (TypedArray.TypedArray.Uint32Array, np.uint32), np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32), @@ -141,7 +141,6 @@ index_encoding_type_map = { # array protocol string: ( array_type, as_type ) np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32), np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32), - np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32), np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32) } @@ -192,7 +191,8 @@ def encode_matrix_fbs(matrix, row_idx=None, col_idx=None): columns = [] for cidx in range(n_cols - 1, -1, -1): # serialize the typed array - col = matrix.iloc[:, cidx] if isinstance(matrix, pd.DataFrame) else matrix[:, cidx] + col = matrix.iloc[:, cidx] if isinstance( + matrix, pd.DataFrame) else matrix[:, cidx] typed_arr = serialize_typed_array(builder, col, column_encoding) # serialize the Column union @@ -218,12 +218,18 @@ def encode_matrix_fbs(matrix, row_idx=None, col_idx=None): def deserialize_typed_array(tarr): type_map = { - TypedArray.TypedArray.NONE: None, - TypedArray.TypedArray.Uint32Array: Uint32Array.Uint32Array, - TypedArray.TypedArray.Int32Array: Int32Array.Int32Array, - TypedArray.TypedArray.Float32Array: Float32Array.Float32Array, - TypedArray.TypedArray.Float64Array: Float64Array.Float64Array, - TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray + TypedArray.TypedArray.NONE: + None, + TypedArray.TypedArray.Uint32Array: + Uint32Array.Uint32Array, + TypedArray.TypedArray.Int32Array: + Int32Array.Int32Array, + TypedArray.TypedArray.Float32Array: + Float32Array.Float32Array, + TypedArray.TypedArray.Float64Array: + Float64Array.Float64Array, + TypedArray.TypedArray.JSONEncodedArray: + JSONEncodedArray.JSONEncodedArray } (u_type, u) = tarr if u_type is TypedArray.TypedArray.NONE: @@ -257,13 +263,16 @@ def decode_matrix_fbs(fbs): columns_length = matrix.ColumnsLength() - columns_index = deserialize_typed_array((matrix.ColIndexType(), matrix.ColIndex())) + columns_index = deserialize_typed_array( + (matrix.ColIndexType(), matrix.ColIndex())) if columns_index is None: columns_index = range(0, n_cols) # sanity checks if len(columns_index) != n_cols or columns_length != n_cols: - raise ValueError("FBS column count does not match number of columns in underlying matrix") + raise ValueError( + "FBS column count does not match number of columns in underlying matrix" + ) columns_data = {} columns_type = {} @@ -277,7 +286,8 @@ def decode_matrix_fbs(fbs): if col.UType() is TypedArray.TypedArray.JSONEncodedArray: columns_type[columns_index[col_idx]] = "category" - df = pd.DataFrame.from_dict(data=columns_data).astype(columns_type, copy=False) + df = pd.DataFrame.from_dict(data=columns_data).astype(columns_type, + copy=False) # more sanity checks if not df.columns.is_unique or len(df.columns) != n_cols: diff --git a/server/app/util/matrix_proxy.py b/server/app/util/matrix_proxy.py index 32414c1d..a3dfdc30 100644 --- a/server/app/util/matrix_proxy.py +++ b/server/app/util/matrix_proxy.py @@ -2,7 +2,6 @@ import abc from itertools import zip_longest from copy import copy import numpy as np - """ cellxgene deals with a variety of matrix data types, many of which do not support a consistent API. This framework allows proxies to be created @@ -18,6 +17,7 @@ class _ArrayProxyBase(abc.ABC): Private base class for array or matrix proxy. This summarizes the interface used by the rest of cellxgene. """ + @property @abc.abstractmethod def dtype(self): @@ -59,7 +59,6 @@ class MatrixProxy(_ArrayProxyBase): This class primarily provides the factory method and related support. All other functionality is delegated to subclasses. """ - """ Registry of types to proxy class, where values are: * None: unsupported @@ -128,21 +127,25 @@ class MatrixProxyView(MatrixProxy): """ 2D matrix view to a 2D matrix """ - def __init__(self, arg1, shape=None, index=(), - transposed=False, copy=False): + + def __init__(self, + arg1, + shape=None, + index=(), + transposed=False, + copy=False): if not copy: m = arg1 super().__init__(m) if shape is None: shape = m.shape - assert(len(shape) == 2) + assert (len(shape) == 2) index = tuple( - map(lambda s_i: - slice(0, s_i[0], 1) if s_i[1] is None else s_i[1], - zip_longest(shape, index)) - ) + map( + lambda s_i: slice(0, s_i[0], 1) + if s_i[1] is None else s_i[1], zip_longest(shape, index))) self._shape = shape self._index = index @@ -234,21 +237,29 @@ class MatrixProxyView(MatrixProxy): NOTE: these follow the numpy rules for dimensionality reduction when an integer index is specified. """ + def _getitem_intXint(self, row, col): return self.m[row, col] def _getitem_intXslice(self, row, col): - shape = (_slice_length(col, self.m.shape[1]), ) - return self.__class__.create_array(self.m, shape=shape, index=(row, col)) + shape = (_slice_length(col, self.m.shape[1]),) + return self.__class__.create_array(self.m, + shape=shape, + index=(row, col)) def _getitem_sliceXint(self, row, col): - shape = (_slice_length(row, self.m.shape[0]), ) - return self.__class__.create_array(self.m, shape=shape, index=(row, col)) + shape = (_slice_length(row, self.m.shape[0]),) + return self.__class__.create_array(self.m, + shape=shape, + index=(row, col)) def _getitem_sliceXslice(self, row, col): shape = (_slice_length(row, self.m.shape[0]), _slice_length(col, self.m.shape[1])) - return self.__class__(self.m, shape=shape, index=(row, col), transposed=self.transposed) + return self.__class__(self.m, + shape=shape, + index=(row, col), + transposed=self.transposed) def toarray(self): arr = self.m[self._index] @@ -261,22 +272,24 @@ class ArrayProxyView(_ArrayProxyBase): """ 1D array view to a 2D matrix """ + def __init__(self, arg1, shape=None, index=None, copy=False): super().__init__() if not copy: m = arg1 # one index MUST be an integer and the other MUST be a slice - assert(len(index) == 2) - assert(all(isinstance(idx, INT_TYPES + (slice, )) for idx in index)) - assert(isinstance(index[0], INT_TYPES) != isinstance(index[1], INT_TYPES)) + assert (len(index) == 2) + assert (all(isinstance(idx, INT_TYPES + (slice,)) for idx in index)) + assert (isinstance(index[0], INT_TYPES) != isinstance( + index[1], INT_TYPES)) if shape is None: if isinstance(index[0], INT_TYPES): - shape = (m.shape[0], ) + shape = (m.shape[0],) else: - shape = (m.shape[1], ) - assert(len(shape) == 1) + shape = (m.shape[1],) + assert (len(shape) == 1) self._shape = shape self.m = m @@ -336,7 +349,7 @@ class ArrayProxyView(_ArrayProxyBase): elif isinstance(col, slice): return self._getitem_intXslice(row, col) elif isinstance(row, slice): - assert(isinstance(col, INT_TYPES)) + assert (isinstance(col, INT_TYPES)) return self._getitem_sliceXint(row, col) raise IndexError("unsupported column index types") @@ -345,11 +358,11 @@ class ArrayProxyView(_ArrayProxyBase): return self.m[row, col] def _getitem_intXslice(self, row, col): - shape = (_slice_length(col, self.m.shape[1]), ) + shape = (_slice_length(col, self.m.shape[1]),) return self.__class__(self.m, shape=shape, index=(row, col)) def _getitem_sliceXint(self, row, col): - shape = (_slice_length(row, self.m.shape[0]), ) + shape = (_slice_length(row, self.m.shape[0]),) return self.__class__(self.m, shape=shape, index=(row, col)) def toarray(self): @@ -358,7 +371,7 @@ class ArrayProxyView(_ArrayProxyBase): def _unpack_index(index, shape): if not isinstance(index, tuple): - index = (index, ) + index = (index,) if len(shape) < len(index): raise IndexError("invalid index dimensionality - must be 2") @@ -366,7 +379,7 @@ def _unpack_index(index, shape): for shp, idx in zip_longest(shape, index): idx = slice(None) if idx is None else idx idx = _slice_defaults(idx, shp) if isinstance(idx, slice) else idx - unpacked += (idx, ) + unpacked += (idx,) return unpacked @@ -376,7 +389,7 @@ def _slice_slice(outer, outer_len, inner, inner_len): slice a slice - we take advantage of Python 3 range's support for indexing. """ - assert(outer_len >= inner_len) + assert (outer_len >= inner_len) outer_rng = range(*outer.indices(outer_len)) rng = outer_rng[inner] start, stop, step = rng.start, rng.stop, rng.step @@ -387,8 +400,8 @@ def _slice_slice(outer, outer_len, inner, inner_len): def _range_length(start, stop, step): """ return length of range """ - assert(step != 0) - assert(start is not None and stop is not None and step is not None) + assert (step != 0) + assert (start is not None and stop is not None and step is not None) if step > 0 and start < stop: return 1 + (stop - 1 - start) // step elif step < 0 and start > stop: @@ -404,7 +417,7 @@ def _slice_length(s, length): def _slice_defaults(s, length): """ apply slice defaulting conventions """ - assert(length >= 0) + assert (length >= 0) step = 1 if s.step is None else s.step diff --git a/server/app/util/utils.py b/server/app/util/utils.py index 4bf57612..985bd67f 100644 --- a/server/app/util/utils.py +++ b/server/app/util/utils.py @@ -7,6 +7,7 @@ from server.app.util.errors import DriverError class Float32JSONEncoder(json.JSONEncoder): + def __init__(self, *args, **kwargs): """ NaN/Infinities are illegal in standard JSON. Python extends JSON with @@ -35,9 +36,12 @@ def jsonify_scanpy(data): def requires_data(func): + @wraps(func) def wrapped_function(self, *args, **kwargs): if self.data is None: - raise DriverError(f"error data must be loaded before you call {func.__name__}") + raise DriverError( + f"error data must be loaded before you call {func.__name__}") return func(self, *args, **kwargs) + return wrapped_function diff --git a/server/app/web/webapp.py b/server/app/web/webapp.py index 8af94640..224c6b40 100644 --- a/server/app/web/webapp.py +++ b/server/app/web/webapp.py @@ -1,7 +1,6 @@ import os from flask import Blueprint, render_template, send_from_directory, current_app - bp = Blueprint("webapp", __name__, template_folder="templates") @@ -9,9 +8,12 @@ bp = Blueprint("webapp", __name__, template_folder="templates") def index(): dataset_title = current_app.config["DATASET_TITLE"] scripts = current_app.config["SCRIPTS"] - return render_template("index.html", datasetTitle=dataset_title, SCRIPTS=scripts) + return render_template("index.html", + datasetTitle=dataset_title, + SCRIPTS=scripts) @bp.route("/favicon.png") def favicon(): - return send_from_directory(os.path.join(bp.root_path, "static/img/"), "favicon.png") + return send_from_directory(os.path.join(bp.root_path, "static/img/"), + "favicon.png") diff --git a/server/cli/cli.py b/server/cli/cli.py index ba0576b6..1352ead6 100644 --- a/server/cli/cli.py +++ b/server/cli/cli.py @@ -10,11 +10,10 @@ from .prepare import prepare context_settings=dict(max_content_width=85, help_option_names=['-h', '--help'])) @click.help_option("--help", "-h", help="Show this message and exit.") -@click.version_option( - version="0.13.0", - prog_name="cellxgene", - message="[%(prog)s] Version %(version)s", - help="Show the software version and exit.") +@click.version_option(version="0.13.0", + prog_name="cellxgene", + message="[%(prog)s] Version %(version)s", + help="Show the software version and exit.") def cli(): pass diff --git a/server/cli/launch.py b/server/cli/launch.py index c5fa3178..61feccff 100644 --- a/server/cli/launch.py +++ b/server/cli/launch.py @@ -17,7 +17,7 @@ from server.utils.utils import find_available_port, is_port_available, sort_opti from server.app.util.data_locator import DataLocator # anything bigger than this will generate a special message -BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20 # 100MB +BIG_FILE_SIZE_THRESHOLD = 100 * 2**20 # 100MB def common_args(func): @@ -25,16 +25,14 @@ def common_args(func): Decorator to contain CLI args that will be common to both CLI and GUI: title and engine args. """ - @click.option( - "--title", - "-t", - metavar="", - help="Title to display. If omitted will use file name.") - @click.option( - "--about", - metavar="", - help="URL providing more information about the dataset " - "(hint: must be a fully specified absolute URL).") + @click.option("--title", + "-t", + metavar="", + help="Title to display. If omitted will use file name.") + @click.option("--about", + metavar="", + help="URL providing more information about the dataset " + "(hint: must be a fully specified absolute URL).") @click.option( "--embedding", "-e", @@ -42,69 +40,80 @@ def common_args(func): multiple=True, show_default=False, metavar="", - help="Embedding name, eg, 'umap'. Repeat option for multiple embeddings. Defaults to all." + help= + "Embedding name, eg, 'umap'. Repeat option for multiple embeddings. Defaults to all." ) @click.option( "--obs-names", "-obs", default=None, metavar="", - help="Name of annotation field to use for observations. If not specified cellxgene will use the the obs index.") + help= + "Name of annotation field to use for observations. If not specified cellxgene will use the the obs index." + ) @click.option( "--var-names", "-var", default=None, metavar="", - help="Name of annotation to use for variables. If not specified cellxgene will use the the var index.") + help= + "Name of annotation to use for variables. If not specified cellxgene will use the the var index." + ) @click.option( "--max-category-items", default=1000, metavar="", show_default=True, - help="Will not display categories with more distinct values than specified.",) + help= + "Will not display categories with more distinct values than specified.", + ) @click.option( "--diffexp-lfc-cutoff", "-de", default=0.01, show_default=True, metavar="", - help="Minimum log fold change threshold for differential expression.",) - @click.option( - "--experimental-annotations", - is_flag=True, - default=False, - show_default=True, - help="Enable user annotation of data." + help="Minimum log fold change threshold for differential expression.", ) + @click.option("--experimental-annotations", + is_flag=True, + default=False, + show_default=True, + help="Enable user annotation of data.") @click.option( "--experimental-annotations-file", default=None, show_default=True, multiple=False, metavar="", - help="CSV file to initialize editing of existing annotations; will be altered in-place. " - "Incompatible with --annotations-output-dir.",) + help= + "CSV file to initialize editing of existing annotations; will be altered in-place. " + "Incompatible with --annotations-output-dir.", + ) @click.option( "--experimental-annotations-output-dir", default=None, show_default=False, multiple=False, metavar="", - help="Directory of where to save output annotations; filename will be specified in the application. " - "Incompatible with --annotations-input-file.",) + help= + "Directory of where to save output annotations; filename will be specified in the application. " + "Incompatible with --annotations-input-file.", + ) @click.option( "--backed", "-b", is_flag=True, default=False, show_default=False, - help="Load data in file-backed mode. This may save memory, but may result in slower overall performance.") - @click.option( - "--disable-diffexp", - is_flag=True, - default=False, - show_default=False, - help="Disable on-demand differential expression.") + help= + "Load data in file-backed mode. This may save memory, but may result in slower overall performance." + ) + @click.option("--disable-diffexp", + is_flag=True, + default=False, + show_default=False, + help="Disable on-demand differential expression.") @functools.wraps(func) def wrapper(*args, **kwargs): return func(*args, **kwargs) @@ -112,9 +121,11 @@ def common_args(func): return wrapper -def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffexp_lfc_cutoff, - experimental_annotations, experimental_annotations_file, - experimental_annotations_output_dir, backed, disable_diffexp): +def parse_engine_args(embedding, obs_names, var_names, max_category_items, + diffexp_lfc_cutoff, experimental_annotations, + experimental_annotations_file, + experimental_annotations_output_dir, backed, + disable_diffexp): annotations_file = experimental_annotations_file if experimental_annotations else None annotations_output_dir = experimental_annotations_output_dir if experimental_annotations else None return { @@ -132,9 +143,11 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffe @sort_options -@click.command(short_help="Launch the cellxgene data viewer. " - "Run `cellxgene launch --help` for more information.", - options_metavar="",) +@click.command( + short_help="Launch the cellxgene data viewer. " + "Run `cellxgene launch --help` for more information.", + options_metavar="", +) @click.argument("data", nargs=1, metavar="", required=True) @click.option( "--verbose", @@ -142,7 +155,8 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffe is_flag=True, default=False, show_default=True, - help="Provide verbose output, including warnings and all server requests.",) + help="Provide verbose output, including warnings and all server requests.", +) @click.option( "--debug", "-d", @@ -150,7 +164,8 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffe default=False, show_default=True, help="Run in debug mode. This is helpful for cellxgene developers, " - "or when you want more information about an error condition.",) + "or when you want more information about an error condition.", +) @click.option( "--open", "-o", @@ -158,19 +173,24 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffe is_flag=True, default=False, show_default=True, - help="Open web browser after launch.",) + help="Open web browser after launch.", +) @click.option( "--port", "-p", metavar="", show_default=True, - help="Port to run server on. If not specified cellxgene will find an available port.",) + help= + "Port to run server on. If not specified cellxgene will find an available port.", +) @click.option( "--host", metavar="", default="127.0.0.1", show_default=False, - help="Host IP address. By default cellxgene will use localhost (e.g. 127.0.0.1).") + help= + "Host IP address. By default cellxgene will use localhost (e.g. 127.0.0.1)." +) @click.option( "--scripts", "-s", @@ -178,31 +198,15 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffe multiple=True, metavar="", help="Additional script files to include in HTML page. If not specified, " - "no additional script files will be included.", - show_default=False,) + "no additional script files will be included.", + show_default=False, +) @click.help_option("--help", "-h", help="Show this message and exit.") @common_args -def launch( - data, - verbose, - debug, - open_browser, - port, - host, - embedding, - obs_names, - var_names, - max_category_items, - diffexp_lfc_cutoff, - title, - scripts, - about, - experimental_annotations, - experimental_annotations_file, - experimental_annotations_output_dir, - backed, - disable_diffexp -): +def launch(data, verbose, debug, open_browser, port, host, embedding, obs_names, + var_names, max_category_items, diffexp_lfc_cutoff, title, scripts, + about, experimental_annotations, experimental_annotations_file, + experimental_annotations_output_dir, backed, disable_diffexp): """Launch the cellxgene data viewer. This web app lets you explore single-cell expression data. Data must be in a format that cellxgene expects. @@ -217,17 +221,17 @@ def launch( > cellxgene launch """ - e_args = parse_engine_args(embedding, obs_names, var_names, max_category_items, - diffexp_lfc_cutoff, + e_args = parse_engine_args(embedding, obs_names, var_names, + max_category_items, diffexp_lfc_cutoff, experimental_annotations, experimental_annotations_file, - experimental_annotations_output_dir, - backed, + experimental_annotations_output_dir, backed, disable_diffexp) try: data_locator = DataLocator(data) except RuntimeError as re: - raise click.ClickException(f"Unable to access data at {data}. {str(re)}") + raise click.ClickException( + f"Unable to access data at {data}. {str(re)}") # Startup message click.echo("[cellxgene] Starting the CLI...") @@ -244,7 +248,8 @@ def launch( raise click.FileError(data, hint="data is not a file") name, extension = splitext(data) if extension != ".h5ad": - raise click.FileError(basename(data), hint="file type must be .h5ad") + raise click.FileError(basename(data), + hint="file type must be .h5ad") if debug: verbose = True @@ -266,7 +271,9 @@ def launch( security risk by including the --scripts flag. Make sure you trust the scripts that you are including. """) scripts_pretty = ", ".join(scripts) - click.confirm(f"Are you sure you want to inject these scripts: {scripts_pretty}?", abort=True) + click.confirm( + f"Are you sure you want to inject these scripts: {scripts_pretty}?", + abort=True) if not title: file_parts = splitext(basename(data)) @@ -274,7 +281,9 @@ def launch( if port: if debug: - raise click.ClickException("--port and --debug may not be used together (try --verbose for error logging).") + raise click.ClickException( + "--port and --debug may not be used together (try --verbose for error logging)." + ) if not is_port_available(host, int(port)): raise click.ClickException( f"The port selected {port} is in use, please specify an open port using the --port flag." @@ -284,27 +293,36 @@ def launch( if not experimental_annotations: if experimental_annotations_file is not None: - click.echo("Warning: --experimental-annotations-file ignored as --annotations not enabled.") + click.echo( + "Warning: --experimental-annotations-file ignored as --annotations not enabled." + ) if experimental_annotations_output_dir is not None: - click.echo("Warning: --experimental-annotations-output-dir ignored as --annotations not enabled.") + click.echo( + "Warning: --experimental-annotations-output-dir ignored as --annotations not enabled." + ) else: if experimental_annotations_file is not None and experimental_annotations_output_dir is not None: - raise click.ClickException("--experimental-annotations-file and --experimental-annotations-output-dir " - "may not be used together.") + raise click.ClickException( + "--experimental-annotations-file and --experimental-annotations-output-dir " + "may not be used together.") if experimental_annotations_file is not None: lf_name, lf_ext = splitext(experimental_annotations_file) if lf_ext and lf_ext != ".csv": - raise click.FileError(basename(experimental_annotations_file), hint="annotation file type must be .csv") + raise click.FileError(basename(experimental_annotations_file), + hint="annotation file type must be .csv") - if experimental_annotations_output_dir is not None and not isdir(experimental_annotations_output_dir): + if experimental_annotations_output_dir is not None and not isdir( + experimental_annotations_output_dir): try: mkdir(experimental_annotations_output_dir) except OSError: - raise click.ClickException("Unable to create directory specified by " - "--experimental-annotations-output-dir") + raise click.ClickException( + "Unable to create directory specified by " + "--experimental-annotations-output-dir") if about: + def url_check(url): try: result = urlparse(url) @@ -316,7 +334,9 @@ def launch( return False if not url_check(about): - raise click.ClickException("Must provide an absolute URL for --about. (Example format: http://example.com)") + raise click.ClickException( + "Must provide an absolute URL for --about. (Example format: http://example.com)" + ) # Setup app cellxgene_url = f"http://{host}:{port}" @@ -335,14 +355,18 @@ def launch( # if a big file, let the user know it may take a while to load. if file_size > BIG_FILE_SIZE_THRESHOLD: - click.echo(f"[cellxgene] Loading data from {basename(data)}, this may take a while...") + click.echo( + f"[cellxgene] Loading data from {basename(data)}, this may take a while..." + ) else: click.echo(f"[cellxgene] Loading data from {basename(data)}.") from server.app.scanpy_engine.scanpy_engine import ScanpyEngine try: - server.attach_data(ScanpyEngine(data_locator, e_args), title=title, about=about) + server.attach_data(ScanpyEngine(data_locator, e_args), + title=title, + about=about) except ScanpyFileError as e: raise click.ClickException(f"{e}") @@ -351,10 +375,14 @@ def launch( f"running differential expression may take longer or fail.") if open_browser: - click.echo(f"[cellxgene] Launching! Opening your browser to {cellxgene_url} now.") + click.echo( + f"[cellxgene] Launching! Opening your browser to {cellxgene_url} now." + ) webbrowser.open(cellxgene_url) else: - click.echo(f"[cellxgene] Launching! Please go to {cellxgene_url} in your browser.") + click.echo( + f"[cellxgene] Launching! Please go to {cellxgene_url} in your browser." + ) click.echo("[cellxgene] Type CTRL-C at any time to exit.") @@ -363,8 +391,14 @@ def launch( sys.stdout = f try: - server.app.run(host=host, debug=debug, port=port, threaded=False if debug else True, use_debugger=False) + server.app.run(host=host, + debug=debug, + port=port, + threaded=False if debug else True, + use_debugger=False) except OSError as e: if e.errno == errno.EADDRINUSE: - raise click.ClickException("Port is in use, please specify an open port using the --port flag.") from e + raise click.ClickException( + "Port is in use, please specify an open port using the --port flag." + ) from e raise diff --git a/server/cli/prepare.py b/server/cli/prepare.py index 224fa17c..eef2272b 100644 --- a/server/cli/prepare.py +++ b/server/cli/prepare.py @@ -8,9 +8,11 @@ from server.utils.utils import sort_options @sort_options -@click.command(short_help="Preprocess data for use with cellxgene. " - "Run `cellxgene prepare --help` for more information.", - options_metavar="",) +@click.command( + short_help="Preprocess data for use with cellxgene. " + "Run `cellxgene prepare --help` for more information.", + options_metavar="", +) @click.argument("data", nargs=1, metavar="", required=True) @click.option( "--embedding", @@ -29,43 +31,66 @@ from server.utils.utils import sort_options help="Preprocessing to run.", show_default=True, ) -@click.option("--output", "-o", default="", help="Save a new file to filename.", metavar="") -@click.option("--plotting", "-p", default=False, is_flag=True, help="Generate plots.", show_default=True) -@click.option("--sparse", default=False, is_flag=True, help="Force sparsity.", show_default=True) -@click.option("--overwrite", default=False, is_flag=True, help="Allow file overwriting.", show_default=True) -@click.option("--set-obs-names", default="", help="Named field to set as index for obs.", metavar="") -@click.option("--set-var-names", default="", help="Named field to set as index for var.", metavar="") -@click.option("--skip-qc", default=False, is_flag=True, - help="Do not run quality control metrics. By default cellxgene runs them " - "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).") +@click.option("--output", + "-o", + default="", + help="Save a new file to filename.", + metavar="") +@click.option("--plotting", + "-p", + default=False, + is_flag=True, + help="Generate plots.", + show_default=True) +@click.option("--sparse", + default=False, + is_flag=True, + help="Force sparsity.", + show_default=True) +@click.option("--overwrite", + default=False, + is_flag=True, + help="Allow file overwriting.", + show_default=True) +@click.option("--set-obs-names", + default="", + help="Named field to set as index for obs.", + metavar="") +@click.option("--set-var-names", + default="", + help="Named field to set as index for var.", + metavar="") @click.option( - "--make-obs-names-unique", - default=True, + "--skip-qc", + default=False, is_flag=True, - help="Ensure obs index is unique.", - show_default=True -) -@click.option( - "--make-var-names-unique", - default=True, - is_flag=True, - help="Ensure var index is unique.", - show_default=True + help="Do not run quality control metrics. By default cellxgene runs them " + "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details)." ) +@click.option("--make-obs-names-unique", + default=True, + is_flag=True, + help="Ensure obs index is unique.", + show_default=True) +@click.option("--make-var-names-unique", + default=True, + is_flag=True, + help="Ensure var index is unique.", + show_default=True) @click.help_option("--help", "-h", help="Show this message and exit.") def prepare( - data, - embedding, - recipe, - output, - plotting, - sparse, - overwrite, - set_obs_names, - set_var_names, - skip_qc, - make_obs_names_unique, - make_var_names_unique, + data, + embedding, + recipe, + output, + plotting, + sparse, + overwrite, + set_obs_names, + set_var_names, + skip_qc, + make_obs_names_unique, + make_var_names_unique, ): """ Preprocess data for use with cellxgene. @@ -86,8 +111,7 @@ def prepare( except ImportError: raise click.ClickException( "[cellxgene] cellxgene prepare has not been installed. Please run `pip install cellxgene[prepare]` " - "to install the necessary requirements." - ) + "to install the necessary requirements.") # scanpy settings sc.settings.verbosity = 0 @@ -102,10 +126,11 @@ def prepare( if not output: click.echo( "Warning: No file will be saved, to save the results of cellxgene prepare include " - "--output to save output to a new file" - ) + "--output to save output to a new file") if isfile(output) and not overwrite: - raise click.UsageError(f"Cannot overwrite existing file {output}, try using the flag --overwrite") + raise click.UsageError( + f"Cannot overwrite existing file {output}, try using the flag --overwrite" + ) def load_data(data): if isfile(data): @@ -115,7 +140,9 @@ def prepare( elif extension == ".loom": adata = sc.read_loom(data) else: - raise click.FileError(data, hint="does not have a valid extension [.h5ad | .loom]") + raise click.FileError( + data, + hint="does not have a valid extension [.h5ad | .loom]") elif isdir(data): if not data.endswith(sep): data += sep @@ -125,11 +152,15 @@ def prepare( if not set_obs_names == "": if set_obs_names not in adata.obs_keys(): - raise click.UsageError(f"obs {set_obs_names} not found, options are: {adata.obs_keys()}") + raise click.UsageError( + f"obs {set_obs_names} not found, options are: {adata.obs_keys()}" + ) adata.obs_names = adata.obs[set_obs_names] if not set_var_names == "": if set_var_names not in adata.var_keys(): - raise click.UsageError(f"var {set_var_names} not found, options are: {adata.var_keys()}") + raise click.UsageError( + f"var {set_var_names} not found, options are: {adata.var_keys()}" + ) adata.var_names = adata.var[set_var_names] if make_obs_names_unique: adata.obs_names_make_unique() @@ -184,12 +215,18 @@ def prepare( if "umap" in embedding: sc.tl.umap(adata) if plotting: - sc.pl.umap(adata, color="louvain", palette=palette, save="_louvain") + sc.pl.umap(adata, + color="louvain", + palette=palette, + save="_louvain") if "tsne" in embedding: sc.tl.tsne(adata) if plotting: - sc.pl.tsne(adata, color="louvain", palette=palette, save="_louvain") + sc.pl.tsne(adata, + color="louvain", + palette=palette, + save="_louvain") def show_step(item): if not skip_qc: @@ -208,13 +245,19 @@ def prepare( if item is not None: return names[item.__name__] - steps = [calculate_qc_metrics, make_sparse, run_recipe, run_pca, run_neighbors, run_louvain, run_embedding] + steps = [ + calculate_qc_metrics, make_sparse, run_recipe, run_pca, run_neighbors, + run_louvain, run_embedding + ] click.echo(f"[cellxgene] Loading data from {data}, please wait...") adata = load_data(data) click.echo("[cellxgene] Beginning preprocessing...") - with click.progressbar(steps, label="[cellxgene] Progress", show_eta=False, item_show_func=show_step) as bar: + with click.progressbar(steps, + label="[cellxgene] Progress", + show_eta=False, + item_show_func=show_step) as bar: for step in bar: step(adata) diff --git a/server/gui/browser.py b/server/gui/browser.py index d09bc621..0479bbfd 100644 --- a/server/gui/browser.py +++ b/server/gui/browser.py @@ -12,7 +12,9 @@ WindowUtils = cef.WindowUtils() # noinspection PyUnresolvedReferences CefWidgetParent = QWidget + class CefWidget(CefWidgetParent): + def __init__(self, parent=None): super(CefWidget, self).__init__(parent) self.parent = parent @@ -58,8 +60,8 @@ class CefWidget(CefWidgetParent): if WINDOWS: WindowUtils.OnSize(self.getHandle(), 0, 0, 0) elif LINUX: - self.browser.SetBounds(self.x, self.y, - self.width(), self.height()) + self.browser.SetBounds(self.x, self.y, self.width(), + self.height()) self.browser.NotifyMoveOrResizeStarted() def resizeEvent(self, event): @@ -68,12 +70,13 @@ class CefWidget(CefWidgetParent): if WINDOWS: WindowUtils.OnSize(self.getHandle(), 0, 0, 0) elif LINUX: - self.browser.SetBounds(self.x, self.y, - size.width(), size.height()) + self.browser.SetBounds(self.x, self.y, size.width(), + size.height()) self.browser.NotifyMoveOrResizeStarted() class CefApplication(QApplication): + def __init__(self, args): super(CefApplication, self).__init__(args) if not cef.GetAppSetting("external_message_pump"): diff --git a/server/gui/cellxgene_rc.py b/server/gui/cellxgene_rc.py index 07fda04b..67095adb 100644 --- a/server/gui/cellxgene_rc.py +++ b/server/gui/cellxgene_rc.py @@ -431,10 +431,15 @@ qt_resource_struct = b"\ \x00\x00\x00,\x00\x00\x00\x00\x00\x01\x00\x00\x11>\ " + def qInitResources(): - QtCore.qRegisterResourceData(0x01, qt_resource_struct, qt_resource_name, qt_resource_data) + QtCore.qRegisterResourceData(0x01, qt_resource_struct, qt_resource_name, + qt_resource_data) + def qCleanupResources(): - QtCore.qUnregisterResourceData(0x01, qt_resource_struct, qt_resource_name, qt_resource_data) + QtCore.qUnregisterResourceData(0x01, qt_resource_struct, qt_resource_name, + qt_resource_data) + qInitResources() diff --git a/server/gui/hook-cefpython3.py b/server/gui/hook-cefpython3.py index 4c6e0a9d..c0b23c23 100644 --- a/server/gui/hook-cefpython3.py +++ b/server/gui/hook-cefpython3.py @@ -50,22 +50,22 @@ def check_pyinstaller_version(): version = PyInstaller.__version__ match = re.search(r"^\d+\.\d+(\.\d+)?", version) if not (match.group(0) >= PYINSTALLER_MIN_VERSION): - raise SystemExit("Error: pyinstaller %s or higher is required" - % PYINSTALLER_MIN_VERSION) + raise SystemExit("Error: pyinstaller %s or higher is required" % + PYINSTALLER_MIN_VERSION) def check_cefpython3_version(): if not is_module_satisfies("cefpython3 >= %s" % CEFPYTHON_MIN_VERSION): - raise SystemExit("Error: cefpython3 %s or higher is required" - % CEFPYTHON_MIN_VERSION) + raise SystemExit("Error: cefpython3 %s or higher is required" % + CEFPYTHON_MIN_VERSION) def get_cefpython_modules(): """Get all cefpython Cython modules in the cefpython3 package. It returns a list of names without file extension. Eg. 'cefpython_py27'. """ - pyds = glob.glob(os.path.join(CEFPYTHON3_DIR, - "cefpython_py*" + CYTHON_MODULE_EXT)) + pyds = glob.glob( + os.path.join(CEFPYTHON3_DIR, "cefpython_py*" + CYTHON_MODULE_EXT)) assert len(pyds) > 1, "Missing cefpython3 Cython modules" modules = [] for path in pyds: @@ -155,7 +155,8 @@ def get_cefpython3_datas(): absolute_file_path = os.path.join(path, file) dest_path = os.path.relpath(path, CEFPYTHON3_DIR) ret.append((absolute_file_path, dest_path)) - logger.info("Include cefpython3 data: {}/{}".format(dest_path, file)) + logger.info("Include cefpython3 data: {}/{}".format( + dest_path, file)) elif is_win or is_linux: # The .pak files in cefpython3/locales/ directory locales_dir = os.path.join(CEFPYTHON3_DIR, "locales") @@ -164,8 +165,9 @@ def get_cefpython3_datas(): for filename in os.listdir(locales_dir): logger.info("Include cefpython3 data: {}/{}".format( os.path.basename(locales_dir), filename)) - ret.append((os.path.join(locales_dir, filename), - os.path.join(cefdatadir, "locales"))) + ret.append( + (os.path.join(locales_dir, + filename), os.path.join(cefdatadir, "locales"))) # Optional .so/.dll files in cefpython3/swiftshader/ directory swiftshader_dir = os.path.join(CEFPYTHON3_DIR, "swiftshader") diff --git a/server/gui/main.py b/server/gui/main.py index 72f3e2a7..f63aa271 100644 --- a/server/gui/main.py +++ b/server/gui/main.py @@ -34,6 +34,7 @@ LOAD_INDEX = 1 class MainWindow(QMainWindow): + def __init__(self): super(MainWindow, self).__init__(None) self.cef_widget = None @@ -59,14 +60,17 @@ class MainWindow(QMainWindow): # close emitter on error/finished self.parent_conn, self.child_conn = Pipe() self.load_emitter = Emitter(self.parent_conn, WorkerSignals) - self.emitter_thread = threading.Thread(target=self.load_emitter.run, daemon=True) + self.emitter_thread = threading.Thread(target=self.load_emitter.run, + daemon=True) self.emitter_thread.start() # send to load with error message? def setupLayout(self): self.resize(WIDTH, HEIGHT) self.cef_widget = CefWidget(self) - self.cef_widget.setSizePolicy(QSizePolicy(QSizePolicy.MinimumExpanding, QSizePolicy.MinimumExpanding)) + self.cef_widget.setSizePolicy( + QSizePolicy(QSizePolicy.MinimumExpanding, + QSizePolicy.MinimumExpanding)) self.data_widget = LoadWidget(self) self.stacked_layout = QStackedLayout() self.stacked_layout.addWidget(self.cef_widget) @@ -104,7 +108,8 @@ class MainWindow(QMainWindow): # close emitter on error/finished self.parent_conn, self.child_conn = Pipe() self.load_emitter = Emitter(self.parent_conn, WorkerSignals) - self.emitter_thread = threading.Thread(target=self.load_emitter.run, daemon=True) + self.emitter_thread = threading.Thread(target=self.load_emitter.run, + daemon=True) self.emitter_thread.start() # send to load with error message? @@ -144,6 +149,7 @@ class MainWindow(QMainWindow): class LoadWidget(QFrame): + def __init__(self, parent): super(LoadWidget, self).__init__(parent=parent) # Init layout @@ -240,11 +246,18 @@ class LoadWidget(QFrame): def createScanpyEngine(self, file_name): title = splitext(basename(file_name))[0] self.window().setupServer() - worker = Worker(self.window().parent_conn, self.window().child_conn, file_name, host="127.0.0.1", - port=GUI_PORT, title=title, engine_options={}) + worker = Worker(self.window().parent_conn, + self.window().child_conn, + file_name, + host="127.0.0.1", + port=GUI_PORT, + title=title, + engine_options={}) self.window().load_emitter.signals.ready.connect(self.onDataReady) - self.window().load_emitter.signals.engine_error.connect(self.onServerError) - self.window().load_emitter.signals.server_error.connect(self.onServerError) + self.window().load_emitter.signals.engine_error.connect( + self.onServerError) + self.window().load_emitter.signals.server_error.connect( + self.onServerError) # Error is generic error from emitter self.window().load_emitter.signals.error.connect(self.onServerError) self.window().worker = Process(target=worker.run, daemon=True) @@ -266,7 +279,8 @@ class LoadWidget(QFrame): self.site_ready_worker.signals.ready.connect(self.onServerReady) self.site_ready_worker.signals.error.connect(self.onServerError) - srw_thread = threading.Thread(target=self.site_ready_worker.run, daemon=True) + srw_thread = threading.Thread(target=self.site_ready_worker.run, + daemon=True) srw_thread.start() def onServerReady(self): @@ -289,7 +303,9 @@ class LoadWidget(QFrame): onServerError = partialmethod(onError, server_error=True) + class FilePath(QObject): + def __init__(self): super(FilePath, self).__init__() self.value = "" @@ -301,6 +317,7 @@ class FilePath(QObject): class FileArea(QFrame): + def __init__(self, parent): super(FileArea, self).__init__() self.setFrameShape(QFrame.Box) @@ -309,10 +326,12 @@ class FileArea(QFrame): self.setAcceptDrops(True) self.instructions = QLabel(self) self.instructions.setText("Drag & Drop a h5ad file to load or open") - self.instructions.setGeometry(10, 10, MAX_CONTENT_WIDTH, self.instructions.height()) + self.instructions.setGeometry(10, 10, MAX_CONTENT_WIDTH, + self.instructions.height()) self.loadButton = QPushButton("Open...", parent=self) x_pos = (MAX_CONTENT_WIDTH - self.loadButton.width()) / 2 - self.loadButton.setGeometry(x_pos, 50, self.loadButton.width(), self.loadButton.height()) + self.loadButton.setGeometry(x_pos, 50, self.loadButton.width(), + self.loadButton.height()) self.loadButton.clicked.connect(self.fileBrowse) self.label = QLabel(self) self.label.setGeometry(10, 75, MAX_CONTENT_WIDTH, self.label.height()) @@ -321,7 +340,10 @@ class FileArea(QFrame): options = QFileDialog.Options() # options |= QFileDialog.DontUseNativeDialog file_name, _ = QFileDialog.getOpenFileName(self, - "Open H5AD File", "", "H5AD Files (*.h5ad)", options=options) + "Open H5AD File", + "", + "H5AD Files (*.h5ad)", + options=options) if file_name: self.parent().file_name.updateValue(file_name) self.parent().onLoad() diff --git a/server/gui/utils.py b/server/gui/utils.py index b2149461..1abc84e5 100644 --- a/server/gui/utils.py +++ b/server/gui/utils.py @@ -49,6 +49,7 @@ class FileChanged(QObject): class Emitter: + def __init__(self, transport, signals): self.transport = transport self.signals = signals() diff --git a/server/gui/workers.py b/server/gui/workers.py index 62320ab8..62d78143 100644 --- a/server/gui/workers.py +++ b/server/gui/workers.py @@ -7,6 +7,7 @@ from server.gui.utils import SiteReadySignals class EmittingProcess(Process): + def __init__(self, parent_conn, child_conn, *arg, **kwargs): super(EmittingProcess, self).__init__() self.parent_conn = parent_conn @@ -21,7 +22,9 @@ class EmittingProcess(Process): class Worker(EmittingProcess): - def __init__(self, parent_conn, child_conn, data_file, host, port, title, engine_options, *args, **kwargs): + + def __init__(self, parent_conn, child_conn, data_file, host, port, title, + engine_options, *args, **kwargs): super(Worker, self).__init__(parent_conn, child_conn) self.data_file = data_file self.host = host @@ -62,7 +65,10 @@ class Worker(EmittingProcess): return # launch server try: - server.app.run(host=self.host, debug=False, port=self.port, threaded=True) + server.app.run(host=self.host, + debug=False, + port=self.port, + threaded=True) except Exception as e: self.emit("server_error", str(e)) finally: @@ -70,6 +76,7 @@ class Worker(EmittingProcess): class SiteReadyWorker: + def __init__(self, location): super(SiteReadyWorker, self).__init__() self.signals = SiteReadySignals() diff --git a/server/test/decode_fbs.py b/server/test/decode_fbs.py index a9b847ad..1e417dfd 100644 --- a/server/test/decode_fbs.py +++ b/server/test/decode_fbs.py @@ -1,4 +1,3 @@ - """ Code to decode, for testing purposes, the flatbuffer encoded blobs. This code will need to be updated if fbs/matrix.fbs changes. @@ -18,18 +17,23 @@ import server.app.util.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray def decode_typed_array(tarr): type_map = { - TypedArray.TypedArray.Uint32Array: Uint32Array.Uint32Array, - TypedArray.TypedArray.Int32Array: Int32Array.Int32Array, - TypedArray.TypedArray.Float32Array: Float32Array.Float32Array, - TypedArray.TypedArray.Float64Array: Float64Array.Float64Array, - TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray + TypedArray.TypedArray.Uint32Array: + Uint32Array.Uint32Array, + TypedArray.TypedArray.Int32Array: + Int32Array.Int32Array, + TypedArray.TypedArray.Float32Array: + Float32Array.Float32Array, + TypedArray.TypedArray.Float64Array: + Float64Array.Float64Array, + TypedArray.TypedArray.JSONEncodedArray: + JSONEncodedArray.JSONEncodedArray } (u_type, u) = tarr if u_type == TypedArray.TypedArray.NONE: return None TarType = type_map.get(u_type, None) - assert(TarType is not None) + assert (TarType is not None) arr = TarType() arr.Init(u.Bytes, u.Pos) diff --git a/server/test/test_api.py b/server/test/test_api.py index b2c581aa..42b6608b 100644 --- a/server/test/test_api.py +++ b/server/test/test_api.py @@ -19,7 +19,10 @@ class EndPoints(unittest.TestCase): @classmethod def setUpClass(cls): - cls.ps = Popen(["cellxgene", "launch", "../example-dataset/pbmc3k.h5ad", "--verbose", "--port", "5005"]) + cls.ps = Popen([ + "cellxgene", "launch", "../example-dataset/pbmc3k.h5ad", + "--verbose", "--port", "5005" + ]) session = requests.Session() for i in range(90): try: @@ -47,7 +50,8 @@ class EndPoints(unittest.TestCase): result_data = result.json() self.assertEqual(result_data["schema"]["dataframe"]["nObs"], 2638) self.assertEqual(len(result_data["schema"]["annotations"]["obs"]), 2) - self.assertEqual(len(result_data["schema"]["annotations"]["obs"]["columns"]), 5) + self.assertEqual( + len(result_data["schema"]["annotations"]["obs"]["columns"]), 5) def test_config(self): endpoint = "config" @@ -57,7 +61,8 @@ class EndPoints(unittest.TestCase): self.assertEqual(result.headers["Content-Type"], "application/json") result_data = result.json() self.assertIn("library_versions", result_data["config"]) - self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k") + self.assertEqual(result_data["config"]["displayNames"]["dataset"], + "pbmc3k") self.assertEqual(len(result_data["config"]["features"]), 4) def test_get_layout_fbs(self): @@ -66,13 +71,15 @@ class EndPoints(unittest.TestCase): header = {"Accept": "application/octet-stream"} result = self.session.get(url, headers=header) self.assertEqual(result.status_code, HTTPStatus.OK) - self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + self.assertEqual(result.headers["Content-Type"], + "application/octet-stream") df = decode_fbs.decode_matrix_FBS(result.content) self.assertEqual(df['n_rows'], 2638) self.assertEqual(df['n_cols'], 8) self.assertIsNotNone(df['columns']) self.assertListEqual(df['col_idx'], [ - 'pca_0', 'pca_1', 'tsne_0', 'tsne_1', 'umap_0', 'umap_1', 'draw_graph_fr_0', 'draw_graph_fr_1' + 'pca_0', 'pca_1', 'tsne_0', 'tsne_1', 'umap_0', 'umap_1', + 'draw_graph_fr_0', 'draw_graph_fr_1' ]) self.assertIsNone(df['row_idx']) self.assertEqual(len(df['columns']), df['n_cols']) @@ -89,7 +96,8 @@ class EndPoints(unittest.TestCase): header = {"Accept": "application/octet-stream"} result = self.session.get(url, headers=header) self.assertEqual(result.status_code, HTTPStatus.OK) - self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + self.assertEqual(result.headers["Content-Type"], + "application/octet-stream") df = decode_fbs.decode_matrix_FBS(result.content) self.assertEqual(df['n_rows'], 2638) self.assertEqual(df['n_cols'], 5) @@ -97,8 +105,11 @@ class EndPoints(unittest.TestCase): self.assertIsNotNone(df['col_idx']) self.assertIsNone(df['row_idx']) self.assertEqual(len(df['columns']), df['n_cols']) - obs_index_col_name = self.schema["schema"]["annotations"]["obs"]["index"] - self.assertListEqual(df['col_idx'], [obs_index_col_name, 'n_genes', 'percent_mito', 'n_counts', 'louvain']) + obs_index_col_name = self.schema["schema"]["annotations"]["obs"][ + "index"] + self.assertListEqual(df['col_idx'], [ + obs_index_col_name, 'n_genes', 'percent_mito', 'n_counts', 'louvain' + ]) def test_get_annotations_obs_keys_fbs(self): endpoint = "annotations/obs" @@ -107,7 +118,8 @@ class EndPoints(unittest.TestCase): header = {"Accept": "application/octet-stream"} result = self.session.get(url, headers=header) self.assertEqual(result.status_code, HTTPStatus.OK) - self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + self.assertEqual(result.headers["Content-Type"], + "application/octet-stream") df = decode_fbs.decode_matrix_FBS(result.content) self.assertEqual(df['n_rows'], 2638) self.assertEqual(df['n_cols'], 2) @@ -129,8 +141,26 @@ class EndPoints(unittest.TestCase): url = f"{URL_BASE}{endpoint}" params = { "mode": "topN", - "set1": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["NK cells"]}]}}}, - "set2": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["CD8 T cells"]}]}}}, + "set1": { + "filter": { + "obs": { + "annotation_value": [{ + "name": "louvain", + "values": ["NK cells"] + }] + } + } + }, + "set2": { + "filter": { + "obs": { + "annotation_value": [{ + "name": "louvain", + "values": ["CD8 T cells"] + }] + } + } + }, "count": 7, } result = self.session.post(url, json=params) @@ -145,8 +175,20 @@ class EndPoints(unittest.TestCase): params = { "mode": "topN", "count": 10, - "set1": {"filter": {"obs": {"index": [[0, 500]]}}}, - "set2": {"filter": {"obs": {"index": [[500, 1000]]}}}, + "set1": { + "filter": { + "obs": { + "index": [[0, 500]] + } + } + }, + "set2": { + "filter": { + "obs": { + "index": [[500, 1000]] + } + } + }, } result = self.session.post(url, json=params) self.assertEqual(result.status_code, HTTPStatus.OK) @@ -160,7 +202,8 @@ class EndPoints(unittest.TestCase): header = {"Accept": "application/octet-stream"} result = self.session.get(url, headers=header) self.assertEqual(result.status_code, HTTPStatus.OK) - self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + self.assertEqual(result.headers["Content-Type"], + "application/octet-stream") df = decode_fbs.decode_matrix_FBS(result.content) self.assertEqual(df['n_rows'], 1838) self.assertEqual(df['n_cols'], 2) @@ -168,7 +211,8 @@ class EndPoints(unittest.TestCase): self.assertIsNotNone(df['col_idx']) self.assertIsNone(df['row_idx']) self.assertEqual(len(df['columns']), df['n_cols']) - var_index_col_name = self.schema["schema"]["annotations"]["var"]["index"] + var_index_col_name = self.schema["schema"]["annotations"]["var"][ + "index"] self.assertListEqual(df['col_idx'], [var_index_col_name, 'n_cells']) def test_get_annotations_var_keys_fbs(self): @@ -178,7 +222,8 @@ class EndPoints(unittest.TestCase): header = {"Accept": "application/octet-stream"} result = self.session.get(url, headers=header) self.assertEqual(result.status_code, HTTPStatus.OK) - self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + self.assertEqual(result.headers["Content-Type"], + "application/octet-stream") df = decode_fbs.decode_matrix_FBS(result.content) self.assertEqual(df['n_rows'], 1838) self.assertEqual(df['n_cols'], 1) @@ -207,7 +252,8 @@ class EndPoints(unittest.TestCase): url = f"{URL_BASE}{endpoint}" result = self.session.put(url) self.assertEqual(result.status_code, HTTPStatus.OK) - self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + self.assertEqual(result.headers["Content-Type"], + "application/octet-stream") def test_data_put_fbs(self): endpoint = f"data/var" @@ -215,7 +261,8 @@ class EndPoints(unittest.TestCase): header = {"Accept": "application/octet-stream"} result = self.session.put(url, headers=header) self.assertEqual(result.status_code, HTTPStatus.OK) - self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + self.assertEqual(result.headers["Content-Type"], + "application/octet-stream") df = decode_fbs.decode_matrix_FBS(result.content) self.assertEqual(df['n_rows'], 2638) self.assertEqual(df['n_cols'], 1838) @@ -228,16 +275,11 @@ class EndPoints(unittest.TestCase): endpoint = f"data/var" url = f"{URL_BASE}{endpoint}" header = {"Accept": "application/octet-stream"} - filter = { - "filter": { - "var": { - "index": [0, 1, 4] - } - } - } + filter = {"filter": {"var": {"index": [0, 1, 4]}}} result = self.session.put(url, headers=header, json=filter) self.assertEqual(result.status_code, HTTPStatus.OK) - self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + self.assertEqual(result.headers["Content-Type"], + "application/octet-stream") df = decode_fbs.decode_matrix_FBS(result.content) self.assertEqual(df['n_rows'], 2638) self.assertEqual(df['n_cols'], 3) @@ -252,10 +294,20 @@ class EndPoints(unittest.TestCase): url = f"{URL_BASE}{endpoint}" header = {"Accept": "application/octet-stream"} index_col_name = self.schema["schema"]["annotations"]["var"]["index"] - var_filter = {"filter": {"var": {"annotation_value": [{"name": index_col_name, "values": ["RER1"]}]}}} + var_filter = { + "filter": { + "var": { + "annotation_value": [{ + "name": index_col_name, + "values": ["RER1"] + }] + } + } + } result = self.session.put(url, headers=header, json=var_filter) self.assertEqual(result.status_code, HTTPStatus.OK) - self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + self.assertEqual(result.headers["Content-Type"], + "application/octet-stream") df = decode_fbs.decode_matrix_FBS(result.content) self.assertEqual(df["n_rows"], 2638) self.assertEqual(df["n_cols"], 1) diff --git a/server/test/test_fbs.py b/server/test/test_fbs.py index da32903a..d4efb279 100644 --- a/server/test/test_fbs.py +++ b/server/test/test_fbs.py @@ -40,49 +40,52 @@ class FbsTests(unittest.TestCase): def test_encode_DataFrame(self): df = pd.DataFrame( data={ - 'a': np.zeros((10,), dtype=np.float32), - 'b': np.ones((10,), dtype=np.int64), - 'c': np.array([i for i in range(0, 10)], dtype=np.uint16), - 'd': pd.Series(['x', 'y', 'z', 'x', 'y', 'z', 'a', 'x', 'y', 'z'], dtype='category') + 'a': + np.zeros((10,), dtype=np.float32), + 'b': + np.ones((10,), dtype=np.int64), + 'c': + np.array([i for i in range(0, 10)], dtype=np.uint16), + 'd': + pd.Series( + ['x', 'y', 'z', 'x', 'y', 'z', 'a', 'x', 'y', 'z'], + dtype='category') }) - expected_types = ( - (np.ndarray, np.float32), - (np.ndarray, np.int32), - (np.ndarray, np.uint32), - (list, None) - ) + expected_types = ((np.ndarray, np.float32), (np.ndarray, np.int32), + (np.ndarray, np.uint32), (list, None)) fbs = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns) self.fbs_checks(fbs, (10, 4), expected_types, ['a', 'b', 'c', 'd']) def test_encode_ndarray(self): arr = np.zeros((3, 2), dtype=np.float32) - expected_types = ( - (np.ndarray, np.float32), - (np.ndarray, np.float32), - (np.ndarray, np.float32) - ) + expected_types = ((np.ndarray, np.float32), (np.ndarray, np.float32), + (np.ndarray, np.float32)) fbs = encode_matrix_fbs(matrix=arr, row_idx=None, col_idx=None) self.fbs_checks(fbs, (3, 2), expected_types, None) def test_encode_sparse(self): csc = sparse.csc_matrix(np.array([[0, 1, 2], [3, 0, 4]])) - expected_types = ( - (np.ndarray, np.int32), - (np.ndarray, np.int32), - (np.ndarray, np.int32) - ) + expected_types = ((np.ndarray, np.int32), (np.ndarray, np.int32), + (np.ndarray, np.int32)) fbs = encode_matrix_fbs(matrix=csc, row_idx=None, col_idx=None) self.fbs_checks(fbs, (2, 3), expected_types, None) def test_roundtrip(self): dfSrc = pd.DataFrame( data={ - 'a': np.zeros((10,), dtype=np.float32), - 'b': np.ones((10,), dtype=np.int64), - 'c': np.array([i for i in range(0, 10)], dtype=np.uint16), - 'd': pd.Series(['x', 'y', 'z', 'x', 'y', 'z', 'a', 'x', 'y', 'z'], dtype='category') + 'a': + np.zeros((10,), dtype=np.float32), + 'b': + np.ones((10,), dtype=np.int64), + 'c': + np.array([i for i in range(0, 10)], dtype=np.uint16), + 'd': + pd.Series( + ['x', 'y', 'z', 'x', 'y', 'z', 'a', 'x', 'y', 'z'], + dtype='category') }) - dfDst = decode_matrix_fbs(encode_matrix_fbs(matrix=dfSrc, col_idx=dfSrc.columns)) + dfDst = decode_matrix_fbs( + encode_matrix_fbs(matrix=dfSrc, col_idx=dfSrc.columns)) self.assertEqual(dfSrc.shape, dfDst.shape) self.assertEqual(set(dfSrc.columns), set(dfDst.columns)) for c in dfSrc.columns: diff --git a/server/test/test_matrix_proxy.py b/server/test/test_matrix_proxy.py index 624cbda2..fad8fae5 100644 --- a/server/test/test_matrix_proxy.py +++ b/server/test/test_matrix_proxy.py @@ -7,12 +7,14 @@ class NdArrayProxyView(MatrixProxyView): """ Fake test class for matrix proxy - wraps ndarray """ + @classmethod def __supports__(cls): - return ('numpy.ndarray', ) + return ('numpy.ndarray',) class MatrixProxyViewTest(unittest.TestCase): + def test_ismatrixproxy(self): n = np.zeros((2, 4)) mp = MatrixProxy.create(n) @@ -41,18 +43,13 @@ class MatrixProxyViewTest(unittest.TestCase): def test_toarray(self): n = np.arange(15, dtype=np.float32).reshape((3, 5)) mp = MatrixProxy.create(n) - self.assertTrue(np.all(mp.toarray() == [ - [0., 1., 2., 3., 4.], - [5., 6., 7., 8., 9.], - [10., 11., 12., 13., 14.] - ])) - self.assertTrue(np.all(mp.T.toarray() == [ - [0., 5., 10.], - [1., 6., 11.], - [2., 7., 12.], - [3., 8., 13.], - [4., 9., 14.] - ])) + self.assertTrue( + np.all(mp.toarray() == [[0., 1., 2., 3., 4.], [5., 6., 7., 8., 9.], + [10., 11., 12., 13., 14.]])) + self.assertTrue( + np.all( + mp.T.toarray() == [[0., 5., 10.], [1., 6., 11.], [2., 7., 12.], + [3., 8., 13.], [4., 9., 14.]])) def test_indexing(self): """ @@ -95,47 +92,25 @@ class MatrixProxyViewTest(unittest.TestCase): # slice, slice - self.assertTrue(np.all(mp[1:3, 2:4].toarray() == [ - [7, 8], - [12, 13] - ])) - self.assertTrue(np.all(mp[:3, :4].toarray() == [ - [0., 1., 2., 3.], - [5., 6., 7., 8.], - [10., 11., 12., 13.] - ])) - self.assertTrue(np.all(mp[::-1, ::-1].toarray() == [ - [14, 13, 12, 11, 10], - [9, 8, 7, 6, 5], - [4, 3, 2, 1, 0] - ])) - self.assertTrue(np.all(mp[::-2, ::-2].toarray() == [ - [14, 12, 10], - [4, 2, 0] - ])) + self.assertTrue(np.all(mp[1:3, 2:4].toarray() == [[7, 8], [12, 13]])) + self.assertTrue( + np.all(mp[:3, :4].toarray() == [[0., 1., 2., 3.], [5., 6., 7., 8.], + [10., 11., 12., 13.]])) + self.assertTrue( + np.all(mp[::-1, ::-1].toarray() == + [[14, 13, 12, 11, 10], [9, 8, 7, 6, 5], [4, 3, 2, 1, 0]])) + self.assertTrue( + np.all(mp[::-2, ::-2].toarray() == [[14, 12, 10], [4, 2, 0]])) - self.assertTrue(np.all(mp.T[2:4, 1:3].toarray() == [ - [7, 12], - [8, 13] - ])) - self.assertTrue(np.all(mp.T[:4, :3].toarray() == [ - [0, 5, 10], - [1, 6, 11], - [2, 7, 12], - [3, 8, 13] - ])) - self.assertTrue(np.all(mp.T[::-1, ::-1].toarray() == [ - [14, 9, 4], - [13, 8, 3], - [12, 7, 2], - [11, 6, 1], - [10, 5, 0] - ])) - self.assertTrue(np.all(mp.T[::-2, ::-2].toarray() == [ - [14, 4], - [12, 2], - [10, 0] - ])) + self.assertTrue(np.all(mp.T[2:4, 1:3].toarray() == [[7, 12], [8, 13]])) + self.assertTrue( + np.all(mp.T[:4, :3].toarray() == [[0, 5, 10], [1, 6, 11], + [2, 7, 12], [3, 8, 13]])) + self.assertTrue( + np.all(mp.T[::-1, ::-1].toarray( + ) == [[14, 9, 4], [13, 8, 3], [12, 7, 2], [11, 6, 1], [10, 5, 0]])) + self.assertTrue( + np.all(mp.T[::-2, ::-2].toarray() == [[14, 4], [12, 2], [10, 0]])) def test_repeated_indexing(self): """ @@ -149,31 +124,16 @@ class MatrixProxyViewTest(unittest.TestCase): self.assertEqual(mp[0][1], 1) self.assertEqual(mp.T[0][1], 5) - self.assertTrue(np.all(mp[0::-1, ::-1][0, 2:4].toarray() == [ - 2, 1 - ])) - self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 1:3:1].toarray() == [ - 2, 3 - ])) - self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 2:0:-1].toarray() == [ - 3, 2 - ])) - self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, 1:3:1].toarray() == [ - 3, 2 - ])) - self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, 2:0:-1].toarray() == [ - 2, 3 - ])) + self.assertTrue(np.all(mp[0::-1, ::-1][0, 2:4].toarray() == [2, 1])) + self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 1:3:1].toarray() == [2, 3])) + self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 2:0:-1].toarray() == [3, 2])) + self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, 1:3:1].toarray() == [3, 2])) + self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, + 2:0:-1].toarray() == [2, 3])) - self.assertTrue(np.all(mp.T[::-1, 0::-1][2:4, 0].toarray() == [ - 2, 1 - ])) - self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 1:3:1].toarray() == [ - 10 - ])) - self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 2:0:-1].toarray() == [ - 10 - ])) + self.assertTrue(np.all(mp.T[::-1, 0::-1][2:4, 0].toarray() == [2, 1])) + self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 1:3:1].toarray() == [10])) + self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 2:0:-1].toarray() == [10])) self.assertTrue(np.all(mp.T[0::-1, 5:1:-1][0, 1:3:1].toarray() == [])) self.assertTrue(np.all(mp.T[0::-1, 5:1:-1][0, 2:0:-1].toarray() == [])) @@ -190,20 +150,12 @@ class MatrixProxyViewTest(unittest.TestCase): self.assertEqual(mp[0, 0], 0) # drop 1 dimension, to an array - self.assertTrue(np.all(mp[0, :].toarray() == [ - 0, 1, 2, 3, 4 - ])) - self.assertTrue(np.all(mp[:, 0].toarray() == [ - 0, 5, 10 - ])) + self.assertTrue(np.all(mp[0, :].toarray() == [0, 1, 2, 3, 4])) + self.assertTrue(np.all(mp[:, 0].toarray() == [0, 5, 10])) # with .T - self.assertTrue(np.all(mp[0:2].T[-1:].toarray() == [ - [4, 9] - ])) - self.assertTrue(np.all(mp[0:2].T[-1].toarray() == [ - 4, 9 - ])) + self.assertTrue(np.all(mp[0:2].T[-1:].toarray() == [[4, 9]])) + self.assertTrue(np.all(mp[0:2].T[-1].toarray() == [4, 9])) def test_iter(self): """ @@ -214,17 +166,13 @@ class MatrixProxyViewTest(unittest.TestCase): rows = [r for r in mp] self.assertEqual(len(rows), 3) - self.assertTrue(np.all(rows[0].toarray() == [ - 0, 1, 2, 3, 4 - ])) + self.assertTrue(np.all(rows[0].toarray() == [0, 1, 2, 3, 4])) for i, r in enumerate(rows): self.assertTrue(np.all(mp[i].toarray() == r.toarray())) cols = [c for c in mp.T] self.assertEqual(len(cols), 5) - self.assertTrue(np.all(cols[0].toarray() == [ - 0, 5, 10 - ])) + self.assertTrue(np.all(cols[0].toarray() == [0, 5, 10])) for i, c in enumerate(cols): self.assertTrue(np.all(mp.T[i].toarray() == c.toarray())) diff --git a/server/test/test_nan_rest.py b/server/test/test_nan_rest.py index 45895811..61b2c4c4 100644 --- a/server/test/test_nan_rest.py +++ b/server/test/test_nan_rest.py @@ -20,9 +20,10 @@ class WithNaNs(unittest.TestCase): @classmethod def setUpClass(cls): - cls.ps = Popen( - ["cellxgene", "launch", "test/test_datasets/nan.h5ad", "--verbose", "--port", "5006"] - ) + cls.ps = Popen([ + "cellxgene", "launch", "test/test_datasets/nan.h5ad", "--verbose", + "--port", "5006" + ]) session = requests.Session() for i in range(90): try: @@ -51,7 +52,8 @@ class WithNaNs(unittest.TestCase): url = f"{URL_BASE}{endpoint}" result = self.session.put(url) self.assertEqual(result.status_code, HTTPStatus.OK) - self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + self.assertEqual(result.headers["Content-Type"], + "application/octet-stream") df = decode_fbs.decode_matrix_FBS(result.content) self.assertTrue(math.isnan(df["columns"][3][3])) @@ -60,7 +62,8 @@ class WithNaNs(unittest.TestCase): url = f"{URL_BASE}{endpoint}" result = self.session.get(url) self.assertEqual(result.status_code, HTTPStatus.OK) - self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + self.assertEqual(result.headers["Content-Type"], + "application/octet-stream") df = decode_fbs.decode_matrix_FBS(result.content) self.assertTrue(math.isnan(df["columns"][2][0])) @@ -69,6 +72,7 @@ class WithNaNs(unittest.TestCase): url = f"{URL_BASE}{endpoint}" result = self.session.get(url) self.assertEqual(result.status_code, HTTPStatus.OK) - self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + self.assertEqual(result.headers["Content-Type"], + "application/octet-stream") df = decode_fbs.decode_matrix_FBS(result.content) self.assertTrue(math.isnan(df["columns"][2][0])) diff --git a/server/test/test_nan_scanpy_engine.py b/server/test/test_nan_scanpy_engine.py index 7c193f55..27af1c9c 100644 --- a/server/test/test_nan_scanpy_engine.py +++ b/server/test/test_nan_scanpy_engine.py @@ -11,6 +11,7 @@ from server.app.util.data_locator import DataLocator class NaNTest(unittest.TestCase): + def setUp(self): self.args = { "layout": ["umap"], @@ -21,7 +22,8 @@ class NaNTest(unittest.TestCase): } with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UserWarning) - self.data = ScanpyEngine(DataLocator("test/test_datasets/nan.h5ad"), self.args) + self.data = ScanpyEngine(DataLocator("test/test_datasets/nan.h5ad"), + self.args) self.data._create_schema() def test_load(self): @@ -35,7 +37,8 @@ class NaNTest(unittest.TestCase): self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon) def test_dataframe(self): - data_frame_var = decode_fbs.decode_matrix_FBS(self.data.data_frame_to_fbs_matrix(None, "var")) + data_frame_var = decode_fbs.decode_matrix_FBS( + self.data.data_frame_to_fbs_matrix(None, "var")) self.assertIsNotNone(data_frame_var) self.assertEqual(data_frame_var["n_rows"], 100) self.assertEqual(data_frame_var["n_cols"], 100) @@ -44,30 +47,29 @@ class NaNTest(unittest.TestCase): with pytest.raises(FilterError): self.data.data_frame_to_fbs_matrix("an erroneous filter", "var") with pytest.raises(FilterError): - filter_ = { - "filter": { - "obs": {"index": [1, 99, [200, 300]]} - } - } + filter_ = {"filter": {"obs": {"index": [1, 99, [200, 300]]}}} self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") def test_dataframe_obs_not_implemented(self): with self.assertRaises(ValueError) as cm: - decode_fbs.decode_matrix_FBS(self.data.data_frame_to_fbs_matrix(None, "obs")) + decode_fbs.decode_matrix_FBS( + self.data.data_frame_to_fbs_matrix(None, "obs")) self.assertIsNotNone(cm.exception) def test_annotation(self): - annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("obs")) + annotations = decode_fbs.decode_matrix_FBS( + self.data.annotation_to_fbs_matrix("obs")) obs_index_col_name = self.data.schema["annotations"]["obs"]["index"] - self.assertEqual( - annotations["col_idx"], - [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"] - ) + self.assertEqual(annotations["col_idx"], [ + obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain" + ]) self.assertEqual(annotations["n_rows"], 100) self.assertTrue(math.isnan(annotations["columns"][2][0])) - annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("var")) + annotations = decode_fbs.decode_matrix_FBS( + self.data.annotation_to_fbs_matrix("var")) var_index_col_name = self.data.schema["annotations"]["var"]["index"] - self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells", "var_with_nans"]) + self.assertEqual(annotations["col_idx"], + [var_index_col_name, "n_cells", "var_with_nans"]) self.assertEqual(annotations["n_rows"], 100) self.assertTrue(math.isnan(annotations["columns"][2][0])) diff --git a/server/test/test_scanpy_engine.py b/server/test/test_scanpy_engine.py index bf81d125..6ae4c860 100644 --- a/server/test/test_scanpy_engine.py +++ b/server/test/test_scanpy_engine.py @@ -12,7 +12,6 @@ import pandas as pd from server.app.scanpy_engine.scanpy_engine import ScanpyEngine from server.app.util.errors import FilterError, DisabledFeatureError from server.app.util.data_locator import DataLocator - """ Test the scanpy engine using the pbmc3k data set. """ @@ -22,12 +21,12 @@ Test the scanpy engine using the pbmc3k data set. ("../example-dataset/pbmc3k.h5ad", False), ("test/test_datasets/pbmc3k-CSC-gz.h5ad", False), ("test/test_datasets/pbmc3k-CSR-gz.h5ad", False), - ("../example-dataset/pbmc3k.h5ad", True), ("test/test_datasets/pbmc3k-CSC-gz.h5ad", True), ("test/test_datasets/pbmc3k-CSR-gz.h5ad", True), ]) class EngineTest(unittest.TestCase): + def setUp(self): args = { "layout": ["umap"], @@ -47,10 +46,12 @@ class EngineTest(unittest.TestCase): self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon) def test_mandatory_annotations(self): - obs_index_col_name = self.data.get_schema()["annotations"]["obs"]["index"] + obs_index_col_name = self.data.get_schema( + )["annotations"]["obs"]["index"] self.assertIn(obs_index_col_name, self.data.data.obs) self.assertEqual(list(self.data.data.obs.index), list(range(2638))) - var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"] + var_index_col_name = self.data.get_schema( + )["annotations"]["var"]["index"] self.assertIn(var_index_col_name, self.data.data.var) self.assertEqual(list(self.data.data.var.index), list(range(1838))) @@ -64,11 +65,7 @@ class EngineTest(unittest.TestCase): self.data._validate_data_types() def test_filter_idx(self): - filter_ = { - "filter": { - "var": {"index": [1, 99, [200, 300]]} - } - } + filter_ = {"filter": {"var": {"index": [1, 99, [200, 300]]}}} fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") data = decode_fbs.decode_matrix_FBS(fbs) self.assertEqual(data["n_rows"], 2638) @@ -78,9 +75,10 @@ class EngineTest(unittest.TestCase): filter_ = { "filter": { "var": { - "annotation_value": [ - {"name": "n_cells", "min": 10} - ], + "annotation_value": [{ + "name": "n_cells", + "min": 10 + }], "index": [1, 99, [200, 300]] } } @@ -91,8 +89,12 @@ class EngineTest(unittest.TestCase): self.assertEqual(data["n_cols"], 91) def test_obs_and_var_names(self): - self.assertEqual(np.sum(self.data.data.var[self.data.get_schema()["annotations"]["var"]["index"]].isna()), 0) - self.assertEqual(np.sum(self.data.data.obs[self.data.get_schema()["annotations"]["obs"]["index"]].isna()), 0) + self.assertEqual( + np.sum(self.data.data.var[self.data.get_schema()["annotations"] + ["var"]["index"]].isna()), 0) + self.assertEqual( + np.sum(self.data.data.obs[self.data.get_schema()["annotations"] + ["obs"]["index"]].isna()), 0) def test_get_schema(self): with open(path.join(path.dirname(__file__), "schema.json")) as fh: @@ -110,7 +112,10 @@ class EngineTest(unittest.TestCase): def test_config(self): self.assertEqual( self.data.features["layout"]["obs"], - {"available": True, "interactiveLimit": 50000}, + { + "available": True, + "interactiveLimit": 50000 + }, ) def test_layout(self): @@ -129,18 +134,24 @@ class EngineTest(unittest.TestCase): annotations = decode_fbs.decode_matrix_FBS(fbs) self.assertEqual(annotations["n_rows"], 2638) self.assertEqual(annotations["n_cols"], 5) - obs_index_col_name = self.data.get_schema()["annotations"]["obs"]["index"] + obs_index_col_name = self.data.get_schema( + )["annotations"]["obs"]["index"] self.assertEqual( annotations["col_idx"], - [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"], + [ + obs_index_col_name, "n_genes", "percent_mito", "n_counts", + "louvain" + ], ) fbs = self.data.annotation_to_fbs_matrix("var") annotations = decode_fbs.decode_matrix_FBS(fbs) self.assertEqual(annotations['n_rows'], 1838) self.assertEqual(annotations['n_cols'], 2) - var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"] - self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells"]) + var_index_col_name = self.data.get_schema( + )["annotations"]["var"]["index"] + self.assertEqual(annotations["col_idx"], + [var_index_col_name, "n_cells"]) def test_annotation_fields(self): fbs = self.data.annotation_to_fbs_matrix("obs", ["n_genes", "n_counts"]) @@ -148,7 +159,8 @@ class EngineTest(unittest.TestCase): self.assertEqual(annotations["n_rows"], 2638) self.assertEqual(annotations['n_cols'], 2) - var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"] + var_index_col_name = self.data.get_schema( + )["annotations"]["var"]["index"] fbs = self.data.annotation_to_fbs_matrix("var", [var_index_col_name]) annotations = decode_fbs.decode_matrix_FBS(fbs) self.assertEqual(annotations['n_rows'], 1838) @@ -163,7 +175,8 @@ class EngineTest(unittest.TestCase): f2 = {"filter": {"obs": {"index": [[500, 1000]]}}} result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"])) self.assertEqual(len(result), 10) - result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20)) + result = json.loads( + self.data.diffexp_topN(f1["filter"], f2["filter"], 20)) self.assertEqual(len(result), 20) def test_data_frame(self): @@ -178,7 +191,14 @@ class EngineTest(unittest.TestCase): def test_filtered_data_frame(self): filter_ = { - "filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 100}]}} + "filter": { + "var": { + "annotation_value": [{ + "name": "n_cells", + "min": 100 + }] + } + } } fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") data = decode_fbs.decode_matrix_FBS(fbs) @@ -186,16 +206,29 @@ class EngineTest(unittest.TestCase): self.assertEqual(data["n_cols"], 1040) filter_ = { - "filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}} + "filter": { + "obs": { + "annotation_value": [{ + "name": "n_counts", + "min": 3000 + }] + } + } } with self.assertRaises(FilterError): self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") def test_data_named_gene(self): - var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"] + var_index_col_name = self.data.get_schema( + )["annotations"]["var"]["index"] filter_ = { "filter": { - "var": {"annotation_value": [{"name": var_index_col_name, "values": ["RER1"]}]} + "var": { + "annotation_value": [{ + "name": var_index_col_name, + "values": ["RER1"] + }] + } } } fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") @@ -206,7 +239,12 @@ class EngineTest(unittest.TestCase): filter_ = { "filter": { - "var": {"annotation_value": [{"name": var_index_col_name, "values": ["SPEN", "TYMP", "PRMT2"]}]} + "var": { + "annotation_value": [{ + "name": var_index_col_name, + "values": ["SPEN", "TYMP", "PRMT2"] + }] + } } } fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") diff --git a/server/test/test_scanpy_engine_data_load.py b/server/test/test_scanpy_engine_data_load.py index 0d03b070..90b33b96 100644 --- a/server/test/test_scanpy_engine_data_load.py +++ b/server/test/test_scanpy_engine_data_load.py @@ -10,6 +10,7 @@ class DataLoadEngineTest(unittest.TestCase): """ Test file loading, including deferred loading/update. """ + def setUp(self): self.data_file = DataLocator("../example-dataset/pbmc3k.h5ad") self.data = ScanpyEngine() @@ -52,7 +53,8 @@ class DataLoadEngineTest(unittest.TestCase): f2 = {"filter": {"obs": {"index": [[500, 1000]]}}} result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"])) self.assertEqual(len(result), 10) - result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20)) + result = json.loads( + self.data.diffexp_topN(f1["filter"], f2["filter"], 20)) self.assertEqual(len(result), 20) @@ -60,6 +62,7 @@ class DataLocatorEngineTest(unittest.TestCase): """ Test various types of data locators we expect to consume """ + def setUp(self): self.args = { "layout": ["umap"], diff --git a/server/test/test_writable_annotation.py b/server/test/test_writable_annotation.py index f63e8304..9fdc4145 100644 --- a/server/test/test_writable_annotation.py +++ b/server/test/test_writable_annotation.py @@ -14,6 +14,7 @@ from server.app.util.data_locator import DataLocator class WritableAnnotationTest(unittest.TestCase): + def setUp(self): self.tmpDir = tempfile.mkdtemp() self.annotations_file = path.join(self.tmpDir, "test_annotations.csv") @@ -27,7 +28,8 @@ class WritableAnnotationTest(unittest.TestCase): "annotations_file": self.annotations_file, "annotations_output_dir": None } - self.data = ScanpyEngine(DataLocator("../example-dataset/pbmc3k.h5ad"), args) + self.data = ScanpyEngine(DataLocator("../example-dataset/pbmc3k.h5ad"), + args) def tearDown(self): shutil.rmtree(self.tmpDir) @@ -41,7 +43,9 @@ class WritableAnnotationTest(unittest.TestCase): n_rows = self.data.data.obs.shape[0] fbs_bad = self.make_fbs({ - 'louvain': pd.Series(['undefined' for l in range(0, n_rows)], dtype='category') + 'louvain': + pd.Series(['undefined' for l in range(0, n_rows)], + dtype='category') }) # ensure attempt to change VAR annotation @@ -56,31 +60,49 @@ class WritableAnnotationTest(unittest.TestCase): # verify the file is written as expected n_rows = self.data.data.obs.shape[0] fbs = self.make_fbs({ - 'cat_A': pd.Series(['label_A' for l in range(0, n_rows)], dtype='category'), - 'cat_B': pd.Series(['label_B' for l in range(0, n_rows)], dtype='category') + 'cat_A': + pd.Series(['label_A' for l in range(0, n_rows)], + dtype='category'), + 'cat_B': + pd.Series(['label_B' for l in range(0, n_rows)], + dtype='category') }) res = self.data.annotation_put_fbs("obs", fbs) self.assertEqual(res, json.dumps({"status": "OK"})) self.assertTrue(path.exists(self.annotations_file)) - df = pd.read_csv(self.annotations_file, index_col=0, header=0, comment='#') + df = pd.read_csv(self.annotations_file, + index_col=0, + header=0, + comment='#') self.assertEqual(df.shape, (n_rows, 2)) self.assertEqual(set(df.columns), set(['cat_A', 'cat_B'])) self.assertTrue(self.data.original_obs_index.equals(df.index)) - self.assertTrue(np.all(df['cat_A'] == ['label_A' for l in range(0, n_rows)])) - self.assertTrue(np.all(df['cat_B'] == ['label_B' for l in range(0, n_rows)])) + self.assertTrue( + np.all(df['cat_A'] == ['label_A' for l in range(0, n_rows)])) + self.assertTrue( + np.all(df['cat_B'] == ['label_B' for l in range(0, n_rows)])) # verify complete overwrite on second attempt, AND rotation occurs fbs = self.make_fbs({ - 'cat_A': pd.Series(['label_A1' for l in range(0, n_rows)], dtype='category'), - 'cat_C': pd.Series(['label_C' for l in range(0, n_rows)], dtype='category') + 'cat_A': + pd.Series(['label_A1' for l in range(0, n_rows)], + dtype='category'), + 'cat_C': + pd.Series(['label_C' for l in range(0, n_rows)], + dtype='category') }) res = self.data.annotation_put_fbs("obs", fbs) self.assertEqual(res, json.dumps({"status": "OK"})) self.assertTrue(path.exists(self.annotations_file)) - df = pd.read_csv(self.annotations_file, index_col=0, header=0, comment='#') + df = pd.read_csv(self.annotations_file, + index_col=0, + header=0, + comment='#') self.assertEqual(set(df.columns), set(['cat_A', 'cat_C'])) - self.assertTrue(np.all(df['cat_A'] == ['label_A1' for l in range(0, n_rows)])) - self.assertTrue(np.all(df['cat_C'] == ['label_C' for l in range(0, n_rows)])) + self.assertTrue( + np.all(df['cat_A'] == ['label_A1' for l in range(0, n_rows)])) + self.assertTrue( + np.all(df['cat_C'] == ['label_C' for l in range(0, n_rows)])) # rotation name, ext = path.splitext(self.annotations_file) @@ -93,8 +115,12 @@ class WritableAnnotationTest(unittest.TestCase): # verify we stop rotation at 9 n_rows = self.data.data.obs.shape[0] fbs = self.make_fbs({ - 'cat_A': pd.Series(['label_A' for l in range(0, n_rows)], dtype='category'), - 'cat_B': pd.Series(['label_B' for l in range(0, n_rows)], dtype='category') + 'cat_A': + pd.Series(['label_A' for l in range(0, n_rows)], + dtype='category'), + 'cat_B': + pd.Series(['label_B' for l in range(0, n_rows)], + dtype='category') }) for i in range(0, 11): res = self.data.annotation_put_fbs("obs", fbs) @@ -112,8 +138,12 @@ class WritableAnnotationTest(unittest.TestCase): n_rows = self.data.data.obs.shape[0] fbs = self.make_fbs({ - 'cat_A': pd.Series(['label_A' for l in range(0, n_rows)], dtype='category'), - 'cat_B': pd.Series(['label_B' for l in range(0, n_rows)], dtype='category') + 'cat_A': + pd.Series(['label_A' for l in range(0, n_rows)], + dtype='category'), + 'cat_B': + pd.Series(['label_B' for l in range(0, n_rows)], + dtype='category') }) # put @@ -129,27 +159,30 @@ class WritableAnnotationTest(unittest.TestCase): self.assertEqual(annotations["n_cols"], 7) self.assertIsNone(annotations["row_idx"]) self.assertEqual(annotations["col_idx"], [ - obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain", "cat_A", "cat_B" + obs_index_col_name, "n_genes", "percent_mito", "n_counts", + "louvain", "cat_A", "cat_B" ]) col_idx = annotations["col_idx"] - self.assertEqual(annotations["columns"][col_idx.index('cat_A')], [ - 'label_A' for l in range(0, n_rows) - ]) - self.assertEqual(annotations["columns"][col_idx.index('cat_B')], [ - 'label_B' for l in range(0, n_rows) - ]) + self.assertEqual(annotations["columns"][col_idx.index('cat_A')], + ['label_A' for l in range(0, n_rows)]) + self.assertEqual(annotations["columns"][col_idx.index('cat_B')], + ['label_B' for l in range(0, n_rows)]) # verify the schema was updated - all_col_schema = {c["name"]: c for c in schema["annotations"]["obs"]["columns"]} - self.assertEqual(all_col_schema["cat_A"], { - "name": "cat_A", - "type": "categorical", - "categories": ["label_A"], - "writable": True - }) - self.assertEqual(all_col_schema["cat_B"], { - "name": "cat_B", - "type": "categorical", - "categories": ["label_B"], - "writable": True - }) + all_col_schema = { + c["name"]: c for c in schema["annotations"]["obs"]["columns"] + } + self.assertEqual( + all_col_schema["cat_A"], { + "name": "cat_A", + "type": "categorical", + "categories": ["label_A"], + "writable": True + }) + self.assertEqual( + all_col_schema["cat_B"], { + "name": "cat_B", + "type": "categorical", + "categories": ["label_B"], + "writable": True + }) diff --git a/server/utils/utils.py b/server/utils/utils.py index 417aeb7b..c8e473c0 100644 --- a/server/utils/utils.py +++ b/server/utils/utils.py @@ -12,12 +12,15 @@ def find_available_port(host, port=5005): for port_to_try in range(port, port + num_ports_to_try): if is_port_available(host, port_to_try): return port_to_try - raise socket.error(errno.EADDRINUSE, f"No port in range {port} - {port + num_ports_to_try - 1} available.") + raise socket.error( + errno.EADDRINUSE, + f"No port in range {port} - {port + num_ports_to_try - 1} available.") def is_port_available(host, port): is_available = False - with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: + with contextlib.closing(socket.socket(socket.AF_INET, + socket.SOCK_STREAM)) as s: try: s.bind((host, port)) is_available = True