diff --git a/docs/faq.md b/docs/faq.md index e2db7c7a..013bce77 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -40,6 +40,14 @@ Currently this is not supported directly, but you should be able to do this your - `.X` is used to display expression (histograms, scatterplot & colorscale) and to compute differential expression - `.obsm` is used for layout +#### I have a BIG dataset - how can I make cellxgene run as fast as possible? + +If your dataset requires gigabytes of disk space, you may need to select an appropriate storage format in order to effectively utilize `cellxgene`. Tips and tricks: + +- `cellxgene` is optimized for columnar data access. For large datasets, format the expression matrix (`.X`) as either a [SciPy CSC sparse matrix](https://docs.scipy.org/doc/scipy/reference/generated/scipy.sparse.csc_matrix.html) or a dense Numpy array (whichever creates a smaller `h5ad` file). If you are using `cellxgene prepare`, include the `--sparse` flag to ensure `.X` is formatted as a CSC sparse matrix (by default, `.X` will be a dense matrix). +- `cellxgene` start time is directly proportional to `h5ad` file size and the speed of your file system. Expect that large (eg, million cell) datasets will take minutes to load, even on relatively fast computers with a high performance local hard drive. Once loaded, exploring metadata should still be quick. +- If your dataset size exceeds the size of memory (RAM) on the host computer, differential expression calculations will be extremely slow (or fail, if you run out of virtual memory). + # Algorithms #### How are you computing and sorting differential expression results? diff --git a/server/app/scanpy_engine/scanpy_engine.py b/server/app/scanpy_engine/scanpy_engine.py index 8519b225..e6446693 100644 --- a/server/app/scanpy_engine/scanpy_engine.py +++ b/server/app/scanpy_engine/scanpy_engine.py @@ -3,6 +3,7 @@ import warnings import numpy as np from pandas.core.dtypes.dtypes import CategoricalDtype import scanpy as sc +from scipy import sparse from server.app.driver.driver import CXGDriver from server.app.util.constants import Axis, DEFAULT_TOP_N @@ -291,6 +292,22 @@ class ScanpyEngine(CXGDriver): df = df[fields] return encode_matrix_fbs(df, col_idx=df.columns) + @staticmethod + def slice_columns(X, var_mask): + """ + Slice columns from the matrix X, as specified by the mask + Semantically equivalent to X[:, var_mask], but handles sparse + matrices in a more performant manner. + """ + if var_mask is None: # noop + return X + if sparse.issparse(X): # use tuned getcol/hstack for performance + indices = np.nonzero(var_mask)[0] + cols = [X.getcol(i) for i in indices] + return sparse.hstack(cols) + else: # else, just use standard slicing, which is fine for dense arrays + return X[:, var_mask] + @requires_data def data_frame_to_fbs_matrix(self, filter, axis): """ @@ -313,9 +330,7 @@ class ScanpyEngine(CXGDriver): raise FilterError("filtering on obs unsupported") # Currently only handles VAR dimension - X = self.data._X - if var_selector is not None: - X = X[:, var_selector] + X = self.slice_columns(self.data._X, var_selector) return encode_matrix_fbs(X, col_idx=np.nonzero(var_selector)[0], row_idx=None) @requires_data