diff --git a/server/app/scanpy_engine/scanpy_engine.py b/server/app/scanpy_engine/scanpy_engine.py index 58c0843d..092a9b72 100644 --- a/server/app/scanpy_engine/scanpy_engine.py +++ b/server/app/scanpy_engine/scanpy_engine.py @@ -2,7 +2,7 @@ import warnings import numpy as np from pandas.core.dtypes.dtypes import CategoricalDtype -import scanpy as sc +import anndata from scipy import sparse from server.app.driver.driver import CXGDriver @@ -140,11 +140,10 @@ class ScanpyEngine(CXGDriver): self.schema["annotations"][ax].append(ann_schema) def _load_data(self, data): - # Based on benchmarking, cache=True has no impact on perf. - # Note: as of current scanpy/anndata release, setting backed='r' will - # result in an error. https://github.com/theislab/anndata/issues/79 + # as of AnnData 0.6.19, backed mode performs initial load fast, but at the + # cost of significantly slower access to X data. try: - self.data = sc.read(data, cache=True) + self.data = anndata.read_h5ad(data) except ValueError: raise ScanpyFileError( "File must be in the .h5ad format. Please read " @@ -174,6 +173,11 @@ class ScanpyEngine(CXGDriver): @requires_data def _validate_data_types(self): + if sparse.isspmatrix(self.data.X) and not sparse.isspmatrix_csc(self.data.X): + warnings.warn( + f"Scanpy data matrix is sparse, but not a CSC (columnar) matrix. " + f"Performance may be improved by using CSC." + ) if self.data.X.dtype != "float32": warnings.warn( f"Scanpy data matrix is in {self.data.X.dtype} format not float32. " diff --git a/server/cli/launch.py b/server/cli/launch.py index 402fb431..1c4f1005 100644 --- a/server/cli/launch.py +++ b/server/cli/launch.py @@ -1,11 +1,12 @@ import logging from os import devnull -from os.path import splitext, basename +from os.path import splitext, basename, getsize import sys import warnings import webbrowser import click +import psutil from server.app.app import Server from server.app.util.errors import ScanpyFileError @@ -13,6 +14,10 @@ from server.app.util.utils import custom_format_warning from server.utils.constants import MODES +# anything bigger than this will generate a special message +BIG_FILE_SIZE_THRESHOLD = 100 * 2**20 # 100MB + + @click.command() @click.argument("data", metavar="", type=click.Path(exists=True, file_okay=True, dir_okay=False)) @click.option( @@ -148,7 +153,16 @@ security risk by including the --scripts flag. Make sure you trust the scripts t log = logging.getLogger("werkzeug") log.setLevel(logging.ERROR) - click.echo(f"[cellxgene] Loading data from {basename(data)}, this may take awhile...") + file_size = getsize(data) + + # if a big file, let the user know it may take a while to load. + if file_size > BIG_FILE_SIZE_THRESHOLD: + click.echo(f"[cellxgene] Loading data from {basename(data)}, this may take awhile...") + else: + click.echo(f"[cellxgene] Loading data from {basename(data)}.") + # if file is larger than main memory, let the user know performance may suffer + if file_size > .95 * psutil.virtual_memory().total: + click.echo(f"[cellxgene] Warning: data file is larger than RAM - application may be very slow.") # Fix for anaconda python. matplotlib typically expects python to be installed as a framework TKAgg is usually # available and fixes this issue. See https://matplotlib.org/faq/virtualenv_faq.html diff --git a/server/requirements.txt b/server/requirements.txt index 6219c809..fa30fa0d 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -1,3 +1,4 @@ +anndata>=0.6.15 click>=6.7 Flask>=1.0.2 Flask-Caching>=1.4.0 @@ -8,6 +9,7 @@ flatbuffers>=1.10.0 matplotlib>=2.2 numpy>=1.15.2 pandas>=0.23.1 +psutil>=5.6.2 scanpy>=1.3.7 scipy>=1.1.0 scikit-learn>=0.19.1,!=0.20.0