move common code into server, update tests and makefile (#2425)

* move common code into server, update tests and makefile

remove backend directory, refactor

update smoke tests

server/Makefile

@@ -0,0 +1,12 @@
+include ../common.mk
+
+.PHONY: clean
+clean:
+	rm -f common/web/templates/index.html
+	rm -rf common/web/static
+	rm -f common/web/csp-hashes.json
+
+
+.PHONY: test-annotations-performance
+test-annotations-performance:
+	python ../test/performance/performance_test_annotations.py

server/__init__.py

@@ -0,0 +1,13 @@
+import logging
+import sys
+from server.common.utils.utils import import_plugins
+
+__version__ = "0.18.0"
+display_version = "cellxgene v" + __version__
+
+try:
+    import_plugins("server.plugins")
+except Exception as e:
+    # Make sure to exit in this case, as the server may not be configured as expected.
+    logging.critical(f"Error in import_plugins: {str(e)}")
+    sys.exit(1)

server/__main__.py

@@ -0,0 +1,15 @@
+# Work around bug https://github.com/pallets/werkzeug/issues/461
+if __package__ is None:
+    import sys
+    from pathlib import Path
+
+    PKG_PATH = Path(__file__).parent
+    sys.path.insert(0, str(PKG_PATH.parent))
+    import server  # noqa F401
+
+    __package__ = PKG_PATH.name
+
+# Main thing
+from .cli.cli import cli  # noqa F402
+
+cli()

server/app/app.py

@@ -0,0 +1,267 @@
+import datetime
+import logging
+from functools import wraps
+
+from flask import (
+    Flask,
+    current_app,
+    make_response,
+    render_template,
+    Blueprint,
+    request,
+    send_from_directory,
+)
+from flask_restful import Api, Resource
+
+import server.common.rest as common_rest
+from server.common.errors import DatasetAccessError, RequestException
+from server.common.health import health_check
+from server.common.utils.utils import StrictJSONEncoder
+
+webbp = Blueprint("webapp", "server.common.web", template_folder="templates")
+
+ONE_WEEK = 7 * 24 * 60 * 60
+
+
+def _cache_control(always, **cache_kwargs):
+    """
+    Used to easily manage cache control headers on responses.
+    See Werkzeug for attributes that can be set, eg, no_cache, private, max_age, etc.
+    https://werkzeug.palletsprojects.com/en/1.0.x/datastructures/#werkzeug.datastructures.ResponseCacheControl
+    """
+
+    def inner_cache_control(f):
+        @wraps(f)
+        def wrapper(*args, **kwargs):
+            response = make_response(f(*args, **kwargs))
+            if not always and not current_app.app_config.server_config.app__generate_cache_control_headers:
+                return response
+            if response.status_code >= 400:
+                return response
+            for k, v in cache_kwargs.items():
+                setattr(response.cache_control, k, v)
+            return response
+
+        return wrapper
+
+    return inner_cache_control
+
+
+def cache_control(**cache_kwargs):
+    """ config driven """
+    return _cache_control(False, **cache_kwargs)
+
+
+def cache_control_always(**cache_kwargs):
+    """ always generate headers, regardless of the config """
+    return _cache_control(True, **cache_kwargs)
+
+
+@webbp.route("/", methods=["GET"])
+def dataset_index():
+    app_config = current_app.app_config
+
+    dataset_config = app_config.get_dataset_config()
+    scripts = dataset_config.app__scripts
+    inline_scripts = dataset_config.app__inline_scripts
+
+    try:
+        args = {"SCRIPTS": scripts, "INLINE_SCRIPTS": inline_scripts}
+        return render_template("index.html", **args)
+
+    except DatasetAccessError as e:
+        return common_rest.abort_and_log(
+            e.status_code, f"Invalid dataset: {e.message}", loglevel=logging.INFO, include_exc_info=True
+        )
+
+
+@webbp.errorhandler(RequestException)
+def handle_request_exception(error):
+    return common_rest.abort_and_log(error.status_code, error.message, loglevel=logging.INFO, include_exc_info=True)
+
+
+def rest_get_data_adaptor(func):
+    @wraps(func)
+    def wrapped_function(self):
+        try:
+            return func(self, current_app.data_adaptor)
+        except DatasetAccessError as e:
+            return common_rest.abort_and_log(
+                e.status_code, f"Invalid dataset: {e.message}", loglevel=logging.INFO, include_exc_info=True
+            )
+
+    return wrapped_function
+
+
+class HealthAPI(Resource):
+    @cache_control_always(no_store=True)
+    def get(self):
+        config = current_app.app_config
+        return health_check(config)
+
+
+class SchemaAPI(Resource):
+    # TODO @mdunitz separate dataset schema and user schema
+    @cache_control(public=True, max_age=ONE_WEEK)
+    @rest_get_data_adaptor
+    def get(self, data_adaptor):
+        return common_rest.schema_get(data_adaptor)
+
+
+class ConfigAPI(Resource):
+    @cache_control(public=True, max_age=ONE_WEEK)
+    @rest_get_data_adaptor
+    def get(self, data_adaptor):
+        return common_rest.config_get(current_app.app_config, data_adaptor)
+
+
+class AnnotationsObsAPI(Resource):
+    @cache_control(public=True, max_age=ONE_WEEK)
+    @rest_get_data_adaptor
+    def get(self, data_adaptor):
+        return common_rest.annotations_obs_get(request, data_adaptor)
+
+    @cache_control(no_store=True)
+    @rest_get_data_adaptor
+    def put(self, data_adaptor):
+        return common_rest.annotations_obs_put(request, data_adaptor)
+
+
+class AnnotationsVarAPI(Resource):
+    @cache_control(public=True, max_age=ONE_WEEK)
+    @rest_get_data_adaptor
+    def get(self, data_adaptor):
+        return common_rest.annotations_var_get(request, data_adaptor)
+
+
+class DataVarAPI(Resource):
+    @cache_control(no_store=True)
+    @rest_get_data_adaptor
+    def put(self, data_adaptor):
+        return common_rest.data_var_put(request, data_adaptor)
+
+    @cache_control(public=True, max_age=ONE_WEEK)
+    @rest_get_data_adaptor
+    def get(self, data_adaptor):
+        return common_rest.data_var_get(request, data_adaptor)
+
+
+class ColorsAPI(Resource):
+    @cache_control(public=True, max_age=ONE_WEEK)
+    @rest_get_data_adaptor
+    def get(self, data_adaptor):
+        return common_rest.colors_get(data_adaptor)
+
+
+class DiffExpObsAPI(Resource):
+    @cache_control(no_store=True)
+    @rest_get_data_adaptor
+    def post(self, data_adaptor):
+        return common_rest.diffexp_obs_post(request, data_adaptor)
+
+
+class LayoutObsAPI(Resource):
+    @cache_control(public=True, max_age=ONE_WEEK)
+    @rest_get_data_adaptor
+    def get(self, data_adaptor):
+        return common_rest.layout_obs_get(request, data_adaptor)
+
+
+class GenesetsAPI(Resource):
+    @cache_control(public=True, max_age=ONE_WEEK)
+    @rest_get_data_adaptor
+    def get(self, data_adaptor):
+        return common_rest.genesets_get(request, data_adaptor)
+
+    @cache_control(no_store=True)
+    @rest_get_data_adaptor
+    def put(self, data_adaptor):
+        return common_rest.genesets_put(request, data_adaptor)
+
+
+class SummarizeVarAPI(Resource):
+    @rest_get_data_adaptor
+    @cache_control(public=True, max_age=ONE_WEEK)
+    def get(self, data_adaptor):
+        return common_rest.summarize_var_get(request, data_adaptor)
+
+    @rest_get_data_adaptor
+    @cache_control(no_store=True)
+    def post(self, data_adaptor):
+        return common_rest.summarize_var_post(request, data_adaptor)
+
+
+def get_api_base_resources(bp_base):
+    """Add resources that are accessed from the api url"""
+    api = Api(bp_base)
+
+    # Diagnostics routes
+    api.add_resource(HealthAPI, "/health")
+    return api
+
+
+def get_api_dataroot_resources(bp_dataroot):
+    """Add resources that refer to a dataset"""
+    api = Api(bp_dataroot)
+
+    def add_resource(resource, url):
+        """convenience function to make the outer function less verbose"""
+        api.add_resource(resource, url)
+
+    # Initialization routes
+    add_resource(SchemaAPI, "/schema")
+    add_resource(ConfigAPI, "/config")
+    # Data routes
+    add_resource(AnnotationsObsAPI, "/annotations/obs")
+    add_resource(AnnotationsVarAPI, "/annotations/var")
+    add_resource(DataVarAPI, "/data/var")
+    add_resource(GenesetsAPI, "/genesets")
+    add_resource(SummarizeVarAPI, "/summarize/var")
+    # Display routes
+    add_resource(ColorsAPI, "/colors")
+    # Computation routes
+    add_resource(DiffExpObsAPI, "/diffexp/obs")
+    add_resource(LayoutObsAPI, "/layout/obs")
+    return api
+
+
+class Server:
+    @staticmethod
+    def _before_adding_routes(app, app_config):
+        """ will be called before routes are added, during __init__.  Subclass protocol """
+        pass
+
+    def __init__(self, app_config):
+        self.app = Flask(__name__, static_folder=None)
+        self._before_adding_routes(self.app, app_config)
+        self.app.json_encoder = StrictJSONEncoder
+        server_config = app_config.server_config
+
+        # enable session data
+        self.app.permanent_session_lifetime = datetime.timedelta(days=50 * 365)
+
+        # Config
+        secret_key = server_config.app__flask_secret_key
+        self.app.config.update(SECRET_KEY=secret_key)
+
+        self.app.register_blueprint(webbp)
+
+        api_version = "/api/v0.2"
+        api_path = "/"
+
+        bp_base = Blueprint("bp_base", __name__, url_prefix=api_path)
+        base_resources = get_api_base_resources(bp_base)
+        self.app.register_blueprint(base_resources.blueprint)
+
+        bp_api = Blueprint("api", __name__, url_prefix=f"{api_path}{api_version}")
+        resources = get_api_dataroot_resources(bp_api)
+        self.app.register_blueprint(resources.blueprint)
+        self.app.add_url_rule(
+            "/static/<path:filename>",
+            "static_assets",
+            view_func=lambda filename: send_from_directory("../common/web/static", filename),
+            methods=["GET"],
+        )
+
+        self.app.data_adaptor = server_config.data_adaptor
+        self.app.app_config = app_config

server/app/session.py

@@ -0,0 +1,13 @@
+from uuid import uuid4
+
+from flask.sessions import SessionMixin
+
+CXGUID = "cxguid"
+
+
+def get_user_id(session: SessionMixin) -> str:
+    """ Gets a session-persistent user id. Creates one in the Flask session if non-extant """
+    if CXGUID not in session:
+        session[CXGUID] = uuid4().hex
+        session.permanent = True
+    return session[CXGUID]

server/cli/cli.py

@@ -0,0 +1,34 @@
+import click
+
+from .launch import launch
+from .prepare import prepare
+from .upgrade import log_upgrade_check
+from .. import __version__
+
+
+@click.group(
+    name="cellxgene",
+    subcommand_metavar="COMMAND <args>",
+    options_metavar="<options>",
+    context_settings=dict(max_content_width=85, help_option_names=["-h", "--help"]),
+)
+@click.help_option("--help", "-h", help="Show this message and exit.")
+@click.version_option(
+    version=__version__,
+    prog_name="cellxgene",
+    message="[%(prog)s] Version %(version)s",
+    help="Show the software version and exit.",
+)
+@click.option(
+    "--upgrade-check/--no-upgrade-check",
+    default=True,
+    show_default=True,
+    help="Check for release upgrades on start.",
+)
+def cli(upgrade_check):
+    if upgrade_check:
+        log_upgrade_check()
+
+
+cli.add_command(launch)
+cli.add_command(prepare)

server/cli/launch.py

@@ -0,0 +1,449 @@
+import errno
+import functools
+import logging
+import sys
+import webbrowser
+import os
+import click
+from flask_compress import Compress
+from flask_cors import CORS
+
+from server.default_config import default_config
+from server.app.app import Server
+from server.common.config.app_config import AppConfig
+from server.common.errors import DatasetAccessError, ConfigurationError
+from server.common.utils.utils import sort_options
+
+DEFAULT_CONFIG = AppConfig()
+
+
+def annotation_args(func):
+    @click.option(
+        "--disable-annotations",
+        is_flag=True,
+        default=not DEFAULT_CONFIG.dataset_config.user_annotations__enable,
+        show_default=True,
+        help="Disable user annotation of data.",
+    )
+    @click.option(
+        "--annotations-file",
+        default=DEFAULT_CONFIG.dataset_config.user_annotations__local_file_csv__file,
+        show_default=True,
+        multiple=False,
+        metavar="<path>",
+        help="CSV file to initialize editing of existing annotations; will be altered in-place. "
+        "Incompatible with --user-generated-data-dir.",
+    )
+    @click.option(
+        "--user-generated-data-dir",
+        "--annotations-dir",
+        default=DEFAULT_CONFIG.dataset_config.user_annotations__local_file_csv__directory,
+        show_default=False,
+        multiple=False,
+        metavar="<directory path>",
+        help="Directory of where to save output annotations; filename will be specified in the application. "
+        "Incompatible with --annotations-file and --gene-sets-file.",
+    )
+    @click.option(
+        "--disable-gene-sets-save",
+        is_flag=True,
+        default=DEFAULT_CONFIG.dataset_config.user_annotations__gene_sets__readonly,
+        show_default=False,
+        help="Disable saving gene sets. If disabled, users will be able to make changes to gene sets but all "
+        "changes will be lost on browser refresh.",
+    )
+    @click.option(
+        "--gene-sets-file",
+        default=DEFAULT_CONFIG.dataset_config.user_annotations__local_file_csv__gene_sets_file,
+        show_default=True,
+        multiple=False,
+        metavar="<path>",
+        help="CSV file to initialize editing of gene sets; will be altered in-place. Incompatible with "
+        "--user-generated-data-dir.",
+    )
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
+def config_args(func):
+    @click.option(
+        "--max-category-items",
+        default=DEFAULT_CONFIG.dataset_config.presentation__max_categories,
+        metavar="<integer>",
+        show_default=True,
+        help="Will not display categories with more distinct values than specified.",
+    )
+    @click.option(
+        "--disable-custom-colors",
+        is_flag=True,
+        default=False,
+        show_default=False,
+        help="Disable user-defined category-label colors drawn from source data file.",
+    )
+    @click.option(
+        "--diffexp-lfc-cutoff",
+        "-de",
+        default=DEFAULT_CONFIG.dataset_config.diffexp__lfc_cutoff,
+        show_default=True,
+        metavar="<float>",
+        help="Minimum log fold change threshold for differential expression.",
+    )
+    @click.option(
+        "--disable-diffexp",
+        is_flag=True,
+        default=not DEFAULT_CONFIG.dataset_config.diffexp__enable,
+        show_default=False,
+        help="Disable on-demand differential expression.",
+    )
+    @click.option(
+        "--embedding",
+        "-e",
+        default=DEFAULT_CONFIG.dataset_config.embeddings__names,
+        multiple=True,
+        show_default=False,
+        metavar="<text>",
+        help="Embedding name, eg, 'umap'. Repeat option for multiple embeddings. Defaults to all.",
+    )
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
+def dataset_args(func):
+    @click.option(
+        "--obs-names",
+        "-obs",
+        default=DEFAULT_CONFIG.server_config.single_dataset__obs_names,
+        metavar="<text>",
+        help="Name of annotation field to use for observations. If not specified cellxgene will use the the obs index.",
+    )
+    @click.option(
+        "--var-names",
+        "-var",
+        default=DEFAULT_CONFIG.server_config.single_dataset__var_names,
+        metavar="<text>",
+        help="Name of annotation to use for variables. If not specified cellxgene will use the the var index.",
+    )
+    @click.option(
+        "--backed",
+        "-b",
+        is_flag=True,
+        default=DEFAULT_CONFIG.server_config.adaptor__anndata_adaptor__backed,
+        show_default=False,
+        help="Load anndata in file-backed mode. " "This may save memory, but may result in slower overall performance.",
+    )
+    @click.option(
+        "--title",
+        "-t",
+        default=DEFAULT_CONFIG.server_config.single_dataset__title,
+        metavar="<text>",
+        help="Title to display. If omitted will use file name.",
+    )
+    @click.option(
+        "--about",
+        default=DEFAULT_CONFIG.server_config.single_dataset__about,
+        metavar="<URL>",
+        help="URL providing more information about the dataset (hint: must be a fully specified absolute URL).",
+    )
+    @click.option(
+        "--X-approximate-distribution",
+        default=DEFAULT_CONFIG.dataset_config.X_approximate_distribution,
+        show_default=True,
+        type=click.Choice(["auto", "normal", "count"], case_sensitive=False),
+        help="Specify the approximate distribution of X matrix values. 'auto' will use a heuristic "
+        "to determine the approximate distribution.  Mode 'auto' is incompatible with --backed.",
+    )
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
+def server_args(func):
+    @click.option(
+        "--debug",
+        "-d",
+        is_flag=True,
+        default=DEFAULT_CONFIG.server_config.app__debug,
+        show_default=True,
+        help="Run in debug mode. This is helpful for cellxgene developers, "
+        "or when you want more information about an error condition.",
+    )
+    @click.option(
+        "--verbose",
+        "-v",
+        is_flag=True,
+        default=DEFAULT_CONFIG.server_config.app__verbose,
+        show_default=True,
+        help="Provide verbose output, including warnings and all server requests.",
+    )
+    @click.option(
+        "--port",
+        "-p",
+        metavar="<port>",
+        default=DEFAULT_CONFIG.server_config.app__port,
+        type=int,
+        show_default=True,
+        help="Port to run server on. If not specified cellxgene will find an available port.",
+    )
+    @click.option(
+        "--host",
+        metavar="<IP address>",
+        default=DEFAULT_CONFIG.server_config.app__host,
+        show_default=False,
+        help="Host IP address. By default cellxgene will use localhost (e.g. 127.0.0.1).",
+    )
+    @click.option(
+        "--scripts",
+        "-s",
+        default=DEFAULT_CONFIG.dataset_config.app__scripts,
+        multiple=True,
+        metavar="<text>",
+        help="Additional script files to include in HTML page. If not specified, "
+        "no additional script files will be included.",
+        show_default=False,
+    )
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
+def launch_args(func):
+    @annotation_args
+    @config_args
+    @dataset_args
+    @server_args
+    @click.argument("datapath", required=False, metavar="<path to data file>")
+    @click.option(
+        "--open",
+        "-o",
+        "open_browser",
+        is_flag=True,
+        default=DEFAULT_CONFIG.server_config.app__open_browser,
+        show_default=True,
+        help="Open web browser after launch.",
+    )
+    @click.option(
+        "--config-file",
+        "-c",
+        "config_file",
+        default=None,
+        show_default=True,
+        help="Location to yaml file with configuration settings",
+    )
+    @click.option(
+        "--dump-default-config",
+        "dump_default_config",
+        is_flag=True,
+        default=False,
+        show_default=True,
+        help="Print default configuration settings and exit",
+    )
+    @click.help_option("--help", "-h", help="Show this message and exit.")
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
+def handle_scripts(scripts):
+    if scripts:
+        click.echo(
+            r"""
+    / / /\ \ \__ _ _ __ _ __ (_)_ __   __ _
+    \ \/  \/ / _` | '__| '_ \| | '_ \ / _` |
+     \  /\  / (_| | |  | | | | | | | | (_| |
+      \/  \/ \__,_|_|  |_| |_|_|_| |_|\__, |
+                                      |___/
+    The --scripts flag is intended for developers to include google analytics etc. You could be opening yourself to a
+    security risk by including the --scripts flag. Make sure you trust the scripts that you are including.
+            """
+        )
+        scripts_pretty = ", ".join(scripts)
+        click.confirm(f"Are you sure you want to inject these scripts: {scripts_pretty}?", abort=True)
+
+
+class CliLaunchServer(Server):
+    """
+    the CLI runs a local web server, and needs to enable a few more features.
+    """
+
+    def __init__(self, app_config):
+        super().__init__(app_config)
+
+    @staticmethod
+    def _before_adding_routes(app, app_config):
+        app.config["COMPRESS_MIMETYPES"] = [
+            "text/html",
+            "text/css",
+            "text/xml",
+            "application/json",
+            "application/javascript",
+            "application/octet-stream",
+        ]
+        Compress(app)
+        if app_config.server_config.app__debug:
+            CORS(app, supports_credentials=True)
+
+
+@sort_options
+@click.command(
+    short_help="Launch the cellxgene data viewer. " "Run `cellxgene launch --help` for more information.",
+    options_metavar="<options>",
+)
+@launch_args
+def launch(
+    datapath,
+    verbose,
+    debug,
+    open_browser,
+    port,
+    host,
+    embedding,
+    obs_names,
+    var_names,
+    max_category_items,
+    disable_custom_colors,
+    diffexp_lfc_cutoff,
+    title,
+    scripts,
+    about,
+    disable_annotations,
+    annotations_file,
+    user_generated_data_dir,
+    gene_sets_file,
+    disable_gene_sets_save,
+    backed,
+    disable_diffexp,
+    config_file,
+    dump_default_config,
+    x_approximate_distribution,
+):
+    """Launch the cellxgene data viewer.
+    This web app lets you explore single-cell expression data.
+    Data must be in a format that cellxgene expects.
+    Read the "getting started" guide to learn more:
+    https://chanzuckerberg.github.io/cellxgene/getting-started.html
+
+    Examples:
+
+    > cellxgene launch example-dataset/pbmc3k.h5ad --title pbmc3k
+
+    > cellxgene launch <your data file> --title <your title>
+
+    > cellxgene launch <url>"""
+
+    if dump_default_config:
+        print(default_config)
+        sys.exit(0)
+
+    # Startup message
+    click.echo("[cellxgene] Starting the CLI...")
+
+    # app config
+    app_config = AppConfig()
+    server_config = app_config.server_config
+
+    try:
+        if config_file:
+            app_config.update_from_config_file(config_file)
+
+        # Determine which config options were give on the command line.
+        # Those will override the ones provided in the config file (if provided).
+        cli_config = AppConfig()
+        cli_config.update_server_config(
+            app__verbose=verbose,
+            app__debug=debug,
+            app__host=host,
+            app__port=port,
+            app__open_browser=open_browser,
+            single_dataset__datapath=datapath,
+            single_dataset__title=title,
+            single_dataset__about=about,
+            single_dataset__obs_names=obs_names,
+            single_dataset__var_names=var_names,
+            adaptor__anndata_adaptor__backed=backed,
+        )
+        cli_config.update_dataset_config(
+            app__scripts=scripts,
+            user_annotations__enable=not disable_annotations,
+            user_annotations__local_file_csv__file=annotations_file,
+            user_annotations__local_file_csv__directory=user_generated_data_dir,
+            user_annotations__local_file_csv__gene_sets_file=gene_sets_file,
+            user_annotations__gene_sets__readonly=disable_gene_sets_save,
+            presentation__max_categories=max_category_items,
+            presentation__custom_colors=not disable_custom_colors,
+            embeddings__names=embedding,
+            diffexp__enable=not disable_diffexp,
+            diffexp__lfc_cutoff=diffexp_lfc_cutoff,
+            X_approximate_distribution=x_approximate_distribution,
+        )
+
+        diff = cli_config.server_config.changes_from_default()
+        changes = {key: val for key, val, _ in diff}
+        app_config.update_server_config(**changes)
+
+        diff = cli_config.dataset_config.changes_from_default()
+        changes = {key: val for key, val, _ in diff}
+        app_config.update_dataset_config(**changes)
+
+        # process the configuration
+        #  any errors will be thrown as an exception.
+        #  any info messages will be passed to the messagefn function.
+
+        def messagefn(message):
+            click.echo("[cellxgene] " + message)
+
+        # Use a default secret if one is not provided
+        if not server_config.app__flask_secret_key:
+            app_config.update_server_config(app__flask_secret_key="SparkleAndShine")
+
+        app_config.complete_config(messagefn)
+
+    except (ConfigurationError, DatasetAccessError) as e:
+        raise click.ClickException(e)
+
+    handle_scripts(scripts)
+
+    # create the server
+    server = CliLaunchServer(app_config)
+
+    if not server_config.app__verbose:
+        log = logging.getLogger("werkzeug")
+        log.setLevel(logging.ERROR)
+
+    cellxgene_url = f"http://{app_config.server_config.app__host}:{app_config.server_config.app__port}"
+    if server_config.app__open_browser:
+        click.echo(f"[cellxgene] Launching! Opening your browser to {cellxgene_url} now.")
+        webbrowser.open(cellxgene_url)
+    else:
+        click.echo(f"[cellxgene] Launching! Please go to {cellxgene_url} in your browser.")
+
+    click.echo("[cellxgene] Type CTRL-C at any time to exit.")
+
+    if not server_config.app__verbose:
+        f = open(os.devnull, "w")
+        sys.stdout = f
+
+    try:
+        server.app.run(
+            host=server_config.app__host,
+            debug=server_config.app__debug,
+            port=server_config.app__port,
+            threaded=not server_config.app__debug,
+            use_debugger=False,
+            use_reloader=False,
+        )
+    except OSError as e:
+        if e.errno == errno.EADDRINUSE:
+            raise click.ClickException("Port is in use, please specify an open port using the --port flag.") from e
+        raise

server/cli/prepare.py

@@ -0,0 +1,278 @@
+from os.path import expanduser, isdir, isfile, sep, splitext
+
+import click
+import pandas as pd
+from numpy import ndarray, unique
+from scipy.sparse.csc import csc_matrix
+
+from server.common.utils.utils import sort_options
+
+
+@sort_options
+@click.command(
+    short_help="Preprocess data for use with cellxgene. " "Run `cellxgene prepare --help` for more information.",
+    options_metavar="<options>",
+)
+@click.argument("data", nargs=1, metavar="<path to data file>", required=True)
+@click.option(
+    "--embedding",
+    "-e",
+    default=["umap", "tsne"],
+    multiple=True,
+    type=click.Choice(["umap", "tsne"]),
+    help="Embedding algorithm(s). Repeat option for multiple embeddings.",
+    show_default=True,
+)
+@click.option(
+    "--recipe",
+    "-r",
+    default="none",
+    type=click.Choice(["none", "seurat", "zheng17"]),
+    show_default=True,
+)
+@click.option("--output", "-o", default="", help="Save a new file to filename.", metavar="<filename>")
+@click.option("--plotting", "-p", default=False, is_flag=True, help="Generate plots.", show_default=True)
+@click.option("--sparse", default=False, is_flag=True, help="Force sparsity.", show_default=True)
+@click.option("--overwrite", default=False, is_flag=True, help="Allow file overwriting.", show_default=True)
+@click.option("--set-obs-names", default="", help="Named field to set as index for obs.", metavar="<name>")
+@click.option("--set-var-names", default="", help="Named field to set as index for var.", metavar="<name>")
+@click.option(
+    "--skip-qc",
+    default=False,
+    is_flag=True,
+    help="Do not run quality control metrics. By default cellxgene runs them "
+    "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).",
+)
+@click.option(
+    "--make-obs-names-unique/--no-make-obs-names-unique",
+    default=True,
+    help="Ensure obs index is unique.",
+    show_default=True,
+)
+@click.option(
+    "--make-var-names-unique/--no-make-var-names-unique",
+    default=True,
+    help="Ensure var index is unique.",
+    show_default=True,
+)
+@click.help_option("--help", "-h", help="Show this message and exit.")
+def prepare(
+    data,
+    embedding,
+    recipe,
+    output,
+    plotting,
+    sparse,
+    overwrite,
+    set_obs_names,
+    set_var_names,
+    skip_qc,
+    make_obs_names_unique,
+    make_var_names_unique,
+):
+    """
+    Preprocess data for use with cellxgene.
+    This tool runs a series of scanpy routines for preparing a dataset for use
+    with cellxgene. It loads data from different formats
+    (h5ad, loom, or a 10x directory), runs dimensionality reduction,
+    computes nearest neighbors, computes an embedding, performs clustering,
+    and saves the results. Includes additional options for naming annotations,
+    ensuring sparsity, and plotting results.
+    """
+
+    # collect slow imports here to make CLI startup more responsive
+    click.echo("[cellxgene] Starting CLI...")
+    try:
+        import matplotlib
+
+        matplotlib.use("Agg")
+        import scanpy as sc
+    except ImportError:
+        raise click.ClickException(
+            "[cellxgene] cellxgene prepare has not been installed. Please run `pip install 'cellxgene[prepare]'` "
+            "to install the necessary requirements."
+        )
+
+    # scanpy settings
+    sc.settings.verbosity = 0
+    sc.settings.autosave = True
+
+    # check args
+    if sparse and not recipe == "none":
+        raise click.UsageError("Cannot use a recipe when forcing sparsity")
+
+    output = expanduser(output)
+
+    if not output:
+        click.echo(
+            "Warning: No file will be saved, to save the results of cellxgene prepare include "
+            "--output <filename> to save output to a new file"
+        )
+    if isfile(output) and not overwrite:
+        raise click.UsageError(f"Cannot overwrite existing file {output}, try using the flag --overwrite")
+
+    def load_data(data):
+        if isfile(data):
+            name, extension = splitext(data)
+            if extension == ".h5ad":
+                adata = sc.read_h5ad(data)
+            elif extension == ".loom":
+                adata = sc.read_loom(data)
+            else:
+                raise click.FileError(data, hint="does not have a valid extension [.h5ad | .loom]")
+        elif isdir(data):
+            if not data.endswith(sep):
+                data += sep
+            adata = sc.read_10x_mtx(data)
+        else:
+            raise click.FileError(data, hint="not a valid file or path")
+
+        if not set_obs_names == "":
+            if set_obs_names not in adata.obs_keys():
+                raise click.UsageError(f"obs {set_obs_names} not found, options are: {adata.obs_keys()}")
+            adata.obs_names = adata.obs[set_obs_names]
+        if not set_var_names == "":
+            if set_var_names not in adata.var_keys():
+                raise click.UsageError(f"var {set_var_names} not found, options are: {adata.var_keys()}")
+            adata.var_names = adata.var[set_var_names]
+        if make_obs_names_unique:
+            adata.obs.index = make_index_unique(adata.obs.index)
+        if make_var_names_unique:
+            adata.var.index = make_index_unique(adata.var.index)
+        if not adata._obs.index.is_unique:
+            click.echo("Warning: obs index is not unique")
+        if not adata._var.index.is_unique:
+            click.echo("Warning: var index is not unique")
+        return adata
+
+    def calculate_qc_metrics(adata):
+        if not skip_qc:
+            sc.pp.calculate_qc_metrics(adata, inplace=True)
+        return adata
+
+    def make_sparse(adata):
+        if (type(adata.X) is ndarray) and sparse:
+            adata.X = csc_matrix(adata.X)
+
+    def run_recipe(adata):
+        if recipe == "seurat":
+            sc.pp.recipe_seurat(adata)
+        elif recipe == "zheng17":
+            sc.pp.recipe_zheng17(adata)
+        else:
+            sc.pp.filter_cells(adata, min_genes=5)
+            sc.pp.filter_genes(adata, min_cells=25)
+            if sparse:
+                sc.pp.scale(adata, zero_center=False)
+            else:
+                sc.pp.scale(adata)
+
+    def run_pca(adata):
+        if sparse:
+            sc.pp.pca(adata, svd_solver="arpack", zero_center=False)
+        else:
+            sc.pp.pca(adata, svd_solver="arpack")
+
+    def run_neighbors(adata):
+        sc.pp.neighbors(adata)
+
+    def run_louvain(adata):
+        sc.tl.louvain(adata)
+
+    def run_embedding(adata):
+        if len(unique(adata.obs["louvain"].values)) < 10:
+            palette = "tab10"
+        else:
+            palette = "tab20"
+
+        if "umap" in embedding:
+            sc.tl.umap(adata)
+            if plotting:
+                sc.pl.umap(adata, color="louvain", palette=palette, save="_louvain")
+
+        if "tsne" in embedding:
+            sc.tl.tsne(adata)
+            if plotting:
+                sc.pl.tsne(adata, color="louvain", palette=palette, save="_louvain")
+
+    def show_step(item):
+        if not skip_qc:
+            qc_name = "Calculating QC metrics"
+        else:
+            qc_name = "Skipping QC"
+        names = {
+            "calculate_qc_metrics": qc_name,
+            "make_sparse": "Ensuring sparsity",
+            "run_recipe": f'Running preprocessing recipe "{recipe}"',
+            "run_pca": "Running PCA",
+            "run_neighbors": "Calculating neighbors",
+            "run_louvain": "Calculating clusters",
+            "run_embedding": "Computing embedding",
+        }
+        if item is not None:
+            return names[item.__name__]
+
+    steps = [calculate_qc_metrics, make_sparse, run_recipe, run_pca, run_neighbors, run_louvain, run_embedding]
+
+    click.echo(f"[cellxgene] Loading data from {data}, please wait...")
+    adata = load_data(data)
+
+    click.echo("[cellxgene] Beginning preprocessing...")
+    with click.progressbar(steps, label="[cellxgene] Progress", show_eta=False, item_show_func=show_step) as bar:
+        for step in bar:
+            step(adata)
+
+    # saving
+    if not output == "":
+        click.echo(f"[cellxgene] Saving results to {output}...")
+        adata.write(output)
+
+    click.echo("[cellxgene] Success!")
+
+
+# TODO (mweiden): remove this once this issue is resolved https://github.com/theislab/anndata/issues/344
+# Note: tentative solution here https://github.com/theislab/anndata/pull/345
+def make_index_unique(index: pd.Index, join: str = "-"):
+    """
+    Makes the index unique by appending a number string to each duplicate index element: '1', '2', etc.
+
+    If a tentative name created by the algorithm already exists in the index, it tries the next integer in the sequence.
+
+    The first occurrence of a non-unique value is ignored.
+    Parameters
+    ----------
+    join
+         The connecting string between name and integer.
+    Examples
+    --------
+    >>> from anndata import AnnData
+    >>> adata1 = AnnData(np.ones((3, 2)), dict(obs_names=['a', 'b', 'c']))
+    >>> adata2 = AnnData(np.zeros((3, 2)), dict(obs_names=['d', 'b', 'b']))
+    >>> adata = adata1.concatenate(adata2)
+    >>> adata.obs_names
+    Index(['a', 'b', 'c', 'd', 'b', 'b'], dtype='object')
+    >>> adata.obs_names_make_unique()
+    >>> adata.obs_names
+    Index(['a', 'b', 'c', 'd', 'b-1', 'b-2'], dtype='object')
+    """
+    if index.is_unique:
+        return index
+    from collections import defaultdict
+
+    values = index.values
+    values_set = set(values)
+    indices_dup = index.duplicated(keep="first")
+    values_dup = values[indices_dup]
+    counter = defaultdict(lambda: 0)
+    for i, v in enumerate(values_dup):
+        while True:
+            counter[v] += 1
+            tentative_new_name = v + join + str(counter[v])
+            if tentative_new_name not in values_set:
+                values_set.add(tentative_new_name)
+                values_dup[i] = tentative_new_name
+                break
+
+    values[indices_dup] = values_dup
+    index = pd.Index(values)
+    return index

server/cli/upgrade.py

@@ -0,0 +1,85 @@
+import re
+
+import click
+import requests
+from requests.exceptions import ConnectionError
+
+from .. import __version__
+
+# Official SemVer regex: https://semver.org/
+SEMVER_FORMAT = re.compile(
+    r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*["
+    r"a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+("
+    r"?:\.[0-9a-zA-Z-]+)*))?$"
+)
+
+
+def log_upgrade_check():
+    # Sanity-check that the CLI version is a properly-formatted SemVer string
+    assert validate_version_str(__version__, release_only=False)
+
+    # Get the current latest release
+    try:
+        release_tag_generator = (r["tag_name"] for r in _request_cellxgene_releases())
+        latest_release = next(release_tag_generator, lambda tag_name: validate_version_str(tag_name))
+        if version_gt(latest_release, __version__):
+            click.echo(f"There's a new version of cellxgene available ({latest_release})!", err=True)
+            click.echo("To upgrade, run the following: pip install --upgrade cellxgene\n", err=True)
+    except (ConnectionError, RateLimitException):
+        click.echo("Upgrade check failed.\n")
+
+
+class RateLimitException(Exception):
+    """
+    Github API Rate Limit Exception
+    """
+
+
+def _request_cellxgene_releases():
+    def raise_on_rate_limit(response):
+        if response.status_code == 403 and res.headers.get("X-RateLimit-Remaining") == "0":
+            raise RateLimitException
+
+    url = "https://api.github.com/repos/chanzuckerberg/cellxgene/releases"
+    res = requests.get(url)
+    raise_on_rate_limit(res)
+    for release in res.json():
+        yield release
+    while "next" in res.links.keys():
+        res = requests.get(res.links["next"]["url"])
+        raise_on_rate_limit(res)
+        for release in res.json():
+            yield release
+
+
+def validate_version_str(version_str, release_only=True):
+    """
+    Test if a string conforms to SemVer format (https://semver.org/)
+    :param version_str: a string to be validated
+    :param release_only: only declare releases (not prereleases) valid
+    :return: True if the version string is of a valid SemVer format else False
+    """
+    match = SEMVER_FORMAT.match(version_str)
+    has_match = match is not None
+    if has_match and release_only:
+        return not match.group("prerelease")
+    return has_match
+
+
+def split_version(version_string):
+    """
+    Split a SemVer-formatted string into its component integers
+    :param version_string: a SemVer string to be split
+    :return: an array of three integers
+    """
+    match = SEMVER_FORMAT.match(version_string)
+    return [int(match.group(group)) for group in ["major", "minor", "patch"]]
+
+
+def version_gt(left_version, right_version):
+    for left, right in zip(split_version(left_version), split_version(right_version)):
+        if left > right:
+            return True
+        elif right > left:
+            return False
+    return False

server/common/annotations/annotations.py

@@ -0,0 +1,90 @@
+from abc import ABCMeta, abstractmethod
+
+from server.common.errors import DisabledFeatureError
+from server.common.utils.type_conversion_utils import get_schema_type_hint_of_array
+from server.common.genesets import write_gene_sets_tidycsv
+
+
+class Annotations(metaclass=ABCMeta):
+    """baseclass for annotations and gene sets"""
+
+    def __init__(self, config={}):
+        self.config = config
+
+    def user_annotations_enabled(self):
+        return self.config.get("user-annotations", False)
+
+    def gene_sets_save_enabled(self):
+        return self.config.get("genesets-save", False)
+
+    def check_user_annotations_enabled(self):
+        if not self.user_annotations_enabled():
+            raise DisabledFeatureError("User annotations are disabled.")
+
+    def check_gene_sets_save_enabled(self):
+        if not self.gene_sets_save_enabled():
+            raise DisabledFeatureError("User gene sets save is disabled.")
+
+    def get_schema(self, data_adaptor):
+        schema = []
+        labels = self.read_labels(data_adaptor)
+        if labels is not None and not labels.empty:
+            for col in labels.columns:
+                col_schema = dict(name=col, writable=True)
+                col_schema.update(get_schema_type_hint_of_array(labels[col]))
+                schema.append(col_schema)
+
+        return schema
+
+    @abstractmethod
+    def set_collection(self, name):
+        """set or create a new annotation collection"""
+        pass
+
+    @abstractmethod
+    def read_labels(self, data_adaptor):
+        """Return the labels as a pandas.DataFrame"""
+        pass
+
+    @abstractmethod
+    def write_labels(self, df, data_adaptor):
+        """Write the labels (df) to a persistent storage such that it can later be read"""
+        pass
+
+    @abstractmethod
+    def read_gene_sets(self, data_adaptor):
+        """Return the gene sets from persistent storage"""
+        pass
+
+    @abstractmethod
+    def write_gene_sets(self, gs, data_adaptor):
+        """Write the gene sets (gs) to a persistent storage such that it can later be read"""
+        pass
+
+    @abstractmethod
+    def update_parameters(self, parameters, data_adaptor):
+        """Update configuration parameters that describe information about the annotations feature"""
+        pass
+
+    @staticmethod
+    def gene_sets_to_csv(genesets):
+        """
+        Convert the internal gene sets format (returned by read_gene_set) into
+        the simple Tidy CSV.
+        """
+        from io import StringIO
+
+        if isinstance(genesets, dict):
+            genesets = genesets.values()
+
+        with StringIO() as sio:
+            write_gene_sets_tidycsv(sio, genesets)
+            return sio.getvalue()
+
+    @staticmethod
+    def gene_sets_to_response(genesets):
+        """
+        Convert the internal gene sets format (returned by read_gene_set) into
+        the dict expected by the JSON REST API
+        """
+        return list(genesets.values())

server/common/annotations/local_file_csv.py

@@ -0,0 +1,273 @@
+import base64
+import os
+import re
+import threading
+from datetime import datetime
+from hashlib import blake2b
+
+import pandas as pd
+from flask import session
+
+from server import __version__ as cellxgene_version
+from server.app.session import get_user_id
+from server.common.annotations.annotations import Annotations
+from server.common.genesets import read_gene_sets_tidycsv
+from server.common.errors import AnnotationsError, ObsoleteRequest
+from server.common.utils.data_locator import DataLocator
+
+
+class AnnotationsLocalFile(Annotations):
+    CXG_ANNO_COLLECTION = "cxg_anno_collection"
+
+    def __init__(self, config, output_dir, label_output_file, gene_sets_output_file):
+        super().__init__(config)
+        self.output_dir = output_dir
+        self.label_output_file = label_output_file
+        self.gene_sets_output_file = gene_sets_output_file
+        # lock used to protect label file write ops
+        self.label_lock = threading.RLock()
+        self.gene_sets_lock = threading.RLock()
+
+        # cache the most recent cell labels/annotations.
+        self.last_label_fname = None
+        self.last_labels = None
+
+        # cache the most recent gene sets.
+        self.last_geneset_fname = None
+        self.last_geneset = None
+
+        # txn ID - used to de-dup geneset writes
+        self.last_geneset_tid = 0
+
+    def is_safe_collection_name(self, name):
+        """
+        return true if this is a safe collection name
+        this is ultra conservative. If we want to allow full legal file name syntax,
+        we could look at modules like `pathvalidate`
+        """
+        if name is None:
+            return False
+        return re.match(r"^[\w\-]+$", name) is not None
+
+    def set_collection(self, name):
+        session[self.CXG_ANNO_COLLECTION] = name
+        session.permanent = True
+
+    def get_collection(self):
+        if session is None:
+            return None
+        return session.get(self.CXG_ANNO_COLLECTION)
+
+    def read_labels(self, data_adaptor):
+        self.check_user_annotations_enabled()  # raises
+
+        fname = self._get_celllabels_filename(data_adaptor)
+        with self.label_lock:
+            if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0:
+                # returned the cached labels if possible, otherwise read them from the file
+                if fname == self.last_label_fname:
+                    return self.last_labels
+                else:
+                    labels = pd.read_csv(
+                        fname, dtype="category", index_col=0, header=0, comment="#", keep_default_na=False
+                    )
+                    # update the cache
+                    self.last_label_fname = fname
+                    self.last_labels = labels
+                    return labels
+            else:
+                return pd.DataFrame()
+
+    def write_labels(self, df, data_adaptor):
+        self.check_user_annotations_enabled()  # raises
+
+        # update our internal state and save it.  Multi-threading often enabled,
+        # so treat this as a critical section.
+        with self.label_lock:
+            lastmod = data_adaptor.get_last_mod_time()
+            lastmodstr = "'unknown'" if lastmod is None else lastmod.isoformat(timespec="seconds")
+            header = (
+                f"# Annotations generated on {datetime.now().isoformat(timespec='seconds')} "
+                f"using cellxgene version {cellxgene_version}\n"
+                f"# Input data file was {data_adaptor.get_location()}, "
+                f"which was last modified on {lastmodstr}\n"
+            )
+
+            fname = self._get_celllabels_filename(data_adaptor)
+            self._backup(fname)
+            if not df.empty:
+                with open(fname, "w", newline="") as f:
+                    if header is not None:
+                        f.write(header)
+                    df.to_csv(f)
+            else:
+                open(fname, "w").close()
+
+            # update the cache
+            self.last_label_fname = fname
+            self.last_labels = df
+
+    def read_gene_sets(self, data_adaptor, context=None):
+        fname = self._get_genesets_filename(data_adaptor)
+        gene_sets = {}
+        tid = None
+        with self.gene_sets_lock:
+            tid = self.last_geneset_tid  # inside the critical section
+            if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0:
+                # return the cached genesets if possible, otherwise read from file and validate them
+                if fname == self.last_geneset_fname:
+                    gene_sets = self.last_geneset
+                else:
+                    # read
+                    gene_sets = read_gene_sets_tidycsv(DataLocator(fname), context)
+
+                    # validate
+                    gene_sets = data_adaptor.check_new_gene_sets(gene_sets, context)
+
+                    # update cache
+                    self.last_geneset_fname = fname
+                    self.last_geneset = gene_sets
+
+        return (gene_sets, tid)
+
+    def write_gene_sets(self, gene_sets, tid, data_adaptor):
+        self.check_gene_sets_save_enabled()  # raises
+
+        if type(tid) != int or tid < 0:
+            raise ValueError("tid must be a positive integer")
+
+        # may raise
+        gene_sets = data_adaptor.check_new_gene_sets(gene_sets)
+
+        with self.gene_sets_lock:
+            # skip if the request is stale
+            if tid is not None:
+                if tid <= self.last_geneset_tid:
+                    raise ObsoleteRequest("TID is stale.")
+                self.last_geneset_tid = tid
+
+            lastmod = data_adaptor.get_last_mod_time()
+            lastmodstr = "'unknown'" if lastmod is None else lastmod.isoformat(timespec="seconds")
+            header = (
+                f"# Gene set generated on {datetime.now().isoformat(timespec='seconds')} "
+                f"using cellxgene version {cellxgene_version}\n"
+                f"# Input data file was {data_adaptor.get_location()}, "
+                f"which was last modified on {lastmodstr}\n"
+            )
+
+            fname = self._get_genesets_filename(data_adaptor)
+            self._backup(fname)
+            with open(fname, "w", newline="") as f:
+                f.write(header)
+                f.write(self.gene_sets_to_csv(gene_sets))
+
+            # update the cache
+            self.last_geneset_fname = fname
+            self.last_geneset = gene_sets if type(gene_sets) == dict else {g["geneset_name"]: g for g in gene_sets}
+
+    def _get_userdata_idhash(self, data_adaptor):
+        """
+        Return a short hash that weakly identifies the user and dataset.
+        Used to create safe annotations output file names.
+        """
+        uid = get_user_id(session)
+        id = (uid + data_adaptor.get_location()).encode()
+        idhash = base64.b32encode(blake2b(id, digest_size=5).digest()).decode("utf-8")
+        return idhash
+
+    def _get_output_dir(self):
+        if self.output_dir:
+            return self.output_dir
+
+        output_file = self.label_output_file or self.gene_sets_output_file
+        if output_file:
+            return os.path.dirname(os.path.abspath(output_file))
+
+        return os.getcwd()
+
+    def _get_celllabels_filename(self, data_adaptor):
+        """return the current annotation file name"""
+        if self.label_output_file:
+            return self.label_output_file
+
+        return self._get_filename(data_adaptor, "cell-labels")
+
+    def _get_genesets_filename(self, data_adaptor):
+        """return the current gene sets file name"""
+        if self.gene_sets_output_file:
+            return self.gene_sets_output_file
+
+        return self._get_filename(data_adaptor, "gene-sets")
+
+    def _get_filename(self, data_adaptor, anno_name):
+        # we need to generate a file name, which we can only do if we have a UID and collection name
+        if session is None:
+            raise AnnotationsError("unable to determine file name for annotations")
+
+        collection = self.get_collection()
+        if collection is None:
+            return None
+
+        if data_adaptor is None:
+            raise AnnotationsError("unable to determine file name for annotations")
+
+        idhash = self._get_userdata_idhash(data_adaptor)
+        return os.path.join(self._get_output_dir(), f"{collection}-{anno_name}-{idhash}.csv")
+
+    def _backup(self, fname, max_backups=9):
+        """
+        save N backups of file to backup_dir.
+            1. fname -> backup_dir/fname-TIME
+            2. delete excess files in backup_dir
+        """
+        root, ext = os.path.splitext(fname)
+        backup_dir = f"{root}-backups"
+
+        # Make sure there is work to do
+        if not os.path.exists(fname):
+            return
+
+        # Ensure backup_dir exists
+        if not os.path.exists(backup_dir):
+            os.mkdir(backup_dir)
+
+        # Save current file to backup_dir
+        fname_base = os.path.basename(fname)
+        fname_base_root, fname_base_ext = os.path.splitext(fname_base)
+        # don't use ISO standard time format, as it contains characters illegal on some filesytems.
+        nowish = datetime.now().strftime("%Y-%m-%dT%H-%M-%S")
+        backup_fname = os.path.join(backup_dir, f"{fname_base_root}-{nowish}{fname_base_ext}")
+        if os.path.exists(backup_fname):
+            os.remove(backup_fname)
+        os.rename(fname, backup_fname)
+
+        # prune the backup_dir to max number of backup files, keeping the most recent backups
+        backups = list(filter(lambda s: s.startswith(fname_base_root), os.listdir(backup_dir)))
+        excess_count = len(backups) - max_backups
+        if excess_count > 0:
+            backups.sort()
+            for bu in backups[0:excess_count]:
+                os.remove(os.path.join(backup_dir, bu))
+
+    def update_parameters(self, parameters, data_adaptor):
+        params = {}
+        params["annotations"] = self.user_annotations_enabled()
+        params["annotations_genesets_readonly"] = not self.gene_sets_save_enabled()
+        params["annotations_genesets_name_is_read_only"] = self.gene_sets_output_file is not None
+        params["user_annotation_collection_name_enabled"] = True
+
+        if self.label_output_file is not None:
+            # user has hard-wired the name of the annotation cell label data collection
+            fname = os.path.basename(self.label_output_file)
+            collection_fname = os.path.splitext(fname)[0]
+            params["annotations-data-collection-is-read-only"] = True
+            params["annotations-data-collection-name"] = collection_fname
+
+        elif session is not None:
+            collection = self.get_collection()
+            params["annotations-data-collection-is-read-only"] = not self.user_annotations_enabled()
+            params["annotations-data-collection-name"] = collection
+
+        params["annotations-user-data-idhash"] = self._get_userdata_idhash(data_adaptor)
+
+        parameters.update(params)

server/common/colors.py

@@ -0,0 +1,233 @@
+import re
+
+from server.common.errors import ColorFormatException
+
+HEX_COLOR_FORMAT = re.compile("^#[a-fA-F0-9]{6,6}$")
+
+# https://www.w3.org/TR/css-color-4/#named-colors
+CSS4_NAMED_COLORS = dict(
+    aliceblue="#f0f8ff",
+    antiquewhite="#faebd7",
+    aqua="#00ffff",
+    aquamarine="#7fffd4",
+    azure="#f0ffff",
+    beige="#f5f5dc",
+    bisque="#ffe4c4",
+    black="#000000",
+    blanchedalmond="#ffebcd",
+    blue="#0000ff",
+    blueviolet="#8a2be2",
+    brown="#a52a2a",
+    burlywood="#deb887",
+    cadetblue="#5f9ea0",
+    chartreuse="#7fff00",
+    chocolate="#d2691e",
+    coral="#ff7f50",
+    cornflowerblue="#6495ed",
+    cornsilk="#fff8dc",
+    crimson="#dc143c",
+    cyan="#00ffff",
+    darkblue="#00008b",
+    darkcyan="#008b8b",
+    darkgoldenrod="#b8860b",
+    darkgray="#a9a9a9",
+    darkgreen="#006400",
+    darkgrey="#a9a9a9",
+    darkkhaki="#bdb76b",
+    darkmagenta="#8b008b",
+    darkolivegreen="#556b2f",
+    darkorange="#ff8c00",
+    darkorchid="#9932cc",
+    darkred="#8b0000",
+    darksalmon="#e9967a",
+    darkseagreen="#8fbc8f",
+    darkslateblue="#483d8b",
+    darkslategray="#2f4f4f",
+    darkslategrey="#2f4f4f",
+    darkturquoise="#00ced1",
+    darkviolet="#9400d3",
+    deeppink="#ff1493",
+    deepskyblue="#00bfff",
+    dimgray="#696969",
+    dimgrey="#696969",
+    dodgerblue="#1e90ff",
+    firebrick="#b22222",
+    floralwhite="#fffaf0",
+    forestgreen="#228b22",
+    fuchsia="#ff00ff",
+    gainsboro="#dcdcdc",
+    ghostwhite="#f8f8ff",
+    gold="#ffd700",
+    goldenrod="#daa520",
+    gray="#808080",
+    green="#008000",
+    greenyellow="#adff2f",
+    grey="#808080",
+    honeydew="#f0fff0",
+    hotpink="#ff69b4",
+    indianred="#cd5c5c",
+    indigo="#4b0082",
+    ivory="#fffff0",
+    khaki="#f0e68c",
+    lavender="#e6e6fa",
+    lavenderblush="#fff0f5",
+    lawngreen="#7cfc00",
+    lemonchiffon="#fffacd",
+    lightblue="#add8e6",
+    lightcoral="#f08080",
+    lightcyan="#e0ffff",
+    lightgoldenrodyellow="#fafad2",
+    lightgray="#d3d3d3",
+    lightgreen="#90ee90",
+    lightgrey="#d3d3d3",
+    lightpink="#ffb6c1",
+    lightsalmon="#ffa07a",
+    lightseagreen="#20b2aa",
+    lightskyblue="#87cefa",
+    lightslategray="#778899",
+    lightslategrey="#778899",
+    lightsteelblue="#b0c4de",
+    lightyellow="#ffffe0",
+    lime="#00ff00",
+    limegreen="#32cd32",
+    linen="#faf0e6",
+    magenta="#ff00ff",
+    maroon="#800000",
+    mediumaquamarine="#66cdaa",
+    mediumblue="#0000cd",
+    mediumorchid="#ba55d3",
+    mediumpurple="#9370db",
+    mediumseagreen="#3cb371",
+    mediumslateblue="#7b68ee",
+    mediumspringgreen="#00fa9a",
+    mediumturquoise="#48d1cc",
+    mediumvioletred="#c71585",
+    midnightblue="#191970",
+    mintcream="#f5fffa",
+    mistyrose="#ffe4e1",
+    moccasin="#ffe4b5",
+    navajowhite="#ffdead",
+    navy="#000080",
+    oldlace="#fdf5e6",
+    olive="#808000",
+    olivedrab="#6b8e23",
+    orange="#ffa500",
+    orangered="#ff4500",
+    orchid="#da70d6",
+    palegoldenrod="#eee8aa",
+    palegreen="#98fb98",
+    paleturquoise="#afeeee",
+    palevioletred="#db7093",
+    papayawhip="#ffefd5",
+    peachpuff="#ffdab9",
+    peru="#cd853f",
+    pink="#ffc0cb",
+    plum="#dda0dd",
+    powderblue="#b0e0e6",
+    purple="#800080",
+    rebeccapurple="#663399",
+    red="#ff0000",
+    rosybrown="#bc8f8f",
+    royalblue="#4169e1",
+    saddlebrown="#8b4513",
+    salmon="#fa8072",
+    sandybrown="#f4a460",
+    seagreen="#2e8b57",
+    seashell="#fff5ee",
+    sienna="#a0522d",
+    silver="#c0c0c0",
+    skyblue="#87ceeb",
+    slateblue="#6a5acd",
+    slategray="#708090",
+    slategrey="#708090",
+    snow="#fffafa",
+    springgreen="#00ff7f",
+    steelblue="#4682b4",
+    tan="#d2b48c",
+    teal="#008080",
+    thistle="#d8bfd8",
+    tomato="#ff6347",
+    turquoise="#40e0d0",
+    violet="#ee82ee",
+    wheat="#f5deb3",
+    white="#ffffff",
+    whitesmoke="#f5f5f5",
+    yellow="#ffff00",
+    yellowgreen="#9acd32",
+)
+
+
+def convert_color_to_hex_format(unknown):
+    """
+    Try to convert color info to a hex triplet string https://en.wikipedia.org/wiki/Web_colors#Hex_triplet.
+
+    The function accepts for the following formats:
+    - A CSS4 color name, as supported by matplotlib https://matplotlib.org/3.1.0/gallery/color/named_colors.html
+    - RGB tuple/list with values ranging from 0.0 to 1.0, as in [0.5, 0.75, 1.0]
+    - RFB tuple/list with values ranging from 0 to 255, as in [128, 192, 255]
+    - Hex triplet string, as in "#08c0ff"
+
+    :param unknown: color info of unknown format
+    :return: a hex triplet representing that color
+    """
+    try:
+        if type(unknown) in (list, tuple) and len(unknown) == 3:
+            if all(0.0 <= ele <= 1.0 for ele in unknown):
+                tup = tuple(int(ele * 255) for ele in unknown)
+            elif all(0 <= ele <= 255 and isinstance(ele, int) for ele in unknown):
+                tup = tuple(unknown)
+            else:
+                raise ColorFormatException("Unknown color iterable format!")
+            return "#%02x%02x%02x" % tup
+        elif isinstance(unknown, str) and unknown.lower() in CSS4_NAMED_COLORS:
+            return CSS4_NAMED_COLORS[unknown.lower()]
+        elif isinstance(unknown, str) and HEX_COLOR_FORMAT.match(unknown):
+            return unknown.lower()
+        else:
+            raise ColorFormatException("Unknown color format type!")
+    except Exception as e:
+        raise ColorFormatException(e)
+
+
+def convert_anndata_category_colors_to_cxg_category_colors(data):
+    """
+    Convert color information from anndata files to the cellxgene color data format as described below:
+        {
+            "<category_name>": {
+                "<label_name>": "<color_hex_code>",
+                ...
+            },
+            ...
+        }
+
+    For more on the cxg color data structure, see https://github.com/chanzuckerberg/cellxgene/issues/1307.
+
+    For more on the anndata color data structure, see
+    https://github.com/chanzuckerberg/cellxgene/issues/1152#issuecomment-587276178.
+
+    Handling of malformed data:
+    - For any color info in a adata.uns[f"{category}_colors"] color array that convert_color_to_hex_format cannot
+      convert to a hex triplet string, a ColorFormatException is raised
+    - No category_name key group is returned for adata.uns[f"{category}_colors"] keys for which there is no
+      adata.obs[f"{category}"] key
+
+    :param data: the anndata file
+    :return: cellxgene color data structure as described above
+    """
+    cxg_colors = dict()
+    color_key_suffix = "_colors"
+    for uns_key in data.uns.keys():
+        # find uns array that describes colors for a category
+        if not uns_key.endswith(color_key_suffix):
+            continue
+
+        # check to see if we actually have observations for that category
+        category_name = uns_key[: -len(color_key_suffix)]
+        if category_name not in data.obs.keys():
+            continue
+
+        # create the cellxgene color entry for this category
+        cxg_colors[category_name] = dict(
+            zip(data.obs[category_name].cat.categories, [convert_color_to_hex_format(c) for c in data.uns[uns_key]])
+        )
+    return cxg_colors

server/common/compute/diffexp_generic.py

@@ -0,0 +1,163 @@
+import numpy as np
+from scipy import sparse, stats
+from server.common.constants import XApproximateDistribution
+
+
+def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
+    """
+    Return differential expression statistics for top N variables.
+
+    Algorithm:
+    - compute fold change
+    - compute Welch's t-test statistic and pvalue (w/ Bonferroni correction)
+    - return top N abs(logfoldchange) where lfc > diffexp_lfc_cutoff
+
+    If there are not N which meet criteria, augment by removing the logfoldchange
+    threshold requirement.
+
+    Notes on alogrithm:
+    - Welch's ttest provides basic statistics test.
+      https://en.wikipedia.org/wiki/Welch%27s_t-test
+    - p-values adjusted with Bonferroni correction.
+      https://en.wikipedia.org/wiki/Bonferroni_correction
+
+    :param adaptor: DataAdaptor instance
+    :param maskA: observation selection mask for set 1
+    :param maskB: observation selection mask for set 2
+    :param top_n: number of variables to return stats for
+    :param diffexp_lfc_cutoff: minimum
+    absolute value returning [ varindex, logfoldchange, pval, pval_adj ] for top N genes
+    :return:  for top N genes, {"positive": for top N genes, [ varindex, foldchange, pval, pval_adj ],
+    "negative": for top N genes, [ varindex, foldchange, pval, pval_adj ]}
+    """
+
+    X_approximate_distribution = adaptor.get_X_approximate_distribution()
+    dataA = adaptor.get_X_array(maskA, None)
+    dataB = adaptor.get_X_array(maskB, None)
+
+    # mean, variance, N - calculate for both selections
+    meanA, vA, nA = mean_var_n(dataA, X_approximate_distribution)
+    meanB, vB, nB = mean_var_n(dataB, X_approximate_distribution)
+    res = diffexp_ttest_from_mean_var(meanA, vA, nA, meanB, vB, nB, top_n, diffexp_lfc_cutoff)
+
+    return res
+
+
+def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp_lfc_cutoff):
+    # IMPORTANT NOTE: this code assumes the data is normally distributed and/or already logged.
+
+    n_var = meanA.shape[0]
+    top_n = min(top_n, n_var)
+
+    # variance / N
+    vnA = varA / min(nA, nB)  # overestimate variance, would normally be nA
+    vnB = varB / min(nA, nB)  # overestimate variance, would normally be nB
+    sum_vn = vnA + vnB
+
+    # degrees of freedom for Welch's t-test
+    with np.errstate(divide="ignore", invalid="ignore"):
+        dof = sum_vn ** 2 / (vnA ** 2 / (nA - 1) + vnB ** 2 / (nB - 1))
+    dof[np.isnan(dof)] = 1
+
+    # Welch's t-test score calculation
+    with np.errstate(divide="ignore", invalid="ignore"):
+        tscores = (meanA - meanB) / np.sqrt(sum_vn)
+    tscores[np.isnan(tscores)] = 0
+
+    # p-value
+    pvals = stats.t.sf(np.abs(tscores), dof) * 2
+    pvals_adj = pvals * n_var
+    pvals_adj[pvals_adj > 1] = 1  # cap adjusted p-value at 1
+
+    # log fold change. The data is normally distributed/logged, so just subtract the means.
+    logfoldchanges = meanA - meanB
+
+    stats_to_sort = tscores
+    # find all with lfc > cutoff
+    lfc_above_cutoff_idx = np.nonzero(np.abs(logfoldchanges) > diffexp_lfc_cutoff)[0]
+
+    # derive sort order
+    if lfc_above_cutoff_idx.shape[0] > top_n * 2:
+        # partition top N
+        rel_t_partition = np.argpartition(stats_to_sort[lfc_above_cutoff_idx], (top_n, -top_n))
+        rel_t_partition_top_n = np.concatenate((rel_t_partition[-top_n:], rel_t_partition[:top_n]))
+        t_partition = lfc_above_cutoff_idx[rel_t_partition_top_n]
+        # sort the top N partition
+        rel_sort_order = np.argsort(stats_to_sort[t_partition])[::-1]
+        sort_order = t_partition[rel_sort_order]
+    else:
+        # partition and sort top N, ignoring lfc cutoff
+        partition = np.argpartition(stats_to_sort, (top_n, -top_n))
+        partition_top_n = np.concatenate((partition[-top_n:], partition[:top_n]))
+
+        rel_sort_order = np.argsort(stats_to_sort[partition_top_n])[::-1]
+        indices = np.indices(stats_to_sort.shape)[0]
+        sort_order = indices[partition_top_n][rel_sort_order]
+
+    # top n slice based upon sort order
+    logfoldchanges_top_n = logfoldchanges[sort_order]
+    pvals_top_n = pvals[sort_order]
+    pvals_adj_top_n = pvals_adj[sort_order]
+
+    # varIndex, logfoldchange, pval, pval_adj
+    result = {
+        "positive": [
+            [sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in range(top_n)
+        ],
+        "negative": [
+            [sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]]
+            for i in range(-1, -1 - top_n, -1)
+        ],
+    }
+
+    return result
+
+
+# Convenience function which handles sparse data
+def mean_var_n(X, X_approximate_distribution=XApproximateDistribution.NORMAL):
+    """
+    Two-pass variance calculation.  Numerically (more) stable
+    than naive methods (and same method used by numpy.var())
+    https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Two-pass
+    """
+    # fp_err_occurred is a flag indicating that a floating point error
+    # occured somewhere in our compute.  Used to trigger non-finite
+    # number handling.
+    fp_err_occurred = False
+
+    def fp_err_set(err, flag):
+        nonlocal fp_err_occurred
+        fp_err_occurred = True
+
+    with np.errstate(divide="call", invalid="call", call=fp_err_set):
+        n = X.shape[0]
+        if sparse.issparse(X):
+            if X_approximate_distribution == XApproximateDistribution.COUNT:
+                X = X.log1p()
+            mean = X.mean(axis=0).A1
+            dfm = X - mean
+            sumsq = np.sum(np.multiply(dfm, dfm), axis=0).A1
+            v = sumsq / (n - 1)
+        else:
+            if X_approximate_distribution == XApproximateDistribution.COUNT:
+                X = np.log1p(X)
+            mean = X.mean(axis=0)
+            dfm = X - mean
+            sumsq = np.sum(np.multiply(dfm, dfm), axis=0)
+            v = sumsq / (n - 1)
+
+    # AnnData does not guarantee that operations on a view of X will
+    # return an ndarray, so force the cast if it wasn't done for us.
+    if type(mean) is not np.ndarray:
+        mean = mean.toarray()
+    if type(v) is not np.ndarray:
+        v = v.toarray()
+
+    if fp_err_occurred:
+        mean[np.isfinite(mean) == False] = 0  # noqa: E712
+        v[np.isfinite(v) == False] = 0  # noqa: E712
+    else:
+        mean[np.isnan(mean)] = 0
+        v[np.isnan(v)] = 0
+
+    return mean, v, n

server/common/compute/estimate_distribution.py

@@ -0,0 +1,108 @@
+from typing import Tuple
+import numba
+import concurrent.futures
+import numpy as np
+from scipy import sparse
+from server.common.constants import XApproximateDistribution
+
+
+@numba.njit(error_model="numpy", nogil=True)
+def min_max_fast(arr: np.ndarray) -> Tuple[float, float]:
+    """Return (min, max) values for the ndarray."""
+
+    # initialize to first finite value in array.  Normally,
+    # this will exit on the first value.
+    for i in range(arr.size):
+        min_val = max_val = arr[i]
+        if np.isfinite(min_val):
+            break
+
+    # now find min/max, unrolled by two
+    odd = arr.size % 2
+    unrolled_loop_limit = arr.size - 1 if odd else arr.size
+    i = 0
+    while i < unrolled_loop_limit:
+        x = arr[i]
+        y = arr[i + 1]
+
+        # ignore non-finites
+        x = x if np.isfinite(x) else min_val
+        y = y if np.isfinite(y) else min_val
+
+        if x > y:
+            x, y = y, x
+        min_val = min(x, min_val)
+        max_val = max(y, max_val)
+        i += 2
+
+    # handle the tail if any
+    if odd:
+        x = arr[arr.size - 1]
+
+        # ignore non-finites
+        x = x if np.isfinite(x) else min_val
+
+        min_val = min(x, min_val)
+        max_val = max(x, max_val)
+
+    return min_val, max_val
+
+
+def min_max_numpy(arr: np.ndarray) -> Tuple[float, float]:
+    return arr.min(), arr.max()
+
+
+def numba_has_support_for_scalar_type(arr: np.ndarray) -> bool:
+    """Numba does not support half-floats, 128 bit floats, ints > 64 bit or non-scalars."""
+    if arr.dtype == np.float32 or arr.dtype == np.float64:
+        return True
+
+    if np.issubdtype(arr.dtype, np.integer) and arr.dtype <= np.int64:
+        return True
+
+    if arr.dtype == np.bool_:
+        return True
+
+    return False
+
+
+def estimate_approximate_distribution(X) -> XApproximateDistribution:
+    """
+    Estimate the distribution (normal, count) of the X matrix.
+
+    Currently this is based upon the assumption that scRNA-seq data is
+    exponentially distributed in its raw (count) form, and when logged,
+    any (max-min) range in excess of 24 is implies tens of millions of
+    observations of a single feature and so is extremely unlikely.
+    """
+    if X.dtype.kind not in ["i", "u", "f"]:
+        raise TypeError(f"Unsupported matrix dtype: {X.dtype.name}")
+
+    if X.size == 0:
+        # default for empty array
+        return XApproximateDistribution.NORMAL
+
+    if sparse.isspmatrix_csc(X) or sparse.isspmatrix_csr(X):
+        Xdata = X.data
+    elif type(X) is np.ndarray:
+        Xdata = X.reshape(
+            X.size,
+        )
+    else:
+        raise TypeError(f"Unsupported matrix format: {str(type(X))}")
+
+    min_max = min_max_fast if numba_has_support_for_scalar_type(Xdata) else min_max_numpy
+
+    CHUNKSIZE = 1 << 24
+    if Xdata.size > CHUNKSIZE:
+        min_val = max_val = Xdata[0]
+        with concurrent.futures.ThreadPoolExecutor() as tp:
+            for (_min, _max) in tp.map(min_max, [Xdata[i : i + CHUNKSIZE] for i in range(0, Xdata.size, CHUNKSIZE)]):
+                min_val = min(_min, min_val)
+                max_val = max(_max, max_val)
+
+    else:
+        min_val, max_val = min_max(Xdata)
+
+    excess_range = (max_val - min_val) > 24
+    return XApproximateDistribution.COUNT if excess_range else XApproximateDistribution.NORMAL

server/common/config/__init__.py

@@ -0,0 +1,2 @@
+DEFAULT_SERVER_PORT = 5005
+BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20  # 100MB

server/common/config/app_config.py

@@ -0,0 +1,171 @@
+import yaml
+from flatten_dict import unflatten
+
+from server.default_config import get_default_config
+from server.common.config.dataset_config import DatasetConfig
+from server.common.config.server_config import ServerConfig
+from server.common.config.external_config import ExternalConfig
+from server.common.errors import ConfigurationError
+
+
+class AppConfig(object):
+    """
+    AppConfig stores all the configuration for cellxgene.
+    AppConfig contains one or more DatasetConfig(s) and one ServerConfig.
+    The server_config contains attributes that refer to the server process as a whole.
+    The dataset_config refers to attributes that are associated with the features and
+    presentations of a dataset.
+    AppConfig has methods to initialize, modify, and access the configuration.
+    """
+
+    def __init__(self):
+
+        # the default configuration (see default_config.py)
+        # TODO @madison -- if we always read from the default config (hard coded path) can we set those values as
+        #  defaults within the config class?
+        self.default_config = get_default_config()
+        # the server configuration
+        self.server_config = ServerConfig(self, self.default_config["server"])
+        # the dataset config
+        self.dataset_config = DatasetConfig(None, self, self.default_config["dataset"])
+        #  external config
+        self.external_config = ExternalConfig(self, self.default_config["external"])
+
+        # Set to true when config_completed is called
+        self.is_completed = False
+
+    def get_dataset_config(self):
+        return self.dataset_config
+
+    def check_config(self):
+        """Verify all the attributes in the config have been type checked"""
+        if not self.is_completed:
+            raise ConfigurationError("The configuration has not been completed")
+        self.server_config.check_config()
+        self.dataset_config.check_config()
+        self.external_config.check_config()
+
+    def update_server_config(self, **kw):
+        self.server_config.update(**kw)
+        self.is_complete = False
+
+    def update_dataset_config(self, **kw):
+        self.dataset_config.update(**kw)
+        self.is_complete = False
+
+    def update_single_config_from_path_and_value(self, path, value):
+        """Update a single config parameter with the value.
+        Path is a list of string, that gives a path to the config parameter to be updated.
+        For example, path may be ["server","app","port"].
+        """
+        self.is_complete = False
+        if not isinstance(path, list):
+            raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'")
+        for part in path:
+            if not isinstance(part, str):
+                raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'")
+
+        if len(path) < 1 or path[0] not in ("server", "dataset"):
+            raise ConfigurationError("path must start with 'server', or 'dataset'")
+
+        if path[0] == "server":
+            attr = "__".join(path[1:])
+            try:
+                self.update_server_config(**{attr: value})
+            except ConfigurationError:
+                raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
+        elif path[0] == "dataset":
+            attr = "__".join(path[1:])
+            try:
+                self.update_dataset_config(**{attr: value})
+            except ConfigurationError:
+                raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
+
+    def update_from_config_file(self, config_file):
+        try:
+            with open(config_file) as yml_file:
+                config = yaml.safe_load(yml_file)
+        except yaml.YAMLError as e:
+            raise ConfigurationError(f"The specified config file contained an error: {e}")
+        except OSError as e:
+            raise ConfigurationError(f"Issue retrieving the specified config file: {e}")
+
+        if config.get("server"):
+            self.server_config.update_from_config(config["server"], "server")
+        if config.get("dataset"):
+            self.dataset_config.update_from_config(config["dataset"], "dataset")
+
+        if config.get("external"):
+            self.external_config.update_from_config(config["external"], "external")
+
+        self.is_complete = False
+
+    def config_to_dict(self):
+        """return the configuration as an unflattened dict"""
+        server = self.server_config.create_mapping(self.server_config.default_config)
+        dataset = self.dataset_config.create_mapping(self.dataset_config.default_config)
+        external = self.external_config.create_mapping(self.external_config.default_config)
+        config = dict(server={}, dataset={})
+        for attrname in server.keys():
+            config["server__" + attrname] = getattr(self.server_config, attrname)
+        for attrname in dataset.keys():
+            config["dataset__" + attrname] = getattr(self.dataset_config, attrname)
+        for attrname in external.keys():
+            config["external__" + attrname] = getattr(self.external_config, attrname)
+
+        config = unflatten(config, splitter=lambda key: key.split("__"))
+        return config
+
+    def write_config(self, config_file):
+        """output the config to a yaml file"""
+        config = self.config_to_dict()
+        yaml.dump(config, open(config_file, "w"))
+
+    def changes_from_default(self):
+        """Return all the attribute that are different from the default"""
+        diff_server = self.server_config.changes_from_default()
+        diff_dataset = self.dataset_config.changes_from_default()
+        diff_external = self.external.changes_from_default()
+        diff = dict(server=diff_server, dataset=diff_dataset, external=diff_external)
+        return diff
+
+    def complete_config(self, messagefn=None):
+        """The configure options are checked, and any additional setup based on the config
+        parameters is done"""
+
+        if messagefn is None:
+
+            def noop(message):
+                pass
+
+            messagefn = noop
+
+        # TODO: to give better error messages we can add a mapping between where each config
+        # attribute originated (e.g. command line argument or config file), then in the error
+        # messages we can give correct context for attributes with bad value.
+        context = dict(messagefn=messagefn)
+
+        # complete config for external_config first, since this may update values in the other sections
+        self.external_config.complete_config(context)
+        self.server_config.complete_config(context)
+        self.dataset_config.complete_config(context)
+
+        self.is_completed = True
+        self.check_config()
+
+    def get_matrix_data_cache_manager(self):
+        return self.server_config.matrix_data_cache_manager
+
+    def get_title(self, data_adaptor):
+        return (
+            self.server_config.single_dataset__title
+            if self.server_config.single_dataset__title
+            else data_adaptor.get_title()
+        )
+
+    def get_about(self, data_adaptor):
+        return (
+            self.server_config.single_dataset__about
+            if self.server_config.single_dataset__about
+            else data_adaptor.get_about()
+        )

server/common/config/base_config.py

@@ -0,0 +1,99 @@
+import copy
+
+from flatten_dict import flatten
+from server.common.errors import ConfigurationError
+
+
+class BaseConfig(object):
+    """
+    This class handles the mechanics of updating and checking attributes.
+    Derived classes are expected to store the actual attributes
+    Currently DatasetConfig and ServerConfig both inherit from BaseConfig.
+    """
+
+    def __init__(self, app_config, default_config):
+        # reference back to the app_config
+        self.app_config = app_config
+        # the complete set of attributes and their default values (unflattened)
+        self.default_config = default_config
+        # used to make sure every attribute value is checked
+        self.attr_checked = {key_name: False for key_name in self.create_mapping(default_config).keys()}
+
+    def create_mapping(self, config):
+        """
+        Create a dictionary where the keys are the name of attributes (using double underscore convention)
+        For example: app__host
+
+        The values are a tuple,
+        - the first item of the tuple is a tuple of path elements (location in config 'tree')
+        - the second item is the value of the config parameter
+
+        For example: (("app", "host"), "session"))
+        """
+        config_copy = copy.deepcopy(config)
+        mapping = {}
+
+        flat_config = flatten(config_copy)
+        for key, value in flat_config.items():
+            # name of the attribute
+            attr = "__".join(key)
+            mapping[attr] = (key, value)
+
+        return mapping
+
+    def validate_correct_type_of_configuration_attribute(self, attrname, vtype):
+        val = getattr(self, attrname)
+        if type(vtype) in (list, tuple):
+            if type(val) not in vtype:
+                tnames = ",".join([x.__name__ for x in vtype])
+                raise ConfigurationError(
+                    f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}"
+                )
+        else:
+            if type(val) != vtype:
+                raise ConfigurationError(
+                    f"Invalid type for attribute: {attrname}, "
+                    f"expected type {vtype.__name__}, got {type(val).__name__}"
+                )
+
+        self.attr_checked[attrname] = True
+
+    def check_config(self):
+        mapping = self.create_mapping(self.default_config)
+        for key in mapping.keys():
+            if not self.attr_checked[key]:
+                raise ConfigurationError(f"The attr '{key}' has not been checked")
+
+    def update(self, **kw):
+        """Update the attributes defined in kw with their new values."""
+        for key, value in kw.items():
+            if not hasattr(self, key):
+                raise ConfigurationError(f"unknown config parameter {key}.")
+            try:
+                if type(value) == tuple:
+                    # convert tuple values to list values
+                    value = list(value)
+                setattr(self, key, value)
+            except KeyError:
+                raise ConfigurationError(f"Unable to set config parameter {key}.")
+
+            self.attr_checked[key] = False
+
+    def update_from_config(self, config, prefix):
+        mapping = self.create_mapping(config)
+        for attr, (key, value) in mapping.items():
+            if not hasattr(self, attr):
+                raise ConfigurationError(f"Unknown key from config file: {prefix}__{attr}")
+            setattr(self, attr, value)
+
+            self.attr_checked[attr] = False
+
+    def changes_from_default(self):
+        """Return all the attribute that are different from the default"""
+        mapping = self.create_mapping(self.default_config)
+        diff = []
+        for attrname, (key, defval) in mapping.items():
+            curval = getattr(self, attrname)
+            if curval != defval:
+                diff.append((attrname, curval, defval))
+        return diff

server/common/config/client_config.py

@@ -0,0 +1,81 @@
+from server import display_version as cellxgene_display_version
+
+
+def get_client_config(app_config, data_adaptor):
+    """
+    Return the configuration as required by the /config REST route
+    """
+
+    server_config = app_config.server_config
+    dataset_config = data_adaptor.dataset_config
+    annotation = dataset_config.user_annotations
+
+    # FIXME The current set of config is not consistently presented:
+    # we have camalCase, hyphen-text, and underscore_text
+
+    # make sure the configuration has been checked.
+    app_config.check_config()
+
+    # display_names
+    title = app_config.get_title(data_adaptor)
+    about = app_config.get_about(data_adaptor)
+
+    display_names = dict(engine=data_adaptor.get_name(), dataset=title)
+
+    # library_versions
+    library_versions = {}
+    library_versions.update(data_adaptor.get_library_versions())
+    library_versions["cellxgene"] = cellxgene_display_version
+
+    # links
+    links = {"about-dataset": about}
+
+    # parameters
+    parameters = {
+        "layout": dataset_config.embeddings__names,
+        "max-category-items": dataset_config.presentation__max_categories,
+        "obs_names": server_config.single_dataset__obs_names,
+        "var_names": server_config.single_dataset__var_names,
+        "diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff,
+        "backed": server_config.adaptor__anndata_adaptor__backed,
+        "disable-diffexp": not dataset_config.diffexp__enable,
+        "annotations": False,
+        "annotations_file": None,
+        "annotations_dir": None,
+        "annotations_genesets": True,  # feature flag
+        "annotations_genesets_readonly": dataset_config.user_annotations__gene_sets__readonly,
+        "annotations_genesets_summary_methods": ["mean"],
+        "custom_colors": dataset_config.presentation__custom_colors,
+        "diffexp-may-be-slow": False,
+    }
+
+    # corpora dataset_props
+    # TODO/Note: putting info from the dataset into the /config is not ideal.
+    # However, it is definitely not part of /schema, and we do not have a top-level
+    # route for data properties.  Consider creating one at some point.
+    corpora_props = data_adaptor.get_corpora_props()
+    if corpora_props and "default_embedding" in corpora_props:
+        default_embedding = corpora_props["default_embedding"]
+        if isinstance(default_embedding, str) and default_embedding.startswith("X_"):
+            default_embedding = default_embedding[2:]  # drop X_ prefix
+        if default_embedding in data_adaptor.get_embedding_names():
+            parameters["default_embedding"] = default_embedding
+
+    data_adaptor.update_parameters(parameters)
+    if annotation:
+        annotation.update_parameters(parameters, data_adaptor)
+
+    # gather it all together
+    client_config = {}
+    config = client_config["config"] = {}
+    config["displayNames"] = display_names
+    config["library_versions"] = library_versions
+    config["links"] = links
+    config["parameters"] = parameters
+    config["corpora_props"] = corpora_props
+    config["limits"] = {
+        "column_request_max": server_config.limits__column_request_max,
+        "diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max,
+    }
+
+    return client_config

server/common/config/dataset_config.py

@@ -0,0 +1,184 @@
+import os
+from os.path import splitext, isdir
+
+from server.common.annotations.local_file_csv import AnnotationsLocalFile
+from server.common.config.base_config import BaseConfig
+from server.common.errors import ConfigurationError, AnnotationsError
+from server.data_common.matrix_loader import MatrixDataLoader
+
+
+class DatasetConfig(BaseConfig):
+    """Manages the config attribute associated with a dataset."""
+
+    def __init__(self, tag, app_config, default_config):
+        super().__init__(app_config, default_config)
+        self.tag = tag
+        try:
+            self.app__scripts = default_config["app"]["scripts"]
+            self.app__inline_scripts = default_config["app"]["inline_scripts"]
+
+            self.presentation__max_categories = default_config["presentation"]["max_categories"]
+            self.presentation__custom_colors = default_config["presentation"]["custom_colors"]
+
+            self.user_annotations__enable = default_config["user_annotations"]["enable"]
+            self.user_annotations__type = default_config["user_annotations"]["type"]
+            self.user_annotations__local_file_csv__directory = default_config["user_annotations"]["local_file_csv"][
+                "directory"
+            ]
+            self.user_annotations__local_file_csv__file = default_config["user_annotations"]["local_file_csv"]["file"]
+            self.user_annotations__gene_sets__readonly = default_config["user_annotations"]["gene_sets"]["readonly"]
+            self.user_annotations__local_file_csv__gene_sets_file = default_config["user_annotations"][
+                "local_file_csv"
+            ]["gene_sets_file"]
+
+            self.embeddings__names = default_config["embeddings"]["names"]
+
+            self.diffexp__enable = default_config["diffexp"]["enable"]
+            self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
+            self.diffexp__top_n = default_config["diffexp"]["top_n"]
+
+            self.X_approximate_distribution = default_config["X_approximate_distribution"]
+
+        except KeyError as e:
+            raise ConfigurationError(f"Unexpected config: {str(e)}")
+
+        # The annotation object is created during complete_config and stored here.
+        self.user_annotations = None
+
+    def complete_config(self, context):
+        self.handle_app()
+        self.handle_presentation()
+        self.handle_user_annotations(context)
+        self.handle_embeddings()
+        self.handle_diffexp(context)
+        self.handle_X_approximate_distribution()
+
+    def get_data_adaptor(self):
+        server_config = self.app_config.server_config
+        if not server_config.data_adaptor:
+            matrix_data_loader = MatrixDataLoader(server_config.single_dataset__datapath, app_config=self.app_config)
+            server_config.data_adaptor = matrix_data_loader.open(self.app_config)
+
+        return server_config.data_adaptor
+
+    def handle_app(self):
+        self.validate_correct_type_of_configuration_attribute("app__scripts", list)
+        self.validate_correct_type_of_configuration_attribute("app__inline_scripts", list)
+
+        # scripts can be string (filename) or dict (attributes). Convert string to dict.
+        scripts = []
+        for script in self.app__scripts:
+            try:
+                if isinstance(script, str):
+                    scripts.append({"src": script})
+                elif isinstance(script, dict) and isinstance(script["src"], str):
+                    scripts.append(script)
+                else:
+                    raise Exception
+            except Exception as e:
+                raise ConfigurationError(f"Scripts must be string or a dict containing an src key: {e}")
+
+        self.app__scripts = scripts
+
+    def handle_presentation(self):
+        self.validate_correct_type_of_configuration_attribute("presentation__max_categories", int)
+        self.validate_correct_type_of_configuration_attribute("presentation__custom_colors", bool)
+
+    def handle_user_annotations(self, context):
+        self.validate_correct_type_of_configuration_attribute("user_annotations__enable", bool)
+        self.validate_correct_type_of_configuration_attribute("user_annotations__type", str)
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__local_file_csv__directory", (type(None), str)
+        )
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__local_file_csv__file", (type(None), str)
+        )
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__local_file_csv__gene_sets_file", (type(None), str)
+        )
+        self.validate_correct_type_of_configuration_attribute("user_annotations__gene_sets__readonly", bool)
+
+        # Must always have an annotations instance to support genesets. User annotation (cell labels) are optional
+        # as are writable gene sets
+        if self.user_annotations__type == "local_file_csv":
+            self.handle_local_file_csv_annotations(context)
+        else:
+            raise ConfigurationError('The only annotation type support is "local_file_csv"')
+
+        self.check_annotation_config_vars_not_set(context)
+
+    def handle_local_file_csv_annotations(self, context):
+        dirname = self.user_annotations__local_file_csv__directory
+        filename = self.user_annotations__local_file_csv__file
+        genesets_filename = self.user_annotations__local_file_csv__gene_sets_file
+
+        if dirname is not None and (filename is not None or genesets_filename is not None):
+            raise ConfigurationError(
+                "'user-generated-data-dir' may not be used with 'annotations-file' or 'gene-sets-file'."
+            )
+
+        if filename is not None:
+            lf_name, lf_ext = splitext(filename)
+            if lf_ext and lf_ext != ".csv":
+                raise ConfigurationError(f"annotation file type must be .csv: {filename}")
+
+        if genesets_filename is not None:
+            lf_name, lf_ext = splitext(genesets_filename)
+            if lf_ext and lf_ext != ".csv":
+                raise ConfigurationError(f"genesets file type must be .csv: {genesets_filename}")
+
+        if dirname is not None and not isdir(dirname):
+            try:
+                os.mkdir(dirname)
+            except OSError:
+                raise ConfigurationError("Unable to create directory specified by --user-generated-data-dir")
+
+        anno_config = {
+            "user-annotations": self.user_annotations__enable,
+            "genesets-save": not self.user_annotations__gene_sets__readonly,
+        }
+        self.user_annotations = AnnotationsLocalFile(anno_config, dirname, filename, genesets_filename)
+
+        # if the user has specified a fixed label file, go ahead and validate it
+        # so that we can remove errors early in the process.
+        server_config = self.app_config.server_config
+        if server_config.single_dataset__datapath:
+            data_adaptor = self.get_data_adaptor()
+            if self.user_annotations__local_file_csv__file:
+                self.user_annotations.read_labels(data_adaptor)
+            if self.user_annotations__local_file_csv__gene_sets_file:
+                try:
+                    self.user_annotations.read_gene_sets(data_adaptor, context)
+                except (ValueError, AnnotationsError, KeyError) as e:
+                    raise ConfigurationError(f"Unable to read genesets CSV file: {str(e)}") from e
+
+    def check_annotation_config_vars_not_set(self, context):
+        if self.user_annotations__type is not None:
+            dirname = self.user_annotations__local_file_csv__directory
+            filename = self.user_annotations__local_file_csv__file
+            if not self.user_annotations__enable:
+                if filename is not None:
+                    context["messagefn"]("Warning: --annotations-file ignored as annotations are disabled.")
+                if dirname is not None:
+                    context["messagefn"]("Warning: --user-generated-data-dir ignored as annotations are disabled.")
+
+    def handle_embeddings(self):
+        self.validate_correct_type_of_configuration_attribute("embeddings__names", list)
+
+    def handle_diffexp(self, context):
+        self.validate_correct_type_of_configuration_attribute("diffexp__enable", bool)
+        self.validate_correct_type_of_configuration_attribute("diffexp__lfc_cutoff", float)
+        self.validate_correct_type_of_configuration_attribute("diffexp__top_n", int)
+
+        data_adaptor = self.get_data_adaptor()
+        if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False):
+            context["messagefn"](
+                "CAUTION: due to the size of your dataset, " "running differential expression may take longer or fail."
+            )
+
+    def handle_X_approximate_distribution(self):
+        self.validate_correct_type_of_configuration_attribute("X_approximate_distribution", str)
+        if self.X_approximate_distribution not in ["auto", "normal", "count"]:
+            raise ConfigurationError(
+                "X_approximate_distribution has unknown value -- must be 'auto', 'normal' or 'count'."
+            )

server/common/config/external_config.py

@@ -0,0 +1,44 @@
+import os
+
+from server.common.config.base_config import BaseConfig
+from server.common.errors import ConfigurationError
+from server.common.utils.type_conversion_utils import convert_string_to_value
+
+
+class ExternalConfig(BaseConfig):
+    """Manages the config attribute associated with external configuration sources, such as
+    environment variables."""
+
+    def __init__(self, app_config, default_config):
+        super().__init__(app_config, default_config)
+        try:
+            self.environment = default_config["environment"]
+
+        except KeyError as e:
+            raise ConfigurationError(f"Unexpected config: {str(e)}")
+
+    def complete_config(self, context):
+        self.handle_environment(context)
+
+    def handle_environment(self, context):
+        """For each environment variable defined, get the value (if it is set),
+        and set the specified config parameter"""
+        self.validate_correct_type_of_configuration_attribute("environment", list)
+        for envdict in self.environment:
+            name = envdict.get("name")
+            if name is None:
+                raise ConfigurationError("environment: 'name' is missing")
+            required = envdict.get("required", False)
+            if type(required) != bool:
+                raise ConfigurationError("environment: 'required' must be a bool")
+            path = envdict.get("path")
+            if path is None:
+                raise ConfigurationError("environment: 'path' is missing")
+
+            value = os.environ.get(name)
+            if value is None:
+                if required:
+                    raise ConfigurationError(f"required environment variable '{name}' not set")
+            else:
+                value = convert_string_to_value(value)
+                self.app_config.update_single_config_from_path_and_value(path, value)

server/common/config/server_config.py

@@ -0,0 +1,164 @@
+import os
+import sys
+import warnings
+from os.path import basename
+from urllib.parse import urlparse
+
+from server.common.config.base_config import BaseConfig
+from server.common.config import DEFAULT_SERVER_PORT, BIG_FILE_SIZE_THRESHOLD
+from server.common.utils.data_locator import discover_s3_region_name
+from server.common.errors import ConfigurationError, DatasetAccessError
+from server.common.utils.utils import is_port_available, find_available_port, custom_format_warning
+from server.data_common.matrix_loader import MatrixDataLoader
+
+
+class ServerConfig(BaseConfig):
+    """Manages the config attribute associated with the server."""
+
+    def __init__(self, app_config, default_config):
+        super().__init__(app_config, default_config)
+
+        try:
+            self.app__verbose = default_config["app"]["verbose"]
+            self.app__debug = default_config["app"]["debug"]
+            self.app__host = default_config["app"]["host"]
+            self.app__port = default_config["app"]["port"]
+            self.app__open_browser = default_config["app"]["open_browser"]
+            self.app__force_https = default_config["app"]["force_https"]
+            self.app__flask_secret_key = default_config["app"]["flask_secret_key"]
+            self.app__generate_cache_control_headers = default_config["app"]["generate_cache_control_headers"]
+
+            self.single_dataset__datapath = default_config["single_dataset"]["datapath"]
+            self.single_dataset__obs_names = default_config["single_dataset"]["obs_names"]
+            self.single_dataset__var_names = default_config["single_dataset"]["var_names"]
+            self.single_dataset__about = default_config["single_dataset"]["about"]
+            self.single_dataset__title = default_config["single_dataset"]["title"]
+
+            self.data_locator__s3__region_name = default_config["data_locator"]["s3"]["region_name"]
+
+            self.adaptor__anndata_adaptor__backed = default_config["adaptor"]["anndata_adaptor"]["backed"]
+
+            self.limits__diffexp_cellcount_max = default_config["limits"]["diffexp_cellcount_max"]
+            self.limits__column_request_max = default_config["limits"]["column_request_max"]
+
+        except KeyError as e:
+            raise ConfigurationError(f"Unexpected config: {str(e)}")
+
+        self.data_adaptor = None
+
+    def complete_config(self, context):
+        self.handle_app(context)
+        self.handle_data_source()
+        self.handle_data_locator()
+        self.handle_adaptor()  # may depend on data_locator
+        self.handle_single_dataset(context)  # may depend on adaptor
+        self.handle_limits()
+
+        self.check_config()
+
+    def handle_app(self, context):
+        self.validate_correct_type_of_configuration_attribute("app__verbose", bool)
+        self.validate_correct_type_of_configuration_attribute("app__debug", bool)
+        self.validate_correct_type_of_configuration_attribute("app__host", str)
+        self.validate_correct_type_of_configuration_attribute("app__port", (type(None), int))
+        self.validate_correct_type_of_configuration_attribute("app__open_browser", bool)
+        self.validate_correct_type_of_configuration_attribute("app__force_https", bool)
+        self.validate_correct_type_of_configuration_attribute("app__flask_secret_key", str)
+        self.validate_correct_type_of_configuration_attribute("app__generate_cache_control_headers", bool)
+
+        if self.app__port:
+            try:
+                if not is_port_available(self.app__host, self.app__port):
+                    raise ConfigurationError(
+                        f"The port selected {self.app__port} is in use, please configure an open port."
+                    )
+            except OverflowError:
+                raise ConfigurationError(f"Invalid port: {self.app__port}")
+        else:
+            try:
+                default_server_port = int(os.environ.get("CXG_SERVER_PORT", DEFAULT_SERVER_PORT))
+            except ValueError:
+                raise ConfigurationError(
+                    "Invalid port from environment variable CXG_SERVER_PORT: " + os.environ.get("CXG_SERVER_PORT")
+                )
+            try:
+                self.app__port = find_available_port(self.app__host, default_server_port)
+            except OverflowError:
+                raise ConfigurationError(f"Invalid port: {default_server_port}")
+
+        if self.app__debug:
+            context["messagefn"]("in debug mode, setting verbose=True and open_browser=False")
+            self.app__verbose = True
+            self.app__open_browser = False
+        else:
+            warnings.formatwarning = custom_format_warning
+
+        if not self.app__verbose:
+            sys.tracebacklimit = 0
+
+    def handle_data_locator(self):
+        self.validate_correct_type_of_configuration_attribute("data_locator__s3__region_name", (type(None), bool, str))
+        if self.data_locator__s3__region_name is True:
+            path = self.single_dataset__datapath
+
+            if path.startswith("s3://"):
+                region_name = discover_s3_region_name(path)
+                if region_name is None:
+                    raise ConfigurationError(f"Unable to discover s3 region name from {path}")
+            else:
+                region_name = None
+            self.data_locator__s3__region_name = region_name
+
+    def handle_data_source(self):
+        self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", str)
+
+    def handle_single_dataset(self, context):
+        self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None)))
+        self.validate_correct_type_of_configuration_attribute("single_dataset__title", (str, type(None)))
+        self.validate_correct_type_of_configuration_attribute("single_dataset__about", (str, type(None)))
+        self.validate_correct_type_of_configuration_attribute("single_dataset__obs_names", (str, type(None)))
+        self.validate_correct_type_of_configuration_attribute("single_dataset__var_names", (str, type(None)))
+
+        # preload this data set
+        matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath, app_config=self.app_config)
+        try:
+            matrix_data_loader.pre_load_validation()
+        except DatasetAccessError as e:
+            raise ConfigurationError(str(e))
+
+        file_size = matrix_data_loader.file_size()
+        file_basename = basename(self.single_dataset__datapath)
+        if file_size > BIG_FILE_SIZE_THRESHOLD:
+            context["messagefn"](f"Loading data from {file_basename}, this may take a while...")
+        else:
+            context["messagefn"](f"Loading data from {file_basename}.")
+
+        if self.single_dataset__about:
+
+            def url_check(url):
+                try:
+                    result = urlparse(url)
+                    if all([result.scheme, result.netloc]):
+                        return True
+                    else:
+                        return False
+                except ValueError:
+                    return False
+
+            if not url_check(self.single_dataset__about):
+                raise ConfigurationError(
+                    "Must provide an absolute URL for --about. (Example format: http://example.com)"
+                )
+
+    def handle_adaptor(self):
+        self.validate_correct_type_of_configuration_attribute("adaptor__anndata_adaptor__backed", bool)
+
+    def handle_limits(self):
+        self.validate_correct_type_of_configuration_attribute("limits__diffexp_cellcount_max", (type(None), int))
+        self.validate_correct_type_of_configuration_attribute("limits__column_request_max", (type(None), int))
+
+    def exceeds_limit(self, limit_name, value):
+        limit_value = getattr(self, "limits__" + limit_name, None)
+        if limit_value is None:  # disabled
+            return False
+        return value > limit_value

server/common/constants.py

@@ -0,0 +1,35 @@
+from enum import Enum
+
+
+class AugmentedEnum(Enum):
+    def __hash__(self):
+        return self.value.__hash__()
+
+    def __eq__(self, other):
+        if isinstance(other, type(self)) or isinstance(other, str):
+            return self.value == other
+        return False
+
+    def __str__(self) -> str:
+        return self.value
+
+
+class Axis(AugmentedEnum):
+    OBS = "obs"
+    VAR = "var"
+
+
+class DiffExpMode(AugmentedEnum):
+    TOP_N = "topN"
+    VAR_FILTER = "varFilter"
+
+
+class XApproximateDistribution(AugmentedEnum):
+    NORMAL = "normal"
+    COUNT = "count"
+
+
+JSON_NaN_to_num_warning_msg = "JSON encoding failure - please verify all data are finite values (no NaN or Infinities)"
+REACTIVE_LIMIT = 1_000_000
+
+MAX_LAYOUTS = 30

server/common/corpora.py

@@ -0,0 +1,78 @@
+"""
+Corpora schema conventions support.  Helper functions for reading.
+
+https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md
+
+https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema_h5ad_implementation.md
+"""
+import collections
+import json
+
+from server.cli.upgrade import validate_version_str
+from server.common.utils.corpora_constants import CorporaConstants
+
+
+def corpora_get_versions_from_anndata(adata):
+    """
+    Given an AnnData object, return:
+        * None - if not a Corpora object
+        * [ corpora_schema_version, corpora_encoding_version ] - if a Corpora object
+
+    Implements the identification protocol defined in the specification.
+    """
+
+    # per Corpora AnnData spec, this is a corpora file if the following is true
+    if "version" not in adata.uns_keys():
+        return None
+    version = adata.uns["version"]
+    if not isinstance(version, collections.abc.Mapping) or "corpora_schema_version" not in version:
+        return None
+
+    corpora_schema_version = version.get("corpora_schema_version")
+    corpora_encoding_version = version.get("corpora_encoding_version")
+
+    # TODO: spec says these must be SEMVER values, so check.
+    if validate_version_str(corpora_schema_version) and validate_version_str(corpora_encoding_version):
+        return [corpora_schema_version, corpora_encoding_version]
+
+
+def corpora_is_version_supported(corpora_schema_version, corpora_encoding_version):
+    return (
+        corpora_schema_version
+        and corpora_encoding_version
+        and corpora_schema_version.startswith("1.")
+        and corpora_encoding_version.startswith("0.1.")
+    )
+
+
+def corpora_get_props_from_anndata(adata):
+    """
+    Get Corpora dataset properties from an AnnData
+    """
+    versions = corpora_get_versions_from_anndata(adata)
+    if versions is None:
+        return None
+    [corpora_schema_version, corpora_encoding_version] = versions
+    version_is_supported = corpora_is_version_supported(corpora_schema_version, corpora_encoding_version)
+    if not version_is_supported:
+        raise ValueError("Unsupported Corpora schema version")
+
+    corpora_props = {}
+    for key in CorporaConstants.REQUIRED_SIMPLE_METADATA_FIELDS:
+        if key not in adata.uns:
+            raise KeyError(f"missing Corpora schema field {key}")
+        corpora_props[key] = adata.uns[key]
+
+    for key in CorporaConstants.OPTIONAL_JSON_ENCODED_METADATA_FIELD:
+        if key not in adata.uns:
+            continue
+        try:
+            corpora_props[key] = json.loads(adata.uns[key])
+        except json.JSONDecodeError:
+            raise json.JSONDecodeError(f"Corpora schema field {key} is expected to be a valid JSON string")
+
+    for key in CorporaConstants.OPTIONAL_SIMPLE_METADATA_FIELDS:
+        if key in adata.uns:
+            corpora_props[key] = adata.uns[key]
+
+    return corpora_props

server/common/errors.py

@@ -0,0 +1,54 @@
+from http import HTTPStatus
+
+
+class CellxgeneException(Exception):
+    """Base class for cellxgene exceptions"""
+
+    def __init__(self, message):
+        self.message = message
+        super().__init__(message)
+
+
+class RequestException(CellxgeneException):
+    """Baseclass for exceptions that can be raised from a request."""
+
+    # The default status code is 400 (Bad Request)
+    default_status_code = HTTPStatus.BAD_REQUEST
+
+    def __init__(self, message, status_code=None):
+        super().__init__(message)
+        self.status_code = status_code or self.default_status_code
+
+
+def define_exception(name, doc):
+    globals()[name] = type(name, (CellxgeneException,), dict(__doc__=doc))
+
+
+def define_request_exception(name, doc, default_status_code=HTTPStatus.BAD_REQUEST):
+    globals()[name] = type(name, (RequestException,), dict(__doc__=doc, default_status_code=default_status_code))
+
+
+define_request_exception("FilterError", "Raised when filter is malformed")
+define_request_exception("JSONEncodingValueError", "Raised when data cannot be encoded into json")
+define_request_exception("MimeTypeError", "Raised when incompatible MIME type selected")
+define_request_exception("DatasetAccessError", "Raised when file loaded into a DataAdaptor is misformatted")
+define_request_exception("DisabledFeatureError", "Raised when an attempt to use a disabled feature occurs")
+define_request_exception("AnnotationsError", "Raised when an attempt to use the annotations feature fails")
+define_request_exception(
+    "ComputeError",
+    "Raised when an error occurs during a compute algorithm (such as diffexp)",
+    HTTPStatus.INTERNAL_SERVER_ERROR,
+)
+define_request_exception("ExceedsLimitError", "Raised when an HTTP request exceeds a limit/quota")
+define_request_exception("ColorFormatException", "Raised when color helper functions encounter an unknown color format")
+
+define_request_exception(
+    "AnnotationCategoryNameError",
+    "Raised when an annotation category name cant be saved",
+    default_status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
+)
+
+define_exception("ConfigurationError", "Raised when checking configuration errors")
+define_exception("PrepareError", "Raised when data is misprepared")
+define_exception("ObsoleteRequest", "Raised when the request is no longer valid.")
+define_exception("UnsupportedSummaryMethod", "Raised when a gene set summary method is unknown or unsupported.")

server/common/fbs/NetEncoding/Column.py

@@ -0,0 +1,41 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: NetEncoding
+
+import flatbuffers
+
+class Column(object):
+    __slots__ = ['_tab']
+
+    @classmethod
+    def GetRootAsColumn(cls, buf, offset):
+        n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
+        x = Column()
+        x.Init(buf, n + offset)
+        return x
+
+    # Column
+    def Init(self, buf, pos):
+        self._tab = flatbuffers.table.Table(buf, pos)
+
+    # Column
+    def UType(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos)
+        return 0
+
+    # Column
+    def U(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6))
+        if o != 0:
+            from flatbuffers.table import Table
+            obj = Table(bytearray(), 0)
+            self._tab.Union(obj, o)
+            return obj
+        return None
+
+def ColumnStart(builder): builder.StartObject(2)
+def ColumnAddUType(builder, uType): builder.PrependUint8Slot(0, uType, 0)
+def ColumnAddU(builder, u): builder.PrependUOffsetTRelativeSlot(1, flatbuffers.number_types.UOffsetTFlags.py_type(u), 0)
+def ColumnEnd(builder): return builder.EndObject()

server/common/fbs/NetEncoding/Float32Array.py

@@ -0,0 +1,46 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: NetEncoding
+
+import flatbuffers
+
+class Float32Array(object):
+    __slots__ = ['_tab']
+
+    @classmethod
+    def GetRootAsFloat32Array(cls, buf, offset):
+        n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
+        x = Float32Array()
+        x.Init(buf, n + offset)
+        return x
+
+    # Float32Array
+    def Init(self, buf, pos):
+        self._tab = flatbuffers.table.Table(buf, pos)
+
+    # Float32Array
+    def Data(self, j):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            a = self._tab.Vector(o)
+            return self._tab.Get(flatbuffers.number_types.Float32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
+        return 0
+
+    # Float32Array
+    def DataAsNumpy(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Float32Flags, o)
+        return 0
+
+    # Float32Array
+    def DataLength(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.VectorLen(o)
+        return 0
+
+def Float32ArrayStart(builder): builder.StartObject(1)
+def Float32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
+def Float32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4)
+def Float32ArrayEnd(builder): return builder.EndObject()

server/common/fbs/NetEncoding/Float64Array.py

@@ -0,0 +1,46 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: NetEncoding
+
+import flatbuffers
+
+class Float64Array(object):
+    __slots__ = ['_tab']
+
+    @classmethod
+    def GetRootAsFloat64Array(cls, buf, offset):
+        n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
+        x = Float64Array()
+        x.Init(buf, n + offset)
+        return x
+
+    # Float64Array
+    def Init(self, buf, pos):
+        self._tab = flatbuffers.table.Table(buf, pos)
+
+    # Float64Array
+    def Data(self, j):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            a = self._tab.Vector(o)
+            return self._tab.Get(flatbuffers.number_types.Float64Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 8))
+        return 0
+
+    # Float64Array
+    def DataAsNumpy(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Float64Flags, o)
+        return 0
+
+    # Float64Array
+    def DataLength(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.VectorLen(o)
+        return 0
+
+def Float64ArrayStart(builder): builder.StartObject(1)
+def Float64ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
+def Float64ArrayStartDataVector(builder, numElems): return builder.StartVector(8, numElems, 8)
+def Float64ArrayEnd(builder): return builder.EndObject()

server/common/fbs/NetEncoding/Int32Array.py

@@ -0,0 +1,46 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: NetEncoding
+
+import flatbuffers
+
+class Int32Array(object):
+    __slots__ = ['_tab']
+
+    @classmethod
+    def GetRootAsInt32Array(cls, buf, offset):
+        n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
+        x = Int32Array()
+        x.Init(buf, n + offset)
+        return x
+
+    # Int32Array
+    def Init(self, buf, pos):
+        self._tab = flatbuffers.table.Table(buf, pos)
+
+    # Int32Array
+    def Data(self, j):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            a = self._tab.Vector(o)
+            return self._tab.Get(flatbuffers.number_types.Int32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
+        return 0
+
+    # Int32Array
+    def DataAsNumpy(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Int32Flags, o)
+        return 0
+
+    # Int32Array
+    def DataLength(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.VectorLen(o)
+        return 0
+
+def Int32ArrayStart(builder): builder.StartObject(1)
+def Int32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
+def Int32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4)
+def Int32ArrayEnd(builder): return builder.EndObject()

server/common/fbs/NetEncoding/JSONEncodedArray.py

@@ -0,0 +1,46 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: NetEncoding
+
+import flatbuffers
+
+class JSONEncodedArray(object):
+    __slots__ = ['_tab']
+
+    @classmethod
+    def GetRootAsJSONEncodedArray(cls, buf, offset):
+        n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
+        x = JSONEncodedArray()
+        x.Init(buf, n + offset)
+        return x
+
+    # JSONEncodedArray
+    def Init(self, buf, pos):
+        self._tab = flatbuffers.table.Table(buf, pos)
+
+    # JSONEncodedArray
+    def Data(self, j):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            a = self._tab.Vector(o)
+            return self._tab.Get(flatbuffers.number_types.Uint8Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 1))
+        return 0
+
+    # JSONEncodedArray
+    def DataAsNumpy(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Uint8Flags, o)
+        return 0
+
+    # JSONEncodedArray
+    def DataLength(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.VectorLen(o)
+        return 0
+
+def JSONEncodedArrayStart(builder): builder.StartObject(1)
+def JSONEncodedArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
+def JSONEncodedArrayStartDataVector(builder, numElems): return builder.StartVector(1, numElems, 1)
+def JSONEncodedArrayEnd(builder): return builder.EndObject()

server/common/fbs/NetEncoding/Matrix.py

@@ -0,0 +1,98 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: NetEncoding
+
+import flatbuffers
+
+class Matrix(object):
+    __slots__ = ['_tab']
+
+    @classmethod
+    def GetRootAsMatrix(cls, buf, offset):
+        n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
+        x = Matrix()
+        x.Init(buf, n + offset)
+        return x
+
+    # Matrix
+    def Init(self, buf, pos):
+        self._tab = flatbuffers.table.Table(buf, pos)
+
+    # Matrix
+    def NRows(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.Get(flatbuffers.number_types.Uint32Flags, o + self._tab.Pos)
+        return 0
+
+    # Matrix
+    def NCols(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6))
+        if o != 0:
+            return self._tab.Get(flatbuffers.number_types.Uint32Flags, o + self._tab.Pos)
+        return 0
+
+    # Matrix
+    def Columns(self, j):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8))
+        if o != 0:
+            x = self._tab.Vector(o)
+            x += flatbuffers.number_types.UOffsetTFlags.py_type(j) * 4
+            x = self._tab.Indirect(x)
+            from .Column import Column
+            obj = Column()
+            obj.Init(self._tab.Bytes, x)
+            return obj
+        return None
+
+    # Matrix
+    def ColumnsLength(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8))
+        if o != 0:
+            return self._tab.VectorLen(o)
+        return 0
+
+    # Matrix
+    def ColIndexType(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10))
+        if o != 0:
+            return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos)
+        return 0
+
+    # Matrix
+    def ColIndex(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12))
+        if o != 0:
+            from flatbuffers.table import Table
+            obj = Table(bytearray(), 0)
+            self._tab.Union(obj, o)
+            return obj
+        return None
+
+    # Matrix
+    def RowIndexType(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14))
+        if o != 0:
+            return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos)
+        return 0
+
+    # Matrix
+    def RowIndex(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(16))
+        if o != 0:
+            from flatbuffers.table import Table
+            obj = Table(bytearray(), 0)
+            self._tab.Union(obj, o)
+            return obj
+        return None
+
+def MatrixStart(builder): builder.StartObject(7)
+def MatrixAddNRows(builder, nRows): builder.PrependUint32Slot(0, nRows, 0)
+def MatrixAddNCols(builder, nCols): builder.PrependUint32Slot(1, nCols, 0)
+def MatrixAddColumns(builder, columns): builder.PrependUOffsetTRelativeSlot(2, flatbuffers.number_types.UOffsetTFlags.py_type(columns), 0)
+def MatrixStartColumnsVector(builder, numElems): return builder.StartVector(4, numElems, 4)
+def MatrixAddColIndexType(builder, colIndexType): builder.PrependUint8Slot(3, colIndexType, 0)
+def MatrixAddColIndex(builder, colIndex): builder.PrependUOffsetTRelativeSlot(4, flatbuffers.number_types.UOffsetTFlags.py_type(colIndex), 0)
+def MatrixAddRowIndexType(builder, rowIndexType): builder.PrependUint8Slot(5, rowIndexType, 0)
+def MatrixAddRowIndex(builder, rowIndex): builder.PrependUOffsetTRelativeSlot(6, flatbuffers.number_types.UOffsetTFlags.py_type(rowIndex), 0)
+def MatrixEnd(builder): return builder.EndObject()

server/common/fbs/NetEncoding/TypedArray.py

@@ -0,0 +1,12 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: NetEncoding
+
+class TypedArray(object):
+    NONE = 0
+    Float32Array = 1
+    Int32Array = 2
+    Uint32Array = 3
+    Float64Array = 4
+    JSONEncodedArray = 5
+

server/common/fbs/NetEncoding/Uint32Array.py

@@ -0,0 +1,46 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: NetEncoding
+
+import flatbuffers
+
+class Uint32Array(object):
+    __slots__ = ['_tab']
+
+    @classmethod
+    def GetRootAsUint32Array(cls, buf, offset):
+        n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
+        x = Uint32Array()
+        x.Init(buf, n + offset)
+        return x
+
+    # Uint32Array
+    def Init(self, buf, pos):
+        self._tab = flatbuffers.table.Table(buf, pos)
+
+    # Uint32Array
+    def Data(self, j):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            a = self._tab.Vector(o)
+            return self._tab.Get(flatbuffers.number_types.Uint32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
+        return 0
+
+    # Uint32Array
+    def DataAsNumpy(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Uint32Flags, o)
+        return 0
+
+    # Uint32Array
+    def DataLength(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.VectorLen(o)
+        return 0
+
+def Uint32ArrayStart(builder): builder.StartObject(1)
+def Uint32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
+def Uint32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4)
+def Uint32ArrayEnd(builder): return builder.EndObject()

server/common/fbs/matrix.py

@@ -0,0 +1,251 @@
+import json
+
+import numpy as np
+import pandas as pd
+from flatbuffers import Builder
+from scipy import sparse
+
+from server.common.utils.type_conversion_utils import get_encoding_dtype_of_array
+
+import server.common.fbs.NetEncoding.Column as Column
+import server.common.fbs.NetEncoding.Float32Array as Float32Array
+import server.common.fbs.NetEncoding.Float64Array as Float64Array
+import server.common.fbs.NetEncoding.Int32Array as Int32Array
+import server.common.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray
+import server.common.fbs.NetEncoding.Matrix as Matrix
+import server.common.fbs.NetEncoding.TypedArray as TypedArray
+import server.common.fbs.NetEncoding.Uint32Array as Uint32Array
+
+
+# Serialization helper
+def serialize_column(builder, typed_arr):
+    """ Serialize NetEncoding.Column """
+
+    (u_type, u_value) = typed_arr
+    Column.ColumnStart(builder)
+    Column.ColumnAddUType(builder, u_type)
+    Column.ColumnAddU(builder, u_value)
+    return Column.ColumnEnd(builder)
+
+
+# Serialization helper
+def serialize_matrix(builder, n_rows, n_cols, columns, col_idx):
+    """ Serialize NetEncoding.Matrix """
+
+    Matrix.MatrixStart(builder)
+    Matrix.MatrixAddNRows(builder, n_rows)
+    Matrix.MatrixAddNCols(builder, n_cols)
+    Matrix.MatrixAddColumns(builder, columns)
+    if col_idx is not None:
+        (u_type, u_val) = col_idx
+        Matrix.MatrixAddColIndexType(builder, u_type)
+        Matrix.MatrixAddColIndex(builder, u_val)
+    return Matrix.MatrixEnd(builder)
+
+
+# Serialization helper
+def serialize_typed_array(builder, source_array, encoding_info):
+    """
+    Serialize any of the various typed arrays, eg, Float32Array. Specific means of serialization and type conversion
+    are provided by type_info.
+    """
+
+    arr = source_array
+    (array_type, as_type) = encoding_info(source_array)
+
+    if isinstance(arr, pd.Index):
+        arr = arr.to_series()
+
+    # convert to a simple ndarray
+    if as_type == "json":
+        as_json = arr.to_json(orient="records")
+        arr = np.array(bytearray(as_json, "utf-8"))
+    else:
+        if sparse.issparse(arr):
+            arr = arr.toarray()
+        elif isinstance(arr, pd.Series):
+            arr = arr.to_numpy()
+        if arr.dtype != as_type:
+            arr = arr.astype(as_type)
+
+    # serialize the ndarray into a vector
+    if arr.ndim == 2:
+        if arr.shape[0] == 1:
+            arr = arr[0]
+        elif arr.shape[1] == 1:
+            arr = arr.T[0]
+
+    vec = builder.CreateNumpyVector(arr)
+
+    # serialize the typed array table
+    builder.StartObject(1)
+    builder.PrependUOffsetTRelativeSlot(0, vec, 0)
+    array_value = builder.EndObject()
+    return (array_type, array_value)
+
+
+def column_encoding(arr):
+    column_encoding_type_map = {
+        # array protocol string:  ( array_type, as_type )
+        np.dtype(np.float64).str: (TypedArray.TypedArray.Float32Array, np.float32),
+        np.dtype(np.float32).str: (TypedArray.TypedArray.Float32Array, np.float32),
+        np.dtype(np.float16).str: (TypedArray.TypedArray.Float32Array, np.float32),
+        np.dtype(np.int8).str: (TypedArray.TypedArray.Int32Array, np.int32),
+        np.dtype(np.int16).str: (TypedArray.TypedArray.Int32Array, np.int32),
+        np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32),
+        np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32),
+        np.dtype(np.uint8).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
+        np.dtype(np.uint16).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
+        np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
+        np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
+    }
+    column_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, "json")
+
+    encoding_dtype = np.dtype(get_encoding_dtype_of_array(arr))
+    return column_encoding_type_map.get(encoding_dtype.str, column_encoding_default)
+
+
+def index_encoding(arr):
+    index_encoding_type_map = {
+        # array protocol string:  ( array_type, as_type )
+        np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32),
+        np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32),
+        np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
+        np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
+    }
+    index_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, "json")
+
+    return index_encoding_type_map.get(arr.dtype.str, index_encoding_default)
+
+
+def guess_at_mem_needed(matrix):
+    (n_rows, n_cols) = matrix.shape
+    if isinstance(matrix, np.ndarray) or sparse.issparse(matrix):
+        guess = (n_rows * n_cols * matrix.dtype.itemsize) + 1024
+    elif isinstance(matrix, pd.DataFrame):
+        # XXX TODO - DataFrame type estimate
+        guess = 1
+    else:
+        guess = 1
+
+    # round up to nearest 1024 bytes
+    guess = (guess + 0x400) & (~0x3FF)
+    return guess
+
+
+def encode_matrix_fbs(matrix, row_idx=None, col_idx=None):
+    """
+    Given a 2D DataFrame, ndarray or sparse equivalent, create and return a Matrix flatbuffer.
+
+    :param matrix: 2D DataFrame, ndarray or sparse equivalent
+    :param row_idx: index for row dimension, Index or ndarray
+    :param col_idx: index for col dimension, Index or ndarray
+
+    NOTE: row indices are (currently) unsupported and must be None
+    """
+
+    if row_idx is not None:
+        raise ValueError("row indexing not supported for FBS Matrix")
+    if matrix.ndim != 2:
+        raise ValueError("FBS Matrix must be 2D")
+
+    (n_rows, n_cols) = matrix.shape
+
+    # estimate size needed, so we don't unnecessarily realloc.
+    builder = Builder(guess_at_mem_needed(matrix))
+
+    columns = []
+    for cidx in range(n_cols - 1, -1, -1):
+        # serialize the typed array
+        col = matrix.iloc[:, cidx] if isinstance(matrix, pd.DataFrame) else matrix[:, cidx]
+        typed_arr = serialize_typed_array(builder, col, column_encoding)
+
+        # serialize the Column union
+        columns.append(serialize_column(builder, typed_arr))
+
+    # Serialize Matrix.columns[]
+    Matrix.MatrixStartColumnsVector(builder, n_cols)
+    for c in columns:
+        builder.PrependUOffsetTRelative(c)
+    matrix_column_vec = builder.EndVector(n_cols)
+
+    # serialize the colIndex if provided
+    cidx = None
+    if col_idx is not None:
+        cidx = serialize_typed_array(builder, col_idx, index_encoding)
+
+    # Serialize Matrix
+    matrix = serialize_matrix(builder, n_rows, n_cols, matrix_column_vec, cidx)
+
+    builder.Finish(matrix)
+    return builder.Output()
+
+
+def deserialize_typed_array(tarr):
+    type_map = {
+        TypedArray.TypedArray.NONE: None,
+        TypedArray.TypedArray.Uint32Array: Uint32Array.Uint32Array,
+        TypedArray.TypedArray.Int32Array: Int32Array.Int32Array,
+        TypedArray.TypedArray.Float32Array: Float32Array.Float32Array,
+        TypedArray.TypedArray.Float64Array: Float64Array.Float64Array,
+        TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray,
+    }
+    (u_type, u) = tarr
+    if u_type is TypedArray.TypedArray.NONE:
+        return None
+
+    TarType = type_map.get(u_type, None)
+    if TarType is None:
+        raise TypeError(f"FBS contains unknown data type: {u_type}")
+
+    arr = TarType()
+    arr.Init(u.Bytes, u.Pos)
+    narr = arr.DataAsNumpy()
+    if u_type == TypedArray.TypedArray.JSONEncodedArray:
+        narr = json.loads(narr.tobytes().decode("utf-8"))
+    return narr
+
+
+def decode_matrix_fbs(fbs):
+    """
+    Given an FBS-encoded Matrix, return a Pandas DataFrame the contains the data and indices.
+    """
+
+    matrix = Matrix.Matrix.GetRootAsMatrix(fbs, 0)
+    n_rows = matrix.NRows()
+    n_cols = matrix.NCols()
+    if n_rows == 0 or n_cols == 0:
+        return pd.DataFrame()
+
+    if matrix.RowIndexType() is not TypedArray.TypedArray.NONE:
+        raise ValueError("row indexing not supported for FBS Matrix")
+
+    columns_length = matrix.ColumnsLength()
+
+    columns_index = deserialize_typed_array((matrix.ColIndexType(), matrix.ColIndex()))
+    if columns_index is None:
+        columns_index = range(0, n_cols)
+
+    # sanity checks
+    if len(columns_index) != n_cols or columns_length != n_cols:
+        raise ValueError("FBS column count does not match number of columns in underlying matrix")
+
+    columns_data = {}
+    columns_type = {}
+    for col_idx in range(0, columns_length):
+        col = matrix.Columns(col_idx)
+        tarr = (col.UType(), col.U())
+        data = deserialize_typed_array(tarr)
+        columns_data[columns_index[col_idx]] = data
+        if len(data) != n_rows:
+            raise ValueError("FBS column length does not match number of rows")
+        if col.UType() is TypedArray.TypedArray.JSONEncodedArray:
+            columns_type[columns_index[col_idx]] = "category"
+
+    df = pd.DataFrame.from_dict(data=columns_data).astype(columns_type, copy=False)
+
+    # more sanity checks
+    if not df.columns.is_unique or len(df.columns) != n_cols:
+        raise KeyError("FBS column indices are not unique")
+
+    return df

server/common/genesets.py

@@ -0,0 +1,239 @@
+"""
+Utility code for gene sets handling
+"""
+
+import re
+import csv
+import hashlib
+
+from .errors import AnnotationsError
+
+
+GENESETS_TIDYCSV_HEADER = [
+    "gene_set_name",
+    "gene_set_description",
+    "gene_symbol",
+    "gene_description",
+]
+
+
+def read_gene_sets_tidycsv(gs_locator, context=None):
+    """
+    Read & parse the Tidy CSV format, applying validation checks for mandatory
+    values, and de-duping rules.
+
+    Format is a four-column CSV, with a mandatory header row, and optional "#" prefixed
+    comments.  Format:
+
+        gene_set_name, gene_set_description, gene_symbol, gene_description
+
+    gene_set_name must be non-null; others are optional.
+
+    Returns: a dictionary of the shape (values in angle-brackets vary):
+
+        {
+            <string, a gene set name>: {
+                "geneset_name": <string, a gene set name>,
+                "geneset_description": <a string or None>,
+                "genes": [
+                    {
+                        "gene_symbol": <string, a gene symbol or name>,
+                        "gene_description": <a string or None>
+                    },
+                    ...
+                ]
+            },
+            ...
+        }
+    """
+
+    class myDialect(csv.excel):
+        skipinitialspace = False
+
+    def just(n, seq):
+        it = iter(seq)
+        for _ in range(n - 1):
+            yield next(it, "")
+        yield tuple(it)
+
+    messagefn = context["messagefn"] if context else (lambda x: None)
+
+    gene_sets = {}
+    with gs_locator.local_handle() as fname:
+        with open(fname, newline="") as f:
+            reader = csv.reader(f, dialect=myDialect())
+            haveReadHeader = False
+            lineno = 0
+            for row in reader:
+                lineno += 1
+                # ignore empty rows
+                if len(row) == 0:
+                    continue
+                # if row starts with '#' it is a comment
+                if row[0].startswith("#"):
+                    continue
+                # if this is the first non-comment row, assume it is a header and validate
+                # column names.  OK if the user has extra columns after our initial set.
+                if not haveReadHeader:
+                    if row[0 : len(GENESETS_TIDYCSV_HEADER)] != GENESETS_TIDYCSV_HEADER:
+                        raise AnnotationsError("Gene set CSV file missing the required column header.")
+                    haveReadHeader = True
+                    continue
+
+                geneset_name, geneset_description, gene_symbol, gene_description, _ = just(5, row)
+                if not geneset_name:
+                    raise AnnotationsError(f"Gene set CSV missing required gene set name on line {lineno}")
+                if (not gene_symbol) and gene_description:
+                    messagefn(f"Warning: Missing gene name in gene set name {geneset_name} on line {lineno}.")
+
+                if geneset_name in gene_sets:
+                    gs = gene_sets[geneset_name]
+                else:
+                    gs = gene_sets[geneset_name] = {
+                        "geneset_name": geneset_name,
+                        "geneset_description": geneset_description,
+                        "genes": [],
+                    }
+                # Use first geneset_description with a value
+                if not gs["geneset_description"] and geneset_description:
+                    gs["geneset_description"] = geneset_description
+                # add the gene if the gene_symbol is defined
+                if gene_symbol:
+                    gs["genes"].append(
+                        {
+                            "gene_symbol": gene_symbol,
+                            "gene_description": gene_description,
+                        }
+                    )
+
+    return gene_sets
+
+
+def write_gene_sets_tidycsv(f, genesets):
+    """
+    Convert the internal gene sets format (returned by read_gene_set_tidycsv) into
+    the simple Tidy CSV.
+    """
+    writer = csv.writer(f, dialect="excel")
+    writer.writerow(GENESETS_TIDYCSV_HEADER)
+    for geneset in genesets:
+        # genes may be empty, in which case we skip the gene set entirely
+        genes = geneset["genes"]
+        if not genes:
+            writer.writerow([geneset["geneset_name"], geneset.get("geneset_description", ""), "", ""])
+        else:
+            writer.writerows(
+                [
+                    [
+                        geneset["geneset_name"],
+                        geneset.get("geneset_description", ""),
+                        gene["gene_symbol"],
+                        gene.get("gene_description", ""),
+                    ]
+                    for gene in genes
+                ]
+            )
+
+
+def summarizeQueryHash(raw_query):
+    """ generate a cache key (hash) from the raw query string """
+    return hashlib.sha1(raw_query).hexdigest()
+
+
+def validate_gene_sets(genesets, var_names, context=None):
+    """
+    Check validity of gene sets, return if correct, else raise error.
+    May also modify the gene set for conditions that should be resolved,
+    but which do not warrant a hard error.
+
+    Argument gene sets may be either the REST OTA format (list of dicts) or the internal
+    format (dict of dicts, keyed by the gene set name).
+
+    Will return a modified gene sets (eg, remove warnings) of the same type as the
+    provided argument. Ie, dict->dict, list->list
+
+    Rules:
+
+    0. All gene set names must be unique. [error]
+    1. Gene set names must conform to the following: [error]
+        * Names must be comprised of 1 or more ASCII characters 32-126
+        * No leading or trailing spaces (ASCII 32)
+        * No multi-space (ASCII 32) runs
+    2. Gene symbols must be part of the current var_index. [warning]
+       If gene symbol is not in the var_index, generate a warning and remove the symbol
+       from the gene sets.
+    3. Gene symbols must not be duplicated in a gene set.  [warning]
+       Duplications will be silently de-duped.
+
+    Items marked [error] will generate a hard error, causing the validation to fail.
+
+    Items marked [warning] will generate a warning, and will be resolved without failing
+    the validation (typically by removing the offending item from the gene sets).
+    """
+
+    messagefn = context["messagefn"] if context else (lambda x: None)
+
+    # accept genesets args as either the internal (dict) or REST (list) format,
+    # as they are identical except for the dict being keyed by geneset_name.
+    if not isinstance(genesets, dict) and not isinstance(genesets, list):
+        raise ValueError("Gene sets must be either dict or list.")
+    genesets_iterable = genesets if isinstance(genesets, list) else genesets.values()
+
+    # 0. check for uniqueness of geneset names
+    geneset_names = [gs["geneset_name"] for gs in genesets_iterable]
+    if len(set(geneset_names)) != len(geneset_names):
+        raise KeyError("All gene set names must be unique.")
+
+    # 1. check gene set character set and format
+    illegal_name = re.compile(r"^\s|  |[\u0000-\u001F\u007F-\uFFFF]|\s$")
+    for name in geneset_names:
+        if type(name) != str or len(name) == 0:
+            raise KeyError("Gene set names must be non-null string.")
+        if illegal_name.search(name):
+            messagefn(
+                "Error: "
+                f"Gene set name {name} "
+                "is not valid. Leading, trailing, and multiple spaces within a name are not allowed."
+            )
+            raise KeyError(
+                "Gene set name is not valid. Leading, trailing, and multiple spaces within a name are not allowed."
+            )
+
+    # 2. & 3. check for duplicate gene symbols, and those not present in the dataset. They will
+    # generate a warning and be removed.
+    for geneset in genesets_iterable:
+        if not isinstance(geneset, dict):
+            raise ValueError("Each gene set must be a dict.")
+        geneset_name = geneset["geneset_name"]
+        genes = geneset["genes"]
+        if not isinstance(genes, list):
+            raise ValueError("Gene set genes field must be a list")
+        geneset.setdefault("geneset_description", "")
+        gene_symbol_already_seen = set()
+        new_genes = []
+        for gene in genes:
+            gene_symbol = gene["gene_symbol"]
+            if not isinstance(gene_symbol, str) or len(gene_symbol) == 0:
+                raise ValueError("Gene symbol must be non-null string.")
+            if gene_symbol in gene_symbol_already_seen:
+                # duplicate check
+                messagefn(
+                    f"Warning: a duplicate of gene {gene_symbol} was found in gene set {geneset_name}, "
+                    "and will be ignored."
+                )
+                continue
+
+            if gene_symbol not in var_names:
+                messagefn(
+                    f"Warning: {gene_symbol}, used in gene set {geneset_name}, "
+                    "was not found in the dataset and will be ignored."
+                )
+                continue
+
+            gene_symbol_already_seen.add(gene_symbol)
+            gene.setdefault("gene_description", "")
+            new_genes.append(gene)
+
+        geneset["genes"] = new_genes
+
+    return genesets

server/common/health.py

@@ -0,0 +1,33 @@
+from http import HTTPStatus
+from flask import make_response, jsonify
+
+from server import __version__ as cellxgene_version
+from server.common.utils.data_locator import DataLocator
+
+
+def _is_accessible(path, config):
+    if path is None:
+        return True
+
+    try:
+        dl = DataLocator(path, region_name=config.data_locator__s3__region_name)
+        return dl.exists()
+    except RuntimeError:
+        return False
+
+
+def health_check(config):
+    """
+    simple health check - return HTTP response.
+    See https://tools.ietf.org/id/draft-inadarei-api-health-check-01.html
+    """
+    health = {"status": None, "version": "1", "releaseID": cellxgene_version}
+
+    server_config = config.server_config
+    check = _is_accessible(server_config.single_dataset__datapath, server_config)
+
+    health["status"] = "pass" if check else "fail"
+    code = HTTPStatus.OK if health["status"] == "pass" else HTTPStatus.BAD_REQUEST
+    response = make_response(jsonify(health), code)
+    response.headers["Content-Type"] = "application/health+json"
+    return response

server/common/rest.py

@@ -0,0 +1,399 @@
+import copy
+import logging
+import sys
+from http import HTTPStatus
+import zlib
+import json
+
+from flask import make_response, jsonify, current_app, abort
+from werkzeug.urls import url_unquote
+
+from server.common.config.client_config import get_client_config
+from server.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg
+from server.common.errors import (
+    FilterError,
+    JSONEncodingValueError,
+    PrepareError,
+    DisabledFeatureError,
+    ExceedsLimitError,
+    DatasetAccessError,
+    ColorFormatException,
+    AnnotationsError,
+    ObsoleteRequest,
+    UnsupportedSummaryMethod,
+)
+from server.common.genesets import summarizeQueryHash
+from server.common.fbs.matrix import decode_matrix_fbs
+
+
+def abort_and_log(code, logmsg, loglevel=logging.DEBUG, include_exc_info=False):
+    """
+    Log the message, then abort with HTTP code. If include_exc_info is true,
+    also include current exception via sys.exc_info().
+    """
+    if include_exc_info:
+        exc_info = sys.exc_info()
+    else:
+        exc_info = False
+    current_app.logger.log(loglevel, logmsg, exc_info=exc_info)
+    # Do NOT send log message to HTTP response.
+    return abort(code)
+
+
+def _query_parameter_to_filter(args):
+    """
+    Convert an annotation value filter, if present in the query args,
+    into the standard dict filter format used by internal code.
+
+    Query param filters look like:  <axis>:name=value, where value
+    may be one of:
+        - a range, min,max, where either may be an open range by using an asterisk, eg, 10,*
+        - a value
+    Eg,
+        ...?tissue=lung&obs:tissue=heart&obs:num_reads=1000,*
+    """
+    filters = {
+        "obs": {},
+        "var": {},
+    }
+
+    # args has already been url-unquoted once.  We assume double escaping
+    # on name and value.
+    try:
+        for key, value in args.items(multi=True):
+            axis, name = key.split(":")
+            if axis not in ("obs", "var"):
+                raise FilterError("unknown filter axis")
+            name = url_unquote(name)
+            current = filters[axis].setdefault(name, {"name": name})
+
+            val_split = value.split(",")
+            if len(val_split) == 1:
+                if "min" in current or "max" in current:
+                    raise FilterError("do not mix range and value filters")
+                value = url_unquote(value)
+                values = current.setdefault("values", [])
+                values.append(value)
+
+            elif len(val_split) == 2:
+                if len(current) > 1:
+                    raise FilterError("duplicate range specification")
+                min = url_unquote(val_split[0])
+                max = url_unquote(val_split[1])
+                if min != "*":
+                    current["min"] = float(min)
+                if max != "*":
+                    current["max"] = float(max)
+                if len(current) < 2:
+                    raise FilterError("must specify at least min or max in range filter")
+
+            else:
+                raise FilterError("badly formated filter value")
+
+    except ValueError as e:
+        raise FilterError(str(e))
+
+    result = {}
+    for axis in ("obs", "var"):
+        axis_filter = filters[axis]
+        if len(axis_filter) > 0:
+            result[axis] = {"annotation_value": [val for val in axis_filter.values()]}
+
+    return result
+
+
+def schema_get_helper(data_adaptor):
+    """helper function to gather the schema from the data source and annotations"""
+    schema = data_adaptor.get_schema()
+    schema = copy.deepcopy(schema)
+
+    # add label obs annotations as needed
+    annotations = data_adaptor.dataset_config.user_annotations
+    if annotations.user_annotations_enabled():
+        label_schema = annotations.get_schema(data_adaptor)
+        schema["annotations"]["obs"]["columns"].extend(label_schema)
+
+    return schema
+
+
+def schema_get(data_adaptor):
+    schema = schema_get_helper(data_adaptor)
+    return make_response(jsonify({"schema": schema}), HTTPStatus.OK)
+
+
+def config_get(app_config, data_adaptor):
+    config = get_client_config(app_config, data_adaptor)
+    return make_response(jsonify(config), HTTPStatus.OK)
+
+
+def annotations_obs_get(request, data_adaptor):
+    fields = request.args.getlist("annotation-name", None)
+    num_columns_requested = len(data_adaptor.get_obs_keys()) if len(fields) == 0 else len(fields)
+    if data_adaptor.server_config.exceeds_limit("column_request_max", num_columns_requested):
+        return abort(HTTPStatus.BAD_REQUEST)
+    preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
+    if preferred_mimetype != "application/octet-stream":
+        return abort(HTTPStatus.NOT_ACCEPTABLE)
+
+    try:
+        labels = None
+        annotations = data_adaptor.dataset_config.user_annotations
+        if annotations.user_annotations_enabled():
+            labels = annotations.read_labels(data_adaptor)
+        fbs = data_adaptor.annotation_to_fbs_matrix(Axis.OBS, fields, labels)
+        return make_response(fbs, HTTPStatus.OK, {"Content-Type": "application/octet-stream"})
+    except KeyError as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+
+
+def annotations_put_fbs_helper(data_adaptor, fbs):
+    """helper function to write annotations from fbs"""
+    annotations = data_adaptor.dataset_config.user_annotations
+    if not annotations.user_annotations_enabled():
+        raise DisabledFeatureError("Writable annotations are not enabled")
+
+    new_label_df = decode_matrix_fbs(fbs)
+    if not new_label_df.empty:
+        new_label_df = data_adaptor.check_new_labels(new_label_df)
+    annotations.write_labels(new_label_df, data_adaptor)
+
+
+def inflate(data):
+    return zlib.decompress(data)
+
+
+def annotations_obs_put(request, data_adaptor):
+    annotations = data_adaptor.dataset_config.user_annotations
+    if not annotations.user_annotations_enabled():
+        return abort(HTTPStatus.NOT_IMPLEMENTED)
+
+    anno_collection = request.args.get("annotation-collection-name", default=None)
+    fbs = inflate(request.get_data())
+
+    if anno_collection is not None:
+        if not annotations.is_safe_collection_name(anno_collection):
+            return abort(HTTPStatus.BAD_REQUEST, "Bad annotation collection name")
+        annotations.set_collection(anno_collection)
+
+    try:
+        annotations_put_fbs_helper(data_adaptor, fbs)
+        res = json.dumps({"status": "OK"})
+        return make_response(res, HTTPStatus.OK, {"Content-Type": "application/json"})
+    except (ValueError, DisabledFeatureError, KeyError) as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+
+
+def annotations_var_get(request, data_adaptor):
+    fields = request.args.getlist("annotation-name", None)
+    num_columns_requested = len(data_adaptor.get_var_keys()) if len(fields) == 0 else len(fields)
+    if data_adaptor.server_config.exceeds_limit("column_request_max", num_columns_requested):
+        return abort(HTTPStatus.BAD_REQUEST)
+    preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
+    if preferred_mimetype != "application/octet-stream":
+        return abort(HTTPStatus.NOT_ACCEPTABLE)
+
+    try:
+        labels = None
+        return make_response(
+            data_adaptor.annotation_to_fbs_matrix(Axis.VAR, fields, labels),
+            HTTPStatus.OK,
+            {"Content-Type": "application/octet-stream"},
+        )
+    except KeyError as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+
+
+def data_var_put(request, data_adaptor):
+    preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
+    if preferred_mimetype != "application/octet-stream":
+        return abort(HTTPStatus.NOT_ACCEPTABLE)
+
+    filter_json = request.get_json()
+    filter = filter_json["filter"] if filter_json else None
+    try:
+        return make_response(
+            data_adaptor.data_frame_to_fbs_matrix(filter, axis=Axis.VAR),
+            HTTPStatus.OK,
+            {"Content-Type": "application/octet-stream"},
+        )
+    except (FilterError, ValueError, ExceedsLimitError) as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+
+
+def data_var_get(request, data_adaptor):
+    preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
+    if preferred_mimetype != "application/octet-stream":
+        return abort(HTTPStatus.NOT_ACCEPTABLE)
+
+    try:
+        filter = _query_parameter_to_filter(request.args)
+        return make_response(
+            data_adaptor.data_frame_to_fbs_matrix(filter, axis=Axis.VAR),
+            HTTPStatus.OK,
+            {"Content-Type": "application/octet-stream"},
+        )
+    except (FilterError, ValueError, ExceedsLimitError) as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+
+
+def colors_get(data_adaptor):
+    if not data_adaptor.dataset_config.presentation__custom_colors:
+        return make_response(jsonify({}), HTTPStatus.OK)
+    try:
+        return make_response(jsonify(data_adaptor.get_colors()), HTTPStatus.OK)
+    except ColorFormatException as e:
+        return abort_and_log(HTTPStatus.NOT_FOUND, str(e), include_exc_info=True)
+
+
+def diffexp_obs_post(request, data_adaptor):
+    if not data_adaptor.dataset_config.diffexp__enable:
+        return abort(HTTPStatus.NOT_IMPLEMENTED)
+
+    args = request.get_json()
+    try:
+        # TODO: implement varfilter mode
+        mode = DiffExpMode(args["mode"])
+
+        if mode == DiffExpMode.VAR_FILTER or "varFilter" in args:
+            return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, "varFilter not enabled")
+
+        set1_filter = args.get("set1", {"filter": {}})["filter"]
+        set2_filter = args.get("set2", {"filter": {}})["filter"]
+        count = args.get("count", None)
+
+        if set1_filter is None or set2_filter is None or count is None:
+            return abort_and_log(HTTPStatus.BAD_REQUEST, "missing required parameter")
+        if Axis.VAR in set1_filter or Axis.VAR in set2_filter:
+            return abort_and_log(HTTPStatus.BAD_REQUEST, "var axis filter not enabled")
+
+    except (KeyError, TypeError) as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+
+    try:
+        diffexp = data_adaptor.diffexp_topN(set1_filter, set2_filter, count)
+        return make_response(diffexp, HTTPStatus.OK, {"Content-Type": "application/json"})
+    except (ValueError, DisabledFeatureError, FilterError, ExceedsLimitError) as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+    except JSONEncodingValueError:
+        # JSON encoding failure, usually due to bad data. Just let it ripple up
+        # to default exception handler.
+        current_app.logger.warning(JSON_NaN_to_num_warning_msg)
+        raise
+
+
+def layout_obs_get(request, data_adaptor):
+    fields = request.args.getlist("layout-name", None)
+    num_columns_requested = len(data_adaptor.get_embedding_names()) if len(fields) == 0 else len(fields)
+    if data_adaptor.server_config.exceeds_limit("column_request_max", num_columns_requested):
+        return abort(HTTPStatus.BAD_REQUEST)
+
+    preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
+    if preferred_mimetype != "application/octet-stream":
+        return abort(HTTPStatus.NOT_ACCEPTABLE)
+
+    try:
+        return make_response(
+            data_adaptor.layout_to_fbs_matrix(fields), HTTPStatus.OK, {"Content-Type": "application/octet-stream"}
+        )
+    except (KeyError, DatasetAccessError) as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+    except PrepareError:
+        return abort_and_log(
+            HTTPStatus.NOT_IMPLEMENTED,
+            f"No embedding available {request.path}",
+            loglevel=logging.ERROR,
+            include_exc_info=True,
+        )
+
+
+def genesets_get(request, data_adaptor):
+    preferred_mimetype = request.accept_mimetypes.best_match(["application/json", "text/csv"])
+    if preferred_mimetype not in ("application/json", "text/csv"):
+        return abort(HTTPStatus.NOT_ACCEPTABLE)
+
+    try:
+        annotations = data_adaptor.dataset_config.user_annotations
+        (genesets, tid) = annotations.read_gene_sets(data_adaptor)
+
+        if preferred_mimetype == "text/csv":
+            return make_response(
+                annotations.gene_sets_to_csv(genesets),
+                HTTPStatus.OK,
+                {
+                    "Content-Type": "text/csv",
+                    "Content-Disposition": "attachment; filename=genesets.csv",
+                },
+            )
+        else:
+            return make_response(
+                jsonify({"genesets": annotations.gene_sets_to_response(genesets), "tid": tid}), HTTPStatus.OK
+            )
+    except (ValueError, KeyError, AnnotationsError) as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e))
+
+
+def genesets_put(request, data_adaptor):
+    annotations = data_adaptor.dataset_config.user_annotations
+    if not annotations.gene_sets_save_enabled():
+        return abort(HTTPStatus.NOT_IMPLEMENTED)
+
+    anno_collection = request.args.get("annotation-collection-name", default=None)
+    if anno_collection is not None:
+        if not annotations.is_safe_collection_name(anno_collection):
+            return abort(HTTPStatus.BAD_REQUEST, "Bad annotation collection name")
+        annotations.set_collection(anno_collection)
+
+    args = request.get_json()
+    try:
+        genesets = args.get("genesets", None)
+        tid = args.get("tid", None)
+        if genesets is None:
+            abort(HTTPStatus.BAD_REQUEST)
+
+        annotations.write_gene_sets(genesets, tid, data_adaptor)
+        return make_response(jsonify({"status": "OK"}), HTTPStatus.OK)
+    except (ValueError, DisabledFeatureError, KeyError) as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+    except (ObsoleteRequest, TypeError) as e:
+        return abort(HTTPStatus.NOT_FOUND, description=str(e))
+
+
+def summarize_var_helper(request, data_adaptor, key, raw_query):
+    preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
+    if preferred_mimetype != "application/octet-stream":
+        return abort(HTTPStatus.NOT_ACCEPTABLE)
+
+    summary_method = request.values.get("method", default="mean")
+    query_hash = summarizeQueryHash(raw_query)
+    if key and query_hash != key:
+        return abort(HTTPStatus.BAD_REQUEST, description="query key did not match")
+
+    args_filter_only = request.values.copy()
+    args_filter_only.poplist("method")
+    args_filter_only.poplist("key")
+
+    try:
+        filter = _query_parameter_to_filter(args_filter_only)
+        return make_response(
+            data_adaptor.summarize_var(summary_method, filter, query_hash),
+            HTTPStatus.OK,
+            {"Content-Type": "application/octet-stream"},
+        )
+    except (ValueError) as e:
+        return abort(HTTPStatus.NOT_FOUND, description=str(e))
+    except (UnsupportedSummaryMethod, FilterError) as e:
+        return abort(HTTPStatus.BAD_REQUEST, description=str(e))
+
+
+def summarize_var_get(request, data_adaptor):
+    return summarize_var_helper(request, data_adaptor, None, request.query_string)
+
+
+def summarize_var_post(request, data_adaptor):
+    if not request.content_type or "application/x-www-form-urlencoded" not in request.content_type:
+        return abort(HTTPStatus.UNSUPPORTED_MEDIA_TYPE)
+    if request.content_length > 1_000_000:  # just a sanity check to avoid memory exhaustion
+        return abort(HTTPStatus.BAD_REQUEST)
+
+    key = request.args.get("key", default=None)
+    return summarize_var_helper(request, data_adaptor, key, request.get_data())

server/common/utils/corpora_constants.py

@@ -0,0 +1,22 @@
+class CorporaConstants(object):
+    REQUIRED_SIMPLE_METADATA_FIELDS = [
+        "version",
+        "title",
+        "layer_descriptions",
+        "organism",
+        "organism_ontology_term_id",
+    ]
+
+    # The Corpora specification requires some values encoded as JSON due to the inability of AnnData to store complex
+    # types.
+    OPTIONAL_JSON_ENCODED_METADATA_FIELD = ["contributors", "project_links"]
+
+    OPTIONAL_SIMPLE_METADATA_FIELDS = [
+        "preprint_doi",
+        "publication_doi",
+        "default_embedding",
+        "default_field",
+        "tags",
+        "project_name",
+        "project_description",
+    ]

server/common/utils/data_locator.py

@@ -0,0 +1,154 @@
+import os
+import tempfile
+import fsspec
+from datetime import datetime
+import boto3
+import botocore
+from urllib.parse import urlparse
+
+
+class DataLocator:
+    """
+    DataLocator is a simple wrapper around fsspec functionality, and provides a
+    set of functions to encapsulate a data location (URI or path), interogate
+    metadata about the object at that location (size, existance, etc) and
+    access the underlying data.
+
+    https://filesystem-spec.readthedocs.io/en/latest/index.html
+
+    Example:
+        dl = DataLocator("/tmp/foo.h5ad")
+        if dl.exists():
+            print(dl.size())
+            with dl.open() as f:
+                thecontents = f.read()
+
+    DataLocator will accept a URI or native path.  Error handling is as defined
+    in fsspec.
+
+    """
+
+    def __init__(self, uri_or_path, region_name=None):
+        if isinstance(uri_or_path, DataLocator):
+            locator = uri_or_path
+            self.uri_or_path = locator.uri_or_path
+            self.protocol = locator.protocol
+            self.path = locator.path
+            self.cname = locator.cname
+        else:
+            self.uri_or_path = uri_or_path
+            self.protocol, self.path = DataLocator._get_protocol_and_path(uri_or_path)
+            # work-around for LocalFileSystem not treating file: and None as the same scheme/protocol
+            self.cname = self.path if self.protocol == "file" else self.uri_or_path
+
+        # fsspec.filesystem will throw RuntimeError if the protocol is unsupported
+        if self.protocol == "s3":
+            if region_name:
+                config_kwargs = dict(region_name=region_name)
+                self.fs = fsspec.filesystem(self.protocol, listings_expiry_time=30, config_kwargs=config_kwargs)
+            else:
+                self.fs = fsspec.filesystem(self.protocol, listings_expiry_time=30)
+        else:
+            self.fs = fsspec.filesystem(self.protocol)
+
+    def __repr__(self):
+        return f"DataLocator(protocol={self.protocol}, cname={self.cname}, "
+        f"path={self.path}, uri_or_path={self.uri_or_path})"
+
+    @staticmethod
+    def _get_protocol_and_path(uri_or_path):
+        if "://" in uri_or_path:
+            protocol, path = uri_or_path.split("://", 1)
+            # windows!!!  Ignore single letter drive identifiers,
+            # eg, G:\foo.txt
+            if len(protocol) > 1:
+                return protocol, path
+        return None, uri_or_path
+
+    def exists(self):
+        return self.fs.exists(self.cname)
+
+    def size(self):
+        return self.fs.size(self.cname)
+
+    def lastmodtime(self):
+        """ return datetime object representing last modification time, or None if unavailable """
+        info = self.fs.info(self.cname)
+        if self.islocal() and info is not None:
+            return datetime.fromtimestamp(info["mtime"])
+        else:
+            return getattr(info, "LastModified", None)
+
+    def abspath(self):
+        """
+        return the absolute path for the locator - only really does something
+        for file: protocol, as all others are already absolute
+        """
+        if self.islocal():
+            return os.path.abspath(self.path)
+        else:
+            return self.uri_or_path
+
+    def isfile(self):
+        return self.fs.isfile(self.cname)
+
+    def open(self, *args):
+        return self.fs.open(self.uri_or_path, *args)
+
+    def islocal(self):
+        return self.protocol is None or self.protocol == "file"
+
+    def local_handle(self):
+        if self.islocal():
+            return LocalFilePath(self.path)
+
+        # if not local, create a tmp file system object to contain the data,
+        # and clean it up when done.  If the path has a suffix/extension,
+        # do our best to create a file with the same.
+        ext = os.path.splitext(self.path)
+        suffix = None if ext[1] == "" else ext[1]
+        with self.open() as src, tempfile.NamedTemporaryFile(prefix="cellxgene_", suffix=suffix, delete=False) as tmp:
+            tmp.write(src.read())
+            tmp.close()
+            src.close()
+            tmp_path = tmp.name
+            return LocalFilePath(tmp_path, delete=True)
+
+    def ls(self):
+        paths = self.fs.ls(self.uri_or_path)
+        return [os.path.basename(p) for p in paths]
+
+
+class LocalFilePath:
+    def __init__(self, tmp_path, delete=False):
+        self.tmp_path = tmp_path
+        self.delete = delete
+
+    def __enter__(self):
+        return self.tmp_path
+
+    def __exit__(self, *args):
+        if self.delete:
+            os.unlink(self.tmp_path)
+
+
+def discover_s3_region_name(uri):
+    """If this is an s3 protocol, discover and return the (aws) region name.
+    If a return name could not be discovered, or if the uri is not an s3 protocol, return None."""
+
+    protocol, _ = DataLocator._get_protocol_and_path(uri)
+    if protocol == "s3":
+        bucket = urlparse(uri).netloc
+        client = boto3.client("s3")
+        try:
+            res = client.head_bucket(Bucket=bucket)
+        except botocore.exceptions.ClientError:
+            return None
+
+        region = res.get("ResponseMetadata", {}).get("HTTPHeaders", {}).get("x-amz-bucket-region")
+        if region:
+            return region
+        else:
+            return None
+
+    return None

server/common/utils/type_conversion_utils.py

@@ -0,0 +1,191 @@
+from typing import Union, Tuple
+import logging
+
+import numpy as np
+import pandas as pd
+
+"""
+These routines drive all type inference for the schema generation and the
+FBS (REST OTA) encoding.
+
+
+H5AD Type                       REST              REST
+(ndarray, Series, Index)        FBS encoding      schema type       ERROR/exceptions
+----------------------------    --------------    ---------------   ----------------------
+bool_/bool                      uint8             boolean
+(u)int8, (u)int16, int32        int32             int32
+uint32, (u)int64                int32             int32             CHECKS value bounds
+float16, float32, float64       float32           float32[0]
+
+categorical[T is numeric[4]]:
+    hasna = False               T                 categorical[1]
+    hasna = True                float32           categorical[1]    CHECKS value bounds
+
+categorical[T not numeric]      JSON/str          categorical[1,2]
+
+(other object)                  JSON/str          string
+
+(all other)                                                         Always an ERROR[3]
+
+
+Notes:
+[0] IEEE format, includes non-finite numbers (NaN, Inf, ...)
+[1] with NO categories enumerated (client side does it to handle rounding)
+[2] NA (undefined) categories are assigned a JSON null value
+[3] Includes all other numpy types:  datetime, complex, etc.
+[4] means float, int, uint (dtype.kind in ['i','u','f'])
+
+"""
+
+
+def get_dtypes_and_schemas_of_dataframe(dataframe: pd.DataFrame):
+    dtypes_by_column_name = {}
+    schema_type_hints_by_column_name = {}
+
+    for column_name, column_values in dataframe.items():
+        (
+            dtypes_by_column_name[column_name],
+            schema_type_hints_by_column_name[column_name],
+        ) = get_dtype_and_schema_of_array(column_values)
+
+    return dtypes_by_column_name, schema_type_hints_by_column_name
+
+
+def get_encoding_dtype_of_array(array: Union[np.ndarray, pd.Series, pd.Index]) -> np.dtype:
+    return _get_type_info(array)[0]
+
+
+def get_schema_type_hint_of_array(array: Union[np.ndarray, pd.Series, pd.Index]) -> dict:
+    return _get_type_info(array)[1]
+
+
+def get_dtype_and_schema_of_array(array: Union[np.ndarray, pd.Series, pd.Index]) -> Tuple[np.dtype, dict]:
+    """Return tuple (encoding_dtype, schema_type_hint)"""
+    return _get_type_info(array)
+
+
+def get_schema_type_hint_from_dtype(dtype) -> dict:
+    res = _get_type_info_from_dtype(dtype)
+    if res is None:
+        raise TypeError(f"Annotations of type {dtype} are unsupported.")
+    else:
+        return res[1]
+
+
+def _get_type_info_from_dtype(dtype) -> Union[Tuple[np.dtype, dict], None]:
+    """
+    Best-effort to determine encoding type and schema hint from a dtype.
+    If this is not possible, or the type is unsupported, return None.
+
+    This should be a subset of the cases which are supported by
+    _get_type_info().  The latter should be preferred if the array (values)
+    are available for typing.
+    """
+    if dtype.kind == "b":
+        return (np.uint8, {"type": "boolean"})
+
+    if dtype.kind == "U":
+        return (np.dtype(str), {"type": "string"})
+
+    if dtype.kind in ["i", "u"]:
+        if np.can_cast(dtype, np.int32):
+            return (np.int32, {"type": "int32"})
+
+    if dtype.kind == "f":
+        _float64_warning(dtype)
+        return (np.float32, {"type": "float32"})
+
+    if dtype.kind == "O" and not dtype.name == "category":
+        return (np.dtype(str), {"type": "string"})
+
+    return None
+
+
+def _get_type_info(array: Union[np.ndarray, pd.Series, pd.Index]) -> Tuple[np.dtype, dict]:
+    """
+    Determine encoding type and schema hint from an array.  This allows more
+    flexible casting than may be possible by using just the dtype, as it can
+    account for category types and array values.
+    """
+    if (
+        not isinstance(array, np.ndarray)
+        and not isinstance(array, pd.Series)
+        and not isinstance(array, pd.Index)
+        and not hasattr(array, "dtype")
+    ):
+        raise TypeError("Unsupported data type.")
+
+    dtype = array.dtype
+
+    res = _get_type_info_from_dtype(dtype)
+    if res is not None:
+        return res
+
+    if dtype.kind == "O":
+        if dtype.name == "category":
+            # Sometimes CategoricalDType can be encoded as int or float without further fuss.
+            # Do not specify the categories in the schema - let the client-side figure it out
+            # on its own.  Utilize Series.to_numpy() to do casting that handles categorical
+            # NA/NaN (missing or undefined) categories.
+            if dtype.categories.dtype.kind in ["f", "i", "u"]:
+                return (
+                    _get_type_info(array.to_numpy())[0],
+                    {"type": "categorical"},
+                )
+            else:
+                return (np.dtype(str), {"type": "categorical", "categories": dtype.categories.to_list()})
+
+        # all other extension types are str-encoded
+        return (np.dtype(str), {"type": "string"})
+
+    if dtype.kind in ["i", "u"] and _can_cast_array_values_to_int32(array):
+        return (np.int32, {"type": "int32"})
+
+    if dtype.kind == "f":
+        _float64_warning(array.dtype)
+        return (np.float32, {"type": "float32"})
+
+    raise TypeError(f"Annotations of type {dtype} are unsupported.")
+
+
+def _float64_warning(dtype):
+    """
+    Warn the user if we are down-casting a float64 to float32, and may potentially lose information.
+    """
+    if dtype.kind == "f" and not np.can_cast(dtype, np.float32):
+        logging.warning(f"Type {dtype.name} will be converted to 32 bit float and may lose precision.")
+
+
+def _can_cast_array_values_to_int32(array: Union[np.ndarray, pd.Series, pd.Index]) -> bool:
+    """
+    Return true if the (U)INT array values can be safely cast to int32.  We allow size reducing
+    casts (ie, int64 to int32) if no actual values require the larger size (ie, actual values
+    can be represented by the smaller type).
+    """
+    assert array.dtype.kind in ["u", "i"]
+
+    if np.can_cast(array.dtype, np.int32):
+        return True
+
+    if array.size == 0:
+        return True
+
+    int32_machine_limits = np.iinfo(np.int32)
+    if array.min() >= int32_machine_limits.min and array.max() <= int32_machine_limits.max:
+        return True
+
+    return False
+
+
+def convert_string_to_value(value: str):
+    """convert a string to value with the most appropriate type"""
+    if value.lower() == "true":
+        return True
+    if value.lower() == "false":
+        return False
+    if value == "null":
+        return None
+    try:
+        return eval(value)
+    except:  # noqa E722
+        return value

server/common/utils/utils.py

@@ -0,0 +1,126 @@
+import contextlib
+import errno
+import importlib.util
+import logging
+import os
+import pkgutil
+import socket
+from urllib.parse import urlsplit, urljoin
+
+import numpy as np
+from flask import json
+
+from server.common.errors import ConfigurationError
+
+
+def find_available_port(host, port=5005):
+    """
+    Helper method to find open port on host. Tries 5000 ports incremented from the specified port
+    """
+    # Takes approx 2 seconds to do a scan of 5000 ports on my laptop
+    num_ports_to_try = 5000
+    for port_to_try in range(port, port + num_ports_to_try):
+        if is_port_available(host, port_to_try):
+            return port_to_try
+    raise socket.error(errno.EADDRINUSE, f"No port in range {port} - {port + num_ports_to_try - 1} available.")
+
+
+def is_port_available(host, port):
+    is_available = False
+    with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
+        try:
+            s.bind((host, port))
+            is_available = True
+        except socket.error:
+            pass
+    return is_available
+
+
+def sort_options(command):
+    """
+    Helper for the click options - will sort options in a command, and can
+    be used as a decorator.
+    """
+    command.params.sort(key=lambda p: p.name)
+    return command
+
+
+def path_join(base, *urls):
+    """
+    this is like urllib.parse.urljoin, except it works around the scheme-specific
+    cleverness in the aforementioned code, ignores anything in the url except the path,
+    and accepts more than one url.
+    """
+    if not base.endswith("/"):
+        base += "/"
+    btpl = urlsplit(base)
+    path = btpl.path
+    for url in urls:
+        utpl = urlsplit(url)
+        if btpl.scheme == "":
+            path = os.path.join(path, utpl.path)
+            path = os.path.normpath(path)
+        else:
+            path = urljoin(path, utpl.path)
+    return btpl._replace(path=path).geturl()
+
+
+class StrictJSONEncoder(json.JSONEncoder):
+    """
+    Custom JSON encoder set-up performing two tasks:
+    1. Strict JSON conformance with non-finite floats (NaN, +/-Inf) via allow_nan=False
+    2. Convert various Numpy types into python types so the encoder will correctly encode.
+    """
+
+    def __init__(self, *args, **kwargs):
+        """
+        NaN/Infinities are illegal in standard JSON.  Python extends JSON with
+        non-standard symbols that most JavaScript JSON parsers do not understand.
+        The `allow_nan` parameter will force Python simplejson to throw an ValueError
+        if it runs into non-finite floating point values which are unsupported by
+        standard JSON.
+        """
+        kwargs["allow_nan"] = False
+        super().__init__(*args, **kwargs)
+
+    def default(self, obj):
+        """This helps us convert types not supported by the native JSON encoder into
+        standard python types, eg, np.int64."""
+        if isinstance(obj, np.floating):
+            return float(obj)
+        if isinstance(obj, np.integer):
+            return int(obj)
+        return json.JSONEncoder.default(self, obj)
+
+
+def custom_format_warning(msg, *args, **kwargs):
+    return f"[cellxgene] Warning: {msg} \n"
+
+
+def jsonify_strict(data):
+    return json.dumps(data, cls=StrictJSONEncoder, allow_nan=False)
+
+
+def import_plugins(plugin_module):
+    """
+    Load optional plugin modules from server.common.plugins
+
+    If you would like to customize cellxgene, you can add submodules to server.common.plugins before running the app.
+    This code will import each, loading the code in each. If no plugins are defined, initializing the app continues as
+    normal.
+    """
+    loaded_modules = []
+    try:
+        pkg = importlib.import_module(plugin_module)
+        for loader, name, is_pkg in pkgutil.walk_packages(pkg.__path__):
+            full_name = f"{plugin_module}.{name}"
+            try:
+                module = importlib.import_module(full_name)
+            except Exception as e:
+                raise ConfigurationError(f"Unexpected error while importing plugin: {plugin_module}.{name}: {str(e)}")
+            loaded_modules.append(module)
+    except ModuleNotFoundError as e:
+        #  This exception occurs when the plugin_module does not exist (not an error).
+        logging.debug(f"No plugins found in module: {plugin_module}: {str(e)}")
+
+    return loaded_modules

server/data_anndata/anndata_adaptor.py

@@ -0,0 +1,373 @@
+import warnings
+
+import anndata
+import numpy as np
+from packaging import version
+from pandas.core.dtypes.dtypes import CategoricalDtype
+from scipy import sparse
+
+import server.common.compute.diffexp_generic as diffexp_generic
+import server.common.compute.estimate_distribution as estimate_distribution
+from server.common.colors import convert_anndata_category_colors_to_cxg_category_colors
+from server.common.constants import Axis, MAX_LAYOUTS, XApproximateDistribution
+from server.common.corpora import corpora_get_props_from_anndata
+from server.common.errors import PrepareError, DatasetAccessError
+from server.common.utils.type_conversion_utils import get_schema_type_hint_of_array
+from server.data_common.data_adaptor import DataAdaptor
+from server.common.fbs.matrix import encode_matrix_fbs
+
+anndata_version = version.parse(str(anndata.__version__)).release
+
+
+def anndata_version_is_pre_070():
+    major = anndata_version[0]
+    minor = anndata_version[1] if len(anndata_version) > 1 else 0
+    return major == 0 and minor < 7
+
+
+class AnndataAdaptor(DataAdaptor):
+    def __init__(self, data_locator, app_config=None, dataset_config=None):
+        super().__init__(data_locator, app_config, dataset_config)
+        self.data = None
+        self.X_approximate_distribution = None
+        self._load_data(data_locator)
+        self._validate_and_initialize()
+
+    def cleanup(self):
+        pass
+
+    @staticmethod
+    def pre_load_validation(data_locator):
+        if data_locator.islocal():
+            # if data locator is local, apply file system conventions and other "cheap"
+            # validation checks.  If a URI, defer until we actually fetch the data and
+            # try to read it.  Many of these tests don't make sense for URIs (eg, extension-
+            # based typing).
+            if not data_locator.exists():
+                raise DatasetAccessError("does not exist")
+            if not data_locator.isfile():
+                raise DatasetAccessError("is not a file")
+
+    @staticmethod
+    def file_size(data_locator):
+        return data_locator.size() if data_locator.islocal() else 0
+
+    @staticmethod
+    def open(data_locator, app_config, dataset_config=None):
+        return AnndataAdaptor(data_locator, app_config, dataset_config)
+
+    def get_corpora_props(self):
+        return corpora_get_props_from_anndata(self.data)
+
+    def get_name(self):
+        return "cellxgene anndata adaptor version"
+
+    def get_library_versions(self):
+        return dict(anndata=str(anndata.__version__))
+
+    @staticmethod
+    def _create_unique_column_name(df, col_name_prefix):
+        """given the columns of a dataframe, and a name prefix, return a column name which
+        does not exist in the dataframe, AND which is prefixed by `prefix`
+
+        The approach is to append a numeric suffix, starting at zero and increasing by
+        one, until an unused name is found (eg, prefix_0, prefix_1, ...).
+        """
+        suffix = 0
+        while f"{col_name_prefix}{suffix}" in df:
+            suffix += 1
+        return f"{col_name_prefix}{suffix}"
+
+    def _alias_annotation_names(self):
+        """
+        The front-end relies on the existance of a unique, human-readable
+        index for obs & var (eg, var is typically gene name, obs the cell name).
+        The user can specify these via the --obs-names and --var-names config.
+        If they are not specified, use the existing index to create them, giving
+        the resulting column a unique name (eg, "name").
+
+        In both cases, enforce that the result is unique, and communicate the
+        index column name to the front-end via the obs_names and var_names config
+        (which is incorporated into the schema).
+        """
+        self.original_obs_index = self.data.obs.index
+
+        for (ax_name, var_name) in ((Axis.OBS, "obs"), (Axis.VAR, "var")):
+            config_name = f"single_dataset__{var_name}_names"
+            parameter_name = f"{var_name}_names"
+            name = getattr(self.server_config, config_name)
+            df_axis = getattr(self.data, str(ax_name))
+            if name is None:
+                # Default: create unique names from index
+                if not df_axis.index.is_unique:
+                    raise KeyError(
+                        f"Values in {ax_name}.index must be unique. "
+                        "Please prepare data to contain unique index values, or specify an "
+                        "alternative with --{ax_name}-name."
+                    )
+                name = self._create_unique_column_name(df_axis.columns, "name_")
+                self.parameters[parameter_name] = name
+                # reset index to simple range; alias name to point at the
+                # previously specified index.
+                df_axis.rename_axis(name, inplace=True)
+                df_axis.reset_index(inplace=True)
+            elif name in df_axis.columns:
+                # User has specified alternative column for unique names, and it exists
+                if not df_axis[name].is_unique:
+                    raise KeyError(
+                        f"Values in {ax_name}.{name} must be unique. " "Please prepare data to contain unique values."
+                    )
+                df_axis.reset_index(drop=True, inplace=True)
+                self.parameters[parameter_name] = name
+            else:
+                # user specified a non-existent column name
+                raise KeyError(f"Annotation name {name}, specified in --{ax_name}-name does not exist.")
+
+    def _create_schema(self):
+        self.schema = {
+            "dataframe": {
+                "nObs": self.cell_count,
+                "nVar": self.gene_count,
+                **get_schema_type_hint_of_array(self.data.X),
+            },
+            "annotations": {
+                "obs": {"index": self.parameters.get("obs_names"), "columns": []},
+                "var": {"index": self.parameters.get("var_names"), "columns": []},
+            },
+            "layout": {"obs": []},
+        }
+        for ax in Axis:
+            curr_axis = getattr(self.data, str(ax))
+            for ann in curr_axis:
+                ann_schema = {"name": ann, "writable": False}
+                ann_schema.update(get_schema_type_hint_of_array(curr_axis[ann]))
+                self.schema["annotations"][ax]["columns"].append(ann_schema)
+
+        for layout in self.get_embedding_names():
+            layout_schema = {"name": layout, "type": "float32", "dims": [f"{layout}_0", f"{layout}_1"]}
+            self.schema["layout"]["obs"].append(layout_schema)
+
+    def get_schema(self):
+        return self.schema
+
+    def _load_data(self, data_locator):
+        # as of AnnData 0.6.19, backed mode performs initial load fast, but at the
+        # cost of significantly slower access to X data.
+        try:
+            # there is no guarantee data_locator indicates a local file.  The AnnData
+            # API will only consume local file objects.  If we get a non-local object,
+            # make a copy in tmp, and delete it after we load into memory.
+            with data_locator.local_handle() as lh:
+                # as of AnnData 0.6.19, backed mode performs initial load fast, but at the
+                # cost of significantly slower access to X data.
+                backed = "r" if self.server_config.adaptor__anndata_adaptor__backed else None
+                self.data = anndata.read_h5ad(lh, backed=backed)
+
+        except ValueError:
+            raise DatasetAccessError(
+                "File must be in the .h5ad format. Please read "
+                "https://github.com/theislab/scanpy_usage/blob/master/170505_seurat/info_h5ad.md to "
+                "learn more about this format. You may be able to convert your file into this format "
+                "using `cellxgene prepare`, please run `cellxgene prepare --help` for more "
+                "information."
+            )
+        except MemoryError:
+            raise DatasetAccessError("Out of memory - file is too large for available memory.")
+        except Exception:
+            raise DatasetAccessError(
+                "File not found or is inaccessible. File must be an .h5ad object. "
+                "Please check your input and try again."
+            )
+
+    def _validate_and_initialize(self):
+        if anndata_version_is_pre_070():
+            warnings.warn(
+                "Use of anndata versions older than 0.7 will have serious issues. Please update to at "
+                "least anndata 0.7 or later."
+            )
+
+        # var and obs column names must be unique
+        if not self.data.obs.columns.is_unique or not self.data.var.columns.is_unique:
+            raise KeyError("All annotation column names must be unique.")
+
+        self._alias_annotation_names()
+        self._validate_data_types()
+        self.cell_count = self.data.shape[0]
+        self.gene_count = self.data.shape[1]
+        self._create_schema()
+
+        if self.dataset_config.X_approximate_distribution == "auto":
+            """Lazy evaluate the heuristic if we are backed."""
+            if not self.data.isbacked:
+                self.X_approximate_distribution = estimate_distribution.estimate_approximate_distribution(self.data.X)
+        else:
+            self.X_approximate_distribution = self.dataset_config.X_approximate_distribution
+
+        # heuristic
+        n_values = self.data.shape[0] * self.data.shape[1]
+        if (n_values > 1e8 and self.server_config.adaptor__anndata_adaptor__backed is True) or (n_values > 5e8):
+            self.parameters.update({"diffexp_may_be_slow": True})
+
+    def _is_valid_layout(self, arr):
+        """return True if this layout data is a valid array for front-end presentation:
+        * ndarray, dtype float/int/uint
+        * with shape (n_obs, >= 2)
+        * with all values finite or NaN (no +Inf or -Inf)
+        """
+        is_valid = type(arr) == np.ndarray and arr.dtype.kind in "fiu"
+        is_valid = is_valid and arr.shape[0] == self.data.n_obs and arr.shape[1] >= 2
+        is_valid = is_valid and not np.any(np.isinf(arr)) and not np.all(np.isnan(arr))
+        return is_valid
+
+    def _validate_data_types(self):
+        # The backed API does not support interrogation of the underlying sparsity or sparse matrix type
+        # Fake it by asking for a small subarray and testing it.   NOTE: if the user has ignored our
+        # anndata <= 0.7 warning, opted for the --backed option, and specified a large, sparse dataset,
+        # this "small" indexing request will load the entire X array. This is due to a bug in anndata<=0.7
+        # which will load the entire X matrix to fullfill any slicing request if X is sparse.  See
+        # user warning in _load_data().
+        X0 = self.data.X[0, 0:1]
+        if sparse.isspmatrix(X0) and not sparse.isspmatrix_csc(X0):
+            warnings.warn(
+                "Anndata data matrix is sparse, but not a CSC (columnar) matrix.  "
+                "Performance may be improved by using CSC."
+            )
+        if self.data.X.dtype > np.dtype(np.float32):
+            warnings.warn(
+                f"Anndata data matrix is in {self.data.X.dtype} format not float32. " f"Precision may be truncated."
+            )
+        for ax in Axis:
+            curr_axis = getattr(self.data, str(ax))
+            for ann in curr_axis:
+                datatype = curr_axis[ann].dtype
+                downcast_map = {
+                    "int64": "int32",
+                    "uint32": "int32",
+                    "uint64": "int32",
+                    "float64": "float32",
+                }
+                if datatype in downcast_map:
+                    warnings.warn(
+                        f"Anndata annotation {ax}:{ann} is in unsupported format: {datatype}. "
+                        f"Data will be downcast to {downcast_map[datatype]}."
+                    )
+                if isinstance(datatype, CategoricalDtype):
+                    category_num = len(curr_axis[ann].dtype.categories)
+                    if category_num > 500 and category_num > self.dataset_config.presentation__max_categories:
+                        warnings.warn(
+                            f"{str(ax).title()} annotation '{ann}' has {category_num} categories, this may be "
+                            f"cumbersome or slow to display. We recommend setting the "
+                            f"--max-category-items option to 500, this will hide categorical "
+                            f"annotations with more than 500 categories in the UI"
+                        )
+
+    def annotation_to_fbs_matrix(self, axis, fields=None, labels=None):
+        if axis == Axis.OBS:
+            if labels is not None and not labels.empty:
+                df = self.data.obs.join(labels, self.parameters.get("obs_names"))
+            else:
+                df = self.data.obs
+        else:
+            df = self.data.var
+
+        if fields is not None and len(fields) > 0:
+            df = df[fields]
+        return encode_matrix_fbs(df, col_idx=df.columns)
+
+    def get_embedding_names(self):
+        """
+        Return pre-computed embeddings.
+
+        function:
+            a) generate list of default layouts
+            b) validate layouts are legal.  remove/warn on any that are not
+            c) cap total list of layouts at global const MAX_LAYOUTS
+        """
+        # load default layouts from the data.
+        layouts = self.dataset_config.embeddings__names
+
+        if layouts is None or len(layouts) == 0:
+            layouts = [key[2:] for key in self.data.obsm_keys() if type(key) == str and key.startswith("X_")]
+
+        # remove invalid layouts
+        valid_layouts = []
+        obsm_keys = self.data.obsm_keys()
+        for layout in layouts:
+            layout_name = f"X_{layout}"
+            if layout_name not in obsm_keys:
+                warnings.warn(f"Ignoring unknown layout name: {layout}.")
+            elif not self._is_valid_layout(self.data.obsm[layout_name]):
+                warnings.warn(f"Ignoring layout due to malformed shape or data type: {layout}")
+            else:
+                valid_layouts.append(layout)
+
+        if len(valid_layouts) == 0:
+            raise PrepareError("No valid layout data.")
+
+        # cap layouts to MAX_LAYOUTS
+        return valid_layouts[0:MAX_LAYOUTS]
+
+    def get_embedding_array(self, ename, dims=2):
+        full_embedding = self.data.obsm[f"X_{ename}"]
+        return full_embedding[:, 0:dims]
+
+    def compute_diffexp_ttest(self, maskA, maskB, top_n=None, lfc_cutoff=None):
+        if top_n is None:
+            top_n = self.dataset_config.diffexp__top_n
+        if lfc_cutoff is None:
+            lfc_cutoff = self.dataset_config.diffexp__lfc_cutoff
+        return diffexp_generic.diffexp_ttest(self, maskA, maskB, top_n, lfc_cutoff)
+
+    def get_colors(self):
+        return convert_anndata_category_colors_to_cxg_category_colors(self.data)
+
+    def get_X_array(self, obs_mask=None, var_mask=None):
+        # H5Py does not support boolean indexing (masks), so convert to integer indexing
+        # when backed (ie, when AnnData is using H5Py indexing)
+        if obs_mask is None:
+            obs_mask = slice(None)
+        elif self.data.isbacked and obs_mask.dtype == bool:
+            obs_mask = obs_mask.nonzero()[0]
+        if var_mask is None:
+            var_mask = slice(None)
+        elif self.data.isbacked and var_mask.dtype == bool:
+            var_mask = var_mask.nonzero()[0]
+        X = self.data.X[obs_mask, var_mask]
+        return X
+
+    def get_X_approximate_distribution(self) -> XApproximateDistribution:
+        """return the approximate distribution of the X matrix."""
+        if self.X_approximate_distribution is None:
+            """Not yet evaluated."""
+            assert self.dataset_config.X_approximate_distribution == "auto"
+            self.data = self.data.to_memory()  # loads data
+            self.X_approximate_distribution = estimate_distribution.estimate_approximate_distribution(self.data.X)
+
+        return self.X_approximate_distribution
+
+    def get_shape(self):
+        return self.data.shape
+
+    def query_var_array(self, term_name):
+        return getattr(self.data.var, term_name)
+
+    def query_obs_array(self, term_name):
+        return getattr(self.data.obs, term_name)
+
+    def get_obs_index(self):
+        name = self.server_config.single_dataset__obs_names
+        if name is None:
+            return self.original_obs_index
+        else:
+            return self.data.obs[name]
+
+    def get_obs_columns(self):
+        return self.data.obs.columns
+
+    def get_obs_keys(self):
+        # return list of keys
+        return self.data.obs.keys().to_list()
+
+    def get_var_keys(self):
+        # return list of keys
+        return self.data.var.keys().to_list()

server/data_common/data_adaptor.py

@@ -0,0 +1,423 @@
+from abc import ABCMeta, abstractmethod
+from os.path import basename, splitext
+import numpy as np
+import pandas as pd
+from scipy import sparse
+from server_timing import Timing as ServerTiming
+
+from server.common.config.app_config import AppConfig
+from server.common.constants import Axis, XApproximateDistribution
+from server.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError, UnsupportedSummaryMethod
+from server.common.utils.utils import jsonify_strict
+from server.common.fbs.matrix import encode_matrix_fbs
+from server.common.genesets import validate_gene_sets
+
+
+class DataAdaptor(metaclass=ABCMeta):
+    """Base class for loading and accessing matrix data"""
+
+    def __init__(self, data_locator, app_config, dataset_config=None):
+        if not isinstance(app_config, AppConfig):
+            raise TypeError("config expected to be of type AppConfig")
+
+        # location to the dataset
+        self.data_locator = data_locator
+
+        # config is the application configuration
+        self.app_config = app_config
+        self.server_config = self.app_config.server_config
+        self.dataset_config = dataset_config or app_config.dataset_config
+
+        # parameters set by this data adaptor based on the data.
+        self.parameters = {}
+
+    @staticmethod
+    @abstractmethod
+    def pre_load_validation(data_locator):
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def open(data_locator, app_config, dataset_config):
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def file_size(data_locator):
+        pass
+
+    @abstractmethod
+    def get_name(self):
+        """return a string name for this data adaptor"""
+        pass
+
+    @abstractmethod
+    def get_library_versions(self):
+        """return a dictionary of library name to library versions"""
+        pass
+
+    @abstractmethod
+    def get_embedding_names(self):
+        """return a list of pre-computed embedding names"""
+        pass
+
+    @abstractmethod
+    def get_embedding_array(self, ename, dims=2):
+        """return an numpy array for the given pre-computed embedding name."""
+        pass
+
+    @abstractmethod
+    def get_X_array(self, obs_mask=None, var_mask=None):
+        """return the X array, possibly filtered by obs_mask or var_mask.
+        the return type is either ndarray or scipy.sparse.spmatrix."""
+        pass
+
+    def get_X_approximate_distribution(self) -> XApproximateDistribution:
+        """return the approximate distribution of the X matrix."""
+        return XApproximateDistribution.NORMAL
+
+    @abstractmethod
+    def get_shape(self):
+        pass
+
+    @abstractmethod
+    def query_var_array(self, term_var):
+        pass
+
+    @abstractmethod
+    def query_obs_array(self, term_var):
+        pass
+
+    @abstractmethod
+    def get_colors(self):
+        pass
+
+    @abstractmethod
+    def get_obs_index(self):
+        pass
+
+    @abstractmethod
+    def get_obs_columns(self):
+        pass
+
+    @abstractmethod
+    def get_obs_keys(self):
+        # return list of keys
+        pass
+
+    @abstractmethod
+    def get_var_keys(self):
+        # return list of keys
+        pass
+
+    @abstractmethod
+    def cleanup(self):
+        pass
+
+    def get_data_locator(self):
+        return self.data_locator
+
+    def get_location(self):
+        return self.data_locator.uri_or_path
+
+    def get_about(self):
+        return None
+
+    def get_title(self):
+        # default to file name
+        location = self.get_location()
+        if location.endswith("/"):
+            location = location[:-1]
+        return splitext(basename(location))[0]
+
+    def get_corpora_props(self):
+        return None
+
+    @abstractmethod
+    def get_schema(self):
+        """
+        Return current schema
+        """
+        pass
+
+    @abstractmethod
+    def annotation_to_fbs_matrix(self, axis, field=None, uid=None):
+        """
+        Gets annotation value for each observation
+        :param axis: string obs or var
+        :param fields: list of keys for annotation to return, returns all annotation values if not set.
+        :return: flatbuffer: in fbs/matrix.fbs encoding
+        """
+        pass
+
+    def update_parameters(self, parameters):
+        parameters.update(self.parameters)
+
+    def _index_filter_to_mask(self, filter, count):
+        mask = np.zeros((count,), dtype=np.bool)
+        for i in filter:
+            if isinstance(i, list):
+                mask[i[0] : i[1]] = True
+            else:
+                mask[i] = True
+        return mask
+
+    def _axis_filter_to_mask(self, axis, filter, count):
+        mask = np.ones((count,), dtype=np.bool)
+        if "index" in filter:
+            mask = np.logical_and(mask, self._index_filter_to_mask(filter["index"], count))
+        if "annotation_value" in filter:
+            mask = np.logical_and(mask, self._annotation_filter_to_mask(axis, filter["annotation_value"], count))
+
+        return mask
+
+    def _annotation_filter_to_mask(self, axis, filter, count):
+        mask = np.ones((count,), dtype=np.bool)
+        for v in filter:
+            name = v["name"]
+            if axis == Axis.VAR:
+                anno_data = self.query_var_array(name)
+            elif axis == Axis.OBS:
+                anno_data = self.query_obs_array(name)
+
+            if anno_data.dtype.name in ["boolean", "category", "object"]:
+                values = v.get("values", [])
+                key_idx = np.in1d(anno_data, values)
+                mask = np.logical_and(mask, key_idx)
+
+            else:
+                min_ = v.get("min", None)
+                max_ = v.get("max", None)
+                if min_ is not None:
+                    key_idx = (anno_data >= min_).ravel()
+                    mask = np.logical_and(mask, key_idx)
+                if max_ is not None:
+                    key_idx = (anno_data <= max_).ravel()
+                    mask = np.logical_and(mask, key_idx)
+
+        return mask
+
+    def _filter_to_mask(self, filter):
+        """
+        Return the filter as a row and column selection list.
+        No filter on a dimension means 'all'
+        """
+        shape = self.get_shape()
+        var_selector = None
+        obs_selector = None
+        if filter is not None:
+            if Axis.OBS in filter:
+                obs_selector = self._axis_filter_to_mask(Axis.OBS, filter["obs"], shape[0])
+
+            if Axis.VAR in filter:
+                var_selector = self._axis_filter_to_mask(Axis.VAR, filter["var"], shape[1])
+
+        return (obs_selector, var_selector)
+
+    def check_new_labels(self, labels_df):
+        """Check the new annotations labels, then set the labels_df index"""
+        if labels_df is None or labels_df.empty:
+            return
+
+        labels_df.index = self.get_obs_index()
+        if labels_df.index.name is None:
+            labels_df.index.name = "index"
+
+        # all labels must have a name, which must be unique and not used in obs column names
+        if not labels_df.columns.is_unique:
+            raise KeyError("All column names specified in user annotations must be unique.")
+
+        # the label index must be unique, and must have same values the anndata obs index
+        if not labels_df.index.is_unique:
+            raise KeyError("All row index values specified in user annotations must be unique.")
+
+        obs_columns = self.get_obs_columns()
+
+        duplicate_columns = list(set(labels_df.columns) & set(obs_columns))
+        if len(duplicate_columns) > 0:
+            raise KeyError(
+                "Labels file may not contain column names which overlap " f"with h5ad obs columns {duplicate_columns}"
+            )
+
+        # labels must have same count as obs annotations
+        shape = self.get_shape()
+        if labels_df.shape[0] != shape[0]:
+            raise ValueError("Labels file must have same number of rows as data file.")
+
+        # This will convert a float column that contains integer data into an integer type.
+        # This case can occur when a user makes a copy of a category that originally contained integer data.
+        # The client always copies array data to floats, therefore the copy will contain floats instead of integers.
+        # float data is not allowed as a categorical type.
+        if any([np.issubdtype(coltype.type, np.floating) for coltype in labels_df.dtypes]):
+            labels_df = labels_df.convert_dtypes()
+            for col, dtype in zip(labels_df, labels_df.dtypes):
+                if isinstance(dtype, pd.Int32Dtype):
+                    labels_df[col] = labels_df[col].astype("int32")
+                if isinstance(dtype, pd.Int64Dtype):
+                    labels_df[col] = labels_df[col].astype("int64")
+
+        if any([np.issubdtype(coltype.type, np.floating) for coltype in labels_df.dtypes]):
+            raise ValueError("Columns may not have floating point types")
+
+        return labels_df
+
+    def check_new_gene_sets(self, genesets, context=None):
+        var_names = set(self.query_var_array(self.parameters.get("var_names")))
+        return validate_gene_sets(genesets, var_names)
+
+    def data_frame_to_fbs_matrix(self, filter, axis):
+        """
+        Retrieves data 'X' and returns in a flatbuffer Matrix.
+        :param filter: filter: dictionary with filter params
+        :param axis: string obs or var
+        :return: flatbuffer Matrix
+
+        Caveats:
+        * currently only supports access on VAR axis
+        * currently only supports filtering on VAR axis
+        """
+        if axis != Axis.VAR:
+            raise ValueError("Only VAR dimension access is supported")
+
+        try:
+            obs_selector, var_selector = self._filter_to_mask(filter)
+        except (KeyError, IndexError, TypeError, AttributeError):
+            raise FilterError("Error parsing filter")
+
+        if obs_selector is not None:
+            raise FilterError("filtering on obs unsupported")
+
+        num_columns = self.get_shape()[1] if var_selector is None else np.count_nonzero(var_selector)
+        if self.server_config.exceeds_limit("column_request_max", num_columns):
+            raise ExceedsLimitError("Requested dataframe columns exceed column request limit")
+
+        X = self.get_X_array(obs_selector, var_selector)
+        col_idx = np.nonzero([] if var_selector is None else var_selector)[0]
+        return encode_matrix_fbs(X, col_idx=col_idx, row_idx=None)
+
+    def diffexp_topN(self, obsFilterA, obsFilterB, top_n=None):
+        """
+        Computes the top N differentially expressed variables between two observation sets. If mode
+        is "TOP_N", then stats for the top N
+        dataframes
+        contain a subset of variables, then statistics for all variables will be returned, otherwise
+        only the top N vars will be returned.
+        :param obsFilterA: filter: dictionary with filter params for first set of observations
+        :param obsFilterB: filter: dictionary with filter params for second set of observations
+        :param top_n: Limit results to top N (Top var mode only)
+        :return: top N genes and corresponding stats
+        """
+        if Axis.VAR in obsFilterA or Axis.VAR in obsFilterB:
+            raise FilterError("Observation filters may not contain variable conditions")
+        try:
+            shape = self.get_shape()
+            obs_mask_A = self._axis_filter_to_mask(Axis.OBS, obsFilterA["obs"], shape[0])
+            obs_mask_B = self._axis_filter_to_mask(Axis.OBS, obsFilterB["obs"], shape[0])
+        except (KeyError, IndexError):
+            raise FilterError("Error parsing filter")
+        if top_n is None:
+            top_n = self.dataset_config.diffexp__top_n
+
+        if self.server_config.exceeds_limit(
+            "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B)
+        ):
+            raise ExceedsLimitError("Diffexp request exceeds max cell count limit")
+
+        result = self.compute_diffexp_ttest(
+            maskA=obs_mask_A,
+            maskB=obs_mask_B,
+            top_n=top_n,
+            lfc_cutoff=self.dataset_config.diffexp__lfc_cutoff,
+        )
+
+        try:
+            return jsonify_strict(result)
+        except ValueError:
+            raise JSONEncodingValueError("Error encoding differential expression to JSON")
+
+    @abstractmethod
+    def compute_diffexp_ttest(self, maskA, maskB, top_n, lfc_cutoff):
+        pass
+
+    @staticmethod
+    def normalize_embedding(embedding):
+        """Normalize embedding layout to meet client assumptions.
+        Embedding is an ndarray, shape (n_obs, n)., where n is normally 2
+        """
+
+        # scale isotropically
+        try:
+            min = np.nanmin(embedding, axis=0)
+            max = np.nanmax(embedding, axis=0)
+        except RuntimeError:
+            # indicates entire array was NaN, which should propagate
+            min = np.NaN
+            max = np.NaN
+
+        scale = np.amax(max - min)
+        normalized_layout = (embedding - min) / scale
+
+        # translate to center on both axis
+        translate = 0.5 - ((max - min) / scale / 2)
+        normalized_layout = normalized_layout + translate
+
+        normalized_layout = normalized_layout.astype(dtype=np.float32)
+        return normalized_layout
+
+    def layout_to_fbs_matrix(self, fields):
+        """
+        return specified embeddings as a flatbuffer, using the cellxgene matrix fbs encoding.
+
+        * returns only first two dimensions, with name {ename}_0 and {ename}_1,
+          where {ename} is the embedding name.
+        * client assumes each will be individually centered & scaled (isotropically)
+          to a [0, 1] range.
+        * does not support filtering
+
+        """
+        embeddings = self.get_embedding_names() if fields is None or len(fields) == 0 else fields
+        layout_data = []
+        with ServerTiming.time("layout.query"):
+            for ename in embeddings:
+                embedding = self.get_embedding_array(ename, 2)
+                normalized_layout = DataAdaptor.normalize_embedding(embedding)
+                layout_data.append(pd.DataFrame(normalized_layout, columns=[f"{ename}_0", f"{ename}_1"]))
+
+        with ServerTiming.time("layout.encode"):
+            if layout_data:
+                df = pd.concat(layout_data, axis=1, copy=False)
+            else:
+                df = pd.DataFrame()
+            fbs = encode_matrix_fbs(df, col_idx=df.columns, row_idx=None)
+
+        return fbs
+
+    def get_last_mod_time(self):
+        try:
+            lastmod = self.get_data_locator().lastmodtime()
+        except RuntimeError:
+            lastmod = None
+        return lastmod
+
+    def summarize_var(self, method, filter, query_hash):
+        if method != "mean":
+            raise UnsupportedSummaryMethod("Unknown gene set summary method.")
+
+        obs_selector, var_selector = self._filter_to_mask(filter)
+        if obs_selector is not None:
+            raise FilterError("filtering on obs unsupported")
+
+        # if no filter, just return zeros.  We don't have a use case
+        # for summarizing the entire X without a filter, and it would
+        # potentially be quite compute / memory intensive.
+        if var_selector is None or np.count_nonzero(var_selector) == 0:
+            mean = np.zeros((self.get_shape()[0], 1), dtype=np.float32)
+        else:
+            X = self.get_X_array(obs_selector, var_selector)
+            if sparse.issparse(X):
+                mean = X.mean(axis=1).A
+            else:
+                mean = X.mean(axis=1, keepdims=True)
+
+        col_idx = pd.Index([query_hash])
+        return encode_matrix_fbs(mean, col_idx=col_idx, row_idx=None)

server/data_common/matrix_loader.py

@@ -0,0 +1,56 @@
+from enum import Enum
+
+from server.common.utils.data_locator import DataLocator
+from server.common.errors import DatasetAccessError
+from http import HTTPStatus
+
+
+class MatrixDataType(Enum):
+    H5AD = "h5ad"
+    UNKNOWN = "unknown"
+
+
+class MatrixDataLoader(object):
+    def __init__(self, location, matrix_data_type=None, app_config=None):
+        """ location can be a string or DataLocator """
+        region_name = None if app_config is None else app_config.server_config.data_locator__s3__region_name
+        self.location = DataLocator(location, region_name=region_name)
+        if not self.location.exists():
+            raise DatasetAccessError("Dataset does not exist.", HTTPStatus.NOT_FOUND)
+
+        # matrix_data_type is an enum value of type MatrixDataType
+        self.matrix_data_type = matrix_data_type
+        # matrix_type is a DataAdaptor type, which corresonds to the matrix_data_type
+        self.matrix_type = None
+
+        if matrix_data_type is None:
+            self.matrix_data_type = self.__matrix_data_type()
+
+        if not self.__matrix_data_type_allowed(app_config):
+            raise DatasetAccessError("Dataset does not have an allowed type.")
+
+        if self.matrix_data_type == MatrixDataType.H5AD:
+            from server.data_anndata.anndata_adaptor import AnndataAdaptor
+
+            self.matrix_type = AnndataAdaptor
+
+    def __matrix_data_type(self):
+        if self.location.path.endswith(".h5ad"):
+            return MatrixDataType.H5AD
+        else:
+            return MatrixDataType.UNKNOWN
+
+    def __matrix_data_type_allowed(self, app_config):
+        return self.matrix_data_type != MatrixDataType.UNKNOWN
+
+    def pre_load_validation(self):
+        if self.matrix_data_type == MatrixDataType.UNKNOWN:
+            raise DatasetAccessError("Dataset does not have a recognized type: .h5ad")
+        self.matrix_type.pre_load_validation(self.location)
+
+    def file_size(self):
+        return self.matrix_type.file_size(self.location)
+
+    def open(self, app_config, dataset_config=None):
+        # create and return a DataAdaptor object
+        return self.matrix_type.open(self.location, app_config, dataset_config)

server/default_config.py

@@ -0,0 +1,94 @@
+import yaml
+
+default_config = """
+server:
+  app:
+    verbose: false
+    debug: false
+    host: localhost
+    port : null
+    open_browser: false
+    force_https: false
+    flask_secret_key: null
+    generate_cache_control_headers: false
+
+  single_dataset:
+    # If datapath is set, then cellxgene with serve a single dataset located at datapath.
+    datapath: null
+    obs_names: null
+    var_names: null
+    about: null
+    title: null
+
+  data_locator:
+    s3:
+      # s3 region name.
+      #   if true, then the s3 location is automatically determined from the datapath or dataroot.
+      #   if false/null, then do not set.
+      #   if a string, then use that value (e.g. us-east-1).
+      region_name: true
+
+  adaptor:
+    anndata_adaptor:
+      backed: false
+
+  limits:
+    column_request_max: 32
+    diffexp_cellcount_max: null
+
+
+dataset:
+  app:
+    # Scripts can be a list of either file names (string) or dicts containing keys src, integrity and crossorigin.
+    # these will be injected into the index template as script tags with these attributes set.
+    scripts: []
+    # Inline scripts are a list of file names, where the contents of the file will be injected into the index.
+    inline_scripts: []
+
+  presentation:
+    max_categories: 1000
+    custom_colors: true
+
+  user_annotations:
+    enable: true
+    type: local_file_csv
+    local_file_csv:
+      directory: null
+      file: null            # annotations file name
+      gene_sets_file: null  # gene sets file name
+    gene_sets:
+      readonly: false       # gene sets CRUD enabled/disabled
+
+  embeddings:
+    names : []
+
+  diffexp:
+    enable: true
+    lfc_cutoff: 0.01
+    top_n: 10
+
+  X_approximate_distribution: auto
+
+external:
+  # You can retrieve configuration parameters from this config file, the environment,
+  # or from the "cellxgene launch" command line arguments.
+  # They are applied in that order, meaning that if a parameter is defined in more
+  # than one location, the last one applied takes effect.
+
+  # environment variables:
+  # This section describes how to map environment variables to configuration parameters.
+  # The format is a list defining an environment variable.
+  # Each entry in the list is a dictionary with three entries:
+  # name:  the name of the environment variable
+  # path: the path within the cellxgene configuration to update.
+  # required: (default=False) a boolean.  If true, then it is an error if the environment variable is not set.
+
+  environment:
+     - name: CXG_SECRET_KEY
+       path: [server, app, flask_secret_key]
+       required: false
+"""
+
+
+def get_default_config():
+    return yaml.load(default_config, Loader=yaml.Loader)

server/requirements-dev.txt

@@ -0,0 +1,10 @@
+black
+bumpversion>=0.5
+codecov>=2.0.15
+parameterized>=0.7.0
+psycopg2-binary>=2.8.5
+pytest>=3.6.3
+python-jose>=3.2.0
+twine>=1.12.1
+-r requirements.txt
+-r requirements-prepare.txt

server/requirements-prepare.txt

@@ -0,0 +1,4 @@
+python-igraph>=0.8
+louvain>=0.6
+scanpy
+umap-learn<0.5.0 # The pinned version scanpy is not compatible with latest umap-learn

server/requirements.txt

@@ -0,0 +1,22 @@
+anndata>=0.7.6 # we need to_memory(), added in 0.7.6
+boto3>=1.12.18
+click>=7.1.2
+Flask>=1.0.2
+Flask-Compress>=1.4.0
+Flask-Cors>=3.0.9  # CVE-2020-25032
+Flask-RESTful>=0.3.6
+flask-server-timing>=0.1.2
+flask-talisman>=0.7.0
+flatbuffers>=1.11.0,<2.0.0 # cellxgene is not compatible with 2.0.0. Requires migration
+flatten-dict>=0.2.0
+fsspec>=0.4.4,<0.8.0
+gunicorn>=20.0.4
+h5py>=3.0.0
+numba>=0.51.2
+numpy>=1.17.5
+packaging>=20.0
+pandas>=1.0,!=1.1  # pandas 1.1 breaks tests, https://github.com/pandas-dev/pandas/issues/35446
+PyYAML>=5.4  # CVE-2020-14343
+scipy>=1.4
+requests>=2.22.0
+s3fs==0.4.2