Refactor czi_hosted and server into backend directory, pull common code into backend/common, refactor tests (#2102)

* move local_server -> backend/server server-> backend/czi_hosted, pull common code into backend/common update imports, tests and make commands

backend/czi_hosted/common/annotations/annotations.py

@@ -0,0 +1,70 @@
+from abc import ABCMeta, abstractmethod
+
+import fastobo
+import fsspec
+
+from backend.common.errors import OntologyLoadFailure
+from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
+
+
+class Annotations(metaclass=ABCMeta):
+    """ baseclass for annotations, including ontologies"""
+
+    """ our default ontology is the PURL for the Cell Ontology.
+    See http://www.obofoundry.org/ontology/cl.html """
+    DefaultOnotology = "http://purl.obolibrary.org/obo/cl.obo"
+
+    def __init__(self):
+        self.ontology_data = None
+
+    def load_ontology(self, path):
+        """Load and parse ontologies - currently support OBO files only."""
+        if path is None:
+            path = self.DefaultOnotology
+
+        try:
+            with fsspec.open(path) as f:
+                obo = fastobo.iter(f)
+                terms = filter(lambda stanza: type(stanza) is fastobo.term.TermFrame, obo)
+                names = [tag.name for term in terms for tag in term if type(tag) is fastobo.term.NameClause]
+                self.ontology_data = names
+
+        except FileNotFoundError as e:
+            raise OntologyLoadFailure("Unable to find OBO ontology path") from e
+
+        except SyntaxError as e:
+            raise OntologyLoadFailure(f"{path}:{e.lineno}:{e.offset} OBO syntax error, unable to read ontology") from e
+
+        except Exception as e:
+            raise OntologyLoadFailure(f"{path}:Error loading OBO file") from e
+
+    def get_schema(self, data_adaptor):
+        schema = []
+        labels = self.read_labels(data_adaptor)
+        if labels is not None and not labels.empty:
+            for col in labels.columns:
+                col_schema = dict(name=col, writable=True)
+                col_schema.update(get_schema_type_hint_of_array(labels[col]))
+                schema.append(col_schema)
+
+        return schema
+
+    @abstractmethod
+    def set_collection(self, name):
+        """set or create a new annotation collection"""
+        pass
+
+    @abstractmethod
+    def read_labels(self, data_adaptor):
+        """Return the labels as a pandas.DataFrame"""
+        pass
+
+    @abstractmethod
+    def write_labels(self, df, data_adaptor):
+        """Write the labels (df) to a persistent storage such that it can later be read"""
+        pass
+
+    @abstractmethod
+    def update_parameters(self, parameters, data_adaptor):
+        """Update configuration parameters that describe information about the annotations feature"""
+        pass

backend/czi_hosted/common/annotations/hosted_tiledb.py

@@ -0,0 +1,171 @@
+import json
+import os
+import re
+import time
+
+import pandas as pd
+import tiledb
+from flask import current_app
+
+from backend.czi_hosted.common.annotations.annotations import Annotations
+from backend.common.errors import AnnotationCategoryNameError
+from backend.czi_hosted.common.utils.sanitization_utils import sanitize_values_in_list
+from backend.common.utils.type_conversion_utils import get_dtypes_and_schemas_of_dataframe, get_dtype_of_array
+from backend.czi_hosted.db.cellxgene_orm import Annotation
+
+
+class AnnotationsHostedTileDB(Annotations):
+    CXG_ANNO_COLLECTION = "cxg_anno_collection"
+
+    def __init__(self, directory_path, db):
+        super().__init__()
+        self.db = db
+        if directory_path[-1] == "/":
+            self.directory_path = directory_path
+        else:
+            self.directory_path = directory_path + "/"
+
+    def check_category_names(self, df):
+        original_category_names = df.keys().to_list()
+        sanitized_category_names = set(sanitize_values_in_list(original_category_names).values())
+        unsanitary_original_category_names = set(original_category_names).difference(sanitized_category_names)
+        if unsanitary_original_category_names:
+            raise AnnotationCategoryNameError(
+                f"{unsanitary_original_category_names} are not valid category names, please resubmit"
+            )
+
+    def get_user_name(self):
+        return current_app.auth.get_user_name()
+
+    def get_user_id(self):
+        return current_app.auth.get_user_id()
+
+    def is_safe_collection_name(self, name):
+        """
+        return true if this is a safe collection name
+        this is ultra conservative. If we want to allow full legal file name syntax,
+        we could look at modules like `pathvalidate`
+        """
+        if name is None:
+            return False
+        return re.match(r"^[\w\-]+$", name) is not None
+
+    def set_collection(self, name):
+        self.CXG_ANNO_COLLECTION = name
+
+    def read_labels(self, data_adaptor):
+        user_id = self.get_user_id()
+        if user_id is None:
+            return
+        dataset_name = data_adaptor.get_location()
+        dataset_id = self.db.get_or_create_dataset(dataset_name)
+
+        annotation_object = self.db.query_for_most_recent(
+            Annotation, [Annotation.user_id == user_id, Annotation.dataset_id == dataset_id]
+        )
+        if annotation_object:
+            if annotation_object.tiledb_uri == "":
+                # this mean the user has removed all the categories.
+                return None
+            try:
+                df = tiledb.open(annotation_object.tiledb_uri)
+            except tiledb.TileDBError:
+                # don't crash if the annotations file is missing or can't be read.
+                current_app.logger.warning(f"Cannot read annotation file: {annotation_object.tiledb_uri}")
+                return None
+            pandas_df = self.convert_to_pandas_df(df, annotation_object.schema_hints)
+            return pandas_df
+        else:
+            return None
+
+    def convert_to_pandas_df(self, tileDBArray, schema_hints):
+        repr_meta = None
+        index_dims = None
+        schema_hints = json.loads(schema_hints)
+
+        if "__pandas_attribute_repr" in tileDBArray.meta:
+            # backwards compatibility... unsure if necessary at this point
+            repr_meta = json.loads(tileDBArray.meta["__pandas_attribute_repr"])
+        if "__pandas_index_dims" in tileDBArray.meta:
+            index_dims = json.loads(tileDBArray.meta["__pandas_index_dims"])
+
+        data = tileDBArray[:]
+        indexes = list()
+
+        for col_name, col_val in data.items():
+            # If the column values are byte literals, decode them
+            if isinstance(col_val[0], bytes):
+                col_val = [value.decode("utf-8") for value in col_val]
+
+            if schema_hints and col_name in schema_hints:
+                type = schema_hints.get(col_name).get("type")
+                if type and type == "categorical":
+                    new_col = pd.Series(col_val, dtype="category")
+                    data[col_name] = new_col
+            elif repr_meta and col_name in repr_meta:
+                new_col = pd.Series(col_val, dtype=repr_meta[col_name])
+                data[col_name] = new_col
+            elif index_dims and col_name in index_dims:
+                new_col = pd.Series(col_val, dtype=index_dims[col_name])
+                data[col_name] = new_col
+                indexes.append(col_name)
+
+        new_df = pd.DataFrame.from_dict(data)
+        if len(indexes) > 0:
+            new_df.set_index(indexes, inplace=True)
+
+        return new_df
+
+    def write_labels(self, df, data_adaptor):
+        auth_user_id = self.get_user_id()
+        user_name = self.get_user_name()
+        timestamp = time.time()
+        dataset_location = data_adaptor.get_location()
+        dataset_id = self.db.get_or_create_dataset(dataset_location)
+        dataset_name = data_adaptor.get_title()
+        user_id = self.db.get_or_create_user(auth_user_id)
+        """
+        NOTE: The uri contains the dataset name, user name and a timestamp as a convenience for debugging purposes.
+        People may have the same name and time.time() can be server dependent.
+        See - https://docs.python.org/2/library/time.html#time.time
+
+        The annotations objects in the database should be used as the source of truth about who an annotation belongs
+        to (for authorization purposes) and what time it was created (for garbage collection).
+        """
+        uri = f"{self.directory_path}{dataset_name}/{user_name}/{timestamp}"
+        if uri.startswith("s3://"):
+            pass
+        else:
+            os.makedirs(uri, exist_ok=True)
+        _, dataframe_schema_type_hints = get_dtypes_and_schemas_of_dataframe(df)
+        if not df.empty:
+            self.check_category_names(df)
+            # convert to tiledb datatypes
+
+            for col in df:
+                df[col] = df[col].astype(get_dtype_of_array(df[col]))
+            tiledb.from_pandas(uri, df, sparse=True)
+        else:
+            uri = ""
+
+        annotation = Annotation(
+            tiledb_uri=uri,
+            user_id=user_id,
+            dataset_id=str(dataset_id),
+            schema_hints=json.dumps(dataframe_schema_type_hints),
+        )
+        self.db.session.add(annotation)
+        self.db.session.commit()
+
+    def update_parameters(self, parameters, data_adaptor):
+        params = {}
+        params["annotations"] = True
+        params["user_annotation_collection_name_enabled"] = False
+
+        if self.ontology_data:
+            params["annotations_cell_ontology_enabled"] = True
+            params["annotations_cell_ontology_terms"] = self.ontology_data
+        else:
+            params["annotations_cell_ontology_enabled"] = False
+
+        parameters.update(params)

backend/czi_hosted/common/annotations/local_file_csv.py

@@ -0,0 +1,196 @@
+import base64
+import os
+import re
+import threading
+from datetime import datetime
+from hashlib import blake2b
+
+import pandas as pd
+from flask import session, has_request_context, current_app
+
+from backend.czi_hosted import __version__ as cellxgene_version
+from backend.czi_hosted.common.annotations.annotations import Annotations
+from backend.common.errors import AnnotationsError
+
+
+class AnnotationsLocalFile(Annotations):
+    CXG_ANNO_COLLECTION = "cxg_anno_collection"
+
+    def __init__(self, output_dir, output_file):
+        super().__init__()
+        self.output_dir = output_dir
+        self.output_file = output_file
+        # lock used to protect label file write ops
+        self.label_lock = threading.RLock()
+
+        # cache the most recent annotations
+        self.last_fname = None
+        self.last_labels = None
+
+    def is_safe_collection_name(self, name):
+        """
+        return true if this is a safe collection name
+        this is ultra conservative. If we want to allow full legal file name syntax,
+        we could look at modules like `pathvalidate`
+        """
+        if name is None:
+            return False
+        return re.match(r"^[\w\-]+$", name) is not None
+
+    def set_collection(self, name):
+        session[self.CXG_ANNO_COLLECTION] = name
+        session.permanent = True
+
+    def get_collection(self):
+        if session is None:
+            return None
+        return session.get(self.CXG_ANNO_COLLECTION)
+
+    def read_labels(self, data_adaptor):
+        if has_request_context():
+            if not current_app.auth.is_user_authenticated():
+                return pd.DataFrame()
+
+        fname = self._get_filename(data_adaptor)
+        with self.label_lock:
+            if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0:
+                # returned the cached labels if possible, otherwise read them from the file
+                if fname == self.last_fname:
+                    return self.last_labels
+                else:
+                    labels = pd.read_csv(
+                        fname, dtype="category", index_col=0, header=0, comment="#", keep_default_na=False
+                    )
+                    # update the cache
+                    self.last_fname = fname
+                    self.last_labels = labels
+                    return labels
+            else:
+                return pd.DataFrame()
+
+    def write_labels(self, df, data_adaptor):
+        # update our internal state and save it.  Multi-threading often enabled,
+        # so treat this as a critical section.
+        with self.label_lock:
+            lastmod = data_adaptor.get_last_mod_time()
+            lastmodstr = "'unknown'" if lastmod is None else lastmod.isoformat(timespec="seconds")
+            header = (
+                f"# Annotations generated on {datetime.now().isoformat(timespec='seconds')} "
+                f"using cellxgene version {cellxgene_version}\n"
+                f"# Input data file was {data_adaptor.get_location()}, "
+                f"which was last modified on {lastmodstr}\n"
+            )
+
+            fname = self._get_filename(data_adaptor)
+            self._backup(fname)
+            if not df.empty:
+                with open(fname, "w", newline="") as f:
+                    if header is not None:
+                        f.write(header)
+                    df.to_csv(f)
+            else:
+                open(fname, "w").close()
+
+            # update the cache
+            self.last_fname = fname
+            self.last_labels = df
+
+    def _get_userdata_idhash(self, data_adaptor):
+        """
+        Return a short hash that weakly identifies the user and dataset.
+        Used to create safe annotations output file names.
+        """
+        uid = current_app.auth.get_user_id()
+        id = (uid + data_adaptor.get_location()).encode()
+        idhash = base64.b32encode(blake2b(id, digest_size=5).digest()).decode("utf-8")
+        return idhash
+
+    def _get_output_dir(self):
+        if self.output_dir:
+            return self.output_dir
+
+        if self.output_file:
+            return os.path.dirname(self.path.abspath(self.output_dir))
+
+        return os.getcwd()
+
+    def _get_filename(self, data_adaptor):
+        """ return the current annotation file name """
+        if self.output_file:
+            return self.output_file
+
+        # we need to generate a file name, which we can only do if we have a UID and collection name
+        if session is None:
+            raise AnnotationsError("unable to determine file name for annotations")
+
+        collection = self.get_collection()
+        if collection is None:
+            return None
+
+        if data_adaptor is None:
+            raise AnnotationsError("unable to determine file name for annotations")
+
+        idhash = self._get_userdata_idhash(data_adaptor)
+        return os.path.join(self._get_output_dir(), f"{collection}-{idhash}.csv")
+
+    def _backup(self, fname, max_backups=9):
+        """
+        save N backups of file to backup_dir.
+            1. fname -> backup_dir/fname-TIME
+            2. delete excess files in backup_dir
+        """
+        root, ext = os.path.splitext(fname)
+        backup_dir = f"{root}-backups"
+
+        # Make sure there is work to do
+        if not os.path.exists(fname):
+            return
+
+        # Ensure backup_dir exists
+        if not os.path.exists(backup_dir):
+            os.mkdir(backup_dir)
+
+        # Save current file to backup_dir
+        fname_base = os.path.basename(fname)
+        fname_base_root, fname_base_ext = os.path.splitext(fname_base)
+        # don't use ISO standard time format, as it contains characters illegal on some filesytems.
+        nowish = datetime.now().strftime("%Y-%m-%dT%H-%M-%S")
+        backup_fname = os.path.join(backup_dir, f"{fname_base_root}-{nowish}{fname_base_ext}")
+        if os.path.exists(backup_fname):
+            os.remove(backup_fname)
+        os.rename(fname, backup_fname)
+
+        # prune the backup_dir to max number of backup files, keeping the most recent backups
+        backups = list(filter(lambda s: s.startswith(fname_base_root), os.listdir(backup_dir)))
+        excess_count = len(backups) - max_backups
+        if excess_count > 0:
+            backups.sort()
+            for bu in backups[0:excess_count]:
+                os.remove(os.path.join(backup_dir, bu))
+
+    def update_parameters(self, parameters, data_adaptor):
+        params = {}
+        params["annotations"] = True
+        params["user_annotation_collection_name_enabled"] = True
+
+        if self.ontology_data:
+            params["annotations_cell_ontology_enabled"] = True
+            params["annotations_cell_ontology_terms"] = self.ontology_data
+        else:
+            params["annotations_cell_ontology_enabled"] = False
+
+        if self.output_file is not None:
+            # user has hard-wired the name of the annotation data collection
+            fname = os.path.basename(self.output_file)
+            collection_fname = os.path.splitext(fname)[0]
+            params["annotations-data-collection-is-read-only"] = True
+            params["annotations-data-collection-name"] = collection_fname
+
+        elif session is not None:
+            collection = self.get_collection()
+            if current_app.auth.is_user_authenticated():
+                params["annotations-user-data-idhash"] = self._get_userdata_idhash(data_adaptor)
+                params["annotations-data-collection-is-read-only"] = False
+                params["annotations-data-collection-name"] = collection
+
+        parameters.update(params)

backend/czi_hosted/common/config/__init__.py

@@ -0,0 +1,4 @@
+from backend.common.utils.aws_secret_utils import get_secret_key  # noqa F504
+
+DEFAULT_SERVER_PORT = 5005
+BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20  # 100MB

backend/czi_hosted/common/config/app_config.py

@@ -0,0 +1,247 @@
+import yaml
+from flatten_dict import unflatten
+
+from backend.czi_hosted.common.config.external_config import ExternalConfig
+from backend.czi_hosted.common.config.dataset_config import DatasetConfig
+from backend.czi_hosted.common.config.server_config import ServerConfig
+from backend.common.errors import ConfigurationError
+from backend.czi_hosted.default_config import get_default_config
+
+
+class AppConfig(object):
+    """
+    AppConfig stores all the configuration for cellxgene.
+    AppConfig contains one or more DatasetConfig(s) and one ServerConfig.
+    The server_config contains attributes that refer to the server process as a whole.
+    The default_dataset_config refers to attributes that are associated with the features and
+    presentations of a dataset.
+    The dataset config attributes can be overridden depending on the url by which the
+    dataset was accessed.  These are stored in dataroot_config.
+    AppConfig has methods to initialize, modify, and access the configuration.
+    """
+
+    def __init__(self):
+
+        # the default configuration (see default_config.py)
+        # TODO @madison -- if we always read from the default config (hard coded path) can we set those values as
+        #  defaults within the config class?
+        self.default_config = get_default_config()
+        # the server configuration
+        self.server_config = ServerConfig(self, self.default_config["server"])
+        # the dataset config, unless overridden by an entry in dataroot_config
+        self.default_dataset_config = DatasetConfig(None, self, self.default_config["dataset"])
+        # a dictionary of keys to DatasetConfig objects.  Each key must exist in the multi_dataset__dataroot
+        # attribute of the server_config. The default dataset config will apply to all datasets unless a different set
+        # of config vars was passed for a specific dataset under the multidataset config. For example:
+        """
+        per_dataset_config:
+            d1:
+               user_annotations:
+                   enable:  false
+            d2:
+               user_annotations:
+                   enable:  true
+         """
+        #   dataroot config
+        self.dataroot_config = {}
+
+        #  external config
+        self.external_config = ExternalConfig(self, self.default_config["external"])
+
+        # Set to true when config_completed is called
+        self.is_completed = False
+
+    def get_dataset_config(self, dataroot_key):
+        if self.server_config.single_dataset__datapath:
+            return self.default_dataset_config
+        else:
+            return self.dataroot_config.get(dataroot_key, self.default_dataset_config)
+
+    def check_config(self):
+        """Verify all the attributes in the config have been type checked"""
+        if not self.is_completed:
+            raise ConfigurationError("The configuration has not been completed")
+        self.server_config.check_config()
+        self.default_dataset_config.check_config()
+        for dataset_config in self.dataroot_config.values():
+            dataset_config.check_config()
+        self.external_config.check_config()
+
+    def update_server_config(self, **kw):
+        self.server_config.update(**kw)
+        self.is_complete = False
+
+    def update_default_dataset_config(self, **kw):
+        self.default_dataset_config.update(**kw)
+        # update all the other dataset configs, if any
+        for value in self.dataroot_config.values():
+            value.update(**kw)
+        self.is_complete = False
+
+    def update_single_config_from_path_and_value(self, path, value):
+        """Update a single config parameter with the value.
+        Path is a list of string, that gives a path to the config parameter to be updated.
+        For example, path may be ["server","app","port"].
+        """
+        self.is_complete = False
+        if not isinstance(path, list):
+            raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'")
+        for part in path:
+            if not isinstance(part, str):
+                raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'")
+
+        if len(path) < 1 or path[0] not in ("server", "dataset", "per_dataset_config"):
+            raise ConfigurationError("path must start with 'server', 'dataset', or 'per_dataset_config'")
+
+        if path[0] == "server":
+            attr = "__".join(path[1:])
+            try:
+                self.update_server_config(**{attr: value})
+            except ConfigurationError:
+                raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
+        elif path[0] == "dataset":
+            attr = "__".join(path[1:])
+            try:
+                self.update_default_dataset_config(**{attr: value})
+            except ConfigurationError:
+                raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
+
+        elif path[0] == "per_dataset_config":
+            if len(path) < 2:
+                raise ConfigurationError(f"missing dataroot when using per_dataset_config: got '{path}'")
+            dataroot = path[1]
+            if dataroot not in self.dataroot_config:
+                dataroots = str(list(self.dataroot_config.keys()))
+                raise ConfigurationError(
+                    f"unknown dataroot when using per_dataset_config: got '{path}',"
+                    f" dataroots specified in config are {dataroots}"
+                )
+
+            attr = "__".join(path[2:])
+            try:
+                self.dataroot_config[dataroot].update(**{attr: value})
+            except ConfigurationError:
+                raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
+
+    def update_from_config_file(self, config_file):
+        try:
+            with open(config_file) as yml_file:
+                config = yaml.safe_load(yml_file)
+        except yaml.YAMLError as e:
+            raise ConfigurationError(f"The specified config file contained an error: {e}")
+        except OSError as e:
+            raise ConfigurationError(f"Issue retrieving the specified config file: {e}")
+
+        if config.get("server"):
+            self.server_config.update_from_config(config["server"], "server")
+        if config.get("dataset"):
+            self.default_dataset_config.update_from_config(config["dataset"], "dataset")
+
+        per_dataset_config = config.get("per_dataset_config", {})
+        for key, dataroot_config in per_dataset_config.items():
+            # first create and initialize the dataroot with the default config
+            self.add_dataroot_config(key, **config["dataset"])
+            # then apply the per dataset configuration
+            self.dataroot_config[key].update_from_config(dataroot_config, f"per_dataset_config__{key}")
+
+        if config.get("external"):
+            self.external_config.update_from_config(config["external"], "external")
+
+        self.is_complete = False
+
+    def config_to_dict(self):
+        """return the configuration as an unflattened dict"""
+        server = self.server_config.create_mapping(self.server_config.default_config)
+        dataset = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
+        external = self.external_config.create_mapping(self.external_config.default_config)
+        config = dict(server={}, dataset={})
+        for attrname in server.keys():
+            config["server__" + attrname] = getattr(self.server_config, attrname)
+        for attrname in dataset.keys():
+            config["dataset__" + attrname] = getattr(self.default_dataset_config, attrname)
+        if self.dataroot_config:
+            config["per_dataset_config"] = {}
+        for dataroot_tag, dataroot_config in self.dataroot_config.items():
+            dataset = dataroot_config.create_mapping(dataroot_config.default_config)
+            for attrname in dataset.keys():
+                config[f"per_dataset_config__{dataroot_tag}__" + attrname] = getattr(dataroot_config, attrname)
+        for attrname in external.keys():
+            config["external__" + attrname] = getattr(self.external_config, attrname)
+
+        config = unflatten(config, splitter=lambda key: key.split("__"))
+        return config
+
+    def write_config(self, config_file):
+        """output the config to a yaml file"""
+        config = self.config_to_dict()
+        yaml.dump(config, open(config_file, "w"))
+
+    def changes_from_default(self):
+        """Return all the attribute that are different from the default"""
+        diff_server = self.server_config.changes_from_default()
+        diff_dataset = self.default_dataset_config.changes_from_default()
+        diff_external = self.external.changes_from_default()
+        diff = dict(server=diff_server, dataset=diff_dataset, external=diff_external)
+        return diff
+
+    def add_dataroot_config(self, dataroot_tag, **kw):
+        """Create a new dataset config object based on the default dataset config, and kw parameters"""
+        if dataroot_tag in self.dataroot_config:
+            raise ConfigurationError(f"dataroot config already exists: {dataroot_tag}")
+        if type(self.server_config.multi_dataset__dataroot) != dict:
+            raise ConfigurationError("The server__multi_dataset__dataroot must be a dictionary")
+        if dataroot_tag not in self.server_config.multi_dataset__dataroot:
+            raise ConfigurationError(f"The dataroot_tag ({dataroot_tag}) not found in server__multi_dataset__dataroot")
+
+        self.is_completed = False
+        self.dataroot_config[dataroot_tag] = DatasetConfig(dataroot_tag, self, self.default_config["dataset"])
+        flat_config = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
+        config = {key: value[1] for key, value in flat_config.items()}
+        self.dataroot_config[dataroot_tag].update(**config)
+        self.dataroot_config[dataroot_tag].update_from_config(kw, dataroot_tag)
+
+    def complete_config(self, messagefn=None):
+        """The configure options are checked, and any additional setup based on the config
+        parameters is done"""
+
+        if messagefn is None:
+
+            def noop(message):
+                pass
+
+            messagefn = noop
+
+        # TODO: to give better error messages we can add a mapping between where each config
+        # attribute originated (e.g. command line argument or config file), then in the error
+        # messages we can give correct context for attributes with bad value.
+        context = dict(messagefn=messagefn)
+
+        # complete config for external_config first, since this may update values in the other sections
+        self.external_config.complete_config(context)
+        self.server_config.complete_config(context)
+        self.default_dataset_config.complete_config(context)
+        for dataroot_config in self.dataroot_config.values():
+            dataroot_config.complete_config(context)
+
+        self.is_completed = True
+        self.check_config()
+
+    def get_matrix_data_cache_manager(self):
+        return self.server_config.matrix_data_cache_manager
+
+    def is_multi_dataset(self):
+        return self.server_config.multi_dataset__dataroot is not None
+
+    def get_title(self, data_adaptor):
+        return (
+            self.server_config.single_dataset__title
+            if self.server_config.single_dataset__title
+            else data_adaptor.get_title()
+        )
+
+    def get_about(self, data_adaptor):
+        return (
+            self.server_config.single_dataset__about
+            if self.server_config.single_dataset__about
+            else data_adaptor.get_about()
+        )

backend/czi_hosted/common/config/base_config.py

@@ -0,0 +1,132 @@
+import copy
+
+from flatten_dict import flatten
+from backend.common.errors import ConfigurationError
+
+
+class BaseConfig(object):
+    """
+    This class handles the mechanics of updating and checking attributes.
+    Derived classes are expected to store the actual attributes
+    Currently DatasetConfig and ServerConfig both inherit from BaseConfig.
+    """
+
+    def __init__(self, app_config, default_config, dictval_cases={}):
+        # reference back to the app_config
+        self.app_config = app_config
+        # the complete set of attributes and their default values (unflattened)
+        self.default_config = default_config
+        # attributes where the value may be a dict (and therefore are not flattened)
+        self.dictval_cases = dictval_cases
+        # used to make sure every attribute value is checked
+        self.attr_checked = {key_name: False for key_name in self.create_mapping(default_config).keys()}
+
+    def create_mapping(self, config):
+        """
+        Create a dictionary where the keys are the name of attributes (using double underscore convention)
+        For example: authentication__type
+
+        The values are a tuple,
+        - the first item of the tuple is a tuple of path elements (location in config 'tree')
+        - the second item is the value of the config parameter
+
+        For example: (('authentication', 'type'), 'session'))
+        """
+        config_copy = copy.deepcopy(config)
+        mapping = {}
+
+        # special cases where the value could be a dict.
+        # If its value is not None, the entry is added to the mapping, and not included
+        # in the flattening below.
+        for dictval_case in self.dictval_cases:
+            cur = config_copy
+            for part in dictval_case[:-1]:
+                cur = cur.get(part, {})
+            val = cur.get(dictval_case[-1])
+            if val is not None:
+                key = "__".join(dictval_case)
+                mapping[key] = (dictval_case, val)
+                del cur[dictval_case[-1]]
+
+        flat_config = flatten(config_copy)
+        for key, value in flat_config.items():
+            # name of the attribute
+            attr = "__".join(key)
+            mapping[attr] = (key, value)
+
+        return mapping
+
+    def validate_correct_type_of_configuration_attribute(self, attrname, vtype):
+        val = getattr(self, attrname)
+        if type(vtype) in (list, tuple):
+            if type(val) not in vtype:
+                tnames = ",".join([x.__name__ for x in vtype])
+                raise ConfigurationError(
+                    f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}"
+                )
+        else:
+            if type(val) != vtype:
+                raise ConfigurationError(
+                    f"Invalid type for attribute: {attrname}, "
+                    f"expected type {vtype.__name__}, got {type(val).__name__}"
+                )
+
+        self.attr_checked[attrname] = True
+
+    def check_config(self):
+        mapping = self.create_mapping(self.default_config)
+        for key in mapping.keys():
+            if not self.attr_checked[key]:
+                raise ConfigurationError(f"The attr '{key}' has not been checked")
+
+    def update(self, **kw):
+        """Update the attributes defined in kw with their new values."""
+        for key, value in kw.items():
+            if not hasattr(self, key):
+
+                # check if the key is setting into a dictval entry.
+                found_dictval = False
+                for dictval in self.dictval_cases:
+                    dictvalname = "__".join(dictval)
+                    if dictvalname + "__" in key:
+                        dictkey = key[len(dictvalname) + 2 :]
+                        curdictval = getattr(self, dictvalname)
+                        if curdictval is None:
+                            setattr(self, dictvalname, dict(dictkey=value))
+                        else:
+                            curdictval[dictkey] = value
+
+                        found_dictval = True
+                        break
+
+                if found_dictval:
+                    continue
+                raise ConfigurationError(f"unknown config parameter {key}.")
+            try:
+                if type(value) == tuple:
+                    # convert tuple values to list values
+                    value = list(value)
+                setattr(self, key, value)
+            except KeyError:
+                raise ConfigurationError(f"Unable to set config parameter {key}.")
+
+            self.attr_checked[key] = False
+
+    def update_from_config(self, config, prefix):
+        mapping = self.create_mapping(config)
+        for attr, (key, value) in mapping.items():
+            if not hasattr(self, attr):
+                raise ConfigurationError(f"Unknown key from config file: {prefix}__{attr}")
+            setattr(self, attr, value)
+
+            self.attr_checked[attr] = False
+
+    def changes_from_default(self):
+        """Return all the attribute that are different from the default"""
+        mapping = self.create_mapping(self.default_config)
+        diff = []
+        for attrname, (key, defval) in mapping.items():
+            curval = getattr(self, attrname)
+            if curval != defval:
+                diff.append((attrname, curval, defval))
+        return diff

backend/czi_hosted/common/config/client_config.py

@@ -0,0 +1,122 @@
+from backend.czi_hosted import display_version as cellxgene_display_version
+
+
+def get_client_config(app_config, data_adaptor):
+    """
+    Return the configuration as required by the /config REST route
+    """
+
+    server_config = app_config.server_config
+    dataset_config = data_adaptor.dataset_config
+    annotation = dataset_config.user_annotations
+    auth = server_config.auth
+
+    # FIXME The current set of config is not consistently presented:
+    # we have camalCase, hyphen-text, and underscore_text
+
+    # make sure the configuration has been checked.
+    app_config.check_config()
+
+    # display_names
+    title = app_config.get_title(data_adaptor)
+    about = app_config.get_about(data_adaptor)
+
+    display_names = dict(engine=data_adaptor.get_name(), dataset=title)
+
+    # library_versions
+    library_versions = {}
+    library_versions.update(data_adaptor.get_library_versions())
+    library_versions["cellxgene"] = cellxgene_display_version
+
+    # links
+    links = {"about-dataset": about}
+
+    # parameters
+    parameters = {
+        "layout": dataset_config.embeddings__names,
+        "max-category-items": dataset_config.presentation__max_categories,
+        "obs_names": server_config.single_dataset__obs_names,
+        "var_names": server_config.single_dataset__var_names,
+        "diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff,
+        "backed": server_config.adaptor__anndata_adaptor__backed,
+        "disable-diffexp": not dataset_config.diffexp__enable,
+        "enable-reembedding": dataset_config.embeddings__enable_reembedding,
+        "annotations": False,
+        "annotations_file": None,
+        "annotations_dir": None,
+        "annotations_cell_ontology_enabled": False,
+        "annotations_cell_ontology_obopath": None,
+        "annotations_cell_ontology_terms": None,
+        "custom_colors": dataset_config.presentation__custom_colors,
+        "diffexp-may-be-slow": False,
+        "about_legal_tos": dataset_config.app__about_legal_tos,
+        "about_legal_privacy": dataset_config.app__about_legal_privacy,
+    }
+
+    # corpora dataset_props
+    # TODO/Note: putting info from the dataset into the /config is not ideal.
+    # However, it is definitely not part of /schema, and we do not have a top-level
+    # route for data properties.  Consider creating one at some point.
+    corpora_props = data_adaptor.get_corpora_props()
+    if corpora_props and "default_embedding" in corpora_props:
+        default_embedding = corpora_props["default_embedding"]
+        if isinstance(default_embedding, str) and default_embedding.startswith("X_"):
+            default_embedding = default_embedding[2:]  # drop X_ prefix
+        if default_embedding in data_adaptor.get_embedding_names():
+            parameters["default_embedding"] = default_embedding
+
+    data_adaptor.update_parameters(parameters)
+    if annotation:
+        annotation.update_parameters(parameters, data_adaptor)
+
+    # gather it all together
+    client_config = {}
+    config = client_config["config"] = {}
+    config["displayNames"] = display_names
+    config["library_versions"] = library_versions
+    config["links"] = links
+    config["parameters"] = parameters
+    config["corpora_props"] = corpora_props
+    config["limits"] = {
+        "column_request_max": server_config.limits__column_request_max,
+        "diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max,
+    }
+
+    if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
+        config["authentication"] = {
+            "requires_client_login": auth.requires_client_login(),
+        }
+        if auth.requires_client_login():
+            config["authentication"].update(
+                {
+                    # Todo why are these stored on the data_adaptor?
+                    "login": auth.get_login_url(data_adaptor),
+                    "logout": auth.get_logout_url(data_adaptor),
+                }
+            )
+
+    return client_config
+
+
+def get_client_userinfo(app_config, data_adaptor):
+    """
+    Return the userinfo as required by the /userinfo REST route
+    """
+
+    server_config = app_config.server_config
+    dataset_config = data_adaptor.dataset_config
+    auth = server_config.auth
+
+    # make sure the configuration has been checked.
+    app_config.check_config()
+
+    if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
+        userinfo = {}
+        userinfo["userinfo"] = {
+            "is_authenticated": auth.is_user_authenticated(),
+            "username": auth.get_user_name(),
+            "user_id": auth.get_user_id(),
+            "email": auth.get_user_email(),
+            "picture": auth.get_user_picture(),
+        }
+        return userinfo

backend/czi_hosted/common/config/dataset_config.py

@@ -0,0 +1,230 @@
+import os
+from os.path import splitext, isdir
+
+from backend.czi_hosted.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
+from backend.czi_hosted.common.annotations.local_file_csv import AnnotationsLocalFile
+from backend.czi_hosted.common.config.base_config import BaseConfig
+from backend.common.errors import ConfigurationError, OntologyLoadFailure
+from backend.czi_hosted.compute.scanpy import get_scanpy_module
+from backend.czi_hosted.data_common.matrix_loader import MatrixDataLoader, MatrixDataType
+from backend.czi_hosted.db.db_utils import DbUtils
+
+
+class DatasetConfig(BaseConfig):
+    """Manages the config attribute associated with a dataset."""
+
+    def __init__(self, tag, app_config, default_config):
+        super().__init__(app_config, default_config)
+        self.tag = tag
+        try:
+            self.app__scripts = default_config["app"]["scripts"]
+            self.app__inline_scripts = default_config["app"]["inline_scripts"]
+            self.app__about_legal_tos = default_config["app"]["about_legal_tos"]
+            self.app__about_legal_privacy = default_config["app"]["about_legal_privacy"]
+            self.app__authentication_enable = default_config["app"]["authentication_enable"]
+
+            self.presentation__max_categories = default_config["presentation"]["max_categories"]
+            self.presentation__custom_colors = default_config["presentation"]["custom_colors"]
+
+            self.user_annotations__enable = default_config["user_annotations"]["enable"]
+            self.user_annotations__type = default_config["user_annotations"]["type"]
+            self.user_annotations__local_file_csv__directory = default_config["user_annotations"]["local_file_csv"][
+                "directory"
+            ]
+            self.user_annotations__local_file_csv__file = default_config["user_annotations"]["local_file_csv"]["file"]
+            self.user_annotations__ontology__enable = default_config["user_annotations"]["ontology"]["enable"]
+            self.user_annotations__ontology__obo_location = default_config["user_annotations"]["ontology"][
+                "obo_location"
+            ]
+            self.user_annotations__hosted_tiledb_array__db_uri = default_config["user_annotations"][
+                "hosted_tiledb_array"
+            ]["db_uri"]
+            self.user_annotations__hosted_tiledb_array__hosted_file_directory = default_config["user_annotations"][
+                "hosted_tiledb_array"
+            ]["hosted_file_directory"]
+
+            self.embeddings__names = default_config["embeddings"]["names"]
+            self.embeddings__enable_reembedding = default_config["embeddings"]["enable_reembedding"]
+
+            self.diffexp__enable = default_config["diffexp"]["enable"]
+            self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
+            self.diffexp__top_n = default_config["diffexp"]["top_n"]
+
+        except KeyError as e:
+            raise ConfigurationError(f"Unexpected config: {str(e)}")
+
+        # The annotation object is created during complete_config and stored here.
+        self.user_annotations = None
+
+    def complete_config(self, context):
+        self.handle_app()
+        self.handle_presentation()
+        self.handle_user_annotations(context)
+        self.handle_embeddings()
+        self.handle_diffexp(context)
+
+    def handle_app(self):
+        self.validate_correct_type_of_configuration_attribute("app__scripts", list)
+        self.validate_correct_type_of_configuration_attribute("app__inline_scripts", list)
+        self.validate_correct_type_of_configuration_attribute("app__about_legal_tos", (type(None), str))
+        self.validate_correct_type_of_configuration_attribute("app__about_legal_privacy", (type(None), str))
+        self.validate_correct_type_of_configuration_attribute("app__authentication_enable", bool)
+
+        # scripts can be string (filename) or dict (attributes). Convert string to dict.
+        scripts = []
+        for script in self.app__scripts:
+            try:
+                if isinstance(script, str):
+                    scripts.append({"src": script})
+                elif isinstance(script, dict) and isinstance(script["src"], str):
+                    scripts.append(script)
+                else:
+                    raise Exception
+            except Exception as e:
+                raise ConfigurationError(f"Scripts must be string or a dict containing an src key: {e}")
+
+        self.app__scripts = scripts
+
+    def handle_presentation(self):
+        self.validate_correct_type_of_configuration_attribute("presentation__max_categories", int)
+        self.validate_correct_type_of_configuration_attribute("presentation__custom_colors", bool)
+
+    def handle_user_annotations(self, context):
+        self.validate_correct_type_of_configuration_attribute("user_annotations__enable", bool)
+        self.validate_correct_type_of_configuration_attribute("user_annotations__type", str)
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__local_file_csv__directory", (type(None), str)
+        )
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__local_file_csv__file", (type(None), str)
+        )
+        self.validate_correct_type_of_configuration_attribute("user_annotations__ontology__enable", bool)
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__ontology__obo_location", (type(None), str)
+        )
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__hosted_tiledb_array__db_uri", (type(None), str)
+        )
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__hosted_tiledb_array__hosted_file_directory", (type(None), str)
+        )
+        if self.user_annotations__enable:
+            server_config = self.app_config.server_config
+            if not self.app__authentication_enable:
+                raise ConfigurationError("user annotations requires authentication to be enabled")
+            if not server_config.auth.is_valid_authentication_type():
+                auth_type = server_config.authentication__type
+                raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations")
+
+            if self.user_annotations__type == "local_file_csv":
+                self.handle_local_file_csv_annotations()
+            elif self.user_annotations__type == "hosted_tiledb_array":
+                self.handle_hosted_tiledb_annotations()
+            else:
+                raise ConfigurationError('The only annotation type support is "local_file_csv" or "hosted_tiledb_array')
+            if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location:
+                try:
+                    self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location)
+                except OntologyLoadFailure as e:
+                    raise ConfigurationError("Unable to load ontology terms\n" + str(e))
+        else:
+            self.check_annotation_config_vars_not_set(context)
+
+    def handle_local_file_csv_annotations(self):
+        dirname = self.user_annotations__local_file_csv__directory
+        filename = self.user_annotations__local_file_csv__file
+        if filename is not None and dirname is not None:
+            raise ConfigurationError("'annotations-file' and 'annotations-dir' may not be used together.")
+
+        if filename is not None:
+            lf_name, lf_ext = splitext(filename)
+            if lf_ext and lf_ext != ".csv":
+                raise ConfigurationError(f"annotation file type must be .csv: {filename}")
+
+        if dirname is not None and not isdir(dirname):
+            try:
+                os.mkdir(dirname)
+            except OSError:
+                raise ConfigurationError("Unable to create directory specified by --annotations-dir")
+
+        self.user_annotations = AnnotationsLocalFile(dirname, filename)
+
+        # if the user has specified a fixed label file, go ahead and validate it
+        # so that we can remove errors early in the process.
+        server_config = self.app_config.server_config
+        if server_config.single_dataset__datapath and self.user_annotations__local_file_csv__file:
+            with server_config.matrix_data_cache_manager.data_adaptor(
+                self.tag, server_config.single_dataset__datapath, self.app_config
+            ) as data_adaptor:
+                data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor))
+
+    def handle_hosted_tiledb_annotations(self):
+        self.validate_correct_type_of_configuration_attribute("user_annotations__hosted_tiledb_array__db_uri", str)
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__hosted_tiledb_array__hosted_file_directory", str
+        )
+        self.user_annotations = AnnotationsHostedTileDB(
+            directory_path=self.user_annotations__hosted_tiledb_array__hosted_file_directory,
+            db=DbUtils(self.user_annotations__hosted_tiledb_array__db_uri),
+        )
+
+    def check_annotation_config_vars_not_set(self, context):
+        if self.user_annotations__type is not None:
+            dirname = self.user_annotations__local_file_csv__directory
+            filename = self.user_annotations__local_file_csv__file
+            db_uri = self.user_annotations__hosted_tiledb_array__db_uri
+            hosted_file_dirname = self.user_annotations__hosted_tiledb_array__hosted_file_directory
+            if filename is not None:
+                context["messagefn"]("Warning: --annotations-file ignored as annotations are disabled.")
+            if dirname is not None:
+                context["messagefn"]("Warning: --annotations-dir ignored as annotations are disabled.")
+            if db_uri is not None:
+                context["messagefn"]("Warning: db_uri ignored as annotations are disabled.")
+            if hosted_file_dirname is not None:
+                context["messagefn"](
+                    "Warning: hosted_file_directory for hosted_tiledb_array ignored as annotations are disabled."
+                )
+
+        if self.user_annotations__ontology__enable:
+            context["messagefn"]("Warning: --experimental-annotations-ontology ignored as annotations are disabled.")
+        if self.user_annotations__ontology__obo_location is not None:
+            context["messagefn"](
+                "Warning: --experimental-annotations-ontology-obo ignored as annotations are disabled."
+            )
+
+    def handle_embeddings(self):
+        self.validate_correct_type_of_configuration_attribute("embeddings__names", list)
+        self.validate_correct_type_of_configuration_attribute("embeddings__enable_reembedding", bool)
+
+        server_config = self.app_config.server_config
+        if self.embeddings__enable_reembedding:
+            if server_config.single_dataset__datapath:
+                matrix_data_loader = MatrixDataLoader(
+                    server_config.single_dataset__datapath, app_config=self.app_config
+                )
+                if matrix_data_loader.matrix_data_type != MatrixDataType.H5AD:
+                    raise ConfigurationError("enable-reembedding is only supported with H5AD files.")
+                if server_config.adaptor__anndata_adaptor__backed:
+                    raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.")
+
+            try:
+                get_scanpy_module()
+            except NotImplementedError:
+                # Todo add scanpy to requirements.txt and remove this check once re-embeddings is fully supported
+                raise ConfigurationError("Please install scanpy to enable UMAP re-embedding")
+
+    def handle_diffexp(self, context):
+        self.validate_correct_type_of_configuration_attribute("diffexp__enable", bool)
+        self.validate_correct_type_of_configuration_attribute("diffexp__lfc_cutoff", float)
+        self.validate_correct_type_of_configuration_attribute("diffexp__top_n", int)
+
+        server_config = self.app_config.server_config
+        if server_config.single_dataset__datapath:
+            with server_config.matrix_data_cache_manager.data_adaptor(
+                self.tag, server_config.single_dataset__datapath, self.app_config
+            ) as data_adaptor:
+                if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False):
+                    context["messagefn"](
+                        "CAUTION: due to the size of your dataset, "
+                        "running differential expression may take longer or fail."
+                    )

backend/czi_hosted/common/config/external_config.py

@@ -0,0 +1,95 @@
+import os
+
+from backend.czi_hosted.common.config.base_config import BaseConfig
+from backend.common.errors import ConfigurationError, SecretKeyRetrievalError
+from backend.common.utils.aws_secret_utils import get_secret_key
+from backend.common.utils.type_conversion_utils import convert_string_to_value
+
+
+class ExternalConfig(BaseConfig):
+    """Manages the config attribute associated with external configuration sources, such as
+    environment variables or the AWS Secrets Manager."""
+
+    def __init__(self, app_config, default_config):
+        super().__init__(app_config, default_config)
+        try:
+            self.environment = default_config["environment"]
+            self.aws_secrets_manager__region = default_config["aws_secrets_manager"]["region"]
+            self.aws_secrets_manager__secrets = default_config["aws_secrets_manager"]["secrets"]
+
+        except KeyError as e:
+            raise ConfigurationError(f"Unexpected config: {str(e)}")
+
+    def complete_config(self, context):
+        self.handle_environment(context)
+        self.handle_aws_secrets_manager(context)
+
+    def handle_environment(self, context):
+        """For each environment variable defined, get the value (if it is set),
+        and set the specified config parameter"""
+        self.validate_correct_type_of_configuration_attribute("environment", list)
+        for envdict in self.environment:
+            name = envdict.get("name")
+            if name is None:
+                raise ConfigurationError("environment: 'name' is missing")
+            required = envdict.get("required", False)
+            if type(required) != bool:
+                raise ConfigurationError("environment: 'required' must be a bool")
+            path = envdict.get("path")
+            if path is None:
+                raise ConfigurationError("environment: 'path' is missing")
+
+            value = os.environ.get(name)
+            if value is None:
+                if required:
+                    raise ConfigurationError(f"required environment variable '{name}' not set")
+            else:
+                value = convert_string_to_value(value)
+                self.app_config.update_single_config_from_path_and_value(path, value)
+
+    def handle_aws_secrets_manager(self, context):
+        """For each aws secret defined, get the key/values, and set the specified config parameter"""
+        self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", (type(None), str))
+        self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__secrets", list)
+
+        if not self.aws_secrets_manager__secrets:
+            return
+
+        self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", str)
+
+        for secret in self.aws_secrets_manager__secrets:
+            secret_name = secret.get("name")
+            if secret_name is None:
+                raise ConfigurationError("aws_secrets_manager: 'name' is missing")
+            if not isinstance(secret_name, str):
+                raise ConfigurationError("aws_secrets_manager: 'name' must be a string")
+
+            try:
+                secret_dict = get_secret_key(self.aws_secrets_manager__region, secret_name)
+            except SecretKeyRetrievalError as e:
+                raise ConfigurationError(f"Unable to retrieve secret {secret_name}: {str(e)}")
+
+            values = secret.get("values")
+            if values is None:
+                raise ConfigurationError("aws_secrets_manager: 'values' is missing")
+            if not isinstance(values, list):
+                raise ConfigurationError("aws_secrets_manager: 'values' must be a list")
+
+            for value in values:
+                key = value.get("key")
+                if key is None:
+                    raise ConfigurationError(f"missing 'key' in secret values: {secret_name}")
+                path = value.get("path")
+                if path is None:
+                    raise ConfigurationError(f"missing 'path' in secret values: {secret_name}")
+                required = value.get("required", False)
+                if type(required) != bool:
+                    raise ConfigurationError(f"wrong type for 'required' in secret values: {secret_name}")
+
+                secret_value = secret_dict.get(key)
+                if secret_value is None:
+                    if required:
+                        raise ConfigurationError(f"required secret '{secret_name}:{key}' not set")
+                else:
+                    secret_value = convert_string_to_value(secret_value)
+                    self.app_config.update_single_config_from_path_and_value(path, secret_value)

backend/czi_hosted/common/config/server_config.py

@@ -0,0 +1,387 @@
+import os
+import sys
+import warnings
+from os.path import basename
+from urllib.parse import urlparse, quote_plus
+
+from backend.czi_hosted.auth.auth import AuthTypeFactory
+from backend.czi_hosted.common.config import DEFAULT_SERVER_PORT, BIG_FILE_SIZE_THRESHOLD
+from backend.czi_hosted.common.config.base_config import BaseConfig
+from backend.common.utils.data_locator import discover_s3_region_name
+from backend.common.errors import ConfigurationError, DatasetAccessError
+from backend.common.utils.utils import is_port_available, find_available_port, custom_format_warning
+from backend.czi_hosted.compute import diffexp_cxg as diffexp_tiledb
+from backend.czi_hosted.data_common.matrix_loader import MatrixDataCacheManager, MatrixDataLoader, MatrixDataType
+
+
+class ServerConfig(BaseConfig):
+    """Manages the config attribute associated with the server."""
+
+    def __init__(self, app_config, default_config):
+        dictval_cases = [
+            ("app", "csp_directives"),
+            ("authentication", "params_oauth", "cookie"),
+            ("authentication", "params_oauth", "jwt_decode_options"),
+            ("adaptor", "cxg_adaptor", "tiledb_ctx"),
+            ("multi_dataset", "dataroot"),
+        ]
+        super().__init__(app_config, default_config, dictval_cases)
+
+        try:
+            self.app__verbose = default_config["app"]["verbose"]
+            self.app__debug = default_config["app"]["debug"]
+            self.app__host = default_config["app"]["host"]
+            self.app__port = default_config["app"]["port"]
+            self.app__open_browser = default_config["app"]["open_browser"]
+            self.app__force_https = default_config["app"]["force_https"]
+            self.app__flask_secret_key = default_config["app"]["flask_secret_key"]
+            self.app__generate_cache_control_headers = default_config["app"]["generate_cache_control_headers"]
+            self.app__server_timing_headers = default_config["app"]["server_timing_headers"]
+            self.app__csp_directives = default_config["app"]["csp_directives"]
+            self.app__api_base_url = default_config["app"]["api_base_url"]
+            self.app__web_base_url = default_config["app"]["web_base_url"]
+
+            self.authentication__type = default_config["authentication"]["type"]
+            self.authentication__insecure_test_environment = default_config["authentication"][
+                "insecure_test_environment"
+            ]
+            self.authentication__params_oauth__oauth_api_base_url = default_config["authentication"]["params_oauth"][
+                "oauth_api_base_url"
+            ]
+            self.authentication__params_oauth__client_id = default_config["authentication"]["params_oauth"]["client_id"]
+            self.authentication__params_oauth__client_secret = default_config["authentication"]["params_oauth"][
+                "client_secret"
+            ]
+            self.authentication__params_oauth__jwt_decode_options = default_config["authentication"]["params_oauth"][
+                "jwt_decode_options"
+            ]
+            self.authentication__params_oauth__session_cookie = default_config["authentication"]["params_oauth"][
+                "session_cookie"
+            ]
+            self.authentication__params_oauth__cookie = default_config["authentication"]["params_oauth"]["cookie"]
+
+            self.multi_dataset__dataroot = default_config["multi_dataset"]["dataroot"]
+            self.multi_dataset__index = default_config["multi_dataset"]["index"]
+            self.multi_dataset__allowed_matrix_types = default_config["multi_dataset"]["allowed_matrix_types"]
+            self.multi_dataset__matrix_cache__max_datasets = default_config["multi_dataset"]["matrix_cache"][
+                "max_datasets"
+            ]
+            self.multi_dataset__matrix_cache__timelimit_s = default_config["multi_dataset"]["matrix_cache"][
+                "timelimit_s"
+            ]
+
+            self.single_dataset__datapath = default_config["single_dataset"]["datapath"]
+            self.single_dataset__obs_names = default_config["single_dataset"]["obs_names"]
+            self.single_dataset__var_names = default_config["single_dataset"]["var_names"]
+            self.single_dataset__about = default_config["single_dataset"]["about"]
+            self.single_dataset__title = default_config["single_dataset"]["title"]
+
+            self.diffexp__alg_cxg__max_workers = default_config["diffexp"]["alg_cxg"]["max_workers"]
+            self.diffexp__alg_cxg__cpu_multiplier = default_config["diffexp"]["alg_cxg"]["cpu_multiplier"]
+            self.diffexp__alg_cxg__target_workunit = default_config["diffexp"]["alg_cxg"]["target_workunit"]
+
+            self.data_locator__s3__region_name = default_config["data_locator"]["s3"]["region_name"]
+
+            self.adaptor__cxg_adaptor__tiledb_ctx = default_config["adaptor"]["cxg_adaptor"]["tiledb_ctx"]
+            self.adaptor__anndata_adaptor__backed = default_config["adaptor"]["anndata_adaptor"]["backed"]
+
+            self.limits__diffexp_cellcount_max = default_config["limits"]["diffexp_cellcount_max"]
+            self.limits__column_request_max = default_config["limits"]["column_request_max"]
+
+        except KeyError as e:
+            raise ConfigurationError(f"Unexpected config: {str(e)}")
+
+        # The matrix data cache manager is created during the complete_config and stored here.
+        self.matrix_data_cache_manager = None
+
+        # The authentication object
+        self.auth = None
+
+    def complete_config(self, context):
+        self.handle_app(context)
+        self.handle_data_source()
+        self.handle_authentication()
+        self.handle_data_locator()
+        self.handle_adaptor()  # may depend on data_locator
+        self.handle_single_dataset(context)  # may depend on adaptor
+        self.handle_multi_dataset()  # may depend on adaptor
+        self.handle_diffexp()
+        self.handle_limits()
+
+        self.check_config()
+
+    def handle_app(self, context):
+        self.validate_correct_type_of_configuration_attribute("app__verbose", bool)
+        self.validate_correct_type_of_configuration_attribute("app__debug", bool)
+        self.validate_correct_type_of_configuration_attribute("app__host", str)
+        self.validate_correct_type_of_configuration_attribute("app__port", (type(None), int))
+        self.validate_correct_type_of_configuration_attribute("app__open_browser", bool)
+        self.validate_correct_type_of_configuration_attribute("app__force_https", bool)
+        self.validate_correct_type_of_configuration_attribute("app__flask_secret_key", str)
+        self.validate_correct_type_of_configuration_attribute("app__generate_cache_control_headers", bool)
+        self.validate_correct_type_of_configuration_attribute("app__server_timing_headers", bool)
+        self.validate_correct_type_of_configuration_attribute("app__csp_directives", (type(None), dict))
+        self.validate_correct_type_of_configuration_attribute("app__api_base_url", (type(None), str))
+        self.validate_correct_type_of_configuration_attribute("app__web_base_url", (type(None), str))
+
+        if self.app__port:
+            try:
+                if not is_port_available(self.app__host, self.app__port):
+                    raise ConfigurationError(
+                        f"The port selected {self.app__port} is in use, please configure an open port."
+                    )
+            except OverflowError:
+                raise ConfigurationError(f"Invalid port: {self.app__port}")
+        else:
+            try:
+                default_server_port = int(os.environ.get("CXG_SERVER_PORT", DEFAULT_SERVER_PORT))
+            except ValueError:
+                raise ConfigurationError(
+                    "Invalid port from environment variable CXG_SERVER_PORT: " + os.environ.get("CXG_SERVER_PORT")
+                )
+            try:
+                self.app__port = find_available_port(self.app__host, default_server_port)
+            except OverflowError:
+                raise ConfigurationError(f"Invalid port: {default_server_port}")
+
+        if self.app__debug:
+            context["messagefn"]("in debug mode, setting verbose=True and open_browser=False")
+            self.app__verbose = True
+            self.app__open_browser = False
+        else:
+            warnings.formatwarning = custom_format_warning
+
+        if not self.app__verbose:
+            sys.tracebacklimit = 0
+
+        # CSP Directives are a dict of string: list(string) or string: string
+        if self.app__csp_directives is not None:
+            for k, v in self.app__csp_directives.items():
+                if not isinstance(k, str):
+                    raise ConfigurationError("CSP directive names must be a string.")
+                if isinstance(v, list):
+                    for policy in v:
+                        if not isinstance(policy, str):
+                            raise ConfigurationError("CSP directive value must be a string or list of strings.")
+                elif not isinstance(v, str):
+                    raise ConfigurationError("CSP directive value must be a string or list of strings.")
+
+        if self.app__web_base_url is None:
+            self.app__web_base_url = self.app__api_base_url
+
+    def handle_authentication(self):
+        self.validate_correct_type_of_configuration_attribute("authentication__type", (type(None), str))
+        self.validate_correct_type_of_configuration_attribute("authentication__insecure_test_environment", bool)
+
+        if self.authentication__type == "test" and not self.authentication__insecure_test_environment:
+            raise ConfigurationError("Test auth can only be used in an insecure test environment")
+
+        # oauth
+        ptypes = str if self.authentication__type == "oauth" else (type(None), str)
+        self.validate_correct_type_of_configuration_attribute(
+            "authentication__params_oauth__oauth_api_base_url", ptypes
+        )
+        self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_id", ptypes)
+        self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_secret", ptypes)
+        self.validate_correct_type_of_configuration_attribute(
+            "authentication__params_oauth__jwt_decode_options", (type(None), dict)
+        )
+        self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__session_cookie", bool)
+
+        if self.authentication__params_oauth__session_cookie:
+            self.validate_correct_type_of_configuration_attribute(
+                "authentication__params_oauth__cookie", (type(None), dict)
+            )
+        else:
+            self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__cookie", dict)
+
+        self.auth = AuthTypeFactory.create(self.authentication__type, self)
+        if self.auth is None:
+            raise ConfigurationError(f"Unknown authentication type: {self.authentication__type}")
+
+    def handle_data_locator(self):
+        self.validate_correct_type_of_configuration_attribute("data_locator__s3__region_name", (type(None), bool, str))
+        if self.data_locator__s3__region_name is True:
+            path = self.single_dataset__datapath or self.multi_dataset__dataroot
+
+            if type(path) == dict:
+                # if multi_dataset__dataroot is a dict, then use the first key
+                # that is in s3.   NOTE:  it is not supported to have dataroots
+                # in different regions.
+                paths = [val.get("dataroot") for val in path.values()]
+                for path in paths:
+                    if path.startswith("s3://"):
+                        break
+            if path.startswith("s3://"):
+                region_name = discover_s3_region_name(path)
+                if region_name is None:
+                    raise ConfigurationError(f"Unable to discover s3 region name from {path}")
+            else:
+                region_name = None
+            self.data_locator__s3__region_name = region_name
+
+    def handle_data_source(self):
+        self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None)))
+        self.validate_correct_type_of_configuration_attribute("multi_dataset__dataroot", (type(None), dict, str))
+
+        if self.single_dataset__datapath and self.multi_dataset__dataroot:
+            raise ConfigurationError(
+                "You must supply either a datapath (for single datasets) or a dataroot (for multidatasets). Not both"
+            )
+        if self.single_dataset__datapath is None and self.multi_dataset__dataroot is None:
+            raise ConfigurationError("You must specify a datapath for a single dataset or a dataroot for multidatasets")
+
+    def handle_single_dataset(self, context):
+        self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None)))
+        self.validate_correct_type_of_configuration_attribute("single_dataset__title", (str, type(None)))
+        self.validate_correct_type_of_configuration_attribute("single_dataset__about", (str, type(None)))
+        self.validate_correct_type_of_configuration_attribute("single_dataset__obs_names", (str, type(None)))
+        self.validate_correct_type_of_configuration_attribute("single_dataset__var_names", (str, type(None)))
+
+        if self.single_dataset__datapath is None:
+            return
+
+        # create the matrix data cache manager:
+        if self.matrix_data_cache_manager is None:
+            self.matrix_data_cache_manager = MatrixDataCacheManager(max_cached=1, timelimit_s=None)
+
+        # preload this data set
+        matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath, app_config=self.app_config)
+        try:
+            matrix_data_loader.pre_load_validation()
+        except DatasetAccessError as e:
+            raise ConfigurationError(str(e))
+
+        file_size = matrix_data_loader.file_size()
+        file_basename = basename(self.single_dataset__datapath)
+        if file_size > BIG_FILE_SIZE_THRESHOLD:
+            context["messagefn"](f"Loading data from {file_basename}, this may take a while...")
+        else:
+            context["messagefn"](f"Loading data from {file_basename}.")
+
+        if self.single_dataset__about:
+
+            def url_check(url):
+                try:
+                    result = urlparse(url)
+                    if all([result.scheme, result.netloc]):
+                        return True
+                    else:
+                        return False
+                except ValueError:
+                    return False
+
+            if not url_check(self.single_dataset__about):
+                raise ConfigurationError(
+                    "Must provide an absolute URL for --about. (Example format: http://example.com)"
+                )
+
+    def handle_multi_dataset(self):
+        self.validate_correct_type_of_configuration_attribute("multi_dataset__dataroot", (type(None), dict, str))
+        self.validate_correct_type_of_configuration_attribute("multi_dataset__index", (type(None), bool, str))
+        self.validate_correct_type_of_configuration_attribute("multi_dataset__allowed_matrix_types", list)
+        self.validate_correct_type_of_configuration_attribute("multi_dataset__matrix_cache__max_datasets", int)
+        self.validate_correct_type_of_configuration_attribute(
+            "multi_dataset__matrix_cache__timelimit_s", (type(None), int, float)
+        )
+
+        if self.multi_dataset__dataroot is None:
+            return
+
+        if type(self.multi_dataset__dataroot) == str:
+            default_dict = dict(base_url="d", dataroot=self.multi_dataset__dataroot)
+            self.multi_dataset__dataroot = dict(d=default_dict)
+
+        for tag, dataroot_dict in self.multi_dataset__dataroot.items():
+            if "base_url" not in dataroot_dict:
+                raise ConfigurationError(f"error in multi_dataset__dataroot: missing base_url for tag {tag}")
+            if "dataroot" not in dataroot_dict:
+                raise ConfigurationError(f"error in multi_dataset__dataroot: missing dataroot, for tag {tag}")
+
+            base_url = dataroot_dict["base_url"]
+
+            # sanity check for well formed base urls
+            bad = False
+            if type(base_url) != str:
+                bad = True
+            elif os.path.normpath(base_url) != base_url:
+                bad = True
+            else:
+                base_url_parts = base_url.split("/")
+                if [quote_plus(part) for part in base_url_parts] != base_url_parts:
+                    bad = True
+                if ".." in base_url_parts:
+                    bad = True
+            if bad:
+                raise ConfigurationError(f"error in multi_dataset__dataroot base_url {base_url} for tag {tag}")
+
+        # verify all the base_urls are unique
+        base_urls = [d["base_url"] for d in self.multi_dataset__dataroot.values()]
+        if len(base_urls) > len(set(base_urls)):
+            raise ConfigurationError("error in multi_dataset__dataroot:  base_urls must be unique")
+
+        # error checking
+        for mtype in self.multi_dataset__allowed_matrix_types:
+            try:
+                MatrixDataType(mtype)
+            except ValueError:
+                raise ConfigurationError(f'Invalid matrix type in "allowed_matrix_types": {mtype}')
+
+        # create the matrix data cache manager:
+        if self.matrix_data_cache_manager is None:
+            self.matrix_data_cache_manager = MatrixDataCacheManager(
+                max_cached=self.multi_dataset__matrix_cache__max_datasets,
+                timelimit_s=self.multi_dataset__matrix_cache__timelimit_s,
+            )
+
+    def handle_diffexp(self):
+        self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__max_workers", (str, int))
+        self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__cpu_multiplier", int)
+        self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__target_workunit", int)
+
+        max_workers = self.diffexp__alg_cxg__max_workers
+        cpu_multiplier = self.diffexp__alg_cxg__cpu_multiplier
+        cpu_count = os.cpu_count()
+        max_workers = min(max_workers, cpu_multiplier * cpu_count)
+        diffexp_tiledb.set_config(max_workers, self.diffexp__alg_cxg__target_workunit)
+
+    def handle_adaptor(self):
+        # cxg
+        self.validate_correct_type_of_configuration_attribute("adaptor__cxg_adaptor__tiledb_ctx", dict)
+        regionkey = "vfs.s3.region"
+        if regionkey not in self.adaptor__cxg_adaptor__tiledb_ctx:
+            if type(self.data_locator__s3__region_name) == str:
+                self.adaptor__cxg_adaptor__tiledb_ctx[regionkey] = self.data_locator__s3__region_name
+
+        from backend.czi_hosted.data_cxg.cxg_adaptor import CxgAdaptor
+
+        CxgAdaptor.set_tiledb_context(self.adaptor__cxg_adaptor__tiledb_ctx)
+
+        # anndata
+        self.validate_correct_type_of_configuration_attribute("adaptor__anndata_adaptor__backed", bool)
+
+    def handle_limits(self):
+        self.validate_correct_type_of_configuration_attribute("limits__diffexp_cellcount_max", (type(None), int))
+        self.validate_correct_type_of_configuration_attribute("limits__column_request_max", (type(None), int))
+
+    def exceeds_limit(self, limit_name, value):
+        limit_value = getattr(self, "limits__" + limit_name, None)
+        if limit_value is None:  # disabled
+            return False
+        return value > limit_value
+
+    def get_api_base_url(self):
+        if self.app__api_base_url == "local":
+            return f"http://{self.app__host}:{self.app__port}"
+        if self.app__api_base_url and self.app__api_base_url.endswith("/"):
+            return self.app__api_base_url[:-1]
+        return self.app__api_base_url
+
+    def get_web_base_url(self):
+        if self.app__web_base_url == "local":
+            return f"http://{self.app__host}:{self.app__port}"
+        if self.app__web_base_url is None:
+            return self.get_api_base_url()
+        if self.app__web_base_url.endswith("/"):
+            return self.app__web_base_url[:-1]
+        return self.app__web_base_url

backend/czi_hosted/common/corpora.py

@@ -0,0 +1,78 @@
+"""
+Corpora schema conventions support.  Helper functions for reading.
+
+https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md
+
+https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema_h5ad_implementation.md
+"""
+import collections
+import json
+
+from backend.czi_hosted.cli.upgrade import validate_version_str
+from backend.czi_hosted.common.utils.corpora_constants import CorporaConstants
+
+
+def corpora_get_versions_from_anndata(adata):
+    """
+    Given an AnnData object, return:
+        * None - if not a Corpora object
+        * [ corpora_schema_version, corpora_encoding_version ] - if a Corpora object
+
+    Implements the identification protocol defined in the specification.
+    """
+
+    # per Corpora AnnData spec, this is a corpora file if the following is true
+    if "version" not in adata.uns_keys():
+        return None
+    version = adata.uns["version"]
+    if not isinstance(version, collections.abc.Mapping) or "corpora_schema_version" not in version:
+        return None
+
+    corpora_schema_version = version.get("corpora_schema_version")
+    corpora_encoding_version = version.get("corpora_encoding_version")
+
+    # TODO: spec says these must be SEMVER values, so check.
+    if validate_version_str(corpora_schema_version) and validate_version_str(corpora_encoding_version):
+        return [corpora_schema_version, corpora_encoding_version]
+
+
+def corpora_is_version_supported(corpora_schema_version, corpora_encoding_version):
+    return (
+        corpora_schema_version
+        and corpora_encoding_version
+        and corpora_schema_version.startswith("1.")
+        and corpora_encoding_version.startswith("0.1.")
+    )
+
+
+def corpora_get_props_from_anndata(adata):
+    """
+    Get Corpora dataset properties from an AnnData
+    """
+    versions = corpora_get_versions_from_anndata(adata)
+    if versions is None:
+        return None
+    [corpora_schema_version, corpora_encoding_version] = versions
+    version_is_supported = corpora_is_version_supported(corpora_schema_version, corpora_encoding_version)
+    if not version_is_supported:
+        raise ValueError("Unsupported Corpora schema version")
+
+    corpora_props = {}
+    for key in CorporaConstants.REQUIRED_SIMPLE_METADATA_FIELDS:
+        if key not in adata.uns:
+            raise KeyError(f"missing Corpora schema field {key}")
+        corpora_props[key] = adata.uns[key]
+
+    for key in CorporaConstants.OPTIONAL_JSON_ENCODED_METADATA_FIELD:
+        if key not in adata.uns:
+            continue
+        try:
+            corpora_props[key] = json.loads(adata.uns[key])
+        except json.JSONDecodeError:
+            raise json.JSONDecodeError(f"Corpora schema field {key} is expected to be a valid JSON string")
+
+    for key in CorporaConstants.OPTIONAL_SIMPLE_METADATA_FIELDS:
+        if key in adata.uns:
+            corpora_props[key] = adata.uns[key]
+
+    return corpora_props

backend/czi_hosted/common/health.py

@@ -0,0 +1,38 @@
+from http import HTTPStatus
+from flask import make_response, jsonify
+
+from backend.czi_hosted import __version__ as cellxgene_version
+from backend.common.utils.data_locator import DataLocator
+
+
+def _is_accessible(path, config):
+    if path is None:
+        return True
+
+    try:
+        dl = DataLocator(path, region_name=config.data_locator__s3__region_name)
+        return dl.exists()
+    except RuntimeError:
+        return False
+
+
+def health_check(config):
+    """
+    simple health check - return HTTP response.
+    See https://tools.ietf.org/id/draft-inadarei-api-health-check-01.html
+    """
+    health = {"status": None, "version": "1", "releaseID": cellxgene_version}
+
+    checks = False
+    server_config = config.server_config
+    if config.is_multi_dataset():
+        dataroots = [datapath_dict["dataroot"] for datapath_dict in server_config.multi_dataset__dataroot.values()]
+        checks = all([_is_accessible(dataroot, server_config) for dataroot in dataroots])
+    else:
+        checks = _is_accessible(server_config.single_dataset__datapath, server_config)
+
+    health["status"] = "pass" if checks else "fail"
+    code = HTTPStatus.OK if health["status"] == "pass" else HTTPStatus.BAD_REQUEST
+    response = make_response(jsonify(health), code)
+    response.headers["Content-Type"] = "application/health+json"
+    return response

backend/czi_hosted/common/immutable_kvcache.py

@@ -0,0 +1,71 @@
+import threading
+from collections.abc import MutableMapping
+
+
+class ImmutableKVCache(MutableMapping):
+    """
+    Guarantees that the factory will be called for each key once, and
+    only once.
+    """
+
+    def __init__(self, factory):
+        self.factory = factory  # user-provided factory function
+        self.lock = threading.Lock()  # guards factory_calls
+        self.factory_calls = {}  # per-key factory condition variables
+        self.cache = {}  # result cache, indexed by key
+        super().__init__()
+
+    def __getitem__(self, key):
+        if key in self.cache:
+            return self.cache[key]
+
+        # we need to call factory.  First grab the main lock and the per-key CV.
+        factory_calls = None
+        creation_thr = False
+        with self.lock:
+            if key in self.cache:
+                return self.cache[key]
+            if key not in self.factory_calls:
+                creation_thr = True
+                self.factory_calls[key] = {"cv": threading.Condition(), "is_done": False, "error": None}
+            factory_calls = self.factory_calls[key]
+
+        # with the CV, create the value (or wait for it to be created)
+        cv = factory_calls["cv"]
+        with cv:
+            if creation_thr:
+                try:
+                    self.cache[key] = self.factory(key)
+                except Exception as e:
+                    factory_calls["error"] = e
+
+                factory_calls["is_done"] = True
+                cv.notify_all()
+            else:
+                """ wait for the value to be available """
+                while not factory_calls["is_done"]:
+                    cv.wait()
+
+        with self.lock:
+            if key in self.factory_calls:
+                del self.factory_calls[key]
+
+        return self.cache[key]
+
+    def __iter__(self):
+        """ weak iter, don't call factory """
+        return self.cache.__iter__()
+
+    def __len__(self):
+        return self.cache.__len__()
+
+    def __contains__(self, key):
+        """ weak contain - don't call factory """
+        return self.cache.__contains__(key)
+
+    def __delitem__(self, key):
+        del self.cache[key]
+
+    def __setitem__(self, key, value):
+        """ unsupported """
+        raise NotImplementedError

backend/czi_hosted/common/rest.py

@@ -0,0 +1,330 @@
+import copy
+import logging
+import sys
+from http import HTTPStatus
+import zlib
+
+from flask import make_response, jsonify, current_app, abort
+from werkzeug.urls import url_unquote
+
+from backend.czi_hosted.common.config.client_config import get_client_config, get_client_userinfo
+from backend.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg
+from backend.common.errors import (
+    FilterError,
+    JSONEncodingValueError,
+    PrepareError,
+    DisabledFeatureError,
+    ExceedsLimitError,
+    DatasetAccessError,
+    ColorFormatException,
+)
+
+import json
+from backend.common.fbs.matrix import decode_matrix_fbs
+
+
+def abort_and_log(code, logmsg, loglevel=logging.DEBUG, include_exc_info=False):
+    """
+    Log the message, then abort with HTTP code. If include_exc_info is true,
+    also include current exception via sys.exc_info().
+    """
+    if include_exc_info:
+        exc_info = sys.exc_info()
+    else:
+        exc_info = False
+    current_app.logger.log(loglevel, logmsg, exc_info=exc_info)
+    # Do NOT send log message to HTTP response.
+    return abort(code)
+
+
+def _query_parameter_to_filter(args):
+    """
+    Convert an annotation value filter, if present in the query args,
+    into the standard dict filter format used by internal code.
+
+    Query param filters look like:  <axis>:name=value, where value
+    may be one of:
+        - a range, min,max, where either may be an open range by using an asterisc, eg, 10,*
+        - a value
+    Eg,
+        ...?tissue=lung&obs:tissue=heart&obs:num_reads=1000,*
+    """
+    filters = {
+        "obs": {},
+        "var": {},
+    }
+
+    # args has already been url-unquoted once.  We assume double escaping
+    # on name and value.
+    try:
+        for key, value in args.items(multi=True):
+            axis, name = key.split(":")
+            if axis not in ("obs", "var"):
+                raise FilterError("unknown filter axis")
+            name = url_unquote(name)
+            current = filters[axis].setdefault(name, {"name": name})
+
+            val_split = value.split(",")
+            if len(val_split) == 1:
+                if "min" in current or "max" in current:
+                    raise FilterError("do not mix range and value filters")
+                value = url_unquote(value)
+                values = current.setdefault("values", [])
+                values.append(value)
+
+            elif len(val_split) == 2:
+                if len(current) > 1:
+                    raise FilterError("duplicate range specification")
+                min = url_unquote(val_split[0])
+                max = url_unquote(val_split[1])
+                if min != "*":
+                    current["min"] = float(min)
+                if max != "*":
+                    current["max"] = float(max)
+                if len(current) < 2:
+                    raise FilterError("must specify at least min or max in range filter")
+
+            else:
+                raise FilterError("badly formated filter value")
+
+    except ValueError as e:
+        raise FilterError(str(e))
+
+    result = {}
+    for axis in ("obs", "var"):
+        axis_filter = filters[axis]
+        if len(axis_filter) > 0:
+            result[axis] = {"annotation_value": [val for val in axis_filter.values()]}
+
+    return result
+
+
+def schema_get_helper(data_adaptor):
+    """helper function to gather the schema from the data source and annotations"""
+    schema = data_adaptor.get_schema()
+    schema = copy.deepcopy(schema)
+
+    # add label obs annotations as needed
+    annotations = data_adaptor.dataset_config.user_annotations
+    if annotations is not None:
+        label_schema = annotations.get_schema(data_adaptor)
+        schema["annotations"]["obs"]["columns"].extend(label_schema)
+
+    return schema
+
+
+def schema_get(data_adaptor):
+    schema = schema_get_helper(data_adaptor)
+    return make_response(jsonify({"schema": schema}), HTTPStatus.OK)
+
+
+def config_get(app_config, data_adaptor):
+    config = get_client_config(app_config, data_adaptor)
+    return make_response(jsonify(config), HTTPStatus.OK)
+
+
+def userinfo_get(app_config, data_adaptor):
+    config = get_client_userinfo(app_config, data_adaptor)
+    return make_response(jsonify(config), HTTPStatus.OK)
+
+
+def annotations_obs_get(request, data_adaptor):
+    fields = request.args.getlist("annotation-name", None)
+    num_columns_requested = len(data_adaptor.get_obs_keys()) if len(fields) == 0 else len(fields)
+    if data_adaptor.server_config.exceeds_limit("column_request_max", num_columns_requested):
+        return abort(HTTPStatus.BAD_REQUEST)
+    preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
+    if preferred_mimetype != "application/octet-stream":
+        return abort(HTTPStatus.NOT_ACCEPTABLE)
+
+    try:
+        labels = None
+        annotations = data_adaptor.dataset_config.user_annotations
+        if annotations:
+            labels = annotations.read_labels(data_adaptor)
+        fbs = data_adaptor.annotation_to_fbs_matrix(Axis.OBS, fields, labels)
+        return make_response(fbs, HTTPStatus.OK, {"Content-Type": "application/octet-stream"})
+    except KeyError as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+
+
+def annotations_put_fbs_helper(data_adaptor, fbs):
+    """helper function to write annotations from fbs"""
+    annotations = data_adaptor.dataset_config.user_annotations
+    if annotations is None:
+        raise DisabledFeatureError("Writable annotations are not enabled")
+
+    new_label_df = decode_matrix_fbs(fbs)
+    if not new_label_df.empty:
+        new_label_df = data_adaptor.check_new_labels(new_label_df)
+    annotations.write_labels(new_label_df, data_adaptor)
+
+
+def inflate(data):
+    return zlib.decompress(data)
+
+
+def annotations_obs_put(request, data_adaptor):
+    annotations = data_adaptor.dataset_config.user_annotations
+    if annotations is None:
+        return abort(HTTPStatus.NOT_IMPLEMENTED)
+
+    anno_collection = request.args.get("annotation-collection-name", default=None)
+    fbs = inflate(request.get_data())
+
+    if anno_collection is not None:
+        if not annotations.is_safe_collection_name(anno_collection):
+            return abort(HTTPStatus.BAD_REQUEST, "Bad annotation collection name")
+        annotations.set_collection(anno_collection)
+
+    try:
+        annotations_put_fbs_helper(data_adaptor, fbs)
+        res = json.dumps({"status": "OK"})
+        return make_response(res, HTTPStatus.OK, {"Content-Type": "application/json"})
+    except (ValueError, DisabledFeatureError, KeyError) as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+
+
+def annotations_var_get(request, data_adaptor):
+    fields = request.args.getlist("annotation-name", None)
+    num_columns_requested = len(data_adaptor.get_var_keys()) if len(fields) == 0 else len(fields)
+    if data_adaptor.server_config.exceeds_limit("column_request_max", num_columns_requested):
+        return abort(HTTPStatus.BAD_REQUEST)
+    preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
+    if preferred_mimetype != "application/octet-stream":
+        return abort(HTTPStatus.NOT_ACCEPTABLE)
+
+    try:
+        labels = None
+        annotations = data_adaptor.dataset_config.user_annotations
+        if annotations is not None:
+            labels = annotations.read_labels(data_adaptor)
+        return make_response(
+            data_adaptor.annotation_to_fbs_matrix(Axis.VAR, fields, labels),
+            HTTPStatus.OK,
+            {"Content-Type": "application/octet-stream"},
+        )
+    except KeyError as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+
+
+def data_var_put(request, data_adaptor):
+    preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
+    if preferred_mimetype != "application/octet-stream":
+        return abort(HTTPStatus.NOT_ACCEPTABLE)
+
+    filter_json = request.get_json()
+    filter = filter_json["filter"] if filter_json else None
+    try:
+        return make_response(
+            data_adaptor.data_frame_to_fbs_matrix(filter, axis=Axis.VAR),
+            HTTPStatus.OK,
+            {"Content-Type": "application/octet-stream"},
+        )
+    except (FilterError, ValueError, ExceedsLimitError) as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+
+
+def data_var_get(request, data_adaptor):
+    preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
+    if preferred_mimetype != "application/octet-stream":
+        return abort(HTTPStatus.NOT_ACCEPTABLE)
+
+    try:
+        filter = _query_parameter_to_filter(request.args)
+        return make_response(
+            data_adaptor.data_frame_to_fbs_matrix(filter, axis=Axis.VAR),
+            HTTPStatus.OK,
+            {"Content-Type": "application/octet-stream"},
+        )
+    except (FilterError, ValueError, ExceedsLimitError) as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+
+
+def colors_get(data_adaptor):
+    if not data_adaptor.dataset_config.presentation__custom_colors:
+        return make_response(jsonify({}), HTTPStatus.OK)
+    try:
+        return make_response(jsonify(data_adaptor.get_colors()), HTTPStatus.OK)
+    except ColorFormatException as e:
+        return abort_and_log(HTTPStatus.NOT_FOUND, str(e), include_exc_info=True)
+
+
+def diffexp_obs_post(request, data_adaptor):
+    if not data_adaptor.dataset_config.diffexp__enable:
+        return abort(HTTPStatus.NOT_IMPLEMENTED)
+
+    args = request.get_json()
+    try:
+        # TODO: implement varfilter mode
+        mode = DiffExpMode(args["mode"])
+
+        if mode == DiffExpMode.VAR_FILTER or "varFilter" in args:
+            return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, "varFilter not enabled")
+
+        set1_filter = args.get("set1", {"filter": {}})["filter"]
+        set2_filter = args.get("set2", {"filter": {}})["filter"]
+        count = args.get("count", None)
+
+        if set1_filter is None or set2_filter is None or count is None:
+            return abort_and_log(HTTPStatus.BAD_REQUEST, "missing required parameter")
+        if Axis.VAR in set1_filter or Axis.VAR in set2_filter:
+            return abort_and_log(HTTPStatus.BAD_REQUEST, "var axis filter not enabled")
+
+    except (KeyError, TypeError) as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+
+    try:
+        diffexp = data_adaptor.diffexp_topN(set1_filter, set2_filter, count)
+        return make_response(diffexp, HTTPStatus.OK, {"Content-Type": "application/json"})
+    except (ValueError, DisabledFeatureError, FilterError, ExceedsLimitError) as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+    except JSONEncodingValueError:
+        # JSON encoding failure, usually due to bad data. Just let it ripple up
+        # to default exception handler.
+        current_app.logger.warning(JSON_NaN_to_num_warning_msg)
+        raise
+
+
+def layout_obs_get(request, data_adaptor):
+    fields = request.args.getlist("layout-name", None)
+    num_columns_requested = len(data_adaptor.get_embedding_names()) if len(fields) == 0 else len(fields)
+    if data_adaptor.server_config.exceeds_limit("column_request_max", num_columns_requested):
+        return abort(HTTPStatus.BAD_REQUEST)
+
+    preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
+    if preferred_mimetype != "application/octet-stream":
+        return abort(HTTPStatus.NOT_ACCEPTABLE)
+
+    try:
+        return make_response(
+            data_adaptor.layout_to_fbs_matrix(fields), HTTPStatus.OK, {"Content-Type": "application/octet-stream"}
+        )
+    except (KeyError, DatasetAccessError) as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+    except PrepareError:
+        return abort_and_log(
+            HTTPStatus.NOT_IMPLEMENTED,
+            f"No embedding available {request.path}",
+            loglevel=logging.ERROR,
+            include_exc_info=True,
+        )
+
+
+def layout_obs_put(request, data_adaptor):
+    if not data_adaptor.dataset_config.embeddings__enable_reembedding:
+        return abort(HTTPStatus.NOT_IMPLEMENTED)
+
+    args = request.get_json()
+    filter = args["filter"] if args else None
+    if not filter:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, "obs filter is required")
+    method = args["method"] if args else "umap"
+
+    try:
+        schema = data_adaptor.compute_embedding(method, filter)
+        return make_response(jsonify(schema), HTTPStatus.OK, {"Content-Type": "application/json"})
+    except NotImplementedError as e:
+        return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, str(e))
+    except (ValueError, DisabledFeatureError, FilterError) as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)

backend/czi_hosted/common/utils/corpora_constants.py

@@ -0,0 +1,22 @@
+class CorporaConstants(object):
+    REQUIRED_SIMPLE_METADATA_FIELDS = [
+        "version",
+        "title",
+        "layer_descriptions",
+        "organism",
+        "organism_ontology_term_id",
+    ]
+
+    # The Corpora specification requires some values encoded as JSON due to the inability of AnnData to store complex
+    # types.
+    OPTIONAL_JSON_ENCODED_METADATA_FIELD = ["contributors", "project_links"]
+
+    OPTIONAL_SIMPLE_METADATA_FIELDS = [
+        "preprint_doi",
+        "publication_doi",
+        "default_embedding",
+        "default_field",
+        "tags",
+        "project_name",
+        "project_description",
+    ]

backend/czi_hosted/common/utils/cxg_constants.py

@@ -0,0 +1,4 @@
+class CxgConstants(object):
+    # The CXG container version number.  Must be a semver string (major.minor.patch)
+    # DO NOT UPDATE THIS WITHOUT ALSO UPDATING CXG SPECIFICATION.
+    CXG_VERSION = "0.2.0"

backend/czi_hosted/common/utils/cxg_generation_utils.py

@@ -0,0 +1,178 @@
+import json
+
+import numpy as np
+import tiledb
+
+from backend.common.utils.type_conversion_utils import get_dtype_of_array, get_dtype_and_schema_of_array
+
+
+def convert_dictionary_to_cxg_group(cxg_container, metadata_dict, group_metadata_name="cxg_group_metadata"):
+    """
+    Saves the contents of the dictionary to the CXG output directory specified.
+
+    This function is primarily used to save metadata about a dataset to the CXG directory. At some point, tiledb will
+    have support for metadata on groups at which point the utility of this function should be revisited. Until such
+    feature exists, this function create an empty array and annotate that array.
+
+    For more information, visit https://github.com/TileDB-Inc/TileDB-Py/issues/254.
+    """
+
+    array_name = f"{cxg_container}/{group_metadata_name}"
+
+    # Because TileDB does not allow one to attach metadata directly to a CXG group, we need to have a workaround
+    # where we create an empty array and attached the metadata onto to this empty array. Below we construct this empty
+    # array.
+    tiledb.from_numpy(array_name, np.zeros((1,)))
+
+    with tiledb.DenseArray(array_name, mode="w") as metadata_array:
+        for key, value in metadata_dict.items():
+            metadata_array.meta[key] = value
+
+
+def convert_dataframe_to_cxg_array(cxg_container, dataframe_name, dataframe, index_column_name, ctx):
+    """
+    Saves the contents of the dataframe to the CXG output directory specified.
+
+    Current access patterns are oriented toward reading very large slices of the dataframe, one attribute at a time.
+    Attribute data also tends to be (often) repetitive (bools, categories, strings). Given this, we use a large tile
+    size (1000) and very aggressive compression levels.
+    """
+
+    def create_dataframe_array(array_name, dataframe):
+        tiledb_filter = tiledb.FilterList(
+            [
+                # Attempt aggressive compression as many of these dataframes are very repetitive strings, bools and
+                # other non-float data.
+                tiledb.ZstdFilter(level=22),
+            ]
+        )
+        attrs = [
+            tiledb.Attr(name=column, dtype=get_dtype_of_array(dataframe[column]), filters=tiledb_filter)
+            for column in dataframe
+        ]
+        domain = tiledb.Domain(
+            tiledb.Dim(domain=(0, dataframe.shape[0] - 1), tile=min(dataframe.shape[0], 1000), dtype=np.uint32)
+        )
+        schema = tiledb.ArraySchema(
+            domain=domain, sparse=False, attrs=attrs, cell_order="row-major", tile_order="row-major"
+        )
+        tiledb.DenseArray.create(array_name, schema)
+
+    array_name = f"{cxg_container}/{dataframe_name}"
+
+    create_dataframe_array(array_name, dataframe)
+
+    with tiledb.DenseArray(array_name, mode="w", ctx=ctx) as array:
+        value = {}
+        schema_hints = {}
+        for column_name, column_values in dataframe.items():
+            dtype, hints = get_dtype_and_schema_of_array(column_values)
+            value[column_name] = column_values.to_numpy(dtype=dtype)
+            if hints:
+                schema_hints.update({column_name: hints})
+
+        schema_hints.update({"index": index_column_name})
+        array[:] = value
+        array.meta["cxg_schema"] = json.dumps(schema_hints)
+
+    tiledb.consolidate(array_name, ctx=ctx)
+
+
+def convert_ndarray_to_cxg_dense_array(ndarray_name, ndarray, ctx):
+    """
+    Saves contents of ndarray to the CXG output directory specified.
+
+    Generally this function is used to convert dataset embeddings. Because embeddings are typically accessed with
+    very large slices (or all of the embedding), they do not benefit from overly aggressive compression due to their
+    format.  Given this, we use a large tile size (1000) but only default compression level.
+    """
+
+    def create_ndarray_array(ndarray_name, ndarray):
+        filters = tiledb.FilterList([tiledb.ZstdFilter()])
+        attrs = [tiledb.Attr(dtype=ndarray.dtype, filters=filters)]
+        dimensions = [
+            tiledb.Dim(
+                domain=(0, ndarray.shape[dimension] - 1), tile=min(ndarray.shape[dimension], 1000), dtype=np.uint32
+            )
+            for dimension in range(ndarray.ndim)
+        ]
+        domain = tiledb.Domain(*dimensions)
+        schema = tiledb.ArraySchema(
+            domain=domain, sparse=False, attrs=attrs, capacity=1_000_000, cell_order="row-major", tile_order="row-major"
+        )
+        tiledb.DenseArray.create(ndarray_name, schema)
+
+    create_ndarray_array(ndarray_name, ndarray)
+
+    with tiledb.DenseArray(ndarray_name, mode="w", ctx=ctx) as array:
+        array[:] = ndarray
+
+    tiledb.consolidate(ndarray_name, ctx=ctx)
+
+
+def convert_matrix_to_cxg_array(
+    matrix_name, matrix, encode_as_sparse_array, ctx, column_shift_for_sparse_encoding=None
+):
+    """
+    Converts a numpy array matrix into a TileDB SparseArray of DenseArray based on whether `encode_as_sparse_array`
+    is true or not. Note that when the matrix is encoded as a SparseArray, it only writes the values that are
+    nonzero. This means that if you count the number of elements in the SparseArray, it will not equal the total
+    number of elements in the matrix, only the number of nonzero elements.
+
+    Furthermore, if the `column_shift_for_sparse_encoding` matrix is not None, this function will subtract the sparse
+    encoding from the original given matrix and as previously stated, only write the nonzero values to the TileDB
+    SparseArray.
+    """
+
+    def create_matrix_array(matrix_name, number_of_rows, number_of_columns, encode_as_sparse_array):
+        filters = tiledb.FilterList([tiledb.ZstdFilter()])
+        attrs = [tiledb.Attr(dtype=np.float32, filters=filters)]
+        if encode_as_sparse_array:
+            domain = tiledb.Domain(
+                tiledb.Dim(name="obs", domain=(0, number_of_rows - 1), tile=min(number_of_rows, 512), dtype=np.uint32),
+                tiledb.Dim(
+                    name="var", domain=(0, number_of_columns - 1), tile=min(number_of_columns, 2048), dtype=np.uint32
+                ),
+            )
+        else:
+            domain = tiledb.Domain(
+                tiledb.Dim(name="obs", domain=(0, number_of_rows - 1), tile=min(number_of_rows, 50), dtype=np.uint32),
+                tiledb.Dim(
+                    name="var", domain=(0, number_of_columns - 1), tile=min(number_of_columns, 100), dtype=np.uint32
+                ),
+            )
+        schema = tiledb.ArraySchema(
+            domain=domain, sparse=encode_as_sparse_array, attrs=attrs, cell_order="row-major", tile_order="col-major"
+        )
+        if encode_as_sparse_array:
+            tiledb.SparseArray.create(matrix_name, schema)
+        else:
+            tiledb.DenseArray.create(matrix_name, schema)
+
+    number_of_rows = matrix.shape[0]
+    number_of_columns = matrix.shape[1]
+    stride = min(int(np.power(10, np.around(np.log10(1e9 / number_of_columns)))), 10_000)
+
+    create_matrix_array(matrix_name, number_of_rows, number_of_columns, encode_as_sparse_array)
+
+    if encode_as_sparse_array:
+        with tiledb.SparseArray(matrix_name, mode="w", ctx=ctx) as array:
+            for start_row_index in range(0, number_of_rows, stride):
+                end_row_index = min(start_row_index + stride, number_of_rows)
+                matrix_subset = matrix[start_row_index:end_row_index, :]
+                if not isinstance(matrix_subset, np.ndarray):
+                    matrix_subset = matrix_subset.toarray()
+                if column_shift_for_sparse_encoding is not None:
+                    matrix_subset = matrix_subset - column_shift_for_sparse_encoding
+                indices = np.nonzero(matrix_subset)
+                trow = indices[0] + start_row_index
+                array[trow, indices[1]] = matrix_subset[indices[0], indices[1]]
+
+    else:
+        with tiledb.DenseArray(matrix_name, mode="w", ctx=ctx) as array:
+            for start_row_index in range(0, number_of_rows, stride):
+                end_row_index = min(start_row_index + stride, number_of_rows)
+                matrix_subset = matrix[start_row_index:end_row_index, :]
+                if not isinstance(matrix_subset, np.ndarray):
+                    matrix_subset = matrix_subset.toarray()
+                array[start_row_index:end_row_index, :] = matrix_subset

backend/czi_hosted/common/utils/matrix_utils.py

@@ -0,0 +1,115 @@
+import logging
+
+import numpy as np
+from scipy.stats import mode
+
+
+def is_matrix_sparse(matrix: np.ndarray, sparse_threshold):
+    """
+    Returns whether `matrix` is sparse or not (i.e. dense). This is determined by figuring out whether the matrix has
+    a sparsity percentage below the sparse_threshold, returning the number of non-zeros encountered and number of
+    elements evaluated.  This function may return before evaluating the whole matrix if it can be determined that matrix
+    is not sparse enough.
+    """
+
+    if sparse_threshold == 100.0:
+        return True
+    if sparse_threshold == 0.0:
+        return False
+
+    total_number_of_rows = matrix.shape[0]
+    total_number_of_columns = matrix.shape[1]
+    total_number_of_matrix_elements = total_number_of_rows * total_number_of_columns
+
+    # For efficiency, we count the number of non-zero elements in chunks of the matrix at a time until we hit the
+    # maximum number of non zero values allowed before the matrix is deemed "dense." This allows the function the
+    # quit early for large dense matrices.
+    row_stride = min(int(np.power(10, np.around(np.log10(1e9 / total_number_of_columns)))), 10_000)
+
+    maximum_number_of_non_zero_elements_in_matrix = int(
+        total_number_of_rows * total_number_of_columns * sparse_threshold / 100
+    )
+    number_of_non_zero_elements = 0
+
+    for start_row_index in range(0, total_number_of_rows, row_stride):
+        end_row_index = min(start_row_index + row_stride, total_number_of_rows)
+
+        matrix_subset = matrix[start_row_index:end_row_index, :]
+        if not isinstance(matrix_subset, np.ndarray):
+            matrix_subset = matrix_subset.toarray()
+
+        number_of_non_zero_elements += np.count_nonzero(matrix_subset)
+        if number_of_non_zero_elements > maximum_number_of_non_zero_elements_in_matrix:
+            if end_row_index != total_number_of_rows:
+                percentage_of_non_zero_elements = (
+                    100 * number_of_non_zero_elements / (end_row_index * total_number_of_columns)
+                )
+                logging.info(
+                    f"Matrix is not sparse. Percentage of non-zero elements (estimate): "
+                    f"{percentage_of_non_zero_elements:6.2f}"
+                )
+            else:
+                percentage_of_non_zero_elements = 100 * number_of_non_zero_elements / total_number_of_matrix_elements
+                logging.info(
+                    f"Matrix is not sparse. Percentage of non-zero elements (exact): "
+                    f"{percentage_of_non_zero_elements:6.2f}"
+                )
+            return False
+
+    is_sparse = (100.0 * number_of_non_zero_elements / total_number_of_matrix_elements) < sparse_threshold
+    return is_sparse
+
+
+def get_column_shift_encode_for_matrix(matrix, sparse_threshold):
+    """
+    Returns a column shift if there is a column shift that allows the given matrix to be considered as sparse. Column
+    shift encoding works by taking the most common value in each column, then subtracting that value from each  element
+    of the column.  If each column mostly contains its most common value, then the resulting matrix can be very sparse.
+
+    This function determines if column shift encoding can be used to transform the matrix into a sparse matrix with a
+    sparsity below the sparse_threshold. If so, returns the array that stores this encoding. This function also returns
+    the number of non-zeros encountered and number of elements evaluated.  This function may return before evaluating
+    the whole matrix if it can be determined that the matrix cannot benefit from column shift encoding.
+    """
+
+    total_number_of_rows = matrix.shape[0]
+    total_number_of_columns = matrix.shape[1]
+    total_number_of_matrix_elements = total_number_of_rows * total_number_of_columns
+
+    stride = max(1, 128_000_000 // total_number_of_rows)
+    column_shift = np.zeros(total_number_of_columns)
+
+    maximum_number_of_non_zero_elements_in_matrix = int(
+        total_number_of_rows * total_number_of_columns * sparse_threshold / 100
+    )
+    number_of_non_zero_elements = 0
+
+    for start_column_index in range(0, total_number_of_columns, stride):
+        end_column_index = min(start_column_index + stride, total_number_of_columns)
+
+        matrix_subset = matrix[:, start_column_index:end_column_index]
+        if not isinstance(matrix_subset, np.ndarray):
+            matrix_subset = matrix_subset.toarray()
+
+        matrix_subset_mode = mode(matrix_subset)
+
+        column_shift[start_column_index:end_column_index] = matrix_subset_mode.mode
+        number_of_non_zero_elements += total_number_of_rows * (end_column_index - start_column_index) - np.sum(
+            matrix_subset_mode.count
+        )
+
+        if number_of_non_zero_elements > maximum_number_of_non_zero_elements_in_matrix:
+            if end_column_index != total_number_of_columns:
+                logging.info(
+                    "Matrix is not sparse even with column shift. Percentage of non-zero elements (estimate): %6.2f"
+                    % (100 * number_of_non_zero_elements / end_column_index * total_number_of_rows)
+                )
+            else:
+                logging.info(
+                    "Matrix is not sparse even with column shift. Percentage of non-zero elements (exact): %6.2f"
+                    % (100 * number_of_non_zero_elements / total_number_of_matrix_elements)
+                )
+            return None
+
+    is_sparse = (100.0 * number_of_non_zero_elements / total_number_of_matrix_elements) < sparse_threshold
+    return column_shift if is_sparse else None

backend/czi_hosted/common/utils/sanitization_utils.py

@@ -0,0 +1,40 @@
+import re
+
+
+def sanitize_values_in_list(list_of_keys: list):
+    """
+    Returns a dictionary mapping of the old keys in the list of `list_of_keys` to its new, clean name that is both
+    safe and unique.
+    """
+
+    if not all([isinstance(key, str) for key in list_of_keys]):
+        raise Exception("List of keys to sanitize must contain all strings.")
+
+    # Mask out [~/.] and anything outside the ASCII range.
+    mask = re.compile(r"[^ -\-0-\[\]-\}]")
+    clean_keys_list = [mask.sub("_", key) for key in list_of_keys]
+
+    # Dedupe the clean keys list
+    deduped_clean_keys_list = []
+    for index, clean_key in enumerate(clean_keys_list):
+        total_occurrences_of_clean_key = clean_keys_list.count(clean_key)
+        total_occurrences_up_until_current_index = clean_keys_list[:index].count(clean_key)
+        deduped_clean_keys_list.append(
+            clean_key + "_" + str(total_occurrences_up_until_current_index + 1)
+            if total_occurrences_of_clean_key > 1
+            else clean_key
+        )
+
+    return dict(zip(list_of_keys, deduped_clean_keys_list))
+
+
+def sanitize_keys_in_dictionary(dict_to_sanitize: dict):
+    """
+    Clean and dedupe the keys in the given dictionary.
+    """
+
+    clean_keys = sanitize_values_in_list(dict_to_sanitize.keys())
+    for original_key, sanitized_key in clean_keys.items():
+        if original_key != sanitized_key:
+            dict_to_sanitize[sanitized_key] = dict_to_sanitize[original_key]
+            del dict_to_sanitize[original_key]