Refactor czi_hosted and server into backend directory, pull common code into backend/common, refactor tests (#2102)

* move local_server -> backend/server server-> backend/czi_hosted, pull common code into backend/common update imports, tests and make commands

backend/test/test_server/locust/README.md

@@ -0,0 +1,39 @@
+# Locust Load Test
+
+This directory contains scripts to load test cellxgene's backend. It
+primary simulates initial data loading and expression data fetch, which
+are the most common data routes. It currently does not include tests
+for differential expression or re-clustering routes.
+
+## Prerequisites
+
+You need:
+
+- Python 3.6+, and pip
+- cellxgene installed
+- install the locust dependencies in `requirements-locust.txt`
+
+## To test
+
+1. Choose to run cellxgene in either single dataset or data root mode.
+2. Edit config.py to indicate which datasets to load:
+   - in single dataset mode, just set `DataSets=[""]`
+   - in dataroot (multi-dataset) mode, add the route names, eg, `DataSets=['foo.cxg', 'bar.cxg']`
+3. Launch cellxgene in the appropriate mode
+4. launch locust, specifying the correct --host argument
+5. point your web browser to the locust http server, usually `http://localhost:8089/`
+
+### Single dataset mode
+
+- Edit config.py and set `DataSets=[""]`
+- in a shell, run `cellxgene launch somefile.h5ad`
+- launch locust in another shell, `locust --host http://localhost:5005/` (or wherever you are running cellxgene)
+- point a browser to the locust port, usually http://localhost:8089/
+- run test
+
+### Multi-dataset mode
+
+- Edit config.py and set `DataSets=["datapath1", ...]`
+- in a shell, run `cellxgene launch --dataroot path`
+
+The remainder of the steps are same as single dataset.

backend/test/test_server/locust/config.py

@@ -0,0 +1,15 @@
+"""
+Locust test config
+"""
+
+
+""" Data routes that will be tested """
+
+# single dataset, for non-dataroot tests
+# DataSets = [""]
+
+# multi-dataset, for dataroot tests.  these are varied in size/shape
+DataSets = [
+    "GSE60361.cxg",
+    "WongAdultRetina.cxg",
+]

backend/test/test_server/locust/locustfile.py

@@ -0,0 +1,165 @@
+import json
+import random
+
+import requests
+from config import DataSets
+from locust import HttpUser, SequentialTaskSet, task, between, TaskSet
+from locust.clients import HttpSession
+from requests.packages.urllib3.exceptions import InsecureRequestWarning
+
+import backend.test.decode_fbs as decode_fbs
+
+requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
+
+"""
+Simple locust stress test definition for cellxgene
+"""
+
+API_SUFFIX = "api/v0.2"
+
+
+class CellXGeneTasks(TaskSet):
+    """
+    Simulate use against a single dataset
+    """
+
+    def on_start(self):
+
+        self.client.verify = False
+        self.dataset = random.choice(DataSets)
+
+        with self.client.get(
+            f"{self.dataset}/{API_SUFFIX}/schema", stream=True, catch_response=True
+        ) as schema_response:
+            if schema_response.status_code == 200:
+                self.schema = schema_response.json()["schema"]
+            else:
+                self.schema = None
+
+        with self.client.get(
+            f"{self.dataset}/{API_SUFFIX}/config", stream=True, catch_response=True
+        ) as config_response:
+            if config_response.status_code == 200:
+                self.config = config_response.json()["config"]
+            else:
+                self.config = None
+
+        with self.client.get(
+            f"{self.dataset}/{API_SUFFIX}/annotations/var?annotation-name={self.var_index_name()}",
+            headers={"Accept": "application/octet-stream"},
+            catch_response=True,
+        ) as var_index_response:
+            if var_index_response.status_code == 200:
+                df = decode_fbs.decode_matrix_FBS(var_index_response.content)
+                gene_names_idx = df["col_idx"].index(self.var_index_name())
+                self.gene_names = df["columns"][gene_names_idx]
+            else:
+                self.gene_names = []
+
+    def var_index_name(self):
+        if self.schema is not None:
+            return self.schema["annotations"]["var"]["index"]
+        return None
+
+    def obs_annotation_names(self):
+        if self.schema is not None:
+            return [col["name"] for col in self.schema["annotations"]["obs"]["columns"]]
+        return []
+
+    def layout_names(self):
+        if self.schema is not None:
+            return [layout["name"] for layout in self.schema["layout"]["obs"]]
+        else:
+            return []
+
+    @task(2)
+    class InitializeClient(SequentialTaskSet):
+        """
+        Initial loading of cellxgene - when the user hits the main route.
+
+        Currently this sequence skips some of the static assets, which are quite small and should be served by the
+        HTTP server directly.
+
+        1. Load index.html, etc.
+        2. Concurrently load /config, /schema
+        3. Concurrently load /layout/obs, /annotations/var?annotation-name=<the index>
+        -- Does initial render --
+        4. Concurrently load all /annotations/obs and all /layouts/obs
+        -- Fully initialized --
+        """
+
+        # Users hit all of the init routes as fast as they can, subject to the ordering constraints and network latency.
+        wait_time = between(0.01, 0.1)
+
+        def on_start(self):
+            self.dataset = self.parent.dataset
+            self.client.verify = False
+            self.api_less_client = HttpSession(
+                base_url=self.client.base_url.replace("api.", "").replace("cellxgene/", ""),
+                request_success=self.client.request_success,
+                request_failure=self.client.request_failure,
+            )
+
+        @task
+        def index(self):
+            self.api_less_client.get(f"{self.dataset}", stream=True)
+
+        @task
+        def loadConfigAndSchema(self):
+            self.client.get(f"{self.dataset}/{API_SUFFIX}/schema", stream=True, catch_response=True)
+            self.client.get(f"{self.dataset}/{API_SUFFIX}/config", stream=True, catch_response=True)
+
+        @task
+        def loadBootstrapData(self):
+            self.client.get(
+                f"{self.dataset}/{API_SUFFIX}/layout/obs", headers={"Accept": "application/octet-stream"}, stream=True
+            )
+            self.client.get(
+                f"{self.dataset}/{API_SUFFIX}/annotations/var?annotation-name={self.parent.var_index_name()}",
+                headers={"Accept": "application/octet-stream"},
+                catch_response=True,
+            )
+
+        @task
+        def loadObsAnnotationsAndLayouts(self):
+            obs_names = self.parent.obs_annotation_names()
+            for name in obs_names:
+                self.client.get(
+                    f"{self.dataset}/{API_SUFFIX}/annotations/obs?annotation-name={name}",
+                    headers={"Accept": "application/octet-stream"},
+                    stream=True,
+                )
+
+            layouts = self.parent.layout_names()
+            for name in layouts:
+                self.client.get(
+                    f"{self.dataset}/{API_SUFFIX}/annotations/obs?layout-name={name}",
+                    headers={"Accept": "application/octet-stream"},
+                    stream=True,
+                )
+
+        @task
+        def done(self):
+            self.interrupt()
+
+    @task(1)
+    def load_expression(self):
+        """
+        Simulate user occasionally loading some expression data for a gene
+        """
+
+        gene_name = random.choice(self.gene_names)
+        filter = {"filter": {"var": {"annotation_value": [{"name": self.var_index_name(), "values": [gene_name]}]}}}
+        self.client.put(
+            f"{self.dataset}/{API_SUFFIX}/data/var",
+            data=json.dumps(filter),
+            headers={"Content-Type": "application/json", "Accept": "application/octet-stream"},
+            stream=True,
+        ).close()
+
+
+class CellxgeneUser(HttpUser):
+    tasks = [CellXGeneTasks]
+
+    # Most ops do not require back-end interaction, so slow cadence for users
+    wait_time = between(10, 60)

backend/test/test_server/locust/requirements-locust.txt

@@ -0,0 +1,2 @@
+locust
+-r ../../requirements.txt

backend/test/test_server/performance/create_test_matrix.py

@@ -0,0 +1,44 @@
+import anndata
+import argparse
+import random
+import scipy
+import numpy as np
+
+
+def main():
+    parser = argparse.ArgumentParser("A command to generate test h5ad files")
+    parser.add_argument("output", help="Name of the output file")
+    parser.add_argument("nobs", type=int, help="Number of observations (rows)")
+    parser.add_argument("nvar", type=int, help="Number of variables (columns)")
+    parser.add_argument("-n", "--nnz-percent", type=float, default=100, help="percent of non-zeros")
+    parser.add_argument("-c", "--col-shift", action="store_true", help="add a random value to each column")
+    parser.add_argument("--seed", type=int, default=None, help="add a random value to each column")
+
+    args = parser.parse_args()
+    create_test_h5ad(args.output, args.nobs, args.nvar, args.nnz_percent, args.col_shift, args.seed)
+
+
+def create_test_h5ad(outfile, nobs, nvar, nnz_percent=100, apply_col_shift=False, seed=None):
+    random.seed(seed)
+    np.random.seed(seed)
+    x = create_X_array(nobs, nvar, nnz_percent, apply_col_shift)
+    obsm = {"X_random": np.random.rand(nobs, 2).astype(np.float32)}
+    adata = anndata.AnnData(x, obsm=obsm)
+    adata.write(outfile)
+
+
+def create_X_array(nobs, nvar, nnz_percent, apply_col_shift):
+    if nnz_percent < 100:
+        array = scipy.sparse.random(nobs, nvar, nnz_percent * 0.01, dtype=np.float32, format="csc")
+    else:
+        array = np.random.rand(nobs, nvar).astype(np.float32)
+
+    if apply_col_shift:
+        col_shift = np.random.rand((nvar))
+        array += col_shift
+
+    return array
+
+
+if __name__ == "__main__":
+    main()

backend/test/test_server/performance/run_diffexp.py

@@ -0,0 +1,106 @@
+import sys
+import argparse
+import random
+import time
+import numpy as np
+
+import backend.server.compute.diffexp_generic as diffexp_generic
+
+from backend.server.common.config.app_config import AppConfig
+from backend.server.data_common.matrix_loader import MatrixDataLoader
+
+
+def main():
+    parser = argparse.ArgumentParser("A command to test diffexp")
+    parser.add_argument("dataset", help="name of a dataset to load")
+    parser.add_argument("-na", "--numA", type=int, help="number of rows in group A")
+    parser.add_argument("-nb", "--numB", type=int, help="number of rows in group B")
+    parser.add_argument("-va", "--varA", help="obs variable:value to use for group A")
+    parser.add_argument("-vb", "--varB", help="obs variable:value to use for group B")
+    parser.add_argument("-t", "--trials", default=1, type=int, help="number of trials")
+    parser.add_argument(
+        "-a", "--alg", choices=("default", "generic"), default="default", help="algorithm to use"
+    )
+    parser.add_argument("-s", "--show", default=False, action="store_true", help="show the results")
+    parser.add_argument(
+        "-n", "--new-selection", default=False, action="store_true", help="change the selection between each trial"
+    )
+    parser.add_argument("--seed", default=1, type=int, help="set the random seed")
+
+    args = parser.parse_args()
+
+    app_config = AppConfig()
+    app_config.update_server_config(single_dataset__datapath=args.dataset)
+    app_config.update_server_config(app__verbose=True)
+    app_config.complete_config()
+
+    loader = MatrixDataLoader(args.dataset)
+    adaptor = loader.open(app_config)
+
+    random.seed(args.seed)
+    np.random.seed(args.seed)
+    rows = adaptor.get_shape()[0]
+
+    if args.numA:
+        filterA = random.sample(range(rows), args.numA)
+    elif args.varA:
+        vname, vval = args.varA.split(":")
+        filterA = get_filter_from_obs(adaptor, vname, vval)
+    else:
+        print("must supply numA or varA")
+        sys.exit(1)
+
+    if args.numB:
+        filterB = random.sample(range(rows), args.numB)
+    elif args.varB:
+        vname, vval = args.varB.split(":")
+        filterB = get_filter_from_obs(adaptor, vname, vval)
+    else:
+        print("must supply numB or varB")
+        sys.exit(1)
+
+    for i in range(args.trials):
+        if args.new_selection:
+            if args.numA:
+                filterA = random.sample(range(rows), args.numA)
+            if args.numB:
+                filterB = random.sample(range(rows), args.numB)
+
+        maskA = np.zeros(rows, dtype=bool)
+        maskA[filterA] = True
+        maskB = np.zeros(rows, dtype=bool)
+        maskB[filterB] = True
+
+        t1 = time.time()
+        if args.alg == "default":
+            results = adaptor.compute_diffexp_ttest(maskA, maskB)
+        elif args.alg == "generic":
+            results = diffexp_generic.diffexp_ttest(adaptor, maskA, maskB)
+
+        t2 = time.time()
+        print("TIME=", t2 - t1)
+
+    if args.show:
+        for res in results:
+            print(res)
+
+
+def get_filter_from_obs(adaptor, obsname, obsval):
+    attrs = adaptor.get_obs_columns()
+    if obsname not in attrs:
+        print(f"Unknown obs attr {obsname}: expected on of {attrs}")
+        sys.exit(1)
+    obsvals = adaptor.query_obs_array(obsname)[:]
+    obsval = type(obsvals[0])(obsval)
+
+    vfilter = np.where(obsvals == obsval)[0]
+    if len(vfilter) == 0:
+        u = np.unique(obsvals)
+        print(f"Unknown value in variable {obsname}:{obsval}: expected one of {list(u)}")
+        sys.exit(1)
+
+    return vfilter
+
+
+if __name__ == "__main__":
+    main()

backend/test/test_server/unit/__init__.py

@@ -0,0 +1,162 @@
+import os
+import random
+import shutil
+import tempfile
+import time
+from contextlib import contextmanager
+from os import path, popen
+from subprocess import Popen
+
+import pandas as pd
+import requests
+
+from backend.server.common.annotations.local_file_csv import AnnotationsLocalFile
+from backend.server.common.config.app_config import AppConfig
+from backend.server.common.config import DEFAULT_SERVER_PORT
+from backend.common.utils.data_locator import DataLocator
+from backend.common.utils.utils import find_available_port
+from backend.common.fbs.matrix import encode_matrix_fbs
+from backend.server.data_common.matrix_loader import MatrixDataType, MatrixDataLoader
+from backend.test import PROJECT_ROOT
+
+
+def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False):
+    tmp_dir = tempfile.mkdtemp()
+    annotations_file = path.join(tmp_dir, "test_annotations.csv")
+    if annotations_fixture:
+        shutil.copyfile(f"{PROJECT_ROOT}/backend/test/fixtures/pbmc3k-annotations.csv", annotations_file)
+    fname = {
+        MatrixDataType.H5AD: f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
+    }[ext]
+    data_locator = DataLocator(fname)
+    config = AppConfig()
+    config.update_server_config(
+        app__flask_secret_key="secret",
+        single_dataset__obs_names=None,
+        single_dataset__var_names=None,
+        single_dataset__datapath=data_locator.path,
+    )
+    config.update_dataset_config(
+        embeddings__names=["umap"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01,
+    )
+
+    config.complete_config()
+    data = MatrixDataLoader(data_locator.abspath()).open(config)
+    anno_config = {
+        "user-annotations": True,
+        "genesets-save": False,
+    }
+    annotations = AnnotationsLocalFile(anno_config, None, annotations_file, None)
+    return data, tmp_dir, annotations
+
+
+def make_fbs(data):
+    df = pd.DataFrame(data)
+    return encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
+
+
+def skip_if(condition, reason: str):
+    def decorator(f):
+        def wraps(self, *args, **kwargs):
+            if condition(self):
+                self.skipTest(reason)
+            else:
+                f(self, *args, **kwargs)
+
+        return wraps
+
+    return decorator
+
+
+def app_config(data_locator, backed=False, extra_server_config={}, extra_dataset_config={}):
+    config = AppConfig()
+    config.update_server_config(
+        app__flask_secret_key="secret",
+        single_dataset__obs_names=None,
+        single_dataset__var_names=None,
+        adaptor__anndata_adaptor__backed=backed,
+        single_dataset__datapath=data_locator,
+        limits__diffexp_cellcount_max=None,
+        limits__column_request_max=None,
+    )
+    config.update_dataset_config(
+        embeddings__names=["umap", "tsne", "pca"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01
+    )
+    config.update_server_config(**extra_server_config)
+    config.update_dataset_config(**extra_dataset_config)
+    config.complete_config()
+    return config
+
+
+def start_test_server(command_line_args=[], app_config=None, env=None):
+    """
+    Command line arguments can be passed in, as well as an app_config.
+    This function is meant to be used like this, for example:
+
+    with unit(...) as server:
+        r = requests.get(f"{server}/...")
+        // check r
+
+    where the server can be accessed within the context, and is terminated when
+    the context is exited.
+    The port is automatically set using find_available_port, unless passed in as a command line arg.
+    The verbose flag is automatically set to True.
+    If an app_config is provided, then this function writes a temporary
+    yaml config file, which this server will read and parse.
+    """
+
+    command = ["cellxgene", "--no-upgrade-check", "launch", "--verbose"]
+    if "-p" in command_line_args:
+        port = int(command_line_args[command_line_args.index("-p") + 1])
+    elif "--port" in command_line_args:
+        port = int(command_line_args[command_line_args.index("--port") + 1])
+    else:
+        start = random.randint(DEFAULT_SERVER_PORT, 2 ** 16 - 1)
+        port = int(os.environ.get("CXG_SERVER_PORT", start))
+        port = find_available_port("localhost", port)
+        command += ["--port=%d" % port]
+
+    command += command_line_args
+
+    tempdir = None
+    if app_config:
+        tempdir = tempfile.TemporaryDirectory()
+        config_file = os.path.join(tempdir.name, "config.yaml")
+        app_config.write_config(config_file)
+        command.extend(["-c", config_file])
+
+    server = f"http://localhost:{port}"
+    ps = Popen(command, env=env)
+
+    for _ in range(10):
+        try:
+            requests.get(f"{server}/health")
+            break
+        except requests.exceptions.ConnectionError:
+            time.sleep(1)
+
+    if tempdir:
+        tempdir.cleanup()
+
+    return ps, server
+
+
+def stop_test_server(ps):
+    try:
+        ps.terminate()
+    except ProcessLookupError:
+        pass
+
+
+@contextmanager
+def test_server(command_line_args=[], app_config=None, env=None):
+    """A context to run the cellxgene server."""
+
+    ps, server = start_test_server(command_line_args, app_config, env)
+    try:
+        yield server
+    finally:
+        try:
+            stop_test_server(ps)
+        except ProcessLookupError:
+            pass

backend/test/test_server/unit/auth/test_auth.py

@@ -0,0 +1,89 @@
+import unittest
+
+import requests
+
+from backend.server.common.config.app_config import AppConfig
+from backend.test.test_server.unit import test_server
+from backend.test import H5AD_FIXTURE
+
+
+class AuthTest(unittest.TestCase):
+    def setUp(self):
+        self.dataset_datapath = H5AD_FIXTURE
+
+    def test_auth_none(self):
+        app_config = AppConfig()
+        app_config.update_server_config(app__flask_secret_key="secret")
+        app_config.update_server_config(authentication__type=None, single_dataset__datapath=self.dataset_datapath)
+        app_config.update_dataset_config(user_annotations__enable=False, user_annotations__gene_sets__readonly=True)
+
+        app_config.complete_config()
+
+        with test_server(app_config=app_config) as server:
+            session = requests.Session()
+            config = session.get(f"{server}/api/v0.2/config").json()
+            userinfo = session.get(f"{server}/api/v0.2/userinfo").json()
+            self.assertNotIn("authentication", config["config"])
+            self.assertIsNone(userinfo)
+
+    def test_auth_session(self):
+        app_config = AppConfig()
+        app_config.update_server_config(app__flask_secret_key="secret")
+        app_config.update_server_config(authentication__type="session", single_dataset__datapath=self.dataset_datapath)
+        app_config.update_dataset_config(user_annotations__enable=True)
+        app_config.complete_config()
+        with test_server(app_config=app_config) as server:
+            session = requests.Session()
+            config = session.get(f"{server}/api/v0.2/config").json()
+            userinfo = session.get(f"{server}/api/v0.2/userinfo").json()
+
+            self.assertFalse(config["config"]["authentication"]["requires_client_login"])
+            self.assertTrue(userinfo["userinfo"]["is_authenticated"])
+            self.assertEqual(userinfo["userinfo"]["username"], "anonymous")
+
+    def test_auth_test_single(self):
+        app_config = AppConfig()
+        app_config.update_server_config(app__flask_secret_key="secret")
+        app_config.update_server_config(
+            authentication__type="test",
+            single_dataset__datapath=self.dataset_datapath,
+            authentication__insecure_test_environment=True,
+        )
+
+        app_config.complete_config()
+
+        with test_server(app_config=app_config) as server:
+            session = requests.Session()
+            config = session.get(f"{server}/api/v0.2/config").json()
+            userinfo = session.get(f"{server}/api/v0.2/userinfo").json()
+            self.assertFalse(userinfo["userinfo"]["is_authenticated"])
+            self.assertIsNone(userinfo["userinfo"]["username"])
+            self.assertTrue(config["config"]["authentication"]["requires_client_login"])
+            self.assertTrue(config["config"]["parameters"]["annotations"])
+
+            login_uri = config["config"]["authentication"]["login"]
+            logout_uri = config["config"]["authentication"]["logout"]
+
+            self.assertEqual(login_uri, "/login")
+            self.assertEqual(logout_uri, "/logout")
+
+            response = session.get(f"{server}/{login_uri}")
+            # check that the login redirect worked
+            self.assertEqual(response.history[0].status_code, 302)
+            self.assertEqual(response.url, f"{server}/")
+
+            config = session.get(f"{server}/api/v0.2/config").json()
+            userinfo = session.get(f"{server}/api/v0.2/userinfo").json()
+            self.assertTrue(userinfo["userinfo"]["is_authenticated"])
+            self.assertEqual(userinfo["userinfo"]["username"], "test_account")
+            self.assertTrue(config["config"]["parameters"]["annotations"])
+
+            response = session.get(f"{server}/{logout_uri}")
+            # check that the logout redirect worked
+            self.assertEqual(response.history[0].status_code, 302)
+            self.assertEqual(response.url, f"{server}/")
+            config = session.get(f"{server}/api/v0.2/config").json()
+            userinfo = session.get(f"{server}/api/v0.2/userinfo").json()
+            self.assertFalse(userinfo["userinfo"]["is_authenticated"])
+            self.assertIsNone(userinfo["userinfo"]["username"])
+            self.assertTrue(config["config"]["parameters"]["annotations"])

backend/test/test_server/unit/cli/test_launch.py

@@ -0,0 +1,27 @@
+import filecmp
+import os
+import shutil
+import unittest
+
+import yaml
+
+from backend.server.default_config import default_config
+from backend.test import FIXTURES_ROOT
+
+
+class CLIPLaunchTests(unittest.TestCase):
+    tmp_dir = os.path.join(FIXTURES_ROOT, "dump_configs")
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        os.mkdir(cls.tmp_dir)
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        shutil.rmtree(cls.tmp_dir)
+
+    def test_dump_default_config(self):
+        os.system(f"cellxgene launch --dump-default-config > {self.tmp_dir}/test_config_dump.txt")
+        with open(f"{self.tmp_dir}/expected_config_dump.txt", "w") as expected_config:
+            expected_config.write(yaml.dump(default_config))
+        filecmp.cmp(f"{self.tmp_dir}/expected_config_dump.txt", f"{self.tmp_dir}/test_config_dump.txt")

backend/test/test_server/unit/cli/test_prepare.py

@@ -0,0 +1,15 @@
+import unittest
+
+import pandas as pd
+
+from backend.server.cli.prepare import make_index_unique
+
+
+class CLIPrepareTests(unittest.TestCase):
+    """ Test cases for CLI prepare logic """
+
+    def test_make_index_unique(self):
+        index = pd.Index(["SNORD113", "SNORD113", "SNORD113-1"])
+        result = make_index_unique(index)
+        expected = pd.Index(["SNORD113", "SNORD113-2", "SNORD113-1"])
+        self.assertTrue(all(left == right for left, right in zip(result.values, expected.values)))

backend/test/test_server/unit/cli/test_upgrade.py

@@ -0,0 +1,28 @@
+import unittest
+
+from backend.server.cli.upgrade import validate_version_str, split_version, version_gt
+
+
+class CLIUpgradeTests(unittest.TestCase):
+    """ Test cases for CLI logic """
+
+    def test_validate_version_str(self):
+        self.assertTrue(validate_version_str("0.1.2"))
+        self.assertTrue(validate_version_str("0.1.2-RC", release_only=False))
+        self.assertFalse(validate_version_str("0.1"))
+        self.assertFalse(validate_version_str("0.1.2.3"))
+        self.assertFalse(validate_version_str("0.1.2-RC"))
+
+    def test_split_version_str(self):
+        self.assertEqual(split_version("0.1.2"), [0, 1, 2])
+        with self.assertRaises(AttributeError):
+            split_version("0.1")
+
+    def test_assert_verstion_gt(self):
+        self.assertTrue(version_gt("1.0.0", "0.1.1"))
+        self.assertTrue(version_gt("0.1.0", "0.0.1"))
+        self.assertTrue(version_gt("0.0.1", "0.0.0"))
+        self.assertFalse(version_gt("0.0.0", "0.0.0"))
+        self.assertFalse(version_gt("0.0.0", "0.0.1"))
+        self.assertFalse(version_gt("0.0.1", "0.1.0"))
+        self.assertFalse(version_gt("0.1.1", "1.0.0"))

backend/test/test_server/unit/common/config/__init__.py

@@ -0,0 +1,228 @@
+import os
+import shutil
+import unittest
+import random
+from unittest import mock
+import yaml
+
+from backend.test import FIXTURES_ROOT
+
+
+def mockenv(**envvars):
+    return mock.patch.dict(os.environ, envvars)
+
+
+class ConfigTests(unittest.TestCase):
+    tmp_fixtures_directory = os.path.join(FIXTURES_ROOT, "tmp_dir")
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        shutil.rmtree(cls.tmp_fixtures_directory)
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        os.makedirs(cls.tmp_fixtures_directory)
+
+    def custom_server_config(
+        self,
+        verbose="false",
+        debug="false",
+        host="localhost",
+        port="null",
+        open_browser="false",
+        force_https="false",
+        flask_secret_key="secret",
+        auth_type="session",
+        insecure_test_environment="false",
+        index="false",
+        allowed_matrix_types=[],
+        max_cached_datasets=5,
+        timelimit_s=5,
+        dataset_datapath="null",
+        obs_names="null",
+        var_names="null",
+        about="null",
+        title="null",
+        data_locater_region_name="us-east-1",
+        anndata_backed="false",
+        column_request_max=32,
+        diffexp_cellcount_max="null",
+        config_file_name="server_config.yaml",
+    ):
+        configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
+        server_config_outline_path = os.path.join(FIXTURES_ROOT, "server_config_outline.py")
+        with open(server_config_outline_path, "r") as config_skeleton:
+            config = config_skeleton.read()
+            server_config = eval(config)
+        with open(configfile, "w") as server_config_file:
+            server_config_file.write(server_config)
+        return configfile
+
+    def custom_app_config(
+        self,
+        verbose="false",
+        debug="false",
+        host="localhost",
+        port="null",
+        open_browser="false",
+        force_https="false",
+        flask_secret_key="secret",
+        auth_type="session",
+        index="false",
+        allowed_matrix_types=[],
+        max_cached_datasets=5,
+        timelimit_s=5,
+        dataset_datapath="null",
+        obs_names="null",
+        var_names="null",
+        about="null",
+        title="null",
+        data_locater_region_name="us-east-1",
+        anndata_backed="false",
+        column_request_max=32,
+        diffexp_cellcount_max="null",
+        scripts=[],
+        inline_scripts=[],
+        authentication_enable="true",
+        max_categories=1000,
+        custom_colors="true",
+        enable_users_annotations="true",
+        annotation_type="local_file_csv",
+        db_uri="null",
+        hosted_file_directory="null",
+        local_file_csv_directory="null",
+        local_file_csv_file="null",
+        local_file_csv_gene_sets_file="null",
+        ontology_enabled="false",
+        obo_location="null",
+        gene_sets_readonly="false",
+        embedding_names=[],
+        enable_reembedding="false",
+        enable_difexp="true",
+        lfc_cutoff=0.01,
+        top_n=10,
+        environment=None,
+        aws_secrets_manager_region=None,
+        aws_secrets_manager_secrets=[],
+        config_file_name="app_config.yml",
+    ):
+        random_num = random.randrange(999999)
+        configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
+        server_config = self.custom_server_config(
+            verbose=verbose,
+            debug=debug,
+            host=host,
+            port=port,
+            open_browser=open_browser,
+            force_https=force_https,
+            flask_secret_key=flask_secret_key,
+            auth_type=auth_type,
+            index=index,
+            allowed_matrix_types=allowed_matrix_types,
+            max_cached_datasets=max_cached_datasets,
+            timelimit_s=timelimit_s,
+            dataset_datapath=dataset_datapath,
+            obs_names=obs_names,
+            var_names=var_names,
+            about=about,
+            title=title,
+            data_locater_region_name=data_locater_region_name,
+            anndata_backed=anndata_backed,
+            column_request_max=column_request_max,
+            diffexp_cellcount_max=diffexp_cellcount_max,
+            config_file_name=f"temp_server_config_{random_num}.yml",
+        )
+        dataset_config = self.custom_dataset_config(
+            scripts=scripts,
+            inline_scripts=inline_scripts,
+            authentication_enable=authentication_enable,
+            max_categories=max_categories,
+            custom_colors=custom_colors,
+            enable_users_annotations=enable_users_annotations,
+            annotation_type=annotation_type,
+            db_uri=db_uri,
+            hosted_file_directory=hosted_file_directory,
+            local_file_csv_directory=local_file_csv_directory,
+            local_file_csv_file=local_file_csv_file,
+            local_file_csv_gene_sets_file=local_file_csv_gene_sets_file,
+            ontology_enabled=ontology_enabled,
+            obo_location=obo_location,
+            gene_sets_readonly=gene_sets_readonly,
+            embedding_names=embedding_names,
+            enable_reembedding=enable_reembedding,
+            enable_difexp=enable_difexp,
+            lfc_cutoff=lfc_cutoff,
+            top_n=top_n,
+            config_file_name=f"temp_dataset_config_{random_num}.yml",
+        )
+        external_config = self.custom_external_config(
+            environment=environment,
+            aws_secrets_manager_region=aws_secrets_manager_region,
+            aws_secrets_manager_secrets=aws_secrets_manager_secrets,
+            config_file_name=f"temp_external_config_{random_num}.yml",
+        )
+
+        with open(configfile, "w") as app_config_file:
+            app_config_file.write(open(server_config).read())
+            app_config_file.write(open(dataset_config).read())
+            app_config_file.write(open(external_config).read())
+
+        return configfile
+
+    def custom_dataset_config(
+        self,
+        scripts=[],
+        inline_scripts=[],
+        authentication_enable="true",
+        max_categories=1000,
+        custom_colors="true",
+        enable_users_annotations="true",
+        annotation_type="local_file_csv",
+        db_uri="null",
+        hosted_file_directory="null",
+        local_file_csv_directory="null",
+        local_file_csv_file="null",
+        local_file_csv_gene_sets_file="null",
+        ontology_enabled="false",
+        obo_location="null",
+        gene_sets_readonly="false",
+        embedding_names=[],
+        enable_reembedding="false",
+        enable_difexp="true",
+        lfc_cutoff=0.01,
+        top_n=10,
+        config_file_name="dataset_config.yml",
+    ):
+        configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
+        dataset_config_outline_path = os.path.join(FIXTURES_ROOT, "dataset_config_outline.py")
+        with open(dataset_config_outline_path, "r") as config_skeleton:
+            config = config_skeleton.read()
+            dataset_config = eval(config)
+        with open(configfile, "w") as dataset_config_file:
+            dataset_config_file.write(dataset_config)
+
+        return configfile
+
+    def custom_external_config(
+        self,
+        environment=None,
+        aws_secrets_manager_region=None,
+        aws_secrets_manager_secrets=[],
+        config_file_name="external_config.yaml",
+    ):
+        # set to the default if environment is None
+        if environment is None:
+            environment = [
+                dict(name="CXG_SECRET_KEY", path=["server", "app", "flask_secret_key"], required=False),
+            ]
+        external_config = {
+            "external": {
+                "environment": environment,
+                "aws_secrets_manager": {"region": aws_secrets_manager_region, "secrets": aws_secrets_manager_secrets},
+            }
+        }
+
+        configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
+        with open(configfile, "w") as external_config_file:
+            yaml.dump(external_config, external_config_file)
+        return configfile

backend/test/test_server/unit/common/config/test_app_config.py

@@ -0,0 +1,153 @@
+import os
+import tempfile
+import unittest
+
+import yaml
+
+from backend.server.default_config import default_config
+from backend.server.common.config.app_config import AppConfig
+from backend.common.errors import ConfigurationError
+from backend.test.test_server.unit.common.config import ConfigTests
+from backend.test import FIXTURES_ROOT, H5AD_FIXTURE
+
+
+class AppConfigTest(ConfigTests):
+    def setUp(self):
+        self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
+        self.config = AppConfig()
+        self.config.update_server_config(app__flask_secret_key="secret")
+        self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE)
+        self.server_config = self.config.server_config
+        self.config.complete_config()
+
+        message_list = []
+
+        def noop(message):
+            message_list.append(message)
+
+        messagefn = noop
+        self.context = dict(messagefn=messagefn, messages=message_list)
+
+    def get_config(self, **kwargs):
+        file_name = self.custom_app_config(
+            dataset_datapath=H5AD_FIXTURE, config_file_name=self.config_file_name, **kwargs
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        return config
+
+    def test_get_default_config_correctly_reads_default_config_file(self):
+        app_default_config = AppConfig().default_config
+
+        expected_config = yaml.load(default_config, Loader=yaml.Loader)
+
+        server_config = app_default_config["server"]
+        dataset_config = app_default_config["dataset"]
+
+        expected_server_config = expected_config["server"]
+        expected_dataset_config = expected_config["dataset"]
+
+        self.assertDictEqual(app_default_config, expected_config)
+        self.assertDictEqual(server_config, expected_server_config)
+        self.assertDictEqual(dataset_config, expected_dataset_config)
+
+    def test_get_dataset_config_returns_dataset_config_for_single_datasets(self):
+        datapath = f"{FIXTURES_ROOT}/1e4dfec4-c0b2-46ad-a04e-ff3ffb3c0a8f.h5ad"
+        file_name = self.custom_app_config(dataset_datapath=datapath, config_file_name=self.config_file_name)
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+
+        self.assertEqual(config.get_dataset_config(), config.dataset_config)
+
+    def test_update_server_config_updates_server_config_and_config_status(self):
+        config = self.get_config()
+        config.complete_config()
+        config.check_config()
+        config.update_server_config(single_dataset__datapath=H5AD_FIXTURE)
+        with self.assertRaises(ConfigurationError):
+            config.server_config.check_config()
+
+    def test_write_config_outputs_yaml_with_all_config_vars(self):
+        config = self.get_config()
+        config.write_config(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml")
+        with open(f"{FIXTURES_ROOT}/tmp_dir/{self.config_file_name}", "r") as default_config:
+            default_config_yml = yaml.safe_load(default_config)
+
+        with open(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml", "r") as output_config:
+            output_config_yml = yaml.safe_load(output_config)
+        self.maxDiff = None
+        self.assertEqual(default_config_yml, output_config_yml)
+
+    def test_update_app_config(self):
+        config = AppConfig()
+        config.update_server_config(app__verbose=True, single_dataset__datapath="datapath")
+        vars = config.server_config.changes_from_default()
+        self.assertCountEqual(vars, [("app__verbose", True, False), ("single_dataset__datapath", "datapath", None)])
+
+        config = AppConfig()
+        config.update_dataset_config(app__scripts=(), app__inline_scripts=())
+        vars = config.server_config.changes_from_default()
+        self.assertCountEqual(vars, [])
+
+        config = AppConfig()
+        config.update_dataset_config(app__scripts=[], app__inline_scripts=[])
+        vars = config.dataset_config.changes_from_default()
+        self.assertCountEqual(vars, [])
+
+        config = AppConfig()
+        config.update_dataset_config(app__scripts=("a", "b"), app__inline_scripts=["c", "d"])
+        vars = config.dataset_config.changes_from_default()
+        self.assertCountEqual(vars, [("app__scripts", ["a", "b"], []), ("app__inline_scripts", ["c", "d"], [])])
+
+    def test_configfile_no_server_section(self):
+        # test a config file without a dataset section
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            configfile = os.path.join(tempdir, "config.yaml")
+            with open(configfile, "w") as fconfig:
+                config = """
+                dataset:
+                    user_annotations:
+                        enable: false
+                """
+                fconfig.write(config)
+
+            app_config = AppConfig()
+            app_config.update_from_config_file(configfile)
+            server_changes = app_config.server_config.changes_from_default()
+            dataset_changes = app_config.dataset_config.changes_from_default()
+            self.assertEqual(server_changes, [])
+            self.assertEqual(dataset_changes, [("user_annotations__enable", False, True)])
+
+    def test_simple_update_single_config_from_path_and_value(self):
+        """Update a simple config parameter"""
+
+        config = AppConfig()
+        config.server_config.single_dataset__datapath = "my/data/path"
+
+        # test simple value in server
+        config.update_single_config_from_path_and_value(["server", "app", "flask_secret_key"], "mysecret")
+        self.assertEqual(config.server_config.app__flask_secret_key, "mysecret")
+
+        # test simple value in default dataset
+        config.update_single_config_from_path_and_value(
+            ["dataset", "user_annotations", "ontology", "obo_location"], "dummy_location",
+        )
+        self.assertEqual(config.dataset_config.user_annotations__ontology__obo_location, "dummy_location")
+
+        # error checking
+        bad_paths = [
+            (
+                ["dataset", "does", "not", "exist"],
+                "unknown config parameter at path: '['dataset', 'does', 'not', 'exist']'",
+            ),
+            (["does", "not", "exist"], "path must start with 'server', or 'dataset'"),
+            ([], "path must start with 'server', or 'dataset'"),
+            ([1, 2, 3], "path must be a list of strings, got '[1, 2, 3]'"),
+            ("string", "path must be a list of strings, got 'string'"),
+        ]
+        for bad_path, error_message in bad_paths:
+            with self.assertRaises(ConfigurationError) as config_error:
+                config.update_single_config_from_path_and_value(bad_path, "value")
+
+            self.assertEqual(config_error.exception.message, error_message)

backend/test/test_server/unit/common/config/test_base_config.py

@@ -0,0 +1,63 @@
+import unittest
+
+from backend.server.common.config.app_config import AppConfig
+from backend.test import H5AD_FIXTURE
+from backend.common.errors import ConfigurationError
+from backend.test.test_server.unit.common.config import ConfigTests
+
+
+class BaseConfigTest(ConfigTests):
+    def setUp(self):
+        self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
+        self.config = AppConfig()
+        self.config.update_server_config(app__flask_secret_key="secret")
+        self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE)
+        self.server_config = self.config.server_config
+        self.config.complete_config()
+
+        message_list = []
+
+        def noop(message):
+            message_list.append(message)
+
+        messagefn = noop
+        self.context = dict(messagefn=messagefn, messages=message_list)
+
+    def get_config(self, **kwargs):
+        file_name = self.custom_app_config(
+            dataset_datapath=f"{H5AD_FIXTURE}", config_file_name=self.config_file_name, **kwargs
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        return config
+
+    def test_mapping_creation_returns_map_of_server_and_dataset_config(self):
+        config = AppConfig()
+        mapping = config.dataset_config.create_mapping(config.default_config)
+        self.assertIsNotNone(mapping["server__app__verbose"])
+        self.assertIsNotNone(mapping["dataset__presentation__max_categories"])
+        self.assertIsNotNone(mapping["dataset__user_annotations__ontology__obo_location"])
+
+    def test_changes_from_default_returns_list_of_nondefault_config_values(self):
+        config = self.get_config(verbose="true", lfc_cutoff=0.05)
+        server_changes = config.server_config.changes_from_default()
+        dataset_changes = config.dataset_config.changes_from_default()
+
+        self.assertEqual(
+            server_changes,
+            [
+                ("app__verbose", True, False),
+                ("app__flask_secret_key", "secret", None),
+                ("single_dataset__datapath", H5AD_FIXTURE, None),
+                ('data_locator__s3__region_name', 'us-east-1', True)
+            ],
+        )
+        self.assertEqual(dataset_changes, [("diffexp__lfc_cutoff", 0.05, 0.01)])
+
+    def test_check_config_throws_error_if_attr_has_not_been_checked(self):
+        config = self.get_config(verbose="true")
+        config.complete_config()
+        config.check_config()
+        config.update_server_config(app__verbose=False)
+        with self.assertRaises(ConfigurationError):
+            config.check_config()

backend/test/test_server/unit/common/config/test_dataset_config.py

@@ -0,0 +1,157 @@
+import os
+import tempfile
+
+import unittest
+from unittest.mock import patch
+
+from backend.server.common.annotations.local_file_csv import AnnotationsLocalFile
+from backend.server.common.config.app_config import AppConfig
+from backend.server.common.config.base_config import BaseConfig
+from backend.test import FIXTURES_ROOT, H5AD_FIXTURE
+
+from backend.common.errors import ConfigurationError
+from backend.test.test_server.unit.common.config import ConfigTests
+
+
+class TestDatasetConfig(ConfigTests):
+    def setUp(self):
+        self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
+        self.config = AppConfig()
+        self.config.update_server_config(app__flask_secret_key="secret")
+        self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE)
+        self.dataset_config = self.config.dataset_config
+        self.config.complete_config()
+        message_list = []
+
+        def noop(message):
+            message_list.append(message)
+
+        messagefn = noop
+        self.context = dict(messagefn=messagefn, messages=message_list)
+
+    def get_config(self, **kwargs):
+        file_name = self.custom_app_config(dataset_datapath=H5AD_FIXTURE, **kwargs)
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        return config
+
+    def test_init_datatset_config_sets_vars_from_config(self):
+        config = AppConfig()
+        self.assertEqual(config.dataset_config.presentation__max_categories, 1000)
+        self.assertEqual(config.dataset_config.user_annotations__type, "local_file_csv")
+        self.assertEqual(config.dataset_config.diffexp__lfc_cutoff, 0.01)
+        self.assertIsNone(config.dataset_config.user_annotations__ontology__obo_location)
+
+    @patch("backend.server.common.config.dataset_config.BaseConfig.validate_correct_type_of_configuration_attribute")
+    def test_complete_config_checks_all_attr(self, mock_check_attrs):
+        mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
+        self.dataset_config.complete_config(self.context)
+        self.assertIsNotNone(self.config.server_config.data_adaptor)
+        self.assertEqual(mock_check_attrs.call_count, 19)
+
+    def test_app_sets_script_vars(self):
+        config = self.get_config(scripts=["path/to/script"])
+        config.dataset_config.handle_app()
+
+        self.assertEqual(config.dataset_config.app__scripts, [{"src": "path/to/script"}])
+
+        config = self.get_config(scripts=[{"src": "path/to/script", "more": "different/script/path"}])
+        config.dataset_config.handle_app()
+        self.assertEqual(
+            config.dataset_config.app__scripts, [{"src": "path/to/script", "more": "different/script/path"}]
+        )
+
+        config = self.get_config(scripts=["path/to/script", "different/script/path"])
+        config.dataset_config.handle_app()
+        # TODO @madison -- is this the desired functionality?
+        self.assertEqual(
+            config.dataset_config.app__scripts, [{"src": "path/to/script"}, {"src": "different/script/path"}]
+        )
+
+        config = self.get_config(scripts=[{"more": "different/script/path"}])
+        with self.assertRaises(ConfigurationError):
+            config.dataset_config.handle_app()
+
+    def test_handle_user_annotations_ensures_auth_is_enabled_with_valid_auth_type(self):
+        config = self.get_config(enable_users_annotations="true", authentication_enable="false")
+        config.server_config.complete_config(self.context)
+        with self.assertRaises(ConfigurationError):
+            config.dataset_config.handle_user_annotations(self.context)
+
+        config = self.get_config(enable_users_annotations="true", authentication_enable="true", auth_type="pretend")
+        with self.assertRaises(ConfigurationError):
+            config.server_config.complete_config(self.context)
+
+    def test_handle_user_annotations__instantiates_user_annotations_class_correctly(self):
+        config = self.get_config(
+            enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv"
+        )
+        config.server_config.complete_config(self.context)
+        config.dataset_config.handle_user_annotations(self.context)
+        self.assertIsInstance(config.dataset_config.user_annotations, AnnotationsLocalFile)
+
+        config = self.get_config(
+            enable_users_annotations="true", authentication_enable="true", annotation_type="NOT_REAL"
+        )
+        config.server_config.complete_config(self.context)
+        with self.assertRaises(ConfigurationError):
+            config.dataset_config.handle_user_annotations(self.context)
+
+    def test_handle_local_file_csv_annotations__sets_dir_if_not_passed_in(self):
+        config = self.get_config(
+            enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv"
+        )
+        config.server_config.complete_config(self.context)
+        config.dataset_config.handle_local_file_csv_annotations(self.context)
+        self.assertIsInstance(config.dataset_config.user_annotations, AnnotationsLocalFile)
+        cwd = os.getcwd()
+        self.assertEqual(config.dataset_config.user_annotations._get_output_dir(), cwd)
+
+    def test_handle_embeddings__checks_data_file_types(self):
+        file_name = self.custom_app_config(
+            embedding_names=["name1", "name2"],
+            enable_reembedding="true",
+            dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad",
+            anndata_backed="true",
+            config_file_name=self.config_file_name,
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        config.server_config.complete_config(self.context)
+        with self.assertRaises(ConfigurationError):
+            config.dataset_config.handle_embeddings()
+
+    def test_handle_diffexp__raises_warning_for_large_datasets(self):
+        config = self.get_config(lfc_cutoff=0.02, enable_difexp="true", top_n=15)
+        config.server_config.complete_config(self.context)
+        config.dataset_config.handle_diffexp(self.context)
+        self.assertEqual(len(self.context["messages"]), 1)
+
+    def test_configfile_with_specialization(self):
+        # test that per_dataset_config config load the default config, then the specialized config
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            configfile = os.path.join(tempdir, "config.yaml")
+            with open(configfile, "w") as fconfig:
+                config = """
+                server:
+                    single_dataset:
+                        datapath: fake_datapath
+                dataset:
+                    user_annotations:
+                        enable: false
+                        type: local_file_csv
+                        local_file_csv:
+                            file: fake_file
+                            directory: fake_dir
+                """
+                fconfig.write(config)
+
+            app_config = AppConfig()
+            app_config.update_from_config_file(configfile)
+
+            test_config = app_config.dataset_config
+
+            # test config from default
+            self.assertEqual(test_config.user_annotations__type, "local_file_csv")
+            self.assertEqual(test_config.user_annotations__local_file_csv__file, "fake_file")

backend/test/test_server/unit/common/config/test_external_config.py

@@ -0,0 +1,216 @@
+import os
+from unittest.mock import patch
+
+import requests
+
+from backend.common.errors import ConfigurationError
+from backend.server.common.config.app_config import AppConfig
+from backend.test.test_server.unit import test_server
+from backend.test import FIXTURES_ROOT
+from backend.common.utils.type_conversion_utils import convert_string_to_value
+from backend.test.test_server.unit.common.config import ConfigTests
+
+
+class TestExternalConfig(ConfigTests):
+    def test_type_convert(self):
+        # The values from environment variables and aws secrets are returned as strings.
+        # These values need to be converted to the proper types.
+
+        self.assertEqual(convert_string_to_value("1"), int(1))
+        self.assertEqual(convert_string_to_value("1.1"), float(1.1))
+        self.assertEqual(convert_string_to_value("string"), "string")
+        self.assertEqual(convert_string_to_value("true"), True)
+        self.assertEqual(convert_string_to_value("True"), True)
+        self.assertEqual(convert_string_to_value("false"), False)
+        self.assertEqual(convert_string_to_value("False"), False)
+        self.assertEqual(convert_string_to_value("null"), None)
+        self.assertEqual(convert_string_to_value("None"), None)
+        self.assertEqual(convert_string_to_value("{'a':10, 'b':'string'}"), dict(a=int(10), b="string"))
+
+    def test_environment_variable(self):
+        configfile = self.custom_external_config(
+            environment=[
+                dict(name="DATAPATH", path=["server", "single_dataset", "datapath"], required=True),
+                dict(name="DIFFEXP", path=["dataset", "diffexp", "enable"], required=True),
+            ],
+            config_file_name="environment_external_config.yaml",
+        )
+
+        env = os.environ
+        env["DATAPATH"] = f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad"
+        env["DIFFEXP"] = "False"
+        with test_server(command_line_args=["-c", configfile], env=env) as server:
+            session = requests.Session()
+            response = session.get(f"{server}/api/v0.2/config")
+            data_config = response.json()
+            self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k-CSC-gz")
+            self.assertTrue(data_config["config"]["parameters"]["disable-diffexp"])
+
+        env["DATAPATH"] = f"{FIXTURES_ROOT}/a95c59b4-7f5d-4b80-ad53-a694834ca18b.h5ad"
+        env["DIFFEXP"] = "True"
+        with test_server(command_line_args=["-c", configfile], env=env) as server:
+            session = requests.Session()
+            response = session.get(f"{server}/api/v0.2/config")
+            data_config = response.json()
+            self.assertEqual(data_config["config"]["displayNames"]["dataset"], "a95c59b4-7f5d-4b80-ad53-a694834ca18b")
+            self.assertFalse(data_config["config"]["parameters"]["disable-diffexp"])
+
+    def test_environment_variable_errors(self):
+
+        # no name
+        app_config = AppConfig()
+        app_config.external_config.environment = [dict(required=True, path=["this", "is", "a", "path"])]
+        with self.assertRaises(ConfigurationError) as config_error:
+            app_config.complete_config()
+        self.assertEqual(config_error.exception.message, "environment: 'name' is missing")
+
+        # required has wrong type
+        app_config = AppConfig()
+        app_config.external_config.environment = [
+            dict(name="myenvar", required="optional", path=["this", "is", "a", "path"])
+        ]
+        with self.assertRaises(ConfigurationError) as config_error:
+            app_config.complete_config()
+        self.assertEqual(config_error.exception.message, "environment: 'required' must be a bool")
+
+        # no path
+        app_config = AppConfig()
+        app_config.external_config.environment = [dict(name="myenvar", required=True)]
+        with self.assertRaises(ConfigurationError) as config_error:
+            app_config.complete_config()
+        self.assertEqual(config_error.exception.message, "environment: 'path' is missing")
+
+        # required environment variable is not set
+        app_config = AppConfig()
+        app_config.external_config.environment = [
+            dict(name="THIS_ENV_IS_NOT_SET", required=True, path=["this", "is", "a", "path"])
+        ]
+        with self.assertRaises(ConfigurationError) as config_error:
+            app_config.complete_config()
+        self.assertEqual(config_error.exception.message, "required environment variable 'THIS_ENV_IS_NOT_SET' not set")
+
+    @patch("backend.server.common.config.external_config.get_secret_key")
+    def test_aws_secrets_manager(self, mock_get_secret_key):
+        mock_get_secret_key.return_value = {
+            "flask_secret_key": "mock_flask_secret_key",
+        }
+        configfile = self.custom_external_config(
+            aws_secrets_manager_region="us-west-2",
+            aws_secrets_manager_secrets=[
+                dict(
+                    name="my_secret",
+                    values=[
+                        dict(key="flask_secret_key", path=["server", "app", "flask_secret_key"], required=True),
+                    ],
+                )
+            ],
+            config_file_name="secret_external_config.yaml",
+        )
+
+        app_config = AppConfig()
+        app_config.update_from_config_file(configfile)
+        app_config.server_config.single_dataset__datapath = f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad"
+
+        app_config.complete_config()
+
+        self.assertEqual(app_config.server_config.app__flask_secret_key, "mock_flask_secret_key")
+
+    @patch("backend.server.common.config.external_config.get_secret_key")
+    def test_aws_secrets_manager_error(self, mock_get_secret_key):
+        mock_get_secret_key.return_value = {
+            "db_uri": "mock_db_uri",
+        }
+
+        # no region
+        app_config = AppConfig()
+        app_config.external_config.aws_secrets_manager__region = None
+        app_config.external_config.aws_secrets_manager__secrets = [
+            dict(name="secret1", values=[dict(key="key1", required=True, path=["this", "is", "my", "path"])])
+        ]
+        with self.assertRaises(ConfigurationError) as config_error:
+            app_config.complete_config()
+        self.assertEqual(
+            config_error.exception.message,
+            "Invalid type for attribute: aws_secrets_manager__region, expected type str, got NoneType",
+        )
+
+        # missing secret name
+        app_config = AppConfig()
+        app_config.external_config.aws_secrets_manager__region = "us-west-2"
+        app_config.external_config.aws_secrets_manager__secrets = [
+            dict(values=[dict(key="db_uri", required=True, path=["this", "is", "my", "path"])])
+        ]
+        with self.assertRaises(ConfigurationError) as config_error:
+            app_config.complete_config()
+        self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'name' is missing")
+
+        # secret name wrong type
+        app_config = AppConfig()
+        app_config.external_config.aws_secrets_manager__region = "us-west-2"
+        app_config.external_config.aws_secrets_manager__secrets = [
+            dict(name=1, values=[dict(key="db_uri", required=True, path=["this", "is", "my", "path"])])
+        ]
+        with self.assertRaises(ConfigurationError) as config_error:
+            app_config.complete_config()
+        self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'name' must be a string")
+
+        # missing values name
+        app_config = AppConfig()
+        app_config.external_config.aws_secrets_manager__region = "us-west-2"
+        app_config.external_config.aws_secrets_manager__secrets = [dict(name="mysecret")]
+        with self.assertRaises(ConfigurationError) as config_error:
+            app_config.complete_config()
+        self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'values' is missing")
+
+        # values wrong type
+        app_config = AppConfig()
+        app_config.external_config.aws_secrets_manager__region = "us-west-2"
+        app_config.external_config.aws_secrets_manager__secrets = [
+            dict(name="mysecret", values=dict(key="db_uri", required=True, path=["this", "is", "my", "path"]))
+        ]
+        with self.assertRaises(ConfigurationError) as config_error:
+            app_config.complete_config()
+        self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'values' must be a list")
+
+        # entry missing key
+        app_config = AppConfig()
+        app_config.external_config.aws_secrets_manager__region = "us-west-2"
+        app_config.external_config.aws_secrets_manager__secrets = [
+            dict(name="mysecret", values=[dict(required=True, path=["this", "is", "my", "path"])])
+        ]
+        with self.assertRaises(ConfigurationError) as config_error:
+            app_config.complete_config()
+        self.assertEqual(config_error.exception.message, "missing 'key' in secret values: mysecret")
+
+        # entry required is wrong type
+        app_config = AppConfig()
+        app_config.external_config.aws_secrets_manager__region = "us-west-2"
+        app_config.external_config.aws_secrets_manager__secrets = [
+            dict(name="mysecret", values=[dict(key="db_uri", required="optional", path=["this", "is", "my", "path"])])
+        ]
+        with self.assertRaises(ConfigurationError) as config_error:
+            app_config.complete_config()
+        self.assertEqual(config_error.exception.message, "wrong type for 'required' in secret values: mysecret")
+
+        # entry missing path
+        app_config = AppConfig()
+        app_config.external_config.aws_secrets_manager__region = "us-west-2"
+        app_config.external_config.aws_secrets_manager__secrets = [
+            dict(name="mysecret", values=[dict(key="db_uri", required=True)])
+        ]
+        with self.assertRaises(ConfigurationError) as config_error:
+            app_config.complete_config()
+        self.assertEqual(config_error.exception.message, "missing 'path' in secret values: mysecret")
+
+        # secret missing required key
+        app_config = AppConfig()
+        app_config.external_config.aws_secrets_manager__region = "us-west-2"
+        app_config.external_config.aws_secrets_manager__secrets = [
+            dict(
+                name="mysecret",
+                values=[dict(key="KEY_DOES_NOT_EXIST", required=True, path=["this", "is", "a", "path"])],
+            )
+        ]
+        with self.assertRaises(ConfigurationError) as config_error:
+            app_config.complete_config()
+        self.assertEqual(config_error.exception.message, "required secret 'mysecret:KEY_DOES_NOT_EXIST' not set")

backend/test/test_server/unit/common/config/test_server_config.py

@@ -0,0 +1,122 @@
+import os
+import unittest
+from unittest import mock
+from unittest.mock import patch
+
+from backend.server.common.config.base_config import BaseConfig
+from backend.test import H5AD_FIXTURE
+
+from backend.server.common.config.app_config import AppConfig
+from backend.common.errors import ConfigurationError
+from backend.test.test_server.unit.common.config import ConfigTests
+
+
+def mockenv(**envvars):
+    return mock.patch.dict(os.environ, envvars)
+
+
+class TestServerConfig(ConfigTests):
+    def setUp(self):
+        self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
+        self.config = AppConfig()
+        self.config.update_server_config(app__flask_secret_key="secret")
+        self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE)
+        self.server_config = self.config.server_config
+        self.config.complete_config()
+
+        message_list = []
+
+        def noop(message):
+            message_list.append(message)
+
+        messagefn = noop
+        self.context = dict(messagefn=messagefn, messages=message_list)
+
+    def get_config(self, **kwargs):
+        file_name = self.custom_app_config(
+            dataset_datapath=f"{H5AD_FIXTURE}", config_file_name=self.config_file_name, **kwargs
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        return config
+
+    def test_init_raises_error_if_default_config_is_invalid(self):
+        invalid_config = self.get_config(port="not_valid")
+        with self.assertRaises(ConfigurationError):
+            invalid_config.complete_config()
+
+    @patch("backend.server.common.config.server_config.BaseConfig.validate_correct_type_of_configuration_attribute")
+    def test_complete_config_checks_all_attr(self, mock_check_attrs):
+        mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
+        self.server_config.complete_config(self.context)
+        self.assertEqual(mock_check_attrs.call_count, 20)
+
+    def test_handle_app__throws_error_if_port_doesnt_exist(self):
+        config = self.get_config(port=99999999)
+        with self.assertRaises(ConfigurationError):
+            config.server_config.handle_app(self.context)
+
+    @patch("backend.server.common.config.server_config.discover_s3_region_name")
+    def test_handle_data_locator_works_for_default_types(self, mock_discover_region_name):
+        mock_discover_region_name.return_value = None
+        # Default config
+        self.assertEqual(self.config.server_config.data_locator__s3__region_name, None)
+        # hard coded
+        config = self.get_config()
+        self.assertEqual(config.server_config.data_locator__s3__region_name, "us-east-1")
+        # incorrectly formatted
+        datapath = "s3://shouldnt/work"
+        file_name = self.custom_app_config(
+            dataset_datapath=datapath, config_file_name=self.config_file_name, data_locater_region_name="true"
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        with self.assertRaises(ConfigurationError):
+            config.server_config.handle_data_locator()
+
+    def test_handle_app___can_use_envar_port(self):
+        config = self.get_config(port=24)
+        self.assertEqual(config.server_config.app__port, 24)
+
+        # Note if the port is set in the config file it will NOT be overwritten by a different envvar
+        os.environ["CXG_SERVER_PORT"] = "4008"
+        self.config = AppConfig()
+        self.config.update_server_config(app__flask_secret_key="secret")
+        self.config.server_config.handle_app(self.context)
+        self.assertEqual(self.config.server_config.app__port, 4008)
+        del os.environ["CXG_SERVER_PORT"]
+
+    def test_handle_app__can_get_secret_key_from_envvar_or_config_file_with_envvar_given_preference(self):
+        config = self.get_config(flask_secret_key="KEY_FROM_FILE")
+        self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_FILE")
+
+        os.environ["CXG_SECRET_KEY"] = "KEY_FROM_ENV"
+        config.external_config.handle_environment(self.context)
+        self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_ENV")
+
+    def test_config_for_single_dataset(self):
+        file_name = self.custom_app_config(
+            config_file_name="single_dataset.yml", dataset_datapath=f"{H5AD_FIXTURE}"
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        config.server_config.handle_single_dataset(self.context)
+
+        file_name = self.custom_app_config(
+            config_file_name="single_dataset_with_about.yml",
+            about="www.cziscience.com",
+            dataset_datapath=f"{H5AD_FIXTURE}",
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        with self.assertRaises(ConfigurationError):
+            config.server_config.handle_single_dataset(self.context)
+
+    def test_test_auth_only_in_insecure(self):
+
+        config = self.get_config(auth_type="test")
+        with self.assertRaises(ConfigurationError):
+            config.complete_config()
+
+        config.update_server_config(authentication__insecure_test_environment=True)
+        config.complete_config()

backend/test/test_server/unit/common/test_api.py

@@ -0,0 +1,823 @@
+import shutil
+import time
+import unittest
+import zlib
+from http import HTTPStatus
+import tempfile
+from os import path
+
+import pandas as pd
+import requests
+
+import backend.test.decode_fbs as decode_fbs
+from backend.server.data_common.matrix_loader import MatrixDataType
+from backend.test.test_server.unit import (
+    data_with_tmp_annotations,
+    make_fbs,
+    start_test_server,
+    stop_test_server,
+)
+from backend.test.fixtures.fixtures import pbmc3k_colors
+from backend.test import PROJECT_ROOT, FIXTURES_ROOT
+
+BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
+
+
+# TODO (mweiden): remove ANNOTATIONS_ENABLED and Annotation subclasses when annotations are no longer experimental
+
+
+class EndPoints(object):
+    ANNOTATIONS_ENABLED = True
+    GENESETS_READONLY = False
+
+    def test_initialize(self):
+        endpoint = "schema"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        self.assertEqual(result_data["schema"]["dataframe"]["nObs"], 2638)
+        self.assertEqual(len(result_data["schema"]["annotations"]["obs"]), 2)
+        self.assertEqual(
+            len(result_data["schema"]["annotations"]["obs"]["columns"]), 6 if self.ANNOTATIONS_ENABLED else 5
+        )
+
+    def test_config(self):
+        endpoint = "config"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        self.assertIn("library_versions", result_data["config"])
+        self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k")
+        self.assertIsNotNone(result_data["config"]["parameters"])
+
+    def test_get_layout_fbs(self):
+        endpoint = "layout/obs"
+        url = f"{self.URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 8)
+        self.assertIsNotNone(df["columns"])
+        self.assertSetEqual(
+            set(df["col_idx"]),
+            {"pca_0", "pca_1", "tsne_0", "tsne_1", "umap_0", "umap_1", "draw_graph_fr_0", "draw_graph_fr_1"},
+        )
+        self.assertIsNone(df["row_idx"])
+        self.assertEqual(len(df["columns"]), df["n_cols"])
+
+    def test_put_layout_fbs(self):
+        # first check that re-embedding is turned on
+        result = self.session.get(f"{self.URL_BASE}config")
+        config_data = result.json()
+        re_embed = config_data["config"]["parameters"]["enable-reembedding"]
+        if not re_embed:
+            return
+        # attempt to reembed with umap over 100 cells.
+        endpoint = "layout/obs"
+        url = f"{self.URL_BASE}{endpoint}"
+        data = {}
+        data["filter"] = {}
+        data["filter"]["obs"] = {}
+        data["filter"]["obs"]["index"] = list(range(100))
+        data["method"] = "umap"
+        result = self.session.put(url, json=data)
+
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        result_data = result.json()
+        self.assertIsInstance(result_data, dict)
+        self.assertEqual(result_data["type"], "float32")
+        self.assertTrue(result_data["name"].startswith("reembed:umap_"))
+        self.assertIsInstance(result_data["dims"], list)
+        self.assertEqual(len(result_data["dims"]), 2)
+        dims = result_data["dims"]
+        self.assertTrue(dims[0].startswith("reembed:umap_") and dims[0].endswith("_0"))
+        self.assertTrue(dims[1].startswith("reembed:umap_") and dims[1].endswith("_1"))
+
+    def test_bad_filter(self):
+        endpoint = "data/var"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.put(url, json=BAD_FILTER)
+        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
+
+    def test_get_annotations_obs_fbs(self):
+        endpoint = "annotations/obs"
+        url = f"{self.URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 6 if self.ANNOTATIONS_ENABLED else 5)
+        self.assertIsNotNone(df["columns"])
+        self.assertIsNone(df["row_idx"])
+        self.assertEqual(len(df["columns"]), df["n_cols"])
+        obs_index_col_name = self.schema["schema"]["annotations"]["obs"]["index"]
+        self.assertCountEqual(
+            df["col_idx"],
+            [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"]
+            + (["cluster-test"] if self.ANNOTATIONS_ENABLED else []),
+        )
+
+    def test_get_annotations_obs_keys_fbs(self):
+        endpoint = "annotations/obs"
+        query = "annotation-name=n_genes&annotation-name=percent_mito"
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 2)
+        self.assertIsNotNone(df["columns"])
+        self.assertIsNone(df["row_idx"])
+        self.assertEqual(len(df["columns"]), df["n_cols"])
+        self.assertCountEqual(df["col_idx"], ["n_genes", "percent_mito"])
+
+    def test_get_annotations_obs_error(self):
+        endpoint = "annotations/obs"
+        query = "annotation-name=notakey"
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
+
+    def test_get_annotations_var_fbs(self):
+        endpoint = "annotations/var"
+        url = f"{self.URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 1838)
+        self.assertEqual(df["n_cols"], 2)
+        self.assertIsNotNone(df["columns"])
+        self.assertIsNone(df["row_idx"])
+        self.assertEqual(len(df["columns"]), df["n_cols"])
+        var_index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
+        self.assertCountEqual(df["col_idx"], [var_index_col_name, "n_cells"])
+
+    def test_get_annotations_var_keys_fbs(self):
+        endpoint = "annotations/var"
+        query = "annotation-name=n_cells"
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 1838)
+        self.assertEqual(df["n_cols"], 1)
+        self.assertIsNotNone(df["columns"])
+        self.assertIsNone(df["row_idx"])
+        self.assertEqual(len(df["columns"]), df["n_cols"])
+        self.assertCountEqual(df["col_idx"], ["n_cells"])
+
+    def test_get_annotations_var_error(self):
+        endpoint = "annotations/var"
+        query = "annotation-name=notakey"
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
+
+    def test_data_mimetype_error(self):
+        endpoint = "data/var"
+        header = {"Accept": "xxx"}
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.put(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.NOT_ACCEPTABLE)
+
+    def test_fbs_default(self):
+        endpoint = "data/var"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.put(url)
+        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
+
+        filter = {"filter": {"var": {"index": [0, 1, 4]}}}
+        result = self.session.put(url, json=filter)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+
+    def test_data_put_fbs(self):
+        endpoint = "data/var"
+        url = f"{self.URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.put(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
+
+    def test_data_get_fbs(self):
+        endpoint = "data/var"
+        url = f"{self.URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
+
+    def test_data_put_filter_fbs(self):
+        endpoint = "data/var"
+        url = f"{self.URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        filter = {"filter": {"var": {"index": [0, 1, 4]}}}
+        result = self.session.put(url, headers=header, json=filter)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 3)
+        self.assertIsNotNone(df["columns"])
+        self.assertIsNone(df["row_idx"])
+        self.assertEqual(len(df["columns"]), df["n_cols"])
+        self.assertListEqual(df["col_idx"].tolist(), [0, 1, 4])
+
+    def test_data_get_filter_fbs(self):
+        index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
+        endpoint = "data/var"
+        query = f"var:{index_col_name}=SIK1"
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 1)
+
+    def test_data_get_unknown_filter_fbs(self):
+        index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
+        endpoint = "data/var"
+        query = f"var:{index_col_name}=UNKNOWN"
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 0)
+
+    def test_data_put_single_var(self):
+        endpoint = "data/var"
+        url = f"{self.URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
+        var_filter = {"filter": {"var": {"annotation_value": [{"name": index_col_name, "values": ["RER1"]}]}}}
+        result = self.session.put(url, headers=header, json=var_filter)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 1)
+
+    def test_colors(self):
+        endpoint = "colors"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        self.assertEqual(result_data, pbmc3k_colors)
+
+    def test_static(self):
+        endpoint = "static"
+        file = "assets/favicon.ico"
+        url = f"{self.server}/{endpoint}/{file}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+
+    def test_genesets_config(self):
+        result = self.session.get(f"{self.URL_BASE}config")
+        config_data = result.json()
+        params = config_data["config"]["parameters"]
+        annotations_genesets = params["annotations_genesets"]
+        annotations_genesets_readonly = params["annotations_genesets_readonly"]
+        annotations_genesets_summary_methods = params["annotations_genesets_summary_methods"]
+        self.assertTrue(annotations_genesets)
+        self.assertEqual(annotations_genesets_readonly, self.GENESETS_READONLY)
+        self.assertEqual(annotations_genesets_summary_methods, ["mean"])
+
+    def test_get_genesets(self):
+        endpoint = "genesets"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.get(url, headers={"Accept": "application/json"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        self.assertIsNotNone(result_data["genesets"])
+
+    def _setupClass(child_class, command_line):
+        child_class.ps, child_class.server = start_test_server(command_line)
+        child_class.URL_BASE = f"{child_class.server}/api/v0.2/"
+        child_class.session = requests.Session()
+        for i in range(90):
+            try:
+                result = child_class.session.get(f"{child_class.URL_BASE}schema")
+                child_class.schema = result.json()
+            except requests.exceptions.ConnectionError:
+                time.sleep(1)
+
+
+class EndPointsAnnotations(EndPoints):
+    def test_get_schema_existing_writable(self):
+        self._test_get_schema_writable("cluster-test")
+
+    def test_get_user_annotations_existing_obs_keys_fbs(self):
+        self._test_get_user_annotations_obs_keys_fbs(
+            "cluster-test",
+            {"unassigned", "one", "two", "three", "four", "five", "six", "seven"},
+        )
+
+    def test_put_user_annotations_obs_fbs(self):
+        endpoint = "annotations/obs"
+        query = "annotation-collection-name=test_annotations"
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        n_rows = self.data.get_shape()[0]
+        fbs = make_fbs({"cat_A": pd.Series(["label_A"] * n_rows, dtype="category")})
+        result = self.session.put(url, data=zlib.compress(fbs))
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        self.assertEqual(result.json(), {"status": "OK"})
+        self._test_get_schema_writable("cat_A")
+        self._test_get_user_annotations_obs_keys_fbs("cat_A", {"label_A"})
+
+    def _test_get_user_annotations_obs_keys_fbs(self, annotation_name, columns):
+        endpoint = "annotations/obs"
+        query = f"annotation-name={annotation_name}"
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 1)
+        self.assertListEqual(df["col_idx"], [annotation_name])
+        self.assertEqual(set(df["columns"][0]), columns)
+        self.assertIsNone(df["row_idx"])
+        self.assertEqual(len(df["columns"]), df["n_cols"])
+
+    def _test_get_schema_writable(self, cluster_name):
+        endpoint = "schema"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        columns = result_data["schema"]["annotations"]["obs"]["columns"]
+        matching_columns = [c for c in columns if c["name"] == cluster_name]
+        self.assertEqual(len(matching_columns), 1)
+        self.assertTrue(matching_columns[0]["writable"])
+
+
+class EndPointsAnndata(unittest.TestCase, EndPoints):
+    """Test Case for endpoints"""
+
+    ANNOTATIONS_ENABLED = False
+    GENESETS_READONLY = True
+
+    @classmethod
+    def setUpClass(cls):
+        cls._setupClass(
+            cls,
+            [
+                f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
+                "--disable-annotations",
+                "--disable-gene-sets-save",
+                "--experimental-enable-reembedding",
+            ],
+        )
+
+    @classmethod
+    def tearDownClass(cls):
+        stop_test_server(cls.ps)
+
+    @property
+    def annotations_enabled(self):
+        return False
+
+    def test_diff_exp(self):
+        endpoint = "diffexp/obs"
+        url = f"{self.URL_BASE}{endpoint}"
+        params = {
+            "mode": "topN",
+            "set1": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["NK cells"]}]}}},
+            "set2": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["CD8 T cells"]}]}}},
+            "count": 7,
+        }
+        result = self.session.post(url, json=params)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        self.assertEqual(len(result_data), 7)
+
+    def test_diff_exp_indices(self):
+        endpoint = "diffexp/obs"
+        url = f"{self.URL_BASE}{endpoint}"
+        params = {
+            "mode": "topN",
+            "count": 10,
+            "set1": {"filter": {"obs": {"index": [[0, 500]]}}},
+            "set2": {"filter": {"obs": {"index": [[500, 1000]]}}},
+        }
+        result = self.session.post(url, json=params)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        self.assertEqual(len(result_data), 10)
+
+
+class EndPointsAnndataAnnotations(unittest.TestCase, EndPointsAnnotations):
+    """Test Case for endpoints"""
+
+    ANNOTATIONS_ENABLED = True
+    GENESETS_READONLY = False
+
+    @classmethod
+    def setUpClass(cls):
+        cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations(
+            MatrixDataType.H5AD, annotations_fixture=True
+        )
+        cls._setupClass(cls, ["--annotations-file", cls.annotations.label_output_file, cls.data.get_location()])
+
+    @classmethod
+    def tearDownClass(cls):
+        shutil.rmtree(cls.tmp_dir)
+        stop_test_server(cls.ps)
+
+
+class EndPointsAnnDataGenesets(unittest.TestCase, EndPoints):
+    ANNOTATIONS_ENABLED = False
+    GENESETS_READONLY = False
+
+    @classmethod
+    def setUpClass(cls):
+        cls.tmp_dir = tempfile.mkdtemp()
+        genesets_file = path.join(cls.tmp_dir, "test_genesets.csv")
+        shutil.copyfile(f"{FIXTURES_ROOT}/pbmc3k-genesets.csv", genesets_file)
+        cls._setupClass(
+            cls,
+            [
+                f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
+                "--disable-annotations",
+                "--gene-sets-file",
+                genesets_file,
+            ],
+        )
+
+    @classmethod
+    def tearDownClass(cls):
+        shutil.rmtree(cls.tmp_dir)
+        stop_test_server(cls.ps)
+
+    def test_get_genesets_json(self):
+        endpoint = "genesets"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.get(url, headers={"Accept": "application/json"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        self.assertIsNotNone(result_data["genesets"])
+        self.assertIsNotNone(result_data["tid"])
+
+        self.assertEqual(
+            result_data,
+            {
+                "genesets": [
+                    {
+                        "genes": [
+                            {"gene_description": "a gene_description", "gene_symbol": "F5"},
+                            {"gene_description": "", "gene_symbol": "SUMO3"},
+                            {"gene_description": "", "gene_symbol": "SRM"},
+                        ],
+                        "geneset_description": "a description",
+                        "geneset_name": "first gene set name",
+                    },
+                    {
+                        "genes": [
+                            {"gene_description": "", "gene_symbol": "RER1"},
+                            {"gene_description": "", "gene_symbol": "SIK1"},
+                        ],
+                        "geneset_description": "",
+                        "geneset_name": "second gene set",
+                    },
+                    {"genes": [], "geneset_description": "", "geneset_name": "third gene set"},
+                    {"genes": [], "geneset_description": "fourth description", "geneset_name": "fourth_gene_set"},
+                    {"genes": [], "geneset_description": "", "geneset_name": "fifth_dataset"},
+                    {
+                        "genes": [
+                            {"gene_description": "", "gene_symbol": "ACD"},
+                            {"gene_description": "", "gene_symbol": "AATF"},
+                            {"gene_description": "", "gene_symbol": "F5"},
+                            {"gene_description": "", "gene_symbol": "PIGU"},
+                        ],
+                        "geneset_description": "",
+                        "geneset_name": "summary test",
+                    },
+                ],
+                "tid": 0,
+            },
+        )
+
+    def test_get_genesets_csv(self):
+        endpoint = "genesets"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.get(url, headers={"Accept": "text/csv"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "text/csv")
+        self.assertEqual(
+            result.text,
+            """gene_set_name,gene_set_description,gene_symbol,gene_description\r
+first gene set name,a description,F5,a gene_description\r
+first gene set name,a description,SUMO3,\r
+first gene set name,a description,SRM,\r
+second gene set,,RER1,\r
+second gene set,,SIK1,\r
+third gene set,,,\r
+fourth_gene_set,fourth description,,\r
+fifth_dataset,,,\r
+summary test,,ACD,\r
+summary test,,AATF,\r
+summary test,,F5,\r
+summary test,,PIGU,\r
+""",
+        )
+
+    def test_put_genesets(self):
+        endpoint = "genesets"
+        url = f"{self.URL_BASE}{endpoint}"
+
+        # assume we start with TID 0
+        result = self.session.get(url, headers={"Accept": "application/json"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.json()["tid"], 0)
+
+        test1 = {"tid": 3, "genesets": []}
+        result = self.session.put(url, json=test1)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        result = self.session.get(url, headers={"Accept": "application/json"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.json(), test1)
+
+        # stale TID
+        result = self.session.put(url, json=test1)
+        self.assertEqual(result.status_code, HTTPStatus.NOT_FOUND)
+
+        test2 = {
+            "tid": 4,
+            "genesets": [
+                {"geneset_name": "foobar", "genes": []},
+                {"geneset_name": "contains a space", "genes": []},
+                {"geneset_name": "contains_weird_characters: #$%^&*()_+=-!@<>,./?';:\"[]{}|\\", "genes": []},
+            ],
+        }
+        test2_response = {
+            "tid": 4,
+            "genesets": [
+                {"geneset_name": "foobar", "geneset_description": "", "genes": []},
+                {"geneset_name": "contains a space", "geneset_description": "", "genes": []},
+                {
+                    "geneset_name": "contains_weird_characters: #$%^&*()_+=-!@<>,./?';:\"[]{}|\\",
+                    "geneset_description": "",
+                    "genes": [],
+                },
+            ],
+        }
+        result = self.session.put(url, json=test2)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        result = self.session.get(url, headers={"Accept": "application/json"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.json(), test2_response)
+
+        test3 = {
+            "tid": 5,
+            "genesets": [
+                {
+                    "geneset_name": "foobar",
+                    "geneset_description": "",
+                    "genes": [
+                        {
+                            "gene_symbol": "F5",
+                            "gene_description": "",
+                        }
+                    ],
+                }
+            ],
+        }
+        result = self.session.put(url, json=test3)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        result = self.session.get(url, headers={"Accept": "application/json"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.json(), test3)
+
+    def test_put_genesets_malformed(self):
+        """ test malformed submissions that we expect the backend to catch/tolerate """
+        endpoint = "genesets"
+        url = f"{self.URL_BASE}{endpoint}"
+
+        result = self.session.get(url, headers={"Accept": "application/json"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        original_data = result.json()
+        tid = original_data["tid"]
+
+        def test_case(test, expected_code, original_data):
+            """ check for expected error AND that no change was made to the original state """
+            result = self.session.put(url, json=test)
+            self.assertEqual(result.status_code, expected_code)
+            result = self.session.get(url, headers={"Accept": "application/json"})
+            self.assertEqual(result.status_code, HTTPStatus.OK)
+            self.assertEqual(result.json(), original_data)
+
+        # missing or malformed genesets
+        test_case(
+            {"tid": tid + 1},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+        test_case(
+            {"tid": tid + 1, "genesets": 99},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+
+        # illegal geneset_name
+        test_case(
+            {"tid": tid + 1, "genesets": [{"geneset_name": " foo", "genes": []}]},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+        test_case(
+            {"tid": tid + 1, "genesets": [{"geneset_name": "foo ", "genes": []}]},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+        test_case(
+            {"tid": tid + 1, "genesets": [{"geneset_name": "f  oo", "genes": []}]},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+        test_case(
+            {"tid": tid + 1, "genesets": [{"geneset_name": "f\too", "genes": []}]},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+        test_case(
+            {"tid": tid + 1, "genesets": [{"geneset_name": "f\roo", "genes": []}]},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+        test_case(
+            {"tid": tid + 1, "genesets": [{"geneset_name": "f\noo", "genes": []}]},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+        test_case(
+            {"tid": tid + 1, "genesets": [{"geneset_name": "f\voo", "genes": []}]},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+
+        # duplicate geneset_name
+        test_case(
+            {
+                "tid": tid + 1,
+                "genesets": [
+                    {"geneset_name": "foo", "genes": []},
+                    {"geneset_name": "foo", "genes": []},
+                ],
+            },
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+
+        # missing geneset_name
+        test_case(
+            {"tid": tid + 1, "genesets": [{"genes": []}]},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+
+        # non-numeric TID
+        test_case(
+            {"tid": [], "genesets": [{"geneset_name": "foo", "genes": []}]},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+        test_case(
+            {"tid": None, "genesets": [{"geneset_name": "foo", "genes": []}]},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+        test_case(
+            {"tid": "not a number", "genesets": [{"geneset_name": "foo", "genes": []}]},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+
+        # duplicate gene_symbol
+        test_case(
+            {
+                "tid": "not a number",
+                "genesets": [{"geneset_name": "foo", "genes": [{"gene_symbol": "SIK1"}, {"gene_symbol": "SIK1"}]}],
+            },
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+
+        # gene_symbol is not a string
+        test_case(
+            {
+                "tid": "not a number",
+                "genesets": [{"geneset_name": "foo", "genes": [{"gene_symbol": 99}]}],
+            },
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+
+    def test_get_geneset_summary(self):
+        endpoint = "geneset_summary?geneset_name=summary%20test&method=mean"
+        url = f"{self.URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 1)
+        self.assertEqual(df["col_idx"], ["summary test"])
+        self.assertAlmostEqual(df["columns"][0][0], -0.19863907)
+
+    def test_get_geneset_summary_default_method(self):
+        endpoint = "geneset_summary?geneset_name=summary%20test"
+        url = f"{self.URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 1)
+        self.assertEqual(df["col_idx"], ["summary test"])
+        self.assertAlmostEqual(df["columns"][0][0], -0.19863907)
+
+    def test_get_geneset_summary_check_tid(self):
+        # get the TID
+        result = self.session.get(f"{self.URL_BASE}genesets", headers={"Accept": "application/json"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        tid = result.json()["tid"]
+
+        # current tid
+        endpoint = f"geneset_summary?geneset_name=summary%20test&tid={tid}"
+        result = self.session.get(f"{self.URL_BASE}{endpoint}", headers={"Accept": "application/octet-stream"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+
+        # future tid
+        endpoint = f"geneset_summary?geneset_name=summary%20test&tid={tid+1}"
+        result = self.session.get(f"{self.URL_BASE}{endpoint}", headers={"Accept": "application/octet-stream"})
+        self.assertEqual(result.status_code, HTTPStatus.NOT_FOUND)
+
+        # past tid
+        endpoint = f"geneset_summary?geneset_name=summary%20test&tid={tid-1}"
+        result = self.session.get(f"{self.URL_BASE}{endpoint}", headers={"Accept": "application/octet-stream"})
+        self.assertEqual(result.status_code, HTTPStatus.NOT_FOUND)
+
+        # No tid - ie, skip check
+        endpoint = "geneset_summary?geneset_name=summary%20test"
+        result = self.session.get(f"{self.URL_BASE}{endpoint}", headers={"Accept": "application/octet-stream"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+
+    def test_get_geneset_summary_edge_cases(self):
+        # attempt to summarize _all_ genesets, including edge cases with zero or one gene
+        result = self.session.get(f"{self.URL_BASE}genesets", headers={"Accept": "application/json"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        geneset_names = [gs["geneset_name"] for gs in result.json()["genesets"]]
+
+        for gs in geneset_names:
+            endpoint = f"geneset_summary?geneset_name={gs}"
+            result = self.session.get(f"{self.URL_BASE}{endpoint}", headers={"Accept": "application/octet-stream"})
+            self.assertEqual(result.status_code, HTTPStatus.OK)
+            self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+            df = decode_fbs.decode_matrix_FBS(result.content)
+            self.assertEqual(df["n_rows"], 2638)
+            self.assertEqual(df["n_cols"], 1)
+            self.assertEqual(df["col_idx"], [gs])
+
+    def test_get_geneset_error_handling(self):
+        # no geneset
+        endpoint = "geneset_summary"
+        result = self.session.get(f"{self.URL_BASE}{endpoint}", headers={"Accept": "application/octet-stream"})
+        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
+
+        # unknown geneset
+        endpoint = "geneset_summary?geneset_name=NO_SUCH_GENE_SET"
+        result = self.session.get(f"{self.URL_BASE}{endpoint}", headers={"Accept": "application/octet-stream"})
+        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
+
+        # unknown method
+        endpoint = "geneset_summary?geneset_name=summary%20test&method=NO_SUCH_METHOD"
+        result = self.session.get(f"{self.URL_BASE}{endpoint}", headers={"Accept": "application/octet-stream"})
+        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)

backend/test/test_server/unit/common/test_corpora.py

@@ -0,0 +1,165 @@
+import json
+import shutil
+import tempfile
+import unittest
+from http import HTTPStatus
+
+import anndata
+import requests
+
+from backend.server.common.corpora import (
+    corpora_get_versions_from_anndata,
+    corpora_is_version_supported,
+    corpora_get_props_from_anndata,
+)
+from backend.test.test_server.unit import start_test_server, stop_test_server
+from backend.test import PROJECT_ROOT
+
+VERSION = "v0.2"
+
+
+class CorporaAPITest(unittest.TestCase):
+    def test_corpora_get_versions_from_anndata(self):
+        adata = self._get_h5ad()
+
+        if "version" in adata.uns:
+            del adata.uns["version"]
+        self.assertIsNone(corpora_get_versions_from_anndata(adata))
+
+        # something bogus
+        adata.uns["version"] = 99
+        self.assertIsNone(corpora_get_versions_from_anndata(adata))
+
+        # something legit
+        adata.uns["version"] = {"corpora_schema_version": "0.0.0", "corpora_encoding_version": "9.9.9"}
+        self.assertEqual(corpora_get_versions_from_anndata(adata), ["0.0.0", "9.9.9"])
+
+    def test_corpora_is_version_supported(self):
+        self.assertTrue(corpora_is_version_supported("1.0.0", "0.1.0"))
+        self.assertFalse(corpora_is_version_supported("0.0.0", "0.1.0"))
+        self.assertFalse(corpora_is_version_supported("1.0.0", "0.0.0"))
+
+    def test_corpora_get_props_from_anndata(self):
+        adata = self._get_h5ad()
+
+        if "version" in adata.uns:
+            del adata.uns["version"]
+        self.assertIsNone(corpora_get_props_from_anndata(adata))
+
+        # something bogus
+        adata.uns["version"] = 99
+        self.assertIsNone(corpora_get_props_from_anndata(adata))
+
+        # unsupported version, but missing required values
+        adata.uns["version"] = {"corpora_schema_version": "99.0.0", "corpora_encoding_version": "32.1.0"}
+        with self.assertRaises(ValueError):
+            corpora_get_props_from_anndata(adata)
+
+        # legit version, but missing required values
+        adata.uns["version"] = {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"}
+        with self.assertRaises(KeyError):
+            corpora_get_props_from_anndata(adata)
+
+        some_fields = {
+            "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"},
+            "title": "title",
+            "layer_descriptions": "layer_descriptions",
+            "organism": "organism",
+            "organism_ontology_term_id": "organism_ontology_term_id",
+            "project_name": "project_name",
+            "project_description": "project_description",
+            "contributors": json.dumps([{"contributors": "contributors"}]),
+            "project_links": json.dumps([{"link_name": "link_name", "link_url": "link_url", "link_type": "SUMMARY"}]),
+        }
+        for k in some_fields:
+            adata.uns[k] = some_fields[k]
+        some_fields["contributors"] = json.loads(some_fields["contributors"])
+        some_fields["project_links"] = json.loads(some_fields["project_links"])
+        self.assertEqual(corpora_get_props_from_anndata(adata), some_fields)
+
+    def test_corpora_get_props_from_anndata_v110(self):
+        adata = self._get_h5ad()
+
+        if "version" in adata.uns:
+            del adata.uns["version"]
+        self.assertIsNone(corpora_get_props_from_anndata(adata))
+
+        # legit version, but missing required values
+        adata.uns["version"] = {"corpora_schema_version": "1.1.0", "corpora_encoding_version": "0.1.0"}
+        with self.assertRaises(KeyError):
+            corpora_get_props_from_anndata(adata)
+
+        # Metadata following schema 1.1.0, which removes some fields relative to 1.1.0
+        some_110_fields = {
+            "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"},
+            "title": "title",
+            "layer_descriptions": "layer_descriptions",
+            "organism": "organism",
+            "organism_ontology_term_id": "organism_ontology_term_id",
+        }
+        for k in some_110_fields:
+            adata.uns[k] = some_110_fields[k]
+        self.assertEqual(corpora_get_props_from_anndata(adata), some_110_fields)
+
+    def _get_h5ad(self):
+        return anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
+
+
+class CorporaRESTAPITest(unittest.TestCase):
+    """ Confirm endpoints reflect Corpora-specific features """
+
+    @classmethod
+    def setCorporaFields(cls, path):
+        adata = anndata.read_h5ad(path)
+        corpora_props = {
+            "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"},
+            "title": "PBMC3K",
+            "contributors": json.dumps([{"name": "name"}]),
+            "layer_descriptions": {"X": "raw counts"},
+            "organism": "human",
+            "organism_ontology_term_id": "unknown",
+            "project_name": "test project",
+            "project_description": "test description",
+            "project_links": json.dumps(
+                [{"link_name": "test link", "link_type": "SUMMARY", "link_url": "https://a.u.r.l/"}]
+            ),
+            "default_embedding": "X_tsne",
+        }
+        adata.uns.update(corpora_props)
+        adata.write(path)
+
+    @classmethod
+    def setUpClass(cls):
+        cls.tmp_dir = tempfile.TemporaryDirectory()
+        src = f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad"
+        dst = f"{cls.tmp_dir.name}/pbmc3k.h5ad"
+        shutil.copyfile(src, dst)
+        cls.setCorporaFields(dst)
+        cls.ps, cls.server = start_test_server([dst])
+
+    @classmethod
+    def tearDownClass(cls):
+        stop_test_server(cls.ps)
+        cls.tmp_dir.cleanup()
+
+    def setUp(self):
+        self.session = requests.Session()
+        self.url_base = f"{self.server}/api/{VERSION}/"
+
+    def test_config(self):
+        endpoint = "config"
+        url = f"{self.url_base}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+
+        result_data = result.json()
+        self.assertIsInstance(result_data["config"]["corpora_props"], dict)
+        self.assertIsInstance(result_data["config"]["parameters"], dict)
+
+        corpora_props = result_data["config"]["corpora_props"]
+        parameters = result_data["config"]["parameters"]
+
+        self.assertEqual(corpora_props["version"]["corpora_schema_version"], "1.0.0")
+        self.assertEqual(corpora_props["organism"], "human")
+        self.assertEqual(parameters["default_embedding"], "tsne")

backend/test/test_server/unit/common/test_nan_rest.py

@@ -0,0 +1,61 @@
+from http import HTTPStatus
+import unittest
+import math
+from backend.test.test_server.unit import start_test_server, stop_test_server
+from backend.test import FIXTURES_ROOT
+import backend.test.decode_fbs as decode_fbs
+
+import requests
+
+VERSION = "v0.2"
+BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
+
+
+class WithNaNs(unittest.TestCase):
+    """Test Case for endpoints"""
+
+    @classmethod
+    def setUpClass(cls):
+        cls.ps, cls.server = start_test_server([f"{FIXTURES_ROOT}/nan.h5ad"])
+
+    @classmethod
+    def tearDownClass(cls):
+        stop_test_server(cls.ps)
+
+    def setUp(self):
+        self.session = requests.Session()
+        self.url_base = f"{self.server}/api/{VERSION}/"
+
+    def test_initialize(self):
+        endpoint = "schema"
+        url = f"{self.url_base}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+
+    def test_data(self):
+        endpoint = "data/var"
+        url = f"{self.url_base}{endpoint}"
+        filter = {"filter": {"var": {"index": [[0, 20]]}}}
+        result = self.session.put(url, json=filter)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertTrue(math.isnan(df["columns"][3][3]))
+
+    def test_annotation_obs(self):
+        endpoint = "annotations/obs"
+        url = f"{self.url_base}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertTrue(math.isnan(df["columns"][2][0]))
+
+    def test_annotation_var(self):
+        endpoint = "annotations/var"
+        url = f"{self.url_base}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertTrue(math.isnan(df["columns"][2][0]))

backend/test/test_server/unit/common/test_rest.py

@@ -0,0 +1,79 @@
+import unittest
+from urllib.parse import parse_qs
+from werkzeug.datastructures import MultiDict
+from backend.common.errors import FilterError
+from backend.server.common.rest import _query_parameter_to_filter
+
+
+def _qsparse(qs):
+    """ emulate what Flask/Werkzeug do to our QS """
+    return MultiDict(parse_qs(qs))
+
+
+class FilterParseTests(unittest.TestCase):
+    """ Test cases for various filter parsing """
+
+    def test_queryparam_to_filter_parse(self):
+        # categories
+        self.assertEqual(
+            _query_parameter_to_filter(_qsparse("obs:foo=bar&var:baz=133&var:baz=A&obs:baz=foo")),
+            {
+                "obs": {"annotation_value": [{"name": "foo", "values": ["bar"]}, {"name": "baz", "values": ["foo"]}]},
+                "var": {"annotation_value": [{"name": "baz", "values": ["133", "A"]}]},
+            },
+        )
+
+        # ranges
+        self.assertEqual(
+            _query_parameter_to_filter(_qsparse("obs:A=1,99&obs:B=*,100&obs:C=0,*")),
+            {
+                "obs": {
+                    "annotation_value": [
+                        {"name": "A", "min": 1, "max": 99.0},
+                        {"name": "B", "max": 100.0},
+                        {"name": "C", "min": 0.0},
+                    ]
+                },
+            },
+        )
+
+        # combo
+        self.assertEqual(
+            _query_parameter_to_filter(_qsparse("var:B=YES&var:A=1,99&var:B=NO")),
+            {
+                "var": {
+                    "annotation_value": [
+                        {"name": "B", "values": ["YES", "NO"]},
+                        {"name": "A", "min": 1.0, "max": 99.0},
+                    ]
+                },
+            },
+        )
+
+    def test_queryparam_to_filter_escaping(self):
+        self.assertEqual(
+            _query_parameter_to_filter(_qsparse("obs:var=%2521%252C%253AOK%253D&obs:A%2521=YO")),
+            {"obs": {"annotation_value": [{"name": "var", "values": ["!,:OK="]}, {"name": "A!", "values": ["YO"]}]}},
+        )
+
+    def test_queryparam_to_filter_errors(self):
+
+        # should raise FilterError
+        filter_errors = [
+            "foo=bar",  # no axis
+            "X=&Y=3",  # no value
+            "X&Y=3",  # no value
+            "moo:foo=bar",  # bad axis
+            "obs:x=1,A",  # non-numeric range
+            "var:X=1,2&var:X=3,4",  # duplicate ranges
+            "var:Y=,",
+            "var:Y=2,",
+            "var:Y=,5",
+            "var:Y=*,",
+            "var:Y=,*",
+            "var:Y=*,*",
+        ]
+
+        for qs in filter_errors:
+            with self.assertRaises(FilterError):
+                _query_parameter_to_filter(_qsparse(qs))

backend/test/test_server/unit/common/test_writable_annotation.py

@@ -0,0 +1,168 @@
+import json
+import shutil
+import unittest
+from os import path, listdir
+
+import numpy as np
+import pandas as pd
+
+import backend.test.decode_fbs as decode_fbs
+from backend.server.common.rest import annotations_put_fbs_helper, schema_get_helper
+from backend.server.data_common.matrix_loader import MatrixDataType
+from backend.test.test_server.unit import data_with_tmp_annotations, make_fbs
+
+
+class WritableAnnotationTest(unittest.TestCase):
+    def setUp(self):
+        self.data, self.tmp_dir, self.annotations = data_with_tmp_annotations(MatrixDataType.H5AD)
+        self.data.dataset_config.user_annotations = self.annotations
+
+    def tearDown(self):
+        shutil.rmtree(self.tmp_dir)
+
+    def annotation_put_fbs(self, fbs):
+        annotations_put_fbs_helper(self.data, fbs)
+        res = json.dumps({"status": "OK"})
+        return res
+
+    def test_error_checks(self):
+        # verify that the expected errors are generated
+        n_rows = self.data.get_shape()[0]
+        fbs_bad = make_fbs({"louvain": pd.Series(["undefined"] * n_rows, dtype="category")})
+
+        # ensure we catch attempt to overwrite non-writable data
+        with self.assertRaises(KeyError):
+            self.annotation_put_fbs(fbs_bad)
+
+    def test_write_to_file(self):
+        # verify the file is written as expected
+        n_rows = self.data.get_shape()[0]
+        fbs = make_fbs(
+            {
+                "cat_A": pd.Series(["label_A"] * n_rows, dtype="category"),
+                "cat_B": pd.Series(["label_B"] * n_rows, dtype="category"),
+            }
+        )
+        res = self.annotation_put_fbs(fbs)
+        self.assertEqual(res, json.dumps({"status": "OK"}))
+        self.assertTrue(path.exists(self.annotations.label_output_file))
+        df = pd.read_csv(self.annotations.label_output_file, index_col=0, header=0, comment="#")
+        self.assertEqual(df.shape, (n_rows, 2))
+        self.assertEqual(set(df.columns), {"cat_A", "cat_B"})
+        self.assertTrue(self.data.original_obs_index.equals(df.index))
+        self.assertTrue(np.all(df["cat_A"] == ["label_A"] * n_rows))
+        self.assertTrue(np.all(df["cat_B"] == ["label_B"] * n_rows))
+
+        # verify complete overwrite on second attempt, AND rotation occurs
+        fbs = make_fbs(
+            {
+                "cat_A": pd.Series(["label_A1"] * n_rows, dtype="category"),
+                "cat_C": pd.Series(["label_C"] * n_rows, dtype="category"),
+            }
+        )
+        res = self.annotation_put_fbs(fbs)
+        self.assertEqual(res, json.dumps({"status": "OK"}))
+        self.assertTrue(path.exists(self.annotations.label_output_file))
+        df = pd.read_csv(self.annotations.label_output_file, index_col=0, header=0, comment="#")
+        self.assertEqual(set(df.columns), {"cat_A", "cat_C"})
+        self.assertTrue(np.all(df["cat_A"] == ["label_A1"] * n_rows))
+        self.assertTrue(np.all(df["cat_C"] == ["label_C"] * n_rows))
+
+        # rotation
+        name, ext = path.splitext(self.annotations.label_output_file)
+        backup_dir = f"{name}-backups"
+        self.assertTrue(path.isdir(backup_dir))
+        found_files = listdir(backup_dir)
+        self.assertEqual(len(found_files), 1)
+
+    def test_file_rotation_to_max_9(self):
+        # verify we stop rotation at 9
+        n_rows = self.data.get_shape()[0]
+        fbs = make_fbs(
+            {
+                "cat_A": pd.Series(["label_A"] * n_rows, dtype="category"),
+                "cat_B": pd.Series(["label_B"] * n_rows, dtype="category"),
+            }
+        )
+        for i in range(0, 11):
+            res = self.annotation_put_fbs(fbs)
+            self.assertEqual(res, json.dumps({"status": "OK"}))
+
+        name, ext = path.splitext(self.annotations.label_output_file)
+        backup_dir = f"{name}-backups"
+        self.assertTrue(path.isdir(backup_dir))
+        found_files = listdir(backup_dir)
+        self.assertTrue(len(found_files) <= 9)
+
+    def test_put_get_roundtrip(self):
+        # verify that OBS PUTs (annotation_put_fbs) are accessible via
+        # GET (annotation_to_fbs_matrix)
+
+        n_rows = self.data.get_shape()[0]
+        fbs = make_fbs(
+            {
+                "cat_A": pd.Series(["label_A"] * n_rows, dtype="category"),
+                "cat_B": pd.Series(["label_B"] * n_rows, dtype="category"),
+            }
+        )
+
+        # put
+        res = self.annotation_put_fbs(fbs)
+        self.assertEqual(res, json.dumps({"status": "OK"}))
+
+        # get
+        labels = self.annotations.read_labels(None)
+        fbsAll = self.data.annotation_to_fbs_matrix("obs", None, labels)
+        schema = schema_get_helper(self.data)
+        annotations = decode_fbs.decode_matrix_FBS(fbsAll)
+        obs_index_col_name = schema["annotations"]["obs"]["index"]
+        self.assertEqual(annotations["n_rows"], n_rows)
+        self.assertEqual(annotations["n_cols"], 7)
+        self.assertIsNone(annotations["row_idx"])
+        self.assertEqual(
+            annotations["col_idx"],
+            [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain", "cat_A", "cat_B"],
+        )
+        col_idx = annotations["col_idx"]
+        self.assertEqual(annotations["columns"][col_idx.index("cat_A")], ["label_A"] * n_rows)
+        self.assertEqual(annotations["columns"][col_idx.index("cat_B")], ["label_B"] * n_rows)
+
+        # verify the schema was updated
+        all_col_schema = {c["name"]: c for c in schema["annotations"]["obs"]["columns"]}
+        self.assertEqual(
+            all_col_schema["cat_A"],
+            {"name": "cat_A", "type": "categorical", "categories": ["label_A"], "writable": True},
+        )
+        self.assertEqual(
+            all_col_schema["cat_B"],
+            {"name": "cat_B", "type": "categorical", "categories": ["label_B"], "writable": True},
+        )
+
+    def test_put_float_data(self):
+        # verify that OBS PUTs (annotation_put_fbs) are accessible via
+        # GET (annotation_to_fbs_matrix)
+
+        n_rows = self.data.get_shape()[0]
+
+        # verifies that floating point with decimals fail.
+        fbs = make_fbs({"cat_F_FAIL": pd.Series([1.1] * n_rows, dtype=np.dtype("float"))})
+        with self.assertRaises(ValueError) as exception_context:
+            res = self.annotation_put_fbs(fbs)
+        self.assertEqual(str(exception_context.exception), "Columns may not have floating point types")
+
+        # verifies that floating point that can be converted to int passes
+        fbs = make_fbs({"cat_F_PASS": pd.Series([1.0] * n_rows, dtype="float")})
+        res = self.annotation_put_fbs(fbs)
+        self.assertEqual(res, json.dumps({"status": "OK"}))
+
+        # check read_labels
+        labels = self.annotations.read_labels(None)
+        fbsAll = self.data.annotation_to_fbs_matrix("obs", None, labels)
+        schema = schema_get_helper(self.data)
+        annotations = decode_fbs.decode_matrix_FBS(fbsAll)
+        self.assertEqual(annotations["n_rows"], n_rows)
+        all_col_schema = {c["name"]: c for c in schema["annotations"]["obs"]["columns"]}
+        self.assertEqual(
+            all_col_schema["cat_F_PASS"],
+            {"name": "cat_F_PASS", "type": "int32", "writable": True},
+        )

backend/test/test_server/unit/common/utils/test_utils.py

@@ -0,0 +1,34 @@
+import os
+import shutil
+import unittest
+
+from backend.common.utils.utils import import_plugins
+from backend.test import PROJECT_ROOT, random_string
+
+
+class TestPlugins(unittest.TestCase):
+    """ Test plugin import functionality """
+
+    plugins_dir = f"{PROJECT_ROOT}/backend/test/plugins"
+    test_plugin_path = f"{plugins_dir}/foo.py"
+    secret = random_string(8)
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        if not os.path.isdir(cls.plugins_dir):
+            os.mkdir(cls.plugins_dir)
+        with open(cls.test_plugin_path, "w") as fh:
+            fh.write(f'SECRET = "{cls.secret}"\n')
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        if os.path.isdir(cls.plugins_dir):
+            shutil.rmtree(cls.plugins_dir)
+
+    def test_import_plugins(self):
+        self.assertTrue(os.path.isfile(self.test_plugin_path))
+        loaded_modules = import_plugins("backend.test.plugins")
+        # test that import plugins found the file
+        self.assertEqual(["backend.test.plugins.foo"], [ele.__name__ for ele in loaded_modules])
+        # test that the module was properly executed
+        self.assertEqual(self.secret, loaded_modules[0].SECRET)

backend/test/test_server/unit/compute/test_diffexp_cxg.py

@@ -0,0 +1,63 @@
+import unittest
+
+import numpy as np
+
+from backend.server.data_common.matrix_loader import MatrixDataLoader
+from backend.test.test_server.unit import app_config
+from backend.test import PROJECT_ROOT
+
+
+class DiffExpTest(unittest.TestCase):
+    """Tests the diffexp returns the expected results for one test case, using different
+    adaptor types and different algorithms."""
+
+    def load_dataset(self, path, extra_server_config={}, extra_dataset_config={}):
+        config = app_config(path, extra_server_config=extra_server_config, extra_dataset_config=extra_dataset_config)
+        loader = MatrixDataLoader(path)
+        adaptor = loader.open(config)
+        return adaptor
+
+    def get_mask(self, adaptor, start, stride):
+        """Simple function to return a mask or rows"""
+        rows = adaptor.get_shape()[0]
+        sel = list(range(start, rows, stride))
+        mask = np.zeros(rows, dtype=bool)
+        mask[sel] = True
+        return mask
+
+    def compare_diffexp_results(self, results, expects):
+        self.assertEqual(len(results), len(expects))
+        for result, expect in zip(results, expects):
+            self.assertEqual(result[0], expect[0])
+            self.assertTrue(np.isclose(result[1], expect[1], 1e-6, 1e-4))
+            self.assertTrue(np.isclose(result[2], expect[2], 1e-6, 1e-4))
+            self.assertTrue(np.isclose(result[3], expect[3], 1e-6, 1e-4))
+
+    def check_1_10_2_10(self, results):
+        """Checks the results for a specific set of rows selections"""
+        expects = [
+            [956, 0.016060986, 0.0008649321884808977, 1.0],
+            [1124, 0.96602094, 0.0011717216548271284, 1.0],
+            [1809, 1.1110606, 0.0019304405196777848, 1.0],
+            [1712, -0.5525154, 0.0051788902660723345, 1.0],
+            [1754, 0.5201581, 0.005691734062127954, 1.0],
+            [948, 1.6390722, 0.006622111055981219, 1.0],
+            [1810, 0.78618884, 0.007055917428377063, 1.0],
+            [779, 1.5241305, 0.007202934422407284, 1.0],
+            [1575, 1.0317602, 0.007830310753043345, 1.0],
+            [576, 0.97873515, 0.008272092578813124, 1.0],
+        ]
+        self.compare_diffexp_results(results, expects)
+
+    def get_X_col(self, adaptor, cols):
+        varmask = np.zeros(adaptor.get_shape()[1], dtype=bool)
+        varmask[cols] = True
+        return adaptor.get_X_array(None, varmask)
+
+    def test_anndata_default(self):
+        """Test an anndata adaptor with its default diffexp algorithm (diffexp_generic)"""
+        adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
+        maskA = self.get_mask(adaptor, 1, 10)
+        maskB = self.get_mask(adaptor, 2, 10)
+        results = adaptor.compute_diffexp_ttest(maskA, maskB, 10)
+        self.check_1_10_2_10(results)

backend/test/test_server/unit/converters/schema/test_gene_symbol.py

@@ -0,0 +1,61 @@
+import os
+import unittest
+
+import pandas as pd
+
+from backend.test import FIXTURES_ROOT
+from backend.server.converters.schema import gene_symbol
+
+
+class TestHGNCSymbolChecker(unittest.TestCase):
+
+    def setUp(self):
+        self.test_hgnc_path = os.path.join(FIXTURES_ROOT, "hgnc_example.txt.gz")
+        self.hgnc_checker = gene_symbol.HGNCSymbolChecker.from_hgnc_records(self.test_hgnc_path)
+
+    def test_symbol_upgrade(self):
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("SEPT1"), "SEPTIN1")
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("ADRB2R"), "ADRB2")
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("BAR"), "ADRB2")
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("sept1"), "SEPTIN1")
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("AdRb2R"), "ADRB2")
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("bar"), "ADRB2")
+
+        # Strip off seurat endings when appropriate
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("SEPT1.1"), "SEPTIN1")
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("ADRB2-1"), "ADRB2")
+
+        # DIFF6 is ambiguous so don't upgrade it
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("DIFF6"), "DIFF6")
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("diff6"), "diff6")
+
+        # ARG1 is approved
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("ARG1"), "ARG1")
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("arg1"), "ARG1")
+
+        # HAP1 is both approved and withdrawn
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("HAP1"), "HAP1")
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("hap1"), "HAP1")
+
+        # Leave unknown symbols alone
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("NOTASYMBOL"), "NOTASYMBOL")
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("notasymbol"), "notasymbol")
+
+        # Upgrade HGNC ids unless you can't find it
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("HGNC:286"), "ADRB2")
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("HGNC:4812"), "HAP1")
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("HGNC:123456"), "HGNC:123456")
+
+    def test_check_symbol(self):
+        self.assertEqual(self.hgnc_checker.check_symbol("SEPT1"), gene_symbol.SymbolStatus.UPGRADABLE)
+        self.assertEqual(self.hgnc_checker.check_symbol("DIFF6"), gene_symbol.SymbolStatus.AMBIGUOUS)
+        self.assertEqual(self.hgnc_checker.check_symbol("NOTASYMBOL"), gene_symbol.SymbolStatus.UNKNOWN)
+
+        # HAP1 is one of the approved and withdrawn symbols
+        self.assertEqual(self.hgnc_checker.check_symbol("HAP1"), gene_symbol.SymbolStatus.APPROVED)
+
+    def test_upgrade_index(self):
+        index = pd.Index(["SEPT1", "DIFF6", "NOTASYMBOL", "bar", "SEPTIN1"])
+        var_df = pd.DataFrame([[0] * len(index)], index=index)
+        upgraded_index = gene_symbol.get_upgraded_var_index(var_df, hgnc_path=self.test_hgnc_path)
+        self.assertEqual(upgraded_index.tolist(), ["SEPTIN1", "DIFF6", "NOTASYMBOL", "ADRB2", "SEPTIN1"])

backend/test/test_server/unit/converters/schema/test_ontology.py

@@ -0,0 +1,128 @@
+import json
+
+import unittest.mock
+
+from backend.server.converters.schema import ontology
+
+
+class TestOntologyParsing(unittest.TestCase):
+    def setUp(self):
+
+        self.curies = ["UBERON:0002048", "HsapDv:0000174", "NCBITaxon:9606", "EFO:0008995"]
+
+        self.names = ["UBERON", "HsapDv", "NCBITaxon", "EFO"]
+
+        self.values = ["0002048", "0000174", "9606", "0008995"]
+
+        self.iris = [
+            "http://purl.obolibrary.org/obo/UBERON_0002048",
+            "http://purl.obolibrary.org/obo/HsapDv_0000174",
+            "http://purl.obolibrary.org/obo/NCBITaxon_9606",
+            "http://www.ebi.ac.uk/efo/EFO_0008995",
+        ]
+
+        URL_ROOT = "http://www.ebi.ac.uk/ols/api/ontologies/"
+        self.urls = [
+            URL_ROOT + "UBERON/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FUBERON_0002048",
+            URL_ROOT + "HsapDv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FHsapDv_0000174",
+            URL_ROOT + "NCBITaxon/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_9606",
+            URL_ROOT + "EFO/terms/http%253A%252F%252Fwww.ebi.ac.uk%252Fefo%252FEFO_0008995",
+        ]
+
+        self.responses = {
+            "UBERON:0002048": {
+                "iri": "http://purl.obolibrary.org/obo/UBERON_0002048",
+                "description": ["Respiration organ that develops as an outpocketing of the esophagus."],
+                "label": "lung",
+            },
+            "HsapDv:0000174": {
+                "iri": "http://purl.obolibrary.org/obo/HsapDv_0000174",
+                "description": ["Infant stage that refers to an infant who is over 1 and under 2 months old."],
+                "label": "1-month-old human stage",
+            },
+            "NCBITaxon:9606": {
+                "iri": "http://purl.obolibrary.org/obo/NCBITaxon_9606",
+                "description": None,
+                "label": "Homo sapiens",
+            },
+            "EFO:0008995": {
+                "iri": "http://www.ebi.ac.uk/efo/EFO_0008995",
+                "description": [
+                    (
+                        '10X is a "synthetic long-read" technology and works by capturing a barcoded oligo-coated '
+                        "gel-bead and 0.3x genome copies into a single emulsion droplet, processing the equivalent "
+                        "of 1 million pipetting steps. Successive versions of the 10x chemistry use different "
+                        "barcode locations to improve the sequencing yield and quality of 10x experiments."
+                    )
+                ],
+                "label": "10X sequencing",
+            },
+        }
+
+    def test_ontololgy_name(self):
+        for curie, expected_name in zip(self.curies, self.names):
+            self.assertEqual(ontology._ontology_name(curie), expected_name)
+
+    def test_ontololgy_value(self):
+        for curie, expected_value in zip(self.curies, self.values):
+            self.assertEqual(ontology._ontology_value(curie), expected_value)
+
+    def test_iri(self):
+        for curie, expected_iri in zip(self.curies, self.iris):
+            self.assertEqual(ontology._iri(curie), expected_iri)
+
+    def test_ontology_info_url(self):
+        for curie, expected_url in zip(self.curies, self.urls):
+            self.assertEqual(ontology._ontology_info_url(curie), expected_url)
+
+    def test_empty_ontology_info_url(self):
+        self.assertEqual(ontology._ontology_info_url(""), "")
+
+
+class TestOntologyLookup(unittest.TestCase):
+    def setUp(self):
+        self.responses = {
+            "UBERON:0002048": {
+                "iri": "http://purl.obolibrary.org/obo/UBERON_0002048",
+                "description": ["Respiration organ that develops as an outpocketing of the esophagus."],
+                "label": "lung",
+            },
+            "HsapDv:0000174": {
+                "iri": "http://purl.obolibrary.org/obo/HsapDv_0000174",
+                "description": ["Infant stage that refers to an infant who is over 1 and under 2 months old."],
+                "label": "1-month-old human stage",
+            },
+            "NCBITaxon:9606": {
+                "iri": "http://purl.obolibrary.org/obo/NCBITaxon_9606",
+                "description": None,
+                "label": "Homo sapiens",
+            },
+            "EFO:0008995": {
+                "iri": "http://www.ebi.ac.uk/efo/EFO_0008995",
+                "description": [
+                    ('10X is a "synthetic long-read" technology and works by capturing a barcoded oligo-coated '
+                     'gel-bead and 0.3x genome copies into a single emulsion droplet, processing the equivalent '
+                     'of 1 million pipetting steps. Successive versions of the 10x chemistry use different barcode '
+                     'locations to improve the sequencing yield and quality of 10x experiments.')
+                ],
+                "label": "10X sequencing",
+            },
+        }
+
+        self.labels = {
+            "UBERON:0002048": "lung",
+            "HsapDv:0000174": "1-month-old human stage",
+            "NCBITaxon:9606": "Homo sapiens",
+            "EFO:0008995": "10X sequencing",
+        }
+
+    @unittest.mock.patch("requests.get")
+    def test_lookup_label(self, mock_get):
+
+        for curie, response in self.responses.items():
+            mock_get.return_value.content = json.dumps(response)
+            mock_get.return_value.json.return_value = response
+            mock_get.return_value.status_code = 200
+
+            label = ontology.get_ontology_label(curie)
+            self.assertEqual(label, self.labels[curie])

backend/test/test_server/unit/converters/schema/test_remix.py

@@ -0,0 +1,256 @@
+import json
+import os
+import unittest
+import unittest.mock
+
+import anndata
+import numpy
+import pandas as pd
+import scanpy as sc
+
+from backend.server.converters.schema import remix
+from backend.test import PROJECT_ROOT
+
+
+class TestApplySchema(unittest.TestCase):
+
+    def setUp(self):
+        self.source_h5ad_path = f"{PROJECT_ROOT}/backend/test/fixtures/pbmc3k-CSC-gz.h5ad"
+        self.output_h5ad_path = f"{PROJECT_ROOT}/backend/test/fixtures/test_remix.h5ad"
+        self.config_path = f"{PROJECT_ROOT}/backend/test/fixtures/test_config.yaml"
+        self.bad_config_path = f"{PROJECT_ROOT}/backend/test/fixtures/test_bad_config.yaml"
+
+    def tearDown(self):
+        try:
+            os.remove(self.output_h5ad_path)
+        except OSError:
+            pass
+
+    @unittest.mock.patch("backend.server.converters.schema.ontology.get_ontology_label")
+    def test_apply_schema(self, mock_get_ontology_label):
+        mock_get_ontology_label.return_value = "test label"
+        remix.apply_schema(self.source_h5ad_path, self.config_path, self.output_h5ad_path)
+        new_adata = sc.read_h5ad(self.output_h5ad_path)
+
+        self.assertIn("cell_type", new_adata.obs.columns)
+        self.assertListEqual(["test label"], new_adata.obs["cell_type"].unique().tolist())
+        self.assertListEqual(
+            ["CL:00001", "CL:00002", "CL:00003", "CL:00004", "CL:00005", "CL:00006", "CL:00007", "CL:00008"],
+            sorted(new_adata.obs["cell_type_ontology_term_id"].unique().tolist())
+        )
+
+        self.assertIn("version", new_adata.uns_keys())
+
+    @unittest.mock.patch("backend.server.converters.schema.ontology.get_ontology_label")
+    def test_apply_bad_schema(self, mock_get_ontology_label):
+        mock_get_ontology_label.return_value = "test label"
+        remix.apply_schema(self.source_h5ad_path, self.bad_config_path, self.output_h5ad_path)
+        new_adata = sc.read_h5ad(self.output_h5ad_path)
+
+        # Should refuse to write the version
+        self.assertNotIn("version", new_adata.uns_keys())
+
+class TestFieldParsing(unittest.TestCase):
+
+    def test_is_curie(self):
+        self.assertTrue(remix.is_curie("EFO:00001"))
+        self.assertTrue(remix.is_curie("UBERON:123456"))
+        self.assertTrue(remix.is_curie("HsapDv:0001"))
+        self.assertFalse(remix.is_curie("UBERON"))
+        self.assertFalse(remix.is_curie("UBERON:"))
+        self.assertFalse(remix.is_curie("123456"))
+
+    def test_is_ontology_field(self):
+        self.assertTrue(remix.is_ontology_field("tissue_ontology_term_id"))
+        self.assertTrue(remix.is_ontology_field("cell_type_ontology_term_id"))
+        self.assertFalse(remix.is_ontology_field("cell_ontology"))
+        self.assertFalse(remix.is_ontology_field("method"))
+
+    def test_get_label_field_name(self):
+        self.assertEqual("tissue", remix.get_label_field_name("tissue_ontology_term_id"))
+        self.assertEqual("cell_type", remix.get_label_field_name("cell_type_ontology_term_id"))
+
+    def test_split_suffix(self):
+        self.assertEqual(("UBERON:1234", " (organoid)"), remix.split_suffix("UBERON:1234 (organoid)"))
+        self.assertEqual(("UBERON:1234", " (cell culture)"), remix.split_suffix("UBERON:1234 (cell culture)"))
+        self.assertEqual(("UBERON:1234", ""), remix.split_suffix("UBERON:1234"))
+        self.assertEqual(("UBERON:1234 (something)", ""), remix.split_suffix("UBERON:1234 (something)"))
+
+    @unittest.mock.patch("backend.server.converters.schema.ontology.get_ontology_label")
+    def test_get_curie_and_label(self, mock_get_ontology_label):
+        mock_get_ontology_label.return_value = "test label"
+        self.assertEqual(
+            remix.get_curie_and_label("UBERON:1234"),
+            ("UBERON:1234", "test label")
+        )
+        self.assertEqual(
+            remix.get_curie_and_label("UBERON:1234 (cell culture)"),
+            ("UBERON:1234 (cell culture)", "test label (cell culture)")
+        )
+        self.assertEqual(
+            remix.get_curie_and_label("whatever"),
+            ("", "whatever")
+        )
+
+
+class TestManipulateAnndata(unittest.TestCase):
+
+    def setUp(self):
+
+        self.cell_count = 20
+        self.gene_count = 200
+        X = numpy.random.randint(0, 1000, (self.cell_count, self.gene_count))
+        uns = {"organism": "monkey", "experiment": "monkey experiment"}
+        obs = pd.DataFrame(
+            index=[f"Cell{d}" for d in range(self.cell_count)],
+            columns=["tissue", "CellType"],
+            data=[["lung", "epithelial"]] * (self.cell_count // 2) + [["lung", "endothelial"]] * (self.cell_count // 2)
+        )
+        var = pd.DataFrame(index=[f"SEPT{d}" for d in range(self.gene_count)])
+
+        self.adata = anndata.AnnData(X=X, obs=obs, var=var, uns=uns)
+
+    def test_safe_add_field(self):
+
+        remix.safe_add_field(self.adata.obs, "tissue", ["monkey lung"] * self.cell_count)
+        self.assertEqual(self.adata.obs["tissue_original"].tolist(), ["lung"] * self.cell_count)
+        self.assertEqual(self.adata.obs["tissue"].tolist(), ["monkey lung"] * self.cell_count)
+
+        remix.safe_add_field(self.adata.uns, "contributors", [{"name": "contributor1"}, {"name": "contributor2"}])
+        self.assertEqual(
+            self.adata.uns["contributors"],
+            json.dumps([{"name": "contributor1"}, {"name": "contributor2"}])
+        )
+
+    @unittest.mock.patch("backend.server.converters.schema.ontology.get_ontology_label")
+    def test_remix_uns(self, mock_get_ontology_label):
+        mock_get_ontology_label.return_value = "Pan troglodytes"
+        uns_config = {
+            "version": {
+                "corpora_schema_version": "1.0.0",
+                "corpora_encoding_version": "0.1.0"
+            },
+            "organism_ontology_term_id": "NCBITaxon:9598",
+            "contributors": [
+                {
+                    "name": "scientist",
+                    "email": "scientist@science.com"
+                }
+            ]
+        }
+
+        remix.remix_uns(self.adata, uns_config)
+
+        self.assertEqual(
+            sorted(self.adata.uns_keys()),
+            sorted(["organism_original", "organism", "organism_ontology_term_id",
+                    "contributors", "version", "experiment"])
+        )
+
+        self.assertEqual(self.adata.uns['organism'], "Pan troglodytes")
+        self.assertEqual(self.adata.uns['organism_original'], "monkey")
+        self.assertEqual(self.adata.uns['organism_ontology_term_id'], "NCBITaxon:9598")
+        self.assertEqual(self.adata.uns['contributors'],
+                         json.dumps([{"name": "scientist", "email": "scientist@science.com"}]))
+
+    @unittest.mock.patch("backend.server.converters.schema.ontology.get_ontology_label")
+    def test_remix_obs(self, mock_get_ontology_label):
+        mock_get_ontology_label.return_value = "lung (in a monkey)"
+        obs_config = {
+            "tissue_ontology_term_id": {
+                "tissue": {
+                    "lung": "UBERON:00000"
+                }
+            },
+            "cell_color": {
+                "CellType": {
+                    "epithelial": "fuschia",
+                    "endothelial": "khaki"
+                }
+            },
+            "sex": "male"
+        }
+
+        remix.remix_obs(self.adata, obs_config)
+        self.assertEqual(
+            sorted(self.adata.obs_keys()),
+            sorted(["tissue", "tissue_ontology_term_id", "tissue_original", "CellType", "cell_color", "sex"])
+        )
+
+        self.assertTrue(all(v == "lung" for v in self.adata.obs.tissue_original))
+        self.assertTrue(all(v == "UBERON:00000" for v in self.adata.obs.tissue_ontology_term_id))
+        self.assertTrue(all(v == "lung (in a monkey)" for v in self.adata.obs.tissue))
+        self.assertTrue(all(v == "male" for v in self.adata.obs.sex))
+        self.assertTrue(all(v in (("epithelial", "fuschia"), ("endothelial", "khaki"))
+                            for v in zip(self.adata.obs.CellType, self.adata.obs.cell_color)))
+
+
+class TestFixupGeneSymbols(unittest.TestCase):
+
+    def setUp(self):
+        self.seurat_path = f"{PROJECT_ROOT}/server/test/fixtures/schema_test_data/seurat_tutorial.h5ad"
+        self.seurat_merged_path = f"{PROJECT_ROOT}/server/test/fixtures/schema_test_data/seurat_tutorial_merged.h5ad"
+        self.sctransform_path = f"{PROJECT_ROOT}/server/test/fixtures/schema_test_data/sctransform.h5ad"
+        self.sctransform_merged_path = f"{PROJECT_ROOT}/server/test/fixtures/schema_test_data/sctransform_merged.h5ad"
+
+        # There's lots of MALAT1, but it doesn't collide with any other names,
+        # so it shouldn't change during merging.
+        self.stable_gene = "MALAT1"
+
+    def test_fixup_gene_symbols_seurat(self):
+
+        if not os.path.isfile(self.seurat_path):
+            return unittest.skip(
+                "Skipping gene symbol conversion tests because test h5ads are not present. To create them, "
+                "run server/test/fixtures/schema_test_data/generate_test_data.sh"
+            )
+
+        original_adata = sc.read_h5ad(self.seurat_path)
+        merged_adata = sc.read_h5ad(self.seurat_merged_path)
+
+        fixup_config = {"X": "log1p", "counts": "raw", "scale.data": "log1p"}
+
+        fixed_adata = remix.fixup_gene_symbols(original_adata, fixup_config)
+
+        self.assertEqual(
+            merged_adata.layers["counts"][:, merged_adata.var.index == self.stable_gene].sum(),
+            fixed_adata.raw.X[:, fixed_adata.var.index == self.stable_gene].sum()
+        )
+        self.assertAlmostEqual(
+            merged_adata.X[:, merged_adata.var.index == self.stable_gene].sum(),
+            fixed_adata.X[:, fixed_adata.var.index == self.stable_gene].sum()
+        )
+
+        self.assertAlmostEqual(
+            merged_adata.layers["scale.data"][:, merged_adata.var.index == self.stable_gene].sum(),
+            fixed_adata.layers["scale.data"][:, fixed_adata.var.index == self.stable_gene].sum()
+        )
+
+    def test_fixup_gene_symbols_sctransform(self):
+
+        if not os.path.isfile(self.sctransform_path):
+            return unittest.skip(
+                "Skipping gene symbol conversion tests because test h5ads are not present. To create them, "
+                "run server/test/fixtures/schema_test_data/generate_test_data.sh"
+            )
+
+        original_adata = sc.read_h5ad(self.sctransform_path)
+        merged_adata = sc.read_h5ad(self.sctransform_merged_path)
+
+        fixup_config = {"X": "log1p", "counts": "raw"}
+
+        fixed_adata = remix.fixup_gene_symbols(original_adata, fixup_config)
+
+        # sctransform does a bunch of stuff, including slightly modifying the
+        # raw counts. So we can't assert for exact equality the way we do with
+        # the vanilla seurat tutorial. But, the results should still be very
+        # close.
+        merged_raw_stable = merged_adata.layers["counts"][:, merged_adata.var.index == self.stable_gene].sum()
+        fixed_raw_stable = fixed_adata.raw.X[:, fixed_adata.var.index == self.stable_gene].sum()
+        self.assertLess(abs(merged_raw_stable - fixed_raw_stable), .001 * merged_raw_stable)
+
+        self.assertAlmostEqual(
+            merged_adata.X[:, merged_adata.var.index == self.stable_gene].sum(),
+            fixed_adata.X[:, fixed_adata.var.index == self.stable_gene].sum(),
+            0
+        )

backend/test/test_server/unit/converters/schema/test_validate.py

@@ -0,0 +1,434 @@
+import json
+import unittest
+
+import pandas as pd
+import scanpy as sc
+
+from backend.server.converters.schema import validate
+
+from backend.test import PROJECT_ROOT
+
+
+class TestFieldValidation(unittest.TestCase):
+
+    def test_validate_stringified_list_of_dicts(self):
+
+        good = json.dumps([{"a": 1}, {2: "x", "z": "y"}])
+        not_stringified = [{"a": 1}, {2: "x", "z": "y"}]
+        not_a_list = json.dumps({"bad": "dict"})
+        not_json = "oh hey!"
+
+        self.assertTrue(validate._validate_stringified_list_of_dicts(good))
+
+        self.assertFalse(validate._validate_stringified_list_of_dicts(not_stringified))
+        self.assertFalse(validate._validate_stringified_list_of_dicts(not_a_list))
+        self.assertFalse(validate._validate_stringified_list_of_dicts(not_json))
+
+    def test_validate_human_readable_string(self):
+
+        good = "oh hey!"
+        curie = "EFO:0001"
+        ensg = "ENSG000001234"
+        enst = "ENST000005678"
+
+        self.assertTrue(validate._validate_human_readable_string(good))
+
+        self.assertFalse(validate._validate_human_readable_string(curie))
+        self.assertFalse(validate._validate_human_readable_string(ensg))
+        self.assertFalse(validate._validate_human_readable_string(enst))
+
+    def test_validate_curie(self):
+
+        self.assertTrue(validate._validate_curie("UBERON:00001", ["UBERON", "EFO"]))
+        self.assertTrue(validate._validate_curie("HsapDv:00002", ["HsapDv"]))
+
+        self.assertFalse(validate._validate_curie("HsapDv:00002", ["UBERON", "EFO"]))
+        self.assertFalse(validate._validate_curie("EFO:00002 (organoid)", ["UBERON", "EFO"]))
+        self.assertFalse(validate._validate_curie("EFO:00002 extra", ["UBERON", "EFO"]))
+        self.assertFalse(validate._validate_curie("UBERON:ABCD", ["UBERON", "EFO"]))
+        self.assertFalse(validate._validate_curie("Uberon:00002", ["UBERON", "EFO"]))
+        self.assertFalse(validate._validate_curie("UBERON:", ["UBERON", "EFO"]))
+        self.assertFalse(validate._validate_curie("UBERON", ["UBERON", "EFO"]))
+
+    def test_validate_suffixed_curie(self):
+
+        self.assertTrue(validate._validate_suffixed_curie("EFO:00001", ["UBERON", "EFO"]))
+        self.assertTrue(validate._validate_suffixed_curie("UBERON:00001 (cell culture)", ["UBERON", "EFO"]))
+
+        self.assertFalse(validate._validate_suffixed_curie("HsapDv:00002 (organoid)", ["UBERON", "EFO"]))
+        self.assertFalse(validate._validate_suffixed_curie("HsapDv:00002(organoid)", ["UBERON", "EFO"]))
+        self.assertFalse(validate._validate_suffixed_curie("HsapDv:00002", ["UBERON", "EFO"]))
+        self.assertFalse(validate._validate_suffixed_curie("EFO:00002 extra", ["UBERON", "EFO"]))
+        self.assertFalse(validate._validate_suffixed_curie("UBERON:ABCD", ["UBERON", "EFO"]))
+        self.assertFalse(validate._validate_suffixed_curie("Uberon:00002", ["UBERON", "EFO"]))
+        self.assertFalse(validate._validate_suffixed_curie("UBERON:", ["UBERON", "EFO"]))
+        self.assertFalse(validate._validate_suffixed_curie("UBERON", ["UBERON", "EFO"]))
+
+
+class TestColumnValidation(unittest.TestCase):
+
+    def test_validate_unique(self):
+        unique = pd.DataFrame([["abc", "def"], ["ghi", "jkl"], ["mnop", "qrs"]],
+                              index=["X", "Y", "Z"], columns=["col1", "col2"])
+        duped = pd.DataFrame([["abc", "def"], ["ghi", "qrs"], ["abc", "qrs"]],
+                             index=["X", "Y", "X"], columns=["col1", "col2"])
+
+        schema_def = {"unique": True}
+
+        errors = validate._validate_column(unique.index, "index", "unique_df", schema_def)
+        self.assertFalse(errors)
+
+        errors = validate._validate_column(duped.index, "index", "duped_df", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("is not unique", errors[0])
+
+        errors = validate._validate_column(unique["col1"], "col1", "unique_df", schema_def)
+        self.assertFalse(errors)
+
+        errors = validate._validate_column(duped["col1"], "col1", "duped_df", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("is not unique", errors[0])
+
+        schema_def = {"unique": False}
+        errors = validate._validate_column(duped["col1"], "col1", "duped_df", schema_def)
+        self.assertFalse(errors)
+
+    def test_validate_nullable(self):
+        non_null = pd.DataFrame([["abc", "def"], ["ghi", "jkl"], ["mnop", "qrs"]],
+                                index=["X", "Y", "Z"], columns=["col1", "col2"])
+        has_null = pd.DataFrame([["abc", "", None], ["ghi", "jkl", 1], ["mnop", "qrs", 2]],
+                                index=["X", "Y", "Z"], columns=["col1", "col2", "col3"])
+
+        schema_def = {"nullable": False}
+        errors = validate._validate_column(non_null["col1"], "col1", "nonnull_df", schema_def)
+        self.assertFalse(errors)
+        errors = validate._validate_column(has_null["col1"], "col1", "hasnull_df", schema_def)
+        self.assertFalse(errors)
+
+        errors = validate._validate_column(has_null["col2"], "col2", "hasnull_df", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("contains empty values", errors[0])
+        errors = validate._validate_column(has_null["col3"], "col3", "hasnull_df", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("contains empty values", errors[0])
+
+        schema_def = {"nullable": True}
+        errors = validate._validate_column(has_null["col2"], "col2", "hasnull_df", schema_def)
+        self.assertFalse(errors)
+
+    def test_human_readable(self):
+        hr_df = pd.DataFrame(
+            [["for you, a human", "UBERON:12345", "UBERON:1234 (thundercat)"],
+             ["hope you're well", "bit of lungs", "brain"]],
+            index=["ENSG00001", "ENSG00002"],
+            columns=["good", "curie", "suffixed_curie"])
+
+        schema_def = {"type": "human-readable string"}
+        errors = validate._validate_column(hr_df["good"], "good", "hr", schema_def)
+        self.assertFalse(errors)
+
+        errors = validate._validate_column(hr_df["curie"], "curie", "hr", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("non-human-readable", errors[0])
+
+        errors = validate._validate_column(hr_df["suffixed_curie"], "suffixed_curie", "hr", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("non-human-readable", errors[0])
+
+        errors = validate._validate_column(hr_df.index, "ensg", "hr", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("non-human-readable", errors[0])
+
+    def test_curie(self):
+
+        curie_df = pd.DataFrame(
+            [["EFO:00001", "HsapDv:00001 (cell culture)", "EFO:", "MONDO:0001 cell culture"],
+             ["UBERON:00002", "HsapDv:00002 (organoid)", "EFO:12345", "MONDO:0002 (baba yaga)"],
+             ["EFO:0000000005", "HsapDv:000004 (humanzee)", "EFO:000002", "MONDO:0004 (TMNT)"]],
+            index=["X", "Y", "Z"],
+            columns=["good", "good_suffix", "bad", "bad_suffix"])
+
+        # Good
+        schema_def = {"type": "curie", "prefixes": ["EFO", "UBERON"]}
+        errors = validate._validate_column(curie_df["good"], "good", "curie_df", schema_def)
+        self.assertFalse(errors)
+
+        # Good suffix
+        schema_def = {"type": "suffixed curie", "prefixes": ["HsapDv", "WHATEVER"]}
+        errors = validate._validate_column(curie_df["good_suffix"], "good_suffix", "curie_df", schema_def)
+        self.assertFalse(errors)
+
+        # Bad prefix
+        schema_def = {"type": "curie", "prefixes": ["EFO"]}
+        errors = validate._validate_column(curie_df["good"], "good", "curie_df", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("invalid ontology", errors[0])
+        self.assertIn("must be curies from one of these", errors[0])
+
+        # Bad curies
+        schema_def = {"type": "curie", "prefixes": ["EFO"]}
+        errors = validate._validate_column(curie_df["bad"], "bad", "curie_df", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("invalid ontology", errors[0])
+
+        # Bad suffixes
+        schema_def = {"type": "suffixed curie", "prefixes": ["EFO"]}
+        errors = validate._validate_column(curie_df["bad_suffix"], "bad_suffix", "curie_df", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("invalid ontology", errors[0])
+
+    def test_enum(self):
+        enum_df = pd.DataFrame(
+            [["abc", "ghi"],
+             ["def", "jkl"]],
+            index=["X", "Y"],
+            columns=["col1", "col2"])
+
+        # All match
+        schema_def = {"type": "string", "enum": ["abc", "def", "xyz"]}
+        errors = validate._validate_column(enum_df["col1"], "col1", "enum_df", schema_def)
+        self.assertFalse(errors)
+
+        # Missing value
+        schema_def = {"type": "string", "enum": ["abc", "xyz"]}
+        errors = validate._validate_column(enum_df["col1"], "col1", "enum_df", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("unpermitted values", errors[0])
+
+
+class TestDictValidations(unittest.TestCase):
+
+
+    def test_key_presence(self):
+
+        schema_def = {"keys": {"abc": None, "def": None}}
+
+        dict_ = {"abc": "123", "def": "456"}
+        errors = validate._validate_dict(dict_, "d", schema_def)
+        self.assertFalse(errors)
+
+        # Missing keys are bad
+        dict_ = {"abc": "123"}
+        errors = validate._validate_dict(dict_, "d", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("missing key", errors[0])
+
+        # Extra keys are okay
+        dict_ = {"abc": "123", "def": "456", "xyz": "789"}
+        errors = validate._validate_dict(dict_, "d", schema_def)
+        self.assertFalse(errors)
+
+        # Better not be empty come on
+        dict_ = {}
+        errors = validate._validate_dict(dict_, "d", schema_def)
+        self.assertEqual(len(errors), 2)
+
+    def test_nullable(self):
+
+        schema_def = {"keys": {"abc": {"type": "string", "nullable": False},
+                               "def": {"type": "string", "nullable": True}}}
+
+        dict_ = {"abc": "xyz", "def": ""}
+        errors = validate._validate_dict(dict_, "d", schema_def)
+        self.assertFalse(errors)
+
+        dict_ = {"abc": "", "def": ""}
+        errors = validate._validate_dict(dict_, "d", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("empty value", errors[0])
+
+    def test_recurse(self):
+
+        schema_def = {
+            "keys": {
+                "subdict": {
+                    "type": "dict",
+                    "keys": {
+                        "subdict_key1": None,
+                        "subdict_key2": None
+                    }
+                },
+                "ontology": {
+                    "type": "curie",
+                    "prefixes": ["ONTOLOGY"]
+                },
+                "blob": {
+                    "type": "stringified list of dicts"
+                }
+            }
+        }
+
+        dict_ = {
+            "subdict": {"subdict_key1": "any", "subdict_key2": "any"},
+            "ontology": "ONTOLOGY:123456",
+            "blob": json.dumps([{"abc": 123}, {"def": 456}])
+        }
+        errors = validate._validate_dict(dict_, "d", schema_def)
+        self.assertFalse(errors)
+
+        dict_ = {
+            "subdict": {"subdict_key1": "any"},
+            "ontology": "ONTOLOGY:123456",
+            "blob": json.dumps([{"abc": 123}, {"def": 456}])
+        }
+        errors = validate._validate_dict(dict_, "d", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("missing key", errors[0])
+
+        dict_ = {
+            "subdict": {"subdict_key1": "any", "subdict_key2": "any"},
+            "ontology": "oh no not an ontology term",
+            "blob": json.dumps([{"abc": 123}, {"def": 456}])
+        }
+        errors = validate._validate_dict(dict_, "d", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("invalid ontology", errors[0])
+
+        dict_ = {
+            "subdict": {"subdict_key1": "any", "subdict_key2": "any"},
+            "ontology": "ONTOLOGY:123456",
+            "blob": [{"abc": 123}, {"def": 456}]
+        }
+        errors = validate._validate_dict(dict_, "d", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("JSON-encoded list of dicts", errors[0])
+
+        # Multiple errors
+        dict_ = {
+            "subdict": {"subdict_key1": "any"},
+            "ontology": "oh no not an ontology term",
+            "blob": json.dumps([{"abc": 123}, {"def": 456}])
+        }
+        errors = validate._validate_dict(dict_, "d", schema_def)
+        self.assertEqual(len(errors), 2)
+
+
+class TestDataframeValidation(unittest.TestCase):
+
+    def test_column_presence(self):
+        df = pd.DataFrame(
+            [["abc", "EFO:123"],
+             ["def", "UBERON:456"]],
+            columns=["hr_string", "ontology"],
+            index=["X", "Y"]
+        )
+
+        schema_def = {
+            "columns": {
+                "hr_string": {"type": "human-readable string"},
+                "ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]}
+            }
+        }
+        errors = validate._validate_dataframe(df, "df", schema_def)
+        self.assertFalse(errors)
+
+        schema_def = {
+            "columns": {
+                "hr_string": {"type": "human-readable string"},
+                "another_hr_string": {"type": "human-readable string"},
+                "ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]}
+            }
+        }
+        errors = validate._validate_dataframe(df, "df", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("missing column", errors[0])
+
+        # Extra is okay
+        df = pd.DataFrame(
+            [["abc", "EFO:123", "extra"],
+             ["def", "UBERON:456", "extra"]],
+            columns=["hr_string", "ontology", "extra"],
+            index=["X", "Y"]
+        )
+        schema_def = {
+            "columns": {
+                "hr_string": {"type": "human-readable string"},
+                "ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]}
+            }
+        }
+        errors = validate._validate_dataframe(df, "df", schema_def)
+        self.assertFalse(errors)
+
+
+    def test_index(self):
+        df = pd.DataFrame(
+            [["abc", "123"],
+             ["def", "456"]],
+            columns=["col1", "col2"],
+            index=["ENSG0001", "ENSG0002"]
+        )
+
+        schema_def = {"index": {"unique": True}}
+        errors = validate._validate_dataframe(df, "df", schema_def)
+        self.assertFalse(errors)
+
+        schema_def = {"index": {"type": "human-readable string"}}
+        errors = validate._validate_dataframe(df, "df", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("non-human-readable", errors[0])
+
+        df = pd.DataFrame(
+            [["abc", "123"],
+             ["def", "456"]],
+            columns=["col1", "col2"],
+            index=["ENSG0001", "ENSG0001"]
+        )
+        schema_def = {"index": {"unique": True}}
+        errors = validate._validate_dataframe(df, "df", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("is not unique", errors[0])
+
+    def test_recurse(self):
+
+        df = pd.DataFrame(
+            [["abc", "HsapDv:0001"],
+             ["EFO:123", "UBERON:456"]],
+            columns=["hr_string", "ontology"],
+            index=["X", "Y"]
+        )
+        schema_def = {
+            "columns": {
+                "hr_string": {"type": "human-readable string"},
+                "ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]}
+            }
+        }
+        errors = validate._validate_dataframe(df, "df", schema_def)
+        self.assertEqual(len(errors), 2)
+        self.assertEqual(len([e for e in errors if "non-human-readable" in e]), 1)
+        self.assertEqual(len([e for e in errors if "invalid ontology" in e]), 1)
+
+
+class TestGetSchema(unittest.TestCase):
+
+    def test_get_schema(self):
+        self.assertIsInstance(validate.get_schema_definition("1.0.0"), dict)
+
+        with self.assertRaises(ValueError):
+            validate.get_schema_definition("10.1.5")
+
+
+class TestValidate(unittest.TestCase):
+
+    def setUp(self):
+        self.source_h5ad_path = f"{PROJECT_ROOT}/backend/test/fixtures/pbmc3k-CSC-gz.h5ad"
+
+    def test_shallow(self):
+
+        adata = sc.read_h5ad(self.source_h5ad_path)
+        self.assertFalse(validate.validate_adata(adata, True))
+
+        adata.uns["version"] = {
+            "corpora_schema_version": "1.0.0",
+            "corpora_encoding_version": "0.1.0"
+        }
+        self.assertTrue(validate.validate_adata(adata, True))
+
+    def test_deep(self):
+        adata = sc.read_h5ad(self.source_h5ad_path)
+        self.assertFalse(validate.validate_adata(adata, False))
+
+        adata.uns["version"] = {
+            "corpora_schema_version": "1.0.0",
+            "corpora_encoding_version": "0.1.0"
+        }
+        self.assertFalse(validate.validate_adata(adata, False))

backend/test/test_server/unit/data_anndata/test_anndata_adaptor.py

@@ -0,0 +1,233 @@
+import json
+import sys
+import time
+import unittest
+
+import numpy as np
+import pandas as pd
+import pytest
+from parameterized import parameterized_class
+
+import backend.test.decode_fbs as decode_fbs
+from backend.common.utils.data_locator import DataLocator
+from backend.common.errors import FilterError
+from backend.server.data_anndata.anndata_adaptor import AnndataAdaptor
+from backend.test import PROJECT_ROOT, FIXTURES_ROOT
+from backend.test.test_server.unit import app_config
+from backend.test.fixtures.fixtures import pbmc3k_colors
+
+"""
+Test the anndata adaptor using the pbmc3k data set.
+"""
+
+
+@parameterized_class(
+    ("data_locator", "backed"),
+    [
+        (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", False),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", False),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", False),
+        (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", True),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", True),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", True),
+    ],
+)
+class AdaptorTest(unittest.TestCase):
+    def setUp(self):
+        config = app_config(self.data_locator, self.backed)
+        self.data = AnndataAdaptor(DataLocator(self.data_locator), config)
+
+    def test_init(self):
+        self.assertEqual(self.data.cell_count, 2638)
+        self.assertEqual(self.data.gene_count, 1838)
+        epsilon = 0.000_005
+        self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
+
+    def test_mandatory_annotations(self):
+        obs_index_col_name = self.data.get_schema()["annotations"]["obs"]["index"]
+        self.assertIn(obs_index_col_name, self.data.data.obs)
+        self.assertEqual(list(self.data.data.obs.index), list(range(2638)))
+        var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
+        self.assertIn(var_index_col_name, self.data.data.var)
+        self.assertEqual(list(self.data.data.var.index), list(range(1838)))
+
+    @pytest.mark.filterwarnings("ignore:Anndata data matrix")
+    def test_data_type(self):
+        # don't run the test on the more exotic data types, as they don't
+        # support the astype() interface (used by this test, but not underlying app)
+        if isinstance(self.data.data.X, np.ndarray):
+            self.data.data.X = self.data.data.X.astype("float64")
+            with self.assertWarns(UserWarning):
+                self.data._validate_data_types()
+
+    def test_filter_idx(self):
+        filter_ = {"filter": {"var": {"index": [1, 99, [200, 300]]}}}
+        fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 102)
+
+    def test_filter_complex(self):
+        filter_ = {
+            "filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 10}], "index": [1, 99, [200, 300]]}}
+        }
+        fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 91)
+
+    def test_obs_and_var_names(self):
+        self.assertEqual(np.sum(self.data.data.var[self.data.get_schema()["annotations"]["var"]["index"]].isna()), 0)
+        self.assertEqual(np.sum(self.data.data.obs[self.data.get_schema()["annotations"]["obs"]["index"]].isna()), 0)
+
+    def test_get_colors(self):
+        self.assertEqual(self.data.get_colors(), pbmc3k_colors)
+
+    def test_get_schema(self):
+        with open(f"{FIXTURES_ROOT}/schema.json") as fh:
+            schema = json.load(fh)
+            self.assertDictEqual(self.data.get_schema(), schema)
+
+    def test_schema_produces_error(self):
+        self.data.data.obs["time"] = pd.Series(
+            list([time.time() for i in range(self.data.cell_count)]), dtype="datetime64[ns]",
+        )
+        with pytest.raises(TypeError):
+            self.data._create_schema()
+
+    def test_layout(self):
+        fbs = self.data.layout_to_fbs_matrix(fields=None)
+        layout = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(layout["n_cols"], 6)
+        self.assertEqual(layout["n_rows"], 2638)
+
+        X = layout["columns"][0]
+        self.assertTrue((X >= 0).all() and (X <= 1).all())
+        Y = layout["columns"][1]
+        self.assertTrue((Y >= 0).all() and (Y <= 1).all())
+
+    def test_layout_fields(self):
+        """ X_pca, X_tsne, X_umap are available """
+        fbs = self.data.layout_to_fbs_matrix(["pca"])
+        layout = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(layout["n_cols"], 2)
+        self.assertEqual(layout["n_rows"], 2638)
+        self.assertCountEqual(layout["col_idx"], ["pca_0", "pca_1"])
+
+        fbs = self.data.layout_to_fbs_matrix(["tsne", "pca"])
+        layout = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(layout["n_cols"], 4)
+        self.assertEqual(layout["n_rows"], 2638)
+        self.assertCountEqual(layout["col_idx"], ["tsne_0", "tsne_1", "pca_0", "pca_1"])
+
+    def test_annotations(self):
+        fbs = self.data.annotation_to_fbs_matrix("obs")
+        annotations = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(annotations["n_rows"], 2638)
+        self.assertEqual(annotations["n_cols"], 5)
+        obs_index_col_name = self.data.get_schema()["annotations"]["obs"]["index"]
+        self.assertEqual(
+            annotations["col_idx"], [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"],
+        )
+
+        fbs = self.data.annotation_to_fbs_matrix("var")
+        annotations = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(annotations["n_rows"], 1838)
+        self.assertEqual(annotations["n_cols"], 2)
+        var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
+        self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells"])
+
+    def test_annotation_fields(self):
+        fbs = self.data.annotation_to_fbs_matrix("obs", ["n_genes", "n_counts"])
+        annotations = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(annotations["n_rows"], 2638)
+        self.assertEqual(annotations["n_cols"], 2)
+
+        var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
+        fbs = self.data.annotation_to_fbs_matrix("var", [var_index_col_name])
+        annotations = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(annotations["n_rows"], 1838)
+        self.assertEqual(annotations["n_cols"], 1)
+
+    def test_diffexp_topN(self):
+        f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
+        f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
+        result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
+        self.assertEqual(len(result), 10)
+        result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
+        self.assertEqual(len(result), 20)
+
+    def test_data_frame(self):
+        f1 = {"var": {"index": [[0, 10]]}}
+        fbs = self.data.data_frame_to_fbs_matrix(f1, "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 10)
+
+        with self.assertRaises(ValueError):
+            self.data.data_frame_to_fbs_matrix(None, "obs")
+
+    def test_filtered_data_frame(self):
+        filter_ = {"filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 100}]}}}
+        fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 1040)
+
+        filter_ = {"filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}}
+        with self.assertRaises(FilterError):
+            self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+
+    def test_data_named_gene(self):
+        var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
+        filter_ = {"filter": {"var": {"annotation_value": [{"name": var_index_col_name, "values": ["RER1"]}]}}}
+        fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 1)
+        self.assertEqual(data["col_idx"], [4])
+
+        filter_ = {
+            "filter": {"var": {"annotation_value": [{"name": var_index_col_name, "values": ["SPEN", "TYMP", "PRMT2"]}]}}
+        }
+        fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 3)
+        self.assertTrue((data["col_idx"] == [15, 1818, 1837]).all())
+
+    def test_compute_embedding(self):
+        filter = {"obs": {"index": [[0, 100]]}}
+
+        # Verify that we correctly handle the case where we lack scanpy
+        import unittest.mock
+
+        with unittest.mock.patch.dict(sys.modules, {"scanpy": None}):
+            with self.assertRaises(NotImplementedError):
+                self.data.compute_embedding("umap", filter)
+
+        # if we happen to have scanpy, test the full API, else punt
+        import importlib
+
+        scanpy_spec = importlib.util.find_spec("scanpy")
+        if scanpy_spec is None:
+            print("Skipping compute_embedding test as ScanPy not installed")
+            return
+
+        # this feature is unsupported in backed mode, and we expect an error
+        if self.data.data.isbacked:
+            with self.assertRaises(NotImplementedError):
+                self.data.compute_embedding("umap", filter)
+            return
+
+        schema = self.data.compute_embedding("umap", filter)
+
+        self.assertIsInstance(schema["name"], str)
+        name = schema["name"]
+        self.assertEqual(schema["type"], "float32")
+        self.assertEqual(schema["dims"], [f"{name}_0", f"{name}_1"])
+
+        emb = self.data.data.obsm[f"X_{name}"]
+        self.assertEqual(emb.shape, (2638, 2))
+        self.assertTrue(np.isfinite(emb[0:100]).all())
+        self.assertTrue(np.isnan(emb[100:]).all())

backend/test/test_server/unit/data_anndata/test_anndata_adaptor_data_load.py

@@ -0,0 +1,81 @@
+import unittest
+import json
+
+from backend.common.utils.data_locator import DataLocator
+from backend.server.data_anndata.anndata_adaptor import AnndataAdaptor
+from backend.server.common.config.app_config import AppConfig
+from backend.test import PROJECT_ROOT
+
+
+class DataLoadAdaptorTest(unittest.TestCase):
+    """
+    Test file loading, including deferred loading/update.
+    """
+
+    def setUp(self):
+        self.data_file = DataLocator(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
+        config = AppConfig()
+        config.update_server_config(single_dataset__datapath=self.data_file.path)
+        config.update_server_config(app__flask_secret_key="secret")
+        config.complete_config()
+        self.data = AnndataAdaptor(self.data_file, config)
+
+    def test_delayed_load_data(self):
+        self.data._create_schema()
+        self.assertEqual(self.data.cell_count, 2638)
+        self.assertEqual(self.data.gene_count, 1838)
+        epsilon = 0.000_005
+        self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
+
+    def test_diffexp_topN(self):
+        f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
+        f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
+        result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
+        self.assertEqual(len(result), 10)
+        result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
+        self.assertEqual(len(result), 20)
+
+
+class DataLocatorAdaptorTest(unittest.TestCase):
+    """
+    Test various types of data locators we expect to consume
+    """
+
+    def get_basic_config(self):
+        config = AppConfig()
+        config.update_server_config(
+            single_dataset__obs_names=None, single_dataset__var_names=None,
+        )
+        config.update_server_config(app__flask_secret_key="secret")
+        config.update_dataset_config(
+            embeddings__names=["umap"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01,
+        )
+        return config
+
+    def stdAsserts(self, data):
+        """ run these each time we load the data """
+        self.assertIsNotNone(data)
+        self.assertEqual(data.cell_count, 2638)
+        self.assertEqual(data.gene_count, 1838)
+
+    def test_posix_file(self):
+        locator = DataLocator("../../example-dataset/pbmc3k.h5ad")
+        config = self.get_basic_config()
+        config.update_server_config(single_dataset__datapath=locator.path)
+        config.complete_config()
+        data = AnndataAdaptor(locator, config)
+        self.stdAsserts(data)
+
+    def test_url_https(self):
+        url = "https://raw.githubusercontent.com/chanzuckerberg/cellxgene/main/example-dataset/pbmc3k.h5ad"
+        locator = DataLocator(url)
+        config = self.get_basic_config()
+        data = AnndataAdaptor(locator, config)
+        self.stdAsserts(data)
+
+    def test_url_http(self):
+        url = "http://raw.githubusercontent.com/chanzuckerberg/cellxgene/main/example-dataset/pbmc3k.h5ad"
+        locator = DataLocator(url)
+        config = self.get_basic_config()
+        data = AnndataAdaptor(locator, config)
+        self.stdAsserts(data)

backend/test/test_server/unit/data_anndata/test_nan_anndata_adaptor.py

@@ -0,0 +1,65 @@
+import math
+import unittest
+import warnings
+
+import pytest
+
+import backend.test.decode_fbs as decode_fbs
+from backend.common.utils.data_locator import DataLocator
+from backend.common.errors import FilterError
+from backend.server.data_anndata.anndata_adaptor import AnndataAdaptor
+from backend.test import FIXTURES_ROOT
+from backend.test.test_server.unit import app_config
+
+
+class NaNTest(unittest.TestCase):
+    def setUp(self):
+        self.data_locator = DataLocator(f"{FIXTURES_ROOT}/nan.h5ad")
+        self.config = app_config(self.data_locator.path)
+
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", category=UserWarning)
+            self.data = AnndataAdaptor(self.data_locator, self.config)
+            self.data._create_schema()
+
+    def test_load(self):
+        with self.assertLogs(level="WARN") as logger:
+            self.data = AnndataAdaptor(self.data_locator, self.config)
+            self.assertTrue(logger.output)
+
+    def test_init(self):
+        self.assertEqual(self.data.cell_count, 100)
+        self.assertEqual(self.data.gene_count, 100)
+        epsilon = 0.000_005
+        self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
+
+    def test_dataframe(self):
+        data_frame_var = decode_fbs.decode_matrix_FBS(self.data.data_frame_to_fbs_matrix(None, "var"))
+        self.assertIsNotNone(data_frame_var)
+        self.assertEqual(data_frame_var["n_rows"], 100)
+        self.assertEqual(data_frame_var["n_cols"], 100)
+        self.assertTrue(math.isnan(data_frame_var["columns"][3][3]))
+
+        with pytest.raises(FilterError):
+            self.data.data_frame_to_fbs_matrix("an erroneous filter", "var")
+        with pytest.raises(FilterError):
+            filter_ = {"filter": {"obs": {"index": [1, 99, [200, 300]]}}}
+            self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+
+    def test_dataframe_obs_not_implemented(self):
+        with self.assertRaises(ValueError) as cm:
+            decode_fbs.decode_matrix_FBS(self.data.data_frame_to_fbs_matrix(None, "obs"))
+        self.assertIsNotNone(cm.exception)
+
+    def test_annotation(self):
+        annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("obs"))
+        obs_index_col_name = self.data.schema["annotations"]["obs"]["index"]
+        self.assertEqual(annotations["col_idx"], [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"])
+        self.assertEqual(annotations["n_rows"], 100)
+        self.assertTrue(math.isnan(annotations["columns"][2][0]))
+
+        annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("var"))
+        var_index_col_name = self.data.schema["annotations"]["var"]["index"]
+        self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells", "var_with_nans"])
+        self.assertEqual(annotations["n_rows"], 100)
+        self.assertTrue(math.isnan(annotations["columns"][2][0]))