Refactor czi_hosted and server into backend directory, pull common code into backend/common, refactor tests (#2102)

* move local_server -> backend/server server-> backend/czi_hosted, pull common code into backend/common update imports, tests and make commands

backend/test/test_czi_hosted/unit/__init__.py

@@ -0,0 +1,191 @@
+import os
+import random
+import shutil
+import tempfile
+import time
+from contextlib import contextmanager
+from os import path
+from subprocess import Popen
+
+import pandas as pd
+import requests
+
+from backend.czi_hosted.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
+from backend.czi_hosted.common.annotations.local_file_csv import AnnotationsLocalFile
+from backend.czi_hosted.common.config import DEFAULT_SERVER_PORT
+from backend.czi_hosted.common.config.app_config import AppConfig
+from backend.common.utils.data_locator import DataLocator
+from backend.common.utils.utils import find_available_port
+from backend.common.fbs.matrix import encode_matrix_fbs
+from backend.czi_hosted.data_common.matrix_loader import MatrixDataType, MatrixDataLoader
+from backend.czi_hosted.db.db_utils import DbUtils
+from backend.test import PROJECT_ROOT, FIXTURES_ROOT
+
+
+def data_with_tmp_tiledb_annotations(ext: MatrixDataType):
+    tmp_dir = tempfile.mkdtemp()
+    fname = {
+        MatrixDataType.H5AD: f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
+        MatrixDataType.CXG: "test/fixtures/pbmc3k.cxg",
+    }[ext]
+    data_locator = DataLocator(fname)
+    config = AppConfig()
+    config.update_server_config(
+        app__flask_secret_key="secret",
+        multi_dataset__dataroot=data_locator.path,
+        authentication__type="test",
+        authentication__insecure_test_environment=True,
+    )
+    config.update_default_dataset_config(
+        embeddings__names=["umap"],
+        presentation__max_categories=100,
+        diffexp__lfc_cutoff=0.01,
+        user_annotations__type="hosted_tiledb_array",
+        user_annotations__hosted_tiledb_array__db_uri="postgresql://postgres:test_pw@localhost:5432",
+        user_annotations__hosted_tiledb_array__hosted_file_directory=tmp_dir,
+    )
+
+    config.complete_config()
+
+    data = MatrixDataLoader(data_locator.abspath()).open(config)
+    annotations = AnnotationsHostedTileDB(tmp_dir, DbUtils("postgresql://postgres:test_pw@localhost:5432"),)
+    return data, tmp_dir, annotations
+
+
+def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False):
+    tmp_dir = tempfile.mkdtemp()
+    annotations_file = path.join(tmp_dir, "test_annotations.csv")
+    if annotations_fixture:
+        shutil.copyfile(f"{FIXTURES_ROOT}/pbmc3k-annotations.csv", annotations_file)
+    fname = {
+        MatrixDataType.H5AD: f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
+        MatrixDataType.CXG: f"{FIXTURES_ROOT}/pbmc3k.cxg",
+    }[ext]
+    data_locator = DataLocator(fname)
+    config = AppConfig()
+    config.update_server_config(
+        app__flask_secret_key="secret",
+        single_dataset__obs_names=None,
+        single_dataset__var_names=None,
+        single_dataset__datapath=data_locator.path,
+    )
+    config.update_default_dataset_config(
+        embeddings__names=["umap"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01,
+    )
+
+    config.complete_config()
+    data = MatrixDataLoader(data_locator.abspath()).open(config)
+    annotations = AnnotationsLocalFile(None, annotations_file)
+    return data, tmp_dir, annotations
+
+
+def make_fbs(data):
+    df = pd.DataFrame(data)
+    return encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
+
+
+def skip_if(condition, reason: str):
+    def decorator(f):
+        def wraps(self, *args, **kwargs):
+            if condition(self):
+                self.skipTest(reason)
+            else:
+                f(self, *args, **kwargs)
+
+        return wraps
+
+    return decorator
+
+
+def app_config(data_locator, backed=False, extra_server_config={}, extra_dataset_config={}):
+    config = AppConfig()
+    config.update_server_config(
+        app__flask_secret_key="secret",
+        single_dataset__obs_names=None,
+        single_dataset__var_names=None,
+        adaptor__anndata_adaptor__backed=backed,
+        single_dataset__datapath=data_locator,
+        limits__diffexp_cellcount_max=None,
+        limits__column_request_max=None,
+    )
+    config.update_default_dataset_config(
+        embeddings__names=["umap", "tsne", "pca"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01
+    )
+    config.update_server_config(**extra_server_config)
+    config.update_default_dataset_config(**extra_dataset_config)
+    config.complete_config()
+    return config
+
+
+def start_test_server(command_line_args=[], app_config=None, env=None):
+    """
+    Command line arguments can be passed in, as well as an app_config.
+    This function is meant to be used like this, for example:
+
+    with unit(...) as server:
+        r = requests.get(f"{server}/...")
+        // check r
+
+    where the server can be accessed within the context, and is terminated when
+    the context is exited.
+    The port is automatically set using find_available_port, unless passed in as a command line arg.
+    The verbose flag is automatically set to True.
+    If an app_config is provided, then this function writes a temporary
+    yaml config file, which this server will read and parse.
+    """
+
+    command = ["cellxgene", "--no-upgrade-check", "launch", "--verbose"]
+    if "-p" in command_line_args:
+        port = int(command_line_args[command_line_args.index("-p") + 1])
+    elif "--port" in command_line_args:
+        port = int(command_line_args[command_line_args.index("--port") + 1])
+    else:
+        start = random.randint(DEFAULT_SERVER_PORT, 2 ** 16 - 1)
+        port = int(os.environ.get("CXG_SERVER_PORT", start))
+        port = find_available_port("localhost", port)
+        command += ["--port=%d" % port]
+
+    command += command_line_args
+
+    tempdir = None
+    if app_config:
+        tempdir = tempfile.TemporaryDirectory()
+        config_file = os.path.join(tempdir.name, "config.yaml")
+        app_config.write_config(config_file)
+        command.extend(["-c", config_file])
+
+    server = f"http://localhost:{port}"
+    ps = Popen(command, env=env)
+
+    for _ in range(10):
+        try:
+            requests.get(f"{server}/health")
+            break
+        except requests.exceptions.ConnectionError:
+            time.sleep(1)
+
+    if tempdir:
+        tempdir.cleanup()
+
+    return ps, server
+
+
+def stop_test_server(ps):
+    try:
+        ps.terminate()
+    except ProcessLookupError:
+        pass
+
+
+@contextmanager
+def test_server(command_line_args=[], app_config=None, env=None):
+    """A context to run the cellxgene server."""
+
+    ps, server = start_test_server(command_line_args, app_config, env)
+    try:
+        yield server
+    finally:
+        try:
+            stop_test_server(ps)
+        except ProcessLookupError:
+            pass

backend/test/test_czi_hosted/unit/auth/test_auth.py

@@ -0,0 +1,159 @@
+import unittest
+
+import requests
+
+from backend.czi_hosted.common.config.app_config import AppConfig
+from backend.test.test_czi_hosted.unit import test_server
+from backend.test import FIXTURES_ROOT
+
+
+class AuthTest(unittest.TestCase):
+    def setUp(self):
+        self.dataset_dataroot = FIXTURES_ROOT
+
+    def test_auth_none(self):
+        app_config = AppConfig()
+        app_config.update_server_config(app__flask_secret_key="secret")
+        app_config.update_server_config(authentication__type=None, multi_dataset__dataroot=self.dataset_dataroot)
+        app_config.update_default_dataset_config(user_annotations__enable=False)
+
+        app_config.complete_config()
+
+        with test_server(app_config=app_config) as server:
+            session = requests.Session()
+            config = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config").json()
+            userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
+            self.assertNotIn("authentication", config["config"])
+            self.assertIsNone(userinfo)
+
+    def test_auth_session(self):
+        app_config = AppConfig()
+        app_config.update_server_config(app__flask_secret_key="secret")
+        app_config.update_server_config(authentication__type="session", multi_dataset__dataroot=self.dataset_dataroot)
+        app_config.update_default_dataset_config(user_annotations__enable=True)
+        app_config.complete_config()
+
+        with test_server(app_config=app_config) as server:
+            session = requests.Session()
+            config = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config").json()
+            userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
+
+            self.assertFalse(config["config"]["authentication"]["requires_client_login"])
+            self.assertTrue(userinfo["userinfo"]["is_authenticated"])
+            self.assertEqual(userinfo["userinfo"]["username"], "anonymous")
+
+    def test_auth_test(self):
+        app_config = AppConfig()
+        app_config.update_server_config(app__flask_secret_key="secret")
+        app_config.update_server_config(authentication__type="test")
+        app_config.update_server_config(authentication__insecure_test_environment=True)
+        app_config.update_server_config(
+            multi_dataset__dataroot=dict(
+                a1=dict(dataroot=self.dataset_dataroot, base_url="auth"),
+                a2=dict(dataroot=self.dataset_dataroot, base_url="no-auth"),
+            )
+        )
+
+        # specialize the configs
+        app_config.add_dataroot_config("a1", app__authentication_enable=True, user_annotations__enable=True)
+        app_config.add_dataroot_config("a2", app__authentication_enable=False, user_annotations__enable=False)
+
+        app_config.complete_config()
+
+        with test_server(app_config=app_config) as server:
+            session = requests.Session()
+
+            # auth datasets
+            config = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/config").json()
+            userinfo = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json()
+
+            self.assertFalse(userinfo["userinfo"]["is_authenticated"])
+            self.assertIsNone(userinfo["userinfo"]["username"])
+            self.assertTrue(config["config"]["authentication"]["requires_client_login"])
+            self.assertTrue(config["config"]["parameters"]["annotations"])
+
+            login_uri = config["config"]["authentication"]["login"]
+            logout_uri = config["config"]["authentication"]["logout"]
+
+            self.assertEqual(login_uri, "/login?dataset=auth/pbmc3k.cxg")
+            self.assertEqual(logout_uri, "/logout?dataset=auth/pbmc3k.cxg")
+
+            r = session.get(f"{server}/{login_uri}")
+            # check that the login redirect worked
+            self.assertEqual(r.history[0].status_code, 302)
+            self.assertEqual(r.url, f"{server}/auth/pbmc3k.cxg")
+
+            config = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/config").json()
+            userinfo = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json()
+            self.assertTrue(userinfo["userinfo"]["is_authenticated"])
+            self.assertEqual(userinfo["userinfo"]["username"], "test_account")
+            self.assertEqual(userinfo["userinfo"]["picture"], None)
+            self.assertTrue(config["config"]["parameters"]["annotations"])
+
+            r = session.get(f"{server}/{logout_uri}")
+            # check that the logout redirect worked
+            self.assertEqual(r.history[0].status_code, 302)
+            self.assertEqual(r.url, f"{server}/auth/pbmc3k.cxg")
+            config = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/config").json()
+            userinfo = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json()
+            self.assertFalse(userinfo["userinfo"]["is_authenticated"])
+            self.assertIsNone(userinfo["userinfo"]["username"])
+            self.assertTrue(config["config"]["parameters"]["annotations"])
+
+            # no-auth datasets
+            config = session.get(f"{server}/no-auth/pbmc3k.cxg/api/v0.2/config").json()
+            userinfo = session.get(f"{server}/no-auth/pbmc3k.cxg/api/v0.2/userinfo").json()
+            self.assertIsNone(userinfo)
+            self.assertFalse(config["config"]["parameters"]["annotations"])
+
+            # login with a picture
+            session.get(f"{server}/{login_uri}&picture=myimage.png")
+            userinfo = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json()
+            self.assertTrue(userinfo["userinfo"]["is_authenticated"])
+            self.assertEqual(userinfo["userinfo"]["picture"], "myimage.png")
+
+    def test_auth_test_single(self):
+        app_config = AppConfig()
+        app_config.update_server_config(app__flask_secret_key="secret")
+        app_config.update_server_config(
+            authentication__type="test", single_dataset__datapath=f"{self.dataset_dataroot}/pbmc3k.cxg"
+        )
+        app_config.update_server_config(authentication__insecure_test_environment=True)
+
+        app_config.complete_config()
+
+        with test_server(app_config=app_config) as server:
+            session = requests.Session()
+            config = session.get(f"{server}/api/v0.2/config").json()
+            userinfo = session.get(f"{server}/api/v0.2/userinfo").json()
+            self.assertFalse(userinfo["userinfo"]["is_authenticated"])
+            self.assertIsNone(userinfo["userinfo"]["username"])
+            self.assertTrue(config["config"]["authentication"]["requires_client_login"])
+            self.assertTrue(config["config"]["parameters"]["annotations"])
+
+            login_uri = config["config"]["authentication"]["login"]
+            logout_uri = config["config"]["authentication"]["logout"]
+
+            self.assertEqual(login_uri, "/login")
+            self.assertEqual(logout_uri, "/logout")
+
+            response = session.get(f"{server}/{login_uri}")
+            # check that the login redirect worked
+            self.assertEqual(response.history[0].status_code, 302)
+            self.assertEqual(response.url, f"{server}/")
+
+            config = session.get(f"{server}/api/v0.2/config").json()
+            userinfo = session.get(f"{server}/api/v0.2/userinfo").json()
+            self.assertTrue(userinfo["userinfo"]["is_authenticated"])
+            self.assertEqual(userinfo["userinfo"]["username"], "test_account")
+            self.assertTrue(config["config"]["parameters"]["annotations"])
+
+            response = session.get(f"{server}/{logout_uri}")
+            # check that the logout redirect worked
+            self.assertEqual(response.history[0].status_code, 302)
+            self.assertEqual(response.url, f"{server}/")
+            config = session.get(f"{server}/api/v0.2/config").json()
+            userinfo = session.get(f"{server}/api/v0.2/userinfo").json()
+            self.assertFalse(userinfo["userinfo"]["is_authenticated"])
+            self.assertIsNone(userinfo["userinfo"]["username"])
+            self.assertTrue(config["config"]["parameters"]["annotations"])

backend/test/test_czi_hosted/unit/auth/test_oauth.py

@@ -0,0 +1,222 @@
+import unittest
+import random
+import time
+import base64
+import json
+import requests
+
+from flask import Flask, jsonify, make_response, request, redirect
+from multiprocessing import Process
+
+import jose
+from backend.czi_hosted.common.config.app_config import AppConfig
+from backend.test.test_czi_hosted.unit import test_server
+from backend.test import FIXTURES_ROOT
+
+# This tests the oauth authentication type.
+# This test starts a cellxgene server and a mock oauth server.
+# API requests to login and logout and get the userinfo are made
+# to the cellxgene server, which then sends requests to the mock
+# oauth server.
+
+# number of seconds that the oauth token is valid
+TOKEN_EXPIRES = 2
+
+# Create a mocked out oauth token, which servers all the endpoints needed by the oauth type.
+mock_oauth_app = Flask("mock_oauth_app")
+
+
+@mock_oauth_app.route("/authorize")
+def authorize():
+    callback = request.args.get("redirect_uri")
+    state = request.args.get("state")
+    return redirect(callback + f"?code=fakecode&state={state}")
+
+
+@mock_oauth_app.route("/oauth/token", methods=["POST"])
+def token():
+    now = time.time()
+    expires_at = now + TOKEN_EXPIRES
+    headers = dict(alg="RS256", kid="fake_kid")
+    payload = dict(name="fake_user", sub="fake_id", email="fake_user@email.com", email_verified=True, exp=expires_at)
+    jwt = jose.jwt.encode(claims=payload, key="mysecret", algorithm="HS256", headers=headers)
+    r = {
+        "access_token": f"access-{now}",
+        "id_token": jwt,
+        "refresh_token": f"random-{now}",
+        "scope": "openid profile email",
+        "expires_in": TOKEN_EXPIRES,
+        "token_type": "Bearer",
+        "expires_at": expires_at,
+    }
+    return make_response(jsonify(r))
+
+
+@mock_oauth_app.route("/v2/logout")
+def logout():
+    return_to = request.args.get("returnTo")
+    return redirect(return_to)
+
+
+@mock_oauth_app.route("/.well-known/jwks.json")
+def jwks():
+    data = dict(alg="RS256", kty="RSA", use="sig", kid="fake_kid",)
+    return make_response(jsonify(dict(keys=[data])))
+
+
+# function to launch the mock oauth server
+def launch_mock_oauth(mock_port):
+    mock_oauth_app.run(port=mock_port)
+
+
+class AuthTest(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        # The port that the mock oauth server will listen on
+        cls.mock_port = random.randint(10000, 12000)
+        cls.dataset_dataroot = FIXTURES_ROOT
+        cls.mock_oauth_process = Process(target=launch_mock_oauth, args=(cls.mock_port,))
+        cls.mock_oauth_process.start()
+
+        # Verify that the mock oauth server is ready (accepting requests) before starting the tests.
+
+        # The following lines are polling until the mock server is ready.
+        # The issue is we are starting a mock oauth server, then we are starting a cellxgene server,
+        # which will start making requests to the mock oauth server.
+        # So there is a race condition because the mock oauth server needs to be ready before it gets requests.
+        # We check to see if it is ready, and if not we wait 1 second, then try again.
+        # If it gets to 5 seconds, which is shouldn't, we assume something has gone wrong and fail the test.
+        server_okay = False
+        for _ in range(5):
+            try:
+                response = requests.get(f"http://localhost:{cls.mock_port}/.well-known/jwks.json")
+                if response.status_code == 200:
+                    server_okay = True
+                    break
+            except:  # noqa: E722
+                pass
+
+            # wait one second and try again
+            time.sleep(1)
+
+        assert(server_okay)
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.mock_oauth_process.terminate()
+
+    def auth_flow(self, app_config, cookie_key=None):
+
+        app_config.update_server_config(
+            app__api_base_url="local",
+            authentication__type="oauth",
+            authentication__params_oauth__oauth_api_base_url=f"http://localhost:{self.mock_port}",
+            authentication__params_oauth__client_id="mock_client_id",
+            authentication__params_oauth__client_secret="mock_client_secret",
+            authentication__params_oauth__jwt_decode_options={"verify_signature": False, "verify_iss": False},
+        )
+
+        app_config.update_server_config(multi_dataset__dataroot=self.dataset_dataroot)
+        app_config.complete_config()
+
+        with test_server(app_config=app_config) as server:
+            session = requests.Session()
+
+            # auth datasets
+            config = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config").json()
+            userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
+
+            self.assertFalse(userinfo["userinfo"]["is_authenticated"])
+            self.assertIsNone(userinfo["userinfo"]["username"])
+            self.assertTrue(config["config"]["authentication"]["requires_client_login"])
+            self.assertTrue(config["config"]["parameters"]["annotations"])
+
+            login_uri = config["config"]["authentication"]["login"]
+            logout_uri = config["config"]["authentication"]["logout"]
+
+            self.assertEqual(login_uri, f"{server}/login?dataset=d/pbmc3k.cxg/")
+            self.assertEqual(logout_uri, f"{server}/logout?dataset=d/pbmc3k.cxg/")
+
+            r = session.get(login_uri)
+            # check that the login redirect worked
+            self.assertEqual(r.history[0].status_code, 302)
+            self.assertEqual(r.url, f"{server}/d/pbmc3k.cxg/")
+            config = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config").json()
+            userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
+            self.assertTrue(userinfo["userinfo"]["is_authenticated"])
+            self.assertEqual(userinfo["userinfo"]["username"], "fake_user")
+            self.assertEqual(userinfo["userinfo"]["email"], "fake_user@email.com")
+            self.assertTrue(config["config"]["parameters"]["annotations"])
+
+            if cookie_key:
+                cookie = session.cookies.get(cookie_key)
+                token = json.loads(base64.b64decode(cookie))
+                access_token_before = token.get("access_token")
+                id_token_before = token.get("id_token")
+
+                # let the token expire
+                time.sleep(TOKEN_EXPIRES + 1)
+
+                # check that refresh works
+                session.get(login_uri)
+                userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
+                self.assertTrue(userinfo["userinfo"]["is_authenticated"])
+                self.assertEqual(userinfo["userinfo"]["username"], "fake_user")
+
+                cookie = session.cookies.get(cookie_key)
+                token = json.loads(base64.b64decode(cookie))
+                access_token_after = token.get("access_token")
+                id_token_after = token.get("id_token")
+
+                self.assertNotEqual(access_token_before, access_token_after)
+                self.assertNotEqual(id_token_before, id_token_after)
+
+                # invalid cookie is rejected
+                session.cookies.set(cookie_key, "TEST_" + cookie)
+                self.assertTrue(cookie_key in session.cookies)
+                response = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo")
+                # this is not an error, the invalid cookie is just ignored.
+                self.assertEqual(response.status_code, 200)
+                userinfo = response.json()
+                self.assertFalse(userinfo["userinfo"]["is_authenticated"])
+                self.assertIsNone(userinfo["userinfo"]["username"])
+
+                # invalid id_token is rejected
+                test_token = token
+                test_token["id_token"] = "TEST_" + id_token_after
+                encoded_cookie = base64.b64encode(json.dumps(test_token).encode()).decode()
+                session.cookies.set(cookie_key, encoded_cookie)
+                response = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo")
+                # this is not an error, the invalid id_token is just ignored.
+                self.assertEqual(response.status_code, 200)
+                userinfo = response.json()
+                self.assertFalse(userinfo["userinfo"]["is_authenticated"])
+                self.assertIsNone(userinfo["userinfo"]["username"])
+
+            r = session.get(logout_uri)
+            # check that the logout redirect worked
+            self.assertEqual(r.history[0].status_code, 302)
+            self.assertEqual(r.url, f"{server}/d/pbmc3k.cxg/")
+            config = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config").json()
+            userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
+            self.assertFalse(userinfo["userinfo"]["is_authenticated"])
+            self.assertIsNone(userinfo["userinfo"]["username"])
+            self.assertTrue(config["config"]["parameters"]["annotations"])
+
+    def test_auth_oauth_session(self):
+        # test with session cookies
+        app_config = AppConfig()
+        app_config.update_server_config(app__flask_secret_key="secret")
+        app_config.update_server_config(authentication__params_oauth__session_cookie=True,)
+        self.auth_flow(app_config)
+
+    def test_auth_oauth_cookie(self):
+        # test with specified cookie
+        app_config = AppConfig()
+        app_config.update_server_config(app__flask_secret_key="secret")
+        app_config.update_server_config(
+            authentication__params_oauth__session_cookie=False,
+            authentication__params_oauth__cookie=dict(key="test_cxguser", httponly=True, max_age=60),
+        )
+
+        self.auth_flow(app_config, "test_cxguser")

backend/test/test_czi_hosted/unit/cli/test_launch.py

@@ -0,0 +1,27 @@
+import filecmp
+import os
+import shutil
+import unittest
+
+import yaml
+
+from backend.czi_hosted.default_config import default_config
+from backend.test import FIXTURES_ROOT
+
+
+class CLIPLaunchTests(unittest.TestCase):
+    tmp_dir = os.path.join(FIXTURES_ROOT, "dump_configs")
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        os.mkdir(cls.tmp_dir)
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        shutil.rmtree(cls.tmp_dir)
+
+    def test_dump_default_config(self):
+        os.system(f"cellxgene launch --dump-default-config > {self.tmp_dir}/test_config_dump.txt")
+        with open(f"{self.tmp_dir}/expected_config_dump.txt", "w") as expected_config:
+            expected_config.write(yaml.dump(default_config))
+        filecmp.cmp(f"{self.tmp_dir}/expected_config_dump.txt", f"{self.tmp_dir}/test_config_dump.txt")

backend/test/test_czi_hosted/unit/cli/test_prepare.py

@@ -0,0 +1,15 @@
+import unittest
+
+import pandas as pd
+
+from backend.czi_hosted.cli.prepare import make_index_unique
+
+
+class CLIPrepareTests(unittest.TestCase):
+    """ Test cases for CLI prepare logic """
+
+    def test_make_index_unique(self):
+        index = pd.Index(["SNORD113", "SNORD113", "SNORD113-1"])
+        result = make_index_unique(index)
+        expected = pd.Index(["SNORD113", "SNORD113-2", "SNORD113-1"])
+        self.assertTrue(all(left == right for left, right in zip(result.values, expected.values)))

backend/test/test_czi_hosted/unit/cli/test_upgrade.py

@@ -0,0 +1,28 @@
+import unittest
+
+from backend.czi_hosted.cli.upgrade import validate_version_str, split_version, version_gt
+
+
+class CLIUpgradeTests(unittest.TestCase):
+    """ Test cases for CLI logic """
+
+    def test_validate_version_str(self):
+        self.assertTrue(validate_version_str("0.1.2"))
+        self.assertTrue(validate_version_str("0.1.2-RC", release_only=False))
+        self.assertFalse(validate_version_str("0.1"))
+        self.assertFalse(validate_version_str("0.1.2.3"))
+        self.assertFalse(validate_version_str("0.1.2-RC"))
+
+    def test_split_version_str(self):
+        self.assertEqual(split_version("0.1.2"), [0, 1, 2])
+        with self.assertRaises(AttributeError):
+            split_version("0.1")
+
+    def test_assert_verstion_gt(self):
+        self.assertTrue(version_gt("1.0.0", "0.1.1"))
+        self.assertTrue(version_gt("0.1.0", "0.0.1"))
+        self.assertTrue(version_gt("0.0.1", "0.0.0"))
+        self.assertFalse(version_gt("0.0.0", "0.0.0"))
+        self.assertFalse(version_gt("0.0.0", "0.0.1"))
+        self.assertFalse(version_gt("0.0.1", "0.1.0"))
+        self.assertFalse(version_gt("0.1.1", "1.0.0"))

backend/test/test_czi_hosted/unit/common/config/__init__.py

@@ -0,0 +1,284 @@
+import os
+import shutil
+import unittest
+import random
+from unittest import mock
+import yaml
+
+from backend.test import FIXTURES_ROOT
+
+
+def mockenv(**envvars):
+    return mock.patch.dict(os.environ, envvars)
+
+
+class ConfigTests(unittest.TestCase):
+    tmp_fixtures_directory = os.path.join(FIXTURES_ROOT, "tmp_dir")
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        shutil.rmtree(cls.tmp_fixtures_directory)
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        os.makedirs(cls.tmp_fixtures_directory)
+
+    def custom_server_config(
+        self,
+        verbose="false",
+        debug="false",
+        host="localhost",
+        port="null",
+        open_browser="false",
+        force_https="false",
+        flask_secret_key="secret",
+        generate_cache_control_headers="false",
+        server_timing_headers="false",
+        csp_directives="null",
+        api_base_url="null",
+        web_base_url="null",
+        auth_type="session",
+        insecure_test_environment="false",
+        oauth_api_base_url="null",
+        client_id="null",
+        client_secret="null",
+        jwt_decode_options="null",
+        session_cookie="true",
+        cookie="null",
+        dataroot="null",
+        index="false",
+        allowed_matrix_types=[],
+        max_cached_datasets=5,
+        timelimit_s=5,
+        dataset_datapath="null",
+        obs_names="null",
+        var_names="null",
+        about="null",
+        title="null",
+        diffexp_max_workers=64,
+        cpu_multiplier=4,
+        target_workunit="16_000_000",
+        data_locater_region_name="us-east-1",
+        cxg_tile_cache_size=8589934592,
+        cxg_num_reader_threads=32,
+        anndata_backed="false",
+        column_request_max=32,
+        diffexp_cellcount_max="null",
+        config_file_name="server_config.yaml",
+    ):
+        configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
+        server_config_outline_path = os.path.join(FIXTURES_ROOT, "czi_hosted_server_config_outline.py")
+        with open(server_config_outline_path, "r") as config_skeleton:
+            config = config_skeleton.read()
+            server_config = eval(config)
+        with open(configfile, "w") as server_config_file:
+            server_config_file.write(server_config)
+        return configfile
+
+    def custom_app_config(
+        self,
+        verbose="false",
+        debug="false",
+        host="localhost",
+        port="null",
+        open_browser="false",
+        force_https="false",
+        flask_secret_key="secret",
+        generate_cache_control_headers="false",
+        server_timing_headers="false",
+        csp_directives="null",
+        api_base_url="null",
+        web_base_url="null",
+        auth_type="session",
+        oauth_api_base_url="null",
+        client_id="null",
+        client_secret="null",
+        jwt_decode_options="null",
+        session_cookie="true",
+        cookie="null",
+        dataroot="null",
+        index="false",
+        allowed_matrix_types=[],
+        max_cached_datasets=5,
+        timelimit_s=5,
+        dataset_datapath="null",
+        obs_names="null",
+        var_names="null",
+        about="null",
+        title="null",
+        diffexp_max_workers=64,
+        cpu_multiplier=4,
+        target_workunit="16_000_000",
+        data_locater_region_name="us-east-1",
+        cxg_tile_cache_size=8589934592,
+        cxg_num_reader_threads=32,
+        anndata_backed="false",
+        column_request_max=32,
+        diffexp_cellcount_max="null",
+        scripts=[],
+        inline_scripts=[],
+        about_legal_tos="null",
+        about_legal_privacy="null",
+        authentication_enable="true",
+        max_categories=1000,
+        custom_colors="true",
+        enable_users_annotations="true",
+        annotation_type="local_file_csv",
+        db_uri="null",
+        hosted_file_directory="null",
+        local_file_csv_directory="null",
+        local_file_csv_file="null",
+        ontology_enabled="false",
+        obo_location="null",
+        embedding_names=[],
+        enable_reembedding="false",
+        enable_difexp="true",
+        lfc_cutoff=0.01,
+        top_n=10,
+        environment=None,
+        aws_secrets_manager_region=None,
+        aws_secrets_manager_secrets=[],
+        config_file_name="app_config.yml",
+    ):
+        random_num = random.randrange(999999)
+        configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
+        server_config = self.custom_server_config(
+            verbose=verbose,
+            debug=debug,
+            host=host,
+            port=port,
+            open_browser=open_browser,
+            force_https=force_https,
+            flask_secret_key=flask_secret_key,
+            generate_cache_control_headers=generate_cache_control_headers,
+            server_timing_headers=server_timing_headers,
+            csp_directives=csp_directives,
+            api_base_url=api_base_url,
+            web_base_url=web_base_url,
+            auth_type=auth_type,
+            oauth_api_base_url=oauth_api_base_url,
+            client_id=client_id,
+            client_secret=client_secret,
+            jwt_decode_options=jwt_decode_options,
+            session_cookie=session_cookie,
+            cookie=cookie,
+            dataroot=dataroot,
+            index=index,
+            allowed_matrix_types=allowed_matrix_types,
+            max_cached_datasets=max_cached_datasets,
+            timelimit_s=timelimit_s,
+            dataset_datapath=dataset_datapath,
+            obs_names=obs_names,
+            var_names=var_names,
+            about=about,
+            title=title,
+            diffexp_max_workers=diffexp_max_workers,
+            cpu_multiplier=cpu_multiplier,
+            target_workunit=target_workunit,
+            data_locater_region_name=data_locater_region_name,
+            cxg_tile_cache_size=cxg_tile_cache_size,
+            cxg_num_reader_threads=cxg_num_reader_threads,
+            anndata_backed=anndata_backed,
+            column_request_max=column_request_max,
+            diffexp_cellcount_max=diffexp_cellcount_max,
+            config_file_name=f"temp_server_config_{random_num}.yml",
+        )
+        dataset_config = self.custom_dataset_config(
+            scripts=scripts,
+            inline_scripts=inline_scripts,
+            about_legal_tos=about_legal_tos,
+            about_legal_privacy=about_legal_privacy,
+            authentication_enable=authentication_enable,
+            max_categories=max_categories,
+            custom_colors=custom_colors,
+            enable_users_annotations=enable_users_annotations,
+            annotation_type=annotation_type,
+            db_uri=db_uri,
+            hosted_file_directory=hosted_file_directory,
+            local_file_csv_directory=local_file_csv_directory,
+            local_file_csv_file=local_file_csv_file,
+            ontology_enabled=ontology_enabled,
+            obo_location=obo_location,
+            embedding_names=embedding_names,
+            enable_reembedding=enable_reembedding,
+            enable_difexp=enable_difexp,
+            lfc_cutoff=lfc_cutoff,
+            top_n=top_n,
+            config_file_name=f"temp_dataset_config_{random_num}.yml",
+        )
+        external_config = self.custom_external_config(
+            environment=environment,
+            aws_secrets_manager_region=aws_secrets_manager_region,
+            aws_secrets_manager_secrets=aws_secrets_manager_secrets,
+            config_file_name=f"temp_external_config_{random_num}.yml",
+        )
+
+        with open(configfile, "w") as app_config_file:
+            app_config_file.write(open(server_config).read())
+            app_config_file.write(open(dataset_config).read())
+            app_config_file.write(open(external_config).read())
+
+        return configfile
+
+    def custom_dataset_config(
+        self,
+        scripts=[],
+        inline_scripts=[],
+        about_legal_tos="null",
+        about_legal_privacy="null",
+        authentication_enable="true",
+        max_categories=1000,
+        custom_colors="true",
+        enable_users_annotations="true",
+        annotation_type="local_file_csv",
+        db_uri="null",
+        hosted_file_directory="null",
+        local_file_csv_directory="null",
+        local_file_csv_file="null",
+        ontology_enabled="false",
+        obo_location="null",
+        embedding_names=[],
+        enable_reembedding="false",
+        enable_difexp="true",
+        lfc_cutoff=0.01,
+        top_n=10,
+        config_file_name="dataset_config.yml",
+    ):
+        configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
+        dataset_config_outline_path = os.path.join(FIXTURES_ROOT, "czi_hosted_dataset_config_outline.py")
+        with open(dataset_config_outline_path, "r") as config_skeleton:
+            config = config_skeleton.read()
+            dataset_config = eval(config)
+        with open(configfile, "w") as dataset_config_file:
+            dataset_config_file.write(dataset_config)
+
+        return configfile
+
+    def custom_external_config(
+        self,
+        environment=None,
+        aws_secrets_manager_region=None,
+        aws_secrets_manager_secrets=[],
+        config_file_name="external_config.yaml",
+    ):
+        # set to the default if environment is None
+        if environment is None:
+            environment = [
+                dict(name="CXG_SECRET_KEY", path=["server", "app", "flask_secret_key"], required=False),
+                dict(
+                    name="CXG_OAUTH_CLIENT_SECRET",
+                    path=["server", "authentication", "params_oauth", "client_secret"],
+                    required=False,
+                ),
+            ]
+        external_config = {
+            "external": {
+                "environment": environment,
+                "aws_secrets_manager": {"region": aws_secrets_manager_region, "secrets": aws_secrets_manager_secrets},
+            }
+        }
+
+        configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
+        with open(configfile, "w") as external_config_file:
+            yaml.dump(external_config, external_config_file)
+        return configfile

backend/test/test_czi_hosted/unit/common/config/test_app_config.py

@@ -0,0 +1,220 @@
+import os
+import tempfile
+import unittest
+
+import yaml
+
+from backend.czi_hosted.common.config.app_config import AppConfig
+from backend.common.errors import ConfigurationError
+from backend.czi_hosted.default_config import default_config
+from backend.test import FIXTURES_ROOT
+from backend.test.test_czi_hosted.unit.common.config import ConfigTests
+
+
+class AppConfigTest(ConfigTests):
+    def setUp(self):
+        self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
+        self.config = AppConfig()
+        self.config.update_server_config(app__flask_secret_key="secret")
+        self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
+        self.server_config = self.config.server_config
+        self.config.complete_config()
+
+        message_list = []
+
+        def noop(message):
+            message_list.append(message)
+
+        messagefn = noop
+        self.context = dict(messagefn=messagefn, messages=message_list)
+
+    def get_config(self, **kwargs):
+        file_name = self.custom_app_config(
+            dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        return config
+
+    def test_get_default_config_correctly_reads_default_config_file(self):
+        app_default_config = AppConfig().default_config
+
+        expected_config = yaml.load(default_config, Loader=yaml.Loader)
+
+        server_config = app_default_config["server"]
+        dataset_config = app_default_config["dataset"]
+
+        expected_server_config = expected_config["server"]
+        expected_dataset_config = expected_config["dataset"]
+
+        self.assertDictEqual(app_default_config, expected_config)
+        self.assertDictEqual(server_config, expected_server_config)
+        self.assertDictEqual(dataset_config, expected_dataset_config)
+
+    def test_get_dataset_config_returns_default_dataset_config_for_single_datasets(self):
+        datapath = f"{FIXTURES_ROOT}/1e4dfec4-c0b2-46ad-a04e-ff3ffb3c0a8f.h5ad"
+        file_name = self.custom_app_config(dataset_datapath=datapath, config_file_name=self.config_file_name)
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+
+        self.assertEqual(config.get_dataset_config(""), config.default_dataset_config)
+
+    def test_update_server_config_updates_server_config_and_config_status(self):
+        config = self.get_config()
+        config.complete_config()
+        config.check_config()
+        config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
+        with self.assertRaises(ConfigurationError):
+            config.server_config.check_config()
+
+    def test_write_config_outputs_yaml_with_all_config_vars(self):
+        config = self.get_config()
+        config.write_config(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml")
+        with open(f"{FIXTURES_ROOT}/tmp_dir/{self.config_file_name}", "r") as default_config:
+            default_config_yml = yaml.safe_load(default_config)
+
+        with open(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml", "r") as output_config:
+            output_config_yml = yaml.safe_load(output_config)
+        self.maxDiff = None
+        self.assertEqual(default_config_yml, output_config_yml)
+
+    def test_update_app_config(self):
+        config = AppConfig()
+        config.update_server_config(app__verbose=True, multi_dataset__dataroot="datadir")
+        vars = config.server_config.changes_from_default()
+        self.assertCountEqual(vars, [("app__verbose", True, False), ("multi_dataset__dataroot", "datadir", None)])
+
+        config = AppConfig()
+        config.update_default_dataset_config(app__scripts=(), app__inline_scripts=())
+        vars = config.server_config.changes_from_default()
+        self.assertCountEqual(vars, [])
+
+        config = AppConfig()
+        config.update_default_dataset_config(app__scripts=[], app__inline_scripts=[])
+        vars = config.default_dataset_config.changes_from_default()
+        self.assertCountEqual(vars, [])
+
+        config = AppConfig()
+        config.update_default_dataset_config(app__scripts=("a", "b"), app__inline_scripts=["c", "d"])
+        vars = config.default_dataset_config.changes_from_default()
+        self.assertCountEqual(vars, [("app__scripts", ["a", "b"], []), ("app__inline_scripts", ["c", "d"], [])])
+
+    def test_configfile_no_dataset_section(self):
+        # test a config file without a dataset section
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            configfile = os.path.join(tempdir, "config.yaml")
+            with open(configfile, "w") as fconfig:
+                config = """
+                server:
+                    app:
+                        flask_secret_key: secret
+                    multi_dataset:
+                        dataroot: test_dataroot
+
+                """
+                fconfig.write(config)
+
+            app_config = AppConfig()
+            app_config.update_from_config_file(configfile)
+            server_changes = app_config.server_config.changes_from_default()
+            dataset_changes = app_config.default_dataset_config.changes_from_default()
+            self.assertEqual(
+                server_changes,
+                [("app__flask_secret_key", "secret", None), ("multi_dataset__dataroot", "test_dataroot", None)],
+            )
+            self.assertEqual(dataset_changes, [])
+
+    def test_configfile_no_server_section(self):
+        # test a config file without a dataset section
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            configfile = os.path.join(tempdir, "config.yaml")
+            with open(configfile, "w") as fconfig:
+                config = """
+                dataset:
+                    user_annotations:
+                        enable: false
+                """
+                fconfig.write(config)
+
+            app_config = AppConfig()
+            app_config.update_from_config_file(configfile)
+            server_changes = app_config.server_config.changes_from_default()
+            dataset_changes = app_config.default_dataset_config.changes_from_default()
+            self.assertEqual(server_changes, [])
+            self.assertEqual(dataset_changes, [("user_annotations__enable", False, True)])
+
+    def test_simple_update_single_config_from_path_and_value(self):
+        """Update a simple config parameter"""
+
+        config = AppConfig()
+        config.server_config.multi_dataset__dataroot = dict(
+            s1=dict(dataroot="my_dataroot_s1", base_url="my_baseurl_s1"),
+            s2=dict(dataroot="my_dataroot_s2", base_url="my_baseurl_s2"),
+        )
+        config.add_dataroot_config("s1")
+        config.add_dataroot_config("s2")
+
+        # test simple value in server
+        config.update_single_config_from_path_and_value(["server", "app", "flask_secret_key"], "mysecret")
+        self.assertEqual(config.server_config.app__flask_secret_key, "mysecret")
+
+        # test simple value in default dataset
+        config.update_single_config_from_path_and_value(
+            ["dataset", "user_annotations", "hosted_tiledb_array", "db_uri"], "mydburi",
+        )
+        self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mydburi")
+        self.assertEqual(config.dataroot_config["s1"].user_annotations__hosted_tiledb_array__db_uri, "mydburi")
+        self.assertEqual(config.dataroot_config["s2"].user_annotations__hosted_tiledb_array__db_uri, "mydburi")
+
+        # test simple value in specific dataset
+        config.update_single_config_from_path_and_value(
+            ["per_dataset_config", "s1", "user_annotations", "hosted_tiledb_array", "db_uri"], "s1dburi"
+        )
+        self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mydburi")
+        self.assertEqual(config.dataroot_config["s1"].user_annotations__hosted_tiledb_array__db_uri, "s1dburi")
+        self.assertEqual(config.dataroot_config["s2"].user_annotations__hosted_tiledb_array__db_uri, "mydburi")
+
+        # error checking
+        bad_paths = [
+            (
+                ["dataset", "does", "not", "exist"],
+                "unknown config parameter at path: '['dataset', 'does', 'not', 'exist']'",
+            ),
+            (["does", "not", "exist"], "path must start with 'server', 'dataset', or 'per_dataset_config'"),
+            ([], "path must start with 'server', 'dataset', or 'per_dataset_config'"),
+            (["per_dataset_config"], "missing dataroot when using per_dataset_config: got '['per_dataset_config']'"),
+            (
+                ["per_dataset_config", "unknown"],
+                "unknown dataroot when using per_dataset_config: got '['per_dataset_config', 'unknown']',"
+                " dataroots specified in config are ['s1', 's2']",
+            ),
+            ([1, 2, 3], "path must be a list of strings, got '[1, 2, 3]'"),
+            ("string", "path must be a list of strings, got 'string'"),
+        ]
+        for bad_path, error_message in bad_paths:
+            with self.assertRaises(ConfigurationError) as config_error:
+                config.update_single_config_from_path_and_value(bad_path, "value")
+
+            self.assertEqual(config_error.exception.message, error_message)
+
+    def test_dict_update_single_config_from_path_and_value(self):
+        """Update a config parameter that has a value of dict"""
+
+        # the path leads to a dict config param, set the config parameter to the new value
+        config = AppConfig()
+        config.update_single_config_from_path_and_value(
+            ["server", "authentication", "params_oauth", "cookie"], dict(key="mykey1", max_age=100)
+        )
+        self.assertEqual(config.server_config.authentication__params_oauth__cookie, dict(key="mykey1", max_age=100))
+
+        # the path leads to an entry within a dict config param, the value is simple
+        config = AppConfig()
+        config.server_config.authentication__params_oauth__cookie = dict(key="mykey1", max_age=100)
+        config.update_single_config_from_path_and_value(
+            ["server", "authentication", "params_oauth", "cookie", "httponly"], True,
+        )
+        self.assertEqual(
+            config.server_config.authentication__params_oauth__cookie, dict(key="mykey1", max_age=100, httponly=True)
+        )

backend/test/test_czi_hosted/unit/common/config/test_base_config.py

@@ -0,0 +1,65 @@
+import unittest
+
+from backend.czi_hosted.common.config.app_config import AppConfig
+from backend.test import FIXTURES_ROOT
+from backend.common.errors import ConfigurationError
+from backend.test.test_czi_hosted.unit.common.config import ConfigTests
+
+
+class BaseConfigTest(ConfigTests):
+    def setUp(self):
+        self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
+        self.config = AppConfig()
+        self.config.update_server_config(app__flask_secret_key="secret")
+        self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
+        self.server_config = self.config.server_config
+        self.config.complete_config()
+
+        message_list = []
+
+        def noop(message):
+            message_list.append(message)
+
+        messagefn = noop
+        self.context = dict(messagefn=messagefn, messages=message_list)
+
+    def get_config(self, **kwargs):
+        file_name = self.custom_app_config(
+            dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        return config
+
+    def test_mapping_creation_returns_map_of_server_and_dataset_config(self):
+        config = AppConfig()
+        mapping = config.default_dataset_config.create_mapping(config.default_config)
+        self.assertIsNotNone(mapping["server__app__verbose"])
+        self.assertIsNotNone(mapping["dataset__presentation__max_categories"])
+        self.assertIsNotNone(mapping["dataset__user_annotations__ontology__obo_location"])
+        self.assertIsNotNone(mapping["server__multi_dataset__allowed_matrix_types"])
+
+    def test_changes_from_default_returns_list_of_nondefault_config_values(self):
+        config = self.get_config(verbose="true", lfc_cutoff=0.05)
+        server_changes = config.server_config.changes_from_default()
+        dataset_changes = config.default_dataset_config.changes_from_default()
+
+        self.assertEqual(
+            server_changes,
+            [
+                ("app__verbose", True, False),
+                ("app__flask_secret_key", "secret", None),
+                ("multi_dataset__dataroot", FIXTURES_ROOT, None),
+                ("multi_dataset__matrix_cache__timelimit_s", 5, 30),
+                ("data_locator__s3__region_name", "us-east-1", True),
+            ],
+        )
+        self.assertEqual(dataset_changes, [("diffexp__lfc_cutoff", 0.05, 0.01)])
+
+    def test_check_config_throws_error_if_attr_has_not_been_checked(self):
+        config = self.get_config(verbose="true")
+        config.complete_config()
+        config.check_config()
+        config.update_server_config(app__verbose=False)
+        with self.assertRaises(ConfigurationError):
+            config.check_config()

backend/test/test_czi_hosted/unit/common/config/test_dataset_config.py

@@ -0,0 +1,265 @@
+import os
+import tempfile
+
+import requests
+import unittest
+from unittest.mock import patch
+
+from backend.czi_hosted.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
+from backend.czi_hosted.common.annotations.local_file_csv import AnnotationsLocalFile
+from backend.czi_hosted.common.config.app_config import AppConfig
+from backend.czi_hosted.common.config.base_config import BaseConfig
+from backend.test.test_czi_hosted.unit import test_server
+from backend.test import PROJECT_ROOT, FIXTURES_ROOT
+
+from backend.common.errors import ConfigurationError
+from backend.test.test_czi_hosted.unit.common.config import ConfigTests
+
+
+class TestDatasetConfig(ConfigTests):
+    def setUp(self):
+        self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
+        self.config = AppConfig()
+        self.config.update_server_config(app__flask_secret_key="secret")
+        self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
+        self.dataset_config = self.config.default_dataset_config
+        self.config.complete_config()
+        message_list = []
+
+        def noop(message):
+            message_list.append(message)
+
+        messagefn = noop
+        self.context = dict(messagefn=messagefn, messages=message_list)
+
+    def get_config(self, **kwargs):
+        file_name = self.custom_app_config(
+            dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        return config
+
+    def test_init_datatset_config_sets_vars_from_default_config(self):
+        config = AppConfig()
+        self.assertEqual(config.default_dataset_config.presentation__max_categories, 1000)
+        self.assertEqual(config.default_dataset_config.user_annotations__type, "local_file_csv")
+        self.assertEqual(config.default_dataset_config.diffexp__lfc_cutoff, 0.01)
+        self.assertIsNone(config.default_dataset_config.user_annotations__ontology__obo_location)
+
+    @patch("backend.czi_hosted.common.config.dataset_config.BaseConfig.validate_correct_type_of_configuration_attribute")
+    def test_complete_config_checks_all_attr(self, mock_check_attrs):
+        mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
+        self.dataset_config.complete_config(self.context)
+        self.assertEqual(mock_check_attrs.call_count, 21)
+
+    def test_app_sets_script_vars(self):
+        config = self.get_config(scripts=["path/to/script"])
+        config.default_dataset_config.handle_app()
+
+        self.assertEqual(config.default_dataset_config.app__scripts, [{"src": "path/to/script"}])
+
+        config = self.get_config(scripts=[{"src": "path/to/script", "more": "different/script/path"}])
+        config.default_dataset_config.handle_app()
+        self.assertEqual(
+            config.default_dataset_config.app__scripts, [{"src": "path/to/script", "more": "different/script/path"}]
+        )
+
+        config = self.get_config(scripts=["path/to/script", "different/script/path"])
+        config.default_dataset_config.handle_app()
+        # TODO @madison -- is this the desired functionality?
+        self.assertEqual(
+            config.default_dataset_config.app__scripts, [{"src": "path/to/script"}, {"src": "different/script/path"}]
+        )
+
+        config = self.get_config(scripts=[{"more": "different/script/path"}])
+        with self.assertRaises(ConfigurationError):
+            config.default_dataset_config.handle_app()
+
+    def test_handle_user_annotations_ensures_auth_is_enabled_with_valid_auth_type(self):
+        config = self.get_config(enable_users_annotations="true", authentication_enable="false")
+        config.server_config.complete_config(self.context)
+        with self.assertRaises(ConfigurationError):
+            config.default_dataset_config.handle_user_annotations(self.context)
+
+        config = self.get_config(enable_users_annotations="true", authentication_enable="true", auth_type="pretend")
+        with self.assertRaises(ConfigurationError):
+            config.server_config.complete_config(self.context)
+
+    def test_handle_user_annotations__adds_warning_message_if_annotation_vars_set_when_annotations_disabled(self):
+        config = self.get_config(
+            enable_users_annotations="false", authentication_enable="false", db_uri="shouldnt/be/set"
+        )
+        config.default_dataset_config.handle_user_annotations(self.context)
+
+        self.assertEqual(self.context["messages"], ["Warning: db_uri ignored as annotations are disabled."])
+
+    @patch("backend.czi_hosted.common.config.dataset_config.DbUtils")
+    def test_handle_user_annotations__instantiates_user_annotations_class_correctly(self, mock_db_utils):
+        mock_db_utils.return_value = "123"
+        config = self.get_config(
+            enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv"
+        )
+        config.server_config.complete_config(self.context)
+        config.default_dataset_config.handle_user_annotations(self.context)
+        self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsLocalFile)
+
+        config = self.get_config(
+            enable_users_annotations="true",
+            authentication_enable="true",
+            annotation_type="hosted_tiledb_array",
+            db_uri="gotta/set/this",
+            hosted_file_directory="and/this",
+        )
+        config.server_config.complete_config(self.context)
+        config.default_dataset_config.handle_user_annotations(self.context)
+        self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsHostedTileDB)
+
+        config = self.get_config(
+            enable_users_annotations="true", authentication_enable="true", annotation_type="NOT_REAL"
+        )
+        config.server_config.complete_config(self.context)
+        with self.assertRaises(ConfigurationError):
+            config.default_dataset_config.handle_user_annotations(self.context)
+
+    def test_handle_local_file_csv_annotations__sets_dir_if_not_passed_in(self):
+        config = self.get_config(
+            enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv"
+        )
+        config.server_config.complete_config(self.context)
+        config.default_dataset_config.handle_local_file_csv_annotations()
+        self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsLocalFile)
+        cwd = os.getcwd()
+        self.assertEqual(config.default_dataset_config.user_annotations._get_output_dir(), cwd)
+
+    def test_handle_embeddings__checks_data_file_types(self):
+        file_name = self.custom_app_config(
+            embedding_names=["name1", "name2"],
+            enable_reembedding="true",
+            dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad",
+            anndata_backed="true",
+            config_file_name=self.config_file_name,
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        config.server_config.complete_config(self.context)
+        with self.assertRaises(ConfigurationError):
+            config.default_dataset_config.handle_embeddings()
+
+    def test_handle_diffexp__raises_warning_for_large_datasets(self):
+        config = self.get_config(lfc_cutoff=0.02, enable_difexp="true", top_n=15)
+        config.server_config.complete_config(self.context)
+        config.default_dataset_config.handle_diffexp(self.context)
+        self.assertEqual(len(self.context["messages"]), 0)
+
+    def test_multi_dataset(self):
+        config = AppConfig()
+        # test for illegal url_dataroots
+        for illegal in ("../b", "!$*", "\\n", "", "(bad)"):
+            config.update_server_config(
+                app__flask_secret_key="secret",
+                multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": f"{PROJECT_ROOT}/example-dataset"}},
+            )
+            with self.assertRaises(ConfigurationError):
+                config.complete_config()
+
+        # test for legal url_dataroots
+        for legal in ("d", "this.is-okay_", "a/b"):
+            config.update_server_config(
+                app__flask_secret_key="secret",
+                multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": f"{PROJECT_ROOT}/example-dataset"}},
+            )
+            config.complete_config()
+
+        # test that multi dataroots work end to end
+        config.update_server_config(
+            app__flask_secret_key="secret",
+            multi_dataset__dataroot=dict(
+                s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"),
+                s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"),
+                s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"),
+            ),
+        )
+
+        # Change this default to test if the dataroot overrides below work.
+        config.update_default_dataset_config(app__about_legal_tos="tos_default.html")
+
+        # specialize the configs for set1
+        config.add_dataroot_config(
+            "s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html"
+        )
+
+        # specialize the configs for set2
+        config.add_dataroot_config(
+            "s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html"
+        )
+
+        # no specializations for set3 (they get the default dataset config)
+        config.complete_config()
+
+        with test_server(app_config=config) as server:
+            session = requests.Session()
+
+            response = session.get(f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config")
+            data_config = response.json()
+            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+            assert data_config["config"]["parameters"]["annotations"] is False
+            assert data_config["config"]["parameters"]["disable-diffexp"] is False
+            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html"
+
+            response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config")
+            data_config = response.json()
+            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+            assert data_config["config"]["parameters"]["annotations"] is True
+            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html"
+
+            response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config")
+            data_config = response.json()
+            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+            assert data_config["config"]["parameters"]["annotations"] is True
+            assert data_config["config"]["parameters"]["disable-diffexp"] is False
+            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html"
+
+            response = session.get(f"{server}/health")
+            assert response.json()["status"] == "pass"
+
+    def test_configfile_with_specialization(self):
+        # test that per_dataset_config config load the default config, then the specialized config
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            configfile = os.path.join(tempdir, "config.yaml")
+            with open(configfile, "w") as fconfig:
+                config = """
+                server:
+                    multi_dataset:
+                        dataroot:
+                            test:
+                                base_url: test
+                                dataroot: fake_dataroot
+
+                dataset:
+                    user_annotations:
+                        enable: false
+                        type: hosted_tiledb_array
+                        hosted_tiledb_array:
+                            db_uri: fake_db_uri
+                            hosted_file_directory: fake_dir
+
+                per_dataset_config:
+                    test:
+                        user_annotations:
+                            enable: true
+                """
+                fconfig.write(config)
+
+            app_config = AppConfig()
+            app_config.update_from_config_file(configfile)
+
+            test_config = app_config.dataroot_config["test"]
+
+            # test config from default
+            self.assertEqual(test_config.user_annotations__type, "hosted_tiledb_array")
+            self.assertEqual(test_config.user_annotations__hosted_tiledb_array__db_uri, "fake_db_uri")
+
+            # test config from specialization
+            self.assertTrue(test_config.user_annotations__enable)

backend/test/test_czi_hosted/unit/common/config/test_external_config.py

@@ -0,0 +1,232 @@
+import os
+from unittest.mock import patch
+
+import requests
+
+from backend.common.errors import ConfigurationError
+from backend.czi_hosted.common.config.app_config import AppConfig
+from backend.common.utils.type_conversion_utils import convert_string_to_value
+from backend.test.test_czi_hosted.unit import test_server
+from backend.test import FIXTURES_ROOT
+from backend.test.test_czi_hosted.unit.common.config import ConfigTests
+
+
+class TestExternalConfig(ConfigTests):
+    def test_type_convert(self):
+        # The values from environment variables and aws secrets are returned as strings.
+        # These values need to be converted to the proper types.
+
+        self.assertEqual(convert_string_to_value("1"), int(1))
+        self.assertEqual(convert_string_to_value("1.1"), float(1.1))
+        self.assertEqual(convert_string_to_value("string"), "string")
+        self.assertEqual(convert_string_to_value("true"), True)
+        self.assertEqual(convert_string_to_value("True"), True)
+        self.assertEqual(convert_string_to_value("false"), False)
+        self.assertEqual(convert_string_to_value("False"), False)
+        self.assertEqual(convert_string_to_value("null"), None)
+        self.assertEqual(convert_string_to_value("None"), None)
+        self.assertEqual(convert_string_to_value("{'a':10, 'b':'string'}"), dict(a=int(10), b="string"))
+
+    def test_environment_variable(self):
+        configfile = self.custom_external_config(
+            environment=[
+                dict(name="DATAPATH", path=["server", "single_dataset", "datapath"], required=True),
+                dict(name="DIFFEXP", path=["dataset", "diffexp", "enable"], required=True),
+            ],
+            config_file_name="environment_external_config.yaml",
+        )
+
+        env = os.environ
+        env["DATAPATH"] = f"{FIXTURES_ROOT}/pbmc3k.cxg"
+        env["DIFFEXP"] = "False"
+        with test_server(command_line_args=["-c", configfile], env=env) as server:
+            session = requests.Session()
+            response = session.get(f"{server}/api/v0.2/config")
+            data_config = response.json()
+            self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k")
+            self.assertTrue(data_config["config"]["parameters"]["disable-diffexp"])
+
+        env["DATAPATH"] = f"{FIXTURES_ROOT}/a95c59b4-7f5d-4b80-ad53-a694834ca18b.h5ad"
+        env["DIFFEXP"] = "True"
+        with test_server(command_line_args=["-c", configfile], env=env) as server:
+            session = requests.Session()
+            response = session.get(f"{server}/api/v0.2/config")
+            data_config = response.json()
+            self.assertEqual(data_config["config"]["displayNames"]["dataset"], "a95c59b4-7f5d-4b80-ad53-a694834ca18b")
+            self.assertFalse(data_config["config"]["parameters"]["disable-diffexp"])
+
+    def test_environment_variable_errors(self):
+
+        # no name
+        app_config = AppConfig()
+        app_config.external_config.environment = [dict(required=True, path=["this", "is", "a", "path"])]
+        with self.assertRaises(ConfigurationError) as config_error:
+            app_config.complete_config()
+        self.assertEqual(config_error.exception.message, "environment: 'name' is missing")
+
+        # required has wrong type
+        app_config = AppConfig()
+        app_config.external_config.environment = [
+            dict(name="myenvar", required="optional", path=["this", "is", "a", "path"])
+        ]
+        with self.assertRaises(ConfigurationError) as config_error:
+            app_config.complete_config()
+        self.assertEqual(config_error.exception.message, "environment: 'required' must be a bool")
+
+        # no path
+        app_config = AppConfig()
+        app_config.external_config.environment = [dict(name="myenvar", required=True)]
+        with self.assertRaises(ConfigurationError) as config_error:
+            app_config.complete_config()
+        self.assertEqual(config_error.exception.message, "environment: 'path' is missing")
+
+        # required environment variable is not set
+        app_config = AppConfig()
+        app_config.external_config.environment = [
+            dict(name="THIS_ENV_IS_NOT_SET", required=True, path=["this", "is", "a", "path"])
+        ]
+        with self.assertRaises(ConfigurationError) as config_error:
+            app_config.complete_config()
+        self.assertEqual(config_error.exception.message, "required environment variable 'THIS_ENV_IS_NOT_SET' not set")
+
+    @patch("backend.czi_hosted.common.config.external_config.get_secret_key")
+    def test_aws_secrets_manager(self, mock_get_secret_key):
+        mock_get_secret_key.return_value = {
+            "oauth_client_secret": "mock_oauth_secret",
+            "db_uri": "mock_db_uri",
+        }
+        configfile = self.custom_external_config(
+            aws_secrets_manager_region="us-west-2",
+            aws_secrets_manager_secrets=[
+                dict(
+                    name="my_secret",
+                    values=[
+                        dict(key="flask_secret_key", path=["server", "app", "flask_secret_key"], required=False),
+                        dict(
+                            key="db_uri",
+                            path=["dataset", "user_annotations", "hosted_tiledb_array", "db_uri"],
+                            required=True,
+                        ),
+                        dict(
+                            key="oauth_client_secret",
+                            path=["server", "authentication", "params_oauth", "client_secret"],
+                            required=True,
+                        ),
+                    ],
+                )
+            ],
+            config_file_name="secret_external_config.yaml",
+        )
+
+        app_config = AppConfig()
+        app_config.update_from_config_file(configfile)
+        app_config.server_config.single_dataset__datapath = f"{FIXTURES_ROOT}/pbmc3k.cxg"
+        app_config.server_config.app__flask_secret_key = "original"
+        app_config.server_config.single_dataset__datapath = f"{FIXTURES_ROOT}/pbmc3k.cxg"
+
+        app_config.complete_config()
+
+        self.assertEqual(app_config.server_config.app__flask_secret_key, "original")
+        self.assertEqual(app_config.server_config.authentication__params_oauth__client_secret, "mock_oauth_secret")
+        self.assertEqual(app_config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mock_db_uri")
+
+    @patch("backend.czi_hosted.common.config.external_config.get_secret_key")
+    def test_aws_secrets_manager_error(self, mock_get_secret_key):
+        mock_get_secret_key.return_value = {
+            "oauth_client_secret": "mock_oauth_secret",
+            "db_uri": "mock_db_uri",
+        }
+
+        # no region
+        app_config = AppConfig()
+        app_config.external_config.aws_secrets_manager__region = None
+        app_config.external_config.aws_secrets_manager__secrets = [
+            dict(name="secret1", values=[dict(key="key1", required=True, path=["this", "is", "my", "path"])])
+        ]
+        with self.assertRaises(ConfigurationError) as config_error:
+            app_config.complete_config()
+        self.assertEqual(
+            config_error.exception.message,
+            "Invalid type for attribute: aws_secrets_manager__region, expected type str, got NoneType",
+        )
+
+        # missing secret name
+        app_config = AppConfig()
+        app_config.external_config.aws_secrets_manager__region = "us-west-2"
+        app_config.external_config.aws_secrets_manager__secrets = [
+            dict(values=[dict(key="db_uri", required=True, path=["this", "is", "my", "path"])])
+        ]
+        with self.assertRaises(ConfigurationError) as config_error:
+            app_config.complete_config()
+        self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'name' is missing")
+
+        # secret name wrong type
+        app_config = AppConfig()
+        app_config.external_config.aws_secrets_manager__region = "us-west-2"
+        app_config.external_config.aws_secrets_manager__secrets = [
+            dict(name=1, values=[dict(key="db_uri", required=True, path=["this", "is", "my", "path"])])
+        ]
+        with self.assertRaises(ConfigurationError) as config_error:
+            app_config.complete_config()
+        self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'name' must be a string")
+
+        # missing values name
+        app_config = AppConfig()
+        app_config.external_config.aws_secrets_manager__region = "us-west-2"
+        app_config.external_config.aws_secrets_manager__secrets = [dict(name="mysecret")]
+        with self.assertRaises(ConfigurationError) as config_error:
+            app_config.complete_config()
+        self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'values' is missing")
+
+        # values wrong type
+        app_config = AppConfig()
+        app_config.external_config.aws_secrets_manager__region = "us-west-2"
+        app_config.external_config.aws_secrets_manager__secrets = [
+            dict(name="mysecret", values=dict(key="db_uri", required=True, path=["this", "is", "my", "path"]))
+        ]
+        with self.assertRaises(ConfigurationError) as config_error:
+            app_config.complete_config()
+        self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'values' must be a list")
+
+        # entry missing key
+        app_config = AppConfig()
+        app_config.external_config.aws_secrets_manager__region = "us-west-2"
+        app_config.external_config.aws_secrets_manager__secrets = [
+            dict(name="mysecret", values=[dict(required=True, path=["this", "is", "my", "path"])])
+        ]
+        with self.assertRaises(ConfigurationError) as config_error:
+            app_config.complete_config()
+        self.assertEqual(config_error.exception.message, "missing 'key' in secret values: mysecret")
+
+        # entry required is wrong type
+        app_config = AppConfig()
+        app_config.external_config.aws_secrets_manager__region = "us-west-2"
+        app_config.external_config.aws_secrets_manager__secrets = [
+            dict(name="mysecret", values=[dict(key="db_uri", required="optional", path=["this", "is", "my", "path"])])
+        ]
+        with self.assertRaises(ConfigurationError) as config_error:
+            app_config.complete_config()
+        self.assertEqual(config_error.exception.message, "wrong type for 'required' in secret values: mysecret")
+
+        # entry missing path
+        app_config = AppConfig()
+        app_config.external_config.aws_secrets_manager__region = "us-west-2"
+        app_config.external_config.aws_secrets_manager__secrets = [
+            dict(name="mysecret", values=[dict(key="db_uri", required=True)])
+        ]
+        with self.assertRaises(ConfigurationError) as config_error:
+            app_config.complete_config()
+        self.assertEqual(config_error.exception.message, "missing 'path' in secret values: mysecret")
+
+        # secret missing required key
+        app_config = AppConfig()
+        app_config.external_config.aws_secrets_manager__region = "us-west-2"
+        app_config.external_config.aws_secrets_manager__secrets = [
+            dict(
+                name="mysecret",
+                values=[dict(key="KEY_DOES_NOT_EXIST", required=True, path=["this", "is", "a", "path"])],
+            )
+        ]
+        with self.assertRaises(ConfigurationError) as config_error:
+            app_config.complete_config()
+        self.assertEqual(config_error.exception.message, "required secret 'mysecret:KEY_DOES_NOT_EXIST' not set")

backend/test/test_czi_hosted/unit/common/config/test_server_config.py

@@ -0,0 +1,322 @@
+import os
+import unittest
+from unittest import mock
+from unittest.mock import patch
+
+from backend.czi_hosted.common.config.base_config import BaseConfig
+from backend.common.utils.utils import find_available_port
+from backend.test.test_czi_hosted.unit import test_server
+from backend.test import PROJECT_ROOT, FIXTURES_ROOT
+import requests
+
+from backend.czi_hosted.common.config.app_config import AppConfig
+from backend.common.errors import ConfigurationError
+from backend.test.test_czi_hosted.unit.common.config import ConfigTests
+
+
+def mockenv(**envvars):
+    return mock.patch.dict(os.environ, envvars)
+
+
+class TestServerConfig(ConfigTests):
+    def setUp(self):
+        self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
+        self.config = AppConfig()
+        self.config.update_server_config(app__flask_secret_key="secret")
+        self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
+        self.server_config = self.config.server_config
+        self.config.complete_config()
+
+        message_list = []
+
+        def noop(message):
+            message_list.append(message)
+
+        messagefn = noop
+        self.context = dict(messagefn=messagefn, messages=message_list)
+
+    def get_config(self, **kwargs):
+        file_name = self.custom_app_config(
+            dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        return config
+
+    def test_init_raises_error_if_default_config_is_invalid(self):
+        invalid_config = self.get_config(port="not_valid")
+        with self.assertRaises(ConfigurationError):
+            invalid_config.complete_config()
+
+    @patch("backend.czi_hosted.common.config.server_config.BaseConfig.validate_correct_type_of_configuration_attribute")
+    def test_complete_config_checks_all_attr(self, mock_check_attrs):
+        mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
+        self.server_config.complete_config(self.context)
+        self.assertEqual(mock_check_attrs.call_count, 41)
+
+    def test_handle_app__throws_error_if_port_doesnt_exist(self):
+        config = self.get_config(port=99999999)
+        with self.assertRaises(ConfigurationError):
+            config.server_config.handle_app(self.context)
+
+    @patch("backend.czi_hosted.common.config.server_config.discover_s3_region_name")
+    def test_handle_data_locator_works_for_default_types(self, mock_discover_region_name):
+        mock_discover_region_name.return_value = None
+        # Default config
+        self.assertEqual(self.config.server_config.data_locator__s3__region_name, None)
+        # hard coded
+        config = self.get_config()
+        self.assertEqual(config.server_config.data_locator__s3__region_name, "us-east-1")
+        # incorrectly formatted
+        dataroot = {
+            "d1": {"base_url": "set1", "dataroot": "/path/to/set1_datasets/"},
+            "d2": {"base_url": "set2/subdir", "dataroot": "s3://shouldnt/work"},
+        }
+        file_name = self.custom_app_config(
+            dataroot=dataroot, config_file_name=self.config_file_name, data_locater_region_name="true"
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        with self.assertRaises(ConfigurationError):
+            config.server_config.handle_data_locator()
+
+    @patch("backend.czi_hosted.common.config.server_config.discover_s3_region_name")
+    def test_handle_data_locator_can_read_from_dataroot(self, mock_discover_region_name):
+        mock_discover_region_name.return_value = "us-west-2"
+        dataroot = {
+            "d1": {"base_url": "set1", "dataroot": "/path/to/set1_datasets/"},
+            "d2": {"base_url": "set2/subdir", "dataroot": "s3://hosted-cellxgene-dev"},
+        }
+        file_name = self.custom_app_config(
+            dataroot=dataroot, config_file_name=self.config_file_name, data_locater_region_name="true"
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        config.server_config.handle_data_locator()
+        self.assertEqual(config.server_config.data_locator__s3__region_name, "us-west-2")
+        mock_discover_region_name.assert_called_once_with("s3://hosted-cellxgene-dev")
+
+    def test_handle_app___can_use_envar_port(self):
+        config = self.get_config(port=24)
+        self.assertEqual(config.server_config.app__port, 24)
+
+        # Note if the port is set in the config file it will NOT be overwritten by a different envvar
+        os.environ["CXG_SERVER_PORT"] = "4008"
+        self.config = AppConfig()
+        self.config.update_server_config(app__flask_secret_key="secret")
+        self.config.server_config.handle_app(self.context)
+        self.assertEqual(self.config.server_config.app__port, 4008)
+        del os.environ["CXG_SERVER_PORT"]
+
+    def test_handle_app__can_get_secret_key_from_envvar_or_config_file_with_envvar_given_preference(self):
+        config = self.get_config(flask_secret_key="KEY_FROM_FILE")
+        self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_FILE")
+
+        os.environ["CXG_SECRET_KEY"] = "KEY_FROM_ENV"
+        config.external_config.handle_environment(self.context)
+        self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_ENV")
+
+    def test_handle_app__sets_web_base_url(self):
+        config = self.get_config(web_base_url="anything.com")
+        self.assertEqual(config.server_config.app__web_base_url, "anything.com")
+
+    def test_handle_auth__gets_client_secret_from_envvars_or_config_with_envvars_given_preference(self):
+        config = self.get_config(client_secret="KEY_FROM_FILE")
+        config.server_config.handle_authentication()
+        self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "KEY_FROM_FILE")
+
+        os.environ["CXG_OAUTH_CLIENT_SECRET"] = "KEY_FROM_ENV"
+        config.external_config.handle_environment(self.context)
+
+        self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "KEY_FROM_ENV")
+
+    def test_handle_data_source__errors_when_passed_zero_or_two_dataroots(self):
+        file_name = self.custom_app_config(
+            dataroot=f"{FIXTURES_ROOT}",
+            config_file_name="two_data_roots.yml",
+            dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad",
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        with self.assertRaises(ConfigurationError):
+            config.server_config.handle_data_source()
+
+        file_name = self.custom_app_config(config_file_name="zero_roots.yml")
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        with self.assertRaises(ConfigurationError):
+            config.server_config.handle_data_source()
+
+    def test_get_api_base_url_works(self):
+
+        # test the api_base_url feature, and that it can contain a path
+        config = AppConfig()
+        backend_port = find_available_port("localhost", 10000)
+        config.update_server_config(
+            app__flask_secret_key="secret",
+            app__api_base_url=f"http://localhost:{backend_port}/additional/path",
+            multi_dataset__dataroot=f"{PROJECT_ROOT}/example-dataset",
+        )
+
+        config.complete_config()
+
+        with test_server(["-p", str(backend_port)], app_config=config) as server:
+            session = requests.Session()
+            self.assertEqual(server, f"http://localhost:{backend_port}")
+            response = session.get(f"{server}/additional/path/d/pbmc3k.h5ad/api/v0.2/config")
+            self.assertEqual(response.status_code, 200)
+            data_config = response.json()
+            self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k")
+
+            # test the health check at the correct url
+            response = session.get(f"{server}/additional/path/health")
+            assert response.json()["status"] == "pass"
+
+    def test_get_web_base_url_works(self):
+        config = self.get_config(web_base_url="www.thisisawebsite.com")
+        web_base_url = config.server_config.get_web_base_url()
+        self.assertEqual(web_base_url, "www.thisisawebsite.com")
+
+        config = self.get_config(web_base_url="local", port=12)
+        web_base_url = config.server_config.get_web_base_url()
+        self.assertEqual(web_base_url, "http://localhost:12")
+
+        config = self.get_config(web_base_url="www.thisisawebsite.com/")
+        web_base_url = config.server_config.get_web_base_url()
+        self.assertEqual(web_base_url, "www.thisisawebsite.com")
+
+        config = self.get_config(api_base_url="www.api_base.com/")
+        web_base_url = config.server_config.get_web_base_url()
+        self.assertEqual(web_base_url, "www.api_base.com")
+
+    def test_config_for_single_dataset(self):
+        file_name = self.custom_app_config(
+            config_file_name="single_dataset.yml", dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k.cxg"
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        config.server_config.handle_single_dataset(self.context)
+        self.assertIsNotNone(config.server_config.matrix_data_cache_manager)
+
+        file_name = self.custom_app_config(
+            config_file_name="single_dataset_with_about.yml",
+            about="www.cziscience.com",
+            dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k.cxg",
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        with self.assertRaises(ConfigurationError):
+            config.server_config.handle_single_dataset(self.context)
+
+    def test_multi_dataset_raises_error_for_illegal_routes(self):
+        # test for illegal url_dataroots
+        for illegal in ("../b", "!$*", "\\n", "", "(bad)"):
+            self.config.update_server_config(
+                multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": f"{PROJECT_ROOT}/example-dataset"}}
+            )
+            with self.assertRaises(ConfigurationError):
+                self.config.complete_config()
+
+    def test_multidataset_works_for_legal_routes(self):
+        # test for legal url_dataroots
+        for legal in ("d", "this.is-okay_", "a/b"):
+            self.config.update_server_config(
+                multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": f"{PROJECT_ROOT}/example-dataset"}}
+            )
+            self.config.complete_config()
+
+    def test_mulitdatasets_work_e2e(self):
+        # test that multi dataroots work end to end
+        self.config.update_server_config(
+            multi_dataset__dataroot=dict(
+                s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"),
+                s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"),
+                s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"),
+            )
+        )
+
+        # Change this default to test if the dataroot overrides below work.
+        self.config.update_default_dataset_config(app__about_legal_tos="tos_default.html")
+
+        # specialize the configs for set1
+        self.config.add_dataroot_config(
+            "s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html"
+        )
+
+        # specialize the configs for set2
+        self.config.add_dataroot_config(
+            "s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html"
+        )
+
+        # no specializations for set3 (they get the default dataset config)
+        self.config.complete_config()
+
+        with test_server(app_config=self.config) as server:
+            session = requests.Session()
+
+            response = session.get(f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config")
+            data_config = response.json()
+            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+            assert data_config["config"]["parameters"]["annotations"] is False
+            assert data_config["config"]["parameters"]["disable-diffexp"] is False
+            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html"
+
+            response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config")
+            data_config = response.json()
+            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+            assert data_config["config"]["parameters"]["annotations"] is True
+            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html"
+
+            response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config")
+            data_config = response.json()
+            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+            assert data_config["config"]["parameters"]["annotations"] is True
+            assert data_config["config"]["parameters"]["disable-diffexp"] is False
+            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html"
+
+            response = session.get(f"{server}/health")
+            assert response.json()["status"] == "pass"
+
+            # access a dataset (no slash)
+            response = session.get(f"{server}/set2/pbmc3k.cxg")
+            self.assertEqual(response.status_code, 200)
+
+            # access a dataset (with slash)
+            response = session.get(f"{server}/set2/pbmc3k.cxg/")
+            self.assertEqual(response.status_code, 200)
+
+    @patch("backend.czi_hosted.common.config.server_config.diffexp_tiledb.set_config")
+    def test_handle_diffexp(self, mock_tiledb_config):
+        custom_config_file = self.custom_app_config(
+            dataroot=f"{FIXTURES_ROOT}",
+            cpu_multiplier=3,
+            diffexp_max_workers=1,
+            target_workunit=4,
+            config_file_name=self.config_file_name,
+        )
+        config = AppConfig()
+        config.update_from_config_file(custom_config_file)
+        config.server_config.handle_diffexp()
+        # called with the min of diffexp_max_workers and cpus*cpu_multiplier
+        mock_tiledb_config.assert_called_once_with(1, 4)
+
+    @patch("backend.czi_hosted.data_cxg.cxg_adaptor.CxgAdaptor.set_tiledb_context")
+    def test_handle_adaptor(self, mock_tiledb_context):
+        custom_config = self.custom_app_config(
+            dataroot=f"{FIXTURES_ROOT}", cxg_tile_cache_size=10, cxg_num_reader_threads=2
+        )
+        config = AppConfig()
+        config.update_from_config_file(custom_config)
+        config.server_config.handle_adaptor()
+        mock_tiledb_context.assert_called_once_with(
+            {"sm.tile_cache_size": 10, "sm.num_reader_threads": 2, "vfs.s3.region": "us-east-1"}
+        )
+
+    def test_test_auth_only_in_insecure(self):
+
+        config = self.get_config(auth_type="test")
+        with self.assertRaises(ConfigurationError):
+            config.complete_config()
+
+        config.update_server_config(authentication__insecure_test_environment=True)
+        config.complete_config()

backend/test/test_czi_hosted/unit/common/test_api.py

@@ -0,0 +1,451 @@
+import shutil
+import time
+import unittest
+import zlib
+from http import HTTPStatus
+
+import pandas as pd
+import requests
+
+from backend.czi_hosted.data_common.matrix_loader import MatrixDataType
+from backend.test.test_czi_hosted.unit import (
+    data_with_tmp_annotations,
+    make_fbs,
+    start_test_server,
+    stop_test_server,
+)
+from backend.test import PROJECT_ROOT, FIXTURES_ROOT, decode_fbs
+from backend.test.fixtures.fixtures import pbmc3k_colors
+
+BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
+
+
+# TODO (mweiden): remove ANNOTATIONS_ENABLED and Annotation subclasses when annotations are no longer experimental
+
+
+class EndPoints(object):
+    ANNOTATIONS_ENABLED = True
+
+    def test_initialize(self):
+        endpoint = "schema"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        self.assertEqual(result_data["schema"]["dataframe"]["nObs"], 2638)
+        self.assertEqual(len(result_data["schema"]["annotations"]["obs"]), 2)
+        self.assertEqual(
+            len(result_data["schema"]["annotations"]["obs"]["columns"]), 6 if self.ANNOTATIONS_ENABLED else 5
+        )
+
+    def test_config(self):
+        endpoint = "config"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        self.assertIn("library_versions", result_data["config"])
+        self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k")
+
+    def test_get_layout_fbs(self):
+        endpoint = "layout/obs"
+        url = f"{self.URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 8)
+        self.assertIsNotNone(df["columns"])
+        self.assertSetEqual(
+            set(df["col_idx"]),
+            {"pca_0", "pca_1", "tsne_0", "tsne_1", "umap_0", "umap_1", "draw_graph_fr_0", "draw_graph_fr_1"},
+        )
+        self.assertIsNone(df["row_idx"])
+        self.assertEqual(len(df["columns"]), df["n_cols"])
+
+    def test_put_layout_fbs(self):
+        # first check that re-embedding is turned on
+        result = self.session.get(f"{self.URL_BASE}config")
+        config_data = result.json()
+        re_embed = config_data["config"]["parameters"]["enable-reembedding"]
+        if not re_embed:
+            return
+        # attempt to reembed with umap over 100 cells.
+        endpoint = "layout/obs"
+        url = f"{self.URL_BASE}{endpoint}"
+        data = {}
+        data["filter"] = {}
+        data["filter"]["obs"] = {}
+        data["filter"]["obs"]["index"] = list(range(100))
+        data["method"] = "umap"
+        result = self.session.put(url, json=data)
+
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        result_data = result.json()
+        self.assertIsInstance(result_data, dict)
+        self.assertEqual(result_data["type"], "float32")
+        self.assertTrue(result_data["name"].startswith("reembed:umap_"))
+        self.assertIsInstance(result_data["dims"], list)
+        self.assertEqual(len(result_data["dims"]), 2)
+        dims = result_data["dims"]
+        self.assertTrue(dims[0].startswith("reembed:umap_") and dims[0].endswith("_0"))
+        self.assertTrue(dims[1].startswith("reembed:umap_") and dims[1].endswith("_1"))
+
+    def test_bad_filter(self):
+        endpoint = "data/var"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.put(url, json=BAD_FILTER)
+        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
+
+    def test_get_annotations_obs_fbs(self):
+        endpoint = "annotations/obs"
+        url = f"{self.URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 6 if self.ANNOTATIONS_ENABLED else 5)
+        self.assertIsNotNone(df["columns"])
+        self.assertIsNone(df["row_idx"])
+        self.assertEqual(len(df["columns"]), df["n_cols"])
+        obs_index_col_name = self.schema["schema"]["annotations"]["obs"]["index"]
+        self.assertCountEqual(
+            df["col_idx"],
+            [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"]
+            + (["cluster-test"] if self.ANNOTATIONS_ENABLED else []),
+        )
+
+    def test_get_annotations_obs_keys_fbs(self):
+        endpoint = "annotations/obs"
+        query = "annotation-name=n_genes&annotation-name=percent_mito"
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 2)
+        self.assertIsNotNone(df["columns"])
+        self.assertIsNone(df["row_idx"])
+        self.assertEqual(len(df["columns"]), df["n_cols"])
+        self.assertCountEqual(df["col_idx"], ["n_genes", "percent_mito"])
+
+    def test_get_annotations_obs_error(self):
+        endpoint = "annotations/obs"
+        query = "annotation-name=notakey"
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
+
+    def test_diff_exp(self):
+        endpoint = "diffexp/obs"
+        url = f"{self.URL_BASE}{endpoint}"
+        params = {
+            "mode": "topN",
+            "set1": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["NK cells"]}]}}},
+            "set2": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["CD8 T cells"]}]}}},
+            "count": 7,
+        }
+        result = self.session.post(url, json=params)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        self.assertEqual(len(result_data), 7)
+
+    def test_diff_exp_indices(self):
+        endpoint = "diffexp/obs"
+        url = f"{self.URL_BASE}{endpoint}"
+        params = {
+            "mode": "topN",
+            "count": 10,
+            "set1": {"filter": {"obs": {"index": [[0, 500]]}}},
+            "set2": {"filter": {"obs": {"index": [[500, 1000]]}}},
+        }
+        result = self.session.post(url, json=params)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        self.assertEqual(len(result_data), 10)
+
+    def test_get_annotations_var_fbs(self):
+        endpoint = "annotations/var"
+        url = f"{self.URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 1838)
+        self.assertEqual(df["n_cols"], 2)
+        self.assertIsNotNone(df["columns"])
+        self.assertIsNone(df["row_idx"])
+        self.assertEqual(len(df["columns"]), df["n_cols"])
+        var_index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
+        self.assertCountEqual(df["col_idx"], [var_index_col_name, "n_cells"])
+
+    def test_get_annotations_var_keys_fbs(self):
+        endpoint = "annotations/var"
+        query = "annotation-name=n_cells"
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 1838)
+        self.assertEqual(df["n_cols"], 1)
+        self.assertIsNotNone(df["columns"])
+        self.assertIsNone(df["row_idx"])
+        self.assertEqual(len(df["columns"]), df["n_cols"])
+        self.assertCountEqual(df["col_idx"], ["n_cells"])
+
+    def test_get_annotations_var_error(self):
+        endpoint = "annotations/var"
+        query = "annotation-name=notakey"
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
+
+    def test_data_mimetype_error(self):
+        endpoint = "data/var"
+        header = {"Accept": "xxx"}
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.put(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.NOT_ACCEPTABLE)
+
+    def test_fbs_default(self):
+        endpoint = "data/var"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.put(url)
+        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
+
+        filter = {"filter": {"var": {"index": [0, 1, 4]}}}
+        result = self.session.put(url, json=filter)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+
+    def test_data_put_fbs(self):
+        endpoint = "data/var"
+        url = f"{self.URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.put(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
+
+    def test_data_get_fbs(self):
+        endpoint = "data/var"
+        url = f"{self.URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
+
+    def test_data_put_filter_fbs(self):
+        endpoint = "data/var"
+        url = f"{self.URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        filter = {"filter": {"var": {"index": [0, 1, 4]}}}
+        result = self.session.put(url, headers=header, json=filter)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 3)
+        self.assertIsNotNone(df["columns"])
+        self.assertIsNone(df["row_idx"])
+        self.assertEqual(len(df["columns"]), df["n_cols"])
+        self.assertListEqual(df["col_idx"].tolist(), [0, 1, 4])
+
+    def test_data_get_filter_fbs(self):
+        index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
+        endpoint = "data/var"
+        query = f"var:{index_col_name}=SIK1"
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 1)
+
+    def test_data_get_unknown_filter_fbs(self):
+        index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
+        endpoint = "data/var"
+        query = f"var:{index_col_name}=UNKNOWN"
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 0)
+
+    def test_data_put_single_var(self):
+        endpoint = "data/var"
+        url = f"{self.URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
+        var_filter = {"filter": {"var": {"annotation_value": [{"name": index_col_name, "values": ["RER1"]}]}}}
+        result = self.session.put(url, headers=header, json=var_filter)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 1)
+
+    def test_colors(self):
+        endpoint = "colors"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        self.assertEqual(result_data, pbmc3k_colors)
+
+    def test_static(self):
+        endpoint = "static"
+        file = "assets/favicon.ico"
+        url = f"{self.server}/{endpoint}/{file}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+
+    def _setupClass(child_class, command_line):
+        child_class.ps, child_class.server = start_test_server(command_line)
+        child_class.URL_BASE = f"{child_class.server}/api/v0.2/"
+        child_class.session = requests.Session()
+        for i in range(90):
+            try:
+                result = child_class.session.get(f"{child_class.URL_BASE}schema")
+                child_class.schema = result.json()
+            except requests.exceptions.ConnectionError:
+                time.sleep(1)
+
+
+class EndPointsAnnotations(EndPoints):
+    def test_get_schema_existing_writable(self):
+        self._test_get_schema_writable("cluster-test")
+
+    def test_get_user_annotations_existing_obs_keys_fbs(self):
+        self._test_get_user_annotations_obs_keys_fbs(
+            "cluster-test", {"unassigned", "one", "two", "three", "four", "five", "six", "seven"},
+        )
+
+    def test_put_user_annotations_obs_fbs(self):
+        endpoint = "annotations/obs"
+        query = "annotation-collection-name=test_annotations"
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        n_rows = self.data.get_shape()[0]
+        fbs = make_fbs({"cat_A": pd.Series(["label_A"] * n_rows, dtype="category")})
+        result = self.session.put(url, data=zlib.compress(fbs))
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        self.assertEqual(result.json(), {"status": "OK"})
+        self._test_get_schema_writable("cat_A")
+        self._test_get_user_annotations_obs_keys_fbs("cat_A", {"label_A"})
+
+    def _test_get_user_annotations_obs_keys_fbs(self, annotation_name, columns):
+        endpoint = "annotations/obs"
+        query = f"annotation-name={annotation_name}"
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 1)
+        self.assertListEqual(df["col_idx"], [annotation_name])
+        self.assertEqual(set(df["columns"][0]), columns)
+        self.assertIsNone(df["row_idx"])
+        self.assertEqual(len(df["columns"]), df["n_cols"])
+
+    def _test_get_schema_writable(self, cluster_name):
+        endpoint = "schema"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        columns = result_data["schema"]["annotations"]["obs"]["columns"]
+        matching_columns = [c for c in columns if c["name"] == cluster_name]
+        self.assertEqual(len(matching_columns), 1)
+        self.assertTrue(matching_columns[0]["writable"])
+
+
+class EndPointsAnndata(unittest.TestCase, EndPoints):
+    """Test Case for endpoints"""
+
+    ANNOTATIONS_ENABLED = False
+
+    @classmethod
+    def setUpClass(cls):
+        cls._setupClass(
+            cls,
+            [
+                f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
+                "--disable-annotations",
+                "--experimental-enable-reembedding",
+            ],
+        )
+
+    @classmethod
+    def tearDownClass(cls):
+        stop_test_server(cls.ps)
+
+    @property
+    def annotations_enabled(self):
+        return False
+
+
+class EndPointsCxg(unittest.TestCase, EndPoints):
+    """Test Case for endpoints"""
+
+    ANNOTATIONS_ENABLED = False
+
+    @classmethod
+    def setUpClass(cls):
+        cls._setupClass(cls, [f"{FIXTURES_ROOT}/pbmc3k.cxg", "--disable-annotations"])
+
+    @classmethod
+    def tearDownClass(cls):
+        stop_test_server(cls.ps)
+
+
+class EndPointsAnndataAnnotations(unittest.TestCase, EndPointsAnnotations):
+    """Test Case for endpoints"""
+
+    ANNOTATIONS_ENABLED = True
+
+    @classmethod
+    def setUpClass(cls):
+        cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations(
+            MatrixDataType.H5AD, annotations_fixture=True
+        )
+        cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location()])
+
+    @classmethod
+    def tearDownClass(cls):
+        shutil.rmtree(cls.tmp_dir)
+        stop_test_server(cls.ps)
+
+
+class EndPointsCxgAnnotations(unittest.TestCase, EndPointsAnnotations):
+    """Test Case for endpoints"""
+
+    ANNOTATIONS_ENABLED = True
+
+    @classmethod
+    def setUpClass(cls):
+        cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations(MatrixDataType.CXG, annotations_fixture=True)
+        cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location()])
+
+    @classmethod
+    def tearDownClass(cls):
+        shutil.rmtree(cls.tmp_dir)
+        stop_test_server(cls.ps)

backend/test/test_czi_hosted/unit/common/test_corpora.py

@@ -0,0 +1,165 @@
+import json
+import shutil
+import tempfile
+import unittest
+from http import HTTPStatus
+
+import anndata
+import requests
+
+from backend.czi_hosted.common.corpora import (
+    corpora_get_versions_from_anndata,
+    corpora_is_version_supported,
+    corpora_get_props_from_anndata,
+)
+from backend.test.test_czi_hosted.unit import start_test_server, stop_test_server
+from backend.test import PROJECT_ROOT
+
+VERSION = "v0.2"
+
+
+class CorporaAPITest(unittest.TestCase):
+    def test_corpora_get_versions_from_anndata(self):
+        adata = self._get_h5ad()
+
+        if "version" in adata.uns:
+            del adata.uns["version"]
+        self.assertIsNone(corpora_get_versions_from_anndata(adata))
+
+        # something bogus
+        adata.uns["version"] = 99
+        self.assertIsNone(corpora_get_versions_from_anndata(adata))
+
+        # something legit
+        adata.uns["version"] = {"corpora_schema_version": "0.0.0", "corpora_encoding_version": "9.9.9"}
+        self.assertEqual(corpora_get_versions_from_anndata(adata), ["0.0.0", "9.9.9"])
+
+    def test_corpora_is_version_supported(self):
+        self.assertTrue(corpora_is_version_supported("1.0.0", "0.1.0"))
+        self.assertFalse(corpora_is_version_supported("0.0.0", "0.1.0"))
+        self.assertFalse(corpora_is_version_supported("1.0.0", "0.0.0"))
+
+    def test_corpora_get_props_from_anndata(self):
+        adata = self._get_h5ad()
+
+        if "version" in adata.uns:
+            del adata.uns["version"]
+        self.assertIsNone(corpora_get_props_from_anndata(adata))
+
+        # something bogus
+        adata.uns["version"] = 99
+        self.assertIsNone(corpora_get_props_from_anndata(adata))
+
+        # unsupported version, but missing required values
+        adata.uns["version"] = {"corpora_schema_version": "99.0.0", "corpora_encoding_version": "32.1.0"}
+        with self.assertRaises(ValueError):
+            corpora_get_props_from_anndata(adata)
+
+        # legit version, but missing required values
+        adata.uns["version"] = {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"}
+        with self.assertRaises(KeyError):
+            corpora_get_props_from_anndata(adata)
+
+        some_fields = {
+            "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"},
+            "title": "title",
+            "layer_descriptions": "layer_descriptions",
+            "organism": "organism",
+            "organism_ontology_term_id": "organism_ontology_term_id",
+            "project_name": "project_name",
+            "project_description": "project_description",
+            "contributors": json.dumps([{"contributors": "contributors"}]),
+            "project_links": json.dumps([{"link_name": "link_name", "link_url": "link_url", "link_type": "SUMMARY"}]),
+        }
+        for k in some_fields:
+            adata.uns[k] = some_fields[k]
+        some_fields["contributors"] = json.loads(some_fields["contributors"])
+        some_fields["project_links"] = json.loads(some_fields["project_links"])
+        self.assertEqual(corpora_get_props_from_anndata(adata), some_fields)
+
+    def test_corpora_get_props_from_anndata_v110(self):
+        adata = self._get_h5ad()
+
+        if "version" in adata.uns:
+            del adata.uns["version"]
+        self.assertIsNone(corpora_get_props_from_anndata(adata))
+
+        # legit version, but missing required values
+        adata.uns["version"] = {"corpora_schema_version": "1.1.0", "corpora_encoding_version": "0.1.0"}
+        with self.assertRaises(KeyError):
+            corpora_get_props_from_anndata(adata)
+
+        # Metadata following schema 1.1.0, which removes some fields relative to 1.1.0
+        some_110_fields = {
+            "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"},
+            "title": "title",
+            "layer_descriptions": "layer_descriptions",
+            "organism": "organism",
+            "organism_ontology_term_id": "organism_ontology_term_id",
+        }
+        for k in some_110_fields:
+            adata.uns[k] = some_110_fields[k]
+        self.assertEqual(corpora_get_props_from_anndata(adata), some_110_fields)
+
+    def _get_h5ad(self):
+        return anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
+
+
+class CorporaRESTAPITest(unittest.TestCase):
+    """ Confirm endpoints reflect Corpora-specific features """
+
+    @classmethod
+    def setCorporaFields(cls, path):
+        adata = anndata.read_h5ad(path)
+        corpora_props = {
+            "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"},
+            "title": "PBMC3K",
+            "contributors": json.dumps([{"name": "name"}]),
+            "layer_descriptions": {"X": "raw counts"},
+            "organism": "human",
+            "organism_ontology_term_id": "unknown",
+            "project_name": "test project",
+            "project_description": "test description",
+            "project_links": json.dumps(
+                [{"link_name": "test link", "link_type": "SUMMARY", "link_url": "https://a.u.r.l/"}]
+            ),
+            "default_embedding": "X_tsne",
+        }
+        adata.uns.update(corpora_props)
+        adata.write(path)
+
+    @classmethod
+    def setUpClass(cls):
+        cls.tmp_dir = tempfile.TemporaryDirectory()
+        src = f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad"
+        dst = f"{cls.tmp_dir.name}/pbmc3k.h5ad"
+        shutil.copyfile(src, dst)
+        cls.setCorporaFields(dst)
+        cls.ps, cls.server = start_test_server([dst])
+
+    @classmethod
+    def tearDownClass(cls):
+        stop_test_server(cls.ps)
+        cls.tmp_dir.cleanup()
+
+    def setUp(self):
+        self.session = requests.Session()
+        self.url_base = f"{self.server}/api/{VERSION}/"
+
+    def test_config(self):
+        endpoint = "config"
+        url = f"{self.url_base}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+
+        result_data = result.json()
+        self.assertIsInstance(result_data["config"]["corpora_props"], dict)
+        self.assertIsInstance(result_data["config"]["parameters"], dict)
+
+        corpora_props = result_data["config"]["corpora_props"]
+        parameters = result_data["config"]["parameters"]
+
+        self.assertEqual(corpora_props["version"]["corpora_schema_version"], "1.0.0")
+        self.assertEqual(corpora_props["organism"], "human")
+        self.assertEqual(parameters["default_embedding"], "tsne")

backend/test/test_czi_hosted/unit/common/test_nan_rest.py

@@ -0,0 +1,64 @@
+from http import HTTPStatus
+import unittest
+import math
+
+import backend.test.decode_fbs as decode_fbs
+
+
+import requests
+
+from backend.test import FIXTURES_ROOT
+from backend.test.test_czi_hosted.unit import start_test_server, stop_test_server
+
+VERSION = "v0.2"
+BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
+
+
+class WithNaNs(unittest.TestCase):
+    """Test Case for endpoints"""
+
+    @classmethod
+    def setUpClass(cls):
+        cls.ps, cls.server = start_test_server([f"{FIXTURES_ROOT}/nan.h5ad"])
+
+    @classmethod
+    def tearDownClass(cls):
+        stop_test_server(cls.ps)
+
+    def setUp(self):
+        self.session = requests.Session()
+        self.url_base = f"{self.server}/api/{VERSION}/"
+
+    def test_initialize(self):
+        endpoint = "schema"
+        url = f"{self.url_base}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+
+    def test_data(self):
+        endpoint = "data/var"
+        url = f"{self.url_base}{endpoint}"
+        filter = {"filter": {"var": {"index": [[0, 20]]}}}
+        result = self.session.put(url, json=filter)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertTrue(math.isnan(df["columns"][3][3]))
+
+    def test_annotation_obs(self):
+        endpoint = "annotations/obs"
+        url = f"{self.url_base}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertTrue(math.isnan(df["columns"][2][0]))
+
+    def test_annotation_var(self):
+        endpoint = "annotations/var"
+        url = f"{self.url_base}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertTrue(math.isnan(df["columns"][2][0]))

backend/test/test_czi_hosted/unit/common/test_rest.py

@@ -0,0 +1,79 @@
+import unittest
+from urllib.parse import parse_qs
+from werkzeug.datastructures import MultiDict
+from backend.common.errors import FilterError
+from backend.czi_hosted.common.rest import _query_parameter_to_filter
+
+
+def _qsparse(qs):
+    """ emulate what Flask/Werkzeug do to our QS """
+    return MultiDict(parse_qs(qs))
+
+
+class FilterParseTests(unittest.TestCase):
+    """ Test cases for various filter parsing """
+
+    def test_queryparam_to_filter_parse(self):
+        # categories
+        self.assertEqual(
+            _query_parameter_to_filter(_qsparse("obs:foo=bar&var:baz=133&var:baz=A&obs:baz=foo")),
+            {
+                "obs": {"annotation_value": [{"name": "foo", "values": ["bar"]}, {"name": "baz", "values": ["foo"]}]},
+                "var": {"annotation_value": [{"name": "baz", "values": ["133", "A"]}]},
+            },
+        )
+
+        # ranges
+        self.assertEqual(
+            _query_parameter_to_filter(_qsparse("obs:A=1,99&obs:B=*,100&obs:C=0,*")),
+            {
+                "obs": {
+                    "annotation_value": [
+                        {"name": "A", "min": 1, "max": 99.0},
+                        {"name": "B", "max": 100.0},
+                        {"name": "C", "min": 0.0},
+                    ]
+                },
+            },
+        )
+
+        # combo
+        self.assertEqual(
+            _query_parameter_to_filter(_qsparse("var:B=YES&var:A=1,99&var:B=NO")),
+            {
+                "var": {
+                    "annotation_value": [
+                        {"name": "B", "values": ["YES", "NO"]},
+                        {"name": "A", "min": 1.0, "max": 99.0},
+                    ]
+                },
+            },
+        )
+
+    def test_queryparam_to_filter_escaping(self):
+        self.assertEqual(
+            _query_parameter_to_filter(_qsparse("obs:var=%2521%252C%253AOK%253D&obs:A%2521=YO")),
+            {"obs": {"annotation_value": [{"name": "var", "values": ["!,:OK="]}, {"name": "A!", "values": ["YO"]}]}},
+        )
+
+    def test_queryparam_to_filter_errors(self):
+
+        # should raise FilterError
+        filter_errors = [
+            "foo=bar",  # no axis
+            "X=&Y=3",  # no value
+            "X&Y=3",  # no value
+            "moo:foo=bar",  # bad axis
+            "obs:x=1,A",  # non-numeric range
+            "var:X=1,2&var:X=3,4",  # duplicate ranges
+            "var:Y=,",
+            "var:Y=2,",
+            "var:Y=,5",
+            "var:Y=*,",
+            "var:Y=,*",
+            "var:Y=*,*",
+        ]
+
+        for qs in filter_errors:
+            with self.assertRaises(FilterError):
+                _query_parameter_to_filter(_qsparse(qs))

backend/test/test_czi_hosted/unit/common/test_writable_annotation.py

@@ -0,0 +1,317 @@
+import json
+import shutil
+import unittest
+from os import path, listdir
+from unittest.mock import MagicMock
+
+import numpy as np
+import pandas as pd
+import tiledb
+from flask import Flask
+
+from backend.common.errors import AnnotationCategoryNameError
+from backend.czi_hosted.common.rest import annotations_put_fbs_helper, schema_get_helper
+from backend.czi_hosted.data_common.matrix_loader import MatrixDataType
+from backend.czi_hosted.db.cellxgene_orm import CellxGeneDataset, Annotation
+from backend.test.fixtures.database import TestDatabase
+from backend.test.test_czi_hosted.unit import make_fbs, data_with_tmp_tiledb_annotations, data_with_tmp_annotations
+from backend.test import decode_fbs
+
+TestDatabase()
+
+class auth(object):
+    def get_user_id():
+        return "1234"
+
+    def get_user_name():
+        return "person name"
+
+
+class WritableTileDBStoredAnnotationTest(unittest.TestCase):
+    def setUp(self):
+        self.user_id = "1234"
+        self.data, self.tmp_dir, self.annotations = data_with_tmp_tiledb_annotations(MatrixDataType.H5AD)
+        self.data.dataset_config.user_annotations = self.annotations
+        self.db = self.annotations.db
+        self.n_rows = self.data.get_shape()[0]
+        self.test_dict = {
+            "cat_A": pd.Series(["label_A"] * self.n_rows, dtype="category"),
+            "cat_B": pd.Series(["label_B"] * self.n_rows, dtype="category"),
+        }
+        self.fbs = make_fbs(self.test_dict)
+        self.df = pd.DataFrame(self.test_dict)
+        self.app = Flask("fake_app")
+        self.app.__setattr__("auth", auth)
+
+    def tearDown(self):
+        shutil.rmtree(self.tmp_dir)
+
+    def annotation_put_fbs(self, fbs):
+        annotations_put_fbs_helper(self.data, fbs)
+        res = json.dumps({"status": "OK"})
+        return res
+
+    def test_category_name_throws_errors_for_categories_that_cant_be_converted_to_filenames(self):
+        with self.app.test_request_context():
+            bad_category_names = make_fbs(
+                {
+                    "cat_A": pd.Series(["label_A"] * self.n_rows, dtype="category"),
+                    "cat/B": pd.Series(["label_B"] * self.n_rows, dtype="category"),
+                }
+            )
+            with self.assertRaises(AnnotationCategoryNameError):
+                self.annotation_put_fbs(bad_category_names)
+
+    def test_convert_to_pandas__converts_tiledb_to_pandas_df(self):
+        with self.app.test_request_context():
+            self.annotations.write_labels(self.df, self.data)
+            dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id
+            annotation = self.db.query_for_most_recent(
+                Annotation, [Annotation.user_id == self.user_id, Annotation.dataset_id == str(dataset_id)]
+            )
+            # retrieve tiledb array
+            df = tiledb.open(annotation.tiledb_uri)
+            self.assertEqual(type(df), tiledb.array.SparseArray)
+
+            # convert to pandas df
+            pandas_df = self.annotations.convert_to_pandas_df(df, annotation.schema_hints)
+            self.assertEqual(type(pandas_df), pd.DataFrame)
+
+    def test_write_labels_creates_a_dataset_if_it_doesnt_exist(self):
+        with self.app.test_request_context():
+            new_name = "new_dataset/location"
+            self.data.get_location = MagicMock(return_value=new_name)
+            num_datasets = len(self.db.query([CellxGeneDataset]))
+            self.annotation_put_fbs(self.fbs)
+            more_datasets = len(self.db.query([CellxGeneDataset]))
+            self.assertGreater(more_datasets, num_datasets)
+
+            self.assertGreater(len(self.db.query([CellxGeneDataset], [CellxGeneDataset.name == new_name])), 0)
+
+    def test_write_labels_links_to_existing_dataset(self):
+        with self.app.test_request_context():
+            # add dataset to to db
+            self.annotation_put_fbs(self.fbs)
+
+            num_datasets = len(self.db.query([CellxGeneDataset]))
+
+            # create another annotation with the same dataset
+            self.annotation_put_fbs(self.fbs)
+
+            same_num_datasets = len(self.db.query([CellxGeneDataset]))
+
+            self.assertEqual(num_datasets, same_num_datasets)
+
+    def test_read_labels_returns_pandas_df(self):
+        with self.app.test_request_context():
+            self.annotation_put_fbs(self.fbs)
+            pandas_df = self.annotations.read_labels(self.data)
+            self.assertEqual(type(pandas_df), pd.DataFrame)
+
+    def test_read_labels_returns_df_matching_original(self):
+        with self.app.test_request_context():
+            self.annotation_put_fbs(self.fbs)
+            pandas_df = self.annotations.read_labels(self.data)
+
+            self.assertEqual(pandas_df.shape, (self.n_rows, 2))
+            self.assertEqual(set(pandas_df.columns), {"cat_A", "cat_B"})
+
+            self.assertTrue(self.data.original_obs_index.equals(pandas_df.index))
+
+            self.assertTrue(np.all(pandas_df["cat_A"] == ["label_A"] * self.n_rows))
+            self.assertTrue(np.all(pandas_df["cat_B"] == ["label_B"] * self.n_rows))
+
+    def test_error_checks(self):
+        # verify that the expected errors are generated
+        with self.app.test_request_context():
+            n_rows = self.data.get_shape()[0]
+            fbs_bad = make_fbs({"louvain": pd.Series(["undefined"] * n_rows, dtype="category")})
+
+            # ensure we catch attempt to overwrite non-writable data
+            with self.assertRaises(KeyError):
+                self.annotation_put_fbs(fbs_bad)
+
+    def test_write_labels_stores_df_as_tiledb_array(self):
+        with self.app.test_request_context():
+            self.annotations.write_labels(self.df, self.data)
+            # get uri
+            dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id
+            annotation = self.db.query_for_most_recent(
+                Annotation, [Annotation.user_id == "1234", Annotation.dataset_id == str(dataset_id)]
+            )
+
+            df = tiledb.open(annotation.tiledb_uri)
+            self.assertEqual(type(df), tiledb.array.SparseArray)
+
+    def test_remove_categories(self):
+        with self.app.test_request_context():
+            # update empty category data, which is how annotations are removed
+            empty = make_fbs({})
+            self.annotation_put_fbs(empty)
+
+            # verify that the tiledb uri is an empty string.
+            dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id
+            annotation = self.db.query_for_most_recent(
+                Annotation, [Annotation.user_id == self.user_id, Annotation.dataset_id == str(dataset_id)]
+            )
+            self.assertEqual(annotation.tiledb_uri, "")
+
+            # verify that read_labels returns None
+            df = self.annotations.read_labels(self.data)
+            self.assertIsNone(df)
+
+
+class WritableAnnotationTest(unittest.TestCase):
+    def setUp(self):
+        self.data, self.tmp_dir, self.annotations = data_with_tmp_annotations(MatrixDataType.H5AD)
+        self.data.dataset_config.user_annotations = self.annotations
+
+    def tearDown(self):
+        shutil.rmtree(self.tmp_dir)
+
+    def annotation_put_fbs(self, fbs):
+        annotations_put_fbs_helper(self.data, fbs)
+        res = json.dumps({"status": "OK"})
+        return res
+
+    def test_error_checks(self):
+        # verify that the expected errors are generated
+        n_rows = self.data.get_shape()[0]
+        fbs_bad = make_fbs({"louvain": pd.Series(["undefined"] * n_rows, dtype="category")})
+
+        # ensure we catch attempt to overwrite non-writable data
+        with self.assertRaises(KeyError):
+            self.annotation_put_fbs(fbs_bad)
+
+    def test_write_to_file(self):
+        # verify the file is written as expected
+        n_rows = self.data.get_shape()[0]
+        fbs = make_fbs(
+            {
+                "cat_A": pd.Series(["label_A"] * n_rows, dtype="category"),
+                "cat_B": pd.Series(["label_B"] * n_rows, dtype="category"),
+            }
+        )
+        res = self.annotation_put_fbs(fbs)
+        self.assertEqual(res, json.dumps({"status": "OK"}))
+        self.assertTrue(path.exists(self.annotations.output_file))
+        df = pd.read_csv(self.annotations.output_file, index_col=0, header=0, comment="#")
+        self.assertEqual(df.shape, (n_rows, 2))
+        self.assertEqual(set(df.columns), {"cat_A", "cat_B"})
+        self.assertTrue(self.data.original_obs_index.equals(df.index))
+        self.assertTrue(np.all(df["cat_A"] == ["label_A"] * n_rows))
+        self.assertTrue(np.all(df["cat_B"] == ["label_B"] * n_rows))
+
+        # verify complete overwrite on second attempt, AND rotation occurs
+        fbs = make_fbs(
+            {
+                "cat_A": pd.Series(["label_A1"] * n_rows, dtype="category"),
+                "cat_C": pd.Series(["label_C"] * n_rows, dtype="category"),
+            }
+        )
+        res = self.annotation_put_fbs(fbs)
+        self.assertEqual(res, json.dumps({"status": "OK"}))
+        self.assertTrue(path.exists(self.annotations.output_file))
+        df = pd.read_csv(self.annotations.output_file, index_col=0, header=0, comment="#")
+        self.assertEqual(set(df.columns), {"cat_A", "cat_C"})
+        self.assertTrue(np.all(df["cat_A"] == ["label_A1"] * n_rows))
+        self.assertTrue(np.all(df["cat_C"] == ["label_C"] * n_rows))
+
+        # rotation
+        name, ext = path.splitext(self.annotations.output_file)
+        backup_dir = f"{name}-backups"
+        self.assertTrue(path.isdir(backup_dir))
+        found_files = listdir(backup_dir)
+        self.assertEqual(len(found_files), 1)
+
+    def test_file_rotation_to_max_9(self):
+        # verify we stop rotation at 9
+        n_rows = self.data.get_shape()[0]
+        fbs = make_fbs(
+            {
+                "cat_A": pd.Series(["label_A"] * n_rows, dtype="category"),
+                "cat_B": pd.Series(["label_B"] * n_rows, dtype="category"),
+            }
+        )
+        for i in range(0, 11):
+            res = self.annotation_put_fbs(fbs)
+            self.assertEqual(res, json.dumps({"status": "OK"}))
+
+        name, ext = path.splitext(self.annotations.output_file)
+        backup_dir = f"{name}-backups"
+        self.assertTrue(path.isdir(backup_dir))
+        found_files = listdir(backup_dir)
+        self.assertTrue(len(found_files) <= 9)
+
+    def test_put_get_roundtrip(self):
+        # verify that OBS PUTs (annotation_put_fbs) are accessible via
+        # GET (annotation_to_fbs_matrix)
+
+        n_rows = self.data.get_shape()[0]
+        fbs = make_fbs(
+            {
+                "cat_A": pd.Series(["label_A"] * n_rows, dtype="category"),
+                "cat_B": pd.Series(["label_B"] * n_rows, dtype="category"),
+            }
+        )
+
+        # put
+        res = self.annotation_put_fbs(fbs)
+        self.assertEqual(res, json.dumps({"status": "OK"}))
+
+        # get
+        labels = self.annotations.read_labels(None)
+        fbsAll = self.data.annotation_to_fbs_matrix("obs", None, labels)
+        schema = schema_get_helper(self.data)
+        annotations = decode_fbs.decode_matrix_FBS(fbsAll)
+        obs_index_col_name = schema["annotations"]["obs"]["index"]
+        self.assertEqual(annotations["n_rows"], n_rows)
+        self.assertEqual(annotations["n_cols"], 7)
+        self.assertIsNone(annotations["row_idx"])
+        self.assertEqual(
+            annotations["col_idx"],
+            [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain", "cat_A", "cat_B"],
+        )
+        col_idx = annotations["col_idx"]
+        self.assertEqual(annotations["columns"][col_idx.index("cat_A")], ["label_A"] * n_rows)
+        self.assertEqual(annotations["columns"][col_idx.index("cat_B")], ["label_B"] * n_rows)
+
+        # verify the schema was updated
+        all_col_schema = {c["name"]: c for c in schema["annotations"]["obs"]["columns"]}
+        self.assertEqual(
+            all_col_schema["cat_A"],
+            {"name": "cat_A", "type": "categorical", "categories": ["label_A"], "writable": True},
+        )
+        self.assertEqual(
+            all_col_schema["cat_B"],
+            {"name": "cat_B", "type": "categorical", "categories": ["label_B"], "writable": True},
+        )
+
+    def test_put_float_data(self):
+        # verify that OBS PUTs (annotation_put_fbs) are accessible via
+        # GET (annotation_to_fbs_matrix)
+
+        n_rows = self.data.get_shape()[0]
+
+        # verifies that floating point with decimals fail.
+        fbs = make_fbs({"cat_F_FAIL": pd.Series([1.1] * n_rows, dtype=np.dtype("float"))})
+        with self.assertRaises(ValueError) as exception_context:
+            res = self.annotation_put_fbs(fbs)
+        self.assertEqual(str(exception_context.exception), "Columns may not have floating point types")
+
+        # verifies that floating point that can be converted to int passes
+        fbs = make_fbs({"cat_F_PASS": pd.Series([1.0] * n_rows, dtype="float")})
+        res = self.annotation_put_fbs(fbs)
+        self.assertEqual(res, json.dumps({"status": "OK"}))
+
+        # check read_labels
+        labels = self.annotations.read_labels(None)
+        fbsAll = self.data.annotation_to_fbs_matrix("obs", None, labels)
+        schema = schema_get_helper(self.data)
+        annotations = decode_fbs.decode_matrix_FBS(fbsAll)
+        self.assertEqual(annotations["n_rows"], n_rows)
+        all_col_schema = {c["name"]: c for c in schema["annotations"]["obs"]["columns"]}
+        self.assertEqual(
+            all_col_schema["cat_F_PASS"],
+            {"name": "cat_F_PASS", "type": "int32", "writable": True},
+        )

backend/test/test_czi_hosted/unit/common/utils/test_cxg_generation_utils.py

@@ -0,0 +1,158 @@
+import json
+import unittest
+from os import path, mkdir
+from shutil import rmtree
+from uuid import uuid4
+
+import numpy as np
+import tiledb
+from pandas import Series, DataFrame
+
+from backend.czi_hosted.common.utils.cxg_generation_utils import (
+    convert_dictionary_to_cxg_group,
+    convert_dataframe_to_cxg_array,
+    convert_ndarray_to_cxg_dense_array,
+    convert_matrix_to_cxg_array,
+)
+from backend.test import FIXTURES_ROOT
+
+
+class TestCxgGenerationUtils(unittest.TestCase):
+    def setUp(self):
+        self.testing_cxg_temp_directory = f"{FIXTURES_ROOT}/{uuid4()}"
+        mkdir(self.testing_cxg_temp_directory)
+
+    def tearDown(self):
+        if path.isdir(self.testing_cxg_temp_directory):
+            rmtree(self.testing_cxg_temp_directory)
+
+    def test__convert_dictionary_to_cxg_group__writes_successfully(self):
+        random_dictionary = {"cookies": "chocolate_chip", "brownies": "chocolate", "cake": "double chocolate"}
+        dictionary_name = "favorite_desserts"
+        expected_array_directory = f"{self.testing_cxg_temp_directory}/{dictionary_name}"
+
+        convert_dictionary_to_cxg_group(
+            self.testing_cxg_temp_directory, random_dictionary, group_metadata_name=dictionary_name
+        )
+
+        array = tiledb.open(expected_array_directory)
+        actual_stored_metadata = dict(array.meta.items())
+
+        self.assertTrue(path.isdir(expected_array_directory))
+        self.assertTrue(isinstance(array, tiledb.DenseArray))
+        self.assertEqual(random_dictionary, actual_stored_metadata)
+
+    def test__convert_dataframe_to_cxg_array__writes_successfully(self):
+        random_int_category = Series(data=[3, 1, 2, 4], dtype=np.int64)
+        random_bool_category = Series(data=[True, True, False, True], dtype=np.bool_)
+        random_dataframe_name = f"random_dataframe_{uuid4()}"
+        random_dataframe = DataFrame(data={"int_category": random_int_category, "bool_category": random_bool_category})
+
+        convert_dataframe_to_cxg_array(
+            self.testing_cxg_temp_directory, random_dataframe_name, random_dataframe, "int_category", tiledb.Ctx()
+        )
+
+        expected_array_directory = f"{self.testing_cxg_temp_directory}/{random_dataframe_name}"
+        expected_array_metadata = {
+            "cxg_schema": json.dumps(
+                {"int_category": {"type": "int32"}, "bool_category": {"type": "boolean"}, "index": "int_category"}
+            )
+        }
+
+        actual_stored_dataframe_array = tiledb.open(expected_array_directory)
+        actual_stored_dataframe_metadata = dict(actual_stored_dataframe_array.meta.items())
+
+        self.assertTrue(path.isdir(expected_array_directory))
+        self.assertTrue(isinstance(actual_stored_dataframe_array, tiledb.DenseArray))
+        self.assertDictEqual(expected_array_metadata, actual_stored_dataframe_metadata)
+        self.assertTrue((actual_stored_dataframe_array[0:4]["int_category"] == random_int_category.to_numpy()).all())
+        self.assertTrue((actual_stored_dataframe_array[0:4]["bool_category"] == random_bool_category.to_numpy()).all())
+
+    def test__convert_ndarray_to_cxg_dense_array__writes_successfully(self):
+        ndarray = np.random.rand(3, 2)
+        ndarray_name = f"{self.testing_cxg_temp_directory}/awesome_ndarray_{uuid4()}"
+
+        convert_ndarray_to_cxg_dense_array(ndarray_name, ndarray, tiledb.Ctx())
+
+        actual_stored_array = tiledb.open(ndarray_name)
+
+        self.assertTrue(path.isdir(ndarray_name))
+        self.assertTrue(isinstance(actual_stored_array, tiledb.DenseArray))
+        self.assertTrue((actual_stored_array[:, :] == ndarray).all())
+
+    def test__convert_matrix_to_cxg_array__dense_array_writes_successfully(self):
+        matrix = np.float32(np.random.rand(3, 2))
+        matrix_name = f"{self.testing_cxg_temp_directory}/awesome_matrix_{uuid4()}"
+
+        convert_matrix_to_cxg_array(matrix_name, matrix, False, tiledb.Ctx())
+
+        actual_stored_array = tiledb.open(matrix_name)
+
+        self.assertTrue(path.isdir(matrix_name))
+        self.assertTrue(isinstance(actual_stored_array, tiledb.DenseArray))
+        self.assertTrue((actual_stored_array[:, :] == matrix).all())
+
+    def test__convert_matrix_to_cxg_array__sparse_array_only_store_nonzeros_empty_array(self):
+        matrix = np.zeros([3, 2])
+        matrix_name = f"{self.testing_cxg_temp_directory}/awesome_zero_matrix_{uuid4()}"
+
+        convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx())
+
+        actual_stored_array = tiledb.open(matrix_name)
+
+        self.assertTrue(path.isdir(matrix_name))
+        self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
+        self.assertTrue(actual_stored_array[:, :][""].size == 0)
+
+    def test__convert_matrix_to_cxg_array__sparse_array_only_store_nonzeros(self):
+        matrix = np.zeros([3, 3])
+        matrix[0, 0] = 1
+        matrix[1, 1] = 1
+        matrix[2, 2] = 2
+        matrix_name = f"{self.testing_cxg_temp_directory}/awesome_sparse_matrix_{uuid4()}"
+
+        convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx())
+
+        actual_stored_array = tiledb.open(matrix_name)
+
+        self.assertTrue(path.isdir(matrix_name))
+        self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
+        self.assertTrue(actual_stored_array[0, 0][""] == 1)
+        self.assertTrue(actual_stored_array[1, 1][""] == 1)
+        self.assertTrue(actual_stored_array[2, 2][""] == 2)
+        self.assertTrue(actual_stored_array[:, :][""].size == 3)
+
+    def test__convert_matrix_to_cxg_array__sparse_array_with_column_encoding_empty_array(self):
+        matrix_name = f"{self.testing_cxg_temp_directory}/awesome_column_shift_matrix_{uuid4()}"
+        matrix = np.ones((3, 2))
+        # The column shift will be equal to the matrix since subtracting the column shift from the matrix will create
+        # a matrix of zeros which is sparse.
+        column_shift = np.ones((3, 2))
+
+        convert_matrix_to_cxg_array(
+            matrix_name, matrix, True, tiledb.Ctx(), column_shift_for_sparse_encoding=column_shift
+        )
+
+        actual_stored_array = tiledb.open(matrix_name)
+
+        self.assertTrue(path.isdir(matrix_name))
+        self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
+        self.assertTrue(actual_stored_array[:, :][""].size == 0)
+
+    def test__convert_matrix_to_cxg_array__sparse_array_with_column_encoding_partial_array(self):
+        matrix_name = f"{self.testing_cxg_temp_directory}/awesome_column_shift_matrix_{uuid4()}"
+        matrix = np.ones((2, 2))
+        # Only column shift the first column of ones.
+        column_shift = np.array([[1, 0], [1, 0]])
+
+        convert_matrix_to_cxg_array(
+            matrix_name, matrix, True, tiledb.Ctx(), column_shift_for_sparse_encoding=column_shift
+        )
+
+        actual_stored_array = tiledb.open(matrix_name)
+
+        self.assertTrue(path.isdir(matrix_name))
+        self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
+        self.assertTrue(actual_stored_array[0, 1][""] == 1)
+        self.assertTrue(actual_stored_array[1, 1][""] == 1)
+        self.assertTrue(actual_stored_array[:, :][""].size == 2)

backend/test/test_czi_hosted/unit/common/utils/test_matrix_utils.py

@@ -0,0 +1,66 @@
+import unittest
+
+import numpy as np
+
+from backend.czi_hosted.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix
+
+
+class TestMatrixUtils(unittest.TestCase):
+    def test__is_matrix_sparse__zero_and_one_hundred_percent_threshold(self):
+        matrix = np.array([1, 2, 3])
+
+        self.assertFalse(is_matrix_sparse(matrix, 0))
+        self.assertTrue(is_matrix_sparse(matrix, 100))
+
+    def test__is_matrix_sparse__partially_populated_sparse_matrix_returns_true(self):
+        matrix = np.zeros([3, 4])
+        matrix[2][3] = 1.0
+        matrix[1][1] = 2.2
+
+        self.assertTrue(is_matrix_sparse(matrix, 50))
+
+    def test__is_matrix_sparse__partially_populated_dense_matrix_returns_false(self):
+        matrix = np.zeros([2, 2])
+        matrix[0][0] = 1.0
+        matrix[0][1] = 2.2
+        matrix[1][1] = 3.7
+
+        self.assertFalse(is_matrix_sparse(matrix, 50))
+
+    def test__is_matrix_sparse__giant_matrix_returns_false_early(self):
+        matrix = np.ones([20000, 20])
+
+        with self.assertLogs(level="INFO") as logger:
+            self.assertFalse(is_matrix_sparse(matrix, 1))
+
+            # Because the function returns early a log will output the _estimate_ instead of the _exact_ percentage of
+            # non-zero elements in the matrix.
+            self.assertIn("Percentage of non-zero elements (estimate)", logger.output[0])
+
+    def test__is_matrix_sparse_with_column_shift_encoding__regular_sparse_returns_true(self):
+        matrix = np.zeros([2, 2])
+        matrix[0][0] = 1.0
+
+        self.assertIsNotNone(get_column_shift_encode_for_matrix(matrix, 50))
+
+    def test__is_matrix_sparse_with_column_shift_encoding__column_shift_returns_same_value(self):
+        matrix = np.ones([2, 2])
+        expected_column_shift = [1, 1]
+
+        actual_column_shift = get_column_shift_encode_for_matrix(matrix, 50)
+        self.assertTrue((expected_column_shift == actual_column_shift).all())
+
+    def test__is_matrix_sparse_with_column_shift_encoding__impossible_column_shift_returns_none(self):
+        matrix = np.array([[1, 2], [3, 4]])
+
+        self.assertIsNone(get_column_shift_encode_for_matrix(matrix, 50))
+
+    def test__is_matrix_sparse_with_column_shift_encoding__giant_matrix_returns_false_early(self):
+        matrix = np.random.rand(20000, 20)
+
+        with self.assertLogs(level="INFO") as logger:
+            self.assertFalse(is_matrix_sparse(matrix, 1))
+
+            # Because the function returns early a log will output the _estimate_ instead of the _exact_ percentage of
+            # non-zero elements in the matrix.
+            self.assertIn("Percentage of non-zero elements (estimate)", logger.output[0])

backend/test/test_czi_hosted/unit/common/utils/test_sanitization_utils.py

@@ -0,0 +1,55 @@
+import unittest
+
+from backend.czi_hosted.common.utils.sanitization_utils import sanitize_values_in_list, sanitize_keys_in_dictionary
+
+
+class TestSanitizationUtils(unittest.TestCase):
+    def test__sanitize_values_in_list__not_strings_raises_exception(self):
+        keys_to_sanitize = [1, 2, 3]
+
+        with self.assertRaises(Exception) as exception_context:
+            sanitize_values_in_list(keys_to_sanitize)
+
+        self.assertIn("must contain all strings", str(exception_context.exception))
+
+    def test__sanitize_values_in_list__not_all_strings_raises_exception(self):
+        keys_to_sanitize = ["1", "2", 3]
+
+        with self.assertRaises(Exception) as exception_context:
+            sanitize_values_in_list(keys_to_sanitize)
+
+        self.assertIn("must contain all strings", str(exception_context.exception))
+
+    def test__sanitize_values_in_list__replace_non_ascii_character_with_underscore(self):
+        keys_to_sanitize = ["abc.", "~abc", "a~b/c"]
+        expected_sanitized_keys_dict = dict(zip(keys_to_sanitize, ["abc_", "_abc", "a_b_c"]))
+
+        actual_sanitized_keys_dict = sanitize_values_in_list(keys_to_sanitize)
+
+        self.assertEqual(expected_sanitized_keys_dict, actual_sanitized_keys_dict)
+
+    def test__sanitize_keys_in_dictionary__replace_non_ascii_character_with_underscore(self):
+        dictionary_to_sanitize = {"abc.": 3, "~abc": 4, "a~b/c": 5}
+        expected_sanitized_dict = {"abc_": 3, "_abc": 4, "a_b_c": 5}
+
+        actual_sanitized_dict = dictionary_to_sanitize
+        sanitize_keys_in_dictionary(actual_sanitized_dict)
+
+        self.assertEqual(expected_sanitized_dict, actual_sanitized_dict)
+
+    def test__sanitize_keys_in_dictionary__non_string_key_raises_exception(self):
+        dictionary_to_sanitize = {4: 3, "~abc": 4, "a~b/c": 5}
+
+        with self.assertRaises(Exception) as exception_context:
+            sanitize_keys_in_dictionary(dictionary_to_sanitize)
+
+        self.assertIn("must contain all strings", str(exception_context.exception))
+
+    def test__sanitize_keys_in_dictionary__replace_only_some_keys(self):
+        dictionary_to_sanitize = {"abc": 3, "~abc": 4, "a~b/c": 5}
+        expected_sanitized_dict = {"abc": 3, "_abc": 4, "a_b_c": 5}
+
+        actual_sanitized_dict = dictionary_to_sanitize
+        sanitize_keys_in_dictionary(actual_sanitized_dict)
+
+        self.assertEqual(expected_sanitized_dict, actual_sanitized_dict)

backend/test/test_czi_hosted/unit/common/utils/test_utils.py

@@ -0,0 +1,34 @@
+import os
+import shutil
+import unittest
+
+from backend.common.utils.utils import import_plugins
+from backend.test import PROJECT_ROOT, random_string
+
+
+class TestPlugins(unittest.TestCase):
+    """ Test plugin import functionality """
+
+    plugins_dir = f"{PROJECT_ROOT}/backend/test/test_czi_hosted/unit/plugins"
+    test_plugin_path = f"{plugins_dir}/foo.py"
+    secret = random_string(8)
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        if not os.path.isdir(cls.plugins_dir):
+            os.mkdir(cls.plugins_dir)
+        with open(cls.test_plugin_path, "w") as fh:
+            fh.write(f'SECRET = "{cls.secret}"\n')
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        if os.path.isdir(cls.plugins_dir):
+            shutil.rmtree(cls.plugins_dir)
+
+    def test_import_plugins(self):
+        self.assertTrue(os.path.isfile(self.test_plugin_path))
+        loaded_modules = import_plugins("backend.test.test_czi_hosted.unit.plugins")
+        # test that import plugins found the file
+        self.assertEqual(["backend.test.test_czi_hosted.unit.plugins.foo"], [ele.__name__ for ele in loaded_modules])
+        # test that the module was properly executed
+        self.assertEqual(self.secret, loaded_modules[0].SECRET)

backend/test/test_czi_hosted/unit/compute/test_diffexp_cxg.py

@@ -0,0 +1,157 @@
+import os
+import tempfile
+import unittest
+
+import numpy as np
+
+from backend.czi_hosted.compute import diffexp_generic, diffexp_cxg
+from backend.czi_hosted.compute.diffexp_cxg import diffexp_ttest
+from backend.czi_hosted.converters.h5ad_data_file import H5ADDataFile
+from backend.common.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs
+from backend.czi_hosted.data_common.matrix_loader import MatrixDataLoader
+from backend.test.test_czi_hosted.performance.create_test_matrix import create_test_h5ad
+from backend.test.test_czi_hosted.unit import app_config
+from backend.test import PROJECT_ROOT, FIXTURES_ROOT
+
+
+class DiffExpTest(unittest.TestCase):
+    """Tests the diffexp returns the expected results for one test case, using different
+    adaptor types and different algorithms."""
+
+    def load_dataset(self, path, extra_server_config={}, extra_dataset_config={}):
+        config = app_config(path, extra_server_config=extra_server_config, extra_dataset_config=extra_dataset_config)
+        loader = MatrixDataLoader(path)
+        adaptor = loader.open(config)
+        return adaptor
+
+    def get_mask(self, adaptor, start, stride):
+        """Simple function to return a mask or rows"""
+        rows = adaptor.get_shape()[0]
+        sel = list(range(start, rows, stride))
+        mask = np.zeros(rows, dtype=bool)
+        mask[sel] = True
+        return mask
+
+    def compare_diffexp_results(self, results, expects):
+        self.assertEqual(len(results), len(expects))
+        for result, expect in zip(results, expects):
+            self.assertEqual(result[0], expect[0])
+            self.assertTrue(np.isclose(result[1], expect[1], 1e-6, 1e-4))
+            self.assertTrue(np.isclose(result[2], expect[2], 1e-6, 1e-4))
+            self.assertTrue(np.isclose(result[3], expect[3], 1e-6, 1e-4))
+
+    def check_1_10_2_10(self, results):
+        """Checks the results for a specific set of rows selections"""
+        expects = [
+            [956, 0.016060986, 0.0008649321884808977, 1.0],
+            [1124, 0.96602094, 0.0011717216548271284, 1.0],
+            [1809, 1.1110606, 0.0019304405196777848, 1.0],
+            [1712, -0.5525154, 0.0051788902660723345, 1.0],
+            [1754, 0.5201581, 0.005691734062127954, 1.0],
+            [948, 1.6390722, 0.006622111055981219, 1.0],
+            [1810, 0.78618884, 0.007055917428377063, 1.0],
+            [779, 1.5241305, 0.007202934422407284, 1.0],
+            [1575, 1.0317602, 0.007830310753043345, 1.0],
+            [576, 0.97873515, 0.008272092578813124, 1.0],
+        ]
+        self.compare_diffexp_results(results, expects)
+
+    def get_X_col(self, adaptor, cols):
+        varmask = np.zeros(adaptor.get_shape()[1], dtype=bool)
+        varmask[cols] = True
+        return adaptor.get_X_array(None, varmask)
+
+    def test_anndata_default(self):
+        """Test an anndata adaptor with its default diffexp algorithm (diffexp_generic)"""
+        adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
+        maskA = self.get_mask(adaptor, 1, 10)
+        maskB = self.get_mask(adaptor, 2, 10)
+        results = adaptor.compute_diffexp_ttest(maskA, maskB, 10)
+        self.check_1_10_2_10(results)
+
+    def test_cxg_default(self):
+        """Test a cxg adaptor with its default diffexp algorithm (diffexp_cxg)"""
+        adaptor = self.load_dataset(f"{FIXTURES_ROOT}/pbmc3k.cxg")
+        maskA = self.get_mask(adaptor, 1, 10)
+        maskB = self.get_mask(adaptor, 2, 10)
+
+        # run it through the adaptor
+        results = adaptor.compute_diffexp_ttest(maskA, maskB, 10)
+        self.check_1_10_2_10(results)
+
+        # run it directly
+        results = diffexp_ttest(adaptor, maskA, maskB, 10)
+        self.check_1_10_2_10(results)
+
+    def test_cxg_generic(self):
+        """Test a cxg adaptor with the generic adaptor"""
+        adaptor = self.load_dataset(f"{FIXTURES_ROOT}/pbmc3k.cxg")
+        maskA = self.get_mask(adaptor, 1, 10)
+        maskB = self.get_mask(adaptor, 2, 10)
+        # run it directly
+        results = diffexp_generic.diffexp_ttest(adaptor, maskA, maskB, 10)
+        self.check_1_10_2_10(results)
+
+    def test_cxg_sparse(self):
+        self.sparse_diffexp(False)
+
+    def test_cxg_sparse_col_shift(self):
+        self.sparse_diffexp(True)
+
+    def sparse_diffexp(self, apply_col_shift):
+        with tempfile.TemporaryDirectory() as dirname:
+            # create a sparse matrix
+            h5adfile_path = os.path.join(dirname, "sparse.h5ad")
+            create_test_h5ad(h5adfile_path, 2000, 2000, 10, apply_col_shift)
+
+            h5ad_file_to_convert = H5ADDataFile(h5adfile_path, use_corpora_schema=False)
+
+            sparsename = os.path.join(dirname, "sparse.cxg")
+            h5ad_file_to_convert.to_cxg(sparsename, 11, True)
+
+            adaptor_anndata = self.load_dataset(h5adfile_path, extra_dataset_config=dict(embeddings__names=[]))
+
+            adaptor_sparse = self.load_dataset(sparsename)
+            assert adaptor_sparse.open_array("X").schema.sparse
+            assert adaptor_sparse.has_array("X_col_shift") == apply_col_shift
+
+            densename = os.path.join(dirname, "dense.cxg")
+            h5ad_file_to_convert.to_cxg(densename, True, 0)
+            adaptor_dense = self.load_dataset(densename)
+            assert not adaptor_dense.open_array("X").schema.sparse
+            assert not adaptor_dense.has_array("X_col_shift")
+
+            maskA = self.get_mask(adaptor_anndata, 1, 10)
+            maskB = self.get_mask(adaptor_anndata, 2, 10)
+
+            diffexp_results_anndata = diffexp_generic.diffexp_ttest(adaptor_anndata, maskA, maskB, 10)
+            diffexp_results_sparse = diffexp_cxg.diffexp_ttest(adaptor_sparse, maskA, maskB, 10)
+            diffexp_results_dense = diffexp_cxg.diffexp_ttest(adaptor_dense, maskA, maskB, 10)
+
+            self.compare_diffexp_results(diffexp_results_anndata, diffexp_results_sparse)
+            self.compare_diffexp_results(diffexp_results_anndata, diffexp_results_dense)
+
+            topcols = np.array([x[0] for x in diffexp_results_anndata])
+            cols_anndata = self.get_X_col(adaptor_anndata, topcols)
+            cols_sparse = self.get_X_col(adaptor_sparse, topcols)
+            cols_dense = self.get_X_col(adaptor_dense, topcols)
+            assert cols_anndata.shape[0] == adaptor_sparse.get_shape()[0]
+            assert cols_anndata.shape[1] == len(diffexp_results_anndata)
+
+            def convert(mat, cols):
+                return decode_matrix_fbs(encode_matrix_fbs(mat, col_idx=cols)).to_numpy()
+
+            cols_anndata = convert(cols_anndata, topcols)
+            cols_sparse = convert(cols_sparse, topcols)
+            cols_dense = convert(cols_dense, topcols)
+
+            x = adaptor_sparse.get_X_array()
+            assert x.shape == adaptor_sparse.get_shape()
+
+            for row in range(cols_anndata.shape[0]):
+                for col in range(cols_anndata.shape[1]):
+                    vanndata = cols_anndata[row][col]
+                    vsparse = cols_sparse[row][col]
+                    vdense = cols_dense[row][col]
+                    self.assertTrue(np.isclose(vanndata, vsparse, 1e-6, 1e-6))
+                    self.assertTrue(np.isclose(vanndata, vdense, 1e-6, 1e-6))

backend/test/test_czi_hosted/unit/converters/schema/test_gene_symbol.py

@@ -0,0 +1,61 @@
+import os
+import unittest
+
+import pandas as pd
+
+from backend.czi_hosted.converters.schema import gene_symbol
+from backend.test import FIXTURES_ROOT
+
+
+class TestHGNCSymbolChecker(unittest.TestCase):
+
+    def setUp(self):
+        self.test_hgnc_path = os.path.join(FIXTURES_ROOT, "hgnc_example.txt.gz")
+        self.hgnc_checker = gene_symbol.HGNCSymbolChecker.from_hgnc_records(self.test_hgnc_path)
+
+    def test_symbol_upgrade(self):
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("SEPT1"), "SEPTIN1")
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("ADRB2R"), "ADRB2")
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("BAR"), "ADRB2")
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("sept1"), "SEPTIN1")
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("AdRb2R"), "ADRB2")
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("bar"), "ADRB2")
+
+        # Strip off seurat endings when appropriate
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("SEPT1.1"), "SEPTIN1")
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("ADRB2-1"), "ADRB2")
+
+        # DIFF6 is ambiguous so don't upgrade it
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("DIFF6"), "DIFF6")
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("diff6"), "diff6")
+
+        # ARG1 is approved
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("ARG1"), "ARG1")
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("arg1"), "ARG1")
+
+        # HAP1 is both approved and withdrawn
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("HAP1"), "HAP1")
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("hap1"), "HAP1")
+
+        # Leave unknown symbols alone
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("NOTASYMBOL"), "NOTASYMBOL")
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("notasymbol"), "notasymbol")
+
+        # Upgrade HGNC ids unless you can't find it
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("HGNC:286"), "ADRB2")
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("HGNC:4812"), "HAP1")
+        self.assertEqual(self.hgnc_checker.upgrade_symbol("HGNC:123456"), "HGNC:123456")
+
+    def test_check_symbol(self):
+        self.assertEqual(self.hgnc_checker.check_symbol("SEPT1"), gene_symbol.SymbolStatus.UPGRADABLE)
+        self.assertEqual(self.hgnc_checker.check_symbol("DIFF6"), gene_symbol.SymbolStatus.AMBIGUOUS)
+        self.assertEqual(self.hgnc_checker.check_symbol("NOTASYMBOL"), gene_symbol.SymbolStatus.UNKNOWN)
+
+        # HAP1 is one of the approved and withdrawn symbols
+        self.assertEqual(self.hgnc_checker.check_symbol("HAP1"), gene_symbol.SymbolStatus.APPROVED)
+
+    def test_upgrade_index(self):
+        index = pd.Index(["SEPT1", "DIFF6", "NOTASYMBOL", "bar", "SEPTIN1"])
+        var_df = pd.DataFrame([[0] * len(index)], index=index)
+        upgraded_index = gene_symbol.get_upgraded_var_index(var_df, hgnc_path=self.test_hgnc_path)
+        self.assertEqual(upgraded_index.tolist(), ["SEPTIN1", "DIFF6", "NOTASYMBOL", "ADRB2", "SEPTIN1"])

backend/test/test_czi_hosted/unit/converters/schema/test_ontology.py

@@ -0,0 +1,128 @@
+import json
+
+import unittest.mock
+
+from backend.czi_hosted.converters.schema import ontology
+
+
+class TestOntologyParsing(unittest.TestCase):
+    def setUp(self):
+
+        self.curies = ["UBERON:0002048", "HsapDv:0000174", "NCBITaxon:9606", "EFO:0008995"]
+
+        self.names = ["UBERON", "HsapDv", "NCBITaxon", "EFO"]
+
+        self.values = ["0002048", "0000174", "9606", "0008995"]
+
+        self.iris = [
+            "http://purl.obolibrary.org/obo/UBERON_0002048",
+            "http://purl.obolibrary.org/obo/HsapDv_0000174",
+            "http://purl.obolibrary.org/obo/NCBITaxon_9606",
+            "http://www.ebi.ac.uk/efo/EFO_0008995",
+        ]
+
+        URL_ROOT = "http://www.ebi.ac.uk/ols/api/ontologies/"
+        self.urls = [
+            URL_ROOT + "UBERON/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FUBERON_0002048",
+            URL_ROOT + "HsapDv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FHsapDv_0000174",
+            URL_ROOT + "NCBITaxon/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_9606",
+            URL_ROOT + "EFO/terms/http%253A%252F%252Fwww.ebi.ac.uk%252Fefo%252FEFO_0008995",
+        ]
+
+        self.responses = {
+            "UBERON:0002048": {
+                "iri": "http://purl.obolibrary.org/obo/UBERON_0002048",
+                "description": ["Respiration organ that develops as an outpocketing of the esophagus."],
+                "label": "lung",
+            },
+            "HsapDv:0000174": {
+                "iri": "http://purl.obolibrary.org/obo/HsapDv_0000174",
+                "description": ["Infant stage that refers to an infant who is over 1 and under 2 months old."],
+                "label": "1-month-old human stage",
+            },
+            "NCBITaxon:9606": {
+                "iri": "http://purl.obolibrary.org/obo/NCBITaxon_9606",
+                "description": None,
+                "label": "Homo sapiens",
+            },
+            "EFO:0008995": {
+                "iri": "http://www.ebi.ac.uk/efo/EFO_0008995",
+                "description": [
+                    (
+                        '10X is a "synthetic long-read" technology and works by capturing a barcoded oligo-coated '
+                        "gel-bead and 0.3x genome copies into a single emulsion droplet, processing the equivalent "
+                        "of 1 million pipetting steps. Successive versions of the 10x chemistry use different "
+                        "barcode locations to improve the sequencing yield and quality of 10x experiments."
+                    )
+                ],
+                "label": "10X sequencing",
+            },
+        }
+
+    def test_ontololgy_name(self):
+        for curie, expected_name in zip(self.curies, self.names):
+            self.assertEqual(ontology._ontology_name(curie), expected_name)
+
+    def test_ontololgy_value(self):
+        for curie, expected_value in zip(self.curies, self.values):
+            self.assertEqual(ontology._ontology_value(curie), expected_value)
+
+    def test_iri(self):
+        for curie, expected_iri in zip(self.curies, self.iris):
+            self.assertEqual(ontology._iri(curie), expected_iri)
+
+    def test_ontology_info_url(self):
+        for curie, expected_url in zip(self.curies, self.urls):
+            self.assertEqual(ontology._ontology_info_url(curie), expected_url)
+
+    def test_empty_ontology_info_url(self):
+        self.assertEqual(ontology._ontology_info_url(""), "")
+
+
+class TestOntologyLookup(unittest.TestCase):
+    def setUp(self):
+        self.responses = {
+            "UBERON:0002048": {
+                "iri": "http://purl.obolibrary.org/obo/UBERON_0002048",
+                "description": ["Respiration organ that develops as an outpocketing of the esophagus."],
+                "label": "lung",
+            },
+            "HsapDv:0000174": {
+                "iri": "http://purl.obolibrary.org/obo/HsapDv_0000174",
+                "description": ["Infant stage that refers to an infant who is over 1 and under 2 months old."],
+                "label": "1-month-old human stage",
+            },
+            "NCBITaxon:9606": {
+                "iri": "http://purl.obolibrary.org/obo/NCBITaxon_9606",
+                "description": None,
+                "label": "Homo sapiens",
+            },
+            "EFO:0008995": {
+                "iri": "http://www.ebi.ac.uk/efo/EFO_0008995",
+                "description": [
+                    ('10X is a "synthetic long-read" technology and works by capturing a barcoded oligo-coated '
+                     'gel-bead and 0.3x genome copies into a single emulsion droplet, processing the equivalent '
+                     'of 1 million pipetting steps. Successive versions of the 10x chemistry use different barcode '
+                     'locations to improve the sequencing yield and quality of 10x experiments.')
+                ],
+                "label": "10X sequencing",
+            },
+        }
+
+        self.labels = {
+            "UBERON:0002048": "lung",
+            "HsapDv:0000174": "1-month-old human stage",
+            "NCBITaxon:9606": "Homo sapiens",
+            "EFO:0008995": "10X sequencing",
+        }
+
+    @unittest.mock.patch("requests.get")
+    def test_lookup_label(self, mock_get):
+
+        for curie, response in self.responses.items():
+            mock_get.return_value.content = json.dumps(response)
+            mock_get.return_value.json.return_value = response
+            mock_get.return_value.status_code = 200
+
+            label = ontology.get_ontology_label(curie)
+            self.assertEqual(label, self.labels[curie])

backend/test/test_czi_hosted/unit/converters/schema/test_remix.py

@@ -0,0 +1,257 @@
+import json
+import os
+import unittest
+import unittest.mock
+
+import anndata
+import numpy
+import pandas as pd
+import scanpy as sc
+
+from backend.czi_hosted.converters.schema import remix
+
+from backend.test import PROJECT_ROOT, FIXTURES_ROOT
+
+
+class TestApplySchema(unittest.TestCase):
+
+    def setUp(self):
+        self.source_h5ad_path = f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad"
+        self.output_h5ad_path = f"{FIXTURES_ROOT}/test_remix.h5ad"
+        self.config_path = f"{FIXTURES_ROOT}/test_config.yaml"
+        self.bad_config_path = f"{FIXTURES_ROOT}/test_bad_config.yaml"
+
+    def tearDown(self):
+        try:
+            os.remove(self.output_h5ad_path)
+        except OSError:
+            pass
+
+    @unittest.mock.patch("backend.czi_hosted.converters.schema.ontology.get_ontology_label")
+    def test_apply_schema(self, mock_get_ontology_label):
+        mock_get_ontology_label.return_value = "test label"
+        remix.apply_schema(self.source_h5ad_path, self.config_path, self.output_h5ad_path)
+        new_adata = sc.read_h5ad(self.output_h5ad_path)
+
+        self.assertIn("cell_type", new_adata.obs.columns)
+        self.assertListEqual(["test label"], new_adata.obs["cell_type"].unique().tolist())
+        self.assertListEqual(
+            ["CL:00001", "CL:00002", "CL:00003", "CL:00004", "CL:00005", "CL:00006", "CL:00007", "CL:00008"],
+            sorted(new_adata.obs["cell_type_ontology_term_id"].unique().tolist())
+        )
+
+        self.assertIn("version", new_adata.uns_keys())
+
+    @unittest.mock.patch("backend.czi_hosted.converters.schema.ontology.get_ontology_label")
+    def test_apply_bad_schema(self, mock_get_ontology_label):
+        mock_get_ontology_label.return_value = "test label"
+        remix.apply_schema(self.source_h5ad_path, self.bad_config_path, self.output_h5ad_path)
+        new_adata = sc.read_h5ad(self.output_h5ad_path)
+
+        # Should refuse to write the version
+        self.assertNotIn("version", new_adata.uns_keys())
+
+class TestFieldParsing(unittest.TestCase):
+
+    def test_is_curie(self):
+        self.assertTrue(remix.is_curie("EFO:00001"))
+        self.assertTrue(remix.is_curie("UBERON:123456"))
+        self.assertTrue(remix.is_curie("HsapDv:0001"))
+        self.assertFalse(remix.is_curie("UBERON"))
+        self.assertFalse(remix.is_curie("UBERON:"))
+        self.assertFalse(remix.is_curie("123456"))
+
+    def test_is_ontology_field(self):
+        self.assertTrue(remix.is_ontology_field("tissue_ontology_term_id"))
+        self.assertTrue(remix.is_ontology_field("cell_type_ontology_term_id"))
+        self.assertFalse(remix.is_ontology_field("cell_ontology"))
+        self.assertFalse(remix.is_ontology_field("method"))
+
+    def test_get_label_field_name(self):
+        self.assertEqual("tissue", remix.get_label_field_name("tissue_ontology_term_id"))
+        self.assertEqual("cell_type", remix.get_label_field_name("cell_type_ontology_term_id"))
+
+    def test_split_suffix(self):
+        self.assertEqual(("UBERON:1234", " (organoid)"), remix.split_suffix("UBERON:1234 (organoid)"))
+        self.assertEqual(("UBERON:1234", " (cell culture)"), remix.split_suffix("UBERON:1234 (cell culture)"))
+        self.assertEqual(("UBERON:1234", ""), remix.split_suffix("UBERON:1234"))
+        self.assertEqual(("UBERON:1234 (something)", ""), remix.split_suffix("UBERON:1234 (something)"))
+
+    @unittest.mock.patch("backend.czi_hosted.converters.schema.ontology.get_ontology_label")
+    def test_get_curie_and_label(self, mock_get_ontology_label):
+        mock_get_ontology_label.return_value = "test label"
+        self.assertEqual(
+            remix.get_curie_and_label("UBERON:1234"),
+            ("UBERON:1234", "test label")
+        )
+        self.assertEqual(
+            remix.get_curie_and_label("UBERON:1234 (cell culture)"),
+            ("UBERON:1234 (cell culture)", "test label (cell culture)")
+        )
+        self.assertEqual(
+            remix.get_curie_and_label("whatever"),
+            ("", "whatever")
+        )
+
+
+class TestManipulateAnndata(unittest.TestCase):
+
+    def setUp(self):
+
+        self.cell_count = 20
+        self.gene_count = 200
+        X = numpy.random.randint(0, 1000, (self.cell_count, self.gene_count))
+        uns = {"organism": "monkey", "experiment": "monkey experiment"}
+        obs = pd.DataFrame(
+            index=[f"Cell{d}" for d in range(self.cell_count)],
+            columns=["tissue", "CellType"],
+            data=[["lung", "epithelial"]] * (self.cell_count // 2) + [["lung", "endothelial"]] * (self.cell_count // 2)
+        )
+        var = pd.DataFrame(index=[f"SEPT{d}" for d in range(self.gene_count)])
+
+        self.adata = anndata.AnnData(X=X, obs=obs, var=var, uns=uns)
+
+    def test_safe_add_field(self):
+
+        remix.safe_add_field(self.adata.obs, "tissue", ["monkey lung"] * self.cell_count)
+        self.assertEqual(self.adata.obs["tissue_original"].tolist(), ["lung"] * self.cell_count)
+        self.assertEqual(self.adata.obs["tissue"].tolist(), ["monkey lung"] * self.cell_count)
+
+        remix.safe_add_field(self.adata.uns, "contributors", [{"name": "contributor1"}, {"name": "contributor2"}])
+        self.assertEqual(
+            self.adata.uns["contributors"],
+            json.dumps([{"name": "contributor1"}, {"name": "contributor2"}])
+        )
+
+    @unittest.mock.patch("backend.czi_hosted.converters.schema.ontology.get_ontology_label")
+    def test_remix_uns(self, mock_get_ontology_label):
+        mock_get_ontology_label.return_value = "Pan troglodytes"
+        uns_config = {
+            "version": {
+                "corpora_schema_version": "1.0.0",
+                "corpora_encoding_version": "0.1.0"
+            },
+            "organism_ontology_term_id": "NCBITaxon:9598",
+            "contributors": [
+                {
+                    "name": "scientist",
+                    "email": "scientist@science.com"
+                }
+            ]
+        }
+
+        remix.remix_uns(self.adata, uns_config)
+
+        self.assertEqual(
+            sorted(self.adata.uns_keys()),
+            sorted(["organism_original", "organism", "organism_ontology_term_id",
+                    "contributors", "version", "experiment"])
+        )
+
+        self.assertEqual(self.adata.uns['organism'], "Pan troglodytes")
+        self.assertEqual(self.adata.uns['organism_original'], "monkey")
+        self.assertEqual(self.adata.uns['organism_ontology_term_id'], "NCBITaxon:9598")
+        self.assertEqual(self.adata.uns['contributors'],
+                         json.dumps([{"name": "scientist", "email": "scientist@science.com"}]))
+
+    @unittest.mock.patch("backend.czi_hosted.converters.schema.ontology.get_ontology_label")
+    def test_remix_obs(self, mock_get_ontology_label):
+        mock_get_ontology_label.return_value = "lung (in a monkey)"
+        obs_config = {
+            "tissue_ontology_term_id": {
+                "tissue": {
+                    "lung": "UBERON:00000"
+                }
+            },
+            "cell_color": {
+                "CellType": {
+                    "epithelial": "fuschia",
+                    "endothelial": "khaki"
+                }
+            },
+            "sex": "male"
+        }
+
+        remix.remix_obs(self.adata, obs_config)
+        self.assertEqual(
+            sorted(self.adata.obs_keys()),
+            sorted(["tissue", "tissue_ontology_term_id", "tissue_original", "CellType", "cell_color", "sex"])
+        )
+
+        self.assertTrue(all(v == "lung" for v in self.adata.obs.tissue_original))
+        self.assertTrue(all(v == "UBERON:00000" for v in self.adata.obs.tissue_ontology_term_id))
+        self.assertTrue(all(v == "lung (in a monkey)" for v in self.adata.obs.tissue))
+        self.assertTrue(all(v == "male" for v in self.adata.obs.sex))
+        self.assertTrue(all(v in (("epithelial", "fuschia"), ("endothelial", "khaki"))
+                            for v in zip(self.adata.obs.CellType, self.adata.obs.cell_color)))
+
+
+class TestFixupGeneSymbols(unittest.TestCase):
+
+    def setUp(self):
+        self.seurat_path = f"{PROJECT_ROOT}/czi_hosted/test/fixtures/schema_test_data/seurat_tutorial.h5ad"
+        self.seurat_merged_path = f"{PROJECT_ROOT}/czi_hosted/test/fixtures/schema_test_data/seurat_tutorial_merged.h5ad"
+        self.sctransform_path = f"{PROJECT_ROOT}/czi_hosted/test/fixtures/schema_test_data/sctransform.h5ad"
+        self.sctransform_merged_path = f"{PROJECT_ROOT}/czi_hosted/test/fixtures/schema_test_data/sctransform_merged.h5ad"
+
+        # There's lots of MALAT1, but it doesn't collide with any other names,
+        # so it shouldn't change during merging.
+        self.stable_gene = "MALAT1"
+
+    def test_fixup_gene_symbols_seurat(self):
+
+        if not os.path.isfile(self.seurat_path):
+            return unittest.skip(
+                "Skipping gene symbol conversion tests because test h5ads are not present. To create them, "
+                "run czi_hosted/test/fixtures/schema_test_data/generate_test_data.sh"
+            )
+
+        original_adata = sc.read_h5ad(self.seurat_path)
+        merged_adata = sc.read_h5ad(self.seurat_merged_path)
+
+        fixup_config = {"X": "log1p", "counts": "raw", "scale.data": "log1p"}
+
+        fixed_adata = remix.fixup_gene_symbols(original_adata, fixup_config)
+
+        self.assertEqual(
+            merged_adata.layers["counts"][:, merged_adata.var.index == self.stable_gene].sum(),
+            fixed_adata.raw.X[:, fixed_adata.var.index == self.stable_gene].sum()
+        )
+        self.assertAlmostEqual(
+            merged_adata.X[:, merged_adata.var.index == self.stable_gene].sum(),
+            fixed_adata.X[:, fixed_adata.var.index == self.stable_gene].sum()
+        )
+
+        self.assertAlmostEqual(
+            merged_adata.layers["scale.data"][:, merged_adata.var.index == self.stable_gene].sum(),
+            fixed_adata.layers["scale.data"][:, fixed_adata.var.index == self.stable_gene].sum()
+        )
+
+    def test_fixup_gene_symbols_sctransform(self):
+
+        if not os.path.isfile(self.sctransform_path):
+            return unittest.skip(
+                "Skipping gene symbol conversion tests because test h5ads are not present. To create them, "
+                "run czi_hosted/test/fixtures/schema_test_data/generate_test_data.sh"
+            )
+
+        original_adata = sc.read_h5ad(self.sctransform_path)
+        merged_adata = sc.read_h5ad(self.sctransform_merged_path)
+
+        fixup_config = {"X": "log1p", "counts": "raw"}
+
+        fixed_adata = remix.fixup_gene_symbols(original_adata, fixup_config)
+
+        # sctransform does a bunch of stuff, including slightly modifying the
+        # raw counts. So we can't assert for exact equality the way we do with
+        # the vanilla seurat tutorial. But, the results should still be very
+        # close.
+        merged_raw_stable = merged_adata.layers["counts"][:, merged_adata.var.index == self.stable_gene].sum()
+        fixed_raw_stable = fixed_adata.raw.X[:, fixed_adata.var.index == self.stable_gene].sum()
+        self.assertLess(abs(merged_raw_stable - fixed_raw_stable), .001 * merged_raw_stable)
+
+        self.assertAlmostEqual(
+            merged_adata.X[:, merged_adata.var.index == self.stable_gene].sum(),
+            fixed_adata.X[:, fixed_adata.var.index == self.stable_gene].sum(),
+            0
+        )

backend/test/test_czi_hosted/unit/converters/schema/test_validate.py

@@ -0,0 +1,434 @@
+import json
+import unittest
+
+import pandas as pd
+import scanpy as sc
+
+from backend.czi_hosted.converters.schema import validate
+
+from backend.test import PROJECT_ROOT
+
+
+class TestFieldValidation(unittest.TestCase):
+
+    def test_validate_stringified_list_of_dicts(self):
+
+        good = json.dumps([{"a": 1}, {2: "x", "z": "y"}])
+        not_stringified = [{"a": 1}, {2: "x", "z": "y"}]
+        not_a_list = json.dumps({"bad": "dict"})
+        not_json = "oh hey!"
+
+        self.assertTrue(validate._validate_stringified_list_of_dicts(good))
+
+        self.assertFalse(validate._validate_stringified_list_of_dicts(not_stringified))
+        self.assertFalse(validate._validate_stringified_list_of_dicts(not_a_list))
+        self.assertFalse(validate._validate_stringified_list_of_dicts(not_json))
+
+    def test_validate_human_readable_string(self):
+
+        good = "oh hey!"
+        curie = "EFO:0001"
+        ensg = "ENSG000001234"
+        enst = "ENST000005678"
+
+        self.assertTrue(validate._validate_human_readable_string(good))
+
+        self.assertFalse(validate._validate_human_readable_string(curie))
+        self.assertFalse(validate._validate_human_readable_string(ensg))
+        self.assertFalse(validate._validate_human_readable_string(enst))
+
+    def test_validate_curie(self):
+
+        self.assertTrue(validate._validate_curie("UBERON:00001", ["UBERON", "EFO"]))
+        self.assertTrue(validate._validate_curie("HsapDv:00002", ["HsapDv"]))
+
+        self.assertFalse(validate._validate_curie("HsapDv:00002", ["UBERON", "EFO"]))
+        self.assertFalse(validate._validate_curie("EFO:00002 (organoid)", ["UBERON", "EFO"]))
+        self.assertFalse(validate._validate_curie("EFO:00002 extra", ["UBERON", "EFO"]))
+        self.assertFalse(validate._validate_curie("UBERON:ABCD", ["UBERON", "EFO"]))
+        self.assertFalse(validate._validate_curie("Uberon:00002", ["UBERON", "EFO"]))
+        self.assertFalse(validate._validate_curie("UBERON:", ["UBERON", "EFO"]))
+        self.assertFalse(validate._validate_curie("UBERON", ["UBERON", "EFO"]))
+
+    def test_validate_suffixed_curie(self):
+
+        self.assertTrue(validate._validate_suffixed_curie("EFO:00001", ["UBERON", "EFO"]))
+        self.assertTrue(validate._validate_suffixed_curie("UBERON:00001 (cell culture)", ["UBERON", "EFO"]))
+
+        self.assertFalse(validate._validate_suffixed_curie("HsapDv:00002 (organoid)", ["UBERON", "EFO"]))
+        self.assertFalse(validate._validate_suffixed_curie("HsapDv:00002(organoid)", ["UBERON", "EFO"]))
+        self.assertFalse(validate._validate_suffixed_curie("HsapDv:00002", ["UBERON", "EFO"]))
+        self.assertFalse(validate._validate_suffixed_curie("EFO:00002 extra", ["UBERON", "EFO"]))
+        self.assertFalse(validate._validate_suffixed_curie("UBERON:ABCD", ["UBERON", "EFO"]))
+        self.assertFalse(validate._validate_suffixed_curie("Uberon:00002", ["UBERON", "EFO"]))
+        self.assertFalse(validate._validate_suffixed_curie("UBERON:", ["UBERON", "EFO"]))
+        self.assertFalse(validate._validate_suffixed_curie("UBERON", ["UBERON", "EFO"]))
+
+
+class TestColumnValidation(unittest.TestCase):
+
+    def test_validate_unique(self):
+        unique = pd.DataFrame([["abc", "def"], ["ghi", "jkl"], ["mnop", "qrs"]],
+                              index=["X", "Y", "Z"], columns=["col1", "col2"])
+        duped = pd.DataFrame([["abc", "def"], ["ghi", "qrs"], ["abc", "qrs"]],
+                             index=["X", "Y", "X"], columns=["col1", "col2"])
+
+        schema_def = {"unique": True}
+
+        errors = validate._validate_column(unique.index, "index", "unique_df", schema_def)
+        self.assertFalse(errors)
+
+        errors = validate._validate_column(duped.index, "index", "duped_df", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("is not unique", errors[0])
+
+        errors = validate._validate_column(unique["col1"], "col1", "unique_df", schema_def)
+        self.assertFalse(errors)
+
+        errors = validate._validate_column(duped["col1"], "col1", "duped_df", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("is not unique", errors[0])
+
+        schema_def = {"unique": False}
+        errors = validate._validate_column(duped["col1"], "col1", "duped_df", schema_def)
+        self.assertFalse(errors)
+
+    def test_validate_nullable(self):
+        non_null = pd.DataFrame([["abc", "def"], ["ghi", "jkl"], ["mnop", "qrs"]],
+                                index=["X", "Y", "Z"], columns=["col1", "col2"])
+        has_null = pd.DataFrame([["abc", "", None], ["ghi", "jkl", 1], ["mnop", "qrs", 2]],
+                                index=["X", "Y", "Z"], columns=["col1", "col2", "col3"])
+
+        schema_def = {"nullable": False}
+        errors = validate._validate_column(non_null["col1"], "col1", "nonnull_df", schema_def)
+        self.assertFalse(errors)
+        errors = validate._validate_column(has_null["col1"], "col1", "hasnull_df", schema_def)
+        self.assertFalse(errors)
+
+        errors = validate._validate_column(has_null["col2"], "col2", "hasnull_df", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("contains empty values", errors[0])
+        errors = validate._validate_column(has_null["col3"], "col3", "hasnull_df", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("contains empty values", errors[0])
+
+        schema_def = {"nullable": True}
+        errors = validate._validate_column(has_null["col2"], "col2", "hasnull_df", schema_def)
+        self.assertFalse(errors)
+
+    def test_human_readable(self):
+        hr_df = pd.DataFrame(
+            [["for you, a human", "UBERON:12345", "UBERON:1234 (thundercat)"],
+             ["hope you're well", "bit of lungs", "brain"]],
+            index=["ENSG00001", "ENSG00002"],
+            columns=["good", "curie", "suffixed_curie"])
+
+        schema_def = {"type": "human-readable string"}
+        errors = validate._validate_column(hr_df["good"], "good", "hr", schema_def)
+        self.assertFalse(errors)
+
+        errors = validate._validate_column(hr_df["curie"], "curie", "hr", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("non-human-readable", errors[0])
+
+        errors = validate._validate_column(hr_df["suffixed_curie"], "suffixed_curie", "hr", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("non-human-readable", errors[0])
+
+        errors = validate._validate_column(hr_df.index, "ensg", "hr", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("non-human-readable", errors[0])
+
+    def test_curie(self):
+
+        curie_df = pd.DataFrame(
+            [["EFO:00001", "HsapDv:00001 (cell culture)", "EFO:", "MONDO:0001 cell culture"],
+             ["UBERON:00002", "HsapDv:00002 (organoid)", "EFO:12345", "MONDO:0002 (baba yaga)"],
+             ["EFO:0000000005", "HsapDv:000004 (humanzee)", "EFO:000002", "MONDO:0004 (TMNT)"]],
+            index=["X", "Y", "Z"],
+            columns=["good", "good_suffix", "bad", "bad_suffix"])
+
+        # Good
+        schema_def = {"type": "curie", "prefixes": ["EFO", "UBERON"]}
+        errors = validate._validate_column(curie_df["good"], "good", "curie_df", schema_def)
+        self.assertFalse(errors)
+
+        # Good suffix
+        schema_def = {"type": "suffixed curie", "prefixes": ["HsapDv", "WHATEVER"]}
+        errors = validate._validate_column(curie_df["good_suffix"], "good_suffix", "curie_df", schema_def)
+        self.assertFalse(errors)
+
+        # Bad prefix
+        schema_def = {"type": "curie", "prefixes": ["EFO"]}
+        errors = validate._validate_column(curie_df["good"], "good", "curie_df", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("invalid ontology", errors[0])
+        self.assertIn("must be curies from one of these", errors[0])
+
+        # Bad curies
+        schema_def = {"type": "curie", "prefixes": ["EFO"]}
+        errors = validate._validate_column(curie_df["bad"], "bad", "curie_df", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("invalid ontology", errors[0])
+
+        # Bad suffixes
+        schema_def = {"type": "suffixed curie", "prefixes": ["EFO"]}
+        errors = validate._validate_column(curie_df["bad_suffix"], "bad_suffix", "curie_df", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("invalid ontology", errors[0])
+
+    def test_enum(self):
+        enum_df = pd.DataFrame(
+            [["abc", "ghi"],
+             ["def", "jkl"]],
+            index=["X", "Y"],
+            columns=["col1", "col2"])
+
+        # All match
+        schema_def = {"type": "string", "enum": ["abc", "def", "xyz"]}
+        errors = validate._validate_column(enum_df["col1"], "col1", "enum_df", schema_def)
+        self.assertFalse(errors)
+
+        # Missing value
+        schema_def = {"type": "string", "enum": ["abc", "xyz"]}
+        errors = validate._validate_column(enum_df["col1"], "col1", "enum_df", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("unpermitted values", errors[0])
+
+
+class TestDictValidations(unittest.TestCase):
+
+
+    def test_key_presence(self):
+
+        schema_def = {"keys": {"abc": None, "def": None}}
+
+        dict_ = {"abc": "123", "def": "456"}
+        errors = validate._validate_dict(dict_, "d", schema_def)
+        self.assertFalse(errors)
+
+        # Missing keys are bad
+        dict_ = {"abc": "123"}
+        errors = validate._validate_dict(dict_, "d", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("missing key", errors[0])
+
+        # Extra keys are okay
+        dict_ = {"abc": "123", "def": "456", "xyz": "789"}
+        errors = validate._validate_dict(dict_, "d", schema_def)
+        self.assertFalse(errors)
+
+        # Better not be empty come on
+        dict_ = {}
+        errors = validate._validate_dict(dict_, "d", schema_def)
+        self.assertEqual(len(errors), 2)
+
+    def test_nullable(self):
+
+        schema_def = {"keys": {"abc": {"type": "string", "nullable": False},
+                               "def": {"type": "string", "nullable": True}}}
+
+        dict_ = {"abc": "xyz", "def": ""}
+        errors = validate._validate_dict(dict_, "d", schema_def)
+        self.assertFalse(errors)
+
+        dict_ = {"abc": "", "def": ""}
+        errors = validate._validate_dict(dict_, "d", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("empty value", errors[0])
+
+    def test_recurse(self):
+
+        schema_def = {
+            "keys": {
+                "subdict": {
+                    "type": "dict",
+                    "keys": {
+                        "subdict_key1": None,
+                        "subdict_key2": None
+                    }
+                },
+                "ontology": {
+                    "type": "curie",
+                    "prefixes": ["ONTOLOGY"]
+                },
+                "blob": {
+                    "type": "stringified list of dicts"
+                }
+            }
+        }
+
+        dict_ = {
+            "subdict": {"subdict_key1": "any", "subdict_key2": "any"},
+            "ontology": "ONTOLOGY:123456",
+            "blob": json.dumps([{"abc": 123}, {"def": 456}])
+        }
+        errors = validate._validate_dict(dict_, "d", schema_def)
+        self.assertFalse(errors)
+
+        dict_ = {
+            "subdict": {"subdict_key1": "any"},
+            "ontology": "ONTOLOGY:123456",
+            "blob": json.dumps([{"abc": 123}, {"def": 456}])
+        }
+        errors = validate._validate_dict(dict_, "d", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("missing key", errors[0])
+
+        dict_ = {
+            "subdict": {"subdict_key1": "any", "subdict_key2": "any"},
+            "ontology": "oh no not an ontology term",
+            "blob": json.dumps([{"abc": 123}, {"def": 456}])
+        }
+        errors = validate._validate_dict(dict_, "d", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("invalid ontology", errors[0])
+
+        dict_ = {
+            "subdict": {"subdict_key1": "any", "subdict_key2": "any"},
+            "ontology": "ONTOLOGY:123456",
+            "blob": [{"abc": 123}, {"def": 456}]
+        }
+        errors = validate._validate_dict(dict_, "d", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("JSON-encoded list of dicts", errors[0])
+
+        # Multiple errors
+        dict_ = {
+            "subdict": {"subdict_key1": "any"},
+            "ontology": "oh no not an ontology term",
+            "blob": json.dumps([{"abc": 123}, {"def": 456}])
+        }
+        errors = validate._validate_dict(dict_, "d", schema_def)
+        self.assertEqual(len(errors), 2)
+
+
+class TestDataframeValidation(unittest.TestCase):
+
+    def test_column_presence(self):
+        df = pd.DataFrame(
+            [["abc", "EFO:123"],
+             ["def", "UBERON:456"]],
+            columns=["hr_string", "ontology"],
+            index=["X", "Y"]
+        )
+
+        schema_def = {
+            "columns": {
+                "hr_string": {"type": "human-readable string"},
+                "ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]}
+            }
+        }
+        errors = validate._validate_dataframe(df, "df", schema_def)
+        self.assertFalse(errors)
+
+        schema_def = {
+            "columns": {
+                "hr_string": {"type": "human-readable string"},
+                "another_hr_string": {"type": "human-readable string"},
+                "ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]}
+            }
+        }
+        errors = validate._validate_dataframe(df, "df", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("missing column", errors[0])
+
+        # Extra is okay
+        df = pd.DataFrame(
+            [["abc", "EFO:123", "extra"],
+             ["def", "UBERON:456", "extra"]],
+            columns=["hr_string", "ontology", "extra"],
+            index=["X", "Y"]
+        )
+        schema_def = {
+            "columns": {
+                "hr_string": {"type": "human-readable string"},
+                "ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]}
+            }
+        }
+        errors = validate._validate_dataframe(df, "df", schema_def)
+        self.assertFalse(errors)
+
+
+    def test_index(self):
+        df = pd.DataFrame(
+            [["abc", "123"],
+             ["def", "456"]],
+            columns=["col1", "col2"],
+            index=["ENSG0001", "ENSG0002"]
+        )
+
+        schema_def = {"index": {"unique": True}}
+        errors = validate._validate_dataframe(df, "df", schema_def)
+        self.assertFalse(errors)
+
+        schema_def = {"index": {"type": "human-readable string"}}
+        errors = validate._validate_dataframe(df, "df", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("non-human-readable", errors[0])
+
+        df = pd.DataFrame(
+            [["abc", "123"],
+             ["def", "456"]],
+            columns=["col1", "col2"],
+            index=["ENSG0001", "ENSG0001"]
+        )
+        schema_def = {"index": {"unique": True}}
+        errors = validate._validate_dataframe(df, "df", schema_def)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("is not unique", errors[0])
+
+    def test_recurse(self):
+
+        df = pd.DataFrame(
+            [["abc", "HsapDv:0001"],
+             ["EFO:123", "UBERON:456"]],
+            columns=["hr_string", "ontology"],
+            index=["X", "Y"]
+        )
+        schema_def = {
+            "columns": {
+                "hr_string": {"type": "human-readable string"},
+                "ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]}
+            }
+        }
+        errors = validate._validate_dataframe(df, "df", schema_def)
+        self.assertEqual(len(errors), 2)
+        self.assertEqual(len([e for e in errors if "non-human-readable" in e]), 1)
+        self.assertEqual(len([e for e in errors if "invalid ontology" in e]), 1)
+
+
+class TestGetSchema(unittest.TestCase):
+
+    def test_get_schema(self):
+        self.assertIsInstance(validate.get_schema_definition("1.0.0"), dict)
+
+        with self.assertRaises(ValueError):
+            validate.get_schema_definition("10.1.5")
+
+
+class TestValidate(unittest.TestCase):
+
+    def setUp(self):
+        self.source_h5ad_path = f"{PROJECT_ROOT}/backend/test/fixtures/pbmc3k-CSC-gz.h5ad"
+
+    def test_shallow(self):
+
+        adata = sc.read_h5ad(self.source_h5ad_path)
+        self.assertFalse(validate.validate_adata(adata, True))
+
+        adata.uns["version"] = {
+            "corpora_schema_version": "1.0.0",
+            "corpora_encoding_version": "0.1.0"
+        }
+        self.assertTrue(validate.validate_adata(adata, True))
+
+    def test_deep(self):
+        adata = sc.read_h5ad(self.source_h5ad_path)
+        self.assertFalse(validate.validate_adata(adata, False))
+
+        adata.uns["version"] = {
+            "corpora_schema_version": "1.0.0",
+            "corpora_encoding_version": "0.1.0"
+        }
+        self.assertFalse(validate.validate_adata(adata, False))

backend/test/test_czi_hosted/unit/converters/test_h5ad_data_file.py

@@ -0,0 +1,268 @@
+import json
+import unittest
+from glob import glob
+from os import remove, path
+from shutil import rmtree
+from uuid import uuid4
+
+import anndata
+import numpy as np
+from pandas import Series, DataFrame
+
+from backend.czi_hosted.common.corpora import CorporaConstants
+from backend.czi_hosted.converters.h5ad_data_file import H5ADDataFile
+
+from backend.test import PROJECT_ROOT
+
+
+class TestH5ADDataFile(unittest.TestCase):
+    def setUp(self):
+        self.sample_anndata = self._create_sample_anndata_dataset()
+        self.sample_h5ad_filename = self._write_anndata_to_file(self.sample_anndata)
+
+        self.sample_output_directory = path.splitext(self.sample_h5ad_filename)[0] + ".cxg"
+
+    def tearDown(self):
+        if self.sample_h5ad_filename:
+            remove(self.sample_h5ad_filename)
+
+        if path.isdir(self.sample_output_directory):
+            rmtree(self.sample_output_directory)
+
+    def test__create_h5ad_data_file__non_h5ad_raises_exception(self):
+        non_h5ad_filename = "my_fancy_dataset.csv"
+
+        with self.assertRaises(Exception) as exception_context:
+            H5ADDataFile(non_h5ad_filename)
+
+        self.assertIn("File must be an H5AD", str(exception_context.exception))
+
+    def test__create_h5ad_data_file__assert_warning_outputted_if_dataset_title_or_about_given(self):
+        with self.assertLogs(level="WARN") as logger:
+            H5ADDataFile(
+                self.sample_h5ad_filename,
+                dataset_title="My Awesome Dataset",
+                dataset_about="http://www.awesomedataset.com",
+                use_corpora_schema=False,
+            )
+
+        self.assertIn("will override any metadata that is extracted", logger.output[0])
+
+    def test__create_h5ad_data_file__reads_anndata_successfully(self):
+        h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False)
+
+        self.assertTrue((h5ad_file.anndata.X == self.sample_anndata.X).all())
+        self.assertEqual(
+            h5ad_file.anndata.obs.sort_index(inplace=True), self.sample_anndata.obs.sort_index(inplace=True)
+        )
+        self.assertEqual(
+            h5ad_file.anndata.var.sort_index(inplace=True), self.sample_anndata.var.sort_index(inplace=True)
+        )
+
+        for key in h5ad_file.anndata.obsm.keys():
+            self.assertIn(key, self.sample_anndata.obsm.keys())
+            self.assertTrue((h5ad_file.anndata.obsm[key] == self.sample_anndata.obsm[key]).all())
+
+        for key in self.sample_anndata.obsm.keys():
+            self.assertIn(key, h5ad_file.anndata.obsm.keys())
+            self.assertTrue((h5ad_file.anndata.obsm[key] == self.sample_anndata.obsm[key]).all())
+
+    def test__create_h5ad_data_file__copies_index_of_obs_and_var_to_column(self):
+        h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False)
+
+        # The automatic name chosen for the index should be "name_0"
+        self.assertNotIn("name_0", self.sample_anndata.obs.columns)
+        self.assertIn("name_0", h5ad_file.obs.columns)
+
+        self.assertNotIn("name_0", self.sample_anndata.var.columns)
+        self.assertIn("name_0", h5ad_file.var.columns)
+
+    def test__create_h5ad_data_file__no_copy_if_obs_and_var_index_names_specified(self):
+        h5ad_file = H5ADDataFile(
+            self.sample_h5ad_filename,
+            use_corpora_schema=False,
+            obs_index_column_name="float_category",
+            vars_index_column_name="int_category",
+        )
+
+        self.assertNotIn("name_0", h5ad_file.obs.columns)
+        self.assertNotIn("name_0", h5ad_file.var.columns)
+
+    def test__create_h5ad_data_file__obs_and_var_index_names_specified_not_unique_raises_exception(self):
+
+        with self.assertRaises(Exception) as exception_context:
+            H5ADDataFile(
+                self.sample_h5ad_filename,
+                use_corpora_schema=False,
+                obs_index_column_name="float_category",
+                vars_index_column_name="bool_category",
+            )
+
+        self.assertIn("Please prepare data to contain unique values", str(exception_context.exception))
+
+    def test__create_h5ad_data_file__obs_and_var_index_names_specified_doesnt_exist_raises_exception(self):
+        with self.assertRaises(Exception) as exception_context:
+            H5ADDataFile(
+                self.sample_h5ad_filename,
+                use_corpora_schema=False,
+                obs_index_column_name="unknown_category",
+                vars_index_column_name="i_dont_exist",
+            )
+
+        self.assertIn("does not exist", str(exception_context.exception))
+
+    def test__create_h5ad_data_file__extract_about_and_title_from_dataset(self):
+        h5ad_file = H5ADDataFile(self.sample_h5ad_filename)
+
+        self.assertEqual(h5ad_file.dataset_title, "random_link_name")
+        self.assertEqual(h5ad_file.dataset_about, "www.link.com")
+
+    def test__create_h5ad_data_file__inputted_dataset_title_and_about_overrides_extracted(self):
+        h5ad_file = H5ADDataFile(
+            self.sample_h5ad_filename, dataset_about="override_about", dataset_title="override_title"
+        )
+
+        self.assertEqual(h5ad_file.dataset_title, "override_title")
+        self.assertEqual(h5ad_file.dataset_about, "override_about")
+
+    def test__to_cxg__simple_anndata_no_corpora_and_sparse(self):
+        h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False)
+        h5ad_file.to_cxg(self.sample_output_directory, 100)
+
+        self._validate_expected_generated_list_of_tiledb_files()
+
+    def test__to_cxg__simple_anndata_with_corpora_and_sparse(self):
+        h5ad_file = H5ADDataFile(self.sample_h5ad_filename)
+        h5ad_file.to_cxg(self.sample_output_directory, 100)
+
+        self._validate_expected_generated_list_of_tiledb_files()
+
+    def test__to_cxg__simple_anndata_no_corpora_and_dense(self):
+        h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False)
+        h5ad_file.to_cxg(self.sample_output_directory, 0)
+
+        self._validate_expected_generated_list_of_tiledb_files()
+
+    def test__to_cxg__simple_anndata_with_corpora_and_dense(self):
+        h5ad_file = H5ADDataFile(self.sample_h5ad_filename)
+        h5ad_file.to_cxg(self.sample_output_directory, 0)
+
+        self._validate_expected_generated_list_of_tiledb_files()
+
+    def test__to_cxg__with_sparse_column_encoding(self):
+        anndata = self._create_sample_anndata_dataset()
+        anndata.X = np.ones((3, 4))
+        sparse_with_column_shift_filename = self._write_anndata_to_file(anndata)
+
+        h5ad_file = H5ADDataFile(sparse_with_column_shift_filename)
+        h5ad_file.to_cxg(self.sample_output_directory, 50)
+
+        self._validate_expected_generated_list_of_tiledb_files(has_column_encoding=True)
+
+        # Clean up
+        remove(sparse_with_column_shift_filename)
+
+    def _validate_expected_generated_list_of_tiledb_files(self, has_column_encoding=False):
+        (
+            expected_directories,
+            expected_obs_files,
+            expected_var_files,
+        ) = self._get_expected_generated_list_of_tiledb_files()
+
+        for directory in expected_directories:
+            self.assertTrue(path.isdir(directory))
+
+        for obs_file in expected_obs_files:
+            expected_location_of_obs_file = f"{self.sample_output_directory}/obs/*/{obs_file}"
+            self.assertTrue(path.isfile(glob(expected_location_of_obs_file)[0]))
+
+        for var_file in expected_var_files:
+            expected_location_of_var_file = f"{self.sample_output_directory}/var/*/{var_file}"
+            self.assertTrue(path.isfile(glob(expected_location_of_var_file)[0]))
+
+        if has_column_encoding:
+            self.assertTrue(path.isdir(f"{self.sample_output_directory}/X_col_shift"))
+
+    def _get_expected_generated_list_of_tiledb_files(self):
+
+        # Expected directories
+        metadata_directory = f"{self.sample_output_directory}/cxg_group_metadata"
+        main_x_directory = f"{self.sample_output_directory}/X"
+        overall_embedding_directory = f"{self.sample_output_directory}/emb"
+        specific_embedding_directory = f"{self.sample_output_directory}/emb/awesome_embedding"
+        obs_directory = f"{self.sample_output_directory}/obs"
+        var_directory = f"{self.sample_output_directory}/var"
+
+        # Obs files
+        obs_files = []
+        obs_files.append("name_0.tdb")
+        obs_files.append("name_0_var.tdb")
+        obs_files.append("string_category.tdb")
+        obs_files.append("string_category_var.tdb")
+        obs_files.append("float_category.tdb")
+
+        # Var files
+        var_files = []
+        var_files.append("name_0.tdb")
+        var_files.append("name_0_var.tdb")
+        var_files.append("bool_category.tdb")
+        var_files.append("int_category.tdb")
+
+        return (
+            [
+                metadata_directory,
+                main_x_directory,
+                overall_embedding_directory,
+                specific_embedding_directory,
+                obs_directory,
+                var_directory,
+            ],
+            obs_files,
+            var_files,
+        )
+
+    def _write_anndata_to_file(self, anndata):
+        temporary_filename = f"{PROJECT_ROOT}/backend/test/fixtures/{uuid4()}.h5ad"
+        anndata.write(temporary_filename)
+
+        return temporary_filename
+
+    def _create_sample_anndata_dataset(self):
+        # Create X
+        X = np.random.rand(3, 4)
+
+        # Create obs
+        random_string_category = Series(data=["a", "b", "b"], dtype="category")
+        random_float_category = Series(data=[3.2, 1.1, 2.2], dtype=np.float32)
+        obs_dataframe = DataFrame(
+            data={"string_category": random_string_category, "float_category": random_float_category}
+        )
+        obs = obs_dataframe
+
+        # Create vars
+        random_int_category = Series(data=[3, 1, 2, 4], dtype=np.int32)
+        random_bool_category = Series(data=[True, True, False, True], dtype=np.bool_)
+        var_dataframe = DataFrame(data={"int_category": random_int_category, "bool_category": random_bool_category})
+        var = var_dataframe
+
+        # Create embeddings
+        random_embedding = np.random.rand(3, 2)
+        obsm = {"X_awesome_embedding": random_embedding}
+
+        # Create uns corpora metadata
+        uns = {}
+        for metadata_field in CorporaConstants.REQUIRED_SIMPLE_METADATA_FIELDS:
+            uns[metadata_field] = "random"
+
+        for metadata_field in CorporaConstants.OPTIONAL_JSON_ENCODED_METADATA_FIELD:
+            uns[metadata_field] = json.dumps({"random_key": "random_value"})
+
+        # Need to carefully set the corpora schema versions in order for tests to pass.
+        uns["version"] = {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"}
+
+        # Set project links to be a dictionary
+        uns["project_links"] = json.dumps(
+            [{"link_name": "random_link_name", "link_url": "www.link.com", "link_type": "SUMMARY"}]
+        )
+
+        return anndata.AnnData(X=X, obs=obs, var=var, obsm=obsm, uns=uns)

backend/test/test_czi_hosted/unit/data_anndata/test_anndata_adaptor.py

@@ -0,0 +1,232 @@
+import json
+import sys
+import time
+import unittest
+
+import numpy as np
+import pandas as pd
+import pytest
+from parameterized import parameterized_class
+
+from backend.common.utils.data_locator import DataLocator
+from backend.common.errors import FilterError
+from backend.czi_hosted.data_anndata.anndata_adaptor import AnndataAdaptor
+from backend.test.test_czi_hosted.unit import app_config
+from backend.test import PROJECT_ROOT, FIXTURES_ROOT, decode_fbs
+from backend.test.fixtures.fixtures import pbmc3k_colors
+
+"""
+Test the anndata adaptor using the pbmc3k data set.
+"""
+
+
+@parameterized_class(
+    ("data_locator", "backed"),
+    [
+        (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", False),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", False),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", False),
+        (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", True),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", True),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", True),
+    ],
+)
+class AdaptorTest(unittest.TestCase):
+    def setUp(self):
+        config = app_config(self.data_locator, self.backed)
+        self.data = AnndataAdaptor(DataLocator(self.data_locator), config)
+
+    def test_init(self):
+        self.assertEqual(self.data.cell_count, 2638)
+        self.assertEqual(self.data.gene_count, 1838)
+        epsilon = 0.000_005
+        self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
+
+    def test_mandatory_annotations(self):
+        obs_index_col_name = self.data.get_schema()["annotations"]["obs"]["index"]
+        self.assertIn(obs_index_col_name, self.data.data.obs)
+        self.assertEqual(list(self.data.data.obs.index), list(range(2638)))
+        var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
+        self.assertIn(var_index_col_name, self.data.data.var)
+        self.assertEqual(list(self.data.data.var.index), list(range(1838)))
+
+    @pytest.mark.filterwarnings("ignore:Anndata data matrix")
+    def test_data_type(self):
+        # don't run the test on the more exotic data types, as they don't
+        # support the astype() interface (used by this test, but not underlying app)
+        if isinstance(self.data.data.X, np.ndarray):
+            self.data.data.X = self.data.data.X.astype("float64")
+            with self.assertWarns(UserWarning):
+                self.data._validate_data_types()
+
+    def test_filter_idx(self):
+        filter_ = {"filter": {"var": {"index": [1, 99, [200, 300]]}}}
+        fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 102)
+
+    def test_filter_complex(self):
+        filter_ = {
+            "filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 10}], "index": [1, 99, [200, 300]]}}
+        }
+        fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 91)
+
+    def test_obs_and_var_names(self):
+        self.assertEqual(np.sum(self.data.data.var[self.data.get_schema()["annotations"]["var"]["index"]].isna()), 0)
+        self.assertEqual(np.sum(self.data.data.obs[self.data.get_schema()["annotations"]["obs"]["index"]].isna()), 0)
+
+    def test_get_colors(self):
+        self.assertEqual(self.data.get_colors(), pbmc3k_colors)
+
+    def test_get_schema(self):
+        with open(f"{FIXTURES_ROOT}/schema.json") as fh:
+            schema = json.load(fh)
+            self.assertDictEqual(self.data.get_schema(), schema)
+
+    def test_schema_produces_error(self):
+        self.data.data.obs["time"] = pd.Series(
+            list([time.time() for i in range(self.data.cell_count)]), dtype="datetime64[ns]",
+        )
+        with pytest.raises(TypeError):
+            self.data._create_schema()
+
+    def test_layout(self):
+        fbs = self.data.layout_to_fbs_matrix(fields=None)
+        layout = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(layout["n_cols"], 6)
+        self.assertEqual(layout["n_rows"], 2638)
+
+        X = layout["columns"][0]
+        self.assertTrue((X >= 0).all() and (X <= 1).all())
+        Y = layout["columns"][1]
+        self.assertTrue((Y >= 0).all() and (Y <= 1).all())
+
+    def test_layout_fields(self):
+        """ X_pca, X_tsne, X_umap are available """
+        fbs = self.data.layout_to_fbs_matrix(["pca"])
+        layout = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(layout["n_cols"], 2)
+        self.assertEqual(layout["n_rows"], 2638)
+        self.assertCountEqual(layout["col_idx"], ["pca_0", "pca_1"])
+
+        fbs = self.data.layout_to_fbs_matrix(["tsne", "pca"])
+        layout = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(layout["n_cols"], 4)
+        self.assertEqual(layout["n_rows"], 2638)
+        self.assertCountEqual(layout["col_idx"], ["tsne_0", "tsne_1", "pca_0", "pca_1"])
+
+    def test_annotations(self):
+        fbs = self.data.annotation_to_fbs_matrix("obs")
+        annotations = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(annotations["n_rows"], 2638)
+        self.assertEqual(annotations["n_cols"], 5)
+        obs_index_col_name = self.data.get_schema()["annotations"]["obs"]["index"]
+        self.assertEqual(
+            annotations["col_idx"], [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"],
+        )
+
+        fbs = self.data.annotation_to_fbs_matrix("var")
+        annotations = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(annotations["n_rows"], 1838)
+        self.assertEqual(annotations["n_cols"], 2)
+        var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
+        self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells"])
+
+    def test_annotation_fields(self):
+        fbs = self.data.annotation_to_fbs_matrix("obs", ["n_genes", "n_counts"])
+        annotations = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(annotations["n_rows"], 2638)
+        self.assertEqual(annotations["n_cols"], 2)
+
+        var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
+        fbs = self.data.annotation_to_fbs_matrix("var", [var_index_col_name])
+        annotations = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(annotations["n_rows"], 1838)
+        self.assertEqual(annotations["n_cols"], 1)
+
+    def test_diffexp_topN(self):
+        f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
+        f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
+        result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
+        self.assertEqual(len(result), 10)
+        result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
+        self.assertEqual(len(result), 20)
+
+    def test_data_frame(self):
+        f1 = {"var": {"index": [[0, 10]]}}
+        fbs = self.data.data_frame_to_fbs_matrix(f1, "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 10)
+
+        with self.assertRaises(ValueError):
+            self.data.data_frame_to_fbs_matrix(None, "obs")
+
+    def test_filtered_data_frame(self):
+        filter_ = {"filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 100}]}}}
+        fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 1040)
+
+        filter_ = {"filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}}
+        with self.assertRaises(FilterError):
+            self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+
+    def test_data_named_gene(self):
+        var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
+        filter_ = {"filter": {"var": {"annotation_value": [{"name": var_index_col_name, "values": ["RER1"]}]}}}
+        fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 1)
+        self.assertEqual(data["col_idx"], [4])
+
+        filter_ = {
+            "filter": {"var": {"annotation_value": [{"name": var_index_col_name, "values": ["SPEN", "TYMP", "PRMT2"]}]}}
+        }
+        fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 3)
+        self.assertTrue((data["col_idx"] == [15, 1818, 1837]).all())
+
+    def test_compute_embedding(self):
+        filter = {"obs": {"index": [[0, 100]]}}
+
+        # Verify that we correctly handle the case where we lack scanpy
+        import unittest.mock
+
+        with unittest.mock.patch.dict(sys.modules, {"scanpy": None}):
+            with self.assertRaises(NotImplementedError):
+                self.data.compute_embedding("umap", filter)
+
+        # if we happen to have scanpy, test the full API, else punt
+        import importlib
+
+        scanpy_spec = importlib.util.find_spec("scanpy")
+        if scanpy_spec is None:
+            print("Skipping compute_embedding test as ScanPy not installed")
+            return
+
+        # this feature is unsupported in backed mode, and we expect an error
+        if self.data.data.isbacked:
+            with self.assertRaises(NotImplementedError):
+                self.data.compute_embedding("umap", filter)
+            return
+
+        schema = self.data.compute_embedding("umap", filter)
+
+        self.assertIsInstance(schema["name"], str)
+        name = schema["name"]
+        self.assertEqual(schema["type"], "float32")
+        self.assertEqual(schema["dims"], [f"{name}_0", f"{name}_1"])
+
+        emb = self.data.data.obsm[f"X_{name}"]
+        self.assertEqual(emb.shape, (2638, 2))
+        self.assertTrue(np.isfinite(emb[0:100]).all())
+        self.assertTrue(np.isnan(emb[100:]).all())

backend/test/test_czi_hosted/unit/data_anndata/test_anndata_adaptor_data_load.py

@@ -0,0 +1,81 @@
+import unittest
+import json
+
+from backend.common.utils.data_locator import DataLocator
+from backend.czi_hosted.data_anndata.anndata_adaptor import AnndataAdaptor
+from backend.czi_hosted.common.config.app_config import AppConfig
+from backend.test import PROJECT_ROOT
+
+
+class DataLoadAdaptorTest(unittest.TestCase):
+    """
+    Test file loading, including deferred loading/update.
+    """
+
+    def setUp(self):
+        self.data_file = DataLocator(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
+        config = AppConfig()
+        config.update_server_config(single_dataset__datapath=self.data_file.path)
+        config.update_server_config(app__flask_secret_key="secret")
+        config.complete_config()
+        self.data = AnndataAdaptor(self.data_file, config)
+
+    def test_delayed_load_data(self):
+        self.data._create_schema()
+        self.assertEqual(self.data.cell_count, 2638)
+        self.assertEqual(self.data.gene_count, 1838)
+        epsilon = 0.000_005
+        self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
+
+    def test_diffexp_topN(self):
+        f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
+        f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
+        result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
+        self.assertEqual(len(result), 10)
+        result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
+        self.assertEqual(len(result), 20)
+
+
+class DataLocatorAdaptorTest(unittest.TestCase):
+    """
+    Test various types of data locators we expect to consume
+    """
+
+    def get_basic_config(self):
+        config = AppConfig()
+        config.update_server_config(
+            single_dataset__obs_names=None, single_dataset__var_names=None,
+        )
+        config.update_server_config(app__flask_secret_key="secret")
+        config.update_default_dataset_config(
+            embeddings__names=["umap"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01,
+        )
+        return config
+
+    def stdAsserts(self, data):
+        """ run these each time we load the data """
+        self.assertIsNotNone(data)
+        self.assertEqual(data.cell_count, 2638)
+        self.assertEqual(data.gene_count, 1838)
+
+    def test_posix_file(self):
+        locator = DataLocator("../../example-dataset/pbmc3k.h5ad")
+        config = self.get_basic_config()
+        config.update_server_config(single_dataset__datapath=locator.path)
+        config.complete_config()
+        data = AnndataAdaptor(locator, config)
+        self.stdAsserts(data)
+
+    def test_url_https(self):
+        url = "https://raw.githubusercontent.com/chanzuckerberg/cellxgene/main/example-dataset/pbmc3k.h5ad"
+        locator = DataLocator(url)
+        config = self.get_basic_config()
+        data = AnndataAdaptor(locator, config)
+        self.stdAsserts(data)
+
+    def test_url_http(self):
+        url = "http://raw.githubusercontent.com/chanzuckerberg/cellxgene/main/example-dataset/pbmc3k.h5ad"
+        locator = DataLocator(url)
+        config = self.get_basic_config()
+        data = AnndataAdaptor(locator, config)
+        self.stdAsserts(data)

backend/test/test_czi_hosted/unit/data_anndata/test_nan_anndata_adaptor.py

@@ -0,0 +1,64 @@
+import math
+import unittest
+import warnings
+
+import pytest
+
+from backend.common.utils.data_locator import DataLocator
+from backend.common.errors import FilterError
+from backend.czi_hosted.data_anndata.anndata_adaptor import AnndataAdaptor
+from backend.test.test_czi_hosted.unit import app_config
+from backend.test import FIXTURES_ROOT, decode_fbs
+
+
+class NaNTest(unittest.TestCase):
+    def setUp(self):
+        self.data_locator = DataLocator(f"{FIXTURES_ROOT}/nan.h5ad")
+        self.config = app_config(self.data_locator.path)
+
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", category=UserWarning)
+            self.data = AnndataAdaptor(self.data_locator, self.config)
+            self.data._create_schema()
+
+    def test_load(self):
+        with self.assertLogs(level="WARN") as logger:
+            self.data = AnndataAdaptor(self.data_locator, self.config)
+            self.assertTrue(logger.output)
+
+    def test_init(self):
+        self.assertEqual(self.data.cell_count, 100)
+        self.assertEqual(self.data.gene_count, 100)
+        epsilon = 0.000_005
+        self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
+
+    def test_dataframe(self):
+        data_frame_var = decode_fbs.decode_matrix_FBS(self.data.data_frame_to_fbs_matrix(None, "var"))
+        self.assertIsNotNone(data_frame_var)
+        self.assertEqual(data_frame_var["n_rows"], 100)
+        self.assertEqual(data_frame_var["n_cols"], 100)
+        self.assertTrue(math.isnan(data_frame_var["columns"][3][3]))
+
+        with pytest.raises(FilterError):
+            self.data.data_frame_to_fbs_matrix("an erroneous filter", "var")
+        with pytest.raises(FilterError):
+            filter_ = {"filter": {"obs": {"index": [1, 99, [200, 300]]}}}
+            self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+
+    def test_dataframe_obs_not_implemented(self):
+        with self.assertRaises(ValueError) as cm:
+            decode_fbs.decode_matrix_FBS(self.data.data_frame_to_fbs_matrix(None, "obs"))
+        self.assertIsNotNone(cm.exception)
+
+    def test_annotation(self):
+        annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("obs"))
+        obs_index_col_name = self.data.schema["annotations"]["obs"]["index"]
+        self.assertEqual(annotations["col_idx"], [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"])
+        self.assertEqual(annotations["n_rows"], 100)
+        self.assertTrue(math.isnan(annotations["columns"][2][0]))
+
+        annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("var"))
+        var_index_col_name = self.data.schema["annotations"]["var"]["index"]
+        self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells", "var_with_nans"])
+        self.assertEqual(annotations["n_rows"], 100)
+        self.assertTrue(math.isnan(annotations["columns"][2][0]))

backend/test/test_czi_hosted/unit/data_common/fbs/test_matrix.py

@@ -0,0 +1,82 @@
+import unittest
+import pandas as pd
+import numpy as np
+from scipy import sparse
+
+from backend.test import decode_fbs
+from backend.common.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs
+
+
+class FbsTests(unittest.TestCase):
+    """Test Case for Matrix FBS data encode/decode """
+
+    def test_encode_boundary(self):
+        """ test various boundary checks """
+
+        # row indexing is unsupported
+        with self.assertRaises(ValueError):
+            encode_matrix_fbs(matrix=pd.DataFrame(), row_idx=[])
+
+        # matrix must be 2D
+        with self.assertRaises(ValueError):
+            encode_matrix_fbs(matrix=np.zeros((3, 2, 1)))
+        with self.assertRaises(ValueError):
+            encode_matrix_fbs(matrix=np.ones((10,)))
+
+    def fbs_checks(self, fbs, dims, expected_types, expected_column_idx):
+        d = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(d["n_rows"], dims[0])
+        self.assertEqual(d["n_cols"], dims[1])
+        self.assertIsNone(d["row_idx"])
+        self.assertEqual(len(d["columns"]), dims[1])
+        for i in range(0, len(d["columns"])):
+            self.assertEqual(len(d["columns"][i]), dims[0])
+            self.assertIsInstance(d["columns"][i], expected_types[i][0])
+            if expected_types[i][1] is not None:
+                self.assertEqual(d["columns"][i].dtype, expected_types[i][1])
+        if expected_column_idx is not None:
+            self.assertSetEqual(set(expected_column_idx), set(d["col_idx"]))
+
+    def test_encode_DataFrame(self):
+        df = pd.DataFrame(
+            data={
+                "a": np.zeros((10,), dtype=np.float32),
+                "b": np.ones((10,), dtype=np.int64),
+                "c": np.array([i for i in range(0, 10)], dtype=np.uint16),
+                "d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"),
+            }
+        )
+        expected_types = ((np.ndarray, np.float32), (np.ndarray, np.int32), (np.ndarray, np.uint32), (list, None))
+        fbs = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
+        self.fbs_checks(fbs, (10, 4), expected_types, ["a", "b", "c", "d"])
+
+    def test_encode_ndarray(self):
+        arr = np.zeros((3, 2), dtype=np.float32)
+        expected_types = ((np.ndarray, np.float32), (np.ndarray, np.float32), (np.ndarray, np.float32))
+        fbs = encode_matrix_fbs(matrix=arr, row_idx=None, col_idx=None)
+        self.fbs_checks(fbs, (3, 2), expected_types, None)
+
+    def test_encode_sparse(self):
+        csc = sparse.csc_matrix(np.array([[0, 1, 2], [3, 0, 4]]))
+        expected_types = ((np.ndarray, np.int32), (np.ndarray, np.int32), (np.ndarray, np.int32))
+        fbs = encode_matrix_fbs(matrix=csc, row_idx=None, col_idx=None)
+        self.fbs_checks(fbs, (2, 3), expected_types, None)
+
+    def test_roundtrip(self):
+        dfSrc = pd.DataFrame(
+            data={
+                "a": np.zeros((10,), dtype=np.float32),
+                "b": np.ones((10,), dtype=np.int64),
+                "c": np.array([i for i in range(0, 10)], dtype=np.uint16),
+                "d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"),
+            }
+        )
+        dfDst = decode_matrix_fbs(encode_matrix_fbs(matrix=dfSrc, col_idx=dfSrc.columns))
+        self.assertEqual(dfSrc.shape, dfDst.shape)
+        self.assertEqual(set(dfSrc.columns), set(dfDst.columns))
+        for c in dfSrc.columns:
+            self.assertTrue(c in dfDst.columns)
+            if isinstance(dfSrc[c], pd.Series):
+                self.assertTrue(np.all(dfSrc[c] == dfDst[c]))
+            else:
+                self.assertEqual(dfSrc[c], dfDst[c])

backend/test/test_czi_hosted/unit/data_common/test_matrix_loader.py

@@ -0,0 +1,126 @@
+import os
+import shutil
+import tempfile
+import time
+import unittest
+
+from backend.czi_hosted.common.config.app_config import AppConfig
+from backend.common.errors import DatasetAccessError
+from backend.czi_hosted.data_common.matrix_loader import MatrixDataCacheManager
+from backend.test import FIXTURES_ROOT
+
+
+class MatrixCacheTest(unittest.TestCase):
+    def setup(self):
+        pass
+
+    def make_temporay_datasets(self, dirname, num):
+        source = f"{FIXTURES_ROOT}/pbmc3k.cxg"
+        for i in range(num):
+            target = os.path.join(dirname, str(i) + ".cxg")
+            shutil.copytree(source, target)
+
+    def use_dataset(self, matrix_cache, dirname, app_config, dataset_index):
+        with matrix_cache.data_adaptor(None, os.path.join(dirname, str(dataset_index) + ".cxg"), app_config) as adaptor:
+            pass
+        return adaptor
+
+    def use_dataset_with_error(self, matrix_cache, dirname, app_config, dataset_index):
+        try:
+            with matrix_cache.data_adaptor(None, os.path.join(dirname, str(dataset_index) + ".cxg"), app_config):
+                raise DatasetAccessError("something bad happened")
+        except DatasetAccessError:
+            # the MatrixDataCacheManager rethrows the exception, so catch and ignore
+            pass
+
+    def get_datasets(self, matrix_cache, dirname):
+        datasets = matrix_cache.datasets
+        result = {}
+        for k, v in datasets.items():
+            # filter out the dirname and the .cxg from the name
+            newk = int(k[1][len(dirname) + 1 : -4])
+            result[newk] = v
+
+        return result
+
+    def check_datasets(self, matrix_cache, dirname, expected):
+        res = self.get_datasets(matrix_cache, dirname)
+        actual = res.keys()
+        self.assertSetEqual(set(actual), set(expected))
+
+    def test_basic(self):
+        with tempfile.TemporaryDirectory() as dirname:
+            self.make_temporay_datasets(dirname, 5)
+            app_config = AppConfig()
+            m = MatrixDataCacheManager(max_cached=3, timelimit_s=None)
+
+            # should have only dataset 0
+            self.use_dataset(m, dirname, app_config, 0)
+            self.check_datasets(m, dirname, [0])
+
+            # should have datasets 0, 1
+            self.use_dataset(m, dirname, app_config, 1)
+            self.check_datasets(m, dirname, [0, 1])
+
+            # should have datasets 0, 1, 2
+            self.use_dataset(m, dirname, app_config, 2)
+            self.check_datasets(m, dirname, [0, 1, 2])
+
+            # should have datasets 1, 2, 3
+            self.use_dataset(m, dirname, app_config, 3)
+            self.check_datasets(m, dirname, [1, 2, 3])
+
+            # use dataset 1, making is more recent than dataset 2
+            self.use_dataset(m, dirname, app_config, 1)
+            self.check_datasets(m, dirname, [1, 2, 3])
+
+            # use dataset 4, should have 1,3,4
+            self.use_dataset(m, dirname, app_config, 4)
+            self.check_datasets(m, dirname, [1, 3, 4])
+
+            # use dataset 4 a few more times, get the count to 3
+            self.use_dataset(m, dirname, app_config, 4)
+            self.use_dataset(m, dirname, app_config, 4)
+
+            datasets = self.get_datasets(m, dirname)
+            self.assertEqual(datasets[1].num_access, 2)
+            self.assertEqual(datasets[3].num_access, 1)
+            self.assertEqual(datasets[4].num_access, 3)
+
+    def test_timelimit(self):
+        with tempfile.TemporaryDirectory() as dirname:
+            self.make_temporay_datasets(dirname, 2)
+
+            app_config = AppConfig()
+            m = MatrixDataCacheManager(max_cached=3, timelimit_s=1)
+
+            adaptor = self.use_dataset(m, dirname, app_config, 0)
+            adaptor1 = self.use_dataset(m, dirname, app_config, 0)
+            self.assertTrue(adaptor is adaptor1)
+
+            # wait until the timelimit expires and check that there is a new adaptor
+            time.sleep(1.1)
+            adaptor2 = self.use_dataset(m, dirname, app_config, 0)
+            self.assertTrue(adaptor is not adaptor2)
+            self.check_datasets(m, dirname, [0])
+
+            # now load a different dataset and see if dataset 0 gets evicted
+            time.sleep(1.1)
+            self.use_dataset(m, dirname, app_config, 1)
+            self.check_datasets(m, dirname, [1])
+
+    def test_access_error(self):
+        with tempfile.TemporaryDirectory() as dirname:
+            self.make_temporay_datasets(dirname, 1)
+
+            app_config = AppConfig()
+            m = MatrixDataCacheManager(max_cached=3, timelimit_s=1)
+
+            # use the 0 datasets
+            self.use_dataset(m, dirname, app_config, 0)
+            self.check_datasets(m, dirname, [0])
+
+            # use the 0 datasets, but this time a DatasetAccessError is raised.
+            # verify that dataset is removed from the cache.
+            self.use_dataset_with_error(m, dirname, app_config, 0)
+            self.check_datasets(m, dirname, [])

backend/test/test_czi_hosted/unit/data_cxg/test_cxg_adaptor.py

@@ -0,0 +1,20 @@
+import unittest
+
+from backend.common.utils.data_locator import DataLocator
+from backend.czi_hosted.data_cxg.cxg_adaptor import CxgAdaptor
+from backend.test.test_czi_hosted.unit import app_config
+from backend.test import FIXTURES_ROOT
+from backend.test.fixtures.fixtures import pbmc3k_colors
+
+
+class TestCxgAdaptor(unittest.TestCase):
+    def test_get_colors(self):
+        data = self.get_data("pbmc3k.cxg")
+        self.assertDictEqual(data.get_colors(), pbmc3k_colors)
+        data = self.get_data("pbmc3k_v0.cxg")
+        self.assertDictEqual(data.get_colors(), dict())
+
+    def get_data(self, fixture):
+        data_locator = f"{FIXTURES_ROOT}/{fixture}"
+        config = app_config(data_locator)
+        return CxgAdaptor(DataLocator(data_locator), config)

backend/test/test_czi_hosted/unit/eb/test_eb.py

@@ -0,0 +1,81 @@
+import os
+import requests
+import subprocess
+import tempfile
+import time
+import unittest
+
+from contextlib import contextmanager
+from backend.czi_hosted.common.config.app_config import AppConfig
+from backend.test import PROJECT_ROOT, FIXTURES_ROOT
+
+
+@contextmanager
+def run_eb_app(tempdirname):
+    ps = subprocess.Popen(["python", "artifact.dir/application.py"], cwd=tempdirname)
+    server = "http://localhost:5000"
+    for _ in range(10):
+        try:
+            requests.get(f"{server}/health")
+            break
+        except requests.exceptions.ConnectionError:
+            time.sleep(1)
+
+    try:
+        yield server
+    finally:
+        try:
+            ps.terminate()
+        except ProcessLookupError:
+            pass
+
+
+class Elastic_Beanstalk_Test(unittest.TestCase):
+    def test_run(self):
+        tempdir = tempfile.TemporaryDirectory(dir=f"{PROJECT_ROOT}/backend/czi_hosted")
+        tempdirname = tempdir.name
+        config = AppConfig()
+        # test that eb works
+        config.update_server_config(multi_dataset__dataroot=f"{FIXTURES_ROOT}", app__flask_secret_key="open sesame")
+
+        config.complete_config()
+        config.write_config(f"{tempdirname}/config.yaml")
+        subprocess.check_call(f"git ls-files . | cpio -pdm {tempdirname}", cwd=f"{PROJECT_ROOT}/backend/czi_hosted/eb", shell=True)
+
+        subprocess.check_call(["make", "build"], cwd=tempdirname)
+        with run_eb_app(tempdirname) as server:
+            session = requests.Session()
+
+            r = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config")
+            data_config = r.json()
+            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+
+    def test_config(self):
+        check_config_script = os.path.join(PROJECT_ROOT, "backend", "czi_hosted", "eb", "check_config.py")
+        with tempfile.TemporaryDirectory() as tempdir:
+            configfile = os.path.join(tempdir, "config.yaml")
+            app_config = AppConfig()
+            app_config.update_server_config(multi_dataset__dataroot=f"{FIXTURES_ROOT}")
+            app_config.write_config(configfile)
+
+            command = ["python", check_config_script, configfile]
+
+            # test failure mode (flask_secret_key not set)
+            env = os.environ.copy()
+            env.pop("CXG_SECRET_KEY", None)
+            with self.assertRaises(subprocess.CalledProcessError) as exception_context:
+                subprocess.check_output(command, env=env)
+            output = str(exception_context.exception.stdout, "utf-8")
+            self.assertTrue(
+                output.startswith(
+                    "Error: Invalid type for attribute: app__flask_secret_key, expected type str, got NoneType"
+                )
+            )
+            self.assertEqual(exception_context.exception.returncode, 1)
+
+            # test passing case
+            env = os.environ.copy()
+            env["CXG_SECRET_KEY"] = "secret"
+            output = subprocess.check_output(command, env=env)
+            output = str(output, "utf-8")
+            self.assertTrue(output.startswith("PASS"))