add config option to handle multiple dataroots (#1531)

#1513

server/test/__init__.py

@@ -2,13 +2,19 @@ import random
 import shutil
 import string
 import tempfile
+import requests
+import time
+import os
+from subprocess import Popen
 from os import path, popen
+from contextlib import contextmanager
 
 import pandas as pd
 
 from server.common.annotations import AnnotationsLocalFile
 from server.common.data_locator import DataLocator
-from server.common.app_config import AppConfig
+from server.common.app_config import AppConfig, DEFAULT_SERVER_PORT
+from server.common.utils import find_available_port
 from server.data_common.fbs.matrix import encode_matrix_fbs
 from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataType
 
@@ -81,3 +87,54 @@ def app_config(data_locator, backed=False, extra={}):
 
 def random_string(n):
     return "".join(random.choice(string.ascii_letters) for _ in range(n))
+
+
+@contextmanager
+def test_server(command_line_args=[], app_config=None):
+    """A context to run the cellxgene server.
+    Command line arguments can be passed in, as well as an app_config.
+    This function is meant to be used like this, for example:
+
+    with test_server(...) as server:
+       r = requests.get(f"{server}/...")
+       // check r
+
+    where the server can be accessed within the context, and is terminated when
+    the context is exited.
+    The port is automatically set using find_available_port.
+    The verbose flag is automatically set to True.
+    If an app_config is provided, then this function writes a temporary
+    yaml config file, which this server will read and parse.
+    """
+
+    port = DEFAULT_SERVER_PORT
+    port = find_available_port("localhost", port)
+    command = ["cellxgene", "--no-upgrade-check", "launch", "--verbose", "--port=%d" % port] + command_line_args
+
+    tempdir = None
+    if app_config:
+        tempdir = tempfile.TemporaryDirectory()
+        config_file = os.path.join(tempdir.name, "config.yaml")
+        app_config.write_config(config_file)
+        command.extend(["-c", config_file])
+
+    server = f"http://localhost:{port}"
+    ps = Popen(command)
+
+    for _ in range(10):
+        try:
+            requests.get(f"{server}/health")
+            break
+        except requests.exceptions.ConnectionError:
+            time.sleep(1)
+
+    if tempdir:
+        tempdir.cleanup()
+
+    try:
+        yield server
+    finally:
+        try:
+            ps.terminate()
+        except ProcessLookupError:
+            pass

server/test/test_app_config.py

@@ -1,5 +1,8 @@
 import unittest
 from server.common.app_config import AppConfig
+from server.common.errors import ConfigurationError
+from server.test import PROJECT_ROOT, test_server
+import requests
 
 # NOTE, there are more tests that should be written for AppConfig.
 # this is just a start.
@@ -26,3 +29,43 @@ class AppConfigTest(unittest.TestCase):
         c.update(server__scripts=("a", "b"), server__inline_scripts=["c", "d"])
         v = c.changes_from_default()
         self.assertCountEqual(v, [("server__scripts", ["a", "b"], []), ("server__inline_scripts", ["c", "d"], [])])
+
+    def test_multi_dataset(self):
+
+        c = AppConfig()
+        # test for illegal url_dataroots
+        for illegal in ("a/b", "../b", "!$*", "\\n", "", "(bad)"):
+            c.update(multi_dataset__dataroot={illegal: f"{PROJECT_ROOT}/example-dataset"})
+            with self.assertRaises(ConfigurationError):
+                c.complete_config()
+
+        # test for legal url_dataroots
+        for legal in (
+            "d",
+            "this.is-okay_",
+        ):
+            c.update(multi_dataset__dataroot={legal: f"{PROJECT_ROOT}/example-dataset"})
+            c.complete_config()
+
+        # test that multi dataroots work end to end
+        c.update(
+            multi_dataset__dataroot=dict(
+                set1=f"{PROJECT_ROOT}/example-dataset",
+                set2=f"{PROJECT_ROOT}/server/test/test_datasets"
+            )
+        )
+        c.complete_config()
+
+        with test_server(app_config=c) as server:
+            session = requests.Session()
+
+            r = session.get(f"{server}/set1/pbmc3k.h5ad/api/v0.2/config")
+            data_config = r.json()
+            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+
+            r = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config")
+            data_config = r.json()
+            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+
+            r = session.get(f"{server}/health")
+            assert r.json()["status"] == "pass"