Introduce a config file to cellxgene (#1264)

* Introduce a config file to cellxgene

The config file format is in yaml.  The default config is located
in server/common/default_config.py.  A user may create a yaml file
that contains a subset of these fields.  It can be used during cellxgene
launch, or for hosted cellxgene.

The code has also been refactored.  Much of the logic to check arguments
has moved from launch to app config.

It is now possible to set the tiledb context parameters using the config
file.  Other feature will soon be handled in a similar way.

server/test/__init__.py

@@ -6,6 +6,7 @@ import pandas as pd
 
 from server.common.annotations import AnnotationsLocalFile
 from server.common.data_locator import DataLocator
+from server.common.app_config import AppConfig
 from server.data_common.fbs.matrix import encode_matrix_fbs
 from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataType
 
@@ -16,18 +17,22 @@ def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False):
     if annotations_fixture:
         shutil.copyfile(f"test/test_datasets/pbmc3k-annotations.csv", annotations_file)
     args = {
-        "layout": ["umap"],
-        "max_category_items": 100,
-        "obs_names": None,
-        "var_names": None,
-        "diffexp_lfc_cutoff": 0.01,
+        "embeddings__names": ["umap"],
+        "presentation__max_categories": 100,
+        "single_dataset__obs_names": None,
+        "single_dataset__var_names": None,
+        "diffexp__lfc_cutoff": 0.01,
     }
     fname = {
         MatrixDataType.H5AD: "../example-dataset/pbmc3k.h5ad",
         MatrixDataType.CXG: "test/test_datasets/pbmc3k.cxg",
     }[ext]
     data_locator = DataLocator(fname)
-    data = MatrixDataLoader(data_locator.abspath()).open(args)
+    config = AppConfig()
+    config.update(**args)
+    config.update(single_dataset__datapath=data_locator.path)
+    config.complete_config()
+    data = MatrixDataLoader(data_locator.abspath()).open(config)
     annotations = AnnotationsLocalFile(None, annotations_file)
     return data, tmp_dir, annotations
 

server/test/test_anndata_adaptor.py

@@ -13,6 +13,7 @@ import pandas as pd
 from server.data_anndata.anndata_adaptor import AnndataAdaptor
 from server.common.errors import FilterError
 from server.common.data_locator import DataLocator
+from server.common.app_config import AppConfig
 
 """
 Test the anndata adaptor using the pbmc3k data set.
@@ -33,14 +34,18 @@ Test the anndata adaptor using the pbmc3k data set.
 class AdaptorTest(unittest.TestCase):
     def setUp(self):
         args = {
-            "layout": ["umap"],
-            "max_category_items": 100,
-            "obs_names": None,
-            "var_names": None,
-            "diffexp_lfc_cutoff": 0.01,
-            "anndata_backed": self.backed,
+            "embeddings__names": ["umap"],
+            "presentation__max_categories": 100,
+            "single_dataset__obs_names": None,
+            "single_dataset__var_names": None,
+            "diffexp__lfc_cutoff": 0.01,
+            "adaptor__anndata_adaptor__backed": self.backed,
+            "single_dataset__datapath" : self.data_locator
         }
-        self.data = AnndataAdaptor(DataLocator(self.data_locator), args)
+        config = AppConfig()
+        config.update(**args)
+        config.complete_config()
+        self.data = AnndataAdaptor(DataLocator(self.data_locator), config)
 
     def test_init(self):
         self.assertEqual(self.data.cell_count, 2638)
@@ -88,7 +93,7 @@ class AdaptorTest(unittest.TestCase):
     def test_get_schema(self):
         with open(path.join(path.dirname(__file__), "schema.json")) as fh:
             schema = json.load(fh)
-            self.assertEqual(self.data.get_schema(), schema)
+            self.assertDictEqual(self.data.get_schema(), schema)
 
     def test_schema_produces_error(self):
         self.data.data.obs["time"] = pd.Series(
@@ -109,9 +114,9 @@ class AdaptorTest(unittest.TestCase):
             self.assertEqual(len(feature), 1)
 
         check_feature("POST", "/cluster/", False)
-        check_feature("POST", "/diffexp/", not self.data.config.disable_diffexp)
+        check_feature("POST", "/diffexp/", self.data.config.diffexp__enable)
         check_feature("GET", "/layout/obs", True)
-        check_feature("PUT", "/layout/obs", self.data.config.enable_reembedding)
+        check_feature("PUT", "/layout/obs", self.data.config.embeddings__enable_reembedding)
         check_feature("PUT", "/annotations/obs", False)
 
     def test_layout(self):

server/test/test_anndata_adaptor_data_load.py

@@ -3,6 +3,7 @@ import json
 
 from server.data_anndata.anndata_adaptor import AnndataAdaptor
 from server.common.data_locator import DataLocator
+from server.common.app_config import AppConfig
 
 
 class DataLoadAdaptorTest(unittest.TestCase):
@@ -12,7 +13,10 @@ class DataLoadAdaptorTest(unittest.TestCase):
 
     def setUp(self):
         self.data_file = DataLocator("../example-dataset/pbmc3k.h5ad")
-        self.data = AnndataAdaptor(self.data_file)
+        config = AppConfig()
+        config.update(single_dataset__datapath=self.data_file.path)
+        config.complete_config()
+        self.data = AnndataAdaptor(self.data_file, config)
 
     def test_delayed_load_data(self):
         self.data._create_schema()
@@ -37,11 +41,11 @@ class DataLocatorAdaptorTest(unittest.TestCase):
 
     def setUp(self):
         self.args = {
-            "layout": ["umap"],
-            "max_category_items": 100,
-            "obs_names": None,
-            "var_names": None,
-            "diffexp_lfc_cutoff": 0.01,
+            "embeddings__names": ["umap"],
+            "presentation__max_categories": 100,
+            "single_dataset__obs_names": None,
+            "single_dataset__var_names": None,
+            "diffexp__lfc_cutoff": 0.01,
         }
 
     def stdAsserts(self, data):
@@ -52,17 +56,25 @@ class DataLocatorAdaptorTest(unittest.TestCase):
 
     def test_posix_file(self):
         locator = DataLocator("../example-dataset/pbmc3k.h5ad")
-        data = AnndataAdaptor(locator, self.args)
+        config = AppConfig()
+        config.update(**self.args)
+        config.update(single_dataset__datapath=locator.path)
+        config.complete_config()
+        data = AnndataAdaptor(locator, config)
         self.stdAsserts(data)
 
     def test_url_https(self):
         url = "https://raw.githubusercontent.com/chanzuckerberg/cellxgene/master/example-dataset/pbmc3k.h5ad"
         locator = DataLocator(url)
-        data = AnndataAdaptor(locator, self.args)
+        config = AppConfig()
+        config.update(**self.args)
+        data = AnndataAdaptor(locator, config)
         self.stdAsserts(data)
 
     def test_url_http(self):
         url = "http://raw.githubusercontent.com/chanzuckerberg/cellxgene/master/example-dataset/pbmc3k.h5ad"
         locator = DataLocator(url)
-        data = AnndataAdaptor(locator, self.args)
+        config = AppConfig()
+        config.update(**self.args)
+        data = AnndataAdaptor(locator, config)
         self.stdAsserts(data)

server/test/test_nan_anndata_adaptor.py

@@ -8,25 +8,37 @@ import server.test.decode_fbs as decode_fbs
 from server.data_anndata.anndata_adaptor import AnndataAdaptor
 from server.common.errors import FilterError
 from server.common.data_locator import DataLocator
+from server.common.app_config import AppConfig
 
 
 class NaNTest(unittest.TestCase):
     def setUp(self):
         self.args = {
-            "layout": ["umap"],
-            "max_category_items": 100,
-            "obs_names": None,
-            "var_names": None,
-            "diffexp_lfc_cutoff": 0.01,
+            "embeddings__names": ["umap"],
+            "presentation__max_categories": 100,
+            "single_dataset__obs_names": None,
+            "single_dataset__var_names": None,
+            "diffexp__lfc_cutoff": 0.01,
         }
+        config = AppConfig()
+        config.update(**self.args)
+        locator = DataLocator("test/test_datasets/nan.h5ad")
+        config.update(single_dataset__datapath=locator.path)
+        config.complete_config()
+
         with warnings.catch_warnings():
             warnings.simplefilter("ignore", category=UserWarning)
-            self.data = AnndataAdaptor(DataLocator("test/test_datasets/nan.h5ad"), self.args)
+            self.data = AnndataAdaptor(locator, config)
             self.data._create_schema()
 
     def test_load(self):
         with self.assertWarns(UserWarning):
-            AnndataAdaptor(DataLocator("test/test_datasets/nan.h5ad"), self.args)
+            config = AppConfig()
+            config.update(**self.args)
+            locator = DataLocator("test/test_datasets/nan.h5ad")
+            config.update(single_dataset__datapath=locator.path)
+            config.complete_config()
+            self.data = AnndataAdaptor(locator, config)
 
     def test_init(self):
         self.assertEqual(self.data.cell_count, 100)

server/test/test_writable_annotation.py

@@ -149,7 +149,7 @@ class WritableAnnotationTest(unittest.TestCase):
             self.assertEqual(len(feature), 1)
 
         check_feature("POST", "/cluster/", False)
-        check_feature("POST", "/diffexp/", not self.data.config.disable_diffexp)
+        check_feature("POST", "/diffexp/", self.data.config.diffexp__enable)
         check_feature("GET", "/layout/obs", True)
-        check_feature("PUT", "/layout/obs", self.data.config.enable_reembedding)
+        check_feature("PUT", "/layout/obs", self.data.config.embeddings__enable_reembedding)
         check_feature("PUT", "/annotations/obs", True)