initial support for corpora schema conventions (#1676)

* initial support for corpora schema conventions

* remove debugging print

* add corpora util module

* tests

* lint

* PR review edits

* PR changes

* more PR changes

* more PR chnages

* PR fixes

* formatting

* PR updates

* lint

* PR review

server/test/test_anndata_adaptor_data_load.py

@@ -43,13 +43,10 @@ class DataLocatorAdaptorTest(unittest.TestCase):
     def get_basic_config(self):
         config = AppConfig()
         config.update_server_config(
-            single_dataset__obs_names=None,
-            single_dataset__var_names=None,
+            single_dataset__obs_names=None, single_dataset__var_names=None,
         )
         config.update_default_dataset_config(
-            embeddings__names=["umap"],
-            presentation__max_categories=100,
-            diffexp__lfc_cutoff=0.01,
+            embeddings__names=["umap"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01,
         )
         return config
 

server/test/test_corpora.py

@@ -0,0 +1,73 @@
+import unittest
+import anndata
+import json
+
+from server.common.corpora import (
+    corpora_get_versions_from_anndata,
+    corpora_is_version_supported,
+    corpora_get_props_from_anndata,
+)
+from server.test import PROJECT_ROOT
+
+
+class CorporaAPITest(unittest.TestCase):
+    def test_corpora_get_versions_from_anndata(self):
+        adata = self._get_h5ad()
+
+        if "version" in adata.uns:
+            del adata.uns["version"]
+        self.assertIsNone(corpora_get_versions_from_anndata(adata))
+
+        # something bogus
+        adata.uns["version"] = 99
+        self.assertIsNone(corpora_get_versions_from_anndata(adata))
+
+        # something legit
+        adata.uns["version"] = {"corpora_schema_version": "0.0.0", "corpora_encoding_version": "9.9.9"}
+        self.assertEqual(corpora_get_versions_from_anndata(adata), ["0.0.0", "9.9.9"])
+
+    def test_corpora_is_version_supported(self):
+        self.assertTrue(corpora_is_version_supported("1.0.0", "0.1.0"))
+        self.assertFalse(corpora_is_version_supported("0.0.0", "0.1.0"))
+        self.assertFalse(corpora_is_version_supported("1.0.0", "0.0.0"))
+
+    def test_corpora_get_props_from_anndata(self):
+        adata = self._get_h5ad()
+
+        if "version" in adata.uns:
+            del adata.uns["version"]
+        self.assertIsNone(corpora_get_props_from_anndata(adata))
+
+        # something bogus
+        adata.uns["version"] = 99
+        self.assertIsNone(corpora_get_props_from_anndata(adata))
+
+        # unsupported version, but missing required values
+        adata.uns["version"] = {"corpora_schema_version": "99.0.0", "corpora_encoding_version": "32.1.0"}
+        with self.assertRaises(ValueError):
+            corpora_get_props_from_anndata(adata)
+
+        # legit version, but missing required values
+        adata.uns["version"] = {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"}
+        with self.assertRaises(KeyError):
+            corpora_get_props_from_anndata(adata)
+
+        some_fields = {
+            "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"},
+            "title": "title",
+            "layer_descriptions": "layer_descriptions",
+            "organism": "organism",
+            "organism_ontology_term_id": "organism_ontology_term_id",
+            "project_name": "project_name",
+            "project_description": "project_description",
+            "contributors": json.dumps([{"contributors": "contributors"}]),
+            "project_links": json.dumps([{"link_name": "link_name", "link_url": "link_url", "link_type": "SUMMARY"}]),
+        }
+        for k in some_fields:
+            adata.uns[k] = some_fields[k]
+        some_fields["contributors"] = json.loads(some_fields["contributors"])
+        some_fields["project_links"] = json.loads(some_fields["project_links"])
+        self.assertEqual(corpora_get_props_from_anndata(adata), some_fields)
+
+    def _get_h5ad(self):
+        return anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")

server/test/test_cxgtool.py

@@ -4,7 +4,7 @@ import unittest
 import anndata
 
 from server.common.data_locator import DataLocator
-from server.converters.cxgtool import write_cxg
+from server.converters.cxgtool import write_cxg, create_cxg_group_metadata
 from server.data_cxg.cxg_adaptor import CxgAdaptor
 from server.test import PROJECT_ROOT, app_config, random_string
 from server.test.test_datasets.fixtures import pbmc3k_colors
@@ -33,6 +33,9 @@ class TestCxgAdaptor(unittest.TestCase):
         data_locator = f"/tmp/test_{rand_str}.cxg"
         self.fixtures.append(data_locator)
         source_h5ad = anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
-        write_cxg(adata=source_h5ad, container=data_locator, title="pbmc3k", **kwargs)
+        cxg_group_metadata = create_cxg_group_metadata(
+            adata=source_h5ad, basefname="pbmc3k.h5ad", title="pbmc3k", **kwargs
+        )
+        write_cxg(adata=source_h5ad, container=data_locator, cxg_group_metadata=cxg_group_metadata)
         config = app_config(data_locator)
         return CxgAdaptor(DataLocator(data_locator), config)

server/test/test_diffexp.py

@@ -3,7 +3,7 @@ from server.data_common.matrix_loader import MatrixDataLoader
 from server.test import PROJECT_ROOT, app_config
 import server.compute.diffexp_cxg as diffexp_cxg
 import server.compute.diffexp_generic as diffexp_generic
-from server.converters.cxgtool import write_cxg
+from server.converters.cxgtool import write_cxg, create_cxg_group_metadata
 from server.test.create_test_matrix import create_test_h5ad
 from server.data_common.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs
 import numpy as np
@@ -16,8 +16,7 @@ class DiffExpTest(unittest.TestCase):
     adaptor types and different algorithms."""
 
     def load_dataset(self, path, extra_server_config={}, extra_dataset_config={}):
-        config = app_config(path, extra_server_config=extra_server_config,
-                            extra_dataset_config=extra_dataset_config)
+        config = app_config(path, extra_server_config=extra_server_config, extra_dataset_config=extra_dataset_config)
         loader = MatrixDataLoader(path)
         adaptor = loader.open(config)
         return adaptor
@@ -105,13 +104,15 @@ class DiffExpTest(unittest.TestCase):
             adata = adaptor_anndata.data
 
             sparsename = os.path.join(dirname, "sparse.cxg")
-            write_cxg(adata=adata, container=sparsename, title="sparse", sparse_threshold=11)
+            cxg_group_metadata = create_cxg_group_metadata(adata=adata, basefname="sparse.h5ad", title="sparse",)
+            write_cxg(adata=adata, container=sparsename, cxg_group_metadata=cxg_group_metadata, sparse_threshold=11)
             adaptor_sparse = self.load_dataset(sparsename)
             assert adaptor_sparse.open_array("X").schema.sparse
             assert adaptor_sparse.has_array("X_col_shift") == apply_col_shift
 
             densename = os.path.join(dirname, "dense.cxg")
-            write_cxg(adata=adata, container=densename, title="dense", sparse_threshold=0)
+            cxg_group_metadata = create_cxg_group_metadata(adata=adata, basefname="dense.h5ad", title="dense",)
+            write_cxg(adata=adata, container=densename, cxg_group_metadata=cxg_group_metadata, sparse_threshold=0)
             adaptor_dense = self.load_dataset(densename)
             assert not adaptor_dense.open_array("X").schema.sparse
             assert not adaptor_dense.has_array("X_col_shift")