Handle schema v1.1.0 (#2002)

Correctly display datasets that follow schema version 1.1.0

client/src/components/infoDrawer/infoFormat.js

@@ -167,7 +167,7 @@ const renderLinks = (projectLinks, aboutURL) => {
 
 const InfoFormat = React.memo(
   ({ datasetTitle, singleValueCategories, aboutURL, dataPortalProps = {} }) => {
-    if (dataPortalProps.version?.["corpora_schema_version"] !== "1.0.0") {
+    if (["1.0.0", "1.1.0"].indexOf(dataPortalProps.version?.["corpora_schema_version"]) === -1) {
       dataPortalProps = {};
     }
     const {

client/src/components/leftSidebar/topLeftLogoAndTitle.js

@@ -13,7 +13,7 @@ const DATASET_TITLE_FONT_SIZE = 14;
 @connect((state) => {
   const { corpora_props: corporaProps } = state.config;
   const correctVersion =
-    corporaProps?.version?.["corpora_schema_version"] === "1.0.0";
+    ["1.0.0", "1.1.0"].indexOf(corporaProps?.version?.["corpora_schema_version"]) > -1;
   return {
     datasetTitle: state.config?.displayNames?.dataset ?? "",
     libraryVersions: state.config?.["library_versions"],

server/common/corpora.py

@@ -63,9 +63,9 @@ def corpora_get_props_from_anndata(adata):
             raise KeyError(f"missing Corpora schema field {key}")
         corpora_props[key] = adata.uns[key]
 
-    for key in CorporaConstants.REQUIRED_JSON_ENCODED_METADATA_FIELD:
+    for key in CorporaConstants.OPTIONAL_JSON_ENCODED_METADATA_FIELD:
         if key not in adata.uns:
-            raise KeyError(f"missing Corpora schema field {key}")
+            continue
         try:
             corpora_props[key] = json.loads(adata.uns[key])
         except json.JSONDecodeError:

server/common/utils/corpora_constants.py

@@ -5,12 +5,18 @@ class CorporaConstants(object):
         "layer_descriptions",
         "organism",
         "organism_ontology_term_id",
-        "project_name",
-        "project_description",
     ]
 
     # The Corpora specification requires some values encoded as JSON due to the inability of AnnData to store complex
     # types.
-    REQUIRED_JSON_ENCODED_METADATA_FIELD = ["contributors", "project_links"]
+    OPTIONAL_JSON_ENCODED_METADATA_FIELD = ["contributors", "project_links"]
 
-    OPTIONAL_SIMPLE_METADATA_FIELDS = ["preprint_doi", "publication_doi", "default_embedding", "default_field", "tags"]
+    OPTIONAL_SIMPLE_METADATA_FIELDS = [
+        "preprint_doi",
+        "publication_doi",
+        "default_embedding",
+        "default_field",
+        "tags",
+        "project_name",
+        "project_description",
+    ]

server/converters/schema/schema_definitions/1_1_0.yaml

@@ -0,0 +1,93 @@
+title: Corpora schema version 1.1.0
+type: anndata
+components:
+    uns:
+        type: dict
+        keys:
+            version:
+                type: dict
+                keys:
+                    corpora_schema_version: null
+                    corpora_encoding_version: null
+            title:
+                type: string
+            layer_descriptions:
+                type: dict
+                keys:
+                    X: null
+            organism:
+                type: string
+                nullable: false
+            organism_ontology_term_id:
+                type: curie
+                prefixes:
+                    - NCBITaxon
+    var:
+        type: dataframe
+        index:
+            type: human-readable string
+            unique: true
+    obs:
+        type: dataframe
+        index:
+            unique: true
+        columns:
+            tissue:
+                type: human-readable string
+                nullable: false
+            tissue_ontology_term_id:
+                type: suffixed curie
+                nullable: true
+                prefixes:
+                    - UBERON
+            assay:
+                type: human-readable string
+                nullable: false
+            assay_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - EFO
+            disease:
+                type: human-readable string
+                nullable: false
+            disease_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - MONDO
+                    - PATO
+            cell_type:
+                type: human-readable string
+                nullable: false
+            cell_type_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - CL
+                    - UBERON
+            sex:
+                type: string
+                enum:
+                    - male
+                    - female
+                    - mixed
+                    - unknown
+                    - other
+            ethnicity:
+                type: human-readable string
+                nullable: false
+            ethnicity_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - HANCESTRO
+            development_stage:
+                type: human-readable string
+                nullable: false
+            development_stage_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - HsapDv
+                    - EFO

server/test/unit/common/test_corpora.py

@@ -76,6 +76,30 @@ class CorporaAPITest(unittest.TestCase):
         some_fields["project_links"] = json.loads(some_fields["project_links"])
         self.assertEqual(corpora_get_props_from_anndata(adata), some_fields)
 
+    def test_corpora_get_props_from_anndata_v110(self):
+        adata = self._get_h5ad()
+
+        if "version" in adata.uns:
+            del adata.uns["version"]
+        self.assertIsNone(corpora_get_props_from_anndata(adata))
+
+        # legit version, but missing required values
+        adata.uns["version"] = {"corpora_schema_version": "1.1.0", "corpora_encoding_version": "0.1.0"}
+        with self.assertRaises(KeyError):
+            corpora_get_props_from_anndata(adata)
+
+        # Metadata following schema 1.1.0, which removes some fields relative to 1.1.0
+        some_110_fields = {
+            "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"},
+            "title": "title",
+            "layer_descriptions": "layer_descriptions",
+            "organism": "organism",
+            "organism_ontology_term_id": "organism_ontology_term_id",
+        }
+        for k in some_110_fields:
+            adata.uns[k] = some_110_fields[k]
+        self.assertEqual(corpora_get_props_from_anndata(adata), some_110_fields)
+
     def _get_h5ad(self):
         return anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
 

server/test/unit/converters/test_h5ad_data_file.py

@@ -254,7 +254,7 @@ class TestH5ADDataFile(unittest.TestCase):
         for metadata_field in CorporaConstants.REQUIRED_SIMPLE_METADATA_FIELDS:
             uns[metadata_field] = "random"
 
-        for metadata_field in CorporaConstants.REQUIRED_JSON_ENCODED_METADATA_FIELD:
+        for metadata_field in CorporaConstants.OPTIONAL_JSON_ENCODED_METADATA_FIELD:
             uns[metadata_field] = json.dumps({"random_key": "random_value"})
 
         # Need to carefully set the corpora schema versions in order for tests to pass.