Merge branch 'main' into colinmegill/geneset-prototype

.github/workflows/compatibility_tests.yml

@@ -29,7 +29,7 @@ jobs:
     strategy:
       matrix:
         python-version: [3.6, 3.7] # As of Oct 2020 Anndata is not compatible with 3.8
-        anndata-version: [0.7.0, 0.7.1, 0.7.2, 0.7.3, 0.7.4]
+        anndata-version: [0.7.0, 0.7.1, 0.7.2, 0.7.3, 0.7.4, 0.7.5]
         test-suite: [smoke-test, smoke-test-annotations]
     steps:
     - uses: actions/checkout@v2

client/package-lock.json

@@ -12250,9 +12250,9 @@
       "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
     },
     "ini": {
-      "version": "1.3.5",
-      "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz",
-      "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==",
+      "version": "1.3.7",
+      "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.7.tgz",
+      "integrity": "sha512-iKpRpXP+CrP2jyrxvg1kMUpXDyRUFDWurxbnVT1vQPx+Wz9uCYsMIqYuSBLV+PAaZG/d7kRLKRFc9oDMsH+mFQ==",
       "dev": true
     },
     "internal-slot": {
@@ -19857,9 +19857,9 @@
       }
     },
     "urijs": {
-      "version": "1.19.2",
-      "resolved": "https://registry.npmjs.org/urijs/-/urijs-1.19.2.tgz",
-      "integrity": "sha512-s/UIq9ap4JPZ7H1EB5ULo/aOUbWqfDi7FKzMC2Nz+0Si8GiT1rIEaprt8hy3Vy2Ex2aJPpOQv4P4DuOZ+K1c6w==",
+      "version": "1.19.5",
+      "resolved": "https://registry.npmjs.org/urijs/-/urijs-1.19.5.tgz",
+      "integrity": "sha512-48z9VGWwdCV5KfizHsE05DWS5fhK6gFlx5MjO7xu0Krc5FGPWzjlXEVV0nPMrdVuP7xmMHiPZ2HoYZwKOFTZOg==",
       "dev": true
     },
     "urix": {

client/src/components/infoDrawer/infoFormat.js

@@ -167,7 +167,7 @@ const renderLinks = (projectLinks, aboutURL) => {
 
 const InfoFormat = React.memo(
   ({ datasetTitle, singleValueCategories, aboutURL, dataPortalProps = {} }) => {
-    if (dataPortalProps.version?.["corpora_schema_version"] !== "1.0.0") {
+    if (["1.0.0", "1.1.0"].indexOf(dataPortalProps.version?.["corpora_schema_version"]) === -1) {
       dataPortalProps = {};
     }
     const {

client/src/components/leftSidebar/topLeftLogoAndTitle.js

@@ -13,7 +13,7 @@ const DATASET_TITLE_FONT_SIZE = 14;
 @connect((state) => {
   const { corpora_props: corporaProps } = state.config;
   const correctVersion =
-    corporaProps?.version?.["corpora_schema_version"] === "1.0.0";
+    ["1.0.0", "1.1.0"].indexOf(corporaProps?.version?.["corpora_schema_version"]) > -1;
   return {
     datasetTitle: state.config?.displayNames?.dataset ?? "",
     libraryVersions: state.config?.["library_versions"],

server/common/corpora.py

@@ -63,9 +63,9 @@ def corpora_get_props_from_anndata(adata):
             raise KeyError(f"missing Corpora schema field {key}")
         corpora_props[key] = adata.uns[key]
 
-    for key in CorporaConstants.REQUIRED_JSON_ENCODED_METADATA_FIELD:
+    for key in CorporaConstants.OPTIONAL_JSON_ENCODED_METADATA_FIELD:
         if key not in adata.uns:
-            raise KeyError(f"missing Corpora schema field {key}")
+            continue
         try:
             corpora_props[key] = json.loads(adata.uns[key])
         except json.JSONDecodeError:

server/common/utils/corpora_constants.py

@@ -5,12 +5,18 @@ class CorporaConstants(object):
         "layer_descriptions",
         "organism",
         "organism_ontology_term_id",
-        "project_name",
-        "project_description",
     ]
 
     # The Corpora specification requires some values encoded as JSON due to the inability of AnnData to store complex
     # types.
-    REQUIRED_JSON_ENCODED_METADATA_FIELD = ["contributors", "project_links"]
+    OPTIONAL_JSON_ENCODED_METADATA_FIELD = ["contributors", "project_links"]
 
-    OPTIONAL_SIMPLE_METADATA_FIELDS = ["preprint_doi", "publication_doi", "default_embedding", "default_field", "tags"]
+    OPTIONAL_SIMPLE_METADATA_FIELDS = [
+        "preprint_doi",
+        "publication_doi",
+        "default_embedding",
+        "default_field",
+        "tags",
+        "project_name",
+        "project_description",
+    ]

server/converters/schema/schema_definitions/1_1_0.yaml

@@ -0,0 +1,93 @@
+title: Corpora schema version 1.1.0
+type: anndata
+components:
+    uns:
+        type: dict
+        keys:
+            version:
+                type: dict
+                keys:
+                    corpora_schema_version: null
+                    corpora_encoding_version: null
+            title:
+                type: string
+            layer_descriptions:
+                type: dict
+                keys:
+                    X: null
+            organism:
+                type: string
+                nullable: false
+            organism_ontology_term_id:
+                type: curie
+                prefixes:
+                    - NCBITaxon
+    var:
+        type: dataframe
+        index:
+            type: human-readable string
+            unique: true
+    obs:
+        type: dataframe
+        index:
+            unique: true
+        columns:
+            tissue:
+                type: human-readable string
+                nullable: false
+            tissue_ontology_term_id:
+                type: suffixed curie
+                nullable: true
+                prefixes:
+                    - UBERON
+            assay:
+                type: human-readable string
+                nullable: false
+            assay_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - EFO
+            disease:
+                type: human-readable string
+                nullable: false
+            disease_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - MONDO
+                    - PATO
+            cell_type:
+                type: human-readable string
+                nullable: false
+            cell_type_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - CL
+                    - UBERON
+            sex:
+                type: string
+                enum:
+                    - male
+                    - female
+                    - mixed
+                    - unknown
+                    - other
+            ethnicity:
+                type: human-readable string
+                nullable: false
+            ethnicity_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - HANCESTRO
+            development_stage:
+                type: human-readable string
+                nullable: false
+            development_stage_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - HsapDv
+                    - EFO

server/requirements.txt

@@ -12,7 +12,7 @@ flatbuffers>=1.11.0
 flatten-dict>=0.2.0
 fsspec>=0.4.4,<0.8.0
 gunicorn>=20.0.4
-h5py<3.0.0 # h5py returns bytes instead of str, which breaks many assumptions
+h5py<3.0.0 # h5py>=3.0.0 had a breaking change; there is a fix in anndata>=0.7.5
 numba>=0.49.1
 numpy>=1.15.0
 packaging>=20.0

server/test/unit/common/test_corpora.py

@@ -76,6 +76,30 @@ class CorporaAPITest(unittest.TestCase):
         some_fields["project_links"] = json.loads(some_fields["project_links"])
         self.assertEqual(corpora_get_props_from_anndata(adata), some_fields)
 
+    def test_corpora_get_props_from_anndata_v110(self):
+        adata = self._get_h5ad()
+
+        if "version" in adata.uns:
+            del adata.uns["version"]
+        self.assertIsNone(corpora_get_props_from_anndata(adata))
+
+        # legit version, but missing required values
+        adata.uns["version"] = {"corpora_schema_version": "1.1.0", "corpora_encoding_version": "0.1.0"}
+        with self.assertRaises(KeyError):
+            corpora_get_props_from_anndata(adata)
+
+        # Metadata following schema 1.1.0, which removes some fields relative to 1.1.0
+        some_110_fields = {
+            "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"},
+            "title": "title",
+            "layer_descriptions": "layer_descriptions",
+            "organism": "organism",
+            "organism_ontology_term_id": "organism_ontology_term_id",
+        }
+        for k in some_110_fields:
+            adata.uns[k] = some_110_fields[k]
+        self.assertEqual(corpora_get_props_from_anndata(adata), some_110_fields)
+
     def _get_h5ad(self):
         return anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
 

server/test/unit/converters/test_h5ad_data_file.py

@@ -254,7 +254,7 @@ class TestH5ADDataFile(unittest.TestCase):
         for metadata_field in CorporaConstants.REQUIRED_SIMPLE_METADATA_FIELDS:
             uns[metadata_field] = "random"
 
-        for metadata_field in CorporaConstants.REQUIRED_JSON_ENCODED_METADATA_FIELD:
+        for metadata_field in CorporaConstants.OPTIONAL_JSON_ENCODED_METADATA_FIELD:
             uns[metadata_field] = json.dumps({"random_key": "random_value"})
 
         # Need to carefully set the corpora schema versions in order for tests to pass.