From 46d02b1987f3eb033d7356f324571ef84774067f Mon Sep 17 00:00:00 2001 From: Marcus Kinsella Date: Mon, 14 Dec 2020 15:20:29 -0800 Subject: [PATCH 1/5] Handle schema v1.1.0 (#2002) Correctly display datasets that follow schema version 1.1.0 --- .../src/components/infoDrawer/infoFormat.js | 2 +- .../leftSidebar/topLeftLogoAndTitle.js | 2 +- server/common/corpora.py | 4 +- server/common/utils/corpora_constants.py | 14 ++- .../schema/schema_definitions/1_1_0.yaml | 93 +++++++++++++++++++ server/test/unit/common/test_corpora.py | 24 +++++ .../unit/converters/test_h5ad_data_file.py | 2 +- 7 files changed, 132 insertions(+), 9 deletions(-) create mode 100644 server/converters/schema/schema_definitions/1_1_0.yaml diff --git a/client/src/components/infoDrawer/infoFormat.js b/client/src/components/infoDrawer/infoFormat.js index f68cacda..ee21eb8d 100644 --- a/client/src/components/infoDrawer/infoFormat.js +++ b/client/src/components/infoDrawer/infoFormat.js @@ -167,7 +167,7 @@ const renderLinks = (projectLinks, aboutURL) => { const InfoFormat = React.memo( ({ datasetTitle, singleValueCategories, aboutURL, dataPortalProps = {} }) => { - if (dataPortalProps.version?.["corpora_schema_version"] !== "1.0.0") { + if (["1.0.0", "1.1.0"].indexOf(dataPortalProps.version?.["corpora_schema_version"]) === -1) { dataPortalProps = {}; } const { diff --git a/client/src/components/leftSidebar/topLeftLogoAndTitle.js b/client/src/components/leftSidebar/topLeftLogoAndTitle.js index da46bc12..8243c291 100644 --- a/client/src/components/leftSidebar/topLeftLogoAndTitle.js +++ b/client/src/components/leftSidebar/topLeftLogoAndTitle.js @@ -13,7 +13,7 @@ const DATASET_TITLE_FONT_SIZE = 14; @connect((state) => { const { corpora_props: corporaProps } = state.config; const correctVersion = - corporaProps?.version?.["corpora_schema_version"] === "1.0.0"; + ["1.0.0", "1.1.0"].indexOf(corporaProps?.version?.["corpora_schema_version"]) > -1; return { datasetTitle: state.config?.displayNames?.dataset ?? "", libraryVersions: state.config?.["library_versions"], diff --git a/server/common/corpora.py b/server/common/corpora.py index d616f75b..137eb2a1 100644 --- a/server/common/corpora.py +++ b/server/common/corpora.py @@ -63,9 +63,9 @@ def corpora_get_props_from_anndata(adata): raise KeyError(f"missing Corpora schema field {key}") corpora_props[key] = adata.uns[key] - for key in CorporaConstants.REQUIRED_JSON_ENCODED_METADATA_FIELD: + for key in CorporaConstants.OPTIONAL_JSON_ENCODED_METADATA_FIELD: if key not in adata.uns: - raise KeyError(f"missing Corpora schema field {key}") + continue try: corpora_props[key] = json.loads(adata.uns[key]) except json.JSONDecodeError: diff --git a/server/common/utils/corpora_constants.py b/server/common/utils/corpora_constants.py index c0c98168..fa1641d5 100644 --- a/server/common/utils/corpora_constants.py +++ b/server/common/utils/corpora_constants.py @@ -5,12 +5,18 @@ class CorporaConstants(object): "layer_descriptions", "organism", "organism_ontology_term_id", - "project_name", - "project_description", ] # The Corpora specification requires some values encoded as JSON due to the inability of AnnData to store complex # types. - REQUIRED_JSON_ENCODED_METADATA_FIELD = ["contributors", "project_links"] + OPTIONAL_JSON_ENCODED_METADATA_FIELD = ["contributors", "project_links"] - OPTIONAL_SIMPLE_METADATA_FIELDS = ["preprint_doi", "publication_doi", "default_embedding", "default_field", "tags"] + OPTIONAL_SIMPLE_METADATA_FIELDS = [ + "preprint_doi", + "publication_doi", + "default_embedding", + "default_field", + "tags", + "project_name", + "project_description", + ] diff --git a/server/converters/schema/schema_definitions/1_1_0.yaml b/server/converters/schema/schema_definitions/1_1_0.yaml new file mode 100644 index 00000000..7531c72e --- /dev/null +++ b/server/converters/schema/schema_definitions/1_1_0.yaml @@ -0,0 +1,93 @@ +title: Corpora schema version 1.1.0 +type: anndata +components: + uns: + type: dict + keys: + version: + type: dict + keys: + corpora_schema_version: null + corpora_encoding_version: null + title: + type: string + layer_descriptions: + type: dict + keys: + X: null + organism: + type: string + nullable: false + organism_ontology_term_id: + type: curie + prefixes: + - NCBITaxon + var: + type: dataframe + index: + type: human-readable string + unique: true + obs: + type: dataframe + index: + unique: true + columns: + tissue: + type: human-readable string + nullable: false + tissue_ontology_term_id: + type: suffixed curie + nullable: true + prefixes: + - UBERON + assay: + type: human-readable string + nullable: false + assay_ontology_term_id: + type: curie + nullable: true + prefixes: + - EFO + disease: + type: human-readable string + nullable: false + disease_ontology_term_id: + type: curie + nullable: true + prefixes: + - MONDO + - PATO + cell_type: + type: human-readable string + nullable: false + cell_type_ontology_term_id: + type: curie + nullable: true + prefixes: + - CL + - UBERON + sex: + type: string + enum: + - male + - female + - mixed + - unknown + - other + ethnicity: + type: human-readable string + nullable: false + ethnicity_ontology_term_id: + type: curie + nullable: true + prefixes: + - HANCESTRO + development_stage: + type: human-readable string + nullable: false + development_stage_ontology_term_id: + type: curie + nullable: true + prefixes: + - HsapDv + - EFO diff --git a/server/test/unit/common/test_corpora.py b/server/test/unit/common/test_corpora.py index 5f26dae9..ee6e587b 100644 --- a/server/test/unit/common/test_corpora.py +++ b/server/test/unit/common/test_corpora.py @@ -76,6 +76,30 @@ class CorporaAPITest(unittest.TestCase): some_fields["project_links"] = json.loads(some_fields["project_links"]) self.assertEqual(corpora_get_props_from_anndata(adata), some_fields) + def test_corpora_get_props_from_anndata_v110(self): + adata = self._get_h5ad() + + if "version" in adata.uns: + del adata.uns["version"] + self.assertIsNone(corpora_get_props_from_anndata(adata)) + + # legit version, but missing required values + adata.uns["version"] = {"corpora_schema_version": "1.1.0", "corpora_encoding_version": "0.1.0"} + with self.assertRaises(KeyError): + corpora_get_props_from_anndata(adata) + + # Metadata following schema 1.1.0, which removes some fields relative to 1.1.0 + some_110_fields = { + "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"}, + "title": "title", + "layer_descriptions": "layer_descriptions", + "organism": "organism", + "organism_ontology_term_id": "organism_ontology_term_id", + } + for k in some_110_fields: + adata.uns[k] = some_110_fields[k] + self.assertEqual(corpora_get_props_from_anndata(adata), some_110_fields) + def _get_h5ad(self): return anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad") diff --git a/server/test/unit/converters/test_h5ad_data_file.py b/server/test/unit/converters/test_h5ad_data_file.py index 99ad40af..39412351 100644 --- a/server/test/unit/converters/test_h5ad_data_file.py +++ b/server/test/unit/converters/test_h5ad_data_file.py @@ -254,7 +254,7 @@ class TestH5ADDataFile(unittest.TestCase): for metadata_field in CorporaConstants.REQUIRED_SIMPLE_METADATA_FIELDS: uns[metadata_field] = "random" - for metadata_field in CorporaConstants.REQUIRED_JSON_ENCODED_METADATA_FIELD: + for metadata_field in CorporaConstants.OPTIONAL_JSON_ENCODED_METADATA_FIELD: uns[metadata_field] = json.dumps({"random_key": "random_value"}) # Need to carefully set the corpora schema versions in order for tests to pass. From f07e174a06d9eb44e0b92f0cc1e55aa777906c73 Mon Sep 17 00:00:00 2001 From: Trent Smith <1429913+Bento007@users.noreply.github.com> Date: Thu, 17 Dec 2020 16:29:23 -0800 Subject: [PATCH 2/5] Fix PR Template (#1999) --- .../pull request.md => PULL_REQUEST_TEMPLATE.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/PULL_REQUEST_TEMPLATE/pull request.md => PULL_REQUEST_TEMPLATE.md (100%) diff --git a/.github/PULL_REQUEST_TEMPLATE/pull request.md b/PULL_REQUEST_TEMPLATE.md similarity index 100% rename from .github/PULL_REQUEST_TEMPLATE/pull request.md rename to PULL_REQUEST_TEMPLATE.md From 7599af252dd370eb01e4adf97233c6e8dadaf587 Mon Sep 17 00:00:00 2001 From: bmccandless Date: Mon, 21 Dec 2020 12:38:07 -0800 Subject: [PATCH 3/5] Update compatibility test (#2009) Add comment in requirements about h5py and anndata. #1963 --- .github/workflows/compatibility_tests.yml | 2 +- server/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/compatibility_tests.yml b/.github/workflows/compatibility_tests.yml index fff70efa..ccb1075e 100644 --- a/.github/workflows/compatibility_tests.yml +++ b/.github/workflows/compatibility_tests.yml @@ -29,7 +29,7 @@ jobs: strategy: matrix: python-version: [3.6, 3.7] # As of Oct 2020 Anndata is not compatible with 3.8 - anndata-version: [0.7.0, 0.7.1, 0.7.2, 0.7.3, 0.7.4] + anndata-version: [0.7.0, 0.7.1, 0.7.2, 0.7.3, 0.7.4, 0.7.5] test-suite: [smoke-test, smoke-test-annotations] steps: - uses: actions/checkout@v2 diff --git a/server/requirements.txt b/server/requirements.txt index 18561771..1cd9a0e2 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -12,7 +12,7 @@ flatbuffers>=1.11.0 flatten-dict>=0.2.0 fsspec>=0.4.4,<0.8.0 gunicorn>=20.0.4 -h5py<3.0.0 # h5py returns bytes instead of str, which breaks many assumptions +h5py<3.0.0 # h5py>=3.0.0 had a breaking change; there is a fix in anndata>=0.7.5 numba>=0.49.1 numpy>=1.15.0 packaging>=20.0 From 96362b0b98892e5acf444abd456e353e49165f3d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Jan 2021 16:45:23 -0800 Subject: [PATCH 4/5] Bump urijs from 1.19.2 to 1.19.5 in /client (#2012) Bumps [urijs](https://github.com/medialize/URI.js) from 1.19.2 to 1.19.5. - [Release notes](https://github.com/medialize/URI.js/releases) - [Changelog](https://github.com/medialize/URI.js/blob/gh-pages/CHANGELOG.md) - [Commits](https://github.com/medialize/URI.js/compare/v1.19.2...v1.19.5) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- client/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/package-lock.json b/client/package-lock.json index f5a4075f..dfbf8c8b 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -19824,9 +19824,9 @@ } }, "urijs": { - "version": "1.19.2", - "resolved": "https://registry.npmjs.org/urijs/-/urijs-1.19.2.tgz", - "integrity": "sha512-s/UIq9ap4JPZ7H1EB5ULo/aOUbWqfDi7FKzMC2Nz+0Si8GiT1rIEaprt8hy3Vy2Ex2aJPpOQv4P4DuOZ+K1c6w==", + "version": "1.19.5", + "resolved": "https://registry.npmjs.org/urijs/-/urijs-1.19.5.tgz", + "integrity": "sha512-48z9VGWwdCV5KfizHsE05DWS5fhK6gFlx5MjO7xu0Krc5FGPWzjlXEVV0nPMrdVuP7xmMHiPZ2HoYZwKOFTZOg==", "dev": true }, "urix": { From db559467a2e28d4ae01356f3c9ca3b9478b67769 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Jan 2021 17:14:45 -0800 Subject: [PATCH 5/5] Bump ini from 1.3.5 to 1.3.7 in /client (#2000) Bumps [ini](https://github.com/isaacs/ini) from 1.3.5 to 1.3.7.
Commits
  • c74c8af 1.3.7
  • 024b8b5 update deps, add linting
  • 032fbaf Use Object.create(null) to avoid default object property hazards
  • 2da9039 1.3.6
  • cfea636 better git push script, before publish instead of after
  • 56d2805 do not allow invalid hazardous string as section name
  • See full diff in compare view
Maintainer changes

This version was pushed to npm by isaacs, a new releaser for ini since your current version.


[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=ini&package-manager=npm_and_yarn&previous-version=1.3.5&new-version=1.3.7)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) - `@dependabot use these labels` will set the current labels as the default for future PRs for this repo and language - `@dependabot use these reviewers` will set the current reviewers as the default for future PRs for this repo and language - `@dependabot use these assignees` will set the current assignees as the default for future PRs for this repo and language - `@dependabot use this milestone` will set the current milestone as the default for future PRs for this repo and language You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/chanzuckerberg/cellxgene/network/alerts).
--- client/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/package-lock.json b/client/package-lock.json index dfbf8c8b..b4965aa4 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -12250,9 +12250,9 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, "ini": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", - "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==", + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.7.tgz", + "integrity": "sha512-iKpRpXP+CrP2jyrxvg1kMUpXDyRUFDWurxbnVT1vQPx+Wz9uCYsMIqYuSBLV+PAaZG/d7kRLKRFc9oDMsH+mFQ==", "dev": true }, "internal-slot": {