Add version bump

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.16.7
+current_version = 0.16.5
 commit = True
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?:-(?P<prerel>rc)\.(?P<prerelversion>\d+))?
 serialize = 

.github/workflows/codeql-analysis.yml

@@ -1,67 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
-name: "CodeQL Scan"
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [ main ]
-  schedule:
-    - cron: '0 8 * * *'
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: [ 'javascript', 'python' ]
-        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
-        # Learn more:
-        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v2
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
-      with:
-        languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-        # queries: ./path/to/local/query, your-org/your-repo/queries@main
-
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
-
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 https://git.io/JvXDl
-
-    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-    #    and modify them (or add more) to build your code if your project
-    #    uses a compiled language
-
-    #- run: |
-    #   make bootstrap
-    #   make release
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1

.github/workflows/compatibility_tests.yml

@@ -28,8 +28,8 @@ jobs:
     continue-on-error: true
     strategy:
       matrix:
-        python-version: [3.6, 3.7] # As of Oct 2020 Anndata is not compatible with 3.8
-        anndata-version: [0.7.0, 0.7.1, 0.7.2, 0.7.3, 0.7.4, 0.7.5]
+        python-version: [3.6, 3.7, 3.8]
+        anndata-version: [0.6.22.post1, 0.7.1]
         test-suite: [smoke-test, smoke-test-annotations]
     steps:
     - uses: actions/checkout@v2

.github/workflows/scale-test.yml

@@ -1,30 +0,0 @@
-name: "Scale test cellxgene APIs for initial loading"
-
-on:
-  schedule:
-    - cron: "0 0 * * Sun"
-
-jobs:
-  locust-build:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.7
-        uses: actions/setup-python@v1
-        with:
-          python-version: 3.7
-      - name: Install dependencies
-        run: |
-          pip install -r server/test/locust/requirements-locust.txt
-      - name: Dev Scale Test
-        run: |
-          locust -f server/test/locust/locustfile.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt
-      - name: Slack success webhook
-        env:
-          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
-        run: |
-          DEV_STATS=$(tail -n 61 locust_dev_stats.txt)
-          DEV_MSG="\`\`\`CELLXGENE EXPLORER DEV SCALE TEST RESULTS: ${DEV_STATS}\`\`\`"
-          curl -X POST -H 'Content-type: application/json' --data "{'text':'${DEV_MSG}'}" $SLACK_WEBHOOK
-
-

MANIFEST.in

@@ -3,5 +3,3 @@ recursive-include server/common/web/static *
 
 include server/requirements.txt
 include server/requirements-prepare.txt
-include server/converters/schema/hgnc_complete_set.txt.gz
-include server/converters/schema/schema_definitions/*

PULL_REQUEST_TEMPLATE.md

@@ -1,11 +0,0 @@
-#### Reviewers
-**Functional:** 
-
-**Readability:** 
-
----
-
-## Changes
-- add
-- remove
-- modify

README.md

@@ -67,22 +67,13 @@ For any errors, [report bugs on Github](https://github.com/chanzuckerberg/cellxg
 
 ### Contributing
 
-We warmly welcome contributions from the community! Please see our [contributing guide](https://chanzuckerberg.github.io/cellxgene/posts/contribute) and don't hesitate to open an issue or send a pull request to improve cellxgene. Please see the [dev_docs](https://github.com/chanzuckerberg/cellxgene/tree/main/dev_docs) for pull request suggestions, unit test details, local documentation preview, and other development specifics. 
+We warmly welcome contributions from the community! Please see our [contributing guide](https://chanzuckerberg.github.io/cellxgene/posts/contribute) and don't hesitate to open an issue or send a pull request to improve cellxgene.
 
 This project adheres to the Contributor Covenant [code of conduct](https://github.com/chanzuckerberg/.github/blob/master/CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to opensource@chanzuckerberg.com.
 
 ### Reuse
 
-This project was started with the sole goal of empowering the scientific community to explore and understand their data. 
-As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from 
-this project. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT).
-
-
-Before extending cellxgene, we encourage you to reach out to us with ideas or questions. It might be possible that an 
-extension could be directly contributed, which would make it available for a wider audience, or that it's on our 
-[roadmap](./docs/posts/roadmap.md) and under active development. 
-
-See the [cellxgene extensions](./docs/posts/extensions.md) section of our documentation for examples of community use and cellxgene extensions. 
+This project was started with the sole goal of empowering the scientific community to explore and understand their data. As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from this project, and reach out to us with ideas or questions. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT).
 
 ### Security
 

client/__tests__/util/stateManager/colorHelpers.test.js

@@ -1,198 +0,0 @@
-/* eslint-disable no-bitwise -- unsigned right shift better than Math.round */
-
-/*
-test color helpers
-*/
-import {
-  createColorTable,
-  loadUserColorConfig,
-} from "../../../src/util/stateManager/colorHelpers";
-import * as Dataframe from "../../../src/util/dataframe";
-
-describe("categorical color helpers", () => {
-  /*
-  Primary test constraint for categorical colors is that they are ordered/identified
-  by schema order, NOT by value.  Ie,
-
-    scale(schemaIndex) should match rgb[obsOffset]
-  */
-
-  const schema = indexSchema({
-    annotations: {
-      obs: {
-        columns: [
-          {
-            name: "name_0",
-            type: "string",
-            writable: false,
-          },
-          {
-            name: "continuousColumn",
-            type: "float32",
-            writable: false,
-          },
-          {
-            categories: [
-              "CD4 T cells",
-              "CD14+ Monocytes",
-              "B cells",
-              "CD8 T cells",
-              "NK cells",
-              "FCGR3A+ Monocytes",
-              "Dendritic cells",
-              "Megakaryocytes",
-            ],
-            name: "categoricalColumn",
-            type: "categorical",
-            writable: false,
-          },
-        ],
-        index: "name_0",
-      },
-      var: {
-        columns: [
-          {
-            name: "name_0",
-            type: "string",
-            writable: false,
-          },
-        ],
-        index: "name_0",
-      },
-    },
-    dataframe: {
-      nObs: 2638,
-      nVar: 1838,
-      type: "float32",
-    },
-    layout: {},
-  });
-
-  const catColCategories = schema.annotations.obs.columns[2].categories;
-  const obsDataframe = new Dataframe.Dataframe(
-    [schema.dataframe.nObs, 2],
-    [
-      new Float32Array(schema.dataframe.nObs).map(() => Math.random()),
-      new Array(schema.dataframe.nObs)
-        .fill("")
-        .map(
-          () =>
-            catColCategories[(Math.random() * catColCategories.length) >>> 0]
-        ),
-    ],
-    null,
-    new Dataframe.KeyIndex(["continuousColumn", "categoricalColumn"])
-  );
-
-  test("default category order", () => {
-    const ct = createColorTable(
-      "color by categorical metadata",
-      "categoricalColumn",
-      obsDataframe,
-      schema
-    );
-    expect(ct).toBeDefined();
-    const data = obsDataframe.col("categoricalColumn").asArray();
-    const cats = schema.annotations.obsByName.categoricalColumn.categories;
-    for (let i = 0; i < schema.dataframe.nObs; i += 1) {
-      expect(makeScale(ct.rgb[i])).toEqual(ct.scale(cats.indexOf(data[i])));
-    }
-  });
-
-  test("shuffle category order", () => {
-    const schemaClone = indexSchema(JSON.parse(JSON.stringify(schema)));
-    shuffle(schemaClone.annotations.obsByName.categoricalColumn.categories);
-    const ct = createColorTable(
-      "color by categorical metadata",
-      "categoricalColumn",
-      obsDataframe,
-      schemaClone
-    );
-    expect(ct).toBeDefined();
-    const data = obsDataframe.col("categoricalColumn").asArray();
-    const cats = schemaClone.annotations.obsByName.categoricalColumn.categories;
-    for (let i = 0; i < schemaClone.dataframe.nObs; i += 1) {
-      expect(makeScale(ct.rgb[i])).toEqual(ct.scale(cats.indexOf(data[i])));
-    }
-  });
-
-  test("user defined color order", () => {
-    const cats = schema.annotations.obsByName.categoricalColumn.categories;
-    const shuffleCats = shuffle(
-      Array.from(schema.annotations.obsByName.categoricalColumn.categories)
-    );
-    const userDefinedColorTable = {
-      categoricalColumn: shuffleCats.reduce((acc, label) => {
-        acc[label] = randRGBColor();
-        return acc;
-      }, {}),
-    };
-
-    const userColors = loadUserColorConfig(userDefinedColorTable);
-    expect(userColors).toBeDefined();
-
-    const ct = createColorTable(
-      "color by categorical metadata",
-      "categoricalColumn",
-      obsDataframe,
-      schema,
-      userColors
-    );
-    expect(ct).toBeDefined();
-    const data = obsDataframe.col("categoricalColumn").asArray();
-    for (let i = 0; i < schema.dataframe.nObs; i += 1) {
-      expect(makeScale(ct.rgb[i])).toEqual(
-        ct.scale(cats.indexOf(data[i])).toString()
-      );
-    }
-  });
-});
-
-/*
-TODO:
-1. mix up category order in schema to make sure it works with varied order
-2. user defined colors
-*/
-
-function indexSchema(schema) {
-  schema.annotations.obsByName = Object.fromEntries(
-    schema.annotations?.obs?.columns?.map((v) => [v.name, v]) ?? []
-  );
-  schema.annotations.varByName = Object.fromEntries(
-    schema.annotations?.var?.columns?.map((v) => [v.name, v]) ?? []
-  );
-  schema.layout.obsByName = Object.fromEntries(
-    schema.layout?.obs?.map((v) => [v.name, v]) ?? []
-  );
-  schema.layout.varByName = Object.fromEntries(
-    schema.layout?.var?.map((v) => [v.name, v]) ?? []
-  );
-
-  return schema;
-}
-
-function makeScale(rgb) {
-  // make a scale string from a rgb float triple
-  return `rgb(${(rgb[0] * 255) >>> 0}, ${(rgb[1] * 255) >>> 0}, ${
-    (rgb[2] * 256) >>> 0
-  })`;
-}
-
-function shuffle(array) {
-  for (let i = array.length - 1; i > 0; i -= 1) {
-    const j = (Math.random() * (i + 1)) >>> 0;
-    [array[i], array[j]] = [array[j], array[i]];
-  }
-  return array;
-}
-
-function randHexColor() {
-  const hex = ((Math.random() * 255) >>> 0).toString(16);
-  return `0${hex}`.slice(-2);
-}
-
-function randRGBColor() {
-  return `#${randHexColor()}${randHexColor()}${randHexColor()}`;
-}
-
-/* eslint-enable no-bitwise -- unsigned right shift better than Math.round */

client/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "cellxgene",
-  "version": "0.16.7",
+  "version": "0.16.5",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
@@ -7963,15 +7963,6 @@
         "xdg-basedir": "^3.0.0"
       },
       "dependencies": {
-        "dot-prop": {
-          "version": "4.2.1",
-          "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-4.2.1.tgz",
-          "integrity": "sha512-l0p4+mIuJIua0mhxGoh4a+iNL9bmeK5DvnSVQa6T0OhrVmaEa1XScX5Etc673FePCJOArq/4Pa2cLGODUWTPOQ==",
-          "dev": true,
-          "requires": {
-            "is-obj": "^1.0.0"
-          }
-        },
         "make-dir": {
           "version": "1.3.0",
           "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-1.3.0.tgz",
@@ -9283,20 +9274,12 @@
       }
     },
     "dot-prop": {
-      "version": "5.3.0",
-      "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.3.0.tgz",
-      "integrity": "sha512-QM8q3zDe58hqUqjraQOmzZ1LIH9SWQJTlEKCH4kJ2oQvLZk7RbQXvtDM2XEq3fwkV9CCvvH4LA0AV+ogFsBM2Q==",
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-4.2.0.tgz",
+      "integrity": "sha512-tUMXrxlExSW6U2EXiiKGSBVdYgtV8qlHL+C10TsW4PURY/ic+eaysnSkwB4kA/mBlCyy/IKDJ+Lc3wbWeaXtuQ==",
       "dev": true,
       "requires": {
-        "is-obj": "^2.0.0"
-      },
-      "dependencies": {
-        "is-obj": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz",
-          "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==",
-          "dev": true
-        }
+        "is-obj": "^1.0.0"
       }
     },
     "duplexer3": {
@@ -12250,9 +12233,9 @@
       "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
     },
     "ini": {
-      "version": "1.3.7",
-      "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.7.tgz",
-      "integrity": "sha512-iKpRpXP+CrP2jyrxvg1kMUpXDyRUFDWurxbnVT1vQPx+Wz9uCYsMIqYuSBLV+PAaZG/d7kRLKRFc9oDMsH+mFQ==",
+      "version": "1.3.5",
+      "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz",
+      "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==",
       "dev": true
     },
     "internal-slot": {
@@ -15483,7 +15466,8 @@
     "pako": {
       "version": "1.0.11",
       "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz",
-      "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw=="
+      "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==",
+      "dev": true
     },
     "parallel-transform": {
       "version": "1.2.0",
@@ -16011,10 +15995,20 @@
         "vendors": "^1.0.0"
       },
       "dependencies": {
+        "dot-prop": {
+          "version": "5.2.0",
+          "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.2.0.tgz",
+          "integrity": "sha512-uEUyaDKoSQ1M4Oq8l45hSE26SnTxL6snNnqvK/VWx5wJhmff5z0FUVJDKDanor/6w3kzE3i7XZOk+7wC0EXr1A==",
+          "dev": true,
+          "requires": {
+            "is-obj": "^2.0.0"
+          }
+        },
         "is-obj": {
           "version": "2.0.0",
           "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz",
-          "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w=="
+          "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==",
+          "dev": true
         },
         "postcss-selector-parser": {
           "version": "3.1.2",
@@ -16101,10 +16095,20 @@
         "postcss-selector-parser": "^3.0.0"
       },
       "dependencies": {
+        "dot-prop": {
+          "version": "5.2.0",
+          "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.2.0.tgz",
+          "integrity": "sha512-uEUyaDKoSQ1M4Oq8l45hSE26SnTxL6snNnqvK/VWx5wJhmff5z0FUVJDKDanor/6w3kzE3i7XZOk+7wC0EXr1A==",
+          "dev": true,
+          "requires": {
+            "is-obj": "^2.0.0"
+          }
+        },
         "is-obj": {
           "version": "2.0.0",
           "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz",
-          "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w=="
+          "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==",
+          "dev": true
         },
         "postcss-selector-parser": {
           "version": "3.1.2",
@@ -18712,10 +18716,20 @@
         "postcss-selector-parser": "^3.0.0"
       },
       "dependencies": {
+        "dot-prop": {
+          "version": "5.2.0",
+          "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.2.0.tgz",
+          "integrity": "sha512-uEUyaDKoSQ1M4Oq8l45hSE26SnTxL6snNnqvK/VWx5wJhmff5z0FUVJDKDanor/6w3kzE3i7XZOk+7wC0EXr1A==",
+          "dev": true,
+          "requires": {
+            "is-obj": "^2.0.0"
+          }
+        },
         "is-obj": {
           "version": "2.0.0",
           "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz",
-          "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w=="
+          "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==",
+          "dev": true
         },
         "postcss-selector-parser": {
           "version": "3.1.2",
@@ -19824,9 +19838,9 @@
       }
     },
     "urijs": {
-      "version": "1.19.5",
-      "resolved": "https://registry.npmjs.org/urijs/-/urijs-1.19.5.tgz",
-      "integrity": "sha512-48z9VGWwdCV5KfizHsE05DWS5fhK6gFlx5MjO7xu0Krc5FGPWzjlXEVV0nPMrdVuP7xmMHiPZ2HoYZwKOFTZOg==",
+      "version": "1.19.2",
+      "resolved": "https://registry.npmjs.org/urijs/-/urijs-1.19.2.tgz",
+      "integrity": "sha512-s/UIq9ap4JPZ7H1EB5ULo/aOUbWqfDi7FKzMC2Nz+0Si8GiT1rIEaprt8hy3Vy2Ex2aJPpOQv4P4DuOZ+K1c6w==",
       "dev": true
     },
     "urix": {

client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cellxgene",
-  "version": "0.16.7",
+  "version": "0.16.5",
   "license": "MIT",
   "description": "cellxgene is a web application for the interactive exploration of single cell sequence data.",
   "repository": "https://github.com/chanzuckerberg/cellxgene",
@@ -54,7 +54,6 @@
     "is-number": "^7.0.0",
     "lodash": "^4.17.20",
     "memoize-one": "^5.1.1",
-    "pako": "^1.0.11",
     "react": "^16.13.1",
     "react-async": "^10.0.1",
     "react-dom": "^16.13.1",

client/src/actions/annotation.js

@@ -2,7 +2,6 @@
 Action creators for user annotation
 */
 import _ from "lodash";
-import pako from "pako";
 import * as globals from "../globals";
 import { MatrixFBS, AnnotationsHelpers } from "../util/stateManager";
 
@@ -154,7 +153,7 @@ export const annotationCreateLabelInCategory = (
   assignSelected
 ) => async (dispatch, getState) => {
   /*
-  Add a new label to a user-defined category.  If assignSelected is true, assign
+  Add a new label to a user-defined category.  If assignSelected is true, assign 
   the label to all currently selected cells.
   */
   const {
@@ -348,7 +347,6 @@ export const saveObsAnnotationsAction = () => async (dispatch, getState) => {
 
   const df = await annoMatrix.fetch("obs", writableAnnotations(annoMatrix));
   const matrix = MatrixFBS.encodeMatrixFBS(df);
-  const compressedMatrix = pako.deflate(matrix);
   try {
     const queryString =
       !dataCollectionNameIsReadOnly && !!dataCollectionName
@@ -360,7 +358,7 @@ export const saveObsAnnotationsAction = () => async (dispatch, getState) => {
       `${globals.API.prefix}${globals.API.version}annotations/obs${queryString}`,
       {
         method: "PUT",
-        body: compressedMatrix,
+        body: matrix,
         headers: new Headers({
           "Content-Type": "application/octet-stream",
         }),

client/src/components/brushableHistogram/index.js

@@ -146,7 +146,7 @@ const HistogramFooter = React.memo(
           </span>
         </div>
 
-        {logFoldChange !== undefined && pvalAdj !== undefined ? (
+        {logFoldChange && pvalAdj ? (
           <div
             style={{
               display: "flex",
@@ -450,6 +450,7 @@ const Histogram = ({
     isScatterplotYYaccessor: state.controls.scatterplotYYaccessor === field,
     continuousSelectionRange: state.continuousSelection[myName],
     isColorAccessor: state.colors.colorAccessor === field,
+    singleContinuousValues: state.singleContinuousValue.singleContinuousValues,
   };
 })
 class HistogramBrush extends React.PureComponent {
@@ -608,18 +609,44 @@ class HistogramBrush extends React.PureComponent {
   };
 
   fetchAsyncProps = async () => {
-    const { annoMatrix } = this.props;
+    const { annoMatrix, field, dispatch, singleContinuousValues } = this.props;
     const { isClipped } = annoMatrix;
-
+    if (singleContinuousValues.has(field)) {
+      return {
+        histogram: undefined,
+        range: undefined,
+        unclippedRange: undefined,
+        unclippedRangeColor: globals.blue,
+        isSingleValue: true,
+        OK2Render: false,
+      };
+    }
     const query = this.createQuery();
     const df = await annoMatrix.fetch(...query);
     const column = df.icol(0);
 
-    // if we are clipped, fetch both our value and our unclipped value,
-    // as we need the absolute min/max range, not just the clipped min/max.
     const summary = column.summarize();
     const range = [summary.min, summary.max];
 
+    if (summary.min === summary.max && !isClipped) {
+      dispatch({
+        type: "add single continuous value",
+        field,
+        value: summary.min,
+      });
+      return {
+        histogram: undefined,
+        range,
+        unclippedRange: range,
+        unclippedRangeColor: globals.blue,
+        isSingleValue: true,
+        OK2Render: false,
+      };
+    }
+
+    const isSingleValue = summary.min === summary.max;
+    // if we are clipped, fetch both our value and our unclipped value,
+    // as we need the absolute min/max range, not just the clipped min/max.
     let unclippedRange = [...range];
     if (isClipped) {
       const parent = await annoMatrix.viewOf.fetch(...query);
@@ -643,7 +670,6 @@ class HistogramBrush extends React.PureComponent {
       this.height
     );
 
-    const isSingleValue = summary.min === summary.max;
     const nonFiniteExtent =
       summary.min === undefined ||
       summary.max === undefined ||

client/src/components/categorical/value/index.js

@@ -174,7 +174,7 @@ class CategoryValue extends React.Component {
     Checks to see if at least one of the following changed:
     * world state
     * the color accessor (what is currently being colored by)
-    * if this categorical value's selection status has changed
+    * if this catagorical value's selection status has changed
     * the crossfilter (ie, global selection state)
 
     If and only if true, update the component
@@ -201,13 +201,6 @@ class CategoryValue extends React.Component {
     const newCount = newCategorySummary.categoryValueCounts[newCategoryIndex];
     const countChanged = count !== newCount;
 
-    // If the user edits an annotation that is currently colored-by, colors may be re-assigned.
-    // This test is conservative - it may cause re-rendering of entire category (all labels)
-    // if any one changes, but only for the currently colored-by category.
-    const colorMightHaveChanged =
-      nextProps.colorAccessor === nextProps.metadataField &&
-      props.categorySummary !== nextProps.categorySummary;
-
     return (
       labelChanged ||
       valueSelectionChange ||
@@ -215,8 +208,7 @@ class CategoryValue extends React.Component {
       annotationsChange ||
       editingLabel ||
       dilationChange ||
-      countChanged ||
-      colorMightHaveChanged
+      countChanged
     );
   };
 

client/src/components/continuousLegend/index.js

@@ -11,20 +11,20 @@ import {
 
 // create continuous color legend
 // http://bl.ocks.org/syntagmatic/e8ccca52559796be775553b467593a9f
-const continuous = (selectorId, colorScale, colorAccessor) => {
-  const legendHeight = 200;
-  const legendWidth = 80;
+const continuous = (selectorId, colorscale, colorAccessor) => {
+  const legendheight = 200;
+  const legendwidth = 80;
   const margin = { top: 10, right: 60, bottom: 10, left: 2 };
 
   const canvas = d3
     .select(selectorId)
-    .style("height", `${legendHeight}px`)
-    .style("width", `${legendWidth}px`)
+    .style("height", `${legendheight}px`)
+    .style("width", `${legendwidth}px`)
     .append("canvas")
-    .attr("height", legendHeight - margin.top - margin.bottom)
+    .attr("height", legendheight - margin.top - margin.bottom)
     .attr("width", 1)
-    .style("height", `${legendHeight - margin.top - margin.bottom}px`)
-    .style("width", `${legendWidth - margin.left - margin.right}px`)
+    .style("height", `${legendheight - margin.top - margin.bottom}px`)
+    .style("width", `${legendwidth - margin.left - margin.right}px`)
     .style("position", "absolute")
     .style("top", `${margin.top + 1}px`)
     .style("left", `${margin.left + 1}px`)
@@ -37,18 +37,18 @@ const continuous = (selectorId, colorScale, colorAccessor) => {
 
   const ctx = canvas.getContext("2d");
 
-  const legendScale = d3
+  const legendscale = d3
     .scaleLinear()
-    .range([1, legendHeight - margin.top - margin.bottom])
+    .range([1, legendheight - margin.top - margin.bottom])
     .domain([
-      colorScale.domain()[1],
-      colorScale.domain()[0],
+      colorscale.domain()[1],
+      colorscale.domain()[0],
     ]); /* we flip this to make viridis colors dark if high in the color scale */
 
   // image data hackery based on http://bl.ocks.org/mbostock/048d21cf747371b11884f75ad896e5a5
-  const image = ctx.createImageData(1, legendHeight);
-  d3.range(legendHeight).forEach((i) => {
-    const c = d3.rgb(colorScale(legendScale.invert(i)));
+  const image = ctx.createImageData(1, legendheight);
+  d3.range(legendheight).forEach((i) => {
+    const c = d3.rgb(colorscale(legendscale.invert(i)));
     image.data[4 * i] = c.r;
     image.data[4 * i + 1] = c.g;
     image.data[4 * i + 2] = c.b;
@@ -66,20 +66,20 @@ const continuous = (selectorId, colorScale, colorAccessor) => {
   });
   */
 
-  const legendAxis = d3
-    .axisRight(legendScale)
+  const legendaxis = d3
+    .axisRight(legendscale)
     .ticks(6)
     .tickFormat(
       d3.format(
-        legendScale.domain().some((n) => Math.abs(n) >= 10000) ? ".0e" : ","
+        legendscale.domain().some((n) => Math.abs(n) >= 10000) ? ".0e" : ","
       )
     );
 
   const svg = d3
     .select(selectorId)
     .append("svg")
-    .attr("height", `${legendHeight}px`)
-    .attr("width", `${legendWidth}px`)
+    .attr("height", `${legendheight}px`)
+    .attr("width", `${legendwidth}px`)
     .style("position", "absolute")
     .style("left", "0px")
     .style("top", "0px");
@@ -89,16 +89,16 @@ const continuous = (selectorId, colorScale, colorAccessor) => {
     .attr("class", "axis")
     .attr(
       "transform",
-      `translate(${legendWidth - margin.left - margin.right + 3},${margin.top})`
+      `translate(${legendwidth - margin.left - margin.right + 3},${margin.top})`
     )
-    .call(legendAxis);
+    .call(legendaxis);
 
   // text label for the y axis
   svg
     .append("text")
     .attr("transform", "rotate(-90)")
     .attr("y", 2)
-    .attr("x", 0 - legendHeight / 2)
+    .attr("x", 0 - legendheight / 2)
     .attr("dy", "1em")
     .style("text-anchor", "middle")
     .style("fill", "white")
@@ -110,7 +110,24 @@ const continuous = (selectorId, colorScale, colorAccessor) => {
   colors: state.colors,
 }))
 class ContinuousLegend extends React.Component {
-  async componentDidUpdate(prevProps) {
+  constructor(props) {
+    super(props);
+    this.ref = null;
+    this.state = {
+      colorAccessor: null,
+      colorScale: null,
+    };
+  }
+
+  componentDidMount() {
+    this.updateState(null);
+  }
+
+  componentDidUpdate(prevProps) {
+    this.updateState(prevProps);
+  }
+
+  async updateState(prevProps) {
     const { annoMatrix, colors } = this.props;
     if (!colors || !annoMatrix) return;
 
@@ -144,19 +161,35 @@ class ContinuousLegend extends React.Component {
           );
         }
       }
+
+      this.setState({
+        colorAccessor,
+        colorScale: colorTable.scale,
+      });
     }
   }
 
   render() {
+    const { colorAccessor, colorScale } = this.state;
+    if (
+      colorScale?.domain &&
+      colorScale.domain()[1] === colorScale.domain()[0]
+    ) {
+      /* it's a single value, not a distribution, min max are the same */
+      return null;
+    }
     return (
       <div
         id="continuous_legend"
+        ref={(ref) => {
+          this.ref = ref;
+        }}
         style={{
+          display: colorAccessor ? "inherit" : "none",
           position: "absolute",
           left: 8,
           top: 35,
           zIndex: 1,
-          pointerEvents: "none",
         }}
       />
     );

client/src/components/graph/drawPointsRegl.js

@@ -14,7 +14,7 @@ export default function drawPointsRegl(regl) {
     uniform float nPoints;
     uniform float minViewportDimension;
 
-    varying lowp vec4 fragColor;
+    varying vec4 fragColor;
 
     const float zBottom = 0.99;
     const float zMiddle = 0.;
@@ -27,23 +27,23 @@ export default function drawPointsRegl(regl) {
     ${glPointSize}
 
     void main() {
-      bool isBackground, isSelected, isHighlight;
-      getFlags(flag, isBackground, isSelected, isHighlight);
+      bool isNaN, isSelected, isHighlight;
+      getFlags(flag, isNaN, isSelected, isHighlight);
 
       float size = pointSize(nPoints, minViewportDimension, isSelected, isHighlight);
       gl_PointSize = size * pow(distance, 0.5);
 
-      float z = isBackground ? zBottom : (isHighlight ? zTop : zMiddle);
+      float z = isNaN ? zBottom : (isHighlight ? zTop : zMiddle);
       vec3 xy = projView * vec3(position, 1.);
       gl_Position = vec4(xy.xy, z, 1.);
 
-      float alpha = isBackground ? 0.9 : 1.0;
+      float alpha = isNaN ? 0.9 : 1.0;
       fragColor = vec4(color, alpha);
     }`,
 
     frag: `
     precision mediump float;
-    varying lowp vec4 fragColor;
+    varying vec4 fragColor;
     void main() {
       if (length(gl_PointCoord.xy - 0.5) > 0.5) {
         discard;
@@ -67,15 +67,5 @@ export default function drawPointsRegl(regl) {
     count: regl.prop("count"),
 
     primitive: "points",
-
-    blend: {
-      enable: true,
-      func: {
-        srcRGB: "src alpha",
-        srcAlpha: 1,
-        dstRGB: 0,
-        dstAlpha: "zero",
-      },
-    },
   });
 }

client/src/components/graph/graph.js

@@ -22,12 +22,6 @@ import CentroidLabels from "./overlays/centroidLabels";
 import actions from "../../actions";
 import renderThrottle from "../../util/renderThrottle";
 
-import {
-  flagBackground,
-  flagSelected,
-  flagHighlight,
-} from "../../util/glHelpers";
-
 /*
 Simple 2D transforms control all point painting.  There are three:
   * model - convert from underlying per-point coordinate to a layout.
@@ -68,6 +62,10 @@ function createModelTF() {
   return m;
 }
 
+const flagSelected = 1;
+const flagNaN = 2;
+const flagHighlight = 4;
+
 @connect((state) => ({
   annoMatrix: state.annoMatrix,
   crossfilter: state.obsCrossfilter,
@@ -161,9 +159,8 @@ class Graph extends React.Component {
     const flags = new Float32Array(nObs);
     if (colorByData) {
       for (let i = 0, len = flags.length; i < len; i += 1) {
-        const val = colorByData[i];
-        if (typeof val === "number" && !Number.isFinite(val)) {
-          flags[i] = flagBackground;
+        if (!Number.isFinite(colorByData[i])) {
+          flags[i] = flagNaN;
         }
       }
     }
@@ -733,28 +730,14 @@ class Graph extends React.Component {
     );
   });
 
-  updateReglAndRender(asyncProps, prevAsyncProps) {
-    const { positions, colors, flags, height, width } = asyncProps;
+  updateReglAndRender(asyncProps) {
+    const { positions, colors, flags } = asyncProps;
     this.cachedAsyncProps = asyncProps;
     const { pointBuffer, colorBuffer, flagBuffer } = this.state;
-    let needToRenderCanvas = false;
-
-    if (height !== prevAsyncProps?.height || width !== prevAsyncProps?.width) {
-      needToRenderCanvas = true;
-    }
-    if (positions !== prevAsyncProps?.positions) {
-      pointBuffer({ data: positions, dimension: 2 });
-      needToRenderCanvas = true;
-    }
-    if (colors !== prevAsyncProps?.colors) {
-      colorBuffer({ data: colors, dimension: 3 });
-      needToRenderCanvas = true;
-    }
-    if (flags !== prevAsyncProps?.flags) {
-      flagBuffer({ data: flags, dimension: 1 });
-      needToRenderCanvas = true;
-    }
-    if (needToRenderCanvas) this.renderCanvas();
+    pointBuffer({ data: positions, dimension: 2 });
+    colorBuffer({ data: colors, dimension: 3 });
+    flagBuffer({ data: flags, dimension: 1 });
+    this.renderCanvas();
   }
 
   updateColorTable(colors, colorDf) {
@@ -923,7 +906,7 @@ class Graph extends React.Component {
           <Async.Fulfilled>
             {(asyncProps) => {
               if (regl && !shallowEqual(asyncProps, this.cachedAsyncProps)) {
-                this.updateReglAndRender(asyncProps, this.cachedAsyncProps);
+                this.updateReglAndRender(asyncProps);
               }
               return null;
             }}

client/src/components/infoDrawer/infoDrawer.js

@@ -1,20 +1,25 @@
 import React, { PureComponent } from "react";
-import { connect } from "react-redux";
+import { connect, shallowEqual } from "react-redux";
 import { Drawer } from "@blueprintjs/core";
-
 import InfoFormat from "./infoFormat";
 import { selectableCategoryNames } from "../../util/stateManager/controlsHelpers";
 
 @connect((state) => {
   return {
+    annoMatrix: state.annoMatrix,
     schema: state.annoMatrix.schema,
     datasetTitle: state.config?.displayNames?.dataset ?? "",
     aboutURL: state.config?.links?.["about-dataset"],
     isOpen: state.controls.datasetDrawer,
-    dataPortalProps: state.config?.["corpora_props"],
+    dataPortalProps: state.config?.["corpora_props"] ?? {},
+    singleContinuousValues: state.singleContinuousValue.singleContinuousValues,
   };
 })
 class InfoDrawer extends PureComponent {
+  static watchAsync(props, prevProps) {
+    return !shallowEqual(props.watchProps, prevProps.watchProps);
+  }
+
   handleClose = () => {
     const { dispatch } = this.props;
 
@@ -29,18 +34,22 @@ class InfoDrawer extends PureComponent {
       schema,
       isOpen,
       dataPortalProps,
+      singleContinuousValues,
     } = this.props;
 
     const allCategoryNames = selectableCategoryNames(schema).sort();
-    const singleValueCategories = new Map();
+    const allSingleValues = new Map();
 
     allCategoryNames.forEach((catName) => {
       const isUserAnno = schema?.annotations?.obsByName[catName]?.writable;
       const colSchema = schema.annotations.obsByName[catName];
       if (!isUserAnno && colSchema.categories?.length === 1) {
-        singleValueCategories.set(catName, colSchema.categories[0]);
+        allSingleValues.set(catName, colSchema.categories[0]);
       }
     });
+    singleContinuousValues.forEach((value, catName) => {
+      allSingleValues.set(catName, value);
+    });
 
     return (
       <Drawer
@@ -52,8 +61,8 @@ class InfoDrawer extends PureComponent {
           {...{
             datasetTitle,
             aboutURL,
-            singleValueCategories,
-            dataPortalProps: dataPortalProps ?? {},
+            allSingleValues,
+            dataPortalProps,
           }}
         />
       </Drawer>

client/src/components/infoDrawer/infoFormat.js

@@ -1,6 +1,8 @@
-import { H3, H1, UL, HTMLTable, Classes } from "@blueprintjs/core";
+import { H3, H1, UL } from "@blueprintjs/core";
 import React from "react";
 
+import Truncate from "../util/truncate";
+
 const renderContributors = (contributors, affiliations) => {
   // eslint-disable-next-line no-constant-condition --  Temp removed contributor section to avoid publishing PII
   if (!contributors || contributors.length === 0 || true) return null;
@@ -69,63 +71,63 @@ const renderDOILink = (type, doi) => {
   );
 };
 
+const renderOrganism = (organism) => {
+  if (!organism) return null;
+  return (
+    <>
+      <H3>Organism</H3>
+      <p>{organism}</p>
+    </>
+  );
+};
+
 const ONTOLOGY_KEY = "ontology_term_id";
+const CAT_WIDTH = "30%";
+const VAL_WIDTH = "35%";
 // Render list of metadata attributes found in categorical field
-const renderDatasetMetadata = (singleValueCategories, corporaMetadata) => {
-  if (singleValueCategories.size === 0) return null;
+const renderSingleValues = (singleValues) => {
+  if (singleValues.size === 0) return null;
   return (
     <>
       <H3>Dataset Metadata</H3>
-      <HTMLTable
-        striped
-        condensed
-        style={{ display: "block", width: "100%", overflowX: "auto" }}
-      >
-        <thead>
-          <tr>
-            <th>Field</th>
-            <th>Label</th>
-            <th>Ontology ID</th>
-          </tr>
-        </thead>
-        <tbody>
-          {Object.entries(corporaMetadata).map(([key, value]) => {
-            return (
-              <tr {...{ key }}>
-                <td>{`${key}:`}</td>
-                <td>{value}</td>
-                <td />
-              </tr>
-            );
-          })}
-          {Array.from(singleValueCategories).reduce((elems, pair) => {
-            const [category, value] = pair;
-            // If the value is empty skip it
-            if (!value) return elems;
+      <UL>
+        {Array.from(singleValues).reduce((elems, pair) => {
+          const [category, value] = pair;
+          // If the value is empty skip it
+          if (!value) return elems;
 
-            // If this category is a ontology term, let's add its value to the previous node
-            if (String(category).includes(ONTOLOGY_KEY)) {
-              const prevElem = elems.pop();
-              const newChildren = [...prevElem.props.children];
-              newChildren.splice(2, 1, [<td key="ontology">{value}</td>]);
-              // Props aren't extensible so we must clone and alter the component to append the new child
-              elems.push(
-                React.cloneElement(prevElem, prevElem.props, newChildren)
-              );
-            } else {
-              // Create the list item
-              elems.push(
-                <tr key={category}>
-                  <td>{`${category}:`}</td>
-                  <td>{value}</td>
-                  <td />
-                </tr>
-              );
-            }
-            return elems;
-          }, [])}
-        </tbody>
-      </HTMLTable>
+          // If this category is a ontology term, let's add its value to the previous node
+          if (String(category).includes(ONTOLOGY_KEY)) {
+            const prevElem = elems.pop();
+            // Props aren't extensible so we must clone and alter the component to append the new child
+            elems.push(
+              React.cloneElement(
+                prevElem,
+                prevElem.props,
+                // Concat returns a new array
+                prevElem.props.children.concat([
+                  <Truncate key="ontology">
+                    <span style={{ width: VAL_WIDTH }}>{value}</span>
+                  </Truncate>,
+                ])
+              )
+            );
+          } else {
+            // Create the list item
+            elems.push(
+              <li key={category} style={{ width: "100%" }}>
+                <Truncate>
+                  <span style={{ width: CAT_WIDTH }}>{`${category}:`}</span>
+                </Truncate>
+                <Truncate>
+                  <span style={{ width: VAL_WIDTH }}>{value}</span>
+                </Truncate>
+              </li>
+            );
+          }
+          return elems;
+        }, [])}
+      </UL>
     </>
   );
 };
@@ -166,8 +168,8 @@ const renderLinks = (projectLinks, aboutURL) => {
 };
 
 const InfoFormat = React.memo(
-  ({ datasetTitle, singleValueCategories, aboutURL, dataPortalProps = {} }) => {
-    if (["1.0.0", "1.1.0"].indexOf(dataPortalProps.version?.["corpora_schema_version"]) === -1) {
+  ({ datasetTitle, allSingleValues, aboutURL, dataPortalProps = {} }) => {
+    if (dataPortalProps.version?.["corpora_schema_version"] !== "1.0.0") {
       dataPortalProps = {};
     }
     const {
@@ -182,15 +184,14 @@ const InfoFormat = React.memo(
     const affiliations = buildAffiliations(contributors);
 
     return (
-      <div className={Classes.DIALOG_BODY}>
-        <div className={Classes.DIALOG_BODY}>
-          <H1>{title ?? datasetTitle}</H1>
-          {renderContributors(contributors, affiliations)}
-          {renderDatasetMetadata(singleValueCategories, { organism })}
-          {renderLinks(projectLinks, aboutURL)}
-          {renderDOILink("DOI", doi)}
-          {renderDOILink("Preprint DOI", preprintDOI)}
-        </div>
+      <div style={{ margin: 24, overflow: "auto" }}>
+        <H1>{title ?? datasetTitle}</H1>
+        {renderContributors(contributors, affiliations)}
+        {renderDOILink("DOI", doi)}
+        {renderDOILink("Preprint DOI", preprintDOI)}
+        {renderOrganism(organism)}
+        {renderSingleValues(allSingleValues)}
+        {renderLinks(projectLinks, aboutURL)}
       </div>
     );
   }

client/src/components/leftSidebar/topLeftLogoAndTitle.js

@@ -1,3 +1,4 @@
+// jshint esversion: 6
 import React from "react";
 import { connect } from "react-redux";
 import { Button } from "@blueprintjs/core";
@@ -10,19 +11,13 @@ import InformationMenu from "./infoMenu";
 
 const DATASET_TITLE_FONT_SIZE = 14;
 
-@connect((state) => {
-  const { corpora_props: corporaProps } = state.config;
-  const correctVersion =
-    ["1.0.0", "1.1.0"].indexOf(corporaProps?.version?.["corpora_schema_version"]) > -1;
-  return {
-    datasetTitle: state.config?.displayNames?.dataset ?? "",
-    libraryVersions: state.config?.["library_versions"],
-    aboutLink: state.config?.links?.["about-dataset"],
-    tosURL: state.config?.parameters?.["about_legal_tos"],
-    privacyURL: state.config?.parameters?.["about_legal_privacy"],
-    title: correctVersion ? corporaProps?.title : undefined,
-  };
-})
+@connect((state) => ({
+  datasetTitle: state.config?.displayNames?.dataset ?? "",
+  libraryVersions: state.config?.["library_versions"],
+  aboutLink: state.config?.links?.["about-dataset"],
+  tosURL: state.config?.parameters?.["about_legal_tos"],
+  privacyURL: state.config?.parameters?.["about_legal_privacy"],
+}))
 class LeftSideBar extends React.Component {
   handleClick = () => {
     const { dispatch } = this.props;
@@ -37,7 +32,6 @@ class LeftSideBar extends React.Component {
       privacyURL,
       tosURL,
       dispatch,
-      title,
     } = this.props;
 
     return (
@@ -92,7 +86,7 @@ class LeftSideBar extends React.Component {
           >
             <Truncate>
               <span style={{ maxWidth: 155 }} data-testid="header">
-                {title ?? datasetTitle}
+                {datasetTitle}
               </span>
             </Truncate>
           </Button>

client/src/components/scatterplot/drawPointsRegl.js

@@ -13,7 +13,7 @@ export default function drawPointsRegl(regl) {
     uniform float nPoints;
     uniform float minViewportDimension;
 
-    varying lowp vec4 fragColor;
+    varying vec4 fragColor;
 
     const float zBottom = 0.99;
     const float zMiddle = 0.;
@@ -26,22 +26,22 @@ export default function drawPointsRegl(regl) {
     ${glPointSize}
 
     void main() {
-      bool isBackground, isSelected, isHighlight;
-      getFlags(flag, isBackground, isSelected, isHighlight);
+      bool isNaN, isSelected, isHighlight;
+      getFlags(flag, isNaN, isSelected, isHighlight);
 
       gl_PointSize = pointSize(nPoints, minViewportDimension, isSelected, isHighlight);
 
-      float z = isBackground ? zBottom : (isHighlight ? zTop : zMiddle);
+      float z = isNaN ? zBottom : (isHighlight ? zTop : zMiddle);
       vec3 xy = projection * vec3(position, 1.);
       gl_Position = vec4(xy.xy, z, 1.);
 
-      float alpha = isBackground ? 0.9 : 1.0;
+      float alpha = isNaN ? 0.9 : 1.0;
       fragColor = vec4(color, alpha);
     }`,
 
     frag: `
     precision mediump float;
-    varying lowp vec4 fragColor;
+    varying vec4 fragColor;
     void main() {
       if (length(gl_PointCoord.xy - 0.5) > 0.5) {
         discard;
@@ -64,15 +64,5 @@ export default function drawPointsRegl(regl) {
     count: regl.prop("count"),
 
     primitive: "points",
-
-    blend: {
-      enable: true,
-      func: {
-        srcRGB: "src alpha",
-        srcAlpha: 1,
-        dstRGB: 0,
-        dstAlpha: "zero",
-      },
-    },
   });
 }

client/src/components/scatterplot/scatterplot.js

@@ -16,11 +16,10 @@ import {
   createColorQuery,
 } from "../../util/stateManager/colorHelpers";
 import renderThrottle from "../../util/renderThrottle";
-import {
-  flagBackground,
-  flagSelected,
-  flagHighlight,
-} from "../../util/glHelpers";
+
+const flagSelected = 1;
+const flagNaN = 2;
+const flagHighlight = 4;
 
 function createProjectionTF(viewportWidth, viewportHeight) {
   /*
@@ -136,9 +135,8 @@ class Scatterplot extends React.PureComponent {
     const flags = new Float32Array(nObs);
     if (colorByData) {
       for (let i = 0, len = flags.length; i < len; i += 1) {
-        const val = colorByData[i];
-        if (typeof val === "number" && !Number.isFinite(val)) {
-          flags[i] = flagBackground;
+        if (!Number.isFinite(colorByData[i])) {
+          flags[i] = flagNaN;
         }
       }
     }

client/src/reducers/index.js

@@ -21,7 +21,7 @@ import centroidLabels from "./centroidLabels";
 import pointDialation from "./pointDilation";
 import { reembedController } from "./reembed";
 import { gcMiddleware as annoMatrixGC } from "../annoMatrix";
-
+import singleContinuousValue from "./singleContinuousValue";
 import undoableConfig from "./undoableConfig";
 
 const Reducer = undoable(
@@ -32,6 +32,7 @@ const Reducer = undoable(
     ["ontology", ontology],
     ["annotations", annotations],
     ["layoutChoice", layoutChoice],
+    ["singleContinuousValue", singleContinuousValue],
     ["categoricalSelection", categoricalSelection],
     ["continuousSelection", continuousSelection],
     ["graphSelection", graphSelection],

client/src/reducers/singleContinuousValue.js

@@ -0,0 +1,14 @@
+const initialState = {
+  singleContinuousValues: new Map(),
+};
+const singleContinuousValue = (state = initialState, action) => {
+  switch (action.type) {
+    case "add single continuous value":
+      state.singleContinuousValues.set(action.field, action.value);
+      return state;
+    default:
+      return state;
+  }
+};
+
+export default singleContinuousValue;

client/src/util/glHelpers.js

@@ -8,24 +8,18 @@ PointFlags:
 
   We want a bitmask-like flag structure, but due to webgl limitations
   must emulate it with floats.
-
-  Supported flags are:
-
-    selected: the point is currently selected
-    highlight: the point is currently highlighted
-    background: the point is background information
 */
 
 // for JS
 export const flagSelected = 1;
-export const flagBackground = 2;
+export const flagNaN = 2;
 export const flagHighlight = 4;
 
 // for GLSL
 export const glPointFlags = `
 
   const float flagSelected = 1.;
-  const float flagBackground = 2.;
+  const float flagNaN = 2.;
   const float flagHighlight = 4.;
 
   bool isLowBitSet(float f) {
@@ -38,12 +32,12 @@ export const glPointFlags = `
   }
 
   void getFlags(in float flag,
-                out bool isBackground,
+                out bool isNaN,
                 out bool isSelected,
                 out bool isHighlight) {
     isSelected = isLowBitSet(flag);
     flag = shiftRightOne(flag);
-    isBackground = isLowBitSet(flag);
+    isNaN = isLowBitSet(flag);
     flag = shiftRightOne(flag);
     isHighlight = isLowBitSet(flag);
   }

client/src/util/stateManager/colorHelpers.js

@@ -55,8 +55,8 @@ create colors scale and RGB array and return as object. Parameters:
   * userColors - optional user color table
 Returns:
   {
-    scale: function, mapping label index to color scale
-    rgb: cell label to color mapping
+    scale: color scale
+    rgb: cell to color mapping
   }
 */
 function _createColorTable(
@@ -70,7 +70,7 @@ function _createColorTable(
     case "color by categorical metadata": {
       const data = colorByData.col(colorByAccessor).asArray();
       if (userColors && colorByAccessor in userColors) {
-        return createUserColors(data, colorByAccessor, schema, userColors);
+        return createUserColors(data, colorByAccessor, userColors);
       }
       return createColorsByCategoricalMetadata(data, colorByAccessor, schema);
     }
@@ -91,41 +91,42 @@ function _createColorTable(
 }
 export const createColorTable = memoize(_createColorTable);
 
-/**
- * Create two category label-indexed objects:
- *    - colors: maps label to RGB triplet for that label (used by graph, etc)
- *    - scale: function which given label returns d3 color scale for label
- * Order doesn't matter - everything is keyed by label value.
- */
 export function loadUserColorConfig(userColors) {
   const convertedUserColors = {};
   Object.keys(userColors).forEach((category) => {
-    const [colors, scaleMap] = Object.keys(userColors[category]).reduce(
-      (acc, label) => {
-        const color = parseRGB(userColors[category][label]);
-        acc[0][label] = color;
-        acc[1][label] = d3.rgb(255 * color[0], 255 * color[1], 255 * color[2]);
-        return acc;
-      },
-      [{}, {}]
-    );
+    // We cannot iterate over keys without sorting
+    // because we handle categorical values in alphabetical order __ignoring case__
+    //  while Object.keys() _usually_ is ordered alphabetically where all upper characters are less than lowercase (A, B, C, a, b, c)
+    const [colors, scaleMap] = Object.keys(userColors[category])
+      .sort((a, b) => {
+        a = a.toLowerCase();
+        b = b.toLowerCase();
+        if (a === b) return 0;
+        if (a > b) return 1;
+        return -1;
+      })
+      .reduce(
+        (acc, label) => {
+          const color = parseRGB(userColors[category][label]);
+          acc[0][label] = color;
+          acc[1][label] = d3.rgb(
+            255 * color[0],
+            255 * color[1],
+            255 * color[2]
+          );
+          return acc;
+        },
+        [{}, {}]
+      );
     const scale = (label) => scaleMap[label];
     convertedUserColors[category] = { colors, scale };
   });
   return convertedUserColors;
 }
 
-function _createUserColors(data, colorAccessor, schema, userColors) {
-  const { colors, scale: scaleByLabel } = userColors[colorAccessor];
+function _createUserColors(data, colorAccessor, userColors) {
+  const { colors, scale } = userColors[colorAccessor];
   const rgb = createRgbArray(data, colors);
-
-  // color scale function param is INDEX (offset) into schema categories. It is NOT label value.
-  // See createColorsByCategoricalMetadata() for another example.
-  const { categories } = schema.annotations.obsByName[colorAccessor];
-  const categoryMap = new Map();
-  categories.forEach((label, idx) => categoryMap.set(idx, label));
-  const scale = (idx) => scaleByLabel(categoryMap.get(idx));
-
   return { rgb, scale };
 }
 const createUserColors = memoize(_createUserColors);

client/src/util/stateManager/matrix.js

@@ -178,26 +178,26 @@ function promoteTypedArray(o) {
   */
   if (isFpTypedArray(o) || Array.isArray(o)) return o;
 
-  let TypedArrayCtor;
+  let TyepdArrayCtor;
   switch (o.constructor) {
     case Int8Array:
     case Uint8Array:
     case Uint8ClampedArray:
     case Int16Array:
     case Uint16Array:
-      TypedArrayCtor = Float32Array;
+      TyepdArrayCtor = Float32Array;
       break;
 
     case Int32Array:
     case Uint32Array:
-      TypedArrayCtor = Float64Array;
+      TyepdArrayCtor = Float64Array;
       break;
 
     default:
       throw new Error("Unexpected data type returned from server.");
   }
-  if (o.constructor === TypedArrayCtor) return o;
-  return new TypedArrayCtor(o);
+  if (o.constructor === TyepdArrayCtor) return o;
+  return new TyepdArrayCtor(o);
 }
 
 export function matrixFBSToDataframe(arrayBuffers) {

dev_docs/schema_guide.md

@@ -1,175 +0,0 @@
-# Cellxgene Schema Guide
-
-Datasets included in the [data portal](https://cellxgene.cziscience.com/) and hosted cellxgene need to follow the schema
-described [here](https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md). That
-schema defines some required fields, requirements about feature labels, and some optional fields that mostly help with
-presentation.
-
-The number of fields is rather low, and we expect that information needed to populate those fields should either already
-be present in datasets prepared by a submitter or be easy to obtain. However, this still leaves the task of actually
-manipulating the dataset so that it follows the schema: adjusting field names, ensuring proper ontologies are used,
-converting gene symbols to a common set, etc. This can be tedious and error-prone, and at the beginning of the hosted
-cellxgene project, this was always done with engineering support. As we increase the rate at which we add data, we want
-to eliminate the need for engineering support so that ultimately submitters themselves can create files that follow the
-schema.
-
-## `cellxgene schema apply`
-
-To enable this, we have a new cellxgene subcommand, `cellxgene schema`, that handles applying and verifying the schema.
-Its first subcommand, `cellxgene schema apply`, takes three inputs:
-
-1. A source h5ad file. The input needs to be an AnnData file, so if a submitter has, say, a serialized Seurat or
-   SingleCellExperiment object, it needs to be converted to AnnData first. This can be done with
-   [sceasy](https://github.com/cellgeni/sceasy) or via
-   [Seurat](https://satijalab.org/seurat/v3.1/conversion_vignette.html).
-2. A configuration yaml file that describes the fields to add and conversions to apply (see below).
-3. A name for the new h5ad file that should follow the schema.
-
-### Configuration yaml
-
-The configuration yaml file describes how to apply the schema. This is an example of a "skeleton" yaml that has all the
-fields required for the 1.0.0 schema but is not yet filled in with any logic:
-
-```
-uns:
-    version:
-        corpora_schema_version: 1.0.0
-        corpora_encoding_version: 0.1.0
-    contributors:
-    title:
-    layer_descriptions:
-    preprint_doi:
-    publication_doi:
-    organism_ontology_term_id:
-obs:
-    tissue_ontology_term_id:
-    assay_ontology_term_id:
-    disease_ontology_term_id:
-    cell_type_ontology_term_id:
-    sex:
-    ethnicity_ontology_term_id:
-    development_stage_ontology_term_id:
-fixup_gene_symbols:
-```
-
-#### Unstructured metadata
-The first section is `uns`, which includes metadata fields that describe the whole dataset (see 
-[here](https://anndata.readthedocs.io/en/latest/) for further description of `uns` and `obs`.).
-
-The first line is `version`, which is required for most of our tooling to work. The schema version is set at
-1.0.0 in the example above, but of course for future versions that should be changed.
-
-Next is `contributors` which describes who is adding the dataset to the portal. If you consult the schema, you see that
-contributors is a list where each element can have `name`, `email`, and `institution`. So when filled out, the
-`contributors` field should look like this:
-
-```
-contributors:
-  - name: Mary B. Scientist
-    email: mbs@singlecell.edu
-    institution: Single-Cell University
-  - name: Robert J. Scientist
-    email: rjs@usingle.edu
-    institution: University of Single Cell
-```
-
-`title` is the name of the dataset, and is just a string that gets displayed in the portal and cellxgene to identify the
-dataset.
-
-`layer_descriptions` is free text descriptions of the different
-[layers](https://anndata.readthedocs.io/en/latest/anndata.AnnData.layers.html) of the AnnData file. It should look like
-this when complete, depending on what layers are present:
-```
-layer_descriptions:
-  X: CPM and logged
-  raw.X: raw
-```
-Note that one of the layers needs to be "raw", that is, the AnnData file must contain raw counts.
-
-The two DOI fields are optional but can be included if the dataset is associated with a publication or preprint. Note
-that the DOI should be a full url:
-```
-publication_doi: https://doi.org/10.1073%2Fpnas.83.15.5372
-```
-
-Finally, the `organism_ontology_term_id` field is the species of the donor organism from the NCBITaxon ontology. The
-value for _Homo sapiens_ is `NCBITaxon:9606`:
-```
-organism_ontology_term_id: NCBITaxon:9606
-```
-Note that the schema also requires a human-readable `organism` field, but this doesn't need to be included in the yaml.
-When the `cellxgene schema apply` script encounters an ontology field, it looks up the label for the term(s) and inserts it
-into the appropriate field.
-
-
-#### Observation metadata
-The next section is `obs`, which is metadata than can vary for each observation (and "observation" usually means cell).
-These fields are all ontology fields except for `sex`, which has its own enumerated set of permitted values.
-
-There are two ways to fill in the `obs` fields. The first is useful when there is only one value for all the
-observations in the dataset. This is not uncommon, for example all cells often come from the same assay. In that case
-just insert the ontology term:
-```
-assay_ontology_term_id: EFO:0009922
-```
-
-The second is for when there is an existing field in the dataset that needs to be mapped to the schema field. For
-example, the submitter may have included cell type annotations in a field called `CellType`, and those annotations may
-just be free text. This doesn't follow the schema because it needs to be in `cell_type_ontology_term_id` and
-`cell_type`, and it needs ontology terms and labels, not just any text. In that case the field can be a dictionary:
-
-```
-cell_type_ontology_term_id:
-  CellType:
-    t-cell: CL:0000084
-    b-cell: CL:0000236
-```
-
-This will look at the `obs.CellType` field in the dataset, and where it has the value "t-cell", it will insert
-`CL:0000084` into `cell_type_ontology_term_id` and its label `T cell` into `cell_type`.
-
-Now there are often situations where there is no valid ontology term for some field. For example, the dataset may have
-been produced via an assay not present in `EFO`. Or, a particular cell type may have no entry in `CL`. In that case, a
-free text description can be used in the `ontology_term_id` field:
-
-```
-assay_ontology_term_id: Sci-Plex
-cell_type_ontology_term_id:
-  CellType:
-    t-cell: CL:0000084
-    b-cell: CL:0000236
-    new cell type: new cell type
-```
-
-In these cases, the `cellxgene schema apply` script will leave the ontology field blank and move the free text
-description into the label field. So the `assay_ontology_term_id` in the new dataset would be `""` but `assay` would be
-`Sci-Plex`.
-
-
-#### Gene symbol harmonization
-
-The last section describes how gene symbol conversion should be applied to each of the layers. This is similar to the
-`layer_descriptions` field above, but there are only three permitted values: `raw`, `log1p`, and `sqrt`:
-
-```
-fixup_gene_symbols:
-  X: log1p
-  raw.X: raw
-```
-
-This tells the script how each each layer was transformed from raw values that can be directly summed. `raw` means that
-the layer contains raw counts or some linear tranformation of raw counts. `log1p` means that the layer has `log(X + 1)`
-for each the raw `X` values. `sqrt` means `sqrt(X)` (this is not common). For layers produced by Seurat's normalization
-or SCTransform functions, the correct choice is usually `log1p`.
-
-
-### `cellxgene schema validate`
-
-The next `cellxgene schema` subcommand is `cellxgene schema validate`, and it validates that a given h5ad follows a
-version of the schema. It accepts two parameters:
-
-1. The h5ad file to check
-2. The version of the schema to check against.
-
-If the validation succeeds, the command will have a zero exit code. If it does not, it will have a non-zero exit code
-and will print validation failure messages.

docs/_config.yml

@@ -33,7 +33,5 @@ nav:
     url: posts/roadmap
   - title: Contributing (ideas or code)
     url: posts/contribute
-  - title: Extensions
-    url: posts/extensions
   - title: Contact & finding help
     url: posts/contact

docs/_site/deprecated/cellxgene_cziscience_com.html

@@ -1,446 +0,0 @@
-<!DOCTYPE html>
-<html lang="en-US">
-  <head>
-    <meta charset="UTF-8">
-    <meta http-equiv="X-UA-Compatible" content="IE=edge">
-    <meta name="viewport" content="width=device-width, initial-scale=1">
-
-<!-- Begin Jekyll SEO tag v2.6.1 -->
-<title>cellxgene.cziscience.com | cellxgene</title>
-<meta name="generator" content="Jekyll v3.8.7" />
-<meta property="og:title" content="cellxgene.cziscience.com" />
-<meta property="og:locale" content="en_US" />
-<meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
-<meta property="og:description" content="An interactive explorer for single-cell transcriptomics data" />
-<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/deprecated/cellxgene_cziscience_com.html" />
-<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/deprecated/cellxgene_cziscience_com.html" />
-<meta property="og:site_name" content="cellxgene" />
-<script type="application/ld+json">
-{"@type":"WebPage","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"cellxgene.cziscience.com","description":"An interactive explorer for single-cell transcriptomics data","url":"https://chanzuckerberg.github.io/cellxgene/deprecated/cellxgene_cziscience_com.html","@context":"https://schema.org"}</script>
-<!-- End Jekyll SEO tag -->
-
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=f70dffced52a32aada1a22504c841e97e941401a">
-    <!--[if lt IE 9]>
-    <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
-    <![endif]-->
-  </head>
-  <body>
-    <div class="wrapper">
-      <header>
-        <img src="/cellxgene/cellxgene-logo.png" alt="cellxgene" />
-
-        <p>An interactive explorer for single-cell transcriptomics data</p>
-        <p>
-
-        
-        <a href="/cellxgene/" class="btn">Quick start</a><br>
-        
-
-        
-        
-        <a href="/cellxgene/posts/install" class="btn">Installation</a><br>
-        
-        
-        
-        <a href="/cellxgene/posts/gallery" class="btn">Gallery</a><br>
-        
-        
-        
-        <a href="/cellxgene/posts/demo-data" class="btn">Demo datasets</a><br>
-        
-        
-        
-        <a href="https://cellxgene.cziscience.com/" class="btn">All other datasets</a><br>
-        
-        
-        
-        <a href="/cellxgene/posts/prepare" class="btn">Preparing your data</a><br>
-        
-        
-        
-        <a href="/cellxgene/posts/launch" class="btn">Launching cellxgene</a><br>
-        
-        
-        
-        <a href="/cellxgene/posts/hosted" class="btn">Hosting cellxgene</a><br>
-        
-        
-        
-        <a href="/cellxgene/posts/annotations" class="btn">Annotating data</a><br>
-        
-        
-        
-        <a href="/cellxgene/posts/methods" class="btn">Methods</a><br>
-        
-        
-        
-        <a href="/cellxgene/posts/troubleshooting" class="btn">Troubleshooting</a><br>
-        
-        
-        
-        <a href="/cellxgene/posts/roadmap" class="btn">Roadmap</a><br>
-        
-        
-        
-        <a href="/cellxgene/posts/contribute" class="btn">Contributing (ideas or code)</a><br>
-        
-        
-        
-        <a href="/cellxgene/posts/extensions" class="btn">Extensions</a><br>
-        
-        
-        
-        <a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
-        
-        
-
-        <a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
-    </p>
-    </header>
-
-      <section>
-      <h1 id="cellxgeneczisciencecom">cellxgene.cziscience.com</h1>
-
-<p>Chan Zuckerberg has an online repository of public single-cell datasets for exploration with cellxgene.</p>
-
-<p>If you have a public dataset which you would like hosted for visualization on this site,
-with a link to embed on your own site, please drop us a note at <a href="mailto:cellxgene@chanzuckerberg.com">cellxgene@chanzuckerberg.com</a>.</p>
-
-<table class="fixed-layout">
-  <thead style="width: 100%">
-    <tr>
-      <th>cellxgene link</th>
-      <th>More Information</th>
-    </tr>
-  </thead>
-  <tbody style="width: 100%">
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/krasnow_lab_human_lung_cell_atlas_10x-1.cxg/" target="_blank">Krasnow Lab Human Lung Cell Atlas, 10X</a></td>
-      <td>
-        <a href="http://cmgm-new.stanford.edu/krasnow/">Krasnow Lab</a>,
-        <a href="https://github.com/krasnowlab/hlca">HLCA website</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/krasnow_lab_human_lung_cell_atlas_smartseq2-2.cxg/" target="_blank">Krasnow Lab Human Lung Cell Atlas, Smart-seq2</a></td>
-      <td>
-        <a href="http://cmgm-new.stanford.edu/krasnow/">Krasnow Lab</a>, 
-        <a href="https://github.com/krasnowlab/hlca">HLCA website</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/human_cell_landscape-3.cxg/" target="_blank">Human Cell Landscape</a></td>
-      <td>
-        <a href="https://person.zju.edu.cn/en/ggj">Guo Lab</a>, 
-        <a href="http://bis.zju.edu.cn/HCL/">HCL website</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/human_fetal_liver_single_cell_transcriptome-13.cxg/" target="_blank">Human fetal liver single cell transcriptome data</a></td>
-      <td>
-        <a href="https://www.ebi.ac.uk/arrayexpress/experiments/E-MTAB-7407/">E-MTAB-7407</a>,
-        <a href="https://www.covid19cellatlas.org/">covid19cellatlas.org</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/cell_atlas_of_thymic_development-14.cxg/" target="_blank">A cell atlas of human thymic development defines T cell repertoire formation</a></td>
-      <td>
-        <a href="https://www.ebi.ac.uk/arrayexpress/experiments/E-MTAB-8581/">E-MTAB-8581</a>,
-        <a href="https://www.covid19cellatlas.org/">covid19cellatlas.org</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/cellular_census_of_human_lungs_alveoli_and_parenchyma-15.cxg/" target="_blank">A cellular census of human lungs identifies novel cell states in health and in asthma - parenchyma</a></td>
-      <td>
-        <a href="https://asthma.cellgeni.sanger.ac.uk/">asthma.cellgeni.sanger.ac.uk</a>,
-        <a href="https://www.covid19cellatlas.org/">covid19cellatlas.org</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/cellular_census_of_human_lungs_nasal-16.cxg/" target="_blank">A cellular census of human lungs identifies novel cell states in health and in asthma - nasal</a></td>
-      <td>
-        <a href="https://asthma.cellgeni.sanger.ac.uk/">asthma.cellgeni.sanger.ac.uk</a>,
-        <a href="https://www.covid19cellatlas.org/">covid19cellatlas.org</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/cellular_census_of_human_lungs_bronchi-17.cxg/" target="_blank">A cellular census of human lungs identifies novel cell states in health and in asthma - bronchi</a></td>
-      <td>
-        <a href="https://asthma.cellgeni.sanger.ac.uk/">asthma.cellgeni.sanger.ac.uk</a>,
-        <a href="https://www.covid19cellatlas.org/">covid19cellatlas.org</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/ischaemic_sensitivity_of_human_tissue_by_single_cell_RNA_seq_lung-18.cxg/" target="_blank">Ischaemic sensitivity of human tissue by single cell RNA seq - lung</a></td>
-      <td>
-        <a href="https://data.humancellatlas.org/explore/projects/c4077b3c-5c98-4d26-a614-246d12c2e5d7">HCA</a>,
-        <a href="https://www.covid19cellatlas.org/">covid19cellatlas.org</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/ischaemic_sensitivity_of_human_tissue_by_single_cell_RNA_seq_spleen-19.cxg/" target="_blank">Ischaemic sensitivity of human tissue by single cell RNA seq - spleen</a></td>
-      <td>
-        <a href="https://data.humancellatlas.org/explore/projects/c4077b3c-5c98-4d26-a614-246d12c2e5d7">HCA</a>,
-        <a href="https://www.covid19cellatlas.org/">covid19cellatlas.org</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/ischaemic_sensitivity_of_human_tissue_by_single_cell_RNA_seq_oesophagus-20.cxg/" target="_blank">Ischaemic sensitivity of human tissue by single cell RNA seq - oesophagus</a></td>
-      <td>
-        <a href="https://data.humancellatlas.org/explore/projects/c4077b3c-5c98-4d26-a614-246d12c2e5d7">HCA</a>,
-        <a href="https://www.covid19cellatlas.org/">covid19cellatlas.org</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/spatio_temporal_immune_zonation_of_the_human_kidney-21.cxg/" target="_blank">Spatio-temporal immune zonation of the human kidney</a></td>
-      <td>
-        <a href="https://www.kidneycellatlas.org/">www.kidneycellatlas.org</a>,
-        <a href="https://www.covid19cellatlas.org/">covid19cellatlas.org</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/fetal_maternal_interface_10x-22.cxg/" target="_blank">Reconstructing the human first trimester fetal-maternal interface using single cell transcriptomics - 10x</a></td>
-      <td>
-        <a href="https://www.ebi.ac.uk/arrayexpress/experiments/E-MTAB-6701/">E-MTAB-6701</a>,
-        <a href="https://www.covid19cellatlas.org/">covid19cellatlas.org</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/fetal_maternal_interface_smartseq2-23.cxg/" target="_blank">Reconstructing the human first trimester fetal-maternal interface using single cell transcriptomics - SmartSeq2</a></td>
-      <td>
-        <a href="https://www.ebi.ac.uk/arrayexpress/experiments/E-MTAB-6701/">E-MTAB-6701</a>,
-        <a href="https://www.covid19cellatlas.org/">covid19cellatlas.org</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/gut_cell_atlas-24.cxg/" target="_blank">Gut Cell Atlas</a></td>
-      <td>
-        <a href="https://www.gutcellatlas.org/">www.gutcellatlas.org</a>,
-        <a href="https://www.covid19cellatlas.org/">covid19cellatlas.org</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/Single_cell_atlas_of_peripheral_immune_response_to_SARS_CoV_2_infection-25.cxg/" target="_blank">A single-cell atlas of the peripheral immune response to severe COVID-19</a></td>
-      <td>
-        <a href="https://blishlab.sites.stanford.edu/">Blish Lab</a>,
-        <a href="https://www.medrxiv.org/content/10.1101/2020.04.17.20069930v1">medRxiv preprint</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/Atlas_of_Healthy_and_SHIV_Infected_Non_Human_Primate_Lung_and_Ileum_ACE2+_Cells_ileum-12.cxg/" target="_blank">Atlas of Healthy and SHIV-Infected Non-Human Primate Lung and Ileum ACE2+ Cells - Ileum</a></td>
-      <td>
-        <a href="https://singlecell.broadinstitute.org/single_cell/study/SCP807/atlas-of-healthy-and-shiv-infected-non-human-primate-lung-and-ileum-ace2-cells?scpbr=the-alexandria-project">Single Cell Portal</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/Atlas_of_Healthy_and_SHIV_Infected_Non_Human_Primate_Lung_and_Ileum_ACE2+_Cells_lung-11.cxg/" target="_blank">Atlas of Healthy and SHIV-Infected Non-Human Primate Lung and Ileum ACE2+ Cells - Lung</a></td>
-      <td>
-        <a href="https://singlecell.broadinstitute.org/single_cell/study/SCP807/atlas-of-healthy-and-shiv-infected-non-human-primate-lung-and-ileum-ace2-cells?scpbr=the-alexandria-project">Single Cell Portal</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/Allergic_inflammatory_memory_in_human_respiratory_epithelial_progenitor_cells_epithelial-10.cxg/" target="_blank">Allergic inflammatory memory in human respiratory epithelial progenitor cells - epithelial cells</a></td>
-      <td>
-        <a href="https://singlecell.broadinstitute.org/single_cell/study/SCP253/allergic-inflammatory-memory-in-human-respiratory-epithelial-progenitor-cells?scpbr=the-alexandria-project">Single Cell Portal</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/Allergic_inflammatory_memory_in_human_respiratory_epithelial_progenitor_cells_scraping-9.cxg/" target="_blank">Allergic inflammatory memory in human respiratory epithelial progenitor cells - nasal scrapings</a></td>
-      <td>
-        <a href="https://singlecell.broadinstitute.org/single_cell/study/SCP253/allergic-inflammatory-memory-in-human-respiratory-epithelial-progenitor-cells?scpbr=the-alexandria-project">Single Cell Portal</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/Allergic_inflammatory_memory_in_human_respiratory_epithelial_progenitor_cells_surgical-8.cxg/" target="_blank">Allergic inflammatory memory in human respiratory epithelial progenitor cells - surgical</a></td>
-      <td>
-        <a href="https://singlecell.broadinstitute.org/single_cell/study/SCP253/allergic-inflammatory-memory-in-human-respiratory-epithelial-progenitor-cells?scpbr=the-alexandria-project">Single Cell Portal</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/Allergic_inflammatory_memory_in_human_respiratory_epithelial_progenitor_cells_nasalsss-26.cxg/" target="_blank">Allergic inflammatory memory in human respiratory epithelial progenitor cells - nasal SSS</a></td>
-      <td>
-        <a href="https://singlecell.broadinstitute.org/single_cell/study/SCP253/allergic-inflammatory-memory-in-human-respiratory-epithelial-progenitor-cells?scpbr=the-alexandria-project">Single Cell Portal</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/ACE2_and_TMPRSS2_expression_in_human_non_inflamed_terminal_ileum_epithelial-7.cxg/" target="_blank">ACE2 and TMPRSS2 expression in human non-inflamed terminal ileum - epithelial cells</a></td>
-      <td>
-        <a href="https://singlecell.broadinstitute.org/single_cell/study/SCP812/ace2-and-tmprss2-expression-in-human-non-inflamed-terminal-ileum?scpbr=the-alexandria-project">Single Cell Portal</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/ACE2_and_TMPRSS2_expression_in_human_non_inflamed_terminal_ileum-6.cxg/" target="_blank">ACE2 and TMPRSS2 expression in human non-inflamed terminal ileum</a></td>
-      <td>
-        <a href="https://singlecell.broadinstitute.org/single_cell/study/SCP812/ace2-and-tmprss2-expression-in-human-non-inflamed-terminal-ileum?scpbr=the-alexandria-project">Single Cell Portal</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/Human_Lung_HIV_TB_Co_infection_ACE2+_Cells-5.cxg/" target="_blank">Human Lung HIV-TB Co-infection ACE2+ Cells</a></td>
-      <td>
-        <a href="https://singlecell.broadinstitute.org/single_cell/study/SCP814/human-lung-hiv-tb-co-infection-ace2-cells?scpbr=the-alexandria-project">Single Cell Portal</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/Epithelial_Cells_in_NHP_mTB_Granuloma_and_Uninvolved_Lung-4.cxg/" target="_blank">Epithelial Cells in NHP mTB Granuloma and Uninvolved Lung</a></td>
-      <td>
-        <a href="https://singlecell.broadinstitute.org/single_cell/study/SCP806/epithelial-cells-in-nhp-mtb-granuloma-and-uninvolved-lung?scpbr=the-alexandria-project">Single Cell Portal</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/kampmann_lab_human_AD_snRNAseq_EC-49.cxg/
-" target="_blank">Selective Neuronal Vulnerability in Alzheimer's Disease</a></td>
-      <td>
-        <a href="https://kampmannlab.ucsf.edu/">Kampmann Lab</a>, 
-        <a href="https://www.biorxiv.org/content/10.1101/2020.04.04.025825v2">BioRxiv preprint</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/kampmann_lab_human_AD_snRNAseq_SFG-50.cxg/
-" target="_blank">Selective Neuronal Vulnerability in Alzheimer's Disease: Superior Frontal Gyrus</a></td>
-      <td>
-        <a href="https://kampmannlab.ucsf.edu/">Kampmann Lab</a>, 
-        <a href="https://www.biorxiv.org/content/10.1101/2020.04.04.025825v2">BioRxiv preprint</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/kampmann_lab_human_AD_snRNAseq_EC_astrocytes-51.cxg/
-" target="_blank">Selective Neuronal Vulnerability in Alzheimer's Disease: Astrocytes in EC</a></td>
-      <td>
-        <a href="https://kampmannlab.ucsf.edu/">Kampmann Lab</a>, 
-        <a href="https://www.biorxiv.org/content/10.1101/2020.04.04.025825v2">BioRxiv preprint</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/kampmann_lab_human_AD_snRNAseq_EC_excitatoryNeurons-52.cxg/
-" target="_blank">Selective Neuronal Vulnerability in Alzheimer's Disease: Excitatory Neurons in EC</a></td>
-      <td>
-        <a href="https://kampmannlab.ucsf.edu/">Kampmann Lab</a>, 
-        <a href="https://www.biorxiv.org/content/10.1101/2020.04.04.025825v2">BioRxiv preprint</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/kampmann_lab_human_AD_snRNAseq_EC_inhibitoryNeurons-53.cxg/
-" target="_blank">Selective Neuronal Vulnerability in Alzheimer's Disease: Inhibitory Neurons in EC</a></td>
-      <td>
-        <a href="https://kampmannlab.ucsf.edu/">Kampmann Lab</a>, 
-        <a href="https://www.biorxiv.org/content/10.1101/2020.04.04.025825v2">BioRxiv preprint</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/kampmann_lab_human_AD_snRNAseq_EC_microglia-54.cxg/
-" target="_blank">Selective Neuronal Vulnerability in Alzheimer's Disease: Microglia in EC</a></td>
-      <td>
-        <a href="https://kampmannlab.ucsf.edu/">Kampmann Lab</a>, 
-        <a href="https://www.biorxiv.org/content/10.1101/2020.04.04.025825v2">BioRxiv preprint</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/kampmann_lab_human_AD_snRNAseq_SFG_astrocytes-55.cxg/
-" target="_blank">Selective Neuronal Vulnerability in Alzheimer's Disease: Astrocytes in SFG</a></td>
-      <td>
-        <a href="https://kampmannlab.ucsf.edu/">Kampmann Lab</a>, 
-        <a href="https://www.biorxiv.org/content/10.1101/2020.04.04.025825v2">BioRxiv preprint</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/kampmann_lab_human_AD_snRNAseq_SFG_excitatoryNeurons-56.cxg/
-" target="_blank">Selective Neuronal Vulnerability in Alzheimer's Disease: Excitatory Neurons in SFG</a></td>
-      <td>
-        <a href="https://kampmannlab.ucsf.edu/">Kampmann Lab</a>, 
-        <a href="https://www.biorxiv.org/content/10.1101/2020.04.04.025825v2">BioRxiv preprint</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/kampmann_lab_human_AD_snRNAseq_SFG_inhibitoryNeurons-57.cxg/" target="_blank">Selective Neuronal Vulnerability in Alzheimer's Disease: Inhibitory Neurons in SFG</a></td>
-      <td>
-        <a href="https://kampmannlab.ucsf.edu/">Kampmann Lab</a>, 
-        <a href="https://www.biorxiv.org/content/10.1101/2020.04.04.025825v2">BioRxiv preprint</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/kampmann_lab_human_AD_snRNAseq_SFG_microglia-58.cxg/" target="_blank">Selective Neuronal Vulnerability in Alzheimer's Disease: Microglia in SFG</a></td>
-      <td>
-        <a href="https://kampmannlab.ucsf.edu/">Kampmann Lab</a>, 
-        <a href="https://www.biorxiv.org/content/10.1101/2020.04.04.025825v2">BioRxiv preprint</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/Single_cell_gene_expression_profiling_of_SARS_CoV_2_infected_human_cell_lines_H1299-27.cxg/" target="_blank">Single-cell gene expression profiling of SARS-CoV-2 infected human cell lines - H1299</a></td>
-      <td>
-        <a href="https://www.mdc-berlin.de/landthaler#t-single-cellsars-cov-2">Landthaler Lab</a>,
-        <a href="https://www.biorxiv.org/content/10.1101/2020.05.05.079194v1">BioRxiv preprint</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/Single_cell_gene_expression_profiling_of_SARS_CoV_2_infected_human_cell_lines_Calu_3-28.cxg/" target="_blank">Single-cell gene expression profiling of SARS-CoV-2 infected human cell lines - Calu-3</a></td>
-      <td>
-        <a href="https://www.mdc-berlin.de/landthaler#t-single-cellsars-cov-2">Landthaler Lab</a>,
-        <a href="https://www.biorxiv.org/content/10.1101/2020.05.05.079194v1">BioRxiv preprint</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/Single_cell_drug_screening_a549-42.cxg/" target="_blank">Single-cell drug screening - A549</a></td>
-      <td>
-        <a href="https://github.com/cole-trapnell-lab/sci-plex">Trapnell Lab Github</a>,
-        <a href="https://science.sciencemag.org/content/367/6473/45">Science</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/Single_cell_drug_screening_k562-43.cxg/" target="_blank">Single-cell drug screening - K562</a></td>
-      <td>
-        <a href="https://github.com/cole-trapnell-lab/sci-plex">Trapnell Lab Github</a>,
-        <a href="https://science.sciencemag.org/content/367/6473/45">Science</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/Single_cell_drug_screening_mcf7-44.cxg/" target="_blank">Single-cell drug screening - MCF7</a></td>
-      <td>
-        <a href="https://github.com/cole-trapnell-lab/sci-plex">Trapnell Lab Github</a>,
-        <a href="https://science.sciencemag.org/content/367/6473/45">Science</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.prod.single-cell.czi.technology/d/Molecular_atlas_of_cell_types_and_zonation_in_the_brain_vasculature-48.cxg/" target="_blank">A molecular atlas of cell types and zonation in the brain vasculature</a></td>
-      <td>
-        <a href="http://betsholtzlab.org/VascularSingleCells/database.html">Betsholtz Lab</a>,
-        <a href="https://www.nature.com/articles/nature25739">Nature</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/Single_soma_transcriptomics_AT8-45.cxg/" target="_blank">Single Soma Transcriptomics - AT8</a></td>
-      <td>
-        <a href="https://www.biorxiv.org/content/10.1101/2020.05.11.088591v1">bioRxiv preprint</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/Single_soma_transcriptomics_MAP2-46.cxg/" target="_blank">Single Soma Transcriptomics - MAP2</a></td>
-      <td>
-        <a href="https://www.biorxiv.org/content/10.1101/2020.05.11.088591v1">bioRxiv preprint</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/Single_soma_transcriptomics_MAP2AT8-47.cxg/" target="_blank">Single Soma Transcriptomics - MAP2AT8</a></td>
-      <td>
-        <a href="https://www.biorxiv.org/content/10.1101/2020.05.11.088591v1">bioRxiv preprint</a>
-      </td>
-    </tr>
-    <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/Single_cell_longitudinal_analysis_of_SARS_CoV_2_infection_in_human_bronchial_epithelial_cells-29.cxg/" target="_blank">Single-cell longitudinal analysis of SARS-CoV-2 infection in human bronchial epithelial cells</a></td>
-      <td>
-        <a href="https://www.biorxiv.org/content/10.1101/2020.05.06.081695v2">bioRxiv preprint</a>
-      </td>
-    </tr>
-  </tbody>
-</table>
-
-      </section>
-      <footer>
-        
-        <p>This project is maintained by <a href="https://github.com/chanzuckerberg">chanzuckerberg</a></p>
-        
-      </footer>
-    </div>
-    <script src="/cellxgene/assets/js/scale.fix.js"></script>
-    
-  </body>
-</html>

docs/_site/index.html

@@ -7,7 +7,7 @@
 
 <!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>Index | cellxgene</title>
-<meta name="generator" content="Jekyll v3.8.7" />
+<meta name="generator" content="Jekyll v3.9.0" />
 <meta property="og:title" content="Index" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
@@ -16,10 +16,10 @@
 <meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"@type":"WebSite","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Index","description":"An interactive explorer for single-cell transcriptomics data","url":"https://chanzuckerberg.github.io/cellxgene/","name":"cellxgene","@context":"https://schema.org"}</script>
+{"url":"https://chanzuckerberg.github.io/cellxgene/","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Index","name":"cellxgene","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebSite","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=f70dffced52a32aada1a22504c841e97e941401a">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=3718e894edc8a8f6e7776946695ab37c5c96ec9f">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->
@@ -86,10 +86,6 @@
         
         
         
-        <a href="/cellxgene/posts/extensions" class="btn">Extensions</a><br>
-        
-        
-        
         <a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
         
         

docs/_site/posts/extensions.html

@@ -1,156 +0,0 @@
-<!DOCTYPE html>
-<html lang="en-US">
-  <head>
-    <meta charset="UTF-8">
-    <meta http-equiv="X-UA-Compatible" content="IE=edge">
-    <meta name="viewport" content="width=device-width, initial-scale=1">
-
-<!-- Begin Jekyll SEO tag v2.6.1 -->
-<title>Extensions | cellxgene</title>
-<meta name="generator" content="Jekyll v3.8.7" />
-<meta property="og:title" content="Extensions" />
-<meta property="og:locale" content="en_US" />
-<meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
-<meta property="og:description" content="An interactive explorer for single-cell transcriptomics data" />
-<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/extensions.html" />
-<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/extensions.html" />
-<meta property="og:site_name" content="cellxgene" />
-<script type="application/ld+json">
-{"@type":"WebPage","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Extensions","description":"An interactive explorer for single-cell transcriptomics data","url":"https://chanzuckerberg.github.io/cellxgene/posts/extensions.html","@context":"https://schema.org"}</script>
-<!-- End Jekyll SEO tag -->
-
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=f70dffced52a32aada1a22504c841e97e941401a">
-    <!--[if lt IE 9]>
-    <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
-    <![endif]-->
-  </head>
-  <body>
-    <div class="wrapper">
-      <header>
-        <img src="/cellxgene/cellxgene-logo.png" alt="cellxgene" />
-
-        <p>An interactive explorer for single-cell transcriptomics data</p>
-        <p>
-
-        
-        <a href="/cellxgene/" class="btn">Quick start</a><br>
-        
-
-        
-        
-        <a href="/cellxgene/posts/install" class="btn">Installation</a><br>
-        
-        
-        
-        <a href="/cellxgene/posts/gallery" class="btn">Gallery</a><br>
-        
-        
-        
-        <a href="/cellxgene/posts/demo-data" class="btn">Demo datasets</a><br>
-        
-        
-        
-        <a href="https://cellxgene.cziscience.com/" class="btn">All other datasets</a><br>
-        
-        
-        
-        <a href="/cellxgene/posts/prepare" class="btn">Preparing your data</a><br>
-        
-        
-        
-        <a href="/cellxgene/posts/launch" class="btn">Launching cellxgene</a><br>
-        
-        
-        
-        <a href="/cellxgene/posts/hosted" class="btn">Hosting cellxgene</a><br>
-        
-        
-        
-        <a href="/cellxgene/posts/annotations" class="btn">Annotating data</a><br>
-        
-        
-        
-        <a href="/cellxgene/posts/methods" class="btn">Methods</a><br>
-        
-        
-        
-        <a href="/cellxgene/posts/troubleshooting" class="btn">Troubleshooting</a><br>
-        
-        
-        
-        <a href="/cellxgene/posts/roadmap" class="btn">Roadmap</a><br>
-        
-        
-        
-        <a href="/cellxgene/posts/contribute" class="btn">Contributing (ideas or code)</a><br>
-        
-        
-        
-        <a href="/cellxgene/posts/extensions" class="btn"><b>Extensions</b></a><br>
-        
-        
-        
-        <a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
-        
-        
-
-        <a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
-    </p>
-    </header>
-
-      <section>
-      <h1 id="extensions">Extensions</h1>
-
-<p>This project was started with the sole goal of empowering the scientific community to explore and understand their data. 
-As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from 
-this project. All code is freely available for reuse under the <a href="https://opensource.org/licenses/MIT">MIT license</a>.</p>
-
-<p>Before extending cellxgene, we encourage you to reach out to us with ideas or questions. It might be possible that an 
-extension could be directly contributed, which would make it available for a wider audience, or that it’s on our 
-<a href="/cellxgene/posts/roadmap.html">roadmap</a> and under active development.</p>
-
-<p>Please note that cellxgene does not have public APIs. Our development may break extensions. We will document changes to the code base but it is advised that extensions pin the version of cellxgene they develop against.</p>
-
-<h2 id="example-reuse--extensions">Example Reuse &amp; extensions</h2>
-
-<h4 id="cellxgene-gateway">cellxgene Gateway</h4>
-
-<p><a href="https://github.com/Novartis/cellxgene-gateway">cellxgene Gateway</a> allows you to use with multiple datasets. It 
-displays an index of available h5ad (anndata) files. When a user clicks on a file name, it launches a Cellxgene Server 
-instance that loads that particular data file and once it is available proxies requests to that server.</p>
-
-<h4 id="cellxgene-vip-visualization-in-plugin">cellxgene-VIP (Visualization in Plugin)</h4>
-
-<p><a href="https://github.com/interactivereport/cellxgene_VIP">cellxgene-VIP</a> enables cellxgene to generate violin, stacked violin, stacked bar, heatmap, volcano, embedding, dot, track, density, 2D density, sankey and dual-gene plot in high-resolution SVG/PNG format. It also performs differential gene expression analysis and provides a Command Line Interface (CLI) for advanced users to perform analysis using python and R.</p>
-
-<h4 id="galaxy">Galaxy</h4>
-
-<p><a href="https://galaxyproject.org">Galaxy</a> is an open source, collaborative, web-based platform for data intensive biomedical research.
-Galaxy provides various tools for <a href="https://singlecell.usegalaxy.eu/">single-cell data analysis</a> and also infrastructure to the
-<a href="https://humancellatlas.usegalaxy.eu">Galaxy Human Cell Atlas project</a>. cellxgene can be <a href="https://usegalaxy.eu/root?tool_id=interactive_tool_cellxgene">
-  accessed within Galaxy</a> to view analyzed datasets. See also the relevant <a href="https://doi.org/10.1093/gigascience/giaa102">publication</a>
-
-</p>
-
-<h4 id="single-cell-portal">Single Cell Portal</h4>
-
-<p>The <a href="https://singlecell.broadinstitute.org/single_cell">Single Cell Portal</a> is a data hosting and visualization service. cellxgene can be embedded as an additional view to complement the visualizations provided by the. 
-<a href="https://singlecell.broadinstitute.org/single_cell/study/SCP807/atlas-of-healthy-and-shiv-infected-non-human-primate-lung-and-ileum-ace2-cells">Example</a>.</p>
-
-<h4 id="fastgenomics">FASTGenomics</h4>
-
-<p><a href="https://beta.fastgenomics.org/">FASTGenomics</a> is a collaborative research platform that offers easy-to-use data management and reproducible analytics to drive single-cell research forward. Many of the publicly available datasets in FASTGenomics - as well as your private datasets - can be interactively explored with cellxgene. 
-See also this <a href="https://beta.fastgenomics.org/datasets/detail-dataset-952687f71ef34322a850553c4a24e82e#Cellxgene">example</a> for data from <a href="https://beta.fastgenomics.org/p/schulte-schrepping_covid19">Schulte-Schrepping et al. (Cell, 2020)</a>. 
-Note that it is not necessary to create an account, anonymous login is permitted.</p>
-
-      </section>
-      <footer>
-        
-        <p>This project is maintained by <a href="https://github.com/chanzuckerberg">chanzuckerberg</a></p>
-        
-      </footer>
-    </div>
-    <script src="/cellxgene/assets/js/scale.fix.js"></script>
-    
-  </body>
-</html>

docs/_site/posts/extensions.md

@@ -1,39 +0,0 @@
-# Extensions
-
-This project was started with the sole goal of empowering the scientific community to explore and understand their data. 
-As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from 
-this project. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT).
-
-Before extending cellxgene, we encourage you to reach out to us with ideas or questions. It might be possible that an 
-extension could be directly contributed, which would make it available for a wider audience, or that it's on our 
-[roadmap](./roadmap.md) and under active development. 
-
-Please note that cellxgene does not have public APIs. Our development may break extensions. We will document changes to the code base but it is advised that extensions pin the version of cellxgene they develop against. 
-
-## Example Reuse & extensions 
-
-#### cellxgene Gateway
-
-[cellxgene Gateway](https://github.com/Novartis/cellxgene-gateway) allows you to use with multiple datasets. It 
-displays an index of available h5ad (anndata) files. When a user clicks on a file name, it launches a Cellxgene Server 
-instance that loads that particular data file and once it is available proxies requests to that server.
-
-#### cellxgene-VIP (Visualization in Plugin)
-
-[cellxgene-VIP](https://github.com/interactivereport/cellxgene_VIP) enables cellxgene to generate violin, stacked violin, stacked bar, heatmap, volcano, embedding, dot, track, density, 2D density, sankey and dual-gene plot in high-resolution SVG/PNG format. It also performs differential gene expression analysis and provides a Command Line Interface (CLI) for advanced users to perform analysis using python and R.
-
-#### Galaxy
-
-[Galaxy](https://singlecell.usegalaxy.eu/) is an open source, web-based platform for data intensive biomedical research. cellxgene can be accessed within Galaxy to view analyzed datasets. 
-See also the relevant [publication](https://www.biorxiv.org/content/10.1101/2020.06.06.137570v1.full.pdf) 
-
-#### Single Cell Portal
-
-The [Single Cell Portal](https://singlecell.broadinstitute.org/single_cell) is a data hosting and visualization service. cellxgene can be embedded as an additional view to complement the visualizations provided by the. 
-[Example](https://singlecell.broadinstitute.org/single_cell/study/SCP807/atlas-of-healthy-and-shiv-infected-non-human-primate-lung-and-ileum-ace2-cells). 
-
-#### FASTGenomics
-
-[FASTGenomics](https://beta.fastgenomics.org/) is a collaborative research platform that offers easy-to-use data management and reproducible analytics to drive single-cell research forward. Many of the publicly available datasets in FASTGenomics - as well as your private datasets - can be interactively explored with cellxgene. 
-See also this [example](https://beta.fastgenomics.org/datasets/detail-dataset-952687f71ef34322a850553c4a24e82e#Cellxgene) for data from [Schulte-Schrepping et al. (Cell, 2020)](https://beta.fastgenomics.org/p/schulte-schrepping_covid19). 
-Note that it is not necessary to create an account, anonymous login is permitted.

docs/posts/extensions.md

@@ -1,39 +0,0 @@
-# Extensions
-
-This project was started with the sole goal of empowering the scientific community to explore and understand their data. 
-As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from 
-this project. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT).
-
-Before extending cellxgene, we encourage you to reach out to us with ideas or questions. It might be possible that an 
-extension could be directly contributed, which would make it available for a wider audience, or that it's on our 
-[roadmap](./roadmap.md) and under active development. 
-
-Please note that cellxgene does not have public APIs. Our development may break extensions. We will document changes to the code base but it is advised that extensions pin the version of cellxgene they develop against. 
-
-## Example Reuse & extensions 
-
-#### cellxgene Gateway
-
-[cellxgene Gateway](https://github.com/Novartis/cellxgene-gateway) allows you to use with multiple datasets. It 
-displays an index of available h5ad (anndata) files. When a user clicks on a file name, it launches a Cellxgene Server 
-instance that loads that particular data file and once it is available proxies requests to that server.
-
-#### cellxgene-VIP (Visualization in Plugin)
-
-[cellxgene-VIP](https://github.com/interactivereport/cellxgene_VIP) enables cellxgene to generate violin, stacked violin, stacked bar, heatmap, volcano, embedding, dot, track, density, 2D density, sankey and dual-gene plot in high-resolution SVG/PNG format. It also performs differential gene expression analysis and provides a Command Line Interface (CLI) for advanced users to perform analysis using python and R.
-
-#### Galaxy
-
-[Galaxy](https://singlecell.usegalaxy.eu/) is an open source, web-based platform for data intensive biomedical research. cellxgene can be accessed within Galaxy to view analyzed datasets. 
-See also the relevant [publication](https://www.biorxiv.org/content/10.1101/2020.06.06.137570v1.full.pdf) 
-
-#### Single Cell Portal
-
-The [Single Cell Portal](https://singlecell.broadinstitute.org/single_cell) is a data hosting and visualization service. cellxgene can be embedded as an additional view to complement the visualizations provided by the. 
-[Example](https://singlecell.broadinstitute.org/single_cell/study/SCP807/atlas-of-healthy-and-shiv-infected-non-human-primate-lung-and-ileum-ace2-cells). 
-
-#### FASTGenomics
-
-[FASTGenomics](https://beta.fastgenomics.org/) is a collaborative research platform that offers easy-to-use data management and reproducible analytics to drive single-cell research forward. Many of the publicly available datasets in FASTGenomics - as well as your private datasets - can be interactively explored with cellxgene. 
-See also this [example](https://beta.fastgenomics.org/datasets/detail-dataset-952687f71ef34322a850553c4a24e82e#Cellxgene) for data from [Schulte-Schrepping et al. (Cell, 2020)](https://beta.fastgenomics.org/p/schulte-schrepping_covid19). 
-Note that it is not necessary to create an account, anonymous login is permitted.

server/__init__.py

@@ -2,7 +2,7 @@ import logging
 import sys
 from server.common.utils.utils import import_plugins
 
-__version__ = "0.16.7"
+__version__ = "0.16.5"
 display_version = "cellxgene v" + __version__
 
 try:

server/app/app.py

@@ -105,6 +105,17 @@ def dataset_index(url_dataroot=None, dataset=None):
         )
 
 
+# TODO:  This route will be deprecated, but needs to be left for a short time until all the
+# deployments are upgraded to the new location for the health check (or else the upgrade will
+# fail).  Once the upgrade is complete, the deployments can move to the new health check URL
+# and this route will be removed.
+@webbp.route("/health", methods=["GET"])
+@cache_control_always(no_store=True)
+def health():
+    config = current_app.app_config
+    return health_check(config)
+
+
 @webbp.errorhandler(RequestException)
 def handle_request_exception(error):
     return common_rest.abort_and_log(error.status_code, error.message, loglevel=logging.INFO, include_exc_info=True)
@@ -419,15 +430,9 @@ class Server:
                 dataroot_resources = get_api_dataroot_resources(bp_dataroot, url_dataroot)
                 self.app.register_blueprint(dataroot_resources.blueprint)
 
-                self.app.add_url_rule(
-                    f"/{url_dataroot}/<dataset>",
-                    f"dataset_index_{url_dataroot}",
-                    lambda dataset, url_dataroot=url_dataroot: dataset_index(url_dataroot, dataset),
-                    methods=["GET"],
-                )
                 self.app.add_url_rule(
                     f"/{url_dataroot}/<dataset>/",
-                    f"dataset_index_{url_dataroot}/",
+                    f"dataset_index_{url_dataroot}",
                     lambda dataset, url_dataroot=url_dataroot: dataset_index(url_dataroot, dataset),
                     methods=["GET"],
                 )

server/auth/auth_oauth.py

@@ -24,7 +24,7 @@ except ModuleNotFoundError:
 class Tokens:
     """Simple class to represent the tokens that are saved/restored from the cookie"""
 
-    def __init__(self, access_token, id_token, refresh_token, expires_at, **kwargs):
+    def __init__(self, access_token, id_token, refresh_token, expires_at):
         self.access_token = access_token
         self.id_token = id_token
         self.refresh_token = refresh_token
@@ -218,24 +218,22 @@ class AuthTypeOAuth(AuthTypeClientBase):
 
         try:
             if self.session_cookie:
-                value = session.get(self.CXG_TOKENS)
-                if value:
-                    g.tokens = Tokens(**value)
+                tokensdict = session.get(self.CXG_TOKENS)
+                if tokensdict:
+                    g.tokens = Tokens(**tokensdict)
                 else:
                     return None
             else:
                 value = request.cookies.get(self.cookie_params["key"])
-                if value is None:
-                    return None
                 value = base64.b64decode(value)
-                value = json.loads(value)
-                g.tokens = Tokens(**value)
+                try:
+                    tokensdict = json.loads(value)
+                    g.tokens = Tokens(**tokensdict)
+                except (TypeError, KeyError, json.decoder.JSONDecodeError):
+                    g.pop("tokens", None)
+                    return None
 
-        except Exception:
-            # there are many types of exceptions that can be raise in the above section.
-            # It is impractical to list all the exceptions here, since that would be brittle.
-            # If an exception occurs, then return None, meaning that no token could be retrieved.
-            current_app.logger.warning(f"auth cookie is in the wrong format: {str(value)}")
+        except (TypeError, KeyError):
             g.pop("tokens", None)
             return None
 
@@ -333,7 +331,6 @@ class AuthTypeOAuth(AuthTypeClientBase):
 
         # if there is no id_token, return None (user is not authenticated)
         tokens = self.get_tokens()
-
         if tokens is None or tokens.id_token is None:
             return None
 

server/cli/cli.py

@@ -4,7 +4,6 @@ from .convert_to_cxg import convert_to_cxg
 from .launch import launch
 from .prepare import prepare
 from .upgrade import log_upgrade_check
-from .schema import schema_cli
 from .. import __version__
 
 
@@ -32,4 +31,3 @@ def cli(upgrade_check):
 cli.add_command(launch)
 cli.add_command(prepare)
 cli.add_command(convert_to_cxg)
-cli.add_command(schema_cli)

server/cli/schema.py

@@ -1,72 +0,0 @@
-import click
-
-from server.converters.schema import remix, validate
-
-
-@click.group(
-    name="schema",
-    subcommand_metavar="COMMAND <args>",
-    short_help="Apply and validate the cellxgene data integration schema to an h5ad file.",
-    context_settings=dict(max_content_width=85, help_option_names=["-h", "--help"]),
-)
-def schema_cli():
-    try:
-        import scanpy  # noqa: F401
-    except ImportError:
-        raise click.ClickException(
-            "[cellxgene] cellxgene schema requires scanpy"
-        )
-
-
-@click.command(
-    name="apply",
-    short_help="(experimental) Apply the cellxgene data integration schema to an h5ad.",
-    help="(experimental) Using a yaml file that describes schema values to insert or convert and in input "
-    "h5ad file, apply the schema changes and create a new, conforming h5ad.",
-)
-@click.option(
-    "--source-h5ad",
-    help="Input h5ad file.",
-    nargs=1,
-    required=True,
-    type=click.Path(exists=True, dir_okay=False),
-)
-@click.option(
-    "--remix-config",
-    help="Config yaml with information on how to apply the schema.",
-    nargs=1,
-    required=True,
-    type=click.Path(exists=True, dir_okay=False),
-)
-@click.option(
-    "--output-filename",
-    help="Filename for the new, schema-conforming h5ad file.",
-    required=True,
-    nargs=1
-)
-def schema_apply(source_h5ad, remix_config, output_filename):
-    remix.apply_schema(source_h5ad, remix_config, output_filename)
-
-
-@click.command(
-    name="validate",
-    short_help="(experimental) Check that an h5ad follows the cellxgene data integration schema.",
-)
-@click.argument(
-    "h5ad",
-    nargs=1,
-    type=click.Path(exists=True, dir_okay=False),
-)
-@click.option(
-    "--shallow",
-    help="When true, just check that the correct version information is present.",
-    default=False,
-    show_default=True,
-    is_flag=True,
-)
-def schema_validate(h5ad, shallow):
-    validate.validate(h5ad, shallow)
-
-
-schema_cli.add_command(schema_apply)
-schema_cli.add_command(schema_validate)

server/common/annotations/annotations.py

@@ -33,10 +33,10 @@ class Annotations(metaclass=ABCMeta):
             raise OntologyLoadFailure("Unable to find OBO ontology path") from e
 
         except SyntaxError as e:
-            raise OntologyLoadFailure(f"{path}:{e.lineno}:{e.offset} OBO syntax error, unable to read ontology") from e
+            raise OntologyLoadFailure("Syntax error loading OBO ontology") from e
 
         except Exception as e:
-            raise OntologyLoadFailure(f"{path}:Error loading OBO file") from e
+            raise OntologyLoadFailure("Error loading OBO file") from e
 
     def get_schema(self, data_adaptor):
         schema = []

server/common/annotations/hosted_tiledb.py

@@ -34,12 +34,6 @@ class AnnotationsHostedTileDB(Annotations):
                 f"{unsanitary_original_category_names} are not valid category names, please resubmit"
             )
 
-    def get_user_name(self):
-        return current_app.auth.get_user_name()
-
-    def get_user_id(self):
-        return current_app.auth.get_user_id()
-
     def is_safe_collection_name(self, name):
         """
         return true if this is a safe collection name
@@ -54,7 +48,7 @@ class AnnotationsHostedTileDB(Annotations):
         self.CXG_ANNO_COLLECTION = name
 
     def read_labels(self, data_adaptor):
-        user_id = self.get_user_id()
+        user_id = current_app.auth.get_user_id()
         if user_id is None:
             return
         dataset_name = data_adaptor.get_location()
@@ -64,15 +58,7 @@ class AnnotationsHostedTileDB(Annotations):
             Annotation, [Annotation.user_id == user_id, Annotation.dataset_id == dataset_id]
         )
         if annotation_object:
-            if annotation_object.tiledb_uri == "":
-                # this mean the user has removed all the categories.
-                return None
-            try:
-                df = tiledb.open(annotation_object.tiledb_uri)
-            except tiledb.TileDBError:
-                # don't crash if the annotations file is missing or can't be read.
-                current_app.logger.warning(f"Cannot read annotation file: {annotation_object.tiledb_uri}")
-                return None
+            df = tiledb.open(annotation_object.tiledb_uri)
             pandas_df = self.convert_to_pandas_df(df, annotation_object.schema_hints)
             return pandas_df
         else:
@@ -117,8 +103,8 @@ class AnnotationsHostedTileDB(Annotations):
         return new_df
 
     def write_labels(self, df, data_adaptor):
-        auth_user_id = self.get_user_id()
-        user_name = self.get_user_name()
+        auth_user_id = current_app.auth.get_user_id()
+        user_name = current_app.auth.get_user_name()
         timestamp = time.time()
         dataset_location = data_adaptor.get_location()
         dataset_id = self.db.get_or_create_dataset(dataset_location)
@@ -138,22 +124,20 @@ class AnnotationsHostedTileDB(Annotations):
         else:
             os.makedirs(uri, exist_ok=True)
         _, dataframe_schema_type_hints = get_dtypes_and_schemas_of_dataframe(df)
-        if not df.empty:
-            self.check_category_names(df)
-            # convert to tiledb datatypes
-
-            for col in df:
-                df[col] = df[col].astype(get_dtype_of_array(df[col]))
-            tiledb.from_pandas(uri, df, sparse=True)
-        else:
-            uri = ""
-
         annotation = Annotation(
             tiledb_uri=uri,
             user_id=user_id,
             dataset_id=str(dataset_id),
             schema_hints=json.dumps(dataframe_schema_type_hints),
         )
+        if not df.empty:
+            self.check_category_names(df)
+            # convert to tiledb datatypes
+
+            for col in df:
+                df[col] = df[col].astype(get_dtype_of_array(df[col]))
+            tiledb.from_pandas(uri, df)
+
         self.db.session.add(annotation)
         self.db.session.commit()
 

server/common/corpora.py

@@ -63,9 +63,9 @@ def corpora_get_props_from_anndata(adata):
             raise KeyError(f"missing Corpora schema field {key}")
         corpora_props[key] = adata.uns[key]
 
-    for key in CorporaConstants.OPTIONAL_JSON_ENCODED_METADATA_FIELD:
+    for key in CorporaConstants.REQUIRED_JSON_ENCODED_METADATA_FIELD:
         if key not in adata.uns:
-            continue
+            raise KeyError(f"missing Corpora schema field {key}")
         try:
             corpora_props[key] = json.loads(adata.uns[key])
         except json.JSONDecodeError:

server/common/rest.py

@@ -2,7 +2,6 @@ import copy
 import logging
 import sys
 from http import HTTPStatus
-import zlib
 
 from flask import make_response, jsonify, current_app, abort
 from werkzeug.urls import url_unquote
@@ -156,21 +155,17 @@ def annotations_put_fbs_helper(data_adaptor, fbs):
 
     new_label_df = decode_matrix_fbs(fbs)
     if not new_label_df.empty:
-        new_label_df = data_adaptor.check_new_labels(new_label_df)
+        data_adaptor.check_new_labels(new_label_df)
     annotations.write_labels(new_label_df, data_adaptor)
 
 
-def inflate(data):
-    return zlib.decompress(data)
-
-
 def annotations_obs_put(request, data_adaptor):
     annotations = data_adaptor.dataset_config.user_annotations
     if annotations is None:
         return abort(HTTPStatus.NOT_IMPLEMENTED)
 
     anno_collection = request.args.get("annotation-collection-name", default=None)
-    fbs = inflate(request.get_data())
+    fbs = request.get_data()
 
     if anno_collection is not None:
         if not annotations.is_safe_collection_name(anno_collection):

server/common/utils/corpora_constants.py

@@ -5,18 +5,12 @@ class CorporaConstants(object):
         "layer_descriptions",
         "organism",
         "organism_ontology_term_id",
+        "project_name",
+        "project_description",
     ]
 
     # The Corpora specification requires some values encoded as JSON due to the inability of AnnData to store complex
     # types.
-    OPTIONAL_JSON_ENCODED_METADATA_FIELD = ["contributors", "project_links"]
+    REQUIRED_JSON_ENCODED_METADATA_FIELD = ["contributors", "project_links"]
 
-    OPTIONAL_SIMPLE_METADATA_FIELDS = [
-        "preprint_doi",
-        "publication_doi",
-        "default_embedding",
-        "default_field",
-        "tags",
-        "project_name",
-        "project_description",
-    ]
+    OPTIONAL_SIMPLE_METADATA_FIELDS = ["preprint_doi", "publication_doi", "default_embedding", "default_field", "tags"]

server/common/utils/type_conversion_utils.py

@@ -44,7 +44,7 @@ def get_dtype_from_dtype(dtype, array_values=None):
     if dtype_name == "bool":
         return np.uint8
     if dtype_name == "object" and dtype_kind == "O":
-        return str
+        return np.unicode
     if dtype_name == "category":
         return get_dtype_from_dtype(dtype.categories.dtype, array_values)
 
@@ -129,9 +129,9 @@ def can_cast_to_int32(dtype, array_values=None):
             return True
         ii32 = np.iinfo(np.int32)
         if (
-            not ordered_array_values.empty
-            and (ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max)
-            or ordered_array_values.empty
+                not ordered_array_values.empty
+                and (ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max)
+                or ordered_array_values.empty
         ):
             return True
     return False

server/converters/schema/gene_symbol.py

@@ -1,211 +0,0 @@
-"""Helpers for converting and checking HGNC gene symbols."""
-
-import argparse
-import enum
-import logging
-import os
-import re
-import numpy as np
-import pandas as pd
-
-
-def get_upgraded_var_index(var, hgnc_path=None):
-    """Given an anndata var dataframe, return a new index for the dataframe
-    where human gene symbols have been upgraded to the current HGNC set.
-    """
-
-    if not hgnc_path:
-        hgnc_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "hgnc_complete_set.txt.gz")
-
-    hgnc_symbol_checker = HGNCSymbolChecker.from_hgnc_records(hgnc_path)
-
-    return pd.Index([hgnc_symbol_checker.upgrade_symbol(s) for s in var.index])
-
-
-class SymbolStatus(enum.Enum):
-    """The status of a symbol in the HGNC database.
-
-    APPROVED: Currently a valid symbol
-    WITHDRAWN: A previously approved HGNC symbol for a gene that has since been shown
-               not to exist _unless_ that symbol is also approved
-    AMBIGUOUS: A symbol that is not approved but is an alias or previous symbol for
-               multiple approved symbols
-    UPGRADABLE: A symbol that is not approved but unambiguously maps to an approved
-                symbol
-    UNKNOWN: A symbol that does not appear in HGNC
-    """
-
-    APPROVED = 1
-    WITHDRAWN = 2
-    AMBIGUOUS = 3
-    UPGRADABLE = 4
-    UNKNOWN = 5
-
-
-class HGNCSymbolChecker:
-    """Handle checking and correcting HGNC symbols."""
-
-    def __init__(self, approved_symbols, withdrawn_symbols, ambiguous_symbols, symbol_map):
-        self.approved_symbols = approved_symbols
-        self.withdrawn_symbols = withdrawn_symbols
-        self.ambiguous_symbols = ambiguous_symbols
-        self.symbol_map = symbol_map
-
-    def print_symbol_map(self):
-        """Print out a map from old symbol to new symbol."""
-
-        for symbol_pair in self.symbol_map.items():
-            print("\t".join(symbol_pair))
-
-    def check_symbol(self, symbol):
-        """See if a symbol if approved or something else."""
-        if symbol in self.approved_symbols:
-            return SymbolStatus.APPROVED
-
-        if symbol in self.withdrawn_symbols:
-            return SymbolStatus.WITHDRAWN
-
-        if symbol in self.ambiguous_symbols:
-            return SymbolStatus.AMBIGUOUS
-
-        if symbol in self.symbol_map:
-            return SymbolStatus.UPGRADABLE
-
-        return SymbolStatus.UNKNOWN
-
-    def upgrade_symbol(self, symbol):
-        """Return the approved symbol for the given symbol.
-
-        If the symbol cannot be upgraded, just return the original symbol.
-        """
-
-        fixed_symbol, stripped_symbol = format_symbol(symbol)
-
-        if fixed_symbol in self.approved_symbols:
-            return fixed_symbol
-        elif fixed_symbol in self.symbol_map:
-            return self.symbol_map[fixed_symbol]
-        elif stripped_symbol in self.approved_symbols:
-            return stripped_symbol
-        elif stripped_symbol in self.symbol_map:
-            return self.symbol_map[stripped_symbol]
-
-        return symbol
-
-    @classmethod
-    def from_hgnc_records(cls, hgnc_dataset_path):
-        """Parse a hgnc database download into a HGNCSymbolChecker object."""
-
-        def all_symbols(record):
-            """Get all the symbols associated with an HGNC record including previous, alias,
-            and approved."""
-            yield format_symbol(record["symbol"])[0]
-            for symbol in alias_and_previous_symbols(record):
-                yield symbol
-
-        def alias_and_previous_symbols(record):
-            """Get alias and previous symbols from an HGNC record."""
-            for field in ("alias_symbol", "prev_symbol"):
-                if record[field] is not np.nan:
-                    for symbol in record[field].split("|"):
-                        yield format_symbol(symbol)[0]
-            # Sometimes something like HGNC:1234 appears in datasets, which we
-            # want to fix as well.
-            yield record["hgnc_id"]
-
-        hgnc_records = pd.read_csv(hgnc_dataset_path, sep="\t", header=0, low_memory=False).to_dict("records")
-
-        # Get all symbols that are currently approved.
-        approved_symbols = set()
-        for record in hgnc_records:
-            if record["status"] == "Approved":
-                approved_symbols.add(format_symbol(record["symbol"])[0])
-
-        # Get all symbols that have been withdrawn
-        withdrawn_symbols = set()
-        for record in hgnc_records:
-            if record["status"] == "Entry Withdrawn":
-                for symbol in all_symbols(record):
-                    withdrawn_symbols.add(symbol)
-
-        # If a symbol is both approved and withdrawn, be optimistic and call it approved
-        logging.warning(
-            f"Some symbols are simulaneously withdrawn and approved\n"
-            f"We will treat them at approved:\n"
-            f"{withdrawn_symbols.intersection(approved_symbols)}"
-        )
-        withdrawn_symbols = withdrawn_symbols.difference(approved_symbols)
-
-        # Now try to map from symbols that are not approved but are an alias or previous symbol for an approved symbol
-        alias_previous_to_approved = {}
-        ambiguous_symbols = set()
-
-        for record in hgnc_records:
-            if record["status"] == "Approved":
-
-                # The approved symbol is what we'll map to
-                approved_symbol = format_symbol(record["symbol"])[0]
-
-                for symbol in alias_and_previous_symbols(record):
-
-                    # If the alias or previous symbol is also an approved symbol,
-                    # we'll just leave it alone
-                    if symbol in approved_symbols:
-                        continue
-
-                    # If the alias or previous symbol maps to a different approved symbol, mark it as ambiguous
-                    if symbol in alias_previous_to_approved and alias_previous_to_approved[symbol] != approved_symbol:
-                        ambiguous_symbols.add(symbol)
-                    else:
-                        alias_previous_to_approved[symbol] = approved_symbol
-
-        # Remove all the ambiguous symbols from the map
-        for ambiguous_symbol in ambiguous_symbols:
-            alias_previous_to_approved.pop(ambiguous_symbol)
-
-        return HGNCSymbolChecker(approved_symbols, withdrawn_symbols, ambiguous_symbols, alias_previous_to_approved)
-
-
-def format_symbol(symbol):
-    """HGNC rules say symbols should all be upper case except for C#orf#. However, case is
-    variable in both alias and previous symbols as well as in the symbols we get in
-    submissions. So, upper case everything except for the one situation where mixed-case
-    is allowed, which are the genes like C2orf157.
-
-    Also, seurat and scanpy append ".1" or "-1" to duplicated gene names, and these altered
-    names persist throughout the life of the object. They won't match against the HGNC database
-    and we want to merge them, so we need to strip off the suffix and try matching again.
-
-    This function takes a symbol and returns the symbol with the fixed case and also with the
-    seurat/scanpy suffix stripped off.
-    """
-
-    match = re.match(r"^(C)(\d+)(orf)(\d+)$", symbol, re.IGNORECASE)
-
-    if match:
-        fixed_case = f"C{match.group(2)}orf{match.group(4)}"
-    else:
-        fixed_case = symbol.upper()
-
-    suffix_stripped = re.sub(r"[\.\-]\d+$", "", fixed_case)
-
-    return fixed_case, suffix_stripped
-
-
-def main():
-    """When called as main, parse a given hgnc download and print out a map from old to new
-    symbol.
-    """
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "hgnc_dataset", help="HGNC dataset tsv, available from www.genenames.org/download/statistics-and-files/"
-    )
-    args = parser.parse_args()
-
-    hgnc_symbol_checker = HGNCSymbolChecker.from_hgnc_records(args.hgnc_dataset)
-
-    hgnc_symbol_checker.print_symbol_map()
-
-
-if __name__ == "__main__":
-    main()

server/converters/schema/ontology.py

@@ -1,86 +0,0 @@
-"""Methods for working with ontologies and the OLS."""
-from urllib.parse import quote_plus
-
-import requests
-
-OLS_API_ROOT = "http://www.ebi.ac.uk/ols/api"
-
-# Curie means something like CL:0000001
-
-
-def _ontology_name(curie):
-    """Get the name of the ontology from the curie, CL or UBERON for example."""
-    return curie.split(":")[0]
-
-
-def _ontology_value(curie):
-    """Get the id component of the curie, 0000001 from CL:0000001 for example."""
-    return curie.split(":")[1]
-
-
-def _double_encode(url):
-    """Double url encode a url. This is required by the OLS API."""
-    return quote_plus(quote_plus(url))
-
-
-def _iri(curie):
-    """Get the iri from a curie. This is a bit hopeful that they all map to purl.obolibrary.org"""
-    if _ontology_name(curie) == "EFO":
-        return f"http://www.ebi.ac.uk/efo/EFO_{_ontology_value(curie)}"
-    return f"http://purl.obolibrary.org/obo/{_ontology_name(curie)}_{_ontology_value(curie)}"
-
-
-class OntologyLookupError(Exception):
-    """Exception for some problem with looking up ontology information."""
-
-
-def _ontology_info_url(curie):
-    """Get the to make a GET to to get information about an ontology term."""
-
-    # If the curie is empty, just return an empty string. This happens when there is no
-    # valid ontology value.
-    if not curie:
-        return ""
-    else:
-        return f"{OLS_API_ROOT}/ontologies/{_ontology_name(curie)}/terms/{_double_encode(_iri(curie))}"
-
-
-def get_ontology_label(curie):
-    """For a given curie like 'CL:1000413', get the label like 'endothelial cell of artery'"""
-
-    url = _ontology_info_url(curie)
-
-    if not url:
-        return ""
-
-    response = requests.get(url)
-
-    if not response.ok:
-        raise OntologyLookupError(
-            f"Curie {curie} lookup failed, got status code {response.status_code}: {response.text}"
-        )
-    return response.json()["label"]
-
-
-def lookup_candidate_term(label, ontology="cl", method="select"):
-    """Lookup candidate terms for a label. This is useful when there is an existing label in a
-    submitted dataset, and you want to find an appropriate ontology term.
-
-    Args:
-      label: the label to find ontology terms for
-      ontology: the ontology to search in, cl or uberon or efo for example
-      method: select or search. search provides much broader results
-
-    Returns:
-      list of (curie, label) tuples returned by OLS
-    """
-    # using OLS REST API [https://www.ebi.ac.uk/ols/docs/api]
-    url = f"{OLS_API_ROOT}/{method}?q={quote_plus(label)}&ontology={ontology.lower()}"
-    response = requests.get(url)
-
-    if not response.ok:
-        raise OntologyLookupError(
-            f"Label {label} lookup failed, got status code {response.status_code}: {response.text}"
-        )
-
-    return [(r["obo_id"], r["label"]) for r in response.json()["response"]["docs"]]

server/converters/schema/remix.py

@@ -1,264 +0,0 @@
-import argparse
-import collections
-import json
-import logging
-import math
-import string
-
-import anndata
-import numpy as np
-import pandas as pd
-import yaml
-
-from . import gene_symbol
-from . import ontology
-from . import validate
-
-REPLACE_SUFFIX = "_original"
-ONTOLOGY_SUFFIX = "_ontology_term_id"
-
-
-def is_curie(value):
-    """Return True iff the value is an OBO-id CURIE like EFO:000001"""
-    return (value.count(":")
-            and all(len(part) > 0 for part in value.split(":"))
-            and all(c in string.digits for c in value.split(":")[1]))
-
-
-def is_ontology_field(field_name):
-    """Return True iff the field_name is an ontology field like tissue_ontology_term_id"""
-    return field_name.endswith(ONTOLOGY_SUFFIX)
-
-
-def get_label_field_name(field_name):
-    """Get the associated label field from an ontology field, assay_ontology_term_id --> assay"""
-    return field_name[: -len(ONTOLOGY_SUFFIX)]
-
-
-def split_suffix(maybe_curie):
-    """Split off the (cell culture) or (organoid) suffix."""
-
-    suffixes = [" (cell culture)", " (organoid)"]
-    for suffix in suffixes:
-        if maybe_curie.endswith(suffix):
-            return maybe_curie[:-len(suffix)], suffix
-    return maybe_curie, ""
-
-
-def get_curie_and_label(maybe_curie):
-    """Given a string that might be a curie, return a (curie, label) pair"""
-
-    maybe_curie, suffix = split_suffix(maybe_curie)
-    if not is_curie(maybe_curie):
-        return ("", maybe_curie + suffix)
-    return (maybe_curie + suffix, ontology.get_ontology_label(maybe_curie) + suffix)
-
-
-def safe_add_field(adata_attr, field_name, field_value):
-    """Add a field and value to an AnnData, but don't clobber an exising value."""
-
-    if (
-        isinstance(field_value, list)
-        and field_value
-        and isinstance(field_value[0], dict)
-    ):
-        field_value = json.dumps(field_value)
-    if field_name in adata_attr:
-        adata_attr[field_name + REPLACE_SUFFIX] = adata_attr[field_name]
-    adata_attr[field_name] = field_value
-
-
-def remix_uns(adata, uns_config):
-    """Add fields from the config to adata.uns"""
-    for field_name, field_value in uns_config.items():
-
-        if is_ontology_field(field_name):
-            # If it's an ontology field, look it up
-            label_field_name = get_label_field_name(field_name)
-            ontology_term, ontology_label = get_curie_and_label(field_value)
-            safe_add_field(adata.uns, field_name, ontology_term)
-            safe_add_field(adata.uns, label_field_name, ontology_label)
-        else:
-            safe_add_field(adata.uns, field_name, field_value)
-
-
-def remix_obs(adata, obs_config):
-    """Add fields from the config to adata.obs"""
-
-    for field_name, field_value in obs_config.items():
-
-        if isinstance(field_value, dict):
-            # If the value is a dict, that means we are supposed to map from an
-            # existing column to the new one
-            source_column, column_map = next(iter(field_value.items()))
-            nan_value = None
-            for key in column_map:
-                if isinstance(key, float) and math.isnan(key):
-                    nan_value = column_map[key]
-            if nan_value is not None:
-                column_map["nan"] = nan_value
-
-            for key in column_map:
-                if key not in adata.obs[source_column].unique():
-                    logging.warning(f'Key {key} not in adata.obs["{source_column}"]')
-
-            for value in adata.obs[source_column].unique():
-                if value not in column_map:
-                    logging.warning(f'Value {value} in adata.obs["{source_column}"] not in translation dict')
-
-            if is_ontology_field(field_name):
-                ontology_term_map, ontology_label_map = {}, {}
-                logging.info(f"Looking up labels for {field_name}")
-                for original_value, maybe_curie in column_map.items():
-                    curie, label = get_curie_and_label(maybe_curie)
-                    ontology_term_map[original_value] = curie
-                    ontology_label_map[original_value] = label
-                    logging.info(f"Mapping {original_value} -> {curie} -> {label}")
-
-                ontology_column = adata.obs[source_column].replace(
-                    ontology_term_map, inplace=False
-                )
-                label_column = adata.obs[source_column].replace(
-                    ontology_label_map, inplace=False
-                )
-
-                safe_add_field(adata.obs, field_name, ontology_column)
-                safe_add_field(
-                    adata.obs, get_label_field_name(field_name), label_column
-                )
-            else:
-                label_column = adata.obs[source_column].replace(
-                    column_map, inplace=False
-                )
-                safe_add_field(adata.obs, field_name, label_column)
-
-        else:
-            if is_ontology_field(field_name):
-                # If it's an ontology field, look it up
-                label_field_name = get_label_field_name(field_name)
-                ontology_term, ontology_label = get_curie_and_label(field_value)
-                safe_add_field(adata.obs, field_name, ontology_term)
-                safe_add_field(adata.obs, label_field_name, ontology_label)
-            else:
-                safe_add_field(adata.obs, field_name, field_value)
-
-
-def merge_df(df, domain, index, columns):
-    """
-    Given a dataframe with duplicate column labels, merge and return a dataframe where
-    the duplicates have been merged together, resulting in a dataframe with unique column
-    labels.
-
-    "merge" depends on the value of domain. If the domain is "raw", then duplicate columns
-    can just be summed. If it's "log1p" or "sqrt", it needs to be exp1m'd or squared, then
-    summed, and then logged or sqrt'd again.
-    """
-
-    if not isinstance(df, np.ndarray):
-        to_merge = df.toarray()
-    else:
-        to_merge = df
-    if domain == "raw":
-        merged_df = pd.DataFrame(to_merge, index=index, columns=columns).sum(
-            axis=1, level=0, skipna=False
-        )
-    elif domain == "log1p":
-        merged_df = (
-            pd.DataFrame(np.expm1(to_merge, dtype=np.float128), index=index, columns=columns)
-            .sum(axis=1, level=0, skipna=False)
-        )
-        merged_df = pd.DataFrame(np.log1p(merged_df.to_numpy()), index=merged_df.index, columns=merged_df.columns)
-    elif domain == "sqrt":
-        merged_df = (
-            pd.DataFrame(np.square(to_merge), index=index, columns=columns)
-            .sum(axis=1, level=0, skipna=False)
-        )
-        merged_df = pd.DataFrame(np.sqrt(merged_df.to_numpy()), index=merged_df.index, columns=merged_df.columns)
-
-    return merged_df
-
-
-def fixup_gene_symbols(adata, fixup_config):
-    """Update the var index to hold a consistent set of HGNC gene symbols."""
-
-    upgraded_var_index = gene_symbol.get_upgraded_var_index(adata.var)
-
-    merged_X = merge_df(adata.X, fixup_config["X"], adata.obs.index, upgraded_var_index)
-    fixup_adata = anndata.AnnData(
-        X=merged_X,
-        obs=adata.obs,
-        var=merged_X.columns.to_frame(name="hgnc_gene_symbol"),
-        uns=adata.uns,
-        obsm=adata.obsm,
-    )
-
-    for layer, domain in fixup_config.items():
-        if layer == "X":
-            continue
-        if layer == "raw.X":
-            df = adata.raw.X
-        else:
-            df = adata.layers[layer]
-
-        merged_df = merge_df(df, domain, adata.obs.index, upgraded_var_index)
-        assert merged_df.index.equals(merged_X.index)
-        assert merged_df.columns.equals(merged_X.columns)
-
-        if domain == "raw":
-            fixup_raw = anndata.AnnData(
-                X=merged_df,
-                obs=adata.obs,
-                var=merged_X.columns.to_frame(name="hgnc_gene_symbol"),
-            )
-            fixup_adata.raw = fixup_raw
-        else:
-            fixup_adata.layers[layer] = merged_df
-
-    return fixup_adata
-
-def _strip_version(adata):
-    """Remove version information from the AnnData object."""
-
-    if "version" in adata.uns_keys():
-        del adata.uns["version"]
-
-def apply_schema(source_h5ad, remix_config, output_filename):
-
-    try:
-        import scanpy
-    except ImportError:
-        raise ImportError("scanpy must be installed for cellxgene schema")
-    adata = scanpy.read_h5ad(source_h5ad)
-    config = yaml.load(open(remix_config), Loader=yaml.FullLoader)
-    remix_uns(adata, config["uns"])
-    remix_obs(adata, config["obs"])
-
-    if config.get("fixup_gene_symbols"):
-        adata = fixup_gene_symbols(adata, config["fixup_gene_symbols"])
-
-    if ("version" in adata.uns_keys()
-            and isinstance(adata.uns["version"], collections.Mapping)
-            and "corpora_schema_version" in adata.uns["version"]):
-        schema_version = adata.uns["version"]["corpora_schema_version"]
-        try:
-            validate.get_schema_definition(schema_version)
-        except ValueError:
-            logging.warning(f"Stripping version information out of AnnData because schema "
-                            f"version {schema_version} is unknown.")
-            _strip_version(adata)
-
-        if not validate.validate_adata(adata, shallow=False):
-            logging.warning(f"Stripping version information out of AnnData because it does not "
-                            f"follow schema version {schema_version} .")
-            _strip_version(adata)
-
-    adata.write_h5ad(output_filename, compression="gzip")
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--source-h5ad", required=True)
-    parser.add_argument("--remix-config", required=True)
-    parser.add_argument("--output-filename", required=True)
-    args = parser.parse_args()
-    apply_schema(args.source_h5ad, args.remix_config, args.output_filename)

server/converters/schema/schema_definitions/1_0_0.yaml

@@ -1,95 +0,0 @@
-title: Corpora schema version 1.0.0
-type: anndata
-components:
-    uns:
-        type: dict
-        keys:
-            version:
-                type: dict
-                keys:
-                    corpora_schema_version: null
-                    corpora_encoding_version: null
-            title:
-                type: string
-            contributors:
-                type: stringified list of dicts
-            layer_descriptions:
-                type: dict
-                keys:
-                    X: null
-            organism:
-                type: string
-                nullable: false
-            organism_ontology_term_id:
-                type: curie
-                prefixes:
-                    - NCBITaxon
-    var:
-        type: dataframe
-        index:
-            type: human-readable string
-            unique: true
-    obs:
-        type: dataframe
-        index:
-            unique: true
-        columns:
-            tissue:
-                type: human-readable string
-                nullable: false
-            tissue_ontology_term_id:
-                type: suffixed curie
-                nullable: true
-                prefixes:
-                    - UBERON
-            assay:
-                type: human-readable string
-                nullable: false
-            assay_ontology_term_id:
-                type: curie
-                nullable: true
-                prefixes:
-                    - EFO
-            disease:
-                type: human-readable string
-                nullable: false
-            disease_ontology_term_id:
-                type: curie
-                nullable: true
-                prefixes:
-                    - MONDO
-                    - PATO
-            cell_type:
-                type: human-readable string
-                nullable: false
-            cell_type_ontology_term_id:
-                type: curie
-                nullable: true
-                prefixes:
-                    - CL
-                    - UBERON
-            sex:
-                type: string
-                enum:
-                    - male
-                    - female
-                    - mixed
-                    - unknown
-                    - other
-            ethnicity:
-                type: human-readable string
-                nullable: false
-            ethnicity_ontology_term_id:
-                type: curie
-                nullable: true
-                prefixes:
-                    - HANCESTRO
-            development_stage:
-                type: human-readable string
-                nullable: false
-            development_stage_ontology_term_id:
-                type: curie
-                nullable: true
-                prefixes:
-                    - HsapDv
-                    - EFO

server/converters/schema/schema_definitions/1_1_0.yaml

@@ -1,93 +0,0 @@
-title: Corpora schema version 1.1.0
-type: anndata
-components:
-    uns:
-        type: dict
-        keys:
-            version:
-                type: dict
-                keys:
-                    corpora_schema_version: null
-                    corpora_encoding_version: null
-            title:
-                type: string
-            layer_descriptions:
-                type: dict
-                keys:
-                    X: null
-            organism:
-                type: string
-                nullable: false
-            organism_ontology_term_id:
-                type: curie
-                prefixes:
-                    - NCBITaxon
-    var:
-        type: dataframe
-        index:
-            type: human-readable string
-            unique: true
-    obs:
-        type: dataframe
-        index:
-            unique: true
-        columns:
-            tissue:
-                type: human-readable string
-                nullable: false
-            tissue_ontology_term_id:
-                type: suffixed curie
-                nullable: true
-                prefixes:
-                    - UBERON
-            assay:
-                type: human-readable string
-                nullable: false
-            assay_ontology_term_id:
-                type: curie
-                nullable: true
-                prefixes:
-                    - EFO
-            disease:
-                type: human-readable string
-                nullable: false
-            disease_ontology_term_id:
-                type: curie
-                nullable: true
-                prefixes:
-                    - MONDO
-                    - PATO
-            cell_type:
-                type: human-readable string
-                nullable: false
-            cell_type_ontology_term_id:
-                type: curie
-                nullable: true
-                prefixes:
-                    - CL
-                    - UBERON
-            sex:
-                type: string
-                enum:
-                    - male
-                    - female
-                    - mixed
-                    - unknown
-                    - other
-            ethnicity:
-                type: human-readable string
-                nullable: false
-            ethnicity_ontology_term_id:
-                type: curie
-                nullable: true
-                prefixes:
-                    - HANCESTRO
-            development_stage:
-                type: human-readable string
-                nullable: false
-            development_stage_ontology_term_id:
-                type: curie
-                nullable: true
-                prefixes:
-                    - HsapDv
-                    - EFO

server/converters/schema/validate.py

@@ -1,236 +0,0 @@
-import json
-import re
-import os
-import sys
-
-import pandas as pd
-import yaml
-
-
-def _is_null(v):
-    """Return True if v is null, for one of the multiple ways a "null" value shows up in an h5ad."""
-    return pd.isnull(v) or (hasattr(v, "__len__") and len(v) == 0)
-
-
-def _validate_stringified_list_of_dicts(s):
-    """Verify that a string can be parsed into a list.
-
-    We have some types that are lists of dicts. Those cannot be stored directly in an h5ad, so we have to
-    json.dumps them. This verifies that we can load them back.
-    """
-
-    try:
-        list_ = json.loads(s)
-        if not isinstance(list_, list):
-            return False
-        for el in list_:
-            if not isinstance(el, dict):
-                return False
-        return True
-    except (json.JSONDecodeError, TypeError):
-        pass
-    return False
-
-
-def _validate_human_readable_string(s):
-    """Verify that a string is human-readable.
-
-    There are parts of the schema where a "human-readable" string is required. "Human-readable" is kind
-    of vague and subjective. I feel like I can read many strings. So here we just check for the main ways
-    that fails: someone puts in an ontology term id or and ensembl gene/transcript id.
-
-    Returns False if s is not a string or is one of those bad string types.
-    """
-
-    return isinstance(s, str) and (not re.match(r"[A-Z]\w+:\d+", s)) and (not re.match(r"ENS[GT]\d+$", s))
-
-
-def _validate_curie(c, prefixes):
-    """Verify that a string is a valid compact URI, like EFO:000001. If prefixes is not empty, make sure the
-    prefix of the curies is in prefixes.
-    """
-
-    if not c:
-        return True
-
-    match = re.match(r"([A-Z]\w+):\d+$", c)
-
-    if prefixes:
-        return match and match.group(1) in prefixes
-    else:
-        return match
-
-
-def _validate_suffixed_curie(c, prefixes):
-    """Verify that a string is a compact URI with an optional suffix like 'EFO:00001 (cell culture)'"""
-
-    # Pull off the suffix
-    suffix = re.findall(r"\ \(.*\)$", c)
-    if suffix:
-        c = c[: -len(suffix[0])]
-    return _validate_curie(c, prefixes)
-
-
-def _validate_column(column, column_name, df_name, schema_def):
-    """Given a schema definition and the column of a dataframe, verify that the column satifies
-    the schema.
-    """
-
-    errors = []
-
-    if schema_def.get("unique"):
-        if column.nunique() != len(column):
-            errors.append(f"Column {column_name} in dataframe {df_name} is not unique.")
-
-    if "nullable" in schema_def and not schema_def["nullable"]:
-        if any(_is_null(v) for v in column):
-            errors.append(f"Column {column_name} in dataframe {df_name} contains empty values.")
-
-    if schema_def.get("type") == "human-readable string":
-        non_readables = [v for v in column if not _validate_human_readable_string(v)]
-        if non_readables:
-            errors.append(
-                f"Column {column_name} in dataframe {df_name} contains non-human-readable "
-                f"values like {non_readables[0]}"
-            )
-
-    if schema_def.get("type") in ("curie", "suffixed curie"):
-        validation_func = _validate_curie if schema_def.get("type") == "curie" else _validate_suffixed_curie
-        non_valid_curies = [v for v in column if not validation_func(v, schema_def.get("prefixes"))]
-        if non_valid_curies:
-            errors.append(
-                f"Column {column_name} in dataframe {df_name} contains invalid ontology values like "
-                f"{non_valid_curies[0]}."
-            )
-            if "prefixes" in schema_def:
-                errors[-1] += f" Values must be curies from one of these ontologies {schema_def['prefixes']}."
-
-    if "enum" in schema_def:
-        bad_enums = [v for v in column if v not in schema_def["enum"]]
-        if bad_enums:
-            errors.append(
-                f"Column {column_name} in dataframe {df_name} contains unpermitted values like "
-                f"{bad_enums[0]}. Values must be one of {schema_def['enum']}."
-            )
-
-    return errors
-
-
-def _validate_dict(dict_, dict_name, schema_def):
-    """Given a schema definition and dict, verify that the dict satifies the schema."""
-
-    errors = []
-
-    for key in schema_def.get("keys", []):
-        if key not in dict_:
-            errors.append(f"{dict_name} is missing key {key}.")
-        elif schema_def["keys"][key]:
-            if schema_def["keys"][key]["type"] == "stringified list of dicts":
-                if not _validate_stringified_list_of_dicts(dict_[key]):
-                    errors.append(
-                        f"Key {key} in {dict_name} should be a JSON-encoded list of dicts, but it is {dict_[key]}"
-                    )
-            elif schema_def["keys"][key]["type"] == "dict":
-                errors.extend(_validate_dict(dict_[key], key, schema_def["keys"][key]))
-            elif schema_def["keys"][key]["type"] == "curie":
-                if not _validate_curie(dict_[key], schema_def["keys"][key]["prefixes"]):
-                    errors.append(f"Key {key} in {dict_name} contains invalid ontology value.")
-            if "nullable" in schema_def["keys"][key] and not schema_def["keys"][key]["nullable"]:
-                if _is_null(dict_[key]):
-                    errors.append(f"Key {key} in dict {dict_name} is an empty value.")
-
-    return errors
-
-
-def _validate_dataframe(df, df_name, schema_def):
-    """Given a dataframe and schema definition, verify that the dataframe follows the schema."""
-
-    errors = []
-
-    if "index" in schema_def:
-        errors.extend(_validate_column(df.index, "index", df_name, schema_def["index"]))
-
-    for column in schema_def.get("columns", []):
-        if column not in df.columns:
-            errors.append(f"Dataframe {df_name} is missing column {column}.")
-        else:
-            errors.extend(_validate_column(df[column], column, df_name, schema_def["columns"][column]))
-
-    return errors
-
-
-def get_schema_definition(version):
-    """Look up and read a schema definition based on a version number like "1.0.0"."""
-
-    path = os.path.join(
-        os.path.dirname(os.path.realpath(__file__)), "schema_definitions", version.replace(".", "_") + ".yaml"
-    )
-
-    if not os.path.isfile(path):
-        raise ValueError(f"No definition for version {version} found.")
-
-    return yaml.load(open(path), Loader=yaml.FullLoader)
-
-
-def deep_check(adata, schema_def):
-    """Perform a "deep" check of the AnnData object using the schema definition.
-
-    This checks all the columns and unstructured metadata rather than just the version.
-
-    Returns a list of error messages. If that list is empty, the object passed validation.
-    """
-
-    errors = []
-
-    for component, component_def in schema_def["components"].items():
-        if component_def["type"] == "dataframe":
-            errors.extend(_validate_dataframe(getattr(adata, component), component, component_def))
-        elif component_def["type"] == "dict":
-            errors.extend(_validate_dict(getattr(adata, component), component, component_def))
-        else:
-            raise ValueError(f"Unexpected component type {component['type']}")
-
-    return errors
-
-
-def validate_adata(adata, shallow):
-    """Validate an AnnData object. If shallow, just check that the required version information is
-    present.
-    """
-
-    # Does it have the version information written into uns?
-    if "version" not in adata.uns_keys() or "corpora_schema_version" not in adata.uns["version"]:
-        print("AnnData file is missing corpora version information")
-        return False
-
-    # We can stop here if it's a "shallow" check, that is, if we're just
-    # checking that version is present.
-    if shallow:
-        return True
-
-    schema_def = get_schema_definition(adata.uns["version"]["corpora_schema_version"])
-
-    errors = deep_check(adata, schema_def)
-
-    for error in errors:
-        print(error)
-
-    return not errors
-
-
-def validate(h5ad_path, shallow=False):
-    """Entry point for validation."""
-
-    try:
-        import scanpy
-    except ImportError:
-        raise ImportError("scanpy must be installed for cellxgene schema")
-
-    try:
-        adata = scanpy.read_h5ad(h5ad_path, backed="r")
-    except (OSError, TypeError):
-        print(f"Unable to open {h5ad_path} with scanpy.")
-        sys.exit(1)
-
-    if not validate_adata(adata, shallow):
-        sys.exit(1)

server/data_anndata/anndata_adaptor.py

@@ -177,10 +177,10 @@ class AnndataAdaptor(DataAdaptor):
             )
 
     def _validate_and_initialize(self):
-        if anndata_version_is_pre_070():
+        if anndata_version_is_pre_070() and self.server_config.adaptor__anndata_adaptor__backed:
             warnings.warn(
-                "Use of anndata versions older than 0.7 will have serious issues. Please update to at "
-                "least anndata 0.7 or later."
+                "Use of --backed mode with anndata versions older than 0.7 will have serious "
+                "performance issues. Please update to at least anndata 0.7 or later."
             )
 
         # var and obs column names must be unique

server/data_common/data_adaptor.py

@@ -249,23 +249,6 @@ class DataAdaptor(metaclass=ABCMeta):
         if labels_df.shape[0] != shape[0]:
             raise ValueError("Labels file must have same number of rows as data file.")
 
-        # This will convert a float column that contains integer data into an integer type.
-        # This case can occur when a user makes a copy of a category that originally contained integer data.
-        # The client always copies array data to floats, therefore the copy will contain floats instead of integers.
-        # float data is not allowed as a categorical type.
-        if any([np.issubdtype(coltype.type, np.floating) for coltype in labels_df.dtypes]):
-            labels_df = labels_df.convert_dtypes()
-            for col, dtype in zip(labels_df, labels_df.dtypes):
-                if isinstance(dtype, pd.Int32Dtype):
-                    labels_df[col] = labels_df[col].astype("int32")
-                if isinstance(dtype, pd.Int64Dtype):
-                    labels_df[col] = labels_df[col].astype("int64")
-
-        if any([np.issubdtype(coltype.type, np.floating) for coltype in labels_df.dtypes]):
-            raise ValueError("Columns may not have floating point types")
-
-        return labels_df
-
     def data_frame_to_fbs_matrix(self, filter, axis):
         """
         Retrieves data 'X' and returns in a flatbuffer Matrix.

server/eb/Makefile

@@ -27,9 +27,6 @@ build: clean
 	  if [ -f customize/config.yaml ] ; then \
 	     cp customize/config.yaml artifact.dir;  \
 	  fi ; \
-	  if [ -f customize/Dockerfile ] ; then \
-	     cp customize/Dockerfile artifact.dir;  \
-	  fi ; \
 	  if [ -f customize/requirements.txt ] ; then \
 	     pip install requirements-parser ; \
 	     pip install packaging ; \

server/eb/app.py

@@ -185,7 +185,7 @@ else:
 
 if __name__ == "__main__":
     try:
-        application.run(host=app_config.server_config.app__host, debug=debug, threaded=not debug, use_debugger=False)
+        application.run(debug=debug, threaded=not debug, use_debugger=False)
     except Exception:
         logging.critical("Caught exception during initialization", exc_info=True)
         sys.exit(1)

server/requirements-dev.txt

@@ -6,5 +6,6 @@ parameterized>=0.7.0
 psycopg2-binary>=2.8.5
 pytest>=3.6.3
 python-jose>=3.2.0
+scanpy>=1.4.6
 twine>=1.12.1
 -r requirements.txt

server/requirements-prepare.txt

@@ -1,2 +1,3 @@
+scanpy>=1.3.7
 python-igraph
 louvain>=0.6

server/requirements.txt

@@ -1,4 +1,4 @@
-anndata>=0.7.0
+anndata>=0.6.20
 boto3>=1.12.18
 click>=7.1.2
 fastobo>=0.6.1
@@ -12,16 +12,13 @@ flatbuffers>=1.11.0
 flatten-dict>=0.2.0
 fsspec>=0.4.4,<0.8.0
 gunicorn>=20.0.4
-h5py<3.0.0 # h5py>=3.0.0 had a breaking change; there is a fix in anndata>=0.7.5
 numba>=0.49.1
-numpy>=1.15.0
+numpy>=1.16.0
 packaging>=20.0
-pandas>=1.0,!=1.1  # pandas 1.1 breaks tests, https://github.com/pandas-dev/pandas/issues/35446
+pandas>=0.24.2
 PyYAML>=5.3
-scipy>=1.0
+scipy>=1.3.0
 requests>=2.22.0
-tiledb>=0.5.9,>=0.6.2,!=0.7.2
+tiledb>=0.5.9,>=0.6.2
 s3fs==0.4.2
-scanpy==1.4.6 # Until we move to anndata 0.7.4 scanpy needs to be pinned here
 sqlalchemy>=1.3.18
-umap-learn<0.5.0 # The pinned version scanpy is not compatible with latest umap-learn

server/test/fixtures/schema_test_data/generate_test_data.sh

@@ -1,139 +0,0 @@
-#!/bin/bash
-wget "https://s3-us-west-2.amazonaws.com/10x.files/samples/cell/pbmc3k/pbmc3k_filtered_gene_bc_matrices.tar.gz"
-tar xf "pbmc3k_filtered_gene_bc_matrices.tar.gz"
-
-python3 - <<MERGE_GENES
-import os
-from scipy.io import mmread, mmwrite
-import scipy.sparse
-import pandas as pd
-from server.converters.schema import gene_symbol
-
-mat = mmread("filtered_gene_bc_matrices/hg19/matrix.mtx").todense()
-genes = pd.read_csv("filtered_gene_bc_matrices/hg19/genes.tsv", sep='\t', names=["gene_id", "gene_symbol"])
-
-upgraded_genes = gene_symbol.get_upgraded_var_index(pd.DataFrame(index=genes["gene_symbol"]))
-df = pd.DataFrame(data=mat, index=upgraded_genes).T
-merged = df.sum(axis=1, level=0, skipna=False)
-
-os.makedirs("merged")
-merged.columns.to_frame().to_csv("merged/genes.tsv", index=False, header=False)
-mmwrite("merged/matrix.mtx", scipy.sparse.coo_matrix(merged).T)
-MERGE_GENES
-
-cp "filtered_gene_bc_matrices/hg19/barcodes.tsv" "merged/barcodes.tsv"
-awk '{print $1"\t"$1}' merged/genes.tsv > genes_tmp.tsv; mv genes_tmp.tsv merged/genes.tsv
-
-echo -e "\n\n\nRunning tutorial on original\n\n\n"
-Rscript - <<TUTORIAL
-library(Seurat)
-
-pbmc.data <- Read10X(data.dir = "filtered_gene_bc_matrices/hg19/")
-pbmc <- CreateSeuratObject(counts = pbmc.data, project = "pbmc3k", min.features = 200)
-pbmc <- NormalizeData(pbmc, normalization.method = "LogNormalize", scale.factor = 10000)
-pbmc <- FindVariableFeatures(pbmc, selection.method = "vst", nfeatures = 2000)
-pbmc[["percent.mt"]] <- PercentageFeatureSet(pbmc, pattern = "^MT-")
-all.genes <- rownames(pbmc)
-pbmc <- ScaleData(pbmc, features = all.genes)
-
-pbmc <- RunPCA(pbmc, features = VariableFeatures(object = pbmc))
-pbmc <- FindNeighbors(pbmc, dims = 1:10)
-pbmc <- FindClusters(pbmc, resolution = 0.5)
-pbmc <- RunUMAP(pbmc, dims = 1:10)
-saveRDS(pbmc, file = "./seurat_tutorial.rds")
-TUTORIAL
-
-echo -e "\n\n\nRunning tutorial on merged\n\n\n"
-Rscript - <<TUTORIAL_MERGED
-library(Seurat)
-
-pbmc.data <- Read10X(data.dir = "merged/")
-pbmc <- CreateSeuratObject(counts = pbmc.data, project = "pbmc3k", min.features = 200)
-pbmc <- NormalizeData(pbmc, normalization.method = "LogNormalize", scale.factor = 10000)
-pbmc <- FindVariableFeatures(pbmc, selection.method = "vst", nfeatures = 2000)
-pbmc[["percent.mt"]] <- PercentageFeatureSet(pbmc, pattern = "^MT-")
-all.genes <- rownames(pbmc)
-pbmc <- ScaleData(pbmc, features = all.genes)
-
-pbmc <- RunPCA(pbmc, features = VariableFeatures(object = pbmc))
-pbmc <- FindNeighbors(pbmc, dims = 1:10)
-pbmc <- FindClusters(pbmc, resolution = 0.5)
-pbmc <- RunUMAP(pbmc, dims = 1:10)
-saveRDS(pbmc, file = "./seurat_tutorial_merged.rds")
-TUTORIAL_MERGED
-
-echo -e "\n\n\nRunning SCTransform on original\n\n\n"
-Rscript - <<SCTRANSFORM
-library(Seurat)
-library(sctransform)
-
-pbmc.data <- Read10X(data.dir = "filtered_gene_bc_matrices/hg19/")
-pbmc <- CreateSeuratObject(counts = pbmc.data)
-pbmc <- PercentageFeatureSet(pbmc, pattern = "^MT-", col.name = "percent.mt")
-pbmc <- SCTransform(pbmc, vars.to.regress = "percent.mt", verbose = FALSE)
-pbmc <- RunPCA(pbmc, verbose = FALSE)
-pbmc <- RunUMAP(pbmc, dims = 1:30, verbose = FALSE)
-pbmc <- FindNeighbors(pbmc, dims = 1:30, verbose = FALSE)
-pbmc <- FindClusters(pbmc, verbose = FALSE)
-saveRDS(pbmc, file = "./sctransform.rds")
-SCTRANSFORM
-
-echo -e "\n\n\nRunning SCTransform on merged\n\n\n"
-Rscript - <<SCTRANSFORM_MERGED
-library(Seurat)
-library(sctransform)
-
-pbmc.data <- Read10X(data.dir = "merged/")
-pbmc <- CreateSeuratObject(counts = pbmc.data)
-pbmc <- PercentageFeatureSet(pbmc, pattern = "^MT-", col.name = "percent.mt")
-pbmc <- SCTransform(pbmc, vars.to.regress = "percent.mt", verbose = FALSE)
-pbmc <- RunPCA(pbmc, verbose = FALSE)
-pbmc <- RunUMAP(pbmc, dims = 1:30, verbose = FALSE)
-pbmc <- FindNeighbors(pbmc, dims = 1:30, verbose = FALSE)
-pbmc <- FindClusters(pbmc, verbose = FALSE)
-saveRDS(pbmc, file = "./sctransform_merged.rds")
-SCTRANSFORM_MERGED
-
-echo -e "\n\n\nConverting\n\n\n"
-Rscript - <<SCEASY
-library(sceasy)
-srt <- readRDS("seurat_tutorial.rds")
-sceasy::convertFormat(srt,
-                      outFile = "seurat_tutorial.h5ad",
-                      from = "seurat",
-                      to = "anndata",
-                      assay = "RNA",
-                      main_layer = "data",
-                      transfer_layers = c("data", "counts", "scale.data"),
-                      drop_single_values = FALSE)
-
-srt <- readRDS("seurat_tutorial_merged.rds")
-sceasy::convertFormat(srt,
-                      outFile = "seurat_tutorial_merged.h5ad",
-                      from = "seurat",
-                      to = "anndata",
-                      assay = "RNA",
-                      main_layer = "data",
-                      transfer_layers = c("data", "counts", "scale.data"),
-                      drop_single_values = FALSE)
-
-srt <- readRDS("sctransform.rds")
-sceasy::convertFormat(srt,
-                      outFile = "sctransform.h5ad",
-                      from = "seurat",
-                      to = "anndata",
-                      assay = "SCT",
-                      main_layer = "data",
-                      transfer_layers = c("data", "counts", "scale.data"),
-                      drop_single_values = FALSE)
-
-srt <- readRDS("sctransform_merged.rds")
-sceasy::convertFormat(srt,
-                      outFile = "sctransform_merged.h5ad",
-                      from = "seurat",
-                      to = "anndata",
-                      assay = "SCT",
-                      main_layer = "data",
-                      transfer_layers = c("data", "counts", "scale.data"),
-                      drop_single_values = FALSE)
-SCEASY

server/test/fixtures/test_bad_config.yaml

@@ -1,34 +0,0 @@
-fixup_gene_symbols:
-    X: log1p
-obs:
-    cell_type_ontology_term_id:
-        louvain:
-            CD4 T cells: CL:00001
-            B cells: CL:00002
-            CD14+ Monocytes: CL:00003
-            NK cells: CL:00004
-            CD8 T cells: CL:00005
-            FCGR3A+ Monocytes: CL:00006
-            Dendritic cells: CL:00007
-            Megakaryocytes: CL:00008
-    tissue_ontology_term_id: UBERON:12345
-    assay_ontology_term_id: EFO:12345
-    disease_ontology_term_id: MONDO:12345
-    ethnicity_ontology_term_id: MANCESTRO:12345
-    development_stage_ontology_term_id: HsapDv:12345
-    sex: other
-uns:
-    version:
-        corpora_schema_version: 1.0.0
-        corpora_encoding_version: 0.1.0
-    organism_ontology_term_id: NCBITaxon:9606
-    title: Test dataset
-    contributors:
-        - name: Marcus
-          institution: CZI
-    layer_descriptions:
-        X: raw
-    project_links:
-        - link_url: https://chanzuckerberg.com/
-          link_name: CZI
-          link_type: SUMMARY

server/test/fixtures/test_config.yaml

@@ -1,34 +0,0 @@
-fixup_gene_symbols:
-    X: log1p
-obs:
-    cell_type_ontology_term_id:
-        louvain:
-            CD4 T cells: CL:00001
-            B cells: CL:00002
-            CD14+ Monocytes: CL:00003
-            NK cells: CL:00004
-            CD8 T cells: CL:00005
-            FCGR3A+ Monocytes: CL:00006
-            Dendritic cells: CL:00007
-            Megakaryocytes: CL:00008
-    tissue_ontology_term_id: UBERON:12345
-    assay_ontology_term_id: EFO:12345
-    disease_ontology_term_id: MONDO:12345
-    ethnicity_ontology_term_id: HANCESTRO:12345
-    development_stage_ontology_term_id: HsapDv:12345
-    sex: other
-uns:
-    version:
-        corpora_schema_version: 1.0.0
-        corpora_encoding_version: 0.1.0
-    organism_ontology_term_id: NCBITaxon:9606
-    title: Test dataset
-    contributors:
-        - name: Marcus
-          institution: CZI
-    layer_descriptions:
-        X: raw
-    project_links:
-        - link_url: https://chanzuckerberg.com/
-          link_name: CZI
-          link_type: SUMMARY

server/test/locust/config.py

@@ -10,6 +10,12 @@ Locust test config
 
 # multi-dataset, for dataroot tests.  these are varied in size/shape
 DataSets = [
-    "GSE60361.cxg",
-    "WongAdultRetina.cxg",
+    "/d/pbmc3k.cxg",
+    "/d/TM_droplet_processed.cxg",
+    "/d/pancreas.cxg",
+    "/d/immune_bone_marrow_processed.cxg",
+    "/d/10X_mouse_13MM_processed.cxg",
+    "/d/GSE60361.cxg",
+    "/d/Reprogrammed_Dendritic_Cells.cxg",
+    "/d/WongAdultRetina.cxg",
 ]

server/test/locust/locustfile.py

@@ -1,144 +1,143 @@
-import json
+from locust import HttpLocust, TaskSet, TaskSequence, seq_task, task
+from locust.wait_time import between
 import random
-
-import requests
-from config import DataSets
-from locust import HttpUser, SequentialTaskSet, task, between, TaskSet
-from locust.clients import HttpSession
-from requests.packages.urllib3.exceptions import InsecureRequestWarning
+import json
+from gevent.pool import Group
 
 import server.test.unit.decode_fbs as decode_fbs
+from config import DataSets
 
-requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
 
 """
-Simple locust stress test definition for cellxgene
+Simple locust stress test defition for cellxgene
 """
-
-API_SUFFIX = "api/v0.2"
+API = "/api/v0.2"
 
 
-class CellXGeneTasks(TaskSet):
+class ViewDataset(TaskSet):
     """
     Simulate use against a single dataset
     """
 
     def on_start(self):
-
         self.client.verify = False
         self.dataset = random.choice(DataSets)
 
-        with self.client.get(
-            f"{self.dataset}/{API_SUFFIX}/schema", stream=True, catch_response=True
-        ) as schema_response:
-            if schema_response.status_code == 200:
-                self.schema = schema_response.json()["schema"]
-            else:
-                self.schema = None
-
-        with self.client.get(
-            f"{self.dataset}/{API_SUFFIX}/config", stream=True, catch_response=True
-        ) as config_response:
-            if config_response.status_code == 200:
-                self.config = config_response.json()["config"]
+        with self.client.get(f"{self.dataset}{API}/config", catch_response=True) as r:
+            if r.status_code == 200:
+                self.config = r.json()["config"]
+                r.success()
             else:
                 self.config = None
+                r.failure(f"bad response code {r.status_code}")
+
+        with self.client.get(f"{self.dataset}{API}/schema", catch_response=True) as r:
+            if r.status_code == 200:
+                self.schema = r.json()["schema"]
+                r.success()
+            else:
+                self.schema = None
+                r.failure(f"bad response code {r.status_code}")
 
         with self.client.get(
-            f"{self.dataset}/{API_SUFFIX}/annotations/var?annotation-name={self.var_index_name()}",
+            f"{self.dataset}{API}/annotations/var?annotation-name={self.var_index_name()}",
             headers={"Accept": "application/octet-stream"},
             catch_response=True,
-        ) as var_index_response:
-            if var_index_response.status_code == 200:
-                df = decode_fbs.decode_matrix_FBS(var_index_response.content)
+        ) as r:
+            if r.status_code == 200:
+                df = decode_fbs.decode_matrix_FBS(r.content)
                 gene_names_idx = df["col_idx"].index(self.var_index_name())
                 self.gene_names = df["columns"][gene_names_idx]
             else:
-                self.gene_names = []
+                self.gene_names = None
+                r.failure(f"bad response code {r.status_code}")
 
     def var_index_name(self):
-        if self.schema is not None:
-            return self.schema["annotations"]["var"]["index"]
-        return None
+        if self.schema is None:
+            return None
+        return self.schema["annotations"]["var"]["index"]
 
     def obs_annotation_names(self):
-        if self.schema is not None:
-            return [col["name"] for col in self.schema["annotations"]["obs"]["columns"]]
-        return []
-
-    def layout_names(self):
-        if self.schema is not None:
-            return [layout["name"] for layout in self.schema["layout"]["obs"]]
-        else:
+        if self.schema is None:
             return []
+        return [col["name"] for col in self.schema["annotations"]["obs"]["columns"]]
 
     @task(2)
-    class InitializeClient(SequentialTaskSet):
+    class InitializeClient(TaskSequence):
         """
         Initial loading of cellxgene - when the user hits the main route.
 
-        Currently this sequence skips some of the static assets, which are quite small and should be served by the
-        HTTP server directly.
+        Currently this sequence skips some of the static assets, which are quite
+        small and should be served by the HTTP server directly.
 
-        1. Load index.html, etc.
-        2. Concurrently load /config, /schema
-        3. Concurrently load /layout/obs, /annotations/var?annotation-name=<the index>
-        -- Does initial render --
-        4. Concurrently load all /annotations/obs and all /layouts/obs
-        -- Fully initialized --
+        1. load index.html, etc.
+        2. concurrently load /config, /schema
+        3. concurrently load /layout/obs, /annotations/var?annotation-name=<the index>
+        -- does intitial render --
+        4. concurrently load all /annotations/obs
+        -- fully initialized --
         """
 
-        # Users hit all of the init routes as fast as they can, subject to the ordering constraints and network latency.
+        # users hit all of the init routes as fast as they can, subject to the ordering constraints
+        # and network latency
         wait_time = between(0.01, 0.1)
 
         def on_start(self):
             self.dataset = self.parent.dataset
             self.client.verify = False
-            self.api_less_client = HttpSession(
-                base_url=self.client.base_url.replace("api.", "").replace("cellxgene/", ""),
-                request_success=self.client.request_success,
-                request_failure=self.client.request_failure,
-            )
 
-        @task
+        @seq_task(1)
         def index(self):
-            self.api_less_client.get(f"{self.dataset}", stream=True)
+            self.client.get(f"{self.dataset}/", stream=True).close()
 
-        @task
-        def loadConfigAndSchema(self):
-            self.client.get(f"{self.dataset}/{API_SUFFIX}/schema", stream=True, catch_response=True)
-            self.client.get(f"{self.dataset}/{API_SUFFIX}/config", stream=True, catch_response=True)
+        @seq_task(2)
+        def loadConfigSchema(self):
+            def config():
+                self.client.get(f"{self.dataset}{API}/config", stream=True).close()
 
-        @task
+            def schema():
+                self.client.get(f"{self.dataset}{API}/schema", stream=True).close()
+
+            group = Group()
+            group.spawn(config)
+            group.spawn(schema)
+            group.join()
+
+        @seq_task(3)
         def loadBootstrapData(self):
-            self.client.get(
-                f"{self.dataset}/{API_SUFFIX}/layout/obs", headers={"Accept": "application/octet-stream"}, stream=True
-            )
-            self.client.get(
-                f"{self.dataset}/{API_SUFFIX}/annotations/var?annotation-name={self.parent.var_index_name()}",
-                headers={"Accept": "application/octet-stream"},
-                catch_response=True,
-            )
+            def layout():
+                self.client.get(
+                    f"{self.dataset}{API}/layout/obs", headers={"Accept": "application/octet-stream"}, stream=True
+                ).close()
+
+            def varAnnotationIndex():
+                self.client.get(
+                    f"{self.dataset}{API}/annotations/var?annotation-name={self.parent.var_index_name()}",
+                    headers={"Accept": "application/octet-stream"},
+                    stream=True,
+                ).close()
+
+            group = Group()
+            group.spawn(layout)
+            group.spawn(varAnnotationIndex)
+            group.join()
+
+        @seq_task(4)
+        def loadObsAnnotations(self):
+            def obs_annotation(name):
+                self.client.get(
+                    f"{self.dataset}{API}/annotations/obs?annotation-name={name}",
+                    headers={"Accept": "application/octet-stream"},
+                    stream=True,
+                ).close()
 
-        @task
-        def loadObsAnnotationsAndLayouts(self):
             obs_names = self.parent.obs_annotation_names()
+            group = Group()
             for name in obs_names:
-                self.client.get(
-                    f"{self.dataset}/{API_SUFFIX}/annotations/obs?annotation-name={name}",
-                    headers={"Accept": "application/octet-stream"},
-                    stream=True,
-                )
+                group.spawn(obs_annotation, name)
+            group.join()
 
-            layouts = self.parent.layout_names()
-            for name in layouts:
-                self.client.get(
-                    f"{self.dataset}/{API_SUFFIX}/annotations/obs?layout-name={name}",
-                    headers={"Accept": "application/octet-stream"},
-                    stream=True,
-                )
-
-        @task
+        @seq_task(5)
         def done(self):
             self.interrupt()
 
@@ -147,19 +146,19 @@ class CellXGeneTasks(TaskSet):
         """
         Simulate user occasionally loading some expression data for a gene
         """
-
         gene_name = random.choice(self.gene_names)
         filter = {"filter": {"var": {"annotation_value": [{"name": self.var_index_name(), "values": [gene_name]}]}}}
         self.client.put(
-            f"{self.dataset}/{API_SUFFIX}/data/var",
+            f"{self.dataset}{API}/data/var",
             data=json.dumps(filter),
             headers={"Content-Type": "application/json", "Accept": "application/octet-stream"},
             stream=True,
         ).close()
 
 
-class CellxgeneUser(HttpUser):
-    tasks = [CellXGeneTasks]
+class CellxgeneUser(HttpLocust):
+    task_set = ViewDataset
 
-    # Most ops do not require back-end interaction, so slow cadence for users
+    # most ops do not require back-end interaction, so slow cadence
+    # for users
     wait_time = between(10, 60)

server/test/locust/requirements-locust.txt

@@ -1,2 +1 @@
-locust
--r ../../requirements.txt
+locustio

server/test/performance/scale_test_annotations.py

@@ -27,6 +27,7 @@ class WebsiteUser(HttpUser):
     dataset_urls = [
         "human_cell_landscape.cxg",
         "Single_cell_drug_screening_a549-42-remixed.cxg",
+        "kampmann_lab_human_AD_snRNAseq_EC_inhibitoryNeurons-53-remixed.cxg",
         "krasnow_lab_human_lung_cell_atlas_smartseq2-2-remixed.cxg",
         "Single_cell_gene_expression_profiling_of_SARS_CoV_2_infected_human_cell_lines_H1299-27-remixed.cxg",
     ]

server/test/unit/auth/test_auth.py

@@ -79,7 +79,7 @@ class AuthTest(unittest.TestCase):
             r = session.get(f"{server}/{login_uri}")
             # check that the login redirect worked
             self.assertEqual(r.history[0].status_code, 302)
-            self.assertEqual(r.url, f"{server}/auth/pbmc3k.cxg")
+            self.assertEqual(r.url, f"{server}/auth/pbmc3k.cxg/")
 
             config = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/config").json()
             userinfo = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json()
@@ -91,7 +91,7 @@ class AuthTest(unittest.TestCase):
             r = session.get(f"{server}/{logout_uri}")
             # check that the logout redirect worked
             self.assertEqual(r.history[0].status_code, 302)
-            self.assertEqual(r.url, f"{server}/auth/pbmc3k.cxg")
+            self.assertEqual(r.url, f"{server}/auth/pbmc3k.cxg/")
             config = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/config").json()
             userinfo = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json()
             self.assertFalse(userinfo["userinfo"]["is_authenticated"])

server/test/unit/auth/test_oauth.py

@@ -63,53 +63,30 @@ def jwks():
     return make_response(jsonify(dict(keys=[data])))
 
 
+# The port that the mock oauth server will listen on
+PORT = random.randint(10000, 12000)
+
+
 # function to launch the mock oauth server
-def launch_mock_oauth(mock_port):
-    mock_oauth_app.run(port=mock_port)
+def launch_mock_oauth():
+    mock_oauth_app.run(port=PORT)
 
 
 class AuthTest(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        # The port that the mock oauth server will listen on
-        cls.mock_port = random.randint(10000, 12000)
-        cls.dataset_dataroot = FIXTURES_ROOT
-        cls.mock_oauth_process = Process(target=launch_mock_oauth, args=(cls.mock_port,))
-        cls.mock_oauth_process.start()
+    def setUp(self):
+        self.dataset_dataroot = FIXTURES_ROOT
+        self.mock_oauth_process = Process(target=launch_mock_oauth)
+        self.mock_oauth_process.start()
 
-        # Verify that the mock oauth server is ready (accepting requests) before starting the tests.
-
-        # The following lines are polling until the mock server is ready.
-        # The issue is we are starting a mock oauth server, then we are starting a cellxgene server,
-        # which will start making requests to the mock oauth server.
-        # So there is a race condition because the mock oauth server needs to be ready before it gets requests.
-        # We check to see if it is ready, and if not we wait 1 second, then try again.
-        # If it gets to 5 seconds, which is shouldn't, we assume something has gone wrong and fail the test.
-        server_okay = False
-        for _ in range(5):
-            try:
-                response = requests.get(f"http://localhost:{cls.mock_port}/.well-known/jwks.json")
-                if response.status_code == 200:
-                    server_okay = True
-                    break
-            except:  # noqa: E722
-                pass
-
-            # wait one second and try again
-            time.sleep(1)
-
-        assert(server_okay)
-
-    @classmethod
-    def tearDownClass(cls):
-        cls.mock_oauth_process.terminate()
+    def tearDown(self):
+        self.mock_oauth_process.terminate()
 
     def auth_flow(self, app_config, cookie_key=None):
 
         app_config.update_server_config(
             app__api_base_url="local",
             authentication__type="oauth",
-            authentication__params_oauth__oauth_api_base_url=f"http://localhost:{self.mock_port}",
+            authentication__params_oauth__oauth_api_base_url=f"http://localhost:{PORT}",
             authentication__params_oauth__client_id="mock_client_id",
             authentication__params_oauth__client_secret="mock_client_secret",
             authentication__params_oauth__jwt_decode_options={"verify_signature": False, "verify_iss": False},
@@ -170,28 +147,6 @@ class AuthTest(unittest.TestCase):
                 self.assertNotEqual(access_token_before, access_token_after)
                 self.assertNotEqual(id_token_before, id_token_after)
 
-                # invalid cookie is rejected
-                session.cookies.set(cookie_key, "TEST_" + cookie)
-                self.assertTrue(cookie_key in session.cookies)
-                response = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo")
-                # this is not an error, the invalid cookie is just ignored.
-                self.assertEqual(response.status_code, 200)
-                userinfo = response.json()
-                self.assertFalse(userinfo["userinfo"]["is_authenticated"])
-                self.assertIsNone(userinfo["userinfo"]["username"])
-
-                # invalid id_token is rejected
-                test_token = token
-                test_token["id_token"] = "TEST_" + id_token_after
-                encoded_cookie = base64.b64encode(json.dumps(test_token).encode()).decode()
-                session.cookies.set(cookie_key, encoded_cookie)
-                response = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo")
-                # this is not an error, the invalid id_token is just ignored.
-                self.assertEqual(response.status_code, 200)
-                userinfo = response.json()
-                self.assertFalse(userinfo["userinfo"]["is_authenticated"])
-                self.assertIsNone(userinfo["userinfo"]["username"])
-
             r = session.get(logout_uri)
             # check that the logout redirect worked
             self.assertEqual(r.history[0].status_code, 302)

server/test/unit/cli/test_launch.py

@@ -20,8 +20,9 @@ class CLIPLaunchTests(unittest.TestCase):
     def tearDownClass(cls) -> None:
         shutil.rmtree(cls.tmp_dir)
 
-    def test_dump_default_config(self):
-        os.system(f"cellxgene launch --dump-default-config > {self.tmp_dir}/test_config_dump.txt")
-        with open(f"{self.tmp_dir}/expected_config_dump.txt", "w") as expected_config:
-            expected_config.write(yaml.dump(default_config))
-        filecmp.cmp(f"{self.tmp_dir}/expected_config_dump.txt", f"{self.tmp_dir}/test_config_dump.txt")
+
+def test_dump_default_config(self):
+    os.system(f"cellxgene launch --dump-default-config > {self.tmp_dir}/test_config_dump.txt")
+    with open(f"{self.tmp_dir}/expected_config_dump.txt", "w") as expected_config:
+        expected_config.write(yaml.dump(default_config))
+    filecmp.cmp(f"{self.tmp_dir}/expected_config_dump.txt", f"{self.tmp_dir}/test_config_dump.txt")

server/test/unit/common/config/test_server_config.py

@@ -173,6 +173,12 @@ class TestServerConfig(ConfigTests):
             response = session.get(f"{server}/additional/path/health")
             assert response.json()["status"] == "pass"
 
+            # also check that the old URL still works.
+            # NOTE:  this old URL location will soon be deprecated, and when that happens
+            # this check can be removed.
+            response = session.get(f"{server}/health")
+            assert response.json()["status"] == "pass"
+
     def test_get_web_base_url_works(self):
         config = self.get_config(web_base_url="www.thisisawebsite.com")
         web_base_url = config.server_config.get_web_base_url()
@@ -278,14 +284,6 @@ class TestServerConfig(ConfigTests):
             response = session.get(f"{server}/health")
             assert response.json()["status"] == "pass"
 
-            # access a dataset (no slash)
-            response = session.get(f"{server}/set2/pbmc3k.cxg")
-            self.assertEqual(response.status_code, 200)
-
-            # access a dataset (with slash)
-            response = session.get(f"{server}/set2/pbmc3k.cxg/")
-            self.assertEqual(response.status_code, 200)
-
     @patch("server.common.config.server_config.diffexp_tiledb.set_config")
     def test_handle_diffexp(self, mock_tiledb_config):
         custom_config_file = self.custom_app_config(

server/test/unit/common/test_api.py

@@ -1,7 +1,6 @@
 import shutil
 import time
 import unittest
-import zlib
 from http import HTTPStatus
 
 import pandas as pd
@@ -344,7 +343,7 @@ class EndPointsAnnotations(EndPoints):
         url = f"{self.URL_BASE}{endpoint}?{query}"
         n_rows = self.data.get_shape()[0]
         fbs = make_fbs({"cat_A": pd.Series(["label_A"] * n_rows, dtype="category")})
-        result = self.session.put(url, data=zlib.compress(fbs))
+        result = self.session.put(url, data=fbs)
         self.assertEqual(result.status_code, HTTPStatus.OK)
         self.assertEqual(result.headers["Content-Type"], "application/json")
         self.assertEqual(result.json(), {"status": "OK"})

server/test/unit/common/test_corpora.py

@@ -76,30 +76,6 @@ class CorporaAPITest(unittest.TestCase):
         some_fields["project_links"] = json.loads(some_fields["project_links"])
         self.assertEqual(corpora_get_props_from_anndata(adata), some_fields)
 
-    def test_corpora_get_props_from_anndata_v110(self):
-        adata = self._get_h5ad()
-
-        if "version" in adata.uns:
-            del adata.uns["version"]
-        self.assertIsNone(corpora_get_props_from_anndata(adata))
-
-        # legit version, but missing required values
-        adata.uns["version"] = {"corpora_schema_version": "1.1.0", "corpora_encoding_version": "0.1.0"}
-        with self.assertRaises(KeyError):
-            corpora_get_props_from_anndata(adata)
-
-        # Metadata following schema 1.1.0, which removes some fields relative to 1.1.0
-        some_110_fields = {
-            "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"},
-            "title": "title",
-            "layer_descriptions": "layer_descriptions",
-            "organism": "organism",
-            "organism_ontology_term_id": "organism_ontology_term_id",
-        }
-        for k in some_110_fields:
-            adata.uns[k] = some_110_fields[k]
-        self.assertEqual(corpora_get_props_from_anndata(adata), some_110_fields)
-
     def _get_h5ad(self):
         return anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
 

server/test/unit/common/test_writable_annotation.py

@@ -2,7 +2,7 @@ import json
 import shutil
 import unittest
 from os import path, listdir
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, patch
 
 import numpy as np
 import pandas as pd
@@ -129,34 +129,18 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase):
             with self.assertRaises(KeyError):
                 self.annotation_put_fbs(fbs_bad)
 
-    def test_write_labels_stores_df_as_tiledb_array(self):
-        with self.app.test_request_context():
-            self.annotations.write_labels(self.df, self.data)
-            # get uri
-            dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id
-            annotation = self.db.query_for_most_recent(
-                Annotation, [Annotation.user_id == "1234", Annotation.dataset_id == str(dataset_id)]
-            )
+    @patch("server.common.annotations.hosted_tiledb.current_app")
+    def test_write_labels_stores_df_as_tiledb_array(self, mock_user_id):
+        mock_user_id.auth.get_user_id.return_value = "1234"
+        self.annotations.write_labels(self.df, self.data)
+        # get uri
+        dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id
+        annotation = self.db.query_for_most_recent(
+            Annotation, [Annotation.user_id == "1234", Annotation.dataset_id == str(dataset_id)]
+        )
 
-            df = tiledb.open(annotation.tiledb_uri)
-            self.assertEqual(type(df), tiledb.array.SparseArray)
-
-    def test_remove_categories(self):
-        with self.app.test_request_context():
-            # update empty category data, which is how annotations are removed
-            empty = make_fbs({})
-            self.annotation_put_fbs(empty)
-
-            # verify that the tiledb uri is an empty string.
-            dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id
-            annotation = self.db.query_for_most_recent(
-                Annotation, [Annotation.user_id == self.user_id, Annotation.dataset_id == str(dataset_id)]
-            )
-            self.assertEqual(annotation.tiledb_uri, "")
-
-            # verify that read_labels returns None
-            df = self.annotations.read_labels(self.data)
-            self.assertIsNone(df)
+        df = tiledb.open(annotation.tiledb_uri)
+        self.assertEqual(type(df), tiledb.array.SparseArray)
 
 
 class WritableAnnotationTest(unittest.TestCase):
@@ -284,32 +268,3 @@ class WritableAnnotationTest(unittest.TestCase):
             all_col_schema["cat_B"],
             {"name": "cat_B", "type": "categorical", "categories": ["label_B"], "writable": True},
         )
-
-    def test_put_float_data(self):
-        # verify that OBS PUTs (annotation_put_fbs) are accessible via
-        # GET (annotation_to_fbs_matrix)
-
-        n_rows = self.data.get_shape()[0]
-
-        # verifies that floating point with decimals fail.
-        fbs = make_fbs({"cat_F_FAIL": pd.Series([1.1] * n_rows, dtype=np.dtype("float"))})
-        with self.assertRaises(ValueError) as exception_context:
-            res = self.annotation_put_fbs(fbs)
-        self.assertEqual(str(exception_context.exception), "Columns may not have floating point types")
-
-        # verifies that floating point that can be converted to int passes
-        fbs = make_fbs({"cat_F_PASS": pd.Series([1.0] * n_rows, dtype="float")})
-        res = self.annotation_put_fbs(fbs)
-        self.assertEqual(res, json.dumps({"status": "OK"}))
-
-        # check read_labels
-        labels = self.annotations.read_labels(None)
-        fbsAll = self.data.annotation_to_fbs_matrix("obs", None, labels)
-        schema = schema_get_helper(self.data)
-        annotations = decode_fbs.decode_matrix_FBS(fbsAll)
-        self.assertEqual(annotations["n_rows"], n_rows)
-        all_col_schema = {c["name"]: c for c in schema["annotations"]["obs"]["columns"]}
-        self.assertEqual(
-            all_col_schema["cat_F_PASS"],
-            {"name": "cat_F_PASS", "type": "int32", "writable": True},
-        )

server/test/unit/common/utils/test_type_conversion_utils.py

@@ -99,7 +99,7 @@ class TestTypeConversionUtils(unittest.TestCase):
 
     def test__get_dtype_of_array__supported_dtypes_return_as_expected(self):
         types = [np.float32, np.int32, np.bool_, str]
-        expected_dtypes = [np.float32, np.int32, np.uint8, str]
+        expected_dtypes = [np.float32, np.int32, np.uint8, np.unicode]
 
         for test_type_index in range(len(types)):
             with self.subTest(
@@ -110,7 +110,7 @@ class TestTypeConversionUtils(unittest.TestCase):
 
     def test__get_dtype_of_array__categories_return_as_expected(self):
         array = Series(data=["a", "b", "c"], dtype="category")
-        expected_dtype = str
+        expected_dtype = np.unicode
 
         actual_dtype = get_dtype_of_array(array)
 
@@ -179,7 +179,7 @@ class TestTypeConversionUtils(unittest.TestCase):
         category_array = Series(data=["a", "b", "b"], dtype="category")
         dataframe = DataFrame({"float_array": float_array, "category_array": category_array})
 
-        expected_data_types_dict = {"float_array": np.float32, "category_array": str}
+        expected_data_types_dict = {"float_array": np.float32, "category_array": np.unicode}
         expected_schema_type_hints_dict = {
             "float_array": {"type": "float32"},
             "category_array": {"type": "categorical", "categories": ["a", "b"]},

server/test/unit/converters/schema/test_gene_symbol.py

@@ -1,61 +0,0 @@
-import os
-import unittest
-
-import pandas as pd
-
-from server.test import FIXTURES_ROOT
-from server.converters.schema import gene_symbol
-
-
-class TestHGNCSymbolChecker(unittest.TestCase):
-
-    def setUp(self):
-        self.test_hgnc_path = os.path.join(FIXTURES_ROOT, "hgnc_example.txt.gz")
-        self.hgnc_checker = gene_symbol.HGNCSymbolChecker.from_hgnc_records(self.test_hgnc_path)
-
-    def test_symbol_upgrade(self):
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("SEPT1"), "SEPTIN1")
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("ADRB2R"), "ADRB2")
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("BAR"), "ADRB2")
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("sept1"), "SEPTIN1")
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("AdRb2R"), "ADRB2")
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("bar"), "ADRB2")
-
-        # Strip off seurat endings when appropriate
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("SEPT1.1"), "SEPTIN1")
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("ADRB2-1"), "ADRB2")
-
-        # DIFF6 is ambiguous so don't upgrade it
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("DIFF6"), "DIFF6")
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("diff6"), "diff6")
-
-        # ARG1 is approved
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("ARG1"), "ARG1")
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("arg1"), "ARG1")
-
-        # HAP1 is both approved and withdrawn
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("HAP1"), "HAP1")
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("hap1"), "HAP1")
-
-        # Leave unknown symbols alone
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("NOTASYMBOL"), "NOTASYMBOL")
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("notasymbol"), "notasymbol")
-
-        # Upgrade HGNC ids unless you can't find it
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("HGNC:286"), "ADRB2")
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("HGNC:4812"), "HAP1")
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("HGNC:123456"), "HGNC:123456")
-
-    def test_check_symbol(self):
-        self.assertEqual(self.hgnc_checker.check_symbol("SEPT1"), gene_symbol.SymbolStatus.UPGRADABLE)
-        self.assertEqual(self.hgnc_checker.check_symbol("DIFF6"), gene_symbol.SymbolStatus.AMBIGUOUS)
-        self.assertEqual(self.hgnc_checker.check_symbol("NOTASYMBOL"), gene_symbol.SymbolStatus.UNKNOWN)
-
-        # HAP1 is one of the approved and withdrawn symbols
-        self.assertEqual(self.hgnc_checker.check_symbol("HAP1"), gene_symbol.SymbolStatus.APPROVED)
-
-    def test_upgrade_index(self):
-        index = pd.Index(["SEPT1", "DIFF6", "NOTASYMBOL", "bar", "SEPTIN1"])
-        var_df = pd.DataFrame([[0] * len(index)], index=index)
-        upgraded_index = gene_symbol.get_upgraded_var_index(var_df, hgnc_path=self.test_hgnc_path)
-        self.assertEqual(upgraded_index.tolist(), ["SEPTIN1", "DIFF6", "NOTASYMBOL", "ADRB2", "SEPTIN1"])

server/test/unit/converters/schema/test_ontology.py

@@ -1,129 +0,0 @@
-import json
-
-import unittest
-import unittest.mock
-
-from server.converters.schema import ontology
-
-
-class TestOntologyParsing(unittest.TestCase):
-    def setUp(self):
-
-        self.curies = ["UBERON:0002048", "HsapDv:0000174", "NCBITaxon:9606", "EFO:0008995"]
-
-        self.names = ["UBERON", "HsapDv", "NCBITaxon", "EFO"]
-
-        self.values = ["0002048", "0000174", "9606", "0008995"]
-
-        self.iris = [
-            "http://purl.obolibrary.org/obo/UBERON_0002048",
-            "http://purl.obolibrary.org/obo/HsapDv_0000174",
-            "http://purl.obolibrary.org/obo/NCBITaxon_9606",
-            "http://www.ebi.ac.uk/efo/EFO_0008995",
-        ]
-
-        URL_ROOT = "http://www.ebi.ac.uk/ols/api/ontologies/"
-        self.urls = [
-            URL_ROOT + "UBERON/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FUBERON_0002048",
-            URL_ROOT + "HsapDv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FHsapDv_0000174",
-            URL_ROOT + "NCBITaxon/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_9606",
-            URL_ROOT + "EFO/terms/http%253A%252F%252Fwww.ebi.ac.uk%252Fefo%252FEFO_0008995",
-        ]
-
-        self.responses = {
-            "UBERON:0002048": {
-                "iri": "http://purl.obolibrary.org/obo/UBERON_0002048",
-                "description": ["Respiration organ that develops as an outpocketing of the esophagus."],
-                "label": "lung",
-            },
-            "HsapDv:0000174": {
-                "iri": "http://purl.obolibrary.org/obo/HsapDv_0000174",
-                "description": ["Infant stage that refers to an infant who is over 1 and under 2 months old."],
-                "label": "1-month-old human stage",
-            },
-            "NCBITaxon:9606": {
-                "iri": "http://purl.obolibrary.org/obo/NCBITaxon_9606",
-                "description": None,
-                "label": "Homo sapiens",
-            },
-            "EFO:0008995": {
-                "iri": "http://www.ebi.ac.uk/efo/EFO_0008995",
-                "description": [
-                    (
-                        '10X is a "synthetic long-read" technology and works by capturing a barcoded oligo-coated '
-                        "gel-bead and 0.3x genome copies into a single emulsion droplet, processing the equivalent "
-                        "of 1 million pipetting steps. Successive versions of the 10x chemistry use different "
-                        "barcode locations to improve the sequencing yield and quality of 10x experiments."
-                    )
-                ],
-                "label": "10X sequencing",
-            },
-        }
-
-    def test_ontololgy_name(self):
-        for curie, expected_name in zip(self.curies, self.names):
-            self.assertEqual(ontology._ontology_name(curie), expected_name)
-
-    def test_ontololgy_value(self):
-        for curie, expected_value in zip(self.curies, self.values):
-            self.assertEqual(ontology._ontology_value(curie), expected_value)
-
-    def test_iri(self):
-        for curie, expected_iri in zip(self.curies, self.iris):
-            self.assertEqual(ontology._iri(curie), expected_iri)
-
-    def test_ontology_info_url(self):
-        for curie, expected_url in zip(self.curies, self.urls):
-            self.assertEqual(ontology._ontology_info_url(curie), expected_url)
-
-    def test_empty_ontology_info_url(self):
-        self.assertEqual(ontology._ontology_info_url(""), "")
-
-
-class TestOntologyLookup(unittest.TestCase):
-    def setUp(self):
-        self.responses = {
-            "UBERON:0002048": {
-                "iri": "http://purl.obolibrary.org/obo/UBERON_0002048",
-                "description": ["Respiration organ that develops as an outpocketing of the esophagus."],
-                "label": "lung",
-            },
-            "HsapDv:0000174": {
-                "iri": "http://purl.obolibrary.org/obo/HsapDv_0000174",
-                "description": ["Infant stage that refers to an infant who is over 1 and under 2 months old."],
-                "label": "1-month-old human stage",
-            },
-            "NCBITaxon:9606": {
-                "iri": "http://purl.obolibrary.org/obo/NCBITaxon_9606",
-                "description": None,
-                "label": "Homo sapiens",
-            },
-            "EFO:0008995": {
-                "iri": "http://www.ebi.ac.uk/efo/EFO_0008995",
-                "description": [
-                    ('10X is a "synthetic long-read" technology and works by capturing a barcoded oligo-coated '
-                     'gel-bead and 0.3x genome copies into a single emulsion droplet, processing the equivalent '
-                     'of 1 million pipetting steps. Successive versions of the 10x chemistry use different barcode '
-                     'locations to improve the sequencing yield and quality of 10x experiments.')
-                ],
-                "label": "10X sequencing",
-            },
-        }
-
-        self.labels = {
-            "UBERON:0002048": "lung",
-            "HsapDv:0000174": "1-month-old human stage",
-            "NCBITaxon:9606": "Homo sapiens",
-            "EFO:0008995": "10X sequencing",
-        }
-
-    @unittest.mock.patch("requests.get")
-    def test_lookup_label(self, mock_get):
-
-        for curie, response in self.responses.items():
-            mock_get.return_value.content = json.dumps(response)
-            mock_get.return_value.json.return_value = response
-            mock_get.return_value.status_code = 200
-
-            label = ontology.get_ontology_label(curie)
-            self.assertEqual(label, self.labels[curie])

server/test/unit/converters/schema/test_remix.py

@@ -1,257 +0,0 @@
-import json
-import os
-import unittest
-import unittest.mock
-
-import anndata
-import numpy
-import pandas as pd
-import scanpy as sc
-
-from server.converters.schema import remix
-
-PROJECT_ROOT = os.popen("git rev-parse --show-toplevel").read().strip()
-
-
-class TestApplySchema(unittest.TestCase):
-
-    def setUp(self):
-        self.source_h5ad_path = f"{PROJECT_ROOT}/server/test/fixtures/pbmc3k-CSC-gz.h5ad"
-        self.output_h5ad_path = f"{PROJECT_ROOT}/server/test/fixtures/test_remix.h5ad"
-        self.config_path = f"{PROJECT_ROOT}/server/test/fixtures/test_config.yaml"
-        self.bad_config_path = f"{PROJECT_ROOT}/server/test/fixtures/test_bad_config.yaml"
-
-    def tearDown(self):
-        try:
-            os.remove(self.output_h5ad_path)
-        except OSError:
-            pass
-
-    @unittest.mock.patch("server.converters.schema.ontology.get_ontology_label")
-    def test_apply_schema(self, mock_get_ontology_label):
-        mock_get_ontology_label.return_value = "test label"
-        remix.apply_schema(self.source_h5ad_path, self.config_path, self.output_h5ad_path)
-        new_adata = sc.read_h5ad(self.output_h5ad_path)
-
-        self.assertIn("cell_type", new_adata.obs.columns)
-        self.assertListEqual(["test label"], new_adata.obs["cell_type"].unique().tolist())
-        self.assertListEqual(
-            ["CL:00001", "CL:00002", "CL:00003", "CL:00004", "CL:00005", "CL:00006", "CL:00007", "CL:00008"],
-            sorted(new_adata.obs["cell_type_ontology_term_id"].unique().tolist())
-        )
-
-        self.assertIn("version", new_adata.uns_keys())
-
-    @unittest.mock.patch("server.converters.schema.ontology.get_ontology_label")
-    def test_apply_bad_schema(self, mock_get_ontology_label):
-        mock_get_ontology_label.return_value = "test label"
-        remix.apply_schema(self.source_h5ad_path, self.bad_config_path, self.output_h5ad_path)
-        new_adata = sc.read_h5ad(self.output_h5ad_path)
-
-        # Should refuse to write the version
-        self.assertNotIn("version", new_adata.uns_keys())
-
-class TestFieldParsing(unittest.TestCase):
-
-    def test_is_curie(self):
-        self.assertTrue(remix.is_curie("EFO:00001"))
-        self.assertTrue(remix.is_curie("UBERON:123456"))
-        self.assertTrue(remix.is_curie("HsapDv:0001"))
-        self.assertFalse(remix.is_curie("UBERON"))
-        self.assertFalse(remix.is_curie("UBERON:"))
-        self.assertFalse(remix.is_curie("123456"))
-
-    def test_is_ontology_field(self):
-        self.assertTrue(remix.is_ontology_field("tissue_ontology_term_id"))
-        self.assertTrue(remix.is_ontology_field("cell_type_ontology_term_id"))
-        self.assertFalse(remix.is_ontology_field("cell_ontology"))
-        self.assertFalse(remix.is_ontology_field("method"))
-
-    def test_get_label_field_name(self):
-        self.assertEqual("tissue", remix.get_label_field_name("tissue_ontology_term_id"))
-        self.assertEqual("cell_type", remix.get_label_field_name("cell_type_ontology_term_id"))
-
-    def test_split_suffix(self):
-        self.assertEqual(("UBERON:1234", " (organoid)"), remix.split_suffix("UBERON:1234 (organoid)"))
-        self.assertEqual(("UBERON:1234", " (cell culture)"), remix.split_suffix("UBERON:1234 (cell culture)"))
-        self.assertEqual(("UBERON:1234", ""), remix.split_suffix("UBERON:1234"))
-        self.assertEqual(("UBERON:1234 (something)", ""), remix.split_suffix("UBERON:1234 (something)"))
-
-    @unittest.mock.patch("server.converters.schema.ontology.get_ontology_label")
-    def test_get_curie_and_label(self, mock_get_ontology_label):
-        mock_get_ontology_label.return_value = "test label"
-        self.assertEqual(
-            remix.get_curie_and_label("UBERON:1234"),
-            ("UBERON:1234", "test label")
-        )
-        self.assertEqual(
-            remix.get_curie_and_label("UBERON:1234 (cell culture)"),
-            ("UBERON:1234 (cell culture)", "test label (cell culture)")
-        )
-        self.assertEqual(
-            remix.get_curie_and_label("whatever"),
-            ("", "whatever")
-        )
-
-
-class TestManipulateAnndata(unittest.TestCase):
-
-    def setUp(self):
-
-        self.cell_count = 20
-        self.gene_count = 200
-        X = numpy.random.randint(0, 1000, (self.cell_count, self.gene_count))
-        uns = {"organism": "monkey", "experiment": "monkey experiment"}
-        obs = pd.DataFrame(
-            index=[f"Cell{d}" for d in range(self.cell_count)],
-            columns=["tissue", "CellType"],
-            data=[["lung", "epithelial"]] * (self.cell_count // 2) + [["lung", "endothelial"]] * (self.cell_count // 2)
-        )
-        var = pd.DataFrame(index=[f"SEPT{d}" for d in range(self.gene_count)])
-
-        self.adata = anndata.AnnData(X=X, obs=obs, var=var, uns=uns)
-
-    def test_safe_add_field(self):
-
-        remix.safe_add_field(self.adata.obs, "tissue", ["monkey lung"] * self.cell_count)
-        self.assertEqual(self.adata.obs["tissue_original"].tolist(), ["lung"] * self.cell_count)
-        self.assertEqual(self.adata.obs["tissue"].tolist(), ["monkey lung"] * self.cell_count)
-
-        remix.safe_add_field(self.adata.uns, "contributors", [{"name": "contributor1"}, {"name": "contributor2"}])
-        self.assertEqual(
-            self.adata.uns["contributors"],
-            json.dumps([{"name": "contributor1"}, {"name": "contributor2"}])
-        )
-
-    @unittest.mock.patch("server.converters.schema.ontology.get_ontology_label")
-    def test_remix_uns(self, mock_get_ontology_label):
-        mock_get_ontology_label.return_value = "Pan troglodytes"
-        uns_config = {
-            "version": {
-                "corpora_schema_version": "1.0.0",
-                "corpora_encoding_version": "0.1.0"
-            },
-            "organism_ontology_term_id": "NCBITaxon:9598",
-            "contributors": [
-                {
-                    "name": "scientist",
-                    "email": "scientist@science.com"
-                }
-            ]
-        }
-
-        remix.remix_uns(self.adata, uns_config)
-
-        self.assertEqual(
-            sorted(self.adata.uns_keys()),
-            sorted(["organism_original", "organism", "organism_ontology_term_id",
-                    "contributors", "version", "experiment"])
-        )
-
-        self.assertEqual(self.adata.uns['organism'], "Pan troglodytes")
-        self.assertEqual(self.adata.uns['organism_original'], "monkey")
-        self.assertEqual(self.adata.uns['organism_ontology_term_id'], "NCBITaxon:9598")
-        self.assertEqual(self.adata.uns['contributors'],
-                         json.dumps([{"name": "scientist", "email": "scientist@science.com"}]))
-
-    @unittest.mock.patch("server.converters.schema.ontology.get_ontology_label")
-    def test_remix_obs(self, mock_get_ontology_label):
-        mock_get_ontology_label.return_value = "lung (in a monkey)"
-        obs_config = {
-            "tissue_ontology_term_id": {
-                "tissue": {
-                    "lung": "UBERON:00000"
-                }
-            },
-            "cell_color": {
-                "CellType": {
-                    "epithelial": "fuschia",
-                    "endothelial": "khaki"
-                }
-            },
-            "sex": "male"
-        }
-
-        remix.remix_obs(self.adata, obs_config)
-        self.assertEqual(
-            sorted(self.adata.obs_keys()),
-            sorted(["tissue", "tissue_ontology_term_id", "tissue_original", "CellType", "cell_color", "sex"])
-        )
-
-        self.assertTrue(all(v == "lung" for v in self.adata.obs.tissue_original))
-        self.assertTrue(all(v == "UBERON:00000" for v in self.adata.obs.tissue_ontology_term_id))
-        self.assertTrue(all(v == "lung (in a monkey)" for v in self.adata.obs.tissue))
-        self.assertTrue(all(v == "male" for v in self.adata.obs.sex))
-        self.assertTrue(all(v in (("epithelial", "fuschia"), ("endothelial", "khaki"))
-                            for v in zip(self.adata.obs.CellType, self.adata.obs.cell_color)))
-
-
-class TestFixupGeneSymbols(unittest.TestCase):
-
-    def setUp(self):
-        self.seurat_path = f"{PROJECT_ROOT}/server/test/fixtures/schema_test_data/seurat_tutorial.h5ad"
-        self.seurat_merged_path = f"{PROJECT_ROOT}/server/test/fixtures/schema_test_data/seurat_tutorial_merged.h5ad"
-        self.sctransform_path = f"{PROJECT_ROOT}/server/test/fixtures/schema_test_data/sctransform.h5ad"
-        self.sctransform_merged_path = f"{PROJECT_ROOT}/server/test/fixtures/schema_test_data/sctransform_merged.h5ad"
-
-        # There's lots of MALAT1, but it doesn't collide with any other names,
-        # so it shouldn't change during merging.
-        self.stable_gene = "MALAT1"
-
-    def test_fixup_gene_symbols_seurat(self):
-
-        if not os.path.isfile(self.seurat_path):
-            return unittest.skip(
-                "Skipping gene symbol conversion tests because test h5ads are not present. To create them, "
-                "run server/test/fixtures/schema_test_data/generate_test_data.sh"
-            )
-
-        original_adata = sc.read_h5ad(self.seurat_path)
-        merged_adata = sc.read_h5ad(self.seurat_merged_path)
-
-        fixup_config = {"X": "log1p", "counts": "raw", "scale.data": "log1p"}
-
-        fixed_adata = remix.fixup_gene_symbols(original_adata, fixup_config)
-
-        self.assertEqual(
-            merged_adata.layers["counts"][:, merged_adata.var.index == self.stable_gene].sum(),
-            fixed_adata.raw.X[:, fixed_adata.var.index == self.stable_gene].sum()
-        )
-        self.assertAlmostEqual(
-            merged_adata.X[:, merged_adata.var.index == self.stable_gene].sum(),
-            fixed_adata.X[:, fixed_adata.var.index == self.stable_gene].sum()
-        )
-
-        self.assertAlmostEqual(
-            merged_adata.layers["scale.data"][:, merged_adata.var.index == self.stable_gene].sum(),
-            fixed_adata.layers["scale.data"][:, fixed_adata.var.index == self.stable_gene].sum()
-        )
-
-    def test_fixup_gene_symbols_sctransform(self):
-
-        if not os.path.isfile(self.sctransform_path):
-            return unittest.skip(
-                "Skipping gene symbol conversion tests because test h5ads are not present. To create them, "
-                "run server/test/fixtures/schema_test_data/generate_test_data.sh"
-            )
-
-        original_adata = sc.read_h5ad(self.sctransform_path)
-        merged_adata = sc.read_h5ad(self.sctransform_merged_path)
-
-        fixup_config = {"X": "log1p", "counts": "raw"}
-
-        fixed_adata = remix.fixup_gene_symbols(original_adata, fixup_config)
-
-        # sctransform does a bunch of stuff, including slightly modifying the
-        # raw counts. So we can't assert for exact equality the way we do with
-        # the vanilla seurat tutorial. But, the results should still be very
-        # close.
-        merged_raw_stable = merged_adata.layers["counts"][:, merged_adata.var.index == self.stable_gene].sum()
-        fixed_raw_stable = fixed_adata.raw.X[:, fixed_adata.var.index == self.stable_gene].sum()
-        self.assertLess(abs(merged_raw_stable - fixed_raw_stable), .001 * merged_raw_stable)
-
-        self.assertAlmostEqual(
-            merged_adata.X[:, merged_adata.var.index == self.stable_gene].sum(),
-            fixed_adata.X[:, fixed_adata.var.index == self.stable_gene].sum(),
-            0
-        )

server/test/unit/converters/schema/test_validate.py

@@ -1,435 +0,0 @@
-import json
-import os
-import unittest
-
-import pandas as pd
-import scanpy as sc
-
-from server.converters.schema import validate
-
-PROJECT_ROOT = os.popen("git rev-parse --show-toplevel").read().strip()
-
-
-class TestFieldValidation(unittest.TestCase):
-
-    def test_validate_stringified_list_of_dicts(self):
-
-        good = json.dumps([{"a": 1}, {2: "x", "z": "y"}])
-        not_stringified = [{"a": 1}, {2: "x", "z": "y"}]
-        not_a_list = json.dumps({"bad": "dict"})
-        not_json = "oh hey!"
-
-        self.assertTrue(validate._validate_stringified_list_of_dicts(good))
-
-        self.assertFalse(validate._validate_stringified_list_of_dicts(not_stringified))
-        self.assertFalse(validate._validate_stringified_list_of_dicts(not_a_list))
-        self.assertFalse(validate._validate_stringified_list_of_dicts(not_json))
-
-    def test_validate_human_readable_string(self):
-
-        good = "oh hey!"
-        curie = "EFO:0001"
-        ensg = "ENSG000001234"
-        enst = "ENST000005678"
-
-        self.assertTrue(validate._validate_human_readable_string(good))
-
-        self.assertFalse(validate._validate_human_readable_string(curie))
-        self.assertFalse(validate._validate_human_readable_string(ensg))
-        self.assertFalse(validate._validate_human_readable_string(enst))
-
-    def test_validate_curie(self):
-
-        self.assertTrue(validate._validate_curie("UBERON:00001", ["UBERON", "EFO"]))
-        self.assertTrue(validate._validate_curie("HsapDv:00002", ["HsapDv"]))
-
-        self.assertFalse(validate._validate_curie("HsapDv:00002", ["UBERON", "EFO"]))
-        self.assertFalse(validate._validate_curie("EFO:00002 (organoid)", ["UBERON", "EFO"]))
-        self.assertFalse(validate._validate_curie("EFO:00002 extra", ["UBERON", "EFO"]))
-        self.assertFalse(validate._validate_curie("UBERON:ABCD", ["UBERON", "EFO"]))
-        self.assertFalse(validate._validate_curie("Uberon:00002", ["UBERON", "EFO"]))
-        self.assertFalse(validate._validate_curie("UBERON:", ["UBERON", "EFO"]))
-        self.assertFalse(validate._validate_curie("UBERON", ["UBERON", "EFO"]))
-
-    def test_validate_suffixed_curie(self):
-
-        self.assertTrue(validate._validate_suffixed_curie("EFO:00001", ["UBERON", "EFO"]))
-        self.assertTrue(validate._validate_suffixed_curie("UBERON:00001 (cell culture)", ["UBERON", "EFO"]))
-
-        self.assertFalse(validate._validate_suffixed_curie("HsapDv:00002 (organoid)", ["UBERON", "EFO"]))
-        self.assertFalse(validate._validate_suffixed_curie("HsapDv:00002(organoid)", ["UBERON", "EFO"]))
-        self.assertFalse(validate._validate_suffixed_curie("HsapDv:00002", ["UBERON", "EFO"]))
-        self.assertFalse(validate._validate_suffixed_curie("EFO:00002 extra", ["UBERON", "EFO"]))
-        self.assertFalse(validate._validate_suffixed_curie("UBERON:ABCD", ["UBERON", "EFO"]))
-        self.assertFalse(validate._validate_suffixed_curie("Uberon:00002", ["UBERON", "EFO"]))
-        self.assertFalse(validate._validate_suffixed_curie("UBERON:", ["UBERON", "EFO"]))
-        self.assertFalse(validate._validate_suffixed_curie("UBERON", ["UBERON", "EFO"]))
-
-
-class TestColumnValidation(unittest.TestCase):
-
-    def test_validate_unique(self):
-        unique = pd.DataFrame([["abc", "def"], ["ghi", "jkl"], ["mnop", "qrs"]],
-                              index=["X", "Y", "Z"], columns=["col1", "col2"])
-        duped = pd.DataFrame([["abc", "def"], ["ghi", "qrs"], ["abc", "qrs"]],
-                             index=["X", "Y", "X"], columns=["col1", "col2"])
-
-        schema_def = {"unique": True}
-
-        errors = validate._validate_column(unique.index, "index", "unique_df", schema_def)
-        self.assertFalse(errors)
-
-        errors = validate._validate_column(duped.index, "index", "duped_df", schema_def)
-        self.assertEqual(len(errors), 1)
-        self.assertIn("is not unique", errors[0])
-
-        errors = validate._validate_column(unique["col1"], "col1", "unique_df", schema_def)
-        self.assertFalse(errors)
-
-        errors = validate._validate_column(duped["col1"], "col1", "duped_df", schema_def)
-        self.assertEqual(len(errors), 1)
-        self.assertIn("is not unique", errors[0])
-
-        schema_def = {"unique": False}
-        errors = validate._validate_column(duped["col1"], "col1", "duped_df", schema_def)
-        self.assertFalse(errors)
-
-    def test_validate_nullable(self):
-        non_null = pd.DataFrame([["abc", "def"], ["ghi", "jkl"], ["mnop", "qrs"]],
-                                index=["X", "Y", "Z"], columns=["col1", "col2"])
-        has_null = pd.DataFrame([["abc", "", None], ["ghi", "jkl", 1], ["mnop", "qrs", 2]],
-                                index=["X", "Y", "Z"], columns=["col1", "col2", "col3"])
-
-        schema_def = {"nullable": False}
-        errors = validate._validate_column(non_null["col1"], "col1", "nonnull_df", schema_def)
-        self.assertFalse(errors)
-        errors = validate._validate_column(has_null["col1"], "col1", "hasnull_df", schema_def)
-        self.assertFalse(errors)
-
-        errors = validate._validate_column(has_null["col2"], "col2", "hasnull_df", schema_def)
-        self.assertEqual(len(errors), 1)
-        self.assertIn("contains empty values", errors[0])
-        errors = validate._validate_column(has_null["col3"], "col3", "hasnull_df", schema_def)
-        self.assertEqual(len(errors), 1)
-        self.assertIn("contains empty values", errors[0])
-
-        schema_def = {"nullable": True}
-        errors = validate._validate_column(has_null["col2"], "col2", "hasnull_df", schema_def)
-        self.assertFalse(errors)
-
-    def test_human_readable(self):
-        hr_df = pd.DataFrame(
-            [["for you, a human", "UBERON:12345", "UBERON:1234 (thundercat)"],
-             ["hope you're well", "bit of lungs", "brain"]],
-            index=["ENSG00001", "ENSG00002"],
-            columns=["good", "curie", "suffixed_curie"])
-
-        schema_def = {"type": "human-readable string"}
-        errors = validate._validate_column(hr_df["good"], "good", "hr", schema_def)
-        self.assertFalse(errors)
-
-        errors = validate._validate_column(hr_df["curie"], "curie", "hr", schema_def)
-        self.assertEqual(len(errors), 1)
-        self.assertIn("non-human-readable", errors[0])
-
-        errors = validate._validate_column(hr_df["suffixed_curie"], "suffixed_curie", "hr", schema_def)
-        self.assertEqual(len(errors), 1)
-        self.assertIn("non-human-readable", errors[0])
-
-        errors = validate._validate_column(hr_df.index, "ensg", "hr", schema_def)
-        self.assertEqual(len(errors), 1)
-        self.assertIn("non-human-readable", errors[0])
-
-    def test_curie(self):
-
-        curie_df = pd.DataFrame(
-            [["EFO:00001", "HsapDv:00001 (cell culture)", "EFO:", "MONDO:0001 cell culture"],
-             ["UBERON:00002", "HsapDv:00002 (organoid)", "EFO:12345", "MONDO:0002 (baba yaga)"],
-             ["EFO:0000000005", "HsapDv:000004 (humanzee)", "EFO:000002", "MONDO:0004 (TMNT)"]],
-            index=["X", "Y", "Z"],
-            columns=["good", "good_suffix", "bad", "bad_suffix"])
-
-        # Good
-        schema_def = {"type": "curie", "prefixes": ["EFO", "UBERON"]}
-        errors = validate._validate_column(curie_df["good"], "good", "curie_df", schema_def)
-        self.assertFalse(errors)
-
-        # Good suffix
-        schema_def = {"type": "suffixed curie", "prefixes": ["HsapDv", "WHATEVER"]}
-        errors = validate._validate_column(curie_df["good_suffix"], "good_suffix", "curie_df", schema_def)
-        self.assertFalse(errors)
-
-        # Bad prefix
-        schema_def = {"type": "curie", "prefixes": ["EFO"]}
-        errors = validate._validate_column(curie_df["good"], "good", "curie_df", schema_def)
-        self.assertEqual(len(errors), 1)
-        self.assertIn("invalid ontology", errors[0])
-        self.assertIn("must be curies from one of these", errors[0])
-
-        # Bad curies
-        schema_def = {"type": "curie", "prefixes": ["EFO"]}
-        errors = validate._validate_column(curie_df["bad"], "bad", "curie_df", schema_def)
-        self.assertEqual(len(errors), 1)
-        self.assertIn("invalid ontology", errors[0])
-
-        # Bad suffixes
-        schema_def = {"type": "suffixed curie", "prefixes": ["EFO"]}
-        errors = validate._validate_column(curie_df["bad_suffix"], "bad_suffix", "curie_df", schema_def)
-        self.assertEqual(len(errors), 1)
-        self.assertIn("invalid ontology", errors[0])
-
-    def test_enum(self):
-        enum_df = pd.DataFrame(
-            [["abc", "ghi"],
-             ["def", "jkl"]],
-            index=["X", "Y"],
-            columns=["col1", "col2"])
-
-        # All match
-        schema_def = {"type": "string", "enum": ["abc", "def", "xyz"]}
-        errors = validate._validate_column(enum_df["col1"], "col1", "enum_df", schema_def)
-        self.assertFalse(errors)
-
-        # Missing value
-        schema_def = {"type": "string", "enum": ["abc", "xyz"]}
-        errors = validate._validate_column(enum_df["col1"], "col1", "enum_df", schema_def)
-        self.assertEqual(len(errors), 1)
-        self.assertIn("unpermitted values", errors[0])
-
-
-class TestDictValidations(unittest.TestCase):
-
-
-    def test_key_presence(self):
-
-        schema_def = {"keys": {"abc": None, "def": None}}
-
-        dict_ = {"abc": "123", "def": "456"}
-        errors = validate._validate_dict(dict_, "d", schema_def)
-        self.assertFalse(errors)
-
-        # Missing keys are bad
-        dict_ = {"abc": "123"}
-        errors = validate._validate_dict(dict_, "d", schema_def)
-        self.assertEqual(len(errors), 1)
-        self.assertIn("missing key", errors[0])
-
-        # Extra keys are okay
-        dict_ = {"abc": "123", "def": "456", "xyz": "789"}
-        errors = validate._validate_dict(dict_, "d", schema_def)
-        self.assertFalse(errors)
-
-        # Better not be empty come on
-        dict_ = {}
-        errors = validate._validate_dict(dict_, "d", schema_def)
-        self.assertEqual(len(errors), 2)
-
-    def test_nullable(self):
-
-        schema_def = {"keys": {"abc": {"type": "string", "nullable": False},
-                               "def": {"type": "string", "nullable": True}}}
-
-        dict_ = {"abc": "xyz", "def": ""}
-        errors = validate._validate_dict(dict_, "d", schema_def)
-        self.assertFalse(errors)
-
-        dict_ = {"abc": "", "def": ""}
-        errors = validate._validate_dict(dict_, "d", schema_def)
-        self.assertEqual(len(errors), 1)
-        self.assertIn("empty value", errors[0])
-
-    def test_recurse(self):
-
-        schema_def = {
-            "keys": {
-                "subdict": {
-                    "type": "dict",
-                    "keys": {
-                        "subdict_key1": None,
-                        "subdict_key2": None
-                    }
-                },
-                "ontology": {
-                    "type": "curie",
-                    "prefixes": ["ONTOLOGY"]
-                },
-                "blob": {
-                    "type": "stringified list of dicts"
-                }
-            }
-        }
-
-        dict_ = {
-            "subdict": {"subdict_key1": "any", "subdict_key2": "any"},
-            "ontology": "ONTOLOGY:123456",
-            "blob": json.dumps([{"abc": 123}, {"def": 456}])
-        }
-        errors = validate._validate_dict(dict_, "d", schema_def)
-        self.assertFalse(errors)
-
-        dict_ = {
-            "subdict": {"subdict_key1": "any"},
-            "ontology": "ONTOLOGY:123456",
-            "blob": json.dumps([{"abc": 123}, {"def": 456}])
-        }
-        errors = validate._validate_dict(dict_, "d", schema_def)
-        self.assertEqual(len(errors), 1)
-        self.assertIn("missing key", errors[0])
-
-        dict_ = {
-            "subdict": {"subdict_key1": "any", "subdict_key2": "any"},
-            "ontology": "oh no not an ontology term",
-            "blob": json.dumps([{"abc": 123}, {"def": 456}])
-        }
-        errors = validate._validate_dict(dict_, "d", schema_def)
-        self.assertEqual(len(errors), 1)
-        self.assertIn("invalid ontology", errors[0])
-
-        dict_ = {
-            "subdict": {"subdict_key1": "any", "subdict_key2": "any"},
-            "ontology": "ONTOLOGY:123456",
-            "blob": [{"abc": 123}, {"def": 456}]
-        }
-        errors = validate._validate_dict(dict_, "d", schema_def)
-        self.assertEqual(len(errors), 1)
-        self.assertIn("JSON-encoded list of dicts", errors[0])
-
-        # Multiple errors
-        dict_ = {
-            "subdict": {"subdict_key1": "any"},
-            "ontology": "oh no not an ontology term",
-            "blob": json.dumps([{"abc": 123}, {"def": 456}])
-        }
-        errors = validate._validate_dict(dict_, "d", schema_def)
-        self.assertEqual(len(errors), 2)
-
-
-class TestDataframeValidation(unittest.TestCase):
-
-    def test_column_presence(self):
-        df = pd.DataFrame(
-            [["abc", "EFO:123"],
-             ["def", "UBERON:456"]],
-            columns=["hr_string", "ontology"],
-            index=["X", "Y"]
-        )
-
-        schema_def = {
-            "columns": {
-                "hr_string": {"type": "human-readable string"},
-                "ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]}
-            }
-        }
-        errors = validate._validate_dataframe(df, "df", schema_def)
-        self.assertFalse(errors)
-
-        schema_def = {
-            "columns": {
-                "hr_string": {"type": "human-readable string"},
-                "another_hr_string": {"type": "human-readable string"},
-                "ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]}
-            }
-        }
-        errors = validate._validate_dataframe(df, "df", schema_def)
-        self.assertEqual(len(errors), 1)
-        self.assertIn("missing column", errors[0])
-
-        # Extra is okay
-        df = pd.DataFrame(
-            [["abc", "EFO:123", "extra"],
-             ["def", "UBERON:456", "extra"]],
-            columns=["hr_string", "ontology", "extra"],
-            index=["X", "Y"]
-        )
-        schema_def = {
-            "columns": {
-                "hr_string": {"type": "human-readable string"},
-                "ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]}
-            }
-        }
-        errors = validate._validate_dataframe(df, "df", schema_def)
-        self.assertFalse(errors)
-
-
-    def test_index(self):
-        df = pd.DataFrame(
-            [["abc", "123"],
-             ["def", "456"]],
-            columns=["col1", "col2"],
-            index=["ENSG0001", "ENSG0002"]
-        )
-
-        schema_def = {"index": {"unique": True}}
-        errors = validate._validate_dataframe(df, "df", schema_def)
-        self.assertFalse(errors)
-
-        schema_def = {"index": {"type": "human-readable string"}}
-        errors = validate._validate_dataframe(df, "df", schema_def)
-        self.assertEqual(len(errors), 1)
-        self.assertIn("non-human-readable", errors[0])
-
-        df = pd.DataFrame(
-            [["abc", "123"],
-             ["def", "456"]],
-            columns=["col1", "col2"],
-            index=["ENSG0001", "ENSG0001"]
-        )
-        schema_def = {"index": {"unique": True}}
-        errors = validate._validate_dataframe(df, "df", schema_def)
-        self.assertEqual(len(errors), 1)
-        self.assertIn("is not unique", errors[0])
-
-    def test_recurse(self):
-
-        df = pd.DataFrame(
-            [["abc", "HsapDv:0001"],
-             ["EFO:123", "UBERON:456"]],
-            columns=["hr_string", "ontology"],
-            index=["X", "Y"]
-        )
-        schema_def = {
-            "columns": {
-                "hr_string": {"type": "human-readable string"},
-                "ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]}
-            }
-        }
-        errors = validate._validate_dataframe(df, "df", schema_def)
-        self.assertEqual(len(errors), 2)
-        self.assertEqual(len([e for e in errors if "non-human-readable" in e]), 1)
-        self.assertEqual(len([e for e in errors if "invalid ontology" in e]), 1)
-
-
-class TestGetSchema(unittest.TestCase):
-
-    def test_get_schema(self):
-        self.assertIsInstance(validate.get_schema_definition("1.0.0"), dict)
-
-        with self.assertRaises(ValueError):
-            validate.get_schema_definition("10.1.5")
-
-
-class TestValidate(unittest.TestCase):
-
-    def setUp(self):
-        self.source_h5ad_path = f"{PROJECT_ROOT}/server/test/fixtures/pbmc3k-CSC-gz.h5ad"
-
-    def test_shallow(self):
-
-        adata = sc.read_h5ad(self.source_h5ad_path)
-        self.assertFalse(validate.validate_adata(adata, True))
-
-        adata.uns["version"] = {
-            "corpora_schema_version": "1.0.0",
-            "corpora_encoding_version": "0.1.0"
-        }
-        self.assertTrue(validate.validate_adata(adata, True))
-
-    def test_deep(self):
-        adata = sc.read_h5ad(self.source_h5ad_path)
-        self.assertFalse(validate.validate_adata(adata, False))
-
-        adata.uns["version"] = {
-            "corpora_schema_version": "1.0.0",
-            "corpora_encoding_version": "0.1.0"
-        }
-        self.assertFalse(validate.validate_adata(adata, False))

server/test/unit/converters/test_h5ad_data_file.py

@@ -254,7 +254,7 @@ class TestH5ADDataFile(unittest.TestCase):
         for metadata_field in CorporaConstants.REQUIRED_SIMPLE_METADATA_FIELDS:
             uns[metadata_field] = "random"
 
-        for metadata_field in CorporaConstants.OPTIONAL_JSON_ENCODED_METADATA_FIELD:
+        for metadata_field in CorporaConstants.REQUIRED_JSON_ENCODED_METADATA_FIELD:
             uns[metadata_field] = json.dumps({"random_key": "random_value"})
 
         # Need to carefully set the corpora schema versions in order for tests to pass.

server/test/unit/data_anndata/test_anndata_adaptor_data_load.py

@@ -16,7 +16,6 @@ class DataLoadAdaptorTest(unittest.TestCase):
         self.data_file = DataLocator(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
         config = AppConfig()
         config.update_server_config(single_dataset__datapath=self.data_file.path)
-        config.update_server_config(app__flask_secret_key="secret")
         config.complete_config()
         self.data = AnndataAdaptor(self.data_file, config)
 
@@ -46,7 +45,6 @@ class DataLocatorAdaptorTest(unittest.TestCase):
         config.update_server_config(
             single_dataset__obs_names=None, single_dataset__var_names=None,
         )
-        config.update_server_config(app__flask_secret_key="secret")
         config.update_default_dataset_config(
             embeddings__names=["umap"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01,
         )

setup.py

@@ -11,7 +11,7 @@ with open("server/requirements-prepare.txt") as fh:
 
 setup(
     name="cellxgene",
-    version="0.16.7",
+    version="0.16.5",
     packages=find_packages(),
     url="https://github.com/chanzuckerberg/cellxgene",
     license="MIT",