Merge branch 'main' into colinmegill/geneset-prototype

.bumpversion.cfg

@@ -1,17 +1,17 @@
 [bumpversion]
-current_version = 0.16.0
+current_version = 0.16.7
 commit = True
 # The below regex details an acceptable version number by naming the groups (major, minor, patch, prerel, and
 # prerelversion) and also specifying the valid values for each group (integers, `\d+`,  for major, minor, patch, and
 # prerelversion and only `rc` as the acceptable value for prerel).
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?:-(?P<prerel>rc)\.(?P<prerelversion>\d+))?
-serialize =
+serialize = 
 	{major}.{minor}.{patch}-{prerel}.{prerelversion}
 	{major}.{minor}.{patch}
 
 [bumpversion:part:prerel]
 optional_value = release
-values =
+values = 
 	rc
 	release
 

.github/workflows/push_tests.yml

@@ -36,7 +36,7 @@ jobs:
           npm install
       - name: Format with black and lint with flake8
         run: |
-          make lint-server
+          make lint-servers
       - name: Lint src with eslint
         working-directory: ./client
         run: |
@@ -72,6 +72,36 @@ jobs:
           bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k server -cF backend,python,unitTest
           cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest
 
+  unit-test-local:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Python cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Node cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Install dependencies
+        run: make pydist-local install-dist dev-env-local-server
+      - name: Unit tests
+        run: |
+          make unit-test-local
+          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k local_server -cF backend,python,unitTest
+          cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest
+
   smoke-tests:
     runs-on: macos-latest
     timeout-minutes: 20
@@ -126,7 +156,7 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-node-
       - name: Install dependencies
-        run: make pydist install-dist
+        run: make pydist-local install-dist
       - name: Smoke tests (with annotations feature)
         run: |
           cd client && make smoke-test-annotations

MANIFEST.in

@@ -1,7 +1,7 @@
-recursive-include server/common/web/templates *
-recursive-include server/common/web/static *
+recursive-include local_server/common/web/templates *
+recursive-include local_server/common/web/static *
 
-include server/requirements.txt
-include server/requirements-prepare.txt
-include server/converters/schema/hgnc_complete_set.txt.gz
-include server/converters/schema/schema_definitions/*
+include local_server/requirements.txt
+include local_server/requirements-prepare.txt
+include local_server/converters/schema/hgnc_complete_set.txt.gz
+include local_server/converters/schema/schema_definitions/*

MANIFEST_hosted.in

@@ -0,0 +1,7 @@
+recursive-include server/common/web/templates *
+recursive-include server/common/web/static *
+
+include server/requirements.txt
+include server/requirements-prepare.txt
+include server/converters/schema/hgnc_complete_set.txt.gz
+include server/converters/schema/schema_definitions/*

Makefile

@@ -3,13 +3,14 @@ include common.mk
 BUILDDIR := build
 CLIENTBUILD := $(BUILDDIR)/client
 SERVERBUILD := $(BUILDDIR)/server
+LOCALSERVERBUILD := $(BUILDDIR)/local_server
 CLEANFILES :=  $(BUILDDIR)/ client/build build dist cellxgene.egg-info
 
 PART ?= patch
 
 # CLEANING
 .PHONY: clean
-clean: clean-lite clean-server clean-client
+clean: clean-lite clean-local-server clean-server clean-client
 
 # cleaning the client's node_modules is the longest one, so we avoid that if possible
 .PHONY: clean-lite
@@ -17,7 +18,7 @@ clean-lite:
 	rm -rf $(CLEANFILES)
 
 clean-%:
-	cd $(*) && $(MAKE) clean
+	cd $(subst -,_,$*) && $(MAKE) clean
 
 
 # BUILDING PACKAGE
@@ -26,18 +27,35 @@ clean-%:
 build-client:
 	cd client && $(MAKE) ci build
 
+.PHONY: build-local
+build-local: clean build-client
+	git ls-files local_server/ | grep -v 'local_server/test/' | cpio -pdm $(BUILDDIR)
+	cp -r client/build/  $(CLIENTBUILD)
+	$(call copy_client_assets,$(CLIENTBUILD),$(LOCALSERVERBUILD))
+	cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR)
+
 .PHONY: build
 build: clean build-client
 	git ls-files server/ | grep -v 'server/test/' | cpio -pdm $(BUILDDIR)
 	cp -r client/build/  $(CLIENTBUILD)
 	$(call copy_client_assets,$(CLIENTBUILD),$(SERVERBUILD))
-	cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR)
+	cp MANIFEST_hosted.in README.md setup.cfg setup_hosted.py $(BUILDDIR)
+	mv $(BUILDDIR)/setup_hosted.py $(BUILDDIR)/setup.py
+	mv $(BUILDDIR)/MANIFEST_hosted.in $(BUILDDIR)/MANIFEST.in
 
 # If you are actively developing in the server folder use this, dirties the source tree
+.PHONY: build-for-server-dev-local
+build-for-server-dev-local: clean-local-server build-client
+	$(call copy_client_assets,client/build,local_server)
+
 .PHONY: build-for-server-dev
 build-for-server-dev: clean-server build-client
 	$(call copy_client_assets,client/build,server)
 
+.PHONY: copy-client-assets-local
+copy-client-assets-local:
+	$(call copy_client_assets,client/build,local_server)
+
 .PHONY: copy-client-assets
 copy-client-assets:
 	$(call copy_client_assets,client/build,server)
@@ -46,11 +64,17 @@ copy-client-assets:
 .PHONY: test
 test: unit-test smoke-test
 
+.PHONY: test-local
+test-local: unit-test-local smoke-test
+
+.PHONY: unit-test-local
+unit-test-local: unit-test-local-server unit-test-client
+
 .PHONY: unit-test
 unit-test: unit-test-server unit-test-client
 
 unit-test-%:
-	cd $(*) && $(MAKE) unit-test
+	cd $(subst -,_,$*) && $(MAKE) unit-test
 
 .PHONY: smoke-test
 smoke-test:
@@ -64,10 +88,9 @@ smoke-test-annotations:
 test-db:
 	cd server && $(MAKE) test-db
 
-
 # FORMATTING CODE
 
-.PHOHY: fmt
+.PHONY: fmt
 fmt: fmt-client fmt-py
 
 .PHONY: fmt-client
@@ -79,19 +102,30 @@ fmt-py:
 	black .
 
 .PHONY: lint
-lint: lint-server lint-client
+lint: lint-servers lint-client
+
+.PHONY: lint-servers
+lint-servers: lint-local-server lint-server
+
+.PHONY: lint-local-server
+lint-local-server: fmt-py
+	flake8 local_server --per-file-ignores='local_server/test/fixtures/dataset_config_outline.py:F821 local_server/test/fixtures/server_config_outline.py:F821 local_server/test/performance/scale_test_annotations.py:E501'
 
 .PHONY: lint-server
 lint-server: fmt-py
 	flake8 server --per-file-ignores='server/test/fixtures/dataset_config_outline.py:F821 server/test/fixtures/server_config_outline.py:F821 server/test/performance/scale_test_annotations.py:E501'
 
-
 .PHONY: lint-client
 lint-client:
 	cd client && $(MAKE) lint
 
 # CREATING DISTRIBUTION RELEASE
 
+.PHONY: pydist-local
+pydist-local: build-local
+	cd $(BUILDDIR); python setup.py sdist -d ../dist
+	@echo "done"
+
 .PHONY: pydist
 pydist: build
 	cd $(BUILDDIR); python setup.py sdist -d ../dist
@@ -137,16 +171,19 @@ release-directly-to-prod: dev-env pydist twine-prod
 	@echo "    make install-release"
 
 .PHONY: dev-env
-dev-env: dev-env-client dev-env-server
+dev-env: dev-env-client dev-env-local-server
 
 .PHONY: dev-env-client
 dev-env-client:
 	cd client && $(MAKE) ci
 
+.PHONY: dev-env-local-server
+dev-env-local-server:
+	pip install -r local_server/requirements-dev.txt
+
 .PHONY: dev-env-server
 dev-env-server:
 	pip install -r server/requirements-dev.txt
-
 # Set PART=[major, minor, patch] as param to make bump.
 # This will create a release candidate. (i.e. 0.16.1 -> 0.16.2-rc.0 for a patch bump)
 .PHONY: bump-version

README.md

@@ -73,7 +73,16 @@ This project adheres to the Contributor Covenant [code of conduct](https://githu
 
 ### Reuse
 
-This project was started with the sole goal of empowering the scientific community to explore and understand their data. As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from this project, and reach out to us with ideas or questions. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT).
+This project was started with the sole goal of empowering the scientific community to explore and understand their data. 
+As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from 
+this project. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT).
+
+
+Before extending cellxgene, we encourage you to reach out to us with ideas or questions. It might be possible that an 
+extension could be directly contributed, which would make it available for a wider audience, or that it's on our 
+[roadmap](./docs/posts/roadmap.md) and under active development. 
+
+See the [cellxgene extensions](./docs/posts/extensions.md) section of our documentation for examples of community use and cellxgene extensions. 
 
 ### Security
 

client/Makefile

@@ -1,6 +1,6 @@
 include ../common.mk
 
-ANNOTATIONS := $(if $(ANNOTATIONS),$(ANNOTATIONS),../server/test/fixtures/pbmc3k-annotations.csv)
+ANNOTATIONS := $(if $(ANNOTATIONS),$(ANNOTATIONS),../local_server/test/fixtures/pbmc3k-annotations.csv)
 ANNOTATIONS_FILENAME := $(shell basename $(ANNOTATIONS))
 
 CXG_CONFIG := $(if $(CXG_CONFIG), $(CXG_CONFIG), ./__tests__/e2e/test_config.yaml)

client/__tests__/e2e/test_config.yaml

@@ -1,20 +1,7 @@
 server:
   app:
     force_https: true
-
-    # By default, cellxgene will serve api requests from the same base url as the webpage.
-    # In general api_base_url and web_base_url will not need to be set.
-    # There are two reasons to set these parameters:
-    #  1. Oauth authentication is used; the oauth server will redirect back to the api_base_url after login,
-    #     which then redirects back to the web_base_url.  If the web_base_url is not set, it will default to
-    #     the api_base_url.  If oauth authentication is used, the api_base_url must be set.
-    #     For a local test (where the server runs on "http://localhost:<port>"), then the api_base_url may be
-    #     set to the string "local".
-    #  2. The cellxgene deploymnent is in an environment where the webpage and api have
-    #     different base urls.  In this case both api_base_url and web_base_url must be set.
-    #     It is up to the server admin to ensure that the networking is setup correctly for this environment.
-    api_base_url: http://localhost:5005
-    web_base_url: http://localhost:3000
+    port: 5005
 
   authentication:
     # The authentication types may be "none", "session", "oauth"
@@ -22,12 +9,9 @@ server:
     # session:  A session based userid is automatically generated. (no params needed)
     # oauth: oauth2 is used for authentication;  parameters are defined in params_oauth.
     type: test
+    insecure_test_environment: true
 
 dataset:
-  app:
-    about_legal_tos: null
-    about_legal_privacy: null
-
   presentation:
     max_categories: 1000
     custom_colors: true

client/__tests__/reducers/genesets.test.js

@@ -0,0 +1,524 @@
+import genesetsReducer from "../../src/reducers/genesets";
+
+describe("initial reducer state", () => {
+  test("some other action", () => {
+    expect(genesetsReducer(undefined, { type: "foo" })).toMatchObject({
+      initialized: false,
+      lastTid: undefined,
+      genesets: new Map(),
+    });
+  });
+});
+
+describe("geneset: initial load", () => {
+  test("missing JSON response", () => {
+    expect(() =>
+      genesetsReducer(undefined, {
+        type: "geneset: initial load",
+      })
+    ).toThrow("missing or malformed JSON response");
+  });
+
+  test("empty geneset", () => {
+    expect(
+      genesetsReducer(undefined, {
+        type: "geneset: initial load",
+        data: {
+          tid: 0,
+          genesets: [],
+        },
+      })
+    ).toMatchObject({
+      initialized: true,
+      lastTid: 0,
+      genesets: new Map(),
+    });
+  });
+
+  test("non-empty geneset", () => {
+    expect(
+      genesetsReducer(undefined, {
+        type: "geneset: initial load",
+        data: {
+          tid: 99,
+          genesets: [
+            {
+              geneset_name: "G1",
+              genes: [{ gene_symbol: "F5" }],
+            },
+            {
+              geneset_name: "G2",
+              geneset_description: "G2 desc",
+              genes: [{ gene_symbol: "F6" }],
+            },
+            {
+              geneset_name: "G3",
+              geneset_description: "G3 desc",
+              genes: [{ gene_symbol: "F7", gene_description: "gene desc" }],
+            },
+          ],
+        },
+      })
+    ).toMatchObject({
+      initialized: true,
+      lastTid: 99,
+      genesets: new Map([
+        [
+          "G1",
+          {
+            genesetName: "G1",
+            genesetDescription: "",
+            genes: new Map([["F5", { geneSymbol: "F5", geneDescription: "" }]]),
+          },
+        ],
+        [
+          "G2",
+          {
+            genesetName: "G2",
+            genesetDescription: "G2 desc",
+            genes: new Map([["F6", { geneSymbol: "F6", geneDescription: "" }]]),
+          },
+        ],
+        [
+          "G3",
+          {
+            genesetName: "G3",
+            genesetDescription: "G3 desc",
+            genes: new Map([
+              ["F7", { geneSymbol: "F7", geneDescription: "gene desc" }],
+            ]),
+          },
+        ],
+      ]),
+    });
+  });
+});
+
+describe("geneset: create", () => {
+  const initialState = genesetsReducer(undefined, {
+    type: "geneset: initial load",
+    data: {
+      tid: 0,
+      genesets: [],
+    },
+  });
+
+  test("simple create", () => {
+    expect(
+      genesetsReducer(initialState, {
+        type: "geneset: create",
+        genesetName: "a geneset",
+        genesetDescription: "",
+      })
+    ).toMatchObject({
+      ...initialState,
+      genesets: new Map([
+        [
+          "a geneset",
+          {
+            genesetName: "a geneset",
+            genesetDescription: "",
+            genes: new Map(),
+          },
+        ],
+      ]),
+    });
+  });
+
+  test("error - duplicate name", () => {
+    expect(() => {
+      genesetsReducer(
+        genesetsReducer(initialState, {
+          type: "geneset: create",
+          genesetName: "foo",
+          genesetDescription: "foo",
+        }),
+        {
+          type: "geneset: create",
+          genesetName: "foo",
+          genesetDescription: "bar",
+        }
+      );
+    }).toThrow("name already defined");
+  });
+
+  test("error - missing required action values", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: create",
+        genesetDescription: "foo",
+      });
+    }).toThrow();
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: create",
+        genesetName: "foo",
+      });
+    }).toThrow("name or description unspecified");
+  });
+});
+
+describe("geneset: delete", () => {
+  const initialState = genesetsReducer(undefined, {
+    type: "geneset: initial load",
+    data: {
+      tid: 0,
+      genesets: [],
+    },
+  });
+
+  test("simple delete", () => {
+    expect(
+      genesetsReducer(
+        genesetsReducer(initialState, {
+          type: "geneset: create",
+          genesetName: "foo",
+          genesetDescription: "foo",
+        }),
+        {
+          type: "geneset: delete",
+          genesetName: "foo",
+        }
+      )
+    ).toMatchObject({
+      initialized: true,
+      lastTid: 0,
+      genesets: new Map(),
+    });
+  });
+
+  test("error - missing name", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: delete",
+        genesetName: "foo",
+      });
+    }).toThrow("name does not exist");
+  });
+});
+
+describe("geneset: update", () => {
+  const initialState = genesetsReducer(undefined, {
+    type: "geneset: initial load",
+    data: {
+      tid: 0,
+      genesets: [],
+    },
+  });
+
+  test("simple update", () => {
+    expect(
+      genesetsReducer(
+        genesetsReducer(
+          genesetsReducer(initialState, {
+            type: "geneset: create",
+            genesetName: "foo1",
+            genesetDescription: "foo1",
+          }),
+          {
+            type: "geneset: create",
+            genesetName: "foo2",
+            genesetDescription: "foo2",
+          }
+        ),
+        {
+          type: "geneset: update",
+          genesetName: "foo1",
+          update: {
+            genesetName: "bar",
+            genesetDescription: "bar",
+          },
+        }
+      )
+    ).toMatchObject({
+      initialized: true,
+      lastTid: 0,
+      genesets: new Map([
+        [
+          "bar",
+          { genesetName: "bar", genesetDescription: "bar", genes: new Map() },
+        ],
+        [
+          "foo2",
+          { genesetName: "foo2", genesetDescription: "foo2", genes: new Map() },
+        ],
+      ]),
+    });
+  });
+
+  test("error - unknown name", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: update",
+        genesetName: "foo",
+        update: {
+          genesetName: "foo",
+          genesetDescription: "bar",
+        },
+      });
+    }).toThrow("name unspecified or does not exist");
+  });
+
+  test("error - duplicate name", () => {
+    expect(() => {
+      genesetsReducer(
+        genesetsReducer(initialState, {
+          type: "geneset: create",
+          genesetName: "foo",
+          genesetDescription: "foo",
+        }),
+        {
+          type: "geneset: update",
+          genesetName: "foo",
+          update: {
+            genesetName: "foo",
+            genesetDescription: "foo",
+          },
+        }
+      );
+    }).toThrow("update specified existing name");
+  });
+});
+
+describe("geneset: add genes", () => {
+  const initialState = genesetsReducer(
+    genesetsReducer(undefined, {
+      type: "geneset: initial load",
+      data: {
+        tid: 0,
+        genesets: [],
+      },
+    }),
+    {
+      type: "geneset: create",
+      genesetName: "test",
+      genesetDescription: "",
+    }
+  );
+
+  test("add a gene", () => {
+    expect(
+      genesetsReducer(initialState, {
+        type: "geneset: add genes",
+        genesetName: "test",
+        genes: [{ geneSymbol: "F5" }],
+      })
+    ).toMatchObject({
+      ...initialState,
+      genesets: new Map([
+        [
+          "test",
+          {
+            genesetName: "test",
+            genesetDescription: "",
+            genes: new Map([["F5", { geneSymbol: "F5", geneDescription: "" }]]),
+          },
+        ],
+      ]),
+    });
+
+    expect(
+      genesetsReducer(initialState, {
+        type: "geneset: add genes",
+        genesetName: "test",
+        genes: [
+          { geneSymbol: "F5", geneDescription: "desc" },
+          { geneSymbol: "SET1", geneDescription: "" },
+        ],
+      })
+    ).toMatchObject({
+      ...initialState,
+      genesets: new Map([
+        [
+          "test",
+          {
+            genesetName: "test",
+            genesetDescription: "",
+            genes: new Map([
+              ["F5", { geneSymbol: "F5", geneDescription: "desc" }],
+              ["SET1", { geneSymbol: "SET1", geneDescription: "" }],
+            ]),
+          },
+        ],
+      ]),
+    });
+  });
+
+  test("no such geneset error", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: add genes",
+        genesetName: "mumble",
+        genes: [],
+      });
+    }).toThrow("geneset name does not exist");
+  });
+});
+
+describe("geneset: delete genes", () => {
+  const initialState = genesetsReducer(
+    genesetsReducer(
+      genesetsReducer(undefined, {
+        type: "geneset: initial load",
+        data: {
+          tid: 0,
+          genesets: [],
+        },
+      }),
+      {
+        type: "geneset: create",
+        genesetName: "test",
+        genesetDescription: "",
+      }
+    ),
+    {
+      type: "geneset: add genes",
+      genesetName: "test",
+      genes: [{ geneSymbol: "F5" }],
+    }
+  );
+
+  test("simple", () => {
+    expect(
+      genesetsReducer(initialState, {
+        type: "geneset: delete genes",
+        genesetName: "test",
+        geneSymbols: ["F5"],
+      })
+    ).toMatchObject({
+      ...initialState,
+      genesets: new Map([
+        [
+          "test",
+          {
+            genesetName: "test",
+            genesetDescription: "",
+            genes: new Map(),
+          },
+        ],
+      ]),
+    });
+  });
+
+  test("no such geneset error", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: delete genes",
+        genesetName: "mumble",
+        geneSymbols: [],
+      });
+    }).toThrow("name does not exist");
+  });
+});
+
+describe("geneset: set gene description", () => {
+  const initialState = genesetsReducer(
+    genesetsReducer(
+      genesetsReducer(undefined, {
+        type: "geneset: initial load",
+        data: {
+          tid: 0,
+          genesets: [],
+        },
+      }),
+      {
+        type: "geneset: create",
+        genesetName: "test",
+        genesetDescription: "",
+      }
+    ),
+    {
+      type: "geneset: add genes",
+      genesetName: "test",
+      genes: [{ geneSymbol: "F5" }],
+    }
+  );
+
+  test("simple set", () => {
+    expect(
+      genesetsReducer(initialState, {
+        type: "geneset: set gene description",
+        genesetName: "test",
+        update: {
+          geneSymbol: "F5",
+          geneDescription: "mumble",
+        },
+      })
+    ).toMatchObject({
+      ...initialState,
+      genesets: new Map([
+        [
+          "test",
+          {
+            genesetName: "test",
+            genesetDescription: "",
+            genes: new Map([
+              ["F5", { geneSymbol: "F5", geneDescription: "mumble" }],
+            ]),
+          },
+        ],
+      ]),
+    });
+  });
+
+  test("no such geneset error", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: set gene description",
+        genesetName: "does not exist",
+        update: {
+          geneSymbol: "F5",
+          geneDescription: "mumble",
+        },
+      });
+    }).toThrow("geneset name does not exist");
+  });
+
+  test("no such gene error", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: set gene description",
+        genesetName: "test",
+        update: {
+          geneSymbol: "NO SUCH GENE",
+          geneDescription: "mumble",
+        },
+      });
+    }).toThrow("no such gene");
+  });
+});
+
+describe("geneset: set tid", () => {
+  test("simple set", () => {
+    expect(
+      genesetsReducer(undefined, {
+        type: "geneset: set tid",
+        tid: 1,
+      })
+    ).toMatchObject({ lastTid: 1 });
+  });
+
+  test("not a number error", () => {
+    expect(() => {
+      genesetsReducer(
+        { lastTid: 1 },
+        {
+          type: "geneset: set tid",
+          tid: "0",
+        }
+      );
+    }).toThrow("must be a positive integer");
+  });
+
+  test("decrement error", () => {
+    expect(() => {
+      genesetsReducer(
+        { lastTid: 1 },
+        {
+          type: "geneset: set tid",
+          tid: 0,
+        }
+      );
+    }).toThrow("may not be decremented");
+  });
+});

client/__tests__/util/stateManager/colorHelpers.test.js

@@ -0,0 +1,198 @@
+/* eslint-disable no-bitwise -- unsigned right shift better than Math.round */
+
+/*
+test color helpers
+*/
+import {
+  createColorTable,
+  loadUserColorConfig,
+} from "../../../src/util/stateManager/colorHelpers";
+import * as Dataframe from "../../../src/util/dataframe";
+
+describe("categorical color helpers", () => {
+  /*
+  Primary test constraint for categorical colors is that they are ordered/identified
+  by schema order, NOT by value.  Ie,
+
+    scale(schemaIndex) should match rgb[obsOffset]
+  */
+
+  const schema = indexSchema({
+    annotations: {
+      obs: {
+        columns: [
+          {
+            name: "name_0",
+            type: "string",
+            writable: false,
+          },
+          {
+            name: "continuousColumn",
+            type: "float32",
+            writable: false,
+          },
+          {
+            categories: [
+              "CD4 T cells",
+              "CD14+ Monocytes",
+              "B cells",
+              "CD8 T cells",
+              "NK cells",
+              "FCGR3A+ Monocytes",
+              "Dendritic cells",
+              "Megakaryocytes",
+            ],
+            name: "categoricalColumn",
+            type: "categorical",
+            writable: false,
+          },
+        ],
+        index: "name_0",
+      },
+      var: {
+        columns: [
+          {
+            name: "name_0",
+            type: "string",
+            writable: false,
+          },
+        ],
+        index: "name_0",
+      },
+    },
+    dataframe: {
+      nObs: 2638,
+      nVar: 1838,
+      type: "float32",
+    },
+    layout: {},
+  });
+
+  const catColCategories = schema.annotations.obs.columns[2].categories;
+  const obsDataframe = new Dataframe.Dataframe(
+    [schema.dataframe.nObs, 2],
+    [
+      new Float32Array(schema.dataframe.nObs).map(() => Math.random()),
+      new Array(schema.dataframe.nObs)
+        .fill("")
+        .map(
+          () =>
+            catColCategories[(Math.random() * catColCategories.length) >>> 0]
+        ),
+    ],
+    null,
+    new Dataframe.KeyIndex(["continuousColumn", "categoricalColumn"])
+  );
+
+  test("default category order", () => {
+    const ct = createColorTable(
+      "color by categorical metadata",
+      "categoricalColumn",
+      obsDataframe,
+      schema
+    );
+    expect(ct).toBeDefined();
+    const data = obsDataframe.col("categoricalColumn").asArray();
+    const cats = schema.annotations.obsByName.categoricalColumn.categories;
+    for (let i = 0; i < schema.dataframe.nObs; i += 1) {
+      expect(makeScale(ct.rgb[i])).toEqual(ct.scale(cats.indexOf(data[i])));
+    }
+  });
+
+  test("shuffle category order", () => {
+    const schemaClone = indexSchema(JSON.parse(JSON.stringify(schema)));
+    shuffle(schemaClone.annotations.obsByName.categoricalColumn.categories);
+    const ct = createColorTable(
+      "color by categorical metadata",
+      "categoricalColumn",
+      obsDataframe,
+      schemaClone
+    );
+    expect(ct).toBeDefined();
+    const data = obsDataframe.col("categoricalColumn").asArray();
+    const cats = schemaClone.annotations.obsByName.categoricalColumn.categories;
+    for (let i = 0; i < schemaClone.dataframe.nObs; i += 1) {
+      expect(makeScale(ct.rgb[i])).toEqual(ct.scale(cats.indexOf(data[i])));
+    }
+  });
+
+  test("user defined color order", () => {
+    const cats = schema.annotations.obsByName.categoricalColumn.categories;
+    const shuffleCats = shuffle(
+      Array.from(schema.annotations.obsByName.categoricalColumn.categories)
+    );
+    const userDefinedColorTable = {
+      categoricalColumn: shuffleCats.reduce((acc, label) => {
+        acc[label] = randRGBColor();
+        return acc;
+      }, {}),
+    };
+
+    const userColors = loadUserColorConfig(userDefinedColorTable);
+    expect(userColors).toBeDefined();
+
+    const ct = createColorTable(
+      "color by categorical metadata",
+      "categoricalColumn",
+      obsDataframe,
+      schema,
+      userColors
+    );
+    expect(ct).toBeDefined();
+    const data = obsDataframe.col("categoricalColumn").asArray();
+    for (let i = 0; i < schema.dataframe.nObs; i += 1) {
+      expect(makeScale(ct.rgb[i])).toEqual(
+        ct.scale(cats.indexOf(data[i])).toString()
+      );
+    }
+  });
+});
+
+/*
+TODO:
+1. mix up category order in schema to make sure it works with varied order
+2. user defined colors
+*/
+
+function indexSchema(schema) {
+  schema.annotations.obsByName = Object.fromEntries(
+    schema.annotations?.obs?.columns?.map((v) => [v.name, v]) ?? []
+  );
+  schema.annotations.varByName = Object.fromEntries(
+    schema.annotations?.var?.columns?.map((v) => [v.name, v]) ?? []
+  );
+  schema.layout.obsByName = Object.fromEntries(
+    schema.layout?.obs?.map((v) => [v.name, v]) ?? []
+  );
+  schema.layout.varByName = Object.fromEntries(
+    schema.layout?.var?.map((v) => [v.name, v]) ?? []
+  );
+
+  return schema;
+}
+
+function makeScale(rgb) {
+  // make a scale string from a rgb float triple
+  return `rgb(${(rgb[0] * 255) >>> 0}, ${(rgb[1] * 255) >>> 0}, ${
+    (rgb[2] * 256) >>> 0
+  })`;
+}
+
+function shuffle(array) {
+  for (let i = array.length - 1; i > 0; i -= 1) {
+    const j = (Math.random() * (i + 1)) >>> 0;
+    [array[i], array[j]] = [array[j], array[i]];
+  }
+  return array;
+}
+
+function randHexColor() {
+  const hex = ((Math.random() * 255) >>> 0).toString(16);
+  return `0${hex}`.slice(-2);
+}
+
+function randRGBColor() {
+  return `#${randHexColor()}${randHexColor()}${randHexColor()}`;
+}
+
+/* eslint-enable no-bitwise -- unsigned right shift better than Math.round */

client/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "cellxgene",
-  "version": "0.16.0",
+  "version": "0.16.7",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cellxgene",
-  "version": "0.16.0",
+  "version": "0.16.7",
   "license": "MIT",
   "description": "cellxgene is a web application for the interactive exploration of single cell sequence data.",
   "repository": "https://github.com/chanzuckerberg/cellxgene",

client/src/actions/annotation.js

@@ -390,41 +390,51 @@ export const saveObsAnnotationsAction = () => async (dispatch, getState) => {
 
 export const saveGenesetsAction = () => async (dispatch, getState) => {
   const state = getState();
-  const { config, genesets, annotations } = state;
 
   // bail if gene sets not available, or in readonly mode.
+  const { config } = state;
   const genesetsAreAvailable =
     config?.parameters?.["annotations_genesets"] ?? false;
   const genesetsReadonly =
     config?.parameters?.["annotations_genesets_readonly"] ?? true;
   if (!genesetsAreAvailable || genesetsReadonly) {
     // our non-save was completed!
-    dispatch({
+    return dispatch({
       type: "autosave: genesets complete",
-      lastSavedGenesets: genesets,
+      lastSavedGenesets: state.genesets,
     });
   }
 
-  /*
-   JSON data structure is an array of arrays, where the first
-   element is gene set name, remainder are the genes.  Eg,
+  const { lastTid, genesets: lastGenesets } = state.genesets;
 
-    {
-      "genesets": [
-        [ "gs1", ["TNFRSF4","SUMO3","BRWD1"]],
-        [ "gs2", ["DSCR3", "BRWD1", "BACE2", "SIK1", "C21orf33", "ICOSLG", "SUMO3"]]
-      ]
+  /* Create the JSON OTA data structure */
+  const tid = (lastTid ?? 0) + 1;
+  const genesets = [];
+  for (const [name, gs] of lastGenesets) {
+    const genes = [];
+    for (const g of gs.genes.values()) {
+      genes.push({
+        gene_symbol: g.geneSymbol,
+        gene_description: g.geneDescription,
+      });
     }
-
-  Order of gene sets and genes is significant
-  */
-  const gsArr = [];
-  for (const [gsName, gsGenes] of genesets.genesets) {
-    gsArr.push([gsName, Array.from(gsGenes)]);
+    genesets.push({
+      geneset_name: name,
+      geneset_description: gs.genesetDescription,
+      genes,
+    });
   }
+  const ota = {
+    tid,
+    genesets,
+  };
 
+  /* Save to server */
   try {
-    const { dataCollectionNameIsReadOnly, dataCollectionName } = annotations;
+    const {
+      dataCollectionNameIsReadOnly,
+      dataCollectionName,
+    } = state.annotations;
     const queryString =
       !dataCollectionNameIsReadOnly && !!dataCollectionName
         ? `?annotation-collection-name=${encodeURIComponent(
@@ -440,26 +450,29 @@ export const saveGenesetsAction = () => async (dispatch, getState) => {
           Accept: "application/json",
           "Content-Type": "application/json",
         }),
-        body: JSON.stringify({
-          genesets: gsArr,
-        }),
+        body: JSON.stringify(ota),
         credentials: "include",
       }
     );
-    if (res.ok) {
-      dispatch({
-        type: "autosave: genesets complete",
-        lastSavedGenesets: genesets,
-      });
-    } else {
-      dispatch({
+    if (!res.ok) {
+      return dispatch({
         type: "autosave: genesets error",
         message: `HTTP error ${res.status} - ${res.statusText}`,
         res,
       });
     }
+    return Promise.all([
+      dispatch({
+        type: "autosave: genesets complete",
+        lastSavedGenesets: genesets,
+      }),
+      dispatch({
+        type: "geneset: set tid",
+        tid,
+      }),
+    ]);
   } catch (error) {
-    dispatch({
+    return dispatch({
       type: "autosave: genesets error",
       message: error.toString(),
       error,

client/src/actions/index.js

@@ -53,18 +53,22 @@ async function userInfoFetch(dispatch) {
 }
 
 async function genesetsFetch(dispatch, config) {
+  /* request genesets ONLY if the backend supports the feature */
+  const defaultResponse = {
+    genesets: [],
+    tid: 0,
+  };
   if (config?.parameters?.["annotations_genesets"] ?? false) {
     fetchJson("genesets").then((response) => {
-      const genesets = response?.genesets ?? {};
       dispatch({
         type: "geneset: initial load",
-        init: genesets,
+        data: response ?? defaultResponse,
       });
     });
   } else {
     dispatch({
       type: "geneset: initial load",
-      init: [],
+      data: defaultResponse,
     });
   }
 }

client/src/components/autosave/index.js

@@ -11,7 +11,7 @@ import FilenameDialog from "./filenameDialog";
   error: state.autosave?.error,
   writableCategoriesEnabled: state.config?.parameters?.annotations ?? false,
   writableGenesetsEnabled: !(
-    state.config?.parameters?.annotations_genesets_readonly ?? true
+    state.config?.parameters?.["annotations_genesets_readonly"] ?? true
   ),
   annoMatrix: state.annoMatrix,
   genesets: state.genesets,
@@ -27,11 +27,11 @@ class Autosave extends React.Component {
   }
 
   componentDidMount() {
-    const { writableCategoriesEnabled } = this.props;
+    const { writableCategoriesEnabled, writableGenesetsEnabled } = this.props;
 
     let { timer } = this.state;
     if (timer) clearInterval(timer);
-    if (writableCategoriesEnabled) {
+    if (writableCategoriesEnabled || writableGenesetsEnabled) {
       timer = setInterval(this.tick, 2500);
     } else {
       timer = null;
@@ -58,11 +58,6 @@ class Autosave extends React.Component {
     }
   };
 
-  saveInProgress() {
-    const { obsAnnotationSaveInProgress, genesetSaveInProgress } = this.props;
-    return obsAnnotationSaveInProgress || genesetSaveInProgress;
-  }
-
   needToSaveObsAnnotations = () => {
     /* return true if we need to save obs cell labels, false if we don't */
     const { annoMatrix, lastSavedAnnoMatrix } = this.props;
@@ -79,6 +74,11 @@ class Autosave extends React.Component {
     return this.needToSaveGenesets() || this.needToSaveObsAnnotations();
   }
 
+  saveInProgress() {
+    const { obsAnnotationSaveInProgress, genesetSaveInProgress } = this.props;
+    return obsAnnotationSaveInProgress || genesetSaveInProgress;
+  }
+
   statusMessage() {
     const { error } = this.props;
     if (error) {
@@ -88,10 +88,14 @@ class Autosave extends React.Component {
   }
 
   render() {
-    const { writableCategoriesEnabled, lastSavedAnnoMatrix } = this.props;
+    const {
+      writableCategoriesEnabled,
+      writableGenesetsEnabled,
+      lastSavedAnnoMatrix,
+    } = this.props;
     const initialDataLoadComplete = lastSavedAnnoMatrix;
 
-    if (!writableCategoriesEnabled) return null;
+    if (!writableCategoriesEnabled && !writableGenesetsEnabled) return null;
 
     return (
       <div

client/src/components/categorical/value/index.js

@@ -174,7 +174,7 @@ class CategoryValue extends React.Component {
     Checks to see if at least one of the following changed:
     * world state
     * the color accessor (what is currently being colored by)
-    * if this catagorical value's selection status has changed
+    * if this categorical value's selection status has changed
     * the crossfilter (ie, global selection state)
 
     If and only if true, update the component
@@ -201,6 +201,13 @@ class CategoryValue extends React.Component {
     const newCount = newCategorySummary.categoryValueCounts[newCategoryIndex];
     const countChanged = count !== newCount;
 
+    // If the user edits an annotation that is currently colored-by, colors may be re-assigned.
+    // This test is conservative - it may cause re-rendering of entire category (all labels)
+    // if any one changes, but only for the currently colored-by category.
+    const colorMightHaveChanged =
+      nextProps.colorAccessor === nextProps.metadataField &&
+      props.categorySummary !== nextProps.categorySummary;
+
     return (
       labelChanged ||
       valueSelectionChange ||
@@ -208,7 +215,8 @@ class CategoryValue extends React.Component {
       annotationsChange ||
       editingLabel ||
       dilationChange ||
-      countChanged
+      countChanged ||
+      colorMightHaveChanged
     );
   };
 

client/src/components/graph/drawPointsRegl.js

@@ -14,7 +14,7 @@ export default function drawPointsRegl(regl) {
     uniform float nPoints;
     uniform float minViewportDimension;
 
-    varying vec4 fragColor;
+    varying lowp vec4 fragColor;
 
     const float zBottom = 0.99;
     const float zMiddle = 0.;
@@ -27,23 +27,23 @@ export default function drawPointsRegl(regl) {
     ${glPointSize}
 
     void main() {
-      bool isNaN, isSelected, isHighlight;
-      getFlags(flag, isNaN, isSelected, isHighlight);
+      bool isBackground, isSelected, isHighlight;
+      getFlags(flag, isBackground, isSelected, isHighlight);
 
       float size = pointSize(nPoints, minViewportDimension, isSelected, isHighlight);
       gl_PointSize = size * pow(distance, 0.5);
 
-      float z = isNaN ? zBottom : (isHighlight ? zTop : zMiddle);
+      float z = isBackground ? zBottom : (isHighlight ? zTop : zMiddle);
       vec3 xy = projView * vec3(position, 1.);
       gl_Position = vec4(xy.xy, z, 1.);
 
-      float alpha = isNaN ? 0.9 : 1.0;
+      float alpha = isBackground ? 0.9 : 1.0;
       fragColor = vec4(color, alpha);
     }`,
 
     frag: `
     precision mediump float;
-    varying vec4 fragColor;
+    varying lowp vec4 fragColor;
     void main() {
       if (length(gl_PointCoord.xy - 0.5) > 0.5) {
         discard;
@@ -67,5 +67,15 @@ export default function drawPointsRegl(regl) {
     count: regl.prop("count"),
 
     primitive: "points",
+
+    blend: {
+      enable: true,
+      func: {
+        srcRGB: "src alpha",
+        srcAlpha: 1,
+        dstRGB: 0,
+        dstAlpha: "zero",
+      },
+    },
   });
 }

client/src/components/graph/graph.js

@@ -22,6 +22,12 @@ import CentroidLabels from "./overlays/centroidLabels";
 import actions from "../../actions";
 import renderThrottle from "../../util/renderThrottle";
 
+import {
+  flagBackground,
+  flagSelected,
+  flagHighlight,
+} from "../../util/glHelpers";
+
 /*
 Simple 2D transforms control all point painting.  There are three:
   * model - convert from underlying per-point coordinate to a layout.
@@ -62,10 +68,6 @@ function createModelTF() {
   return m;
 }
 
-const flagSelected = 1;
-const flagNaN = 2;
-const flagHighlight = 4;
-
 @connect((state) => ({
   annoMatrix: state.annoMatrix,
   crossfilter: state.obsCrossfilter,
@@ -159,8 +161,9 @@ class Graph extends React.Component {
     const flags = new Float32Array(nObs);
     if (colorByData) {
       for (let i = 0, len = flags.length; i < len; i += 1) {
-        if (!Number.isFinite(colorByData[i])) {
-          flags[i] = flagNaN;
+        const val = colorByData[i];
+        if (typeof val === "number" && !Number.isFinite(val)) {
+          flags[i] = flagBackground;
         }
       }
     }

client/src/components/scatterplot/drawPointsRegl.js

@@ -13,7 +13,7 @@ export default function drawPointsRegl(regl) {
     uniform float nPoints;
     uniform float minViewportDimension;
 
-    varying vec4 fragColor;
+    varying lowp vec4 fragColor;
 
     const float zBottom = 0.99;
     const float zMiddle = 0.;
@@ -26,22 +26,22 @@ export default function drawPointsRegl(regl) {
     ${glPointSize}
 
     void main() {
-      bool isNaN, isSelected, isHighlight;
-      getFlags(flag, isNaN, isSelected, isHighlight);
+      bool isBackground, isSelected, isHighlight;
+      getFlags(flag, isBackground, isSelected, isHighlight);
 
       gl_PointSize = pointSize(nPoints, minViewportDimension, isSelected, isHighlight);
 
-      float z = isNaN ? zBottom : (isHighlight ? zTop : zMiddle);
+      float z = isBackground ? zBottom : (isHighlight ? zTop : zMiddle);
       vec3 xy = projection * vec3(position, 1.);
       gl_Position = vec4(xy.xy, z, 1.);
 
-      float alpha = isNaN ? 0.9 : 1.0;
+      float alpha = isBackground ? 0.9 : 1.0;
       fragColor = vec4(color, alpha);
     }`,
 
     frag: `
     precision mediump float;
-    varying vec4 fragColor;
+    varying lowp vec4 fragColor;
     void main() {
       if (length(gl_PointCoord.xy - 0.5) > 0.5) {
         discard;
@@ -64,5 +64,15 @@ export default function drawPointsRegl(regl) {
     count: regl.prop("count"),
 
     primitive: "points",
+
+    blend: {
+      enable: true,
+      func: {
+        srcRGB: "src alpha",
+        srcAlpha: 1,
+        dstRGB: 0,
+        dstAlpha: "zero",
+      },
+    },
   });
 }

client/src/components/scatterplot/scatterplot.js

@@ -16,10 +16,11 @@ import {
   createColorQuery,
 } from "../../util/stateManager/colorHelpers";
 import renderThrottle from "../../util/renderThrottle";
-
-const flagSelected = 1;
-const flagNaN = 2;
-const flagHighlight = 4;
+import {
+  flagBackground,
+  flagSelected,
+  flagHighlight,
+} from "../../util/glHelpers";
 
 function createProjectionTF(viewportWidth, viewportHeight) {
   /*
@@ -135,8 +136,9 @@ class Scatterplot extends React.PureComponent {
     const flags = new Float32Array(nObs);
     if (colorByData) {
       for (let i = 0, len = flags.length; i < len; i += 1) {
-        if (!Number.isFinite(colorByData[i])) {
-          flags[i] = flagNaN;
+        const val = colorByData[i];
+        if (typeof val === "number" && !Number.isFinite(val)) {
+          flags[i] = flagBackground;
         }
       }
     }

client/src/reducers/genesets.js

@@ -2,14 +2,30 @@
  * Gene set state. Geneset UI state is in a different reducer.
  *
  * geneset reducer state is a Map object, where:
- *  key: the gene set name, a string.
- *  val: the genes in the gene set, which is a Set of string
+ *  key: the geneset name, a string.
+ *  val: the geneset defined as an object ("geneset object")
  *
- * Ie, Map<string, Set<string>>
+ * A geneset object is:
+ * {
+ *    genesetName: <string>  # same as the map key
+ *    genesetDescription: <string>
+ *    genes: Map<<string>, {
+ *      geneSymbol: <string>,  # same as the map key
+ *      geneDescription: <string>
+ *    }>
+ * }
+ *
+ * Geneset and genes Map order is significant, and will be preserved across
+ * CRUD operations on either.
+ *
+ * This reducer does light error checking, but not as much as the backend
+ * routes. Do not rely on it to enforce geneset integrity - eg, no duplicate
+ * genes in a geneset.
  */
 const GeneSets = (
   state = {
     initialized: false,
+    lastTid: undefined,
     genesets: new Map(),
   },
   action
@@ -19,39 +35,73 @@ const GeneSets = (
      * Initial, load-time bootstrap.
      * {
      *    type: "geneset: initial load"
-     *    init: Array<Tuple<string, Array<string>>>
+     *    data: JSON response
      * }
      */
     case "geneset: initial load": {
-      const { init } = action;
+      const { data } = action;
+
+      if (
+        !data ||
+        typeof data.tid !== "number" ||
+        !Array.isArray(data.genesets)
+      )
+        throw new Error("missing or malformed JSON response");
+
+      const lastTid = data.tid;
+      const genesetsData = data.genesets;
       const genesets = new Map();
-      for (const gs of init) {
-        const genes = new Set();
-        for (const gene of gs[1]) {
-          genes.add(gene);
+
+      for (const gsData of genesetsData) {
+        const genes = new Map();
+        for (const gene of gsData.genes) {
+          genes.set(gene.gene_symbol, {
+            geneSymbol: gene.gene_symbol,
+            geneDescription: gene?.["gene_description"] ?? "",
+          });
         }
-        genesets.set(gs[0], genes);
+        const gs = {
+          genesetName: gsData.geneset_name,
+          genesetDescription: gsData?.["geneset_description"] ?? "",
+          genes,
+        };
+        genesets.set(gsData.geneset_name, gs);
       }
+
       return {
-        genesets,
         initialized: true,
+        lastTid,
+        genesets,
       };
     }
 
     /**
+     * Creates a new & empty geneset with the given name and description.
      * {
      *    type: "geneset: create",
-     *    name: string, // gene set name
-     *    genes: Set<string> || Array<string> || undefined
+     *    genesetName: string, // gene set name
+     *    genesetDescription: string, // geneset description
      * }
+     *
      */
     case "geneset: create": {
-      const { name, genes } = action;
-      if (state.genesets.has(name))
+      const { genesetName, genesetDescription } = action;
+      if (
+        typeof genesetName !== "string" ||
+        !genesetName ||
+        genesetDescription === undefined
+      )
+        throw new Error("geneset: create -- name or description unspecified.");
+      if (state.genesets.has(genesetName))
         throw new Error("geneset: create -- name already defined.");
 
       const genesets = new Map(state.genesets); // clone
-      genesets.set(name, new Set(genes || []));
+      genesets.set(genesetName, {
+        genesetName,
+        genesetDescription,
+        genes: new Map(),
+      });
+
       return {
         ...state,
         genesets,
@@ -59,18 +109,19 @@ const GeneSets = (
     }
 
     /**
+     * Deletes the named geneset, if it exists. Throws if it does not.
      * {
      *    type: "geneset: delete",
-     *    name: string
+     *    genesetName: string
      * }
      */
     case "geneset: delete": {
-      const { name } = action;
-      if (!state.genesets.has(name))
-        throw new Error("geneset: delete -- name does not exist.");
+      const { genesetName } = action;
+      if (!state.genesets.has(genesetName))
+        throw new Error("geneset: delete -- geneset name does not exist.");
 
       const genesets = new Map(state.genesets); // clone
-      genesets.delete(name);
+      genesets.delete(genesetName);
       return {
         ...state,
         genesets,
@@ -78,81 +129,142 @@ const GeneSets = (
     }
 
     /**
+     * Update the named geneset with a new name and description.  Preserves the existing
+     * order of the geneset, even when the genesetName changes.
      * {
-     *    type: "geneset: add genes"
-     *    name: string, // gene set name
-     *    genes: Set<string> || Array<string>  // genes to add
+     *    type: "geneset: update",
+     *    genesetName: string, current name of geneset to be updated
+     *    update: {
+     *        genesetName: string, new name
+     *        genesetDescription: string, new description
+     *    }
      * }
-     */
-    case "geneset: add genes": {
-      const { name, genes } = action;
-      if (!state.genesets.has(name))
-        throw new Error("geneset: add genes -- name does not exist.");
-
-      const genesets = new Map(state.genesets); // clone
-      const newGenes = new Set(state.genesets.get(name)); // clone
-      for (const gene of genes) {
-        newGenes.add(gene);
-      }
-      genesets.set(name, newGenes);
-      return {
-        ...state,
-        genesets,
-      };
-    }
-
-    /**
-     * {
-     *    type: "geneset: del genes"
-     *    name: string, // gene set name
-     *    genes: Set<string> || Array<string>  // genes to delete
-     * }
-     */
-    case "geneset: del genes": {
-      const { name, genes } = action;
-
-      if (!state.genesets.has(name))
-        throw new Error("geneset: add genes -- name does not exist.");
-
-      const genesets = new Map(state.genesets); // clone
-      const newGenes = new Set(state.genesets.get(name)); // clone
-      for (const gene of genes) {
-        newGenes.delete(gene);
-      }
-      genesets.set(name, newGenes);
-      return {
-        ...state,
-        genesets,
-      };
-    }
-
-    /**
-     * Rename the gene set, preserving its order in the geneset collection.
      *
-     * {
-     *    type: "geneset: rename",
-     *    name: string, // gene set current (previous) name
-     *    newName: string, // the new gene set name
-     * }
+     * For example, if you want to update JUST the description:
+     *   dispatch({
+     *     action: "geneset: update",
+     *     genesetName: "foo",
+     *     update: { genesetName: "foo", genesetDescription: "a new description"}
+     *   })
      */
-    case "geneset: rename": {
-      const { name, newName } = action;
-      if (!state.genesets.has(name))
-        throw new Error("geneset: rename -- name does not exist.");
+    case "geneset: update": {
+      const { genesetName, update } = action;
       if (
-        typeof newName !== "string" ||
-        !newName.length ||
-        state.genesets.has(newName)
+        typeof genesetName !== "string" ||
+        !genesetName ||
+        !state.genesets.has(genesetName)
       )
         throw new Error(
-          "geneset: rename -- new name must be unique, non-null string."
+          "geneset: update -- geneset name unspecified or does not exist."
         );
+      if (state.genesets.has(update.genesetName))
+        throw new Error("geneset: update -- update specified existing name.");
+
+      const prevGs = state.genesets.get(genesetName);
+      const newGs = {
+        ...update,
+        genes: prevGs.genes,
+      }; // clone
 
       // clone the map, preserving current insert order, but mapping name->newName.
       const genesets = new Map();
-      for (const [gsName, gsGenes] of state.genesets) {
-        if (gsName === name) genesets.set(newName, gsGenes);
-        else genesets.set(gsName, gsGenes);
+      for (const [name, gs] of state.genesets) {
+        if (name === genesetName) genesets.set(newGs.genesetName, newGs);
+        else genesets.set(name, gs);
+      }
+
+      return {
+        ...state,
+        genesets,
+      };
+    }
+
+    /**
+     * Adds genes to the geneset.  They are appended to the END of the geneset, in the
+     * order provided. Duplicates or genes already in the geneset, will be ignored.
+     * {
+     *    type: "geneset: add genes"
+     *    genesetName: <string>, // gene set name
+     *    genes: Array<{
+     *      geneSymbol: <string>,
+     *      geneDescription: <string>
+     *    }>
+     * }
+     *
+     * Example:
+     *   dispatch({
+     *     type: "add genes",
+     *     genesetName: "foo",
+     *     genes: [ { geneSymbol: "FOXP", geneDescription: "test" }]
+     *   });
+     */
+    case "geneset: add genes": {
+      const { genesetName, genes } = action;
+      if (!state.genesets.has(genesetName))
+        throw new Error("geneset: add genes -- geneset name does not exist.");
+
+      // clone
+      const genesets = new Map(state.genesets);
+      const gs = {
+        ...genesets.get(genesetName),
+        genes: new Map(genesets.get(genesetName).genes),
+      };
+      genesets.set(genesetName, gs);
+
+      // add
+      const newGenes = gs.genes;
+      for (const gene of genes) {
+        const { geneSymbol } = gene;
+        const geneDescription = gene?.geneDescription ?? "";
+        // ignore genes already present
+        if (!newGenes.has(geneSymbol))
+          newGenes.set(geneSymbol, {
+            geneSymbol,
+            geneDescription,
+          });
+      }
+
+      return {
+        ...state,
+        genesets,
+      };
+    }
+
+    /**
+     * Delete genes from the named geneset. Will throw if the genesetName does
+     * not exist.  Will ignore geneSymbols that do not exist.
+     * {
+     *    type: "geneset: delete genes",
+     *    genesetName: <string>, // the geneset from which to delete genes
+     *    geneSymbols: [<string>, ...], // the gene symbols to delete.
+     * }
+     *
+     * Example:
+     *  dispatch({
+     *    type: "geneset: delete genes",
+     *    genesetName: "a geneset name",
+     *    geneSymbols: ["F5"]
+     *  })
+     */
+    case "geneset: delete genes": {
+      const { genesetName, geneSymbols } = action;
+      if (!state.genesets.has(genesetName))
+        throw new Error(
+          "geneset: delete genes -- geneset name does not exist."
+        );
+
+      // clone
+      const genesets = new Map(state.genesets);
+      const gs = {
+        ...genesets.get(genesetName),
+        genes: new Map(genesets.get(genesetName).genes),
+      };
+      genesets.set(genesetName, gs);
+
+      // delete
+      const { genes } = gs;
+      for (const geneSymbol of geneSymbols) {
+        genes.delete(geneSymbol);
       }
       return {
         ...state,
@@ -160,6 +272,75 @@ const GeneSets = (
       };
     }
 
+    /**
+     * Set/update the description of the gene.  NOTE that this does not allow the name
+     * of the gene to change - only "geneset: add" and "geneset: delete" can change
+     * the genes in a geneset.  Use this to update a gene description AFTER you add it
+     * to the geneset.
+     * {
+     *    type: "geneset: set gene description",
+     *    genesetName: <string>, // the geneset to update
+     *    update: {
+     *      geneSymbol: <string>, // the gene to update, MUST exist already in the geneset
+     *      geneDescription: <string>
+     *    }
+     * }
+     *
+     * Example:
+     *  dispatch({
+     *      type: "geneset: set gene description",
+     *      genesetName: "my fav geneset",
+     *      update: {
+     *        geneSymbol: "F5",
+     *        geneDescription: "tada, moar description"
+     *      }
+     *  })
+     */
+    case "geneset: set gene description": {
+      const { genesetName, update } = action;
+      if (!state.genesets.has(genesetName))
+        throw new Error(
+          "geneset: set gene description -- geneset name does not exist."
+        );
+
+      // clone
+      const genesets = new Map(state.genesets);
+      const gs = {
+        ...genesets.get(genesetName),
+        genes: new Map(genesets.get(genesetName).genes),
+      };
+      genesets.set(genesetName, gs);
+
+      const { geneSymbol, geneDescription } = update;
+      const gene = gs.genes.get(geneSymbol);
+      if (!gene)
+        throw new Error("geneset: set gene description -- no such gene");
+      gs.genes.set(geneSymbol, {
+        geneSymbol,
+        geneDescription,
+      });
+
+      return {
+        ...state,
+        genesets,
+      };
+    }
+
+    /**
+     * Used by autosave to update the server synchronization TID
+     */
+    case "geneset: set tid": {
+      const { tid } = action;
+      if (!Number.isInteger(tid) || tid < 0)
+        throw new Error("TID must be a positive integer number");
+      if (state.lastTid !== undefined && tid < state.lastTid)
+        throw new Error("TID may not be decremented.");
+      return {
+        ...state,
+        lastTid: tid,
+      };
+    }
+
     default:
       return state;
   }

client/src/reducers/index.js

@@ -4,7 +4,7 @@ import thunk from "redux-thunk";
 import cascadeReducers from "./cascade";
 import undoable from "./undoable";
 import config from "./config";
-import userInfo from "./userInfo";
+import userInfo from "./userinfo";
 import annoMatrix from "./annoMatrix";
 import obsCrossfilter from "./obsCrossfilter";
 import categoricalSelection from "./categoricalSelection";
@@ -59,6 +59,7 @@ const Reducer = undoable(
     "differential",
     "layoutChoice",
     "centroidLabels",
+    "genesets",
     "annotations",
   ],
   undoableConfig

client/src/util/glHelpers.js

@@ -8,18 +8,24 @@ PointFlags:
 
   We want a bitmask-like flag structure, but due to webgl limitations
   must emulate it with floats.
+
+  Supported flags are:
+
+    selected: the point is currently selected
+    highlight: the point is currently highlighted
+    background: the point is background information
 */
 
 // for JS
 export const flagSelected = 1;
-export const flagNaN = 2;
+export const flagBackground = 2;
 export const flagHighlight = 4;
 
 // for GLSL
 export const glPointFlags = `
 
   const float flagSelected = 1.;
-  const float flagNaN = 2.;
+  const float flagBackground = 2.;
   const float flagHighlight = 4.;
 
   bool isLowBitSet(float f) {
@@ -32,12 +38,12 @@ export const glPointFlags = `
   }
 
   void getFlags(in float flag,
-                out bool isNaN,
+                out bool isBackground,
                 out bool isSelected,
                 out bool isHighlight) {
     isSelected = isLowBitSet(flag);
     flag = shiftRightOne(flag);
-    isNaN = isLowBitSet(flag);
+    isBackground = isLowBitSet(flag);
     flag = shiftRightOne(flag);
     isHighlight = isLowBitSet(flag);
   }

client/src/util/stateManager/colorHelpers.js

@@ -55,8 +55,8 @@ create colors scale and RGB array and return as object. Parameters:
   * userColors - optional user color table
 Returns:
   {
-    scale: color scale
-    rgb: cell to color mapping
+    scale: function, mapping label index to color scale
+    rgb: cell label to color mapping
   }
 */
 function _createColorTable(
@@ -70,7 +70,7 @@ function _createColorTable(
     case "color by categorical metadata": {
       const data = colorByData.col(colorByAccessor).asArray();
       if (userColors && colorByAccessor in userColors) {
-        return createUserColors(data, colorByAccessor, userColors);
+        return createUserColors(data, colorByAccessor, schema, userColors);
       }
       return createColorsByCategoricalMetadata(data, colorByAccessor, schema);
     }
@@ -91,42 +91,41 @@ function _createColorTable(
 }
 export const createColorTable = memoize(_createColorTable);
 
+/**
+ * Create two category label-indexed objects:
+ *    - colors: maps label to RGB triplet for that label (used by graph, etc)
+ *    - scale: function which given label returns d3 color scale for label
+ * Order doesn't matter - everything is keyed by label value.
+ */
 export function loadUserColorConfig(userColors) {
   const convertedUserColors = {};
   Object.keys(userColors).forEach((category) => {
-    // We cannot iterate over keys without sorting
-    // because we handle categorical values in alphabetical order __ignoring case__
-    //  while Object.keys() _usually_ is ordered alphabetically where all upper characters are less than lowercase (A, B, C, a, b, c)
-    const [colors, scaleMap] = Object.keys(userColors[category])
-      .sort((a, b) => {
-        a = a.toLowerCase();
-        b = b.toLowerCase();
-        if (a === b) return 0;
-        if (a > b) return 1;
-        return -1;
-      })
-      .reduce(
-        (acc, label) => {
-          const color = parseRGB(userColors[category][label]);
-          acc[0][label] = color;
-          acc[1][label] = d3.rgb(
-            255 * color[0],
-            255 * color[1],
-            255 * color[2]
-          );
-          return acc;
-        },
-        [{}, {}]
-      );
+    const [colors, scaleMap] = Object.keys(userColors[category]).reduce(
+      (acc, label) => {
+        const color = parseRGB(userColors[category][label]);
+        acc[0][label] = color;
+        acc[1][label] = d3.rgb(255 * color[0], 255 * color[1], 255 * color[2]);
+        return acc;
+      },
+      [{}, {}]
+    );
     const scale = (label) => scaleMap[label];
     convertedUserColors[category] = { colors, scale };
   });
   return convertedUserColors;
 }
 
-function _createUserColors(data, colorAccessor, userColors) {
-  const { colors, scale } = userColors[colorAccessor];
+function _createUserColors(data, colorAccessor, schema, userColors) {
+  const { colors, scale: scaleByLabel } = userColors[colorAccessor];
   const rgb = createRgbArray(data, colors);
+
+  // color scale function param is INDEX (offset) into schema categories. It is NOT label value.
+  // See createColorsByCategoricalMetadata() for another example.
+  const { categories } = schema.annotations.obsByName[colorAccessor];
+  const categoryMap = new Map();
+  categories.forEach((label, idx) => categoryMap.set(idx, label));
+  const scale = (idx) => scaleByLabel(categoryMap.get(idx));
+
   return { rgb, scale };
 }
 const createUserColors = memoize(_createUserColors);

docs/_config.yml

@@ -33,5 +33,7 @@ nav:
     url: posts/roadmap
   - title: Contributing (ideas or code)
     url: posts/contribute
+  - title: Extensions
+    url: posts/extensions
   - title: Contact & finding help
     url: posts/contact

docs/_site/deprecated/cellxgene_cziscience_com.html

@@ -0,0 +1,446 @@
+<!DOCTYPE html>
+<html lang="en-US">
+  <head>
+    <meta charset="UTF-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+
+<!-- Begin Jekyll SEO tag v2.6.1 -->
+<title>cellxgene.cziscience.com | cellxgene</title>
+<meta name="generator" content="Jekyll v3.8.7" />
+<meta property="og:title" content="cellxgene.cziscience.com" />
+<meta property="og:locale" content="en_US" />
+<meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
+<meta property="og:description" content="An interactive explorer for single-cell transcriptomics data" />
+<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/deprecated/cellxgene_cziscience_com.html" />
+<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/deprecated/cellxgene_cziscience_com.html" />
+<meta property="og:site_name" content="cellxgene" />
+<script type="application/ld+json">
+{"@type":"WebPage","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"cellxgene.cziscience.com","description":"An interactive explorer for single-cell transcriptomics data","url":"https://chanzuckerberg.github.io/cellxgene/deprecated/cellxgene_cziscience_com.html","@context":"https://schema.org"}</script>
+<!-- End Jekyll SEO tag -->
+
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=f70dffced52a32aada1a22504c841e97e941401a">
+    <!--[if lt IE 9]>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
+    <![endif]-->
+  </head>
+  <body>
+    <div class="wrapper">
+      <header>
+        <img src="/cellxgene/cellxgene-logo.png" alt="cellxgene" />
+
+        <p>An interactive explorer for single-cell transcriptomics data</p>
+        <p>
+
+        
+        <a href="/cellxgene/" class="btn">Quick start</a><br>
+        
+
+        
+        
+        <a href="/cellxgene/posts/install" class="btn">Installation</a><br>
+        
+        
+        
+        <a href="/cellxgene/posts/gallery" class="btn">Gallery</a><br>
+        
+        
+        
+        <a href="/cellxgene/posts/demo-data" class="btn">Demo datasets</a><br>
+        
+        
+        
+        <a href="https://cellxgene.cziscience.com/" class="btn">All other datasets</a><br>
+        
+        
+        
+        <a href="/cellxgene/posts/prepare" class="btn">Preparing your data</a><br>
+        
+        
+        
+        <a href="/cellxgene/posts/launch" class="btn">Launching cellxgene</a><br>
+        
+        
+        
+        <a href="/cellxgene/posts/hosted" class="btn">Hosting cellxgene</a><br>
+        
+        
+        
+        <a href="/cellxgene/posts/annotations" class="btn">Annotating data</a><br>
+        
+        
+        
+        <a href="/cellxgene/posts/methods" class="btn">Methods</a><br>
+        
+        
+        
+        <a href="/cellxgene/posts/troubleshooting" class="btn">Troubleshooting</a><br>
+        
+        
+        
+        <a href="/cellxgene/posts/roadmap" class="btn">Roadmap</a><br>
+        
+        
+        
+        <a href="/cellxgene/posts/contribute" class="btn">Contributing (ideas or code)</a><br>
+        
+        
+        
+        <a href="/cellxgene/posts/extensions" class="btn">Extensions</a><br>
+        
+        
+        
+        <a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
+        
+        
+
+        <a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
+    </p>
+    </header>
+
+      <section>
+      <h1 id="cellxgeneczisciencecom">cellxgene.cziscience.com</h1>
+
+<p>Chan Zuckerberg has an online repository of public single-cell datasets for exploration with cellxgene.</p>
+
+<p>If you have a public dataset which you would like hosted for visualization on this site,
+with a link to embed on your own site, please drop us a note at <a href="mailto:cellxgene@chanzuckerberg.com">cellxgene@chanzuckerberg.com</a>.</p>
+
+<table class="fixed-layout">
+  <thead style="width: 100%">
+    <tr>
+      <th>cellxgene link</th>
+      <th>More Information</th>
+    </tr>
+  </thead>
+  <tbody style="width: 100%">
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/krasnow_lab_human_lung_cell_atlas_10x-1.cxg/" target="_blank">Krasnow Lab Human Lung Cell Atlas, 10X</a></td>
+      <td>
+        <a href="http://cmgm-new.stanford.edu/krasnow/">Krasnow Lab</a>,
+        <a href="https://github.com/krasnowlab/hlca">HLCA website</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/krasnow_lab_human_lung_cell_atlas_smartseq2-2.cxg/" target="_blank">Krasnow Lab Human Lung Cell Atlas, Smart-seq2</a></td>
+      <td>
+        <a href="http://cmgm-new.stanford.edu/krasnow/">Krasnow Lab</a>, 
+        <a href="https://github.com/krasnowlab/hlca">HLCA website</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/human_cell_landscape-3.cxg/" target="_blank">Human Cell Landscape</a></td>
+      <td>
+        <a href="https://person.zju.edu.cn/en/ggj">Guo Lab</a>, 
+        <a href="http://bis.zju.edu.cn/HCL/">HCL website</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/human_fetal_liver_single_cell_transcriptome-13.cxg/" target="_blank">Human fetal liver single cell transcriptome data</a></td>
+      <td>
+        <a href="https://www.ebi.ac.uk/arrayexpress/experiments/E-MTAB-7407/">E-MTAB-7407</a>,
+        <a href="https://www.covid19cellatlas.org/">covid19cellatlas.org</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/cell_atlas_of_thymic_development-14.cxg/" target="_blank">A cell atlas of human thymic development defines T cell repertoire formation</a></td>
+      <td>
+        <a href="https://www.ebi.ac.uk/arrayexpress/experiments/E-MTAB-8581/">E-MTAB-8581</a>,
+        <a href="https://www.covid19cellatlas.org/">covid19cellatlas.org</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/cellular_census_of_human_lungs_alveoli_and_parenchyma-15.cxg/" target="_blank">A cellular census of human lungs identifies novel cell states in health and in asthma - parenchyma</a></td>
+      <td>
+        <a href="https://asthma.cellgeni.sanger.ac.uk/">asthma.cellgeni.sanger.ac.uk</a>,
+        <a href="https://www.covid19cellatlas.org/">covid19cellatlas.org</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/cellular_census_of_human_lungs_nasal-16.cxg/" target="_blank">A cellular census of human lungs identifies novel cell states in health and in asthma - nasal</a></td>
+      <td>
+        <a href="https://asthma.cellgeni.sanger.ac.uk/">asthma.cellgeni.sanger.ac.uk</a>,
+        <a href="https://www.covid19cellatlas.org/">covid19cellatlas.org</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/cellular_census_of_human_lungs_bronchi-17.cxg/" target="_blank">A cellular census of human lungs identifies novel cell states in health and in asthma - bronchi</a></td>
+      <td>
+        <a href="https://asthma.cellgeni.sanger.ac.uk/">asthma.cellgeni.sanger.ac.uk</a>,
+        <a href="https://www.covid19cellatlas.org/">covid19cellatlas.org</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/ischaemic_sensitivity_of_human_tissue_by_single_cell_RNA_seq_lung-18.cxg/" target="_blank">Ischaemic sensitivity of human tissue by single cell RNA seq - lung</a></td>
+      <td>
+        <a href="https://data.humancellatlas.org/explore/projects/c4077b3c-5c98-4d26-a614-246d12c2e5d7">HCA</a>,
+        <a href="https://www.covid19cellatlas.org/">covid19cellatlas.org</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/ischaemic_sensitivity_of_human_tissue_by_single_cell_RNA_seq_spleen-19.cxg/" target="_blank">Ischaemic sensitivity of human tissue by single cell RNA seq - spleen</a></td>
+      <td>
+        <a href="https://data.humancellatlas.org/explore/projects/c4077b3c-5c98-4d26-a614-246d12c2e5d7">HCA</a>,
+        <a href="https://www.covid19cellatlas.org/">covid19cellatlas.org</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/ischaemic_sensitivity_of_human_tissue_by_single_cell_RNA_seq_oesophagus-20.cxg/" target="_blank">Ischaemic sensitivity of human tissue by single cell RNA seq - oesophagus</a></td>
+      <td>
+        <a href="https://data.humancellatlas.org/explore/projects/c4077b3c-5c98-4d26-a614-246d12c2e5d7">HCA</a>,
+        <a href="https://www.covid19cellatlas.org/">covid19cellatlas.org</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/spatio_temporal_immune_zonation_of_the_human_kidney-21.cxg/" target="_blank">Spatio-temporal immune zonation of the human kidney</a></td>
+      <td>
+        <a href="https://www.kidneycellatlas.org/">www.kidneycellatlas.org</a>,
+        <a href="https://www.covid19cellatlas.org/">covid19cellatlas.org</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/fetal_maternal_interface_10x-22.cxg/" target="_blank">Reconstructing the human first trimester fetal-maternal interface using single cell transcriptomics - 10x</a></td>
+      <td>
+        <a href="https://www.ebi.ac.uk/arrayexpress/experiments/E-MTAB-6701/">E-MTAB-6701</a>,
+        <a href="https://www.covid19cellatlas.org/">covid19cellatlas.org</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/fetal_maternal_interface_smartseq2-23.cxg/" target="_blank">Reconstructing the human first trimester fetal-maternal interface using single cell transcriptomics - SmartSeq2</a></td>
+      <td>
+        <a href="https://www.ebi.ac.uk/arrayexpress/experiments/E-MTAB-6701/">E-MTAB-6701</a>,
+        <a href="https://www.covid19cellatlas.org/">covid19cellatlas.org</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/gut_cell_atlas-24.cxg/" target="_blank">Gut Cell Atlas</a></td>
+      <td>
+        <a href="https://www.gutcellatlas.org/">www.gutcellatlas.org</a>,
+        <a href="https://www.covid19cellatlas.org/">covid19cellatlas.org</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/Single_cell_atlas_of_peripheral_immune_response_to_SARS_CoV_2_infection-25.cxg/" target="_blank">A single-cell atlas of the peripheral immune response to severe COVID-19</a></td>
+      <td>
+        <a href="https://blishlab.sites.stanford.edu/">Blish Lab</a>,
+        <a href="https://www.medrxiv.org/content/10.1101/2020.04.17.20069930v1">medRxiv preprint</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/Atlas_of_Healthy_and_SHIV_Infected_Non_Human_Primate_Lung_and_Ileum_ACE2+_Cells_ileum-12.cxg/" target="_blank">Atlas of Healthy and SHIV-Infected Non-Human Primate Lung and Ileum ACE2+ Cells - Ileum</a></td>
+      <td>
+        <a href="https://singlecell.broadinstitute.org/single_cell/study/SCP807/atlas-of-healthy-and-shiv-infected-non-human-primate-lung-and-ileum-ace2-cells?scpbr=the-alexandria-project">Single Cell Portal</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/Atlas_of_Healthy_and_SHIV_Infected_Non_Human_Primate_Lung_and_Ileum_ACE2+_Cells_lung-11.cxg/" target="_blank">Atlas of Healthy and SHIV-Infected Non-Human Primate Lung and Ileum ACE2+ Cells - Lung</a></td>
+      <td>
+        <a href="https://singlecell.broadinstitute.org/single_cell/study/SCP807/atlas-of-healthy-and-shiv-infected-non-human-primate-lung-and-ileum-ace2-cells?scpbr=the-alexandria-project">Single Cell Portal</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/Allergic_inflammatory_memory_in_human_respiratory_epithelial_progenitor_cells_epithelial-10.cxg/" target="_blank">Allergic inflammatory memory in human respiratory epithelial progenitor cells - epithelial cells</a></td>
+      <td>
+        <a href="https://singlecell.broadinstitute.org/single_cell/study/SCP253/allergic-inflammatory-memory-in-human-respiratory-epithelial-progenitor-cells?scpbr=the-alexandria-project">Single Cell Portal</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/Allergic_inflammatory_memory_in_human_respiratory_epithelial_progenitor_cells_scraping-9.cxg/" target="_blank">Allergic inflammatory memory in human respiratory epithelial progenitor cells - nasal scrapings</a></td>
+      <td>
+        <a href="https://singlecell.broadinstitute.org/single_cell/study/SCP253/allergic-inflammatory-memory-in-human-respiratory-epithelial-progenitor-cells?scpbr=the-alexandria-project">Single Cell Portal</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/Allergic_inflammatory_memory_in_human_respiratory_epithelial_progenitor_cells_surgical-8.cxg/" target="_blank">Allergic inflammatory memory in human respiratory epithelial progenitor cells - surgical</a></td>
+      <td>
+        <a href="https://singlecell.broadinstitute.org/single_cell/study/SCP253/allergic-inflammatory-memory-in-human-respiratory-epithelial-progenitor-cells?scpbr=the-alexandria-project">Single Cell Portal</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/Allergic_inflammatory_memory_in_human_respiratory_epithelial_progenitor_cells_nasalsss-26.cxg/" target="_blank">Allergic inflammatory memory in human respiratory epithelial progenitor cells - nasal SSS</a></td>
+      <td>
+        <a href="https://singlecell.broadinstitute.org/single_cell/study/SCP253/allergic-inflammatory-memory-in-human-respiratory-epithelial-progenitor-cells?scpbr=the-alexandria-project">Single Cell Portal</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/ACE2_and_TMPRSS2_expression_in_human_non_inflamed_terminal_ileum_epithelial-7.cxg/" target="_blank">ACE2 and TMPRSS2 expression in human non-inflamed terminal ileum - epithelial cells</a></td>
+      <td>
+        <a href="https://singlecell.broadinstitute.org/single_cell/study/SCP812/ace2-and-tmprss2-expression-in-human-non-inflamed-terminal-ileum?scpbr=the-alexandria-project">Single Cell Portal</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/ACE2_and_TMPRSS2_expression_in_human_non_inflamed_terminal_ileum-6.cxg/" target="_blank">ACE2 and TMPRSS2 expression in human non-inflamed terminal ileum</a></td>
+      <td>
+        <a href="https://singlecell.broadinstitute.org/single_cell/study/SCP812/ace2-and-tmprss2-expression-in-human-non-inflamed-terminal-ileum?scpbr=the-alexandria-project">Single Cell Portal</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/Human_Lung_HIV_TB_Co_infection_ACE2+_Cells-5.cxg/" target="_blank">Human Lung HIV-TB Co-infection ACE2+ Cells</a></td>
+      <td>
+        <a href="https://singlecell.broadinstitute.org/single_cell/study/SCP814/human-lung-hiv-tb-co-infection-ace2-cells?scpbr=the-alexandria-project">Single Cell Portal</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/Epithelial_Cells_in_NHP_mTB_Granuloma_and_Uninvolved_Lung-4.cxg/" target="_blank">Epithelial Cells in NHP mTB Granuloma and Uninvolved Lung</a></td>
+      <td>
+        <a href="https://singlecell.broadinstitute.org/single_cell/study/SCP806/epithelial-cells-in-nhp-mtb-granuloma-and-uninvolved-lung?scpbr=the-alexandria-project">Single Cell Portal</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/kampmann_lab_human_AD_snRNAseq_EC-49.cxg/
+" target="_blank">Selective Neuronal Vulnerability in Alzheimer's Disease</a></td>
+      <td>
+        <a href="https://kampmannlab.ucsf.edu/">Kampmann Lab</a>, 
+        <a href="https://www.biorxiv.org/content/10.1101/2020.04.04.025825v2">BioRxiv preprint</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/kampmann_lab_human_AD_snRNAseq_SFG-50.cxg/
+" target="_blank">Selective Neuronal Vulnerability in Alzheimer's Disease: Superior Frontal Gyrus</a></td>
+      <td>
+        <a href="https://kampmannlab.ucsf.edu/">Kampmann Lab</a>, 
+        <a href="https://www.biorxiv.org/content/10.1101/2020.04.04.025825v2">BioRxiv preprint</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/kampmann_lab_human_AD_snRNAseq_EC_astrocytes-51.cxg/
+" target="_blank">Selective Neuronal Vulnerability in Alzheimer's Disease: Astrocytes in EC</a></td>
+      <td>
+        <a href="https://kampmannlab.ucsf.edu/">Kampmann Lab</a>, 
+        <a href="https://www.biorxiv.org/content/10.1101/2020.04.04.025825v2">BioRxiv preprint</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/kampmann_lab_human_AD_snRNAseq_EC_excitatoryNeurons-52.cxg/
+" target="_blank">Selective Neuronal Vulnerability in Alzheimer's Disease: Excitatory Neurons in EC</a></td>
+      <td>
+        <a href="https://kampmannlab.ucsf.edu/">Kampmann Lab</a>, 
+        <a href="https://www.biorxiv.org/content/10.1101/2020.04.04.025825v2">BioRxiv preprint</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/kampmann_lab_human_AD_snRNAseq_EC_inhibitoryNeurons-53.cxg/
+" target="_blank">Selective Neuronal Vulnerability in Alzheimer's Disease: Inhibitory Neurons in EC</a></td>
+      <td>
+        <a href="https://kampmannlab.ucsf.edu/">Kampmann Lab</a>, 
+        <a href="https://www.biorxiv.org/content/10.1101/2020.04.04.025825v2">BioRxiv preprint</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/kampmann_lab_human_AD_snRNAseq_EC_microglia-54.cxg/
+" target="_blank">Selective Neuronal Vulnerability in Alzheimer's Disease: Microglia in EC</a></td>
+      <td>
+        <a href="https://kampmannlab.ucsf.edu/">Kampmann Lab</a>, 
+        <a href="https://www.biorxiv.org/content/10.1101/2020.04.04.025825v2">BioRxiv preprint</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/kampmann_lab_human_AD_snRNAseq_SFG_astrocytes-55.cxg/
+" target="_blank">Selective Neuronal Vulnerability in Alzheimer's Disease: Astrocytes in SFG</a></td>
+      <td>
+        <a href="https://kampmannlab.ucsf.edu/">Kampmann Lab</a>, 
+        <a href="https://www.biorxiv.org/content/10.1101/2020.04.04.025825v2">BioRxiv preprint</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/kampmann_lab_human_AD_snRNAseq_SFG_excitatoryNeurons-56.cxg/
+" target="_blank">Selective Neuronal Vulnerability in Alzheimer's Disease: Excitatory Neurons in SFG</a></td>
+      <td>
+        <a href="https://kampmannlab.ucsf.edu/">Kampmann Lab</a>, 
+        <a href="https://www.biorxiv.org/content/10.1101/2020.04.04.025825v2">BioRxiv preprint</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/kampmann_lab_human_AD_snRNAseq_SFG_inhibitoryNeurons-57.cxg/" target="_blank">Selective Neuronal Vulnerability in Alzheimer's Disease: Inhibitory Neurons in SFG</a></td>
+      <td>
+        <a href="https://kampmannlab.ucsf.edu/">Kampmann Lab</a>, 
+        <a href="https://www.biorxiv.org/content/10.1101/2020.04.04.025825v2">BioRxiv preprint</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/kampmann_lab_human_AD_snRNAseq_SFG_microglia-58.cxg/" target="_blank">Selective Neuronal Vulnerability in Alzheimer's Disease: Microglia in SFG</a></td>
+      <td>
+        <a href="https://kampmannlab.ucsf.edu/">Kampmann Lab</a>, 
+        <a href="https://www.biorxiv.org/content/10.1101/2020.04.04.025825v2">BioRxiv preprint</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/Single_cell_gene_expression_profiling_of_SARS_CoV_2_infected_human_cell_lines_H1299-27.cxg/" target="_blank">Single-cell gene expression profiling of SARS-CoV-2 infected human cell lines - H1299</a></td>
+      <td>
+        <a href="https://www.mdc-berlin.de/landthaler#t-single-cellsars-cov-2">Landthaler Lab</a>,
+        <a href="https://www.biorxiv.org/content/10.1101/2020.05.05.079194v1">BioRxiv preprint</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/Single_cell_gene_expression_profiling_of_SARS_CoV_2_infected_human_cell_lines_Calu_3-28.cxg/" target="_blank">Single-cell gene expression profiling of SARS-CoV-2 infected human cell lines - Calu-3</a></td>
+      <td>
+        <a href="https://www.mdc-berlin.de/landthaler#t-single-cellsars-cov-2">Landthaler Lab</a>,
+        <a href="https://www.biorxiv.org/content/10.1101/2020.05.05.079194v1">BioRxiv preprint</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/Single_cell_drug_screening_a549-42.cxg/" target="_blank">Single-cell drug screening - A549</a></td>
+      <td>
+        <a href="https://github.com/cole-trapnell-lab/sci-plex">Trapnell Lab Github</a>,
+        <a href="https://science.sciencemag.org/content/367/6473/45">Science</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/Single_cell_drug_screening_k562-43.cxg/" target="_blank">Single-cell drug screening - K562</a></td>
+      <td>
+        <a href="https://github.com/cole-trapnell-lab/sci-plex">Trapnell Lab Github</a>,
+        <a href="https://science.sciencemag.org/content/367/6473/45">Science</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/Single_cell_drug_screening_mcf7-44.cxg/" target="_blank">Single-cell drug screening - MCF7</a></td>
+      <td>
+        <a href="https://github.com/cole-trapnell-lab/sci-plex">Trapnell Lab Github</a>,
+        <a href="https://science.sciencemag.org/content/367/6473/45">Science</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.prod.single-cell.czi.technology/d/Molecular_atlas_of_cell_types_and_zonation_in_the_brain_vasculature-48.cxg/" target="_blank">A molecular atlas of cell types and zonation in the brain vasculature</a></td>
+      <td>
+        <a href="http://betsholtzlab.org/VascularSingleCells/database.html">Betsholtz Lab</a>,
+        <a href="https://www.nature.com/articles/nature25739">Nature</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/Single_soma_transcriptomics_AT8-45.cxg/" target="_blank">Single Soma Transcriptomics - AT8</a></td>
+      <td>
+        <a href="https://www.biorxiv.org/content/10.1101/2020.05.11.088591v1">bioRxiv preprint</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/Single_soma_transcriptomics_MAP2-46.cxg/" target="_blank">Single Soma Transcriptomics - MAP2</a></td>
+      <td>
+        <a href="https://www.biorxiv.org/content/10.1101/2020.05.11.088591v1">bioRxiv preprint</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/Single_soma_transcriptomics_MAP2AT8-47.cxg/" target="_blank">Single Soma Transcriptomics - MAP2AT8</a></td>
+      <td>
+        <a href="https://www.biorxiv.org/content/10.1101/2020.05.11.088591v1">bioRxiv preprint</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/Single_cell_longitudinal_analysis_of_SARS_CoV_2_infection_in_human_bronchial_epithelial_cells-29.cxg/" target="_blank">Single-cell longitudinal analysis of SARS-CoV-2 infection in human bronchial epithelial cells</a></td>
+      <td>
+        <a href="https://www.biorxiv.org/content/10.1101/2020.05.06.081695v2">bioRxiv preprint</a>
+      </td>
+    </tr>
+  </tbody>
+</table>
+
+      </section>
+      <footer>
+        
+        <p>This project is maintained by <a href="https://github.com/chanzuckerberg">chanzuckerberg</a></p>
+        
+      </footer>
+    </div>
+    <script src="/cellxgene/assets/js/scale.fix.js"></script>
+    
+  </body>
+</html>

docs/_site/index.html

@@ -7,7 +7,7 @@
 
 <!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>Index | cellxgene</title>
-<meta name="generator" content="Jekyll v3.9.0" />
+<meta name="generator" content="Jekyll v3.8.7" />
 <meta property="og:title" content="Index" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
@@ -16,10 +16,10 @@
 <meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"url":"https://chanzuckerberg.github.io/cellxgene/","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Index","name":"cellxgene","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebSite","@context":"https://schema.org"}</script>
+{"@type":"WebSite","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Index","description":"An interactive explorer for single-cell transcriptomics data","url":"https://chanzuckerberg.github.io/cellxgene/","name":"cellxgene","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=3718e894edc8a8f6e7776946695ab37c5c96ec9f">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=f70dffced52a32aada1a22504c841e97e941401a">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->
@@ -86,6 +86,10 @@
         
         
         
+        <a href="/cellxgene/posts/extensions" class="btn">Extensions</a><br>
+        
+        
+        
         <a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
         
         

docs/_site/posts/extensions.html

@@ -0,0 +1,156 @@
+<!DOCTYPE html>
+<html lang="en-US">
+  <head>
+    <meta charset="UTF-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+
+<!-- Begin Jekyll SEO tag v2.6.1 -->
+<title>Extensions | cellxgene</title>
+<meta name="generator" content="Jekyll v3.8.7" />
+<meta property="og:title" content="Extensions" />
+<meta property="og:locale" content="en_US" />
+<meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
+<meta property="og:description" content="An interactive explorer for single-cell transcriptomics data" />
+<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/extensions.html" />
+<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/extensions.html" />
+<meta property="og:site_name" content="cellxgene" />
+<script type="application/ld+json">
+{"@type":"WebPage","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Extensions","description":"An interactive explorer for single-cell transcriptomics data","url":"https://chanzuckerberg.github.io/cellxgene/posts/extensions.html","@context":"https://schema.org"}</script>
+<!-- End Jekyll SEO tag -->
+
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=f70dffced52a32aada1a22504c841e97e941401a">
+    <!--[if lt IE 9]>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
+    <![endif]-->
+  </head>
+  <body>
+    <div class="wrapper">
+      <header>
+        <img src="/cellxgene/cellxgene-logo.png" alt="cellxgene" />
+
+        <p>An interactive explorer for single-cell transcriptomics data</p>
+        <p>
+
+        
+        <a href="/cellxgene/" class="btn">Quick start</a><br>
+        
+
+        
+        
+        <a href="/cellxgene/posts/install" class="btn">Installation</a><br>
+        
+        
+        
+        <a href="/cellxgene/posts/gallery" class="btn">Gallery</a><br>
+        
+        
+        
+        <a href="/cellxgene/posts/demo-data" class="btn">Demo datasets</a><br>
+        
+        
+        
+        <a href="https://cellxgene.cziscience.com/" class="btn">All other datasets</a><br>
+        
+        
+        
+        <a href="/cellxgene/posts/prepare" class="btn">Preparing your data</a><br>
+        
+        
+        
+        <a href="/cellxgene/posts/launch" class="btn">Launching cellxgene</a><br>
+        
+        
+        
+        <a href="/cellxgene/posts/hosted" class="btn">Hosting cellxgene</a><br>
+        
+        
+        
+        <a href="/cellxgene/posts/annotations" class="btn">Annotating data</a><br>
+        
+        
+        
+        <a href="/cellxgene/posts/methods" class="btn">Methods</a><br>
+        
+        
+        
+        <a href="/cellxgene/posts/troubleshooting" class="btn">Troubleshooting</a><br>
+        
+        
+        
+        <a href="/cellxgene/posts/roadmap" class="btn">Roadmap</a><br>
+        
+        
+        
+        <a href="/cellxgene/posts/contribute" class="btn">Contributing (ideas or code)</a><br>
+        
+        
+        
+        <a href="/cellxgene/posts/extensions" class="btn"><b>Extensions</b></a><br>
+        
+        
+        
+        <a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
+        
+        
+
+        <a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
+    </p>
+    </header>
+
+      <section>
+      <h1 id="extensions">Extensions</h1>
+
+<p>This project was started with the sole goal of empowering the scientific community to explore and understand their data. 
+As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from 
+this project. All code is freely available for reuse under the <a href="https://opensource.org/licenses/MIT">MIT license</a>.</p>
+
+<p>Before extending cellxgene, we encourage you to reach out to us with ideas or questions. It might be possible that an 
+extension could be directly contributed, which would make it available for a wider audience, or that it’s on our 
+<a href="/cellxgene/posts/roadmap.html">roadmap</a> and under active development.</p>
+
+<p>Please note that cellxgene does not have public APIs. Our development may break extensions. We will document changes to the code base but it is advised that extensions pin the version of cellxgene they develop against.</p>
+
+<h2 id="example-reuse--extensions">Example Reuse &amp; extensions</h2>
+
+<h4 id="cellxgene-gateway">cellxgene Gateway</h4>
+
+<p><a href="https://github.com/Novartis/cellxgene-gateway">cellxgene Gateway</a> allows you to use with multiple datasets. It 
+displays an index of available h5ad (anndata) files. When a user clicks on a file name, it launches a Cellxgene Server 
+instance that loads that particular data file and once it is available proxies requests to that server.</p>
+
+<h4 id="cellxgene-vip-visualization-in-plugin">cellxgene-VIP (Visualization in Plugin)</h4>
+
+<p><a href="https://github.com/interactivereport/cellxgene_VIP">cellxgene-VIP</a> enables cellxgene to generate violin, stacked violin, stacked bar, heatmap, volcano, embedding, dot, track, density, 2D density, sankey and dual-gene plot in high-resolution SVG/PNG format. It also performs differential gene expression analysis and provides a Command Line Interface (CLI) for advanced users to perform analysis using python and R.</p>
+
+<h4 id="galaxy">Galaxy</h4>
+
+<p><a href="https://galaxyproject.org">Galaxy</a> is an open source, collaborative, web-based platform for data intensive biomedical research.
+Galaxy provides various tools for <a href="https://singlecell.usegalaxy.eu/">single-cell data analysis</a> and also infrastructure to the
+<a href="https://humancellatlas.usegalaxy.eu">Galaxy Human Cell Atlas project</a>. cellxgene can be <a href="https://usegalaxy.eu/root?tool_id=interactive_tool_cellxgene">
+  accessed within Galaxy</a> to view analyzed datasets. See also the relevant <a href="https://doi.org/10.1093/gigascience/giaa102">publication</a>
+
+</p>
+
+<h4 id="single-cell-portal">Single Cell Portal</h4>
+
+<p>The <a href="https://singlecell.broadinstitute.org/single_cell">Single Cell Portal</a> is a data hosting and visualization service. cellxgene can be embedded as an additional view to complement the visualizations provided by the. 
+<a href="https://singlecell.broadinstitute.org/single_cell/study/SCP807/atlas-of-healthy-and-shiv-infected-non-human-primate-lung-and-ileum-ace2-cells">Example</a>.</p>
+
+<h4 id="fastgenomics">FASTGenomics</h4>
+
+<p><a href="https://beta.fastgenomics.org/">FASTGenomics</a> is a collaborative research platform that offers easy-to-use data management and reproducible analytics to drive single-cell research forward. Many of the publicly available datasets in FASTGenomics - as well as your private datasets - can be interactively explored with cellxgene. 
+See also this <a href="https://beta.fastgenomics.org/datasets/detail-dataset-952687f71ef34322a850553c4a24e82e#Cellxgene">example</a> for data from <a href="https://beta.fastgenomics.org/p/schulte-schrepping_covid19">Schulte-Schrepping et al. (Cell, 2020)</a>. 
+Note that it is not necessary to create an account, anonymous login is permitted.</p>
+
+      </section>
+      <footer>
+        
+        <p>This project is maintained by <a href="https://github.com/chanzuckerberg">chanzuckerberg</a></p>
+        
+      </footer>
+    </div>
+    <script src="/cellxgene/assets/js/scale.fix.js"></script>
+    
+  </body>
+</html>

docs/_site/posts/extensions.md

@@ -0,0 +1,39 @@
+# Extensions
+
+This project was started with the sole goal of empowering the scientific community to explore and understand their data. 
+As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from 
+this project. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT).
+
+Before extending cellxgene, we encourage you to reach out to us with ideas or questions. It might be possible that an 
+extension could be directly contributed, which would make it available for a wider audience, or that it's on our 
+[roadmap](./roadmap.md) and under active development. 
+
+Please note that cellxgene does not have public APIs. Our development may break extensions. We will document changes to the code base but it is advised that extensions pin the version of cellxgene they develop against. 
+
+## Example Reuse & extensions 
+
+#### cellxgene Gateway
+
+[cellxgene Gateway](https://github.com/Novartis/cellxgene-gateway) allows you to use with multiple datasets. It 
+displays an index of available h5ad (anndata) files. When a user clicks on a file name, it launches a Cellxgene Server 
+instance that loads that particular data file and once it is available proxies requests to that server.
+
+#### cellxgene-VIP (Visualization in Plugin)
+
+[cellxgene-VIP](https://github.com/interactivereport/cellxgene_VIP) enables cellxgene to generate violin, stacked violin, stacked bar, heatmap, volcano, embedding, dot, track, density, 2D density, sankey and dual-gene plot in high-resolution SVG/PNG format. It also performs differential gene expression analysis and provides a Command Line Interface (CLI) for advanced users to perform analysis using python and R.
+
+#### Galaxy
+
+[Galaxy](https://singlecell.usegalaxy.eu/) is an open source, web-based platform for data intensive biomedical research. cellxgene can be accessed within Galaxy to view analyzed datasets. 
+See also the relevant [publication](https://www.biorxiv.org/content/10.1101/2020.06.06.137570v1.full.pdf) 
+
+#### Single Cell Portal
+
+The [Single Cell Portal](https://singlecell.broadinstitute.org/single_cell) is a data hosting and visualization service. cellxgene can be embedded as an additional view to complement the visualizations provided by the. 
+[Example](https://singlecell.broadinstitute.org/single_cell/study/SCP807/atlas-of-healthy-and-shiv-infected-non-human-primate-lung-and-ileum-ace2-cells). 
+
+#### FASTGenomics
+
+[FASTGenomics](https://beta.fastgenomics.org/) is a collaborative research platform that offers easy-to-use data management and reproducible analytics to drive single-cell research forward. Many of the publicly available datasets in FASTGenomics - as well as your private datasets - can be interactively explored with cellxgene. 
+See also this [example](https://beta.fastgenomics.org/datasets/detail-dataset-952687f71ef34322a850553c4a24e82e#Cellxgene) for data from [Schulte-Schrepping et al. (Cell, 2020)](https://beta.fastgenomics.org/p/schulte-schrepping_covid19). 
+Note that it is not necessary to create an account, anonymous login is permitted.

docs/posts/extensions.md

@@ -0,0 +1,39 @@
+# Extensions
+
+This project was started with the sole goal of empowering the scientific community to explore and understand their data. 
+As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from 
+this project. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT).
+
+Before extending cellxgene, we encourage you to reach out to us with ideas or questions. It might be possible that an 
+extension could be directly contributed, which would make it available for a wider audience, or that it's on our 
+[roadmap](./roadmap.md) and under active development. 
+
+Please note that cellxgene does not have public APIs. Our development may break extensions. We will document changes to the code base but it is advised that extensions pin the version of cellxgene they develop against. 
+
+## Example Reuse & extensions 
+
+#### cellxgene Gateway
+
+[cellxgene Gateway](https://github.com/Novartis/cellxgene-gateway) allows you to use with multiple datasets. It 
+displays an index of available h5ad (anndata) files. When a user clicks on a file name, it launches a Cellxgene Server 
+instance that loads that particular data file and once it is available proxies requests to that server.
+
+#### cellxgene-VIP (Visualization in Plugin)
+
+[cellxgene-VIP](https://github.com/interactivereport/cellxgene_VIP) enables cellxgene to generate violin, stacked violin, stacked bar, heatmap, volcano, embedding, dot, track, density, 2D density, sankey and dual-gene plot in high-resolution SVG/PNG format. It also performs differential gene expression analysis and provides a Command Line Interface (CLI) for advanced users to perform analysis using python and R.
+
+#### Galaxy
+
+[Galaxy](https://singlecell.usegalaxy.eu/) is an open source, web-based platform for data intensive biomedical research. cellxgene can be accessed within Galaxy to view analyzed datasets. 
+See also the relevant [publication](https://www.biorxiv.org/content/10.1101/2020.06.06.137570v1.full.pdf) 
+
+#### Single Cell Portal
+
+The [Single Cell Portal](https://singlecell.broadinstitute.org/single_cell) is a data hosting and visualization service. cellxgene can be embedded as an additional view to complement the visualizations provided by the. 
+[Example](https://singlecell.broadinstitute.org/single_cell/study/SCP807/atlas-of-healthy-and-shiv-infected-non-human-primate-lung-and-ileum-ace2-cells). 
+
+#### FASTGenomics
+
+[FASTGenomics](https://beta.fastgenomics.org/) is a collaborative research platform that offers easy-to-use data management and reproducible analytics to drive single-cell research forward. Many of the publicly available datasets in FASTGenomics - as well as your private datasets - can be interactively explored with cellxgene. 
+See also this [example](https://beta.fastgenomics.org/datasets/detail-dataset-952687f71ef34322a850553c4a24e82e#Cellxgene) for data from [Schulte-Schrepping et al. (Cell, 2020)](https://beta.fastgenomics.org/p/schulte-schrepping_covid19). 
+Note that it is not necessary to create an account, anonymous login is permitted.

local_server/Makefile

@@ -0,0 +1,26 @@
+include ../common.mk
+
+.PHONY: clean
+clean:
+	rm -f common/web/templates/index.html
+	rm -rf common/web/static
+	rm -f common/web/csp-hashes.json
+
+.PHONY: unit-test
+unit-test:
+	PYTHONWARNINGS=ignore:ResourceWarning coverage run \
+		--source=app,cli,common,compute,converters,data_anndata,data_common \
+		--omit=.coverage,data_common/fbs/NetEncoding,venv \
+		-m unittest discover \
+		--start-directory test/ \
+		--top-level-directory ../ \
+		--verbose; test_result=$$?; \
+	exit $$test_result \
+
+.PHONY: test-annotations-performance
+test-annotations-performance:
+	python test/performance/performance_test_annotations_backend.py
+
+.PHONY: test-annotations-scale
+test-annotations-scale:
+	locust -f test/performance/scale_test_annotations.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt

local_server/__init__.py

@@ -0,0 +1,13 @@
+import logging
+import sys
+from local_server.common.utils.utils import import_plugins
+
+__version__ = "0.16.0"
+display_version = "cellxgene v" + __version__
+
+try:
+    import_plugins("server.plugins")
+except Exception as e:
+    # Make sure to exit in this case, as the server may not be configured as expected.
+    logging.critical(f"Error in import_plugins: {str(e)}")
+    sys.exit(1)

local_server/__main__.py

@@ -0,0 +1,15 @@
+# Work around bug https://github.com/pallets/werkzeug/issues/461
+if __package__ is None:
+    import sys
+    from pathlib import Path
+
+    PKG_PATH = Path(__file__).parent
+    sys.path.insert(0, str(PKG_PATH.parent))
+    import server  # noqa F401
+
+    __package__ = PKG_PATH.name
+
+# Main thing
+from .cli.cli import cli  # noqa F402
+
+cli()

local_server/app/app.py

@@ -0,0 +1,237 @@
+import datetime
+import logging
+from functools import wraps
+from http import HTTPStatus
+
+from flask import (
+    Flask,
+    current_app,
+    make_response,
+    render_template,
+    Blueprint,
+    request,
+    send_from_directory,
+)
+from flask_restful import Api, Resource
+
+import local_server.common.rest as common_rest
+from local_server.common.errors import DatasetAccessError, RequestException
+from local_server.common.health import health_check
+from local_server.common.utils.utils import Float32JSONEncoder
+
+webbp = Blueprint("webapp", "local_server.common.web", template_folder="templates")
+
+
+@webbp.route("/", methods=["GET"])
+def dataset_index():
+    app_config = current_app.app_config
+
+    dataset_config = app_config.get_dataset_config()
+    scripts = dataset_config.app__scripts
+    inline_scripts = dataset_config.app__inline_scripts
+
+    try:
+        args = {"SCRIPTS": scripts, "INLINE_SCRIPTS": inline_scripts}
+        return render_template("index.html", **args)
+
+    except DatasetAccessError as e:
+        return common_rest.abort_and_log(
+            e.status_code, f"Invalid dataset: {e.message}", loglevel=logging.INFO, include_exc_info=True
+        )
+
+
+@webbp.errorhandler(RequestException)
+def handle_request_exception(error):
+    return common_rest.abort_and_log(error.status_code, error.message, loglevel=logging.INFO, include_exc_info=True)
+
+
+def requires_authentication(func):
+    @wraps(func)
+    def wrapped_function(self, *args, **kwargs):
+        auth = current_app.auth
+        if auth.is_user_authenticated():
+            return func(self, *args, **kwargs)
+        else:
+            return make_response("not authenticated", HTTPStatus.UNAUTHORIZED)
+
+    return wrapped_function
+
+
+def rest_get_data_adaptor(func):
+    @wraps(func)
+    def wrapped_function(self):
+        try:
+            return func(self, current_app.data_adaptor)
+        except DatasetAccessError as e:
+            return common_rest.abort_and_log(
+                e.status_code, f"Invalid dataset: {e.message}", loglevel=logging.INFO, include_exc_info=True
+            )
+
+    return wrapped_function
+
+
+class HealthAPI(Resource):
+    def get(self):
+        config = current_app.app_config
+        return health_check(config)
+
+
+class SchemaAPI(Resource):
+    # TODO @mdunitz separate dataset schema and user schema
+    @rest_get_data_adaptor
+    def get(self, data_adaptor):
+        return common_rest.schema_get(data_adaptor)
+
+
+class ConfigAPI(Resource):
+    @rest_get_data_adaptor
+    def get(self, data_adaptor):
+        return common_rest.config_get(current_app.app_config, data_adaptor)
+
+
+class UserInfoAPI(Resource):
+    @rest_get_data_adaptor
+    def get(self, data_adaptor):
+        return common_rest.userinfo_get(current_app.app_config, data_adaptor)
+
+
+class AnnotationsObsAPI(Resource):
+    @rest_get_data_adaptor
+    def get(self, data_adaptor):
+        return common_rest.annotations_obs_get(request, data_adaptor)
+
+    @requires_authentication
+    @rest_get_data_adaptor
+    def put(self, data_adaptor):
+        return common_rest.annotations_obs_put(request, data_adaptor)
+
+
+class AnnotationsVarAPI(Resource):
+    @rest_get_data_adaptor
+    def get(self, data_adaptor):
+        return common_rest.annotations_var_get(request, data_adaptor)
+
+
+class DataVarAPI(Resource):
+    @rest_get_data_adaptor
+    def put(self, data_adaptor):
+        return common_rest.data_var_put(request, data_adaptor)
+
+    @rest_get_data_adaptor
+    def get(self, data_adaptor):
+        return common_rest.data_var_get(request, data_adaptor)
+
+
+class ColorsAPI(Resource):
+    @rest_get_data_adaptor
+    def get(self, data_adaptor):
+        return common_rest.colors_get(data_adaptor)
+
+
+class DiffExpObsAPI(Resource):
+    @rest_get_data_adaptor
+    def post(self, data_adaptor):
+        return common_rest.diffexp_obs_post(request, data_adaptor)
+
+
+class LayoutObsAPI(Resource):
+    @rest_get_data_adaptor
+    def get(self, data_adaptor):
+        return common_rest.layout_obs_get(request, data_adaptor)
+
+    @rest_get_data_adaptor
+    def put(self, data_adaptor):
+        return common_rest.layout_obs_put(request, data_adaptor)
+
+
+class GenesetsAPI(Resource):
+    @rest_get_data_adaptor
+    def get(self, data_adaptor):
+        return common_rest.genesets_get(request, data_adaptor)
+
+    @requires_authentication
+    @rest_get_data_adaptor
+    def put(self, data_adaptor):
+        return common_rest.genesets_put(request, data_adaptor)
+
+
+def get_api_base_resources(bp_base):
+    """Add resources that are accessed from the api url"""
+    api = Api(bp_base)
+
+    # Diagnostics routes
+    api.add_resource(HealthAPI, "/health")
+    return api
+
+
+def get_api_dataroot_resources(bp_dataroot):
+    """Add resources that refer to a dataset"""
+    api = Api(bp_dataroot)
+
+    def add_resource(resource, url):
+        """convenience function to make the outer function less verbose"""
+        api.add_resource(resource, url)
+
+    # Initialization routes
+    add_resource(SchemaAPI, "/schema")
+    add_resource(ConfigAPI, "/config")
+    add_resource(UserInfoAPI, "/userinfo")
+    # Data routes
+    add_resource(AnnotationsObsAPI, "/annotations/obs")
+    add_resource(AnnotationsVarAPI, "/annotations/var")
+    add_resource(DataVarAPI, "/data/var")
+    add_resource(GenesetsAPI, "/genesets")
+    # Display routes
+    add_resource(ColorsAPI, "/colors")
+    # Computation routes
+    add_resource(DiffExpObsAPI, "/diffexp/obs")
+    add_resource(LayoutObsAPI, "/layout/obs")
+    return api
+
+
+class Server:
+    @staticmethod
+    def _before_adding_routes(app, app_config):
+        """ will be called before routes are added, during __init__.  Subclass protocol """
+        pass
+
+    def __init__(self, app_config):
+        self.app = Flask(__name__, static_folder=None)
+        self._before_adding_routes(self.app, app_config)
+        self.app.json_encoder = Float32JSONEncoder
+        server_config = app_config.server_config
+
+        # enable session data
+        self.app.permanent_session_lifetime = datetime.timedelta(days=50 * 365)
+
+        # Config
+        secret_key = server_config.app__flask_secret_key
+        self.app.config.update(SECRET_KEY=secret_key)
+
+        self.app.register_blueprint(webbp)
+
+        api_version = "/api/v0.2"
+        api_path = "/"
+
+        bp_base = Blueprint("bp_base", __name__, url_prefix=api_path)
+        base_resources = get_api_base_resources(bp_base)
+        self.app.register_blueprint(base_resources.blueprint)
+
+        bp_api = Blueprint("api", __name__, url_prefix=f"{api_path}{api_version}")
+        resources = get_api_dataroot_resources(bp_api)
+        self.app.register_blueprint(resources.blueprint)
+        self.app.add_url_rule(
+            "/static/<path:filename>",
+            "static_assets",
+            view_func=lambda filename: send_from_directory("../common/web/static", filename),
+            methods=["GET"],
+        )
+
+        self.app.data_adaptor = server_config.data_adaptor
+        self.app.app_config = app_config
+
+        auth = server_config.auth
+        self.app.auth = auth
+        if auth.requires_client_login():
+            auth.add_url_rules(self.app)
+        auth.complete_setup(self.app)

local_server/auth/__init__.py

@@ -0,0 +1,5 @@
+# import the built in auth types so they can be registered
+
+import local_server.auth.auth_none  # noqa: F401
+import local_server.auth.auth_test  # noqa: F401
+import local_server.auth.auth_session  # noqa: F401

local_server/auth/auth.py

@@ -0,0 +1,91 @@
+from abc import ABC, abstractmethod
+
+
+class AuthTypeBase(ABC):
+    """Base type for all authentication types."""
+
+    def __init__(self):
+        super().__init__()
+
+    @abstractmethod
+    def is_valid_authentication_type(self):
+        """Return True if the auth type is valid, e.g. it can return userinfo and username.
+          (AuthTypeNone is the only one type that returns False)"""
+        pass
+
+    def requires_client_login(self):
+        """Return True if the user needs to login from the client (e.g. Login button is shown)"""
+        return False
+
+    @abstractmethod
+    def complete_setup(self, app):
+        """complete any setup that may be needed by this auth type.  The Flask app is passed in.
+        This is the last auth function called before the server starts to run."""
+        pass
+
+    @abstractmethod
+    def is_user_authenticated(self):
+        """Return True if the user is authenticated"""
+        pass
+
+    @abstractmethod
+    def get_user_id(self):
+        """Return the id for this user (string)"""
+        pass
+
+    @abstractmethod
+    def get_user_name(self):
+        """Return the name of the user (string)"""
+        pass
+
+    @abstractmethod
+    def get_user_email(self):
+        """Return the name of the user (string)"""
+        pass
+
+    def get_user_picture(self):
+        """Return the location to the user's picture"""
+        return None
+
+
+class AuthTypeClientBase(AuthTypeBase):
+    """Base type for all authentication types that require the client to login"""
+
+    def __init__(self):
+        super().__init__()
+
+    def requires_client_login(self):
+        return True
+
+    @abstractmethod
+    def add_url_rules(self, selfapp):
+        """Add url rules to the app (like /login, /logout, etc)"""
+        pass
+
+    @abstractmethod
+    def get_login_url(self, data_adaptor):
+        """Return the url for the login route"""
+        pass
+
+    @abstractmethod
+    def get_logout_url(self, data_adaptor):
+        """Return the url for the logout route"""
+        pass
+
+
+class AuthTypeFactory:
+    """Factory class to create an authentication type"""
+
+    auth_types = {}
+
+    @staticmethod
+    def register(name, auth_type):
+        assert issubclass(auth_type, AuthTypeBase)
+        AuthTypeFactory.auth_types[name] = auth_type
+
+    @staticmethod
+    def create(name, app_config):
+        auth_type = AuthTypeFactory.auth_types.get(name)
+        if auth_type is None:
+            return None
+        return auth_type(app_config)

local_server/auth/auth_none.py

@@ -0,0 +1,27 @@
+from local_server.auth.auth import AuthTypeBase, AuthTypeFactory
+
+
+class AuthTypeNone(AuthTypeBase):
+    def __init__(self, app_config):
+        super().__init__()
+
+    def is_valid_authentication_type(self):
+        return False
+
+    def complete_setup(self, app):
+        pass
+
+    def is_user_authenticated(self):
+        return True
+
+    def get_user_id(self):
+        return None
+
+    def get_user_name(self):
+        return None
+
+    def get_user_email(self):
+        return None
+
+
+AuthTypeFactory.register(None, AuthTypeNone)

local_server/auth/auth_session.py

@@ -0,0 +1,39 @@
+from local_server.auth.auth import AuthTypeBase, AuthTypeFactory
+from flask import session
+from uuid import uuid4
+
+
+class AuthTypeSession(AuthTypeBase):
+    """Session based authentication.  The user is always logged.  The user id is a random number
+    associated with the session.  This is a good choice for desktop servers."""
+
+    # key in the session token for userid
+    CXGUID = "cxguid"
+
+    def __init__(self, app_config):
+        super().__init__()
+
+    def is_valid_authentication_type(self):
+        return True
+
+    def complete_setup(self, app):
+        pass
+
+    def is_user_authenticated(self):
+        # always authenticated
+        return True
+
+    def get_user_id(self):
+        if self.CXGUID not in session:
+            session[self.CXGUID] = uuid4().hex
+            session.permanent = True
+        return session[self.CXGUID]
+
+    def get_user_name(self):
+        return "anonymous"
+
+    def get_user_email(self):
+        return None
+
+
+AuthTypeFactory.register("session", AuthTypeSession)

local_server/auth/auth_test.py

@@ -0,0 +1,73 @@
+from local_server.auth.auth import AuthTypeClientBase, AuthTypeFactory
+from flask import session, request, redirect
+
+
+class AuthTypeTest(AuthTypeClientBase):
+    """An authentication type for testing client based logins.  When the login route is accessed
+    the user is automatically logged in with a default or configured username"""
+
+    # key in session token with userid and username
+    CXGUID = "cxguid_test"
+    CXGUNAME = "cxguname_test"
+    CXGUEMAIL = "cxguemail_test"
+    CXGUPICTURE = "cxgupicture_test"
+
+    def __init__(self, app_config):
+        super().__init__()
+        self.user_name = "test_account"
+        self.user_id = "id0001"
+        self.user_email = "test_account@test.com"
+        self.user_picture = None
+
+    def is_valid_authentication_type(self):
+        return True
+
+    def requires_client_login(self):
+        return True
+
+    def add_url_rules(self, app):
+        app.add_url_rule("/login", "login", self.login, methods=["GET"])
+        app.add_url_rule("/logout", "logout", self.logout, methods=["GET"])
+
+    def complete_setup(self, app):
+        pass
+
+    def is_user_authenticated(self):
+        return self.CXGUID in session
+
+    def get_user_id(self):
+        return session.get(self.CXGUID)
+
+    def get_user_name(self):
+        return session.get(self.CXGUNAME)
+
+    def get_user_email(self):
+        return session.get(self.CXGUEMAIL)
+
+    def get_user_picture(self):
+        return session.get(self.CXGUPICTURE)
+
+    def login(self):
+        args = request.args
+        return_to = args.get("dataset", "/")
+        session[self.CXGUID] = args.get("userid", self.user_id)
+        session[self.CXGUNAME] = args.get("username", self.user_name)
+        session[self.CXGUEMAIL] = args.get("email", self.user_email)
+        session[self.CXGUPICTURE] = args.get("picture", self.user_picture)
+        return redirect(return_to)
+
+    def logout(self):
+        session.clear()
+        return_to = request.args.get("dataset", "/")
+        return redirect(return_to)
+
+    def get_login_url(self, data_adaptor):
+        """Return the url for the login route"""
+        return "/login"
+
+    def get_logout_url(self, data_adaptor):
+        """Return the url for the logout route"""
+        return "/logout"
+
+
+AuthTypeFactory.register("test", AuthTypeTest)

local_server/cli/cli.py

@@ -0,0 +1,33 @@
+import click
+
+from .launch import launch
+from .prepare import prepare
+from .upgrade import log_upgrade_check
+from .schema import schema_cli
+from .. import __version__
+
+
+@click.group(
+    name="cellxgene",
+    subcommand_metavar="COMMAND <args>",
+    options_metavar="<options>",
+    context_settings=dict(max_content_width=85, help_option_names=["-h", "--help"]),
+)
+@click.help_option("--help", "-h", help="Show this message and exit.")
+@click.version_option(
+    version=__version__,
+    prog_name="cellxgene",
+    message="[%(prog)s] Version %(version)s",
+    help="Show the software version and exit.",
+)
+@click.option(
+    "--upgrade-check/--no-upgrade-check", default=True, show_default=True, help="Check for release upgrades on start.",
+)
+def cli(upgrade_check):
+    if upgrade_check:
+        log_upgrade_check()
+
+
+cli.add_command(launch)
+cli.add_command(prepare)
+cli.add_command(schema_cli)

local_server/cli/launch.py

@@ -0,0 +1,467 @@
+import errno
+import functools
+import logging
+import sys
+import webbrowser
+import os
+import click
+from flask_compress import Compress
+from flask_cors import CORS
+
+from local_server.default_config import default_config
+from local_server.app.app import Server
+from local_server.common.config.app_config import AppConfig
+from local_server.common.errors import DatasetAccessError, ConfigurationError
+from local_server.common.utils.utils import sort_options
+
+DEFAULT_CONFIG = AppConfig()
+
+
+def annotation_args(func):
+    @click.option(
+        "--disable-annotations",
+        is_flag=True,
+        default=not DEFAULT_CONFIG.dataset_config.user_annotations__enable,
+        show_default=True,
+        help="Disable user annotation of data.",
+    )
+    @click.option(
+        "--annotations-file",
+        default=DEFAULT_CONFIG.dataset_config.user_annotations__local_file_csv__file,
+        show_default=True,
+        multiple=False,
+        metavar="<path>",
+        help="CSV file to initialize editing of existing annotations; will be altered in-place. "
+        "Incompatible with --user-generated-data-dir.",
+    )
+    @click.option(
+        "--user-generated-data-dir",
+        "--annotations-dir",
+        default=DEFAULT_CONFIG.dataset_config.user_annotations__local_file_csv__directory,
+        show_default=False,
+        multiple=False,
+        metavar="<directory path>",
+        help="Directory of where to save output annotations; filename will be specified in the application. "
+        "Incompatible with --annotations-file and --gene-sets-file.",
+    )
+    @click.option(
+        "--experimental-annotations-ontology",
+        is_flag=True,
+        default=DEFAULT_CONFIG.dataset_config.user_annotations__ontology__enable,
+        show_default=True,
+        help="When creating annotations, optionally autocomplete names from ontology terms.",
+    )
+    @click.option(
+        "--experimental-annotations-ontology-obo",
+        default=DEFAULT_CONFIG.dataset_config.user_annotations__ontology__obo_location,
+        show_default=True,
+        metavar="<path or url>",
+        help="Location of OBO file defining cell annotation autosuggest terms.",
+    )
+    @click.option(
+        "--disable-gene-sets-save",
+        is_flag=True,
+        default=DEFAULT_CONFIG.dataset_config.user_annotations__gene_sets__readonly,
+        show_default=False,
+        help="Disable saving gene sets. If disabled, users will be able to make changes to gene sets but all "
+        "changes will be lost on browser refresh.",
+    )
+    @click.option(
+        "--gene-sets-file",
+        default=DEFAULT_CONFIG.dataset_config.user_annotations__local_file_csv__gene_sets_file,
+        show_default=True,
+        multiple=False,
+        metavar="<path>",
+        help="CSV file to initialize editing of gene sets; will be altered in-place. Incompatible with "
+        "--user-generated-data-dir.",
+    )
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
+def config_args(func):
+    @click.option(
+        "--max-category-items",
+        default=DEFAULT_CONFIG.dataset_config.presentation__max_categories,
+        metavar="<integer>",
+        show_default=True,
+        help="Will not display categories with more distinct values than specified.",
+    )
+    @click.option(
+        "--disable-custom-colors",
+        is_flag=True,
+        default=False,
+        show_default=False,
+        help="Disable user-defined category-label colors drawn from source data file.",
+    )
+    @click.option(
+        "--diffexp-lfc-cutoff",
+        "-de",
+        default=DEFAULT_CONFIG.dataset_config.diffexp__lfc_cutoff,
+        show_default=True,
+        metavar="<float>",
+        help="Minimum log fold change threshold for differential expression.",
+    )
+    @click.option(
+        "--disable-diffexp",
+        is_flag=True,
+        default=not DEFAULT_CONFIG.dataset_config.diffexp__enable,
+        show_default=False,
+        help="Disable on-demand differential expression.",
+    )
+    @click.option(
+        "--embedding",
+        "-e",
+        default=DEFAULT_CONFIG.dataset_config.embeddings__names,
+        multiple=True,
+        show_default=False,
+        metavar="<text>",
+        help="Embedding name, eg, 'umap'. Repeat option for multiple embeddings. Defaults to all.",
+    )
+    @click.option(
+        "--experimental-enable-reembedding",
+        is_flag=True,
+        default=DEFAULT_CONFIG.dataset_config.embeddings__enable_reembedding,
+        show_default=False,
+        hidden=True,
+        help="Enable experimental on-demand re-embedding using UMAP. WARNING: may be very slow.",
+    )
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
+def dataset_args(func):
+    @click.option(
+        "--obs-names",
+        "-obs",
+        default=DEFAULT_CONFIG.server_config.single_dataset__obs_names,
+        metavar="<text>",
+        help="Name of annotation field to use for observations. If not specified cellxgene will use the the obs index.",
+    )
+    @click.option(
+        "--var-names",
+        "-var",
+        default=DEFAULT_CONFIG.server_config.single_dataset__var_names,
+        metavar="<text>",
+        help="Name of annotation to use for variables. If not specified cellxgene will use the the var index.",
+    )
+    @click.option(
+        "--backed",
+        "-b",
+        is_flag=True,
+        default=DEFAULT_CONFIG.server_config.adaptor__anndata_adaptor__backed,
+        show_default=False,
+        help="Load anndata in file-backed mode. " "This may save memory, but may result in slower overall performance.",
+    )
+    @click.option(
+        "--title",
+        "-t",
+        default=DEFAULT_CONFIG.server_config.single_dataset__title,
+        metavar="<text>",
+        help="Title to display. If omitted will use file name.",
+    )
+    @click.option(
+        "--about",
+        default=DEFAULT_CONFIG.server_config.single_dataset__about,
+        metavar="<URL>",
+        help="URL providing more information about the dataset (hint: must be a fully specified absolute URL).",
+    )
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
+def server_args(func):
+    @click.option(
+        "--debug",
+        "-d",
+        is_flag=True,
+        default=DEFAULT_CONFIG.server_config.app__debug,
+        show_default=True,
+        help="Run in debug mode. This is helpful for cellxgene developers, "
+        "or when you want more information about an error condition.",
+    )
+    @click.option(
+        "--verbose",
+        "-v",
+        is_flag=True,
+        default=DEFAULT_CONFIG.server_config.app__verbose,
+        show_default=True,
+        help="Provide verbose output, including warnings and all server requests.",
+    )
+    @click.option(
+        "--port",
+        "-p",
+        metavar="<port>",
+        default=DEFAULT_CONFIG.server_config.app__port,
+        type=int,
+        show_default=True,
+        help="Port to run server on. If not specified cellxgene will find an available port.",
+    )
+    @click.option(
+        "--host",
+        metavar="<IP address>",
+        default=DEFAULT_CONFIG.server_config.app__host,
+        show_default=False,
+        help="Host IP address. By default cellxgene will use localhost (e.g. 127.0.0.1).",
+    )
+    @click.option(
+        "--scripts",
+        "-s",
+        default=DEFAULT_CONFIG.dataset_config.app__scripts,
+        multiple=True,
+        metavar="<text>",
+        help="Additional script files to include in HTML page. If not specified, "
+        "no additional script files will be included.",
+        show_default=False,
+    )
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
+def launch_args(func):
+    @annotation_args
+    @config_args
+    @dataset_args
+    @server_args
+    @click.argument("datapath", required=False, metavar="<path to data file>")
+    @click.option(
+        "--open",
+        "-o",
+        "open_browser",
+        is_flag=True,
+        default=DEFAULT_CONFIG.server_config.app__open_browser,
+        show_default=True,
+        help="Open web browser after launch.",
+    )
+    @click.option(
+        "--config-file",
+        "-c",
+        "config_file",
+        default=None,
+        show_default=True,
+        help="Location to yaml file with configuration settings",
+    )
+    @click.option(
+        "--dump-default-config",
+        "dump_default_config",
+        is_flag=True,
+        default=False,
+        show_default=True,
+        help="Print default configuration settings and exit",
+    )
+    @click.help_option("--help", "-h", help="Show this message and exit.")
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
+def handle_scripts(scripts):
+    if scripts:
+        click.echo(
+            r"""
+    / / /\ \ \__ _ _ __ _ __ (_)_ __   __ _
+    \ \/  \/ / _` | '__| '_ \| | '_ \ / _` |
+     \  /\  / (_| | |  | | | | | | | | (_| |
+      \/  \/ \__,_|_|  |_| |_|_|_| |_|\__, |
+                                      |___/
+    The --scripts flag is intended for developers to include google analytics etc. You could be opening yourself to a
+    security risk by including the --scripts flag. Make sure you trust the scripts that you are including.
+            """
+        )
+        scripts_pretty = ", ".join(scripts)
+        click.confirm(f"Are you sure you want to inject these scripts: {scripts_pretty}?", abort=True)
+
+
+class CliLaunchServer(Server):
+    """
+    the CLI runs a local web server, and needs to enable a few more features.
+    """
+
+    def __init__(self, app_config):
+        super().__init__(app_config)
+
+    @staticmethod
+    def _before_adding_routes(app, app_config):
+        app.config["COMPRESS_MIMETYPES"] = [
+            "text/html",
+            "text/css",
+            "text/xml",
+            "application/json",
+            "application/javascript",
+            "application/octet-stream",
+        ]
+        Compress(app)
+        if app_config.server_config.app__debug:
+            CORS(app, supports_credentials=True)
+
+
+@sort_options
+@click.command(
+    short_help="Launch the cellxgene data viewer. " "Run `cellxgene launch --help` for more information.",
+    options_metavar="<options>",
+)
+@launch_args
+def launch(
+    datapath,
+    verbose,
+    debug,
+    open_browser,
+    port,
+    host,
+    embedding,
+    obs_names,
+    var_names,
+    max_category_items,
+    disable_custom_colors,
+    diffexp_lfc_cutoff,
+    title,
+    scripts,
+    about,
+    disable_annotations,
+    annotations_file,
+    user_generated_data_dir,
+    gene_sets_file,
+    disable_gene_sets_save,
+    backed,
+    disable_diffexp,
+    experimental_annotations_ontology,
+    experimental_annotations_ontology_obo,
+    experimental_enable_reembedding,
+    config_file,
+    dump_default_config,
+):
+    """Launch the cellxgene data viewer.
+    This web app lets you explore single-cell expression data.
+    Data must be in a format that cellxgene expects.
+    Read the "getting started" guide to learn more:
+    https://chanzuckerberg.github.io/cellxgene/getting-started.html
+
+    Examples:
+
+    > cellxgene launch example-dataset/pbmc3k.h5ad --title pbmc3k
+
+    > cellxgene launch <your data file> --title <your title>
+
+    > cellxgene launch <url>"""
+
+    if dump_default_config:
+        print(default_config)
+        sys.exit(0)
+
+    # Startup message
+    click.echo("[cellxgene] Starting the CLI...")
+
+    # app config
+    app_config = AppConfig()
+    server_config = app_config.server_config
+
+    try:
+        if config_file:
+            app_config.update_from_config_file(config_file)
+
+        # Determine which config options were give on the command line.
+        # Those will override the ones provided in the config file (if provided).
+        cli_config = AppConfig()
+        cli_config.update_server_config(
+            app__verbose=verbose,
+            app__debug=debug,
+            app__host=host,
+            app__port=port,
+            app__open_browser=open_browser,
+            single_dataset__datapath=datapath,
+            single_dataset__title=title,
+            single_dataset__about=about,
+            single_dataset__obs_names=obs_names,
+            single_dataset__var_names=var_names,
+            adaptor__anndata_adaptor__backed=backed,
+        )
+        cli_config.update_dataset_config(
+            app__scripts=scripts,
+            user_annotations__enable=not disable_annotations,
+            user_annotations__local_file_csv__file=annotations_file,
+            user_annotations__local_file_csv__directory=user_generated_data_dir,
+            user_annotations__local_file_csv__gene_sets_file=gene_sets_file,
+            user_annotations__gene_sets__readonly=disable_gene_sets_save,
+            user_annotations__ontology__enable=experimental_annotations_ontology,
+            user_annotations__ontology__obo_location=experimental_annotations_ontology_obo,
+            presentation__max_categories=max_category_items,
+            presentation__custom_colors=not disable_custom_colors,
+            embeddings__names=embedding,
+            embeddings__enable_reembedding=experimental_enable_reembedding,
+            diffexp__enable=not disable_diffexp,
+            diffexp__lfc_cutoff=diffexp_lfc_cutoff,
+        )
+
+        diff = cli_config.server_config.changes_from_default()
+        changes = {key: val for key, val, _ in diff}
+        app_config.update_server_config(**changes)
+
+        diff = cli_config.dataset_config.changes_from_default()
+        changes = {key: val for key, val, _ in diff}
+        app_config.update_dataset_config(**changes)
+
+        # process the configuration
+        #  any errors will be thrown as an exception.
+        #  any info messages will be passed to the messagefn function.
+
+        def messagefn(message):
+            click.echo("[cellxgene] " + message)
+
+        # Use a default secret if one is not provided
+        if not server_config.app__flask_secret_key:
+            app_config.update_server_config(app__flask_secret_key="SparkleAndShine")
+
+        app_config.complete_config(messagefn)
+
+    except (ConfigurationError, DatasetAccessError) as e:
+        raise click.ClickException(e)
+
+    handle_scripts(scripts)
+
+    # create the server
+    server = CliLaunchServer(app_config)
+
+    if not server_config.app__verbose:
+        log = logging.getLogger("werkzeug")
+        log.setLevel(logging.ERROR)
+
+    cellxgene_url = f"http://{app_config.server_config.app__host}:{app_config.server_config.app__port}"
+    if server_config.app__open_browser:
+        click.echo(f"[cellxgene] Launching! Opening your browser to {cellxgene_url} now.")
+        webbrowser.open(cellxgene_url)
+    else:
+        click.echo(f"[cellxgene] Launching! Please go to {cellxgene_url} in your browser.")
+
+    click.echo("[cellxgene] Type CTRL-C at any time to exit.")
+
+    if not server_config.app__verbose:
+        f = open(os.devnull, "w")
+        sys.stdout = f
+
+    try:
+        server.app.run(
+            host=server_config.app__host,
+            debug=server_config.app__debug,
+            port=server_config.app__port,
+            threaded=not server_config.app__debug,
+            use_debugger=False,
+            use_reloader=False,
+        )
+    except OSError as e:
+        if e.errno == errno.EADDRINUSE:
+            raise click.ClickException("Port is in use, please specify an open port using the --port flag.") from e
+        raise

local_server/cli/prepare.py

@@ -0,0 +1,274 @@
+from os.path import expanduser, isdir, isfile, sep, splitext
+
+import click
+import pandas as pd
+from numpy import ndarray, unique
+from scipy.sparse.csc import csc_matrix
+
+from local_server.common.utils.utils import sort_options
+
+
+@sort_options
+@click.command(
+    short_help="Preprocess data for use with cellxgene. " "Run `cellxgene prepare --help` for more information.",
+    options_metavar="<options>",
+)
+@click.argument("data", nargs=1, metavar="<path to data file>", required=True)
+@click.option(
+    "--embedding",
+    "-e",
+    default=["umap", "tsne"],
+    multiple=True,
+    type=click.Choice(["umap", "tsne"]),
+    help="Embedding algorithm(s). Repeat option for multiple embeddings.",
+    show_default=True,
+)
+@click.option(
+    "--recipe", "-r", default="none", type=click.Choice(["none", "seurat", "zheng17"]), show_default=True,
+)
+@click.option("--output", "-o", default="", help="Save a new file to filename.", metavar="<filename>")
+@click.option("--plotting", "-p", default=False, is_flag=True, help="Generate plots.", show_default=True)
+@click.option("--sparse", default=False, is_flag=True, help="Force sparsity.", show_default=True)
+@click.option("--overwrite", default=False, is_flag=True, help="Allow file overwriting.", show_default=True)
+@click.option("--set-obs-names", default="", help="Named field to set as index for obs.", metavar="<name>")
+@click.option("--set-var-names", default="", help="Named field to set as index for var.", metavar="<name>")
+@click.option(
+    "--skip-qc",
+    default=False,
+    is_flag=True,
+    help="Do not run quality control metrics. By default cellxgene runs them "
+    "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).",
+)
+@click.option(
+    "--make-obs-names-unique/--no-make-obs-names-unique",
+    default=True,
+    help="Ensure obs index is unique.",
+    show_default=True,
+)
+@click.option(
+    "--make-var-names-unique/--no-make-var-names-unique",
+    default=True,
+    help="Ensure var index is unique.",
+    show_default=True,
+)
+@click.help_option("--help", "-h", help="Show this message and exit.")
+def prepare(
+    data,
+    embedding,
+    recipe,
+    output,
+    plotting,
+    sparse,
+    overwrite,
+    set_obs_names,
+    set_var_names,
+    skip_qc,
+    make_obs_names_unique,
+    make_var_names_unique,
+):
+    """
+    Preprocess data for use with cellxgene.
+    This tool runs a series of scanpy routines for preparing a dataset for use
+    with cellxgene. It loads data from different formats
+    (h5ad, loom, or a 10x directory), runs dimensionality reduction,
+    computes nearest neighbors, computes an embedding, performs clustering,
+    and saves the results. Includes additional options for naming annotations,
+    ensuring sparsity, and plotting results.
+    """
+
+    # collect slow imports here to make CLI startup more responsive
+    click.echo("[cellxgene] Starting CLI...")
+    try:
+        import matplotlib
+
+        matplotlib.use("Agg")
+        import scanpy as sc
+    except ImportError:
+        raise click.ClickException(
+            "[cellxgene] cellxgene prepare has not been installed. Please run `pip install 'cellxgene[prepare]'` "
+            "to install the necessary requirements."
+        )
+
+    # scanpy settings
+    sc.settings.verbosity = 0
+    sc.settings.autosave = True
+
+    # check args
+    if sparse and not recipe == "none":
+        raise click.UsageError("Cannot use a recipe when forcing sparsity")
+
+    output = expanduser(output)
+
+    if not output:
+        click.echo(
+            "Warning: No file will be saved, to save the results of cellxgene prepare include "
+            "--output <filename> to save output to a new file"
+        )
+    if isfile(output) and not overwrite:
+        raise click.UsageError(f"Cannot overwrite existing file {output}, try using the flag --overwrite")
+
+    def load_data(data):
+        if isfile(data):
+            name, extension = splitext(data)
+            if extension == ".h5ad":
+                adata = sc.read_h5ad(data)
+            elif extension == ".loom":
+                adata = sc.read_loom(data)
+            else:
+                raise click.FileError(data, hint="does not have a valid extension [.h5ad | .loom]")
+        elif isdir(data):
+            if not data.endswith(sep):
+                data += sep
+            adata = sc.read_10x_mtx(data)
+        else:
+            raise click.FileError(data, hint="not a valid file or path")
+
+        if not set_obs_names == "":
+            if set_obs_names not in adata.obs_keys():
+                raise click.UsageError(f"obs {set_obs_names} not found, options are: {adata.obs_keys()}")
+            adata.obs_names = adata.obs[set_obs_names]
+        if not set_var_names == "":
+            if set_var_names not in adata.var_keys():
+                raise click.UsageError(f"var {set_var_names} not found, options are: {adata.var_keys()}")
+            adata.var_names = adata.var[set_var_names]
+        if make_obs_names_unique:
+            adata.obs.index = make_index_unique(adata.obs.index)
+        if make_var_names_unique:
+            adata.var.index = make_index_unique(adata.var.index)
+        if not adata._obs.index.is_unique:
+            click.echo("Warning: obs index is not unique")
+        if not adata._var.index.is_unique:
+            click.echo("Warning: var index is not unique")
+        return adata
+
+    def calculate_qc_metrics(adata):
+        if not skip_qc:
+            sc.pp.calculate_qc_metrics(adata, inplace=True)
+        return adata
+
+    def make_sparse(adata):
+        if (type(adata.X) is ndarray) and sparse:
+            adata.X = csc_matrix(adata.X)
+
+    def run_recipe(adata):
+        if recipe == "seurat":
+            sc.pp.recipe_seurat(adata)
+        elif recipe == "zheng17":
+            sc.pp.recipe_zheng17(adata)
+        else:
+            sc.pp.filter_cells(adata, min_genes=5)
+            sc.pp.filter_genes(adata, min_cells=25)
+            if sparse:
+                sc.pp.scale(adata, zero_center=False)
+            else:
+                sc.pp.scale(adata)
+
+    def run_pca(adata):
+        if sparse:
+            sc.pp.pca(adata, svd_solver="arpack", zero_center=False)
+        else:
+            sc.pp.pca(adata, svd_solver="arpack")
+
+    def run_neighbors(adata):
+        sc.pp.neighbors(adata)
+
+    def run_louvain(adata):
+        sc.tl.louvain(adata)
+
+    def run_embedding(adata):
+        if len(unique(adata.obs["louvain"].values)) < 10:
+            palette = "tab10"
+        else:
+            palette = "tab20"
+
+        if "umap" in embedding:
+            sc.tl.umap(adata)
+            if plotting:
+                sc.pl.umap(adata, color="louvain", palette=palette, save="_louvain")
+
+        if "tsne" in embedding:
+            sc.tl.tsne(adata)
+            if plotting:
+                sc.pl.tsne(adata, color="louvain", palette=palette, save="_louvain")
+
+    def show_step(item):
+        if not skip_qc:
+            qc_name = "Calculating QC metrics"
+        else:
+            qc_name = "Skipping QC"
+        names = {
+            "calculate_qc_metrics": qc_name,
+            "make_sparse": "Ensuring sparsity",
+            "run_recipe": f'Running preprocessing recipe "{recipe}"',
+            "run_pca": "Running PCA",
+            "run_neighbors": "Calculating neighbors",
+            "run_louvain": "Calculating clusters",
+            "run_embedding": "Computing embedding",
+        }
+        if item is not None:
+            return names[item.__name__]
+
+    steps = [calculate_qc_metrics, make_sparse, run_recipe, run_pca, run_neighbors, run_louvain, run_embedding]
+
+    click.echo(f"[cellxgene] Loading data from {data}, please wait...")
+    adata = load_data(data)
+
+    click.echo("[cellxgene] Beginning preprocessing...")
+    with click.progressbar(steps, label="[cellxgene] Progress", show_eta=False, item_show_func=show_step) as bar:
+        for step in bar:
+            step(adata)
+
+    # saving
+    if not output == "":
+        click.echo(f"[cellxgene] Saving results to {output}...")
+        adata.write(output)
+
+    click.echo("[cellxgene] Success!")
+
+
+# TODO (mweiden): remove this once this issue is resolved https://github.com/theislab/anndata/issues/344
+# Note: tentative solution here https://github.com/theislab/anndata/pull/345
+def make_index_unique(index: pd.Index, join: str = "-"):
+    """
+    Makes the index unique by appending a number string to each duplicate index element: '1', '2', etc.
+
+    If a tentative name created by the algorithm already exists in the index, it tries the next integer in the sequence.
+
+    The first occurrence of a non-unique value is ignored.
+    Parameters
+    ----------
+    join
+         The connecting string between name and integer.
+    Examples
+    --------
+    >>> from anndata import AnnData
+    >>> adata1 = AnnData(np.ones((3, 2)), dict(obs_names=['a', 'b', 'c']))
+    >>> adata2 = AnnData(np.zeros((3, 2)), dict(obs_names=['d', 'b', 'b']))
+    >>> adata = adata1.concatenate(adata2)
+    >>> adata.obs_names
+    Index(['a', 'b', 'c', 'd', 'b', 'b'], dtype='object')
+    >>> adata.obs_names_make_unique()
+    >>> adata.obs_names
+    Index(['a', 'b', 'c', 'd', 'b-1', 'b-2'], dtype='object')
+    """
+    if index.is_unique:
+        return index
+    from collections import defaultdict
+
+    values = index.values
+    values_set = set(values)
+    indices_dup = index.duplicated(keep="first")
+    values_dup = values[indices_dup]
+    counter = defaultdict(lambda: 0)
+    for i, v in enumerate(values_dup):
+        while True:
+            counter[v] += 1
+            tentative_new_name = v + join + str(counter[v])
+            if tentative_new_name not in values_set:
+                values_set.add(tentative_new_name)
+                values_dup[i] = tentative_new_name
+                break
+
+    values[indices_dup] = values_dup
+    index = pd.Index(values)
+    return index

local_server/cli/schema.py

@@ -0,0 +1,72 @@
+import click
+
+from local_server.converters.schema import remix, validate
+
+
+@click.group(
+    name="schema",
+    subcommand_metavar="COMMAND <args>",
+    short_help="Apply and validate the cellxgene data integration schema to an h5ad file.",
+    context_settings=dict(max_content_width=85, help_option_names=["-h", "--help"]),
+)
+def schema_cli():
+    try:
+        import scanpy  # noqa: F401
+    except ImportError:
+        raise click.ClickException(
+            "[cellxgene] cellxgene schema requires scanpy"
+        )
+
+
+@click.command(
+    name="apply",
+    short_help="(experimental) Apply the cellxgene data integration schema to an h5ad.",
+    help="(experimental) Using a yaml file that describes schema values to insert or convert and in input "
+    "h5ad file, apply the schema changes and create a new, conforming h5ad.",
+)
+@click.option(
+    "--source-h5ad",
+    help="Input h5ad file.",
+    nargs=1,
+    required=True,
+    type=click.Path(exists=True, dir_okay=False),
+)
+@click.option(
+    "--remix-config",
+    help="Config yaml with information on how to apply the schema.",
+    nargs=1,
+    required=True,
+    type=click.Path(exists=True, dir_okay=False),
+)
+@click.option(
+    "--output-filename",
+    help="Filename for the new, schema-conforming h5ad file.",
+    required=True,
+    nargs=1
+)
+def schema_apply(source_h5ad, remix_config, output_filename):
+    remix.apply_schema(source_h5ad, remix_config, output_filename)
+
+
+@click.command(
+    name="validate",
+    short_help="(experimental) Check that an h5ad follows the cellxgene data integration schema.",
+)
+@click.argument(
+    "h5ad",
+    nargs=1,
+    type=click.Path(exists=True, dir_okay=False),
+)
+@click.option(
+    "--shallow",
+    help="When true, just check that the correct version information is present.",
+    default=False,
+    show_default=True,
+    is_flag=True,
+)
+def schema_validate(h5ad, shallow):
+    validate.validate(h5ad, shallow)
+
+
+schema_cli.add_command(schema_apply)
+schema_cli.add_command(schema_validate)

local_server/cli/upgrade.py

@@ -0,0 +1,85 @@
+import re
+
+import click
+import requests
+from requests.exceptions import ConnectionError
+
+from .. import __version__
+
+# Official SemVer regex: https://semver.org/
+SEMVER_FORMAT = re.compile(
+    r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*["
+    r"a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+("
+    r"?:\.[0-9a-zA-Z-]+)*))?$"
+)
+
+
+def log_upgrade_check():
+    # Sanity-check that the CLI version is a properly-formatted SemVer string
+    assert validate_version_str(__version__, release_only=False)
+
+    # Get the current latest release
+    try:
+        release_tag_generator = (r["tag_name"] for r in _request_cellxgene_releases())
+        latest_release = next(release_tag_generator, lambda tag_name: validate_version_str(tag_name))
+        if version_gt(latest_release, __version__):
+            click.echo(f"There's a new version of cellxgene available ({latest_release})!", err=True)
+            click.echo("To upgrade, run the following: pip install --upgrade cellxgene\n", err=True)
+    except (ConnectionError, RateLimitException):
+        click.echo("Upgrade check failed.\n")
+
+
+class RateLimitException(Exception):
+    """
+    Github API Rate Limit Exception
+    """
+
+
+def _request_cellxgene_releases():
+    def raise_on_rate_limit(response):
+        if response.status_code == 403 and res.headers.get("X-RateLimit-Remaining") == "0":
+            raise RateLimitException
+
+    url = "https://api.github.com/repos/chanzuckerberg/cellxgene/releases"
+    res = requests.get(url)
+    raise_on_rate_limit(res)
+    for release in res.json():
+        yield release
+    while "next" in res.links.keys():
+        res = requests.get(res.links["next"]["url"])
+        raise_on_rate_limit(res)
+        for release in res.json():
+            yield release
+
+
+def validate_version_str(version_str, release_only=True):
+    """
+    Test if a string conforms to SemVer format (https://semver.org/)
+    :param version_str: a string to be validated
+    :param release_only: only declare releases (not prereleases) valid
+    :return: True if the version string is of a valid SemVer format else False
+    """
+    match = SEMVER_FORMAT.match(version_str)
+    has_match = match is not None
+    if has_match and release_only:
+        return not match.group("prerelease")
+    return has_match
+
+
+def split_version(version_string):
+    """
+    Split a SemVer-formatted string into its component integers
+    :param version_string: a SemVer string to be split
+    :return: an array of three integers
+    """
+    match = SEMVER_FORMAT.match(version_string)
+    return [int(match.group(group)) for group in ["major", "minor", "patch"]]
+
+
+def version_gt(left_version, right_version):
+    for left, right in zip(split_version(left_version), split_version(right_version)):
+        if left > right:
+            return True
+        elif right > left:
+            return False
+    return False

local_server/common/annotations/annotations.py

@@ -0,0 +1,144 @@
+from abc import ABCMeta, abstractmethod
+
+import fastobo
+import fsspec
+
+from local_server.common.errors import OntologyLoadFailure, DisabledFeatureError
+from local_server.common.utils.type_conversion_utils import get_schema_type_hint_of_array
+
+
+class Annotations(metaclass=ABCMeta):
+    """ baseclass for annotations, including ontologies and genesets"""
+
+    """ our default ontology is the PURL for the Cell Ontology.
+    See http://www.obofoundry.org/ontology/cl.html """
+    DefaultOnotology = "http://purl.obolibrary.org/obo/cl.obo"
+
+    def __init__(self, config={}):
+        self.ontology_data = None
+        self.config = config
+
+    def user_annotations_enabled(self):
+        return self.config.get("user-annotations", False)
+
+    def gene_sets_save_enabled(self):
+        return self.config.get("genesets-save", False)
+
+    def check_user_annotations_enabled(self):
+        if not self.user_annotations_enabled():
+            raise DisabledFeatureError("User annotations are disabled.")
+
+    def check_gene_sets_save_enabled(self):
+        if not self.gene_sets_save_enabled():
+            raise DisabledFeatureError("User genesets save is disabled.")
+
+    def load_ontology(self, path):
+        """Load and parse ontologies - currently support OBO files only."""
+        if path is None:
+            path = self.DefaultOnotology
+
+        try:
+            with fsspec.open(path) as f:
+                obo = fastobo.iter(f)
+                terms = filter(lambda stanza: type(stanza) is fastobo.term.TermFrame, obo)
+                names = [tag.name for term in terms for tag in term if type(tag) is fastobo.term.NameClause]
+                self.ontology_data = names
+
+        except FileNotFoundError as e:
+            raise OntologyLoadFailure("Unable to find OBO ontology path") from e
+
+        except SyntaxError as e:
+            raise OntologyLoadFailure("Syntax error loading OBO ontology") from e
+
+        except Exception as e:
+            raise OntologyLoadFailure("Error loading OBO file") from e
+
+    def get_schema(self, data_adaptor):
+        schema = []
+        labels = self.read_labels(data_adaptor)
+        if labels is not None and not labels.empty:
+            for col in labels.columns:
+                col_schema = dict(name=col, writable=True)
+                col_schema.update(get_schema_type_hint_of_array(labels[col]))
+                schema.append(col_schema)
+
+        return schema
+
+    @abstractmethod
+    def set_collection(self, name):
+        """set or create a new annotation collection"""
+        pass
+
+    @abstractmethod
+    def read_labels(self, data_adaptor):
+        """Return the labels as a pandas.DataFrame"""
+        pass
+
+    @abstractmethod
+    def write_labels(self, df, data_adaptor):
+        """Write the labels (df) to a persistent storage such that it can later be read"""
+        pass
+
+    @abstractmethod
+    def read_gene_sets(self, data_adaptor):
+        """Return the genesets from persistent storage """
+        pass
+
+    @abstractmethod
+    def write_gene_sets(self, gs, data_adaptor):
+        """Write the genesets (gs) to a persistent storage such that it can later be read"""
+        pass
+
+    @abstractmethod
+    def update_parameters(self, parameters, data_adaptor):
+        """Update configuration parameters that describe information about the annotations feature"""
+        pass
+
+    Genesets_Header = [
+        "gene_set_name",
+        "gene_set_description",
+        "gene_symbol",
+        "gene_description",
+    ]
+
+    @staticmethod
+    def gene_sets_to_csv(genesets):
+        """
+        Convert the internal genesets format (returned by read_gene_set) into
+        the simple Tidy CSV.
+        """
+        from io import StringIO
+        import csv
+
+        if type(genesets) == dict:
+            genesets = genesets.values()
+
+        with StringIO() as sio:
+            writer = csv.writer(sio, dialect='excel')
+            writer.writerow(Annotations.Genesets_Header)
+            for geneset in genesets:
+                # genes may be empty, in which case we skip the geneset entirely
+                genes = geneset["genes"]
+                if not genes:
+                    writer.writerow([geneset["geneset_name"], geneset.get("geneset_description", ""), "", ""])
+                else:
+                    writer.writerows(
+                        [
+                            [
+                                geneset["geneset_name"],
+                                geneset.get("geneset_description", ""),
+                                gene["gene_symbol"],
+                                gene.get("gene_description", ""),
+                            ]
+                            for gene in genes
+                        ]
+                    )
+            return sio.getvalue()
+
+    @staticmethod
+    def gene_sets_to_response(genesets):
+        """
+        Convert the internal genesets format (returned by read_gene_set) into
+        the dict expected by the JSON REST API
+        """
+        return list(genesets.values())

local_server/common/annotations/local_file_csv.py

@@ -0,0 +1,352 @@
+import base64
+import os
+import re
+import threading
+from datetime import datetime
+from hashlib import blake2b
+import csv
+
+import pandas as pd
+from flask import session, has_request_context, current_app
+
+from local_server import __version__ as cellxgene_version
+from local_server.common.annotations.annotations import Annotations
+from local_server.common.errors import AnnotationsError, ObsoleteRequest
+
+
+class AnnotationsLocalFile(Annotations):
+    CXG_ANNO_COLLECTION = "cxg_anno_collection"
+
+    def __init__(self, config, output_dir, label_output_file, gene_sets_output_file):
+        super().__init__(config)
+        self.output_dir = output_dir
+        self.label_output_file = label_output_file
+        self.gene_sets_output_file = gene_sets_output_file
+        # lock used to protect label file write ops
+        self.label_lock = threading.RLock()
+        self.gene_sets_lock = threading.RLock()
+
+        # cache the most recent annotations.
+        self.last_fname = None
+        self.last_labels = None
+
+        # txn ID - used to de-dup geneset writes
+        self.last_geneset_tid = 0
+
+    def is_safe_collection_name(self, name):
+        """
+        return true if this is a safe collection name
+        this is ultra conservative. If we want to allow full legal file name syntax,
+        we could look at modules like `pathvalidate`
+        """
+        if name is None:
+            return False
+        return re.match(r"^[\w\-]+$", name) is not None
+
+    def set_collection(self, name):
+        session[self.CXG_ANNO_COLLECTION] = name
+        session.permanent = True
+
+    def get_collection(self):
+        if session is None:
+            return None
+        return session.get(self.CXG_ANNO_COLLECTION)
+
+    def read_labels(self, data_adaptor):
+        self.check_user_annotations_enabled()  # raises
+
+        if has_request_context():
+            if not current_app.auth.is_user_authenticated():
+                return pd.DataFrame()
+
+        fname = self._get_celllabels_filename(data_adaptor)
+        with self.label_lock:
+            if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0:
+                # returned the cached labels if possible, otherwise read them from the file
+                if fname == self.last_fname:
+                    return self.last_labels
+                else:
+                    labels = pd.read_csv(
+                        fname, dtype="category", index_col=0, header=0, comment="#", keep_default_na=False
+                    )
+                    # update the cache
+                    self.last_fname = fname
+                    self.last_labels = labels
+                    return labels
+            else:
+                return pd.DataFrame()
+
+    def write_labels(self, df, data_adaptor):
+        self.check_user_annotations_enabled()  # raises
+
+        # update our internal state and save it.  Multi-threading often enabled,
+        # so treat this as a critical section.
+        with self.label_lock:
+            lastmod = data_adaptor.get_last_mod_time()
+            lastmodstr = "'unknown'" if lastmod is None else lastmod.isoformat(timespec="seconds")
+            header = (
+                f"# Annotations generated on {datetime.now().isoformat(timespec='seconds')} "
+                f"using cellxgene version {cellxgene_version}\n"
+                f"# Input data file was {data_adaptor.get_location()}, "
+                f"which was last modified on {lastmodstr}\n"
+            )
+
+            fname = self._get_celllabels_filename(data_adaptor)
+            self._backup(fname)
+            if not df.empty:
+                with open(fname, "w", newline="") as f:
+                    if header is not None:
+                        f.write(header)
+                    df.to_csv(f)
+            else:
+                open(fname, "w").close()
+
+            # update the cache
+            self.last_fname = fname
+            self.last_labels = df
+
+    def read_gene_sets(self, data_adaptor, context=None):
+        if has_request_context():
+            if not current_app.auth.is_user_authenticated():
+                return ({}, self.last_geneset_tid)
+
+        fname = self._get_genesets_filename(data_adaptor)
+        gene_sets = {}
+        tid = None
+        with self.gene_sets_lock:
+            tid = self.last_geneset_tid  # inside the critical section
+            if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0:
+                with open(fname, newline="") as f:
+                    gene_sets = read_gene_set_tidycsv(f, context)
+
+        return (gene_sets, tid)
+
+    def write_gene_sets(self, gene_sets, tid, data_adaptor):
+        self.check_gene_sets_save_enabled()  # raises
+
+        if type(tid) != int or tid < 0:
+            raise ValueError("tid must be a positive integer")
+
+        with self.gene_sets_lock:
+            # skip if the request is stale
+            if tid is not None:
+                if tid <= self.last_geneset_tid:
+                    raise ObsoleteRequest("TID is stale.")
+                self.last_geneset_tid = tid
+
+            lastmod = data_adaptor.get_last_mod_time()
+            lastmodstr = "'unknown'" if lastmod is None else lastmod.isoformat(timespec="seconds")
+            header = (
+                f"# Gene set generated on {datetime.now().isoformat(timespec='seconds')} "
+                f"using cellxgene version {cellxgene_version}\n"
+                f"# Input data file was {data_adaptor.get_location()}, "
+                f"which was last modified on {lastmodstr}\n"
+            )
+
+            fname = self._get_genesets_filename(data_adaptor)
+            self._backup(fname)
+            with open(fname, "w", newline="") as f:
+                f.write(header)
+                f.write(self.gene_sets_to_csv(gene_sets))
+
+    def _get_userdata_idhash(self, data_adaptor):
+        """
+        Return a short hash that weakly identifies the user and dataset.
+        Used to create safe annotations output file names.
+        """
+        uid = current_app.auth.get_user_id() or ""
+        id = (uid + data_adaptor.get_location()).encode()
+        idhash = base64.b32encode(blake2b(id, digest_size=5).digest()).decode("utf-8")
+        return idhash
+
+    def _get_output_dir(self):
+        if self.output_dir:
+            return self.output_dir
+
+        output_file = self.label_output_file or self.gene_sets_output_file
+        if output_file:
+            return os.path.dirname(os.path.abspath(output_file))
+
+        return os.getcwd()
+
+    def _get_celllabels_filename(self, data_adaptor):
+        """ return the current annotation file name """
+        if self.label_output_file:
+            return self.label_output_file
+
+        return self._get_filename(data_adaptor, "celllabels")
+
+    def _get_genesets_filename(self, data_adaptor):
+        """ return the current gene sets file name """
+        if self.gene_sets_output_file:
+            return self.gene_sets_output_file
+
+        return self._get_filename(data_adaptor, "genesets")
+
+    def _get_filename(self, data_adaptor, anno_name):
+        # we need to generate a file name, which we can only do if we have a UID and collection name
+        if session is None:
+            raise AnnotationsError("unable to determine file name for annotations")
+
+        collection = self.get_collection()
+        if collection is None:
+            return None
+
+        if data_adaptor is None:
+            raise AnnotationsError("unable to determine file name for annotations")
+
+        idhash = self._get_userdata_idhash(data_adaptor)
+        return os.path.join(self._get_output_dir(), f"{collection}-{anno_name}-{idhash}.csv")
+
+    def _backup(self, fname, max_backups=9):
+        """
+        save N backups of file to backup_dir.
+            1. fname -> backup_dir/fname-TIME
+            2. delete excess files in backup_dir
+        """
+        root, ext = os.path.splitext(fname)
+        backup_dir = f"{root}-backups"
+
+        # Make sure there is work to do
+        if not os.path.exists(fname):
+            return
+
+        # Ensure backup_dir exists
+        if not os.path.exists(backup_dir):
+            os.mkdir(backup_dir)
+
+        # Save current file to backup_dir
+        fname_base = os.path.basename(fname)
+        fname_base_root, fname_base_ext = os.path.splitext(fname_base)
+        # don't use ISO standard time format, as it contains characters illegal on some filesytems.
+        nowish = datetime.now().strftime("%Y-%m-%dT%H-%M-%S")
+        backup_fname = os.path.join(backup_dir, f"{fname_base_root}-{nowish}{fname_base_ext}")
+        if os.path.exists(backup_fname):
+            os.remove(backup_fname)
+        os.rename(fname, backup_fname)
+
+        # prune the backup_dir to max number of backup files, keeping the most recent backups
+        backups = list(filter(lambda s: s.startswith(fname_base_root), os.listdir(backup_dir)))
+        excess_count = len(backups) - max_backups
+        if excess_count > 0:
+            backups.sort()
+            for bu in backups[0:excess_count]:
+                os.remove(os.path.join(backup_dir, bu))
+
+    def update_parameters(self, parameters, data_adaptor):
+        params = {}
+        params["annotations"] = self.user_annotations_enabled()
+        params["annotations_genesets_readonly"] = not self.gene_sets_save_enabled()
+        params["user_annotation_collection_name_enabled"] = True
+
+        if self.ontology_data:
+            params["annotations_cell_ontology_enabled"] = True
+            params["annotations_cell_ontology_terms"] = self.ontology_data
+        else:
+            params["annotations_cell_ontology_enabled"] = False
+
+        if self.label_output_file is not None:
+            # user has hard-wired the name of the annotation cell label data collection
+            fname = os.path.basename(self.label_output_file)
+            collection_fname = os.path.splitext(fname)[0]
+            params["annotations-data-collection-is-read-only"] = True
+            params["annotations-data-collection-name"] = collection_fname
+
+        elif session is not None:
+            collection = self.get_collection()
+            params["annotations-data-collection-is-read-only"] = False
+            params["annotations-data-collection-name"] = collection
+
+        if current_app.auth.is_user_authenticated():
+            params["annotations-user-data-idhash"] = self._get_userdata_idhash(data_adaptor)
+
+        parameters.update(params)
+
+
+def read_gene_set_tidycsv(f, context=None):
+    """
+    Read & parse the Tidy CSV format, applying validation checks for mandatory
+    values, and de-duping rules.
+
+    Format is a four-column CSV, with a mandatory header row, and optional "#" prefixed
+    comments.  Format:
+
+        gene_set_name, gene_set_description, gene_symbol, gene_description
+
+    gene_set_name must be non-null; others are optional.
+
+    Returns: a dictionary of the shape (values in angle-brackets vary):
+
+        {
+            <string, a gene set name>: {
+                "geneset_name": <string, a gene set name>,
+                "geneset_description": <a string or None>,
+                "genes": [
+                    {
+                        "gene_symbol": <string, a gene symbol or name>,
+                        "gene_description": <a string or None>
+                    },
+                    ...
+                ]
+            },
+            ...
+        }
+    """
+
+    class myDialect(csv.excel):
+        skipinitialspace = True
+
+    def just(n, seq):
+        it = iter(seq)
+        for _ in range(n - 1):
+            yield next(it, "")
+        yield tuple(it)
+
+    messagefn = context["messagefn"] if context else (lambda x: None)
+
+    reader = csv.reader(f, dialect=myDialect())
+    gene_sets = {}
+    haveReadHeader = False
+    lineno = 0
+    for row in reader:
+        lineno += 1
+        # ignore empty rows
+        if len(row) == 0:
+            continue
+        # if row starts with '#' it is a comment
+        if row[0].startswith("#"):
+            continue
+        # if this is the first non-comment row, assume it is a header
+        if not haveReadHeader:
+            if row != Annotations.Genesets_Header:
+                raise AnnotationsError("Geneset CSV file missing the required column header.")
+            haveReadHeader = True
+            continue
+
+        geneset_name, geneset_description, gene_symbol, gene_description, _ = just(5, row)
+        if not geneset_name:
+            raise AnnotationsError(f"Geneset CSV missing required geneset or gene name on line {lineno}")
+        if (not gene_symbol) and gene_description:
+            messagefn(f"Warning: Missing gene name in geneset name {geneset_name} on line {lineno}.")
+
+        if geneset_name in gene_sets:
+            gs = gene_sets[geneset_name]
+        else:
+            gs = gene_sets[geneset_name] = {
+                "geneset_name": geneset_name,
+                "geneset_description": geneset_description,
+                "genes": [],
+            }
+        # Use first geneset_description with a value
+        if not gs["geneset_description"] and geneset_description:
+            gs["geneset_description"] = geneset_description
+        # add the gene if the gene_symbol is defined
+        if gene_symbol:
+            gs["genes"].append(
+                {
+                    "gene_symbol": gene_symbol,
+                    "gene_description": gene_description,
+                }
+            )
+
+    return gene_sets

local_server/common/aws_secret_utils.py

@@ -0,0 +1,23 @@
+import logging
+
+import boto3
+from flask import json
+
+from local_server.common.errors import SecretKeyRetrievalError
+
+
+def get_secret_key(region_name, secret_name):
+    session = boto3.session.Session()
+    client = session.client(service_name="secretsmanager", region_name=region_name)
+
+    try:
+        get_secret_value_response = client.get_secret_value(SecretId=secret_name)
+        if "SecretString" in get_secret_value_response:
+            var = get_secret_value_response["SecretString"]
+            secret = json.loads(var)
+            return secret
+    except Exception as e:
+        logging.critical(f"Caught exception during get_secret_key, {e}", exc_info=True)
+        raise SecretKeyRetrievalError(str(e))
+
+    return None

local_server/common/colors.py

@@ -0,0 +1,233 @@
+import re
+
+from local_server.common.errors import ColorFormatException
+
+HEX_COLOR_FORMAT = re.compile("^#[a-fA-F0-9]{6,6}$")
+
+# https://www.w3.org/TR/css-color-4/#named-colors
+CSS4_NAMED_COLORS = dict(
+    aliceblue="#f0f8ff",
+    antiquewhite="#faebd7",
+    aqua="#00ffff",
+    aquamarine="#7fffd4",
+    azure="#f0ffff",
+    beige="#f5f5dc",
+    bisque="#ffe4c4",
+    black="#000000",
+    blanchedalmond="#ffebcd",
+    blue="#0000ff",
+    blueviolet="#8a2be2",
+    brown="#a52a2a",
+    burlywood="#deb887",
+    cadetblue="#5f9ea0",
+    chartreuse="#7fff00",
+    chocolate="#d2691e",
+    coral="#ff7f50",
+    cornflowerblue="#6495ed",
+    cornsilk="#fff8dc",
+    crimson="#dc143c",
+    cyan="#00ffff",
+    darkblue="#00008b",
+    darkcyan="#008b8b",
+    darkgoldenrod="#b8860b",
+    darkgray="#a9a9a9",
+    darkgreen="#006400",
+    darkgrey="#a9a9a9",
+    darkkhaki="#bdb76b",
+    darkmagenta="#8b008b",
+    darkolivegreen="#556b2f",
+    darkorange="#ff8c00",
+    darkorchid="#9932cc",
+    darkred="#8b0000",
+    darksalmon="#e9967a",
+    darkseagreen="#8fbc8f",
+    darkslateblue="#483d8b",
+    darkslategray="#2f4f4f",
+    darkslategrey="#2f4f4f",
+    darkturquoise="#00ced1",
+    darkviolet="#9400d3",
+    deeppink="#ff1493",
+    deepskyblue="#00bfff",
+    dimgray="#696969",
+    dimgrey="#696969",
+    dodgerblue="#1e90ff",
+    firebrick="#b22222",
+    floralwhite="#fffaf0",
+    forestgreen="#228b22",
+    fuchsia="#ff00ff",
+    gainsboro="#dcdcdc",
+    ghostwhite="#f8f8ff",
+    gold="#ffd700",
+    goldenrod="#daa520",
+    gray="#808080",
+    green="#008000",
+    greenyellow="#adff2f",
+    grey="#808080",
+    honeydew="#f0fff0",
+    hotpink="#ff69b4",
+    indianred="#cd5c5c",
+    indigo="#4b0082",
+    ivory="#fffff0",
+    khaki="#f0e68c",
+    lavender="#e6e6fa",
+    lavenderblush="#fff0f5",
+    lawngreen="#7cfc00",
+    lemonchiffon="#fffacd",
+    lightblue="#add8e6",
+    lightcoral="#f08080",
+    lightcyan="#e0ffff",
+    lightgoldenrodyellow="#fafad2",
+    lightgray="#d3d3d3",
+    lightgreen="#90ee90",
+    lightgrey="#d3d3d3",
+    lightpink="#ffb6c1",
+    lightsalmon="#ffa07a",
+    lightseagreen="#20b2aa",
+    lightskyblue="#87cefa",
+    lightslategray="#778899",
+    lightslategrey="#778899",
+    lightsteelblue="#b0c4de",
+    lightyellow="#ffffe0",
+    lime="#00ff00",
+    limegreen="#32cd32",
+    linen="#faf0e6",
+    magenta="#ff00ff",
+    maroon="#800000",
+    mediumaquamarine="#66cdaa",
+    mediumblue="#0000cd",
+    mediumorchid="#ba55d3",
+    mediumpurple="#9370db",
+    mediumseagreen="#3cb371",
+    mediumslateblue="#7b68ee",
+    mediumspringgreen="#00fa9a",
+    mediumturquoise="#48d1cc",
+    mediumvioletred="#c71585",
+    midnightblue="#191970",
+    mintcream="#f5fffa",
+    mistyrose="#ffe4e1",
+    moccasin="#ffe4b5",
+    navajowhite="#ffdead",
+    navy="#000080",
+    oldlace="#fdf5e6",
+    olive="#808000",
+    olivedrab="#6b8e23",
+    orange="#ffa500",
+    orangered="#ff4500",
+    orchid="#da70d6",
+    palegoldenrod="#eee8aa",
+    palegreen="#98fb98",
+    paleturquoise="#afeeee",
+    palevioletred="#db7093",
+    papayawhip="#ffefd5",
+    peachpuff="#ffdab9",
+    peru="#cd853f",
+    pink="#ffc0cb",
+    plum="#dda0dd",
+    powderblue="#b0e0e6",
+    purple="#800080",
+    rebeccapurple="#663399",
+    red="#ff0000",
+    rosybrown="#bc8f8f",
+    royalblue="#4169e1",
+    saddlebrown="#8b4513",
+    salmon="#fa8072",
+    sandybrown="#f4a460",
+    seagreen="#2e8b57",
+    seashell="#fff5ee",
+    sienna="#a0522d",
+    silver="#c0c0c0",
+    skyblue="#87ceeb",
+    slateblue="#6a5acd",
+    slategray="#708090",
+    slategrey="#708090",
+    snow="#fffafa",
+    springgreen="#00ff7f",
+    steelblue="#4682b4",
+    tan="#d2b48c",
+    teal="#008080",
+    thistle="#d8bfd8",
+    tomato="#ff6347",
+    turquoise="#40e0d0",
+    violet="#ee82ee",
+    wheat="#f5deb3",
+    white="#ffffff",
+    whitesmoke="#f5f5f5",
+    yellow="#ffff00",
+    yellowgreen="#9acd32",
+)
+
+
+def convert_color_to_hex_format(unknown):
+    """
+    Try to convert color info to a hex triplet string https://en.wikipedia.org/wiki/Web_colors#Hex_triplet.
+
+    The function accepts for the following formats:
+    - A CSS4 color name, as supported by matplotlib https://matplotlib.org/3.1.0/gallery/color/named_colors.html
+    - RGB tuple/list with values ranging from 0.0 to 1.0, as in [0.5, 0.75, 1.0]
+    - RFB tuple/list with values ranging from 0 to 255, as in [128, 192, 255]
+    - Hex triplet string, as in "#08c0ff"
+
+    :param unknown: color info of unknown format
+    :return: a hex triplet representing that color
+    """
+    try:
+        if type(unknown) in (list, tuple) and len(unknown) == 3:
+            if all(0.0 <= ele <= 1.0 for ele in unknown):
+                tup = tuple(int(ele * 255) for ele in unknown)
+            elif all(0 <= ele <= 255 and isinstance(ele, int) for ele in unknown):
+                tup = tuple(unknown)
+            else:
+                raise ColorFormatException("Unknown color iterable format!")
+            return "#%02x%02x%02x" % tup
+        elif isinstance(unknown, str) and unknown.lower() in CSS4_NAMED_COLORS:
+            return CSS4_NAMED_COLORS[unknown.lower()]
+        elif isinstance(unknown, str) and HEX_COLOR_FORMAT.match(unknown):
+            return unknown.lower()
+        else:
+            raise ColorFormatException("Unknown color format type!")
+    except Exception as e:
+        raise ColorFormatException(e)
+
+
+def convert_anndata_category_colors_to_cxg_category_colors(data):
+    """
+    Convert color information from anndata files to the cellxgene color data format as described below:
+        {
+            "<category_name>": {
+                "<label_name>": "<color_hex_code>",
+                ...
+            },
+            ...
+        }
+
+    For more on the cxg color data structure, see https://github.com/chanzuckerberg/cellxgene/issues/1307.
+
+    For more on the anndata color data structure, see
+    https://github.com/chanzuckerberg/cellxgene/issues/1152#issuecomment-587276178.
+
+    Handling of malformed data:
+    - For any color info in a adata.uns[f"{category}_colors"] color array that convert_color_to_hex_format cannot
+      convert to a hex triplet string, a ColorFormatException is raised
+    - No category_name key group is returned for adata.uns[f"{category}_colors"] keys for which there is no
+      adata.obs[f"{category}"] key
+
+    :param data: the anndata file
+    :return: cellxgene color data structure as described above
+    """
+    cxg_colors = dict()
+    color_key_suffix = "_colors"
+    for uns_key in data.uns.keys():
+        # find uns array that describes colors for a category
+        if not uns_key.endswith(color_key_suffix):
+            continue
+
+        # check to see if we actually have observations for that category
+        category_name = uns_key[: -len(color_key_suffix)]
+        if category_name not in data.obs.keys():
+            continue
+
+        # create the cellxgene color entry for this category
+        cxg_colors[category_name] = dict(
+            zip(data.obs[category_name].cat.categories, [convert_color_to_hex_format(c) for c in data.uns[uns_key]])
+        )
+    return cxg_colors

local_server/common/config/__init__.py

@@ -0,0 +1,4 @@
+from local_server.common.aws_secret_utils import get_secret_key  # noqa F504
+
+DEFAULT_SERVER_PORT = 5005
+BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20  # 100MB

local_server/common/config/app_config.py

@@ -0,0 +1,171 @@
+import yaml
+from flatten_dict import unflatten
+
+from local_server.default_config import get_default_config
+from local_server.common.config.dataset_config import DatasetConfig
+from local_server.common.config.server_config import ServerConfig
+from local_server.common.config.external_config import ExternalConfig
+from local_server.common.errors import ConfigurationError
+
+
+class AppConfig(object):
+    """
+    AppConfig stores all the configuration for cellxgene.
+    AppConfig contains one or more DatasetConfig(s) and one ServerConfig.
+    The server_config contains attributes that refer to the server process as a whole.
+    The dataset_config refers to attributes that are associated with the features and
+    presentations of a dataset.
+    AppConfig has methods to initialize, modify, and access the configuration.
+    """
+
+    def __init__(self):
+
+        # the default configuration (see default_config.py)
+        # TODO @madison -- if we always read from the default config (hard coded path) can we set those values as
+        #  defaults within the config class?
+        self.default_config = get_default_config()
+        # the server configuration
+        self.server_config = ServerConfig(self, self.default_config["server"])
+        # the dataset config
+        self.dataset_config = DatasetConfig(None, self, self.default_config["dataset"])
+        #  external config
+        self.external_config = ExternalConfig(self, self.default_config["external"])
+
+        # Set to true when config_completed is called
+        self.is_completed = False
+
+    def get_dataset_config(self):
+        return self.dataset_config
+
+    def check_config(self):
+        """Verify all the attributes in the config have been type checked"""
+        if not self.is_completed:
+            raise ConfigurationError("The configuration has not been completed")
+        self.server_config.check_config()
+        self.dataset_config.check_config()
+        self.external_config.check_config()
+
+    def update_server_config(self, **kw):
+        self.server_config.update(**kw)
+        self.is_complete = False
+
+    def update_dataset_config(self, **kw):
+        self.dataset_config.update(**kw)
+        self.is_complete = False
+
+    def update_single_config_from_path_and_value(self, path, value):
+        """Update a single config parameter with the value.
+        Path is a list of string, that gives a path to the config parameter to be updated.
+        For example, path may be ["server","app","port"].
+        """
+        self.is_complete = False
+        if not isinstance(path, list):
+            raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'")
+        for part in path:
+            if not isinstance(part, str):
+                raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'")
+
+        if len(path) < 1 or path[0] not in ("server", "dataset"):
+            raise ConfigurationError("path must start with 'server', or 'dataset'")
+
+        if path[0] == "server":
+            attr = "__".join(path[1:])
+            try:
+                self.update_server_config(**{attr: value})
+            except ConfigurationError:
+                raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
+        elif path[0] == "dataset":
+            attr = "__".join(path[1:])
+            try:
+                self.update_dataset_config(**{attr: value})
+            except ConfigurationError:
+                raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
+
+    def update_from_config_file(self, config_file):
+        try:
+            with open(config_file) as yml_file:
+                config = yaml.safe_load(yml_file)
+        except yaml.YAMLError as e:
+            raise ConfigurationError(f"The specified config file contained an error: {e}")
+        except OSError as e:
+            raise ConfigurationError(f"Issue retrieving the specified config file: {e}")
+
+        if config.get("server"):
+            self.server_config.update_from_config(config["server"], "server")
+        if config.get("dataset"):
+            self.dataset_config.update_from_config(config["dataset"], "dataset")
+
+        if config.get("external"):
+            self.external_config.update_from_config(config["external"], "external")
+
+        self.is_complete = False
+
+    def config_to_dict(self):
+        """return the configuration as an unflattened dict"""
+        server = self.server_config.create_mapping(self.server_config.default_config)
+        dataset = self.dataset_config.create_mapping(self.dataset_config.default_config)
+        external = self.external_config.create_mapping(self.external_config.default_config)
+        config = dict(server={}, dataset={})
+        for attrname in server.keys():
+            config["server__" + attrname] = getattr(self.server_config, attrname)
+        for attrname in dataset.keys():
+            config["dataset__" + attrname] = getattr(self.dataset_config, attrname)
+        for attrname in external.keys():
+            config["external__" + attrname] = getattr(self.external_config, attrname)
+
+        config = unflatten(config, splitter=lambda key: key.split("__"))
+        return config
+
+    def write_config(self, config_file):
+        """output the config to a yaml file"""
+        config = self.config_to_dict()
+        yaml.dump(config, open(config_file, "w"))
+
+    def changes_from_default(self):
+        """Return all the attribute that are different from the default"""
+        diff_server = self.server_config.changes_from_default()
+        diff_dataset = self.dataset_config.changes_from_default()
+        diff_external = self.external.changes_from_default()
+        diff = dict(server=diff_server, dataset=diff_dataset, external=diff_external)
+        return diff
+
+    def complete_config(self, messagefn=None):
+        """The configure options are checked, and any additional setup based on the config
+        parameters is done"""
+
+        if messagefn is None:
+
+            def noop(message):
+                pass
+
+            messagefn = noop
+
+        # TODO: to give better error messages we can add a mapping between where each config
+        # attribute originated (e.g. command line argument or config file), then in the error
+        # messages we can give correct context for attributes with bad value.
+        context = dict(messagefn=messagefn)
+
+        # complete config for external_config first, since this may update values in the other sections
+        self.external_config.complete_config(context)
+        self.server_config.complete_config(context)
+        self.dataset_config.complete_config(context)
+
+        self.is_completed = True
+        self.check_config()
+
+    def get_matrix_data_cache_manager(self):
+        return self.server_config.matrix_data_cache_manager
+
+    def get_title(self, data_adaptor):
+        return (
+            self.server_config.single_dataset__title
+            if self.server_config.single_dataset__title
+            else data_adaptor.get_title()
+        )
+
+    def get_about(self, data_adaptor):
+        return (
+            self.server_config.single_dataset__about
+            if self.server_config.single_dataset__about
+            else data_adaptor.get_about()
+        )

local_server/common/config/base_config.py

@@ -0,0 +1,99 @@
+import copy
+
+from flatten_dict import flatten
+from local_server.common.errors import ConfigurationError
+
+
+class BaseConfig(object):
+    """
+    This class handles the mechanics of updating and checking attributes.
+    Derived classes are expected to store the actual attributes
+    Currently DatasetConfig and ServerConfig both inherit from BaseConfig.
+    """
+
+    def __init__(self, app_config, default_config):
+        # reference back to the app_config
+        self.app_config = app_config
+        # the complete set of attributes and their default values (unflattened)
+        self.default_config = default_config
+        # used to make sure every attribute value is checked
+        self.attr_checked = {key_name: False for key_name in self.create_mapping(default_config).keys()}
+
+    def create_mapping(self, config):
+        """
+        Create a dictionary where the keys are the name of attributes (using double underscore convention)
+        For example: authentication__type
+
+        The values are a tuple,
+        - the first item of the tuple is a tuple of path elements (location in config 'tree')
+        - the second item is the value of the config parameter
+
+        For example: (('authentication', 'type'), 'session'))
+        """
+        config_copy = copy.deepcopy(config)
+        mapping = {}
+
+        flat_config = flatten(config_copy)
+        for key, value in flat_config.items():
+            # name of the attribute
+            attr = "__".join(key)
+            mapping[attr] = (key, value)
+
+        return mapping
+
+    def validate_correct_type_of_configuration_attribute(self, attrname, vtype):
+        val = getattr(self, attrname)
+        if type(vtype) in (list, tuple):
+            if type(val) not in vtype:
+                tnames = ",".join([x.__name__ for x in vtype])
+                raise ConfigurationError(
+                    f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}"
+                )
+        else:
+            if type(val) != vtype:
+                raise ConfigurationError(
+                    f"Invalid type for attribute: {attrname}, "
+                    f"expected type {vtype.__name__}, got {type(val).__name__}"
+                )
+
+        self.attr_checked[attrname] = True
+
+    def check_config(self):
+        mapping = self.create_mapping(self.default_config)
+        for key in mapping.keys():
+            if not self.attr_checked[key]:
+                raise ConfigurationError(f"The attr '{key}' has not been checked")
+
+    def update(self, **kw):
+        """Update the attributes defined in kw with their new values."""
+        for key, value in kw.items():
+            if not hasattr(self, key):
+                raise ConfigurationError(f"unknown config parameter {key}.")
+            try:
+                if type(value) == tuple:
+                    # convert tuple values to list values
+                    value = list(value)
+                setattr(self, key, value)
+            except KeyError:
+                raise ConfigurationError(f"Unable to set config parameter {key}.")
+
+            self.attr_checked[key] = False
+
+    def update_from_config(self, config, prefix):
+        mapping = self.create_mapping(config)
+        for attr, (key, value) in mapping.items():
+            if not hasattr(self, attr):
+                raise ConfigurationError(f"Unknown key from config file: {prefix}__{attr}")
+            setattr(self, attr, value)
+
+            self.attr_checked[attr] = False
+
+    def changes_from_default(self):
+        """Return all the attribute that are different from the default"""
+        mapping = self.create_mapping(self.default_config)
+        diff = []
+        for attrname, (key, defval) in mapping.items():
+            curval = getattr(self, attrname)
+            if curval != defval:
+                diff.append((attrname, curval, defval))
+        return diff

local_server/common/config/client_config.py

@@ -0,0 +1,123 @@
+from local_server import display_version as cellxgene_display_version
+
+
+def get_client_config(app_config, data_adaptor):
+    """
+    Return the configuration as required by the /config REST route
+    """
+
+    server_config = app_config.server_config
+    dataset_config = data_adaptor.dataset_config
+    annotation = dataset_config.user_annotations
+    auth = server_config.auth
+
+    # FIXME The current set of config is not consistently presented:
+    # we have camalCase, hyphen-text, and underscore_text
+
+    # make sure the configuration has been checked.
+    app_config.check_config()
+
+    # display_names
+    title = app_config.get_title(data_adaptor)
+    about = app_config.get_about(data_adaptor)
+
+    display_names = dict(engine=data_adaptor.get_name(), dataset=title)
+
+    # library_versions
+    library_versions = {}
+    library_versions.update(data_adaptor.get_library_versions())
+    library_versions["cellxgene"] = cellxgene_display_version
+
+    # links
+    links = {"about-dataset": about}
+
+    # parameters
+    parameters = {
+        "layout": dataset_config.embeddings__names,
+        "max-category-items": dataset_config.presentation__max_categories,
+        "obs_names": server_config.single_dataset__obs_names,
+        "var_names": server_config.single_dataset__var_names,
+        "diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff,
+        "backed": server_config.adaptor__anndata_adaptor__backed,
+        "disable-diffexp": not dataset_config.diffexp__enable,
+        "enable-reembedding": dataset_config.embeddings__enable_reembedding,
+        "annotations": False,
+        "annotations_file": None,
+        "annotations_dir": None,
+        "annotations_genesets": True,  # feature flag
+        "annotations_genesets_readonly": dataset_config.user_annotations__gene_sets__readonly,
+        "annotations_genesets_summary_methods": ["mean"],
+        "annotations_cell_ontology_enabled": False,
+        "annotations_cell_ontology_obopath": None,
+        "annotations_cell_ontology_terms": None,
+        "custom_colors": dataset_config.presentation__custom_colors,
+        "diffexp-may-be-slow": False,
+    }
+
+    # corpora dataset_props
+    # TODO/Note: putting info from the dataset into the /config is not ideal.
+    # However, it is definitely not part of /schema, and we do not have a top-level
+    # route for data properties.  Consider creating one at some point.
+    corpora_props = data_adaptor.get_corpora_props()
+    if corpora_props and "default_embedding" in corpora_props:
+        default_embedding = corpora_props["default_embedding"]
+        if isinstance(default_embedding, str) and default_embedding.startswith("X_"):
+            default_embedding = default_embedding[2:]  # drop X_ prefix
+        if default_embedding in data_adaptor.get_embedding_names():
+            parameters["default_embedding"] = default_embedding
+
+    data_adaptor.update_parameters(parameters)
+    if annotation:
+        annotation.update_parameters(parameters, data_adaptor)
+
+    # gather it all together
+    client_config = {}
+    config = client_config["config"] = {}
+    config["displayNames"] = display_names
+    config["library_versions"] = library_versions
+    config["links"] = links
+    config["parameters"] = parameters
+    config["corpora_props"] = corpora_props
+    config["limits"] = {
+        "column_request_max": server_config.limits__column_request_max,
+        "diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max,
+    }
+
+    if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
+        config["authentication"] = {
+            "requires_client_login": auth.requires_client_login(),
+        }
+        if auth.requires_client_login():
+            config["authentication"].update(
+                {
+                    # Todo why are these stored on the data_adaptor?
+                    "login": auth.get_login_url(data_adaptor),
+                    "logout": auth.get_logout_url(data_adaptor),
+                }
+            )
+
+    return client_config
+
+
+def get_client_userinfo(app_config, data_adaptor):
+    """
+    Return the userinfo as required by the /userinfo REST route
+    """
+
+    server_config = app_config.server_config
+    dataset_config = data_adaptor.dataset_config
+    auth = server_config.auth
+
+    # make sure the configuration has been checked.
+    app_config.check_config()
+
+    if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
+        userinfo = {}
+        userinfo["userinfo"] = {
+            "is_authenticated": auth.is_user_authenticated(),
+            "username": auth.get_user_name(),
+            "user_id": auth.get_user_id(),
+            "email": auth.get_user_email(),
+            "picture": auth.get_user_picture(),
+        }
+        return userinfo

local_server/common/config/dataset_config.py

@@ -0,0 +1,222 @@
+import os
+from os.path import splitext, isdir
+
+from local_server.common.annotations.local_file_csv import AnnotationsLocalFile
+from local_server.common.config.base_config import BaseConfig
+from local_server.common.errors import ConfigurationError, OntologyLoadFailure, AnnotationsError
+from local_server.compute.scanpy import get_scanpy_module
+from local_server.data_common.matrix_loader import MatrixDataLoader
+
+
+class DatasetConfig(BaseConfig):
+    """Manages the config attribute associated with a dataset."""
+
+    def __init__(self, tag, app_config, default_config):
+        super().__init__(app_config, default_config)
+        self.tag = tag
+        try:
+            self.app__scripts = default_config["app"]["scripts"]
+            self.app__inline_scripts = default_config["app"]["inline_scripts"]
+            self.app__authentication_enable = default_config["app"]["authentication_enable"]
+
+            self.presentation__max_categories = default_config["presentation"]["max_categories"]
+            self.presentation__custom_colors = default_config["presentation"]["custom_colors"]
+
+            self.user_annotations__enable = default_config["user_annotations"]["enable"]
+            self.user_annotations__type = default_config["user_annotations"]["type"]
+            self.user_annotations__local_file_csv__directory = default_config["user_annotations"]["local_file_csv"][
+                "directory"
+            ]
+            self.user_annotations__local_file_csv__file = default_config["user_annotations"]["local_file_csv"]["file"]
+            self.user_annotations__ontology__enable = default_config["user_annotations"]["ontology"]["enable"]
+            self.user_annotations__ontology__obo_location = default_config["user_annotations"]["ontology"][
+                "obo_location"
+            ]
+            self.user_annotations__gene_sets__readonly = default_config["user_annotations"]["gene_sets"]["readonly"]
+            self.user_annotations__local_file_csv__gene_sets_file = default_config["user_annotations"]["local_file_csv"][
+                "gene_sets_file"
+            ]
+
+            self.embeddings__names = default_config["embeddings"]["names"]
+            self.embeddings__enable_reembedding = default_config["embeddings"]["enable_reembedding"]
+
+            self.diffexp__enable = default_config["diffexp"]["enable"]
+            self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
+            self.diffexp__top_n = default_config["diffexp"]["top_n"]
+
+        except KeyError as e:
+            raise ConfigurationError(f"Unexpected config: {str(e)}")
+
+        # The annotation object is created during complete_config and stored here.
+        self.user_annotations = None
+
+    def complete_config(self, context):
+        self.handle_app()
+        self.handle_presentation()
+        self.handle_user_annotations(context)
+        self.handle_embeddings()
+        self.handle_diffexp(context)
+
+    def get_data_adaptor(self):
+        server_config = self.app_config.server_config
+        if not server_config.data_adaptor:
+            matrix_data_loader = MatrixDataLoader(server_config.single_dataset__datapath, app_config=self.app_config)
+            server_config.data_adaptor = matrix_data_loader.open(self.app_config)
+
+        return server_config.data_adaptor
+
+    def handle_app(self):
+        self.validate_correct_type_of_configuration_attribute("app__scripts", list)
+        self.validate_correct_type_of_configuration_attribute("app__inline_scripts", list)
+        self.validate_correct_type_of_configuration_attribute("app__authentication_enable", bool)
+
+        # scripts can be string (filename) or dict (attributes). Convert string to dict.
+        scripts = []
+        for script in self.app__scripts:
+            try:
+                if isinstance(script, str):
+                    scripts.append({"src": script})
+                elif isinstance(script, dict) and isinstance(script["src"], str):
+                    scripts.append(script)
+                else:
+                    raise Exception
+            except Exception as e:
+                raise ConfigurationError(f"Scripts must be string or a dict containing an src key: {e}")
+
+        self.app__scripts = scripts
+
+    def handle_presentation(self):
+        self.validate_correct_type_of_configuration_attribute("presentation__max_categories", int)
+        self.validate_correct_type_of_configuration_attribute("presentation__custom_colors", bool)
+
+    def handle_user_annotations(self, context):
+        self.validate_correct_type_of_configuration_attribute("user_annotations__enable", bool)
+        self.validate_correct_type_of_configuration_attribute("user_annotations__type", str)
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__local_file_csv__directory", (type(None), str)
+        )
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__local_file_csv__file", (type(None), str)
+        )
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__local_file_csv__gene_sets_file", (type(None), str)
+        )
+        self.validate_correct_type_of_configuration_attribute("user_annotations__ontology__enable", bool)
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__ontology__obo_location", (type(None), str)
+        )
+        self.validate_correct_type_of_configuration_attribute("user_annotations__gene_sets__readonly", bool)
+
+        if self.user_annotations__enable or not self.user_annotations__gene_sets__readonly:
+            server_config = self.app_config.server_config
+            if not self.app__authentication_enable:
+                raise ConfigurationError("user annotations requires authentication to be enabled")
+            if not server_config.auth.is_valid_authentication_type():
+                auth_type = server_config.authentication__type
+                raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations")
+
+        # Must always have an annotations instance to support genesets. User annotation (cell labels) are optional
+        # as are writable gene sets
+        if self.user_annotations__type == "local_file_csv":
+            self.handle_local_file_csv_annotations(context)
+        else:
+            raise ConfigurationError('The only annotation type support is "local_file_csv"')
+
+        if self.user_annotations__enable:
+            if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location:
+                try:
+                    self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location)
+                except OntologyLoadFailure as e:
+                    raise ConfigurationError("Unable to load ontology terms\n" + str(e))
+
+        self.check_annotation_config_vars_not_set(context)
+
+    def handle_local_file_csv_annotations(self, context):
+        dirname = self.user_annotations__local_file_csv__directory
+        filename = self.user_annotations__local_file_csv__file
+        genesets_filename = self.user_annotations__local_file_csv__gene_sets_file
+
+        if dirname is not None and (filename is not None or genesets_filename is not None):
+            raise ConfigurationError(
+                "'user-generated-data-dir' may not be used with annotations-file' or 'genesets-file'."
+            )
+
+        if filename is not None:
+            lf_name, lf_ext = splitext(filename)
+            if lf_ext and lf_ext != ".csv":
+                raise ConfigurationError(f"annotation file type must be .csv: {filename}")
+
+        if genesets_filename is not None:
+            lf_name, lf_ext = splitext(genesets_filename)
+            if lf_ext and lf_ext != ".csv":
+                raise ConfigurationError(f"genesets file type must be .csv: {genesets_filename}")
+
+        if dirname is not None and not isdir(dirname):
+            try:
+                os.mkdir(dirname)
+            except OSError:
+                raise ConfigurationError("Unable to create directory specified by --user-generated-data-dir")
+
+        anno_config = {
+            "user-annotations": self.user_annotations__enable,
+            "genesets-save": not self.user_annotations__gene_sets__readonly,
+        }
+        self.user_annotations = AnnotationsLocalFile(anno_config, dirname, filename, genesets_filename)
+
+        # if the user has specified a fixed label file, go ahead and validate it
+        # so that we can remove errors early in the process.
+        server_config = self.app_config.server_config
+        if server_config.single_dataset__datapath:
+            data_adaptor = self.get_data_adaptor()
+            if self.user_annotations__local_file_csv__file:
+                data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor))
+            if self.user_annotations__local_file_csv__gene_sets_file:
+                try:
+                    data_adaptor.check_new_gene_sets(self.user_annotations.read_gene_sets(data_adaptor, context), context)
+                except (ValueError, AnnotationsError, KeyError) as e:
+                    raise ConfigurationError(f"Unable to read genesets CSV file: {str(e)}") from e
+
+    def check_annotation_config_vars_not_set(self, context):
+        if self.user_annotations__type is not None:
+            dirname = self.user_annotations__local_file_csv__directory
+            filename = self.user_annotations__local_file_csv__file
+            if not self.user_annotations__enable:
+                if filename is not None:
+                    context["messagefn"]("Warning: --annotations-file ignored as annotations are disabled.")
+                if self.user_annotations__ontology__enable:
+                    context["messagefn"](
+                        "Warning: --experimental-annotations-ontology ignored as annotations are disabled."
+                    )
+                if self.user_annotations__ontology__obo_location is not None:
+                    context["messagefn"](
+                        "Warning: --experimental-annotations-ontology-obo ignored as annotations are disabled."
+                    )
+                if dirname is not None:
+                    context["messagefn"]("Warning: --user-generated-data-dir ignored as annotations are disabled.")
+
+    def handle_embeddings(self):
+        self.validate_correct_type_of_configuration_attribute("embeddings__names", list)
+        self.validate_correct_type_of_configuration_attribute("embeddings__enable_reembedding", bool)
+
+        server_config = self.app_config.server_config
+        if self.embeddings__enable_reembedding:
+            if server_config.single_dataset__datapath:
+                if server_config.adaptor__anndata_adaptor__backed:
+                    raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.")
+
+            try:
+                get_scanpy_module()
+            except NotImplementedError:
+                # Todo add scanpy to requirements.txt and remove this check once re-embeddings is fully supported
+                raise ConfigurationError("Please install scanpy to enable UMAP re-embedding")
+
+    def handle_diffexp(self, context):
+        self.validate_correct_type_of_configuration_attribute("diffexp__enable", bool)
+        self.validate_correct_type_of_configuration_attribute("diffexp__lfc_cutoff", float)
+        self.validate_correct_type_of_configuration_attribute("diffexp__top_n", int)
+
+        data_adaptor = self.get_data_adaptor()
+        if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False):
+            context["messagefn"](
+                "CAUTION: due to the size of your dataset, " "running differential expression may take longer or fail."
+            )

local_server/common/config/external_config.py

@@ -0,0 +1,96 @@
+import os
+
+from local_server.common.config.base_config import BaseConfig
+from local_server.common.errors import ConfigurationError
+from local_server.common.config import get_secret_key
+from local_server.common.errors import SecretKeyRetrievalError
+from local_server.common.utils.type_conversion_utils import convert_string_to_value
+
+
+class ExternalConfig(BaseConfig):
+    """Manages the config attribute associated with external configuration sources, such as
+    environment variables or the AWS Secrets Manager."""
+
+    def __init__(self, app_config, default_config):
+        super().__init__(app_config, default_config)
+        try:
+            self.environment = default_config["environment"]
+            self.aws_secrets_manager__region = default_config["aws_secrets_manager"]["region"]
+            self.aws_secrets_manager__secrets = default_config["aws_secrets_manager"]["secrets"]
+
+        except KeyError as e:
+            raise ConfigurationError(f"Unexpected config: {str(e)}")
+
+    def complete_config(self, context):
+        self.handle_environment(context)
+        self.handle_aws_secrets_manager(context)
+
+    def handle_environment(self, context):
+        """For each environment variable defined, get the value (if it is set),
+        and set the specified config parameter"""
+        self.validate_correct_type_of_configuration_attribute("environment", list)
+        for envdict in self.environment:
+            name = envdict.get("name")
+            if name is None:
+                raise ConfigurationError("environment: 'name' is missing")
+            required = envdict.get("required", False)
+            if type(required) != bool:
+                raise ConfigurationError("environment: 'required' must be a bool")
+            path = envdict.get("path")
+            if path is None:
+                raise ConfigurationError("environment: 'path' is missing")
+
+            value = os.environ.get(name)
+            if value is None:
+                if required:
+                    raise ConfigurationError(f"required environment variable '{name}' not set")
+            else:
+                value = convert_string_to_value(value)
+                self.app_config.update_single_config_from_path_and_value(path, value)
+
+    def handle_aws_secrets_manager(self, context):
+        """For each aws secret defined, get the key/values, and set the specified config parameter"""
+        self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", (type(None), str))
+        self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__secrets", list)
+
+        if not self.aws_secrets_manager__secrets:
+            return
+
+        self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", str)
+
+        for secret in self.aws_secrets_manager__secrets:
+            secret_name = secret.get("name")
+            if secret_name is None:
+                raise ConfigurationError("aws_secrets_manager: 'name' is missing")
+            if not isinstance(secret_name, str):
+                raise ConfigurationError("aws_secrets_manager: 'name' must be a string")
+
+            try:
+                secret_dict = get_secret_key(self.aws_secrets_manager__region, secret_name)
+            except SecretKeyRetrievalError as e:
+                raise ConfigurationError(f"Unable to retrieve secret {secret_name}: {str(e)}")
+
+            values = secret.get("values")
+            if values is None:
+                raise ConfigurationError("aws_secrets_manager: 'values' is missing")
+            if not isinstance(values, list):
+                raise ConfigurationError("aws_secrets_manager: 'values' must be a list")
+
+            for value in values:
+                key = value.get("key")
+                if key is None:
+                    raise ConfigurationError(f"missing 'key' in secret values: {secret_name}")
+                path = value.get("path")
+                if path is None:
+                    raise ConfigurationError(f"missing 'path' in secret values: {secret_name}")
+                required = value.get("required", False)
+                if type(required) != bool:
+                    raise ConfigurationError(f"wrong type for 'required' in secret values: {secret_name}")
+
+                secret_value = secret_dict.get(key)
+                if secret_value is None:
+                    if required:
+                        raise ConfigurationError(f"required secret '{secret_name}:{key}' not set")
+                else:
+                    secret_value = convert_string_to_value(secret_value)
+                    self.app_config.update_single_config_from_path_and_value(path, secret_value)

local_server/common/config/server_config.py

@@ -0,0 +1,183 @@
+import os
+import sys
+import warnings
+from os.path import basename
+from urllib.parse import urlparse
+
+from local_server.auth.auth import AuthTypeFactory
+from local_server.common.config.base_config import BaseConfig
+from local_server.common.config import DEFAULT_SERVER_PORT, BIG_FILE_SIZE_THRESHOLD
+from local_server.common.errors import ConfigurationError, DatasetAccessError
+from local_server.common.data_locator import discover_s3_region_name
+from local_server.common.utils.utils import is_port_available, find_available_port, custom_format_warning
+from local_server.data_common.matrix_loader import MatrixDataLoader
+
+
+class ServerConfig(BaseConfig):
+    """Manages the config attribute associated with the server."""
+
+    def __init__(self, app_config, default_config):
+        super().__init__(app_config, default_config)
+
+        try:
+            self.app__verbose = default_config["app"]["verbose"]
+            self.app__debug = default_config["app"]["debug"]
+            self.app__host = default_config["app"]["host"]
+            self.app__port = default_config["app"]["port"]
+            self.app__open_browser = default_config["app"]["open_browser"]
+            self.app__force_https = default_config["app"]["force_https"]
+            self.app__flask_secret_key = default_config["app"]["flask_secret_key"]
+
+            self.authentication__type = default_config["authentication"]["type"]
+            self.authentication__insecure_test_environment = default_config["authentication"][
+                "insecure_test_environment"
+            ]
+
+            self.single_dataset__datapath = default_config["single_dataset"]["datapath"]
+            self.single_dataset__obs_names = default_config["single_dataset"]["obs_names"]
+            self.single_dataset__var_names = default_config["single_dataset"]["var_names"]
+            self.single_dataset__about = default_config["single_dataset"]["about"]
+            self.single_dataset__title = default_config["single_dataset"]["title"]
+
+            self.data_locator__s3__region_name = default_config["data_locator"]["s3"]["region_name"]
+
+            self.adaptor__anndata_adaptor__backed = default_config["adaptor"]["anndata_adaptor"]["backed"]
+
+            self.limits__diffexp_cellcount_max = default_config["limits"]["diffexp_cellcount_max"]
+            self.limits__column_request_max = default_config["limits"]["column_request_max"]
+
+        except KeyError as e:
+            raise ConfigurationError(f"Unexpected config: {str(e)}")
+
+        self.data_adaptor = None
+
+        # The authentication object
+        self.auth = None
+
+    def complete_config(self, context):
+        self.handle_app(context)
+        self.handle_data_source()
+        self.handle_authentication()
+        self.handle_data_locator()
+        self.handle_adaptor()  # may depend on data_locator
+        self.handle_single_dataset(context)  # may depend on adaptor
+        self.handle_limits()
+
+        self.check_config()
+
+    def handle_app(self, context):
+        self.validate_correct_type_of_configuration_attribute("app__verbose", bool)
+        self.validate_correct_type_of_configuration_attribute("app__debug", bool)
+        self.validate_correct_type_of_configuration_attribute("app__host", str)
+        self.validate_correct_type_of_configuration_attribute("app__port", (type(None), int))
+        self.validate_correct_type_of_configuration_attribute("app__open_browser", bool)
+        self.validate_correct_type_of_configuration_attribute("app__force_https", bool)
+        self.validate_correct_type_of_configuration_attribute("app__flask_secret_key", str)
+
+        if self.app__port:
+            try:
+                if not is_port_available(self.app__host, self.app__port):
+                    raise ConfigurationError(
+                        f"The port selected {self.app__port} is in use, please configure an open port."
+                    )
+            except OverflowError:
+                raise ConfigurationError(f"Invalid port: {self.app__port}")
+        else:
+            try:
+                default_server_port = int(os.environ.get("CXG_SERVER_PORT", DEFAULT_SERVER_PORT))
+            except ValueError:
+                raise ConfigurationError(
+                    "Invalid port from environment variable CXG_SERVER_PORT: " + os.environ.get("CXG_SERVER_PORT")
+                )
+            try:
+                self.app__port = find_available_port(self.app__host, default_server_port)
+            except OverflowError:
+                raise ConfigurationError(f"Invalid port: {default_server_port}")
+
+        if self.app__debug:
+            context["messagefn"]("in debug mode, setting verbose=True and open_browser=False")
+            self.app__verbose = True
+            self.app__open_browser = False
+        else:
+            warnings.formatwarning = custom_format_warning
+
+        if not self.app__verbose:
+            sys.tracebacklimit = 0
+
+    def handle_authentication(self):
+        self.validate_correct_type_of_configuration_attribute("authentication__type", (type(None), str))
+        self.validate_correct_type_of_configuration_attribute("authentication__insecure_test_environment", bool)
+
+        if self.authentication__type == "test" and not self.authentication__insecure_test_environment:
+            raise ConfigurationError("Test auth can only be used in an insecure test environment")
+
+        self.auth = AuthTypeFactory.create(self.authentication__type, self)
+        if self.auth is None:
+            raise ConfigurationError(f"Unknown authentication type: {self.authentication__type}")
+
+    def handle_data_locator(self):
+        self.validate_correct_type_of_configuration_attribute("data_locator__s3__region_name", (type(None), bool, str))
+        if self.data_locator__s3__region_name is True:
+            path = self.single_dataset__datapath
+
+            if path.startswith("s3://"):
+                region_name = discover_s3_region_name(path)
+                if region_name is None:
+                    raise ConfigurationError(f"Unable to discover s3 region name from {path}")
+            else:
+                region_name = None
+            self.data_locator__s3__region_name = region_name
+
+    def handle_data_source(self):
+        self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", str)
+
+    def handle_single_dataset(self, context):
+        self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None)))
+        self.validate_correct_type_of_configuration_attribute("single_dataset__title", (str, type(None)))
+        self.validate_correct_type_of_configuration_attribute("single_dataset__about", (str, type(None)))
+        self.validate_correct_type_of_configuration_attribute("single_dataset__obs_names", (str, type(None)))
+        self.validate_correct_type_of_configuration_attribute("single_dataset__var_names", (str, type(None)))
+
+        # preload this data set
+        matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath, app_config=self.app_config)
+        try:
+            matrix_data_loader.pre_load_validation()
+        except DatasetAccessError as e:
+            raise ConfigurationError(str(e))
+
+        file_size = matrix_data_loader.file_size()
+        file_basename = basename(self.single_dataset__datapath)
+        if file_size > BIG_FILE_SIZE_THRESHOLD:
+            context["messagefn"](f"Loading data from {file_basename}, this may take a while...")
+        else:
+            context["messagefn"](f"Loading data from {file_basename}.")
+
+        if self.single_dataset__about:
+
+            def url_check(url):
+                try:
+                    result = urlparse(url)
+                    if all([result.scheme, result.netloc]):
+                        return True
+                    else:
+                        return False
+                except ValueError:
+                    return False
+
+            if not url_check(self.single_dataset__about):
+                raise ConfigurationError(
+                    "Must provide an absolute URL for --about. (Example format: http://example.com)"
+                )
+
+    def handle_adaptor(self):
+        self.validate_correct_type_of_configuration_attribute("adaptor__anndata_adaptor__backed", bool)
+
+    def handle_limits(self):
+        self.validate_correct_type_of_configuration_attribute("limits__diffexp_cellcount_max", (type(None), int))
+        self.validate_correct_type_of_configuration_attribute("limits__column_request_max", (type(None), int))
+
+    def exceeds_limit(self, limit_name, value):
+        limit_value = getattr(self, "limits__" + limit_name, None)
+        if limit_value is None:  # disabled
+            return False
+        return value > limit_value

local_server/common/constants.py

@@ -0,0 +1,30 @@
+from enum import Enum
+
+
+class AugmentedEnum(Enum):
+    def __hash__(self):
+        return self.value.__hash__()
+
+    def __eq__(self, other):
+        if isinstance(other, type(self)) or isinstance(other, str):
+            return self.value == other
+        return False
+
+    def __str__(self) -> str:
+        return self.value
+
+
+class Axis(AugmentedEnum):
+    OBS = "obs"
+    VAR = "var"
+
+
+class DiffExpMode(AugmentedEnum):
+    TOP_N = "topN"
+    VAR_FILTER = "varFilter"
+
+
+JSON_NaN_to_num_warning_msg = "JSON encoding failure - please verify all data are finite values (no NaN or Infinities)"
+REACTIVE_LIMIT = 1_000_000
+
+MAX_LAYOUTS = 30

local_server/common/corpora.py

@@ -0,0 +1,78 @@
+"""
+Corpora schema conventions support.  Helper functions for reading.
+
+https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md
+
+https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema_h5ad_implementation.md
+"""
+import collections
+import json
+
+from local_server.cli.upgrade import validate_version_str
+from local_server.common.utils.corpora_constants import CorporaConstants
+
+
+def corpora_get_versions_from_anndata(adata):
+    """
+    Given an AnnData object, return:
+        * None - if not a Corpora object
+        * [ corpora_schema_version, corpora_encoding_version ] - if a Corpora object
+
+    Implements the identification protocol defined in the specification.
+    """
+
+    # per Corpora AnnData spec, this is a corpora file if the following is true
+    if "version" not in adata.uns_keys():
+        return None
+    version = adata.uns["version"]
+    if not isinstance(version, collections.abc.Mapping) or "corpora_schema_version" not in version:
+        return None
+
+    corpora_schema_version = version.get("corpora_schema_version")
+    corpora_encoding_version = version.get("corpora_encoding_version")
+
+    # TODO: spec says these must be SEMVER values, so check.
+    if validate_version_str(corpora_schema_version) and validate_version_str(corpora_encoding_version):
+        return [corpora_schema_version, corpora_encoding_version]
+
+
+def corpora_is_version_supported(corpora_schema_version, corpora_encoding_version):
+    return (
+        corpora_schema_version
+        and corpora_encoding_version
+        and corpora_schema_version.startswith("1.")
+        and corpora_encoding_version.startswith("0.1.")
+    )
+
+
+def corpora_get_props_from_anndata(adata):
+    """
+    Get Corpora dataset properties from an AnnData
+    """
+    versions = corpora_get_versions_from_anndata(adata)
+    if versions is None:
+        return None
+    [corpora_schema_version, corpora_encoding_version] = versions
+    version_is_supported = corpora_is_version_supported(corpora_schema_version, corpora_encoding_version)
+    if not version_is_supported:
+        raise ValueError("Unsupported Corpora schema version")
+
+    corpora_props = {}
+    for key in CorporaConstants.REQUIRED_SIMPLE_METADATA_FIELDS:
+        if key not in adata.uns:
+            raise KeyError(f"missing Corpora schema field {key}")
+        corpora_props[key] = adata.uns[key]
+
+    for key in CorporaConstants.OPTIONAL_JSON_ENCODED_METADATA_FIELD:
+        if key not in adata.uns:
+            continue
+        try:
+            corpora_props[key] = json.loads(adata.uns[key])
+        except json.JSONDecodeError:
+            raise json.JSONDecodeError(f"Corpora schema field {key} is expected to be a valid JSON string")
+
+    for key in CorporaConstants.OPTIONAL_SIMPLE_METADATA_FIELDS:
+        if key in adata.uns:
+            corpora_props[key] = adata.uns[key]
+
+    return corpora_props

local_server/common/data_locator.py

@@ -0,0 +1,154 @@
+import os
+import tempfile
+import fsspec
+from datetime import datetime
+import boto3
+import botocore
+from urllib.parse import urlparse
+
+
+class DataLocator:
+    """
+    DataLocator is a simple wrapper around fsspec functionality, and provides a
+    set of functions to encapsulate a data location (URI or path), interogate
+    metadata about the object at that location (size, existance, etc) and
+    access the underlying data.
+
+    https://filesystem-spec.readthedocs.io/en/latest/index.html
+
+    Example:
+        dl = DataLocator("/tmp/foo.h5ad")
+        if dl.exists():
+            print(dl.size())
+            with dl.open() as f:
+                thecontents = f.read()
+
+    DataLocator will accept a URI or native path.  Error handling is as defined
+    in fsspec.
+
+    """
+
+    def __init__(self, uri_or_path, region_name=None):
+        if isinstance(uri_or_path, DataLocator):
+            locator = uri_or_path
+            self.uri_or_path = locator.uri_or_path
+            self.protocol = locator.protocol
+            self.path = locator.path
+            self.cname = locator.cname
+        else:
+            self.uri_or_path = uri_or_path
+            self.protocol, self.path = DataLocator._get_protocol_and_path(uri_or_path)
+            # work-around for LocalFileSystem not treating file: and None as the same scheme/protocol
+            self.cname = self.path if self.protocol == "file" else self.uri_or_path
+
+        # fsspec.filesystem will throw RuntimeError if the protocol is unsupported
+        if self.protocol == "s3":
+            if region_name:
+                config_kwargs = dict(region_name=region_name)
+                self.fs = fsspec.filesystem(self.protocol, listings_expiry_time=30, config_kwargs=config_kwargs)
+            else:
+                self.fs = fsspec.filesystem(self.protocol, listings_expiry_time=30)
+        else:
+            self.fs = fsspec.filesystem(self.protocol)
+
+    def __repr__(self):
+        return f"DataLocator(protocol={self.protocol}, cname={self.cname}, "
+        f"path={self.path}, uri_or_path={self.uri_or_path})"
+
+    @staticmethod
+    def _get_protocol_and_path(uri_or_path):
+        if "://" in uri_or_path:
+            protocol, path = uri_or_path.split("://", 1)
+            # windows!!!  Ignore single letter drive identifiers,
+            # eg, G:\foo.txt
+            if len(protocol) > 1:
+                return protocol, path
+        return None, uri_or_path
+
+    def exists(self):
+        return self.fs.exists(self.cname)
+
+    def size(self):
+        return self.fs.size(self.cname)
+
+    def lastmodtime(self):
+        """ return datetime object representing last modification time, or None if unavailable """
+        info = self.fs.info(self.cname)
+        if self.islocal() and info is not None:
+            return datetime.fromtimestamp(info["mtime"])
+        else:
+            return getattr(info, "LastModified", None)
+
+    def abspath(self):
+        """
+        return the absolute path for the locator - only really does something
+        for file: protocol, as all others are already absolute
+        """
+        if self.islocal():
+            return os.path.abspath(self.path)
+        else:
+            return self.uri_or_path
+
+    def isfile(self):
+        return self.fs.isfile(self.cname)
+
+    def open(self, *args):
+        return self.fs.open(self.uri_or_path, *args)
+
+    def islocal(self):
+        return self.protocol is None or self.protocol == "file"
+
+    def local_handle(self):
+        if self.islocal():
+            return LocalFilePath(self.path)
+
+        # if not local, create a tmp file system object to contain the data,
+        # and clean it up when done.  If the path has a suffix/extension,
+        # do our best to create a file with the same.
+        ext = os.path.splitext(self.path)
+        suffix = None if ext[1] == "" else ext[1]
+        with self.open() as src, tempfile.NamedTemporaryFile(prefix="cellxgene_", suffix=suffix, delete=False) as tmp:
+            tmp.write(src.read())
+            tmp.close()
+            src.close()
+            tmp_path = tmp.name
+            return LocalFilePath(tmp_path, delete=True)
+
+    def ls(self):
+        paths = self.fs.ls(self.uri_or_path)
+        return [os.path.basename(p) for p in paths]
+
+
+class LocalFilePath:
+    def __init__(self, tmp_path, delete=False):
+        self.tmp_path = tmp_path
+        self.delete = delete
+
+    def __enter__(self):
+        return self.tmp_path
+
+    def __exit__(self, *args):
+        if self.delete:
+            os.unlink(self.tmp_path)
+
+
+def discover_s3_region_name(uri):
+    """If this is an s3 protocol, discover and return the (aws) region name.
+    If a return name could not be discovered, or if the uri is not an s3 protocol, return None."""
+
+    protocol, _ = DataLocator._get_protocol_and_path(uri)
+    if protocol == "s3":
+        bucket = urlparse(uri).netloc
+        client = boto3.client("s3")
+        try:
+            res = client.head_bucket(Bucket=bucket)
+        except botocore.exceptions.ClientError:
+            return None
+
+        region = res.get("ResponseMetadata", {}).get("HTTPHeaders", {}).get("x-amz-bucket-region")
+        if region:
+            return region
+        else:
+            return None
+
+    return None

local_server/common/errors.py

@@ -0,0 +1,58 @@
+from http import HTTPStatus
+
+
+class CellxgeneException(Exception):
+    """Base class for cellxgene exceptions"""
+
+    def __init__(self, message):
+        self.message = message
+        super().__init__(message)
+
+
+class RequestException(CellxgeneException):
+    """Baseclass for exceptions that can be raised from a request."""
+
+    # The default status code is 400 (Bad Request)
+    default_status_code = HTTPStatus.BAD_REQUEST
+
+    def __init__(self, message, status_code=None):
+        super().__init__(message)
+        self.status_code = status_code or self.default_status_code
+
+
+def define_exception(name, doc):
+    globals()[name] = type(name, (CellxgeneException,), dict(__doc__=doc))
+
+
+def define_request_exception(name, doc, default_status_code=HTTPStatus.BAD_REQUEST):
+    globals()[name] = type(name, (RequestException,), dict(__doc__=doc, default_status_code=default_status_code))
+
+
+define_request_exception("FilterError", "Raised when filter is malformed")
+define_request_exception("JSONEncodingValueError", "Raised when data cannot be encoded into json")
+define_request_exception("MimeTypeError", "Raised when incompatible MIME type selected")
+define_request_exception("DatasetAccessError", "Raised when file loaded into a DataAdaptor is misformatted")
+define_request_exception("DisabledFeatureError", "Raised when an attempt to use a disabled feature occurs")
+define_request_exception("AnnotationsError", "Raised when an attempt to use the annotations feature fails")
+define_request_exception(
+    "ComputeError",
+    "Raised when an error occurs during a compute algorithm (such as diffexp)",
+    HTTPStatus.INTERNAL_SERVER_ERROR,
+)
+define_request_exception("ExceedsLimitError", "Raised when an HTTP request exceeds a limit/quota")
+define_request_exception("ColorFormatException", "Raised when color helper functions encounter an unknown color format")
+define_request_exception(
+    "AuthenticationError", "Raised when there is an authentication error", default_status_code=HTTPStatus.UNAUTHORIZED
+)
+
+define_request_exception(
+    "AnnotationCategoryNameError",
+    "Raised when an annotation category name cant be saved",
+    default_status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
+)
+
+define_exception("OntologyLoadFailure", "Raised when reading the ontology file fails")
+define_exception("ConfigurationError", "Raised when checking configuration errors")
+define_exception("PrepareError", "Raised when data is misprepared")
+define_exception("SecretKeyRetrievalError", "Raised when get_secret_key from AWS fails")
+define_exception("ObsoleteRequest", "Raised when the request is no longer valid.")

local_server/common/health.py

@@ -0,0 +1,33 @@
+from http import HTTPStatus
+from flask import make_response, jsonify
+
+from local_server import __version__ as cellxgene_version
+from local_server.common.data_locator import DataLocator
+
+
+def _is_accessible(path, config):
+    if path is None:
+        return True
+
+    try:
+        dl = DataLocator(path, region_name=config.data_locator__s3__region_name)
+        return dl.exists()
+    except RuntimeError:
+        return False
+
+
+def health_check(config):
+    """
+    simple health check - return HTTP response.
+    See https://tools.ietf.org/id/draft-inadarei-api-health-check-01.html
+    """
+    health = {"status": None, "version": "1", "releaseID": cellxgene_version}
+
+    server_config = config.server_config
+    check = _is_accessible(server_config.single_dataset__datapath, server_config)
+
+    health["status"] = "pass" if check else "fail"
+    code = HTTPStatus.OK if health["status"] == "pass" else HTTPStatus.BAD_REQUEST
+    response = make_response(jsonify(health), code)
+    response.headers["Content-Type"] = "application/health+json"
+    return response

local_server/common/rest.py

@@ -0,0 +1,382 @@
+import copy
+import logging
+import sys
+from http import HTTPStatus
+import zlib
+
+from flask import make_response, jsonify, current_app, abort
+from werkzeug.urls import url_unquote
+
+from local_server.common.config.client_config import get_client_config, get_client_userinfo
+from local_server.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg
+from local_server.common.errors import (
+    FilterError,
+    JSONEncodingValueError,
+    PrepareError,
+    DisabledFeatureError,
+    ExceedsLimitError,
+    DatasetAccessError,
+    ColorFormatException,
+    AnnotationsError,
+    ObsoleteRequest,
+)
+
+import json
+from local_server.data_common.fbs.matrix import decode_matrix_fbs
+
+
+def abort_and_log(code, logmsg, loglevel=logging.DEBUG, include_exc_info=False):
+    """
+    Log the message, then abort with HTTP code. If include_exc_info is true,
+    also include current exception via sys.exc_info().
+    """
+    if include_exc_info:
+        exc_info = sys.exc_info()
+    else:
+        exc_info = False
+    current_app.logger.log(loglevel, logmsg, exc_info=exc_info)
+    # Do NOT send log message to HTTP response.
+    return abort(code)
+
+
+def _query_parameter_to_filter(args):
+    """
+    Convert an annotation value filter, if present in the query args,
+    into the standard dict filter format used by internal code.
+
+    Query param filters look like:  <axis>:name=value, where value
+    may be one of:
+        - a range, min,max, where either may be an open range by using an asterisk, eg, 10,*
+        - a value
+    Eg,
+        ...?tissue=lung&obs:tissue=heart&obs:num_reads=1000,*
+    """
+    filters = {
+        "obs": {},
+        "var": {},
+    }
+
+    # args has already been url-unquoted once.  We assume double escaping
+    # on name and value.
+    try:
+        for key, value in args.items(multi=True):
+            axis, name = key.split(":")
+            if axis not in ("obs", "var"):
+                raise FilterError("unknown filter axis")
+            name = url_unquote(name)
+            current = filters[axis].setdefault(name, {"name": name})
+
+            val_split = value.split(",")
+            if len(val_split) == 1:
+                if "min" in current or "max" in current:
+                    raise FilterError("do not mix range and value filters")
+                value = url_unquote(value)
+                values = current.setdefault("values", [])
+                values.append(value)
+
+            elif len(val_split) == 2:
+                if len(current) > 1:
+                    raise FilterError("duplicate range specification")
+                min = url_unquote(val_split[0])
+                max = url_unquote(val_split[1])
+                if min != "*":
+                    current["min"] = float(min)
+                if max != "*":
+                    current["max"] = float(max)
+                if len(current) < 2:
+                    raise FilterError("must specify at least min or max in range filter")
+
+            else:
+                raise FilterError("badly formated filter value")
+
+    except ValueError as e:
+        raise FilterError(str(e))
+
+    result = {}
+    for axis in ("obs", "var"):
+        axis_filter = filters[axis]
+        if len(axis_filter) > 0:
+            result[axis] = {"annotation_value": [val for val in axis_filter.values()]}
+
+    return result
+
+
+def schema_get_helper(data_adaptor):
+    """helper function to gather the schema from the data source and annotations"""
+    schema = data_adaptor.get_schema()
+    schema = copy.deepcopy(schema)
+
+    # add label obs annotations as needed
+    annotations = data_adaptor.dataset_config.user_annotations
+    if annotations.user_annotations_enabled():
+        label_schema = annotations.get_schema(data_adaptor)
+        schema["annotations"]["obs"]["columns"].extend(label_schema)
+
+    return schema
+
+
+def schema_get(data_adaptor):
+    schema = schema_get_helper(data_adaptor)
+    return make_response(jsonify({"schema": schema}), HTTPStatus.OK)
+
+
+def config_get(app_config, data_adaptor):
+    config = get_client_config(app_config, data_adaptor)
+    return make_response(jsonify(config), HTTPStatus.OK)
+
+
+def userinfo_get(app_config, data_adaptor):
+    config = get_client_userinfo(app_config, data_adaptor)
+    return make_response(jsonify(config), HTTPStatus.OK)
+
+
+def annotations_obs_get(request, data_adaptor):
+    fields = request.args.getlist("annotation-name", None)
+    num_columns_requested = len(data_adaptor.get_obs_keys()) if len(fields) == 0 else len(fields)
+    if data_adaptor.server_config.exceeds_limit("column_request_max", num_columns_requested):
+        return abort(HTTPStatus.BAD_REQUEST)
+    preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
+    if preferred_mimetype != "application/octet-stream":
+        return abort(HTTPStatus.NOT_ACCEPTABLE)
+
+    try:
+        labels = None
+        annotations = data_adaptor.dataset_config.user_annotations
+        if annotations.user_annotations_enabled():
+            labels = annotations.read_labels(data_adaptor)
+        fbs = data_adaptor.annotation_to_fbs_matrix(Axis.OBS, fields, labels)
+        return make_response(fbs, HTTPStatus.OK, {"Content-Type": "application/octet-stream"})
+    except KeyError as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+
+
+def annotations_put_fbs_helper(data_adaptor, fbs):
+    """helper function to write annotations from fbs"""
+    annotations = data_adaptor.dataset_config.user_annotations
+    if not annotations.user_annotations_enabled():
+        raise DisabledFeatureError("Writable annotations are not enabled")
+
+    new_label_df = decode_matrix_fbs(fbs)
+    if not new_label_df.empty:
+        new_label_df = data_adaptor.check_new_labels(new_label_df)
+    annotations.write_labels(new_label_df, data_adaptor)
+
+
+def inflate(data):
+    return zlib.decompress(data)
+
+
+def annotations_obs_put(request, data_adaptor):
+    annotations = data_adaptor.dataset_config.user_annotations
+    if not annotations.user_annotations_enabled():
+        return abort(HTTPStatus.NOT_IMPLEMENTED)
+
+    anno_collection = request.args.get("annotation-collection-name", default=None)
+    fbs = inflate(request.get_data())
+
+    if anno_collection is not None:
+        if not annotations.is_safe_collection_name(anno_collection):
+            return abort(HTTPStatus.BAD_REQUEST, "Bad annotation collection name")
+        annotations.set_collection(anno_collection)
+
+    try:
+        annotations_put_fbs_helper(data_adaptor, fbs)
+        res = json.dumps({"status": "OK"})
+        return make_response(res, HTTPStatus.OK, {"Content-Type": "application/json"})
+    except (ValueError, DisabledFeatureError, KeyError) as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+
+
+def annotations_var_get(request, data_adaptor):
+    fields = request.args.getlist("annotation-name", None)
+    num_columns_requested = len(data_adaptor.get_var_keys()) if len(fields) == 0 else len(fields)
+    if data_adaptor.server_config.exceeds_limit("column_request_max", num_columns_requested):
+        return abort(HTTPStatus.BAD_REQUEST)
+    preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
+    if preferred_mimetype != "application/octet-stream":
+        return abort(HTTPStatus.NOT_ACCEPTABLE)
+
+    try:
+        labels = None
+        return make_response(
+            data_adaptor.annotation_to_fbs_matrix(Axis.VAR, fields, labels),
+            HTTPStatus.OK,
+            {"Content-Type": "application/octet-stream"},
+        )
+    except KeyError as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+
+
+def data_var_put(request, data_adaptor):
+    preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
+    if preferred_mimetype != "application/octet-stream":
+        return abort(HTTPStatus.NOT_ACCEPTABLE)
+
+    filter_json = request.get_json()
+    filter = filter_json["filter"] if filter_json else None
+    try:
+        return make_response(
+            data_adaptor.data_frame_to_fbs_matrix(filter, axis=Axis.VAR),
+            HTTPStatus.OK,
+            {"Content-Type": "application/octet-stream"},
+        )
+    except (FilterError, ValueError, ExceedsLimitError) as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+
+
+def data_var_get(request, data_adaptor):
+    preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
+    if preferred_mimetype != "application/octet-stream":
+        return abort(HTTPStatus.NOT_ACCEPTABLE)
+
+    try:
+        filter = _query_parameter_to_filter(request.args)
+        return make_response(
+            data_adaptor.data_frame_to_fbs_matrix(filter, axis=Axis.VAR),
+            HTTPStatus.OK,
+            {"Content-Type": "application/octet-stream"},
+        )
+    except (FilterError, ValueError, ExceedsLimitError) as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+
+
+def colors_get(data_adaptor):
+    if not data_adaptor.dataset_config.presentation__custom_colors:
+        return make_response(jsonify({}), HTTPStatus.OK)
+    try:
+        return make_response(jsonify(data_adaptor.get_colors()), HTTPStatus.OK)
+    except ColorFormatException as e:
+        return abort_and_log(HTTPStatus.NOT_FOUND, str(e), include_exc_info=True)
+
+
+def diffexp_obs_post(request, data_adaptor):
+    if not data_adaptor.dataset_config.diffexp__enable:
+        return abort(HTTPStatus.NOT_IMPLEMENTED)
+
+    args = request.get_json()
+    try:
+        # TODO: implement varfilter mode
+        mode = DiffExpMode(args["mode"])
+
+        if mode == DiffExpMode.VAR_FILTER or "varFilter" in args:
+            return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, "varFilter not enabled")
+
+        set1_filter = args.get("set1", {"filter": {}})["filter"]
+        set2_filter = args.get("set2", {"filter": {}})["filter"]
+        count = args.get("count", None)
+
+        if set1_filter is None or set2_filter is None or count is None:
+            return abort_and_log(HTTPStatus.BAD_REQUEST, "missing required parameter")
+        if Axis.VAR in set1_filter or Axis.VAR in set2_filter:
+            return abort_and_log(HTTPStatus.BAD_REQUEST, "var axis filter not enabled")
+
+    except (KeyError, TypeError) as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+
+    try:
+        diffexp = data_adaptor.diffexp_topN(set1_filter, set2_filter, count)
+        return make_response(diffexp, HTTPStatus.OK, {"Content-Type": "application/json"})
+    except (ValueError, DisabledFeatureError, FilterError, ExceedsLimitError) as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+    except JSONEncodingValueError:
+        # JSON encoding failure, usually due to bad data. Just let it ripple up
+        # to default exception handler.
+        current_app.logger.warning(JSON_NaN_to_num_warning_msg)
+        raise
+
+
+def layout_obs_get(request, data_adaptor):
+    fields = request.args.getlist("layout-name", None)
+    num_columns_requested = len(data_adaptor.get_embedding_names()) if len(fields) == 0 else len(fields)
+    if data_adaptor.server_config.exceeds_limit("column_request_max", num_columns_requested):
+        return abort(HTTPStatus.BAD_REQUEST)
+
+    preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
+    if preferred_mimetype != "application/octet-stream":
+        return abort(HTTPStatus.NOT_ACCEPTABLE)
+
+    try:
+        return make_response(
+            data_adaptor.layout_to_fbs_matrix(fields), HTTPStatus.OK, {"Content-Type": "application/octet-stream"}
+        )
+    except (KeyError, DatasetAccessError) as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+    except PrepareError:
+        return abort_and_log(
+            HTTPStatus.NOT_IMPLEMENTED,
+            f"No embedding available {request.path}",
+            loglevel=logging.ERROR,
+            include_exc_info=True,
+        )
+
+
+def layout_obs_put(request, data_adaptor):
+    if not data_adaptor.dataset_config.embeddings__enable_reembedding:
+        return abort(HTTPStatus.NOT_IMPLEMENTED)
+
+    args = request.get_json()
+    filter = args["filter"] if args else None
+    if not filter:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, "obs filter is required")
+    method = args["method"] if args else "umap"
+
+    try:
+        schema = data_adaptor.compute_embedding(method, filter)
+        return make_response(jsonify(schema), HTTPStatus.OK, {"Content-Type": "application/json"})
+    except NotImplementedError as e:
+        return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, str(e))
+    except (ValueError, DisabledFeatureError, FilterError) as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+
+
+def genesets_get(request, data_adaptor):
+    preferred_mimetype = request.accept_mimetypes.best_match(["application/json", "text/csv"])
+    if preferred_mimetype not in ("application/json", "text/csv"):
+        return abort(HTTPStatus.NOT_ACCEPTABLE)
+
+    try:
+        annotations = data_adaptor.dataset_config.user_annotations
+        (genesets, tid) = data_adaptor.check_new_gene_sets(annotations.read_gene_sets(data_adaptor))
+
+        if preferred_mimetype == "text/csv":
+            return make_response(
+                annotations.gene_sets_to_csv(genesets),
+                HTTPStatus.OK,
+                {
+                    "Content-Type": "text/csv",
+                    "Content-Disposition": "attachment; filename=genesets.csv",
+                },
+            )
+        else:
+            return make_response(
+                jsonify({"genesets": annotations.gene_sets_to_response(genesets), "tid": tid}), HTTPStatus.OK
+            )
+    except (ValueError, KeyError, AnnotationsError) as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e))
+
+
+def genesets_put(request, data_adaptor):
+    annotations = data_adaptor.dataset_config.user_annotations
+    if not annotations.gene_sets_save_enabled():
+        return abort(HTTPStatus.NOT_IMPLEMENTED)
+
+    anno_collection = request.args.get("annotation-collection-name", default=None)
+    if anno_collection is not None:
+        if not annotations.is_safe_collection_name(anno_collection):
+            return abort(HTTPStatus.BAD_REQUEST, "Bad annotation collection name")
+        annotations.set_collection(anno_collection)
+
+    args = request.get_json()
+    try:
+        genesets = args.get("genesets", None)
+        tid = args.get("tid", None)
+        if genesets is None:
+            abort(HTTPStatus.BAD_REQUEST)
+
+        (gs, _) = data_adaptor.check_new_gene_sets((genesets, tid))
+        annotations.write_gene_sets(gs, tid, data_adaptor)
+        return make_response(jsonify({"status": "OK"}), HTTPStatus.OK)
+    except (ValueError, DisabledFeatureError, KeyError) as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+    except (ObsoleteRequest, TypeError) as e:
+        return abort(HTTPStatus.NOT_FOUND, description=str(e))

local_server/common/utils/corpora_constants.py

@@ -0,0 +1,22 @@
+class CorporaConstants(object):
+    REQUIRED_SIMPLE_METADATA_FIELDS = [
+        "version",
+        "title",
+        "layer_descriptions",
+        "organism",
+        "organism_ontology_term_id",
+    ]
+
+    # The Corpora specification requires some values encoded as JSON due to the inability of AnnData to store complex
+    # types.
+    OPTIONAL_JSON_ENCODED_METADATA_FIELD = ["contributors", "project_links"]
+
+    OPTIONAL_SIMPLE_METADATA_FIELDS = [
+        "preprint_doi",
+        "publication_doi",
+        "default_embedding",
+        "default_field",
+        "tags",
+        "project_name",
+        "project_description",
+    ]

local_server/common/utils/type_conversion_utils.py

@@ -0,0 +1,158 @@
+import logging
+
+import numpy as np
+import pandas as pd
+
+
+def get_dtypes_and_schemas_of_dataframe(dataframe: pd.DataFrame):
+    dtypes_by_column_name = {}
+    schema_type_hints_by_column_name = {}
+
+    for column_name, column_values in dataframe.items():
+        (
+            dtypes_by_column_name[column_name],
+            schema_type_hints_by_column_name[column_name],
+        ) = get_dtype_and_schema_of_array(column_values)
+
+    return dtypes_by_column_name, schema_type_hints_by_column_name
+
+
+def get_dtype_of_array(array: pd.Series):
+    return get_dtype_and_schema_of_array(array)[0]
+
+
+def get_schema_type_hint_of_array(array: pd.Series):
+    return get_dtype_and_schema_of_array(array)[1]
+
+
+def get_dtype_and_schema_of_array(array: pd.Series):
+    return (
+        get_dtype_from_dtype(array.dtype, array_values=array),
+        get_schema_type_hint_from_dtype(array.dtype, array_values=array),
+    )
+
+
+def get_dtype_from_dtype(dtype, array_values=None):
+    """
+    Given a data type, finds the equivalent data type that the array should be encoded as. Notably, this is relevant
+    for 64 bit values which will get downcast to 32 bit.
+    """
+
+    dtype_name = dtype.name
+    dtype_kind = dtype.kind
+
+    if dtype_name == "bool":
+        return np.uint8
+    if dtype_name == "object" and dtype_kind == "O":
+        return str
+    if dtype_name == "category":
+        return get_dtype_from_dtype(dtype.categories.dtype, array_values)
+
+    if can_cast_to_int32(dtype, array_values):
+        return np.int32
+    if can_cast_to_float32(dtype, array_values):
+        return np.float32
+    if not can_cast_to_float32(dtype, array_values):
+        return np.float64
+
+    raise TypeError(f"Annotations of type {dtype} are unsupported.")
+
+
+def get_schema_type_hint_from_dtype(dtype, array_values=None):
+    """
+    Returns a dictionary that contains type hints about the data type given, especially if the data type is 64 bit
+    and will be downcast to 32 bit.
+    """
+
+    dtype_name = dtype.name
+    dtype_kind = dtype.kind
+
+    if dtype == np.float32 or dtype == np.int32:
+        return {"type": dtype_name}
+    if dtype_name == "bool":
+        return {"type": "boolean"}
+    if dtype_name == "object" and dtype_kind == "O":
+        return {"type": "string"}
+    if dtype_name == "category":
+        return {"type": "categorical", "categories": dtype.categories.tolist()}
+
+    if can_cast_to_int32(dtype, array_values):
+        return {"type": "int32"}
+    if can_cast_to_float32(dtype, array_values):
+        return {"type": "float32"}
+    if dtype_kind == "f" and not can_cast_to_float32(dtype, array_values):
+        return {"type": "float64"}
+
+    raise TypeError(f"Annotations of type {dtype} are unsupported.")
+
+
+def can_cast_to_float32(dtype, array_values):
+    """
+    Optimistically returns True signifying that a type downcast to float32 is possible whenever the incoming type is
+    a float.
+
+    We also handle a special case here where the array is a Series object with integer categorical values AND NaNs.
+    Since NaNs are floating points in numpy, we upcast the integer array to float32 and return True.
+    """
+
+    if dtype.kind == "f":
+        if not np.can_cast(dtype, np.float32):
+            logging.warning(f"Type {dtype.name} will be converted to 32 bit float and may lose precision.")
+
+        return True
+
+    if dtype.kind == "O" and array_values.hasnans:
+        return True
+
+    return False
+
+
+def can_cast_to_int32(dtype, array_values=None):
+    """
+    A type can be cast to 32 bit, overriding the numpy `cast_cast` function if the values in the array that are of
+    the higher precision type has values that are entirely within the range of the downcast type.
+    """
+
+    # Since a NaN is technically a float, any array that contains NaNs cannot be cast to an integer so immediately
+    # return False.
+    if array_values.hasnans:
+        return False
+
+    # If the array is categorical, then we need to order the array values so that functions min and max that occur
+    # later, can function. They do not function on unordered categories.
+    ordered_array_values = array_values
+    if array_values.dtype.name == "category" and not array_values.cat.ordered:
+        ordered_array_values = array_values.cat.as_ordered()
+
+    if dtype.kind in ["i", "u"]:
+        if np.can_cast(dtype, np.int32):
+            return True
+        ii32 = np.iinfo(np.int32)
+        if (
+            not ordered_array_values.empty
+            and (ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max)
+            or ordered_array_values.empty
+        ):
+            return True
+    return False
+
+
+def convert_pandas_series_to_numpy(series_to_convert: pd.Series, dtype):
+    if series_to_convert.hasnans and dtype == np.int32:
+        logging.error("Cannot convert a pandas Series object to an integer dtype if it contains NaNs.")
+
+    return series_to_convert.to_numpy(dtype)
+
+
+def convert_string_to_value(value: str):
+    """convert a string to value with the most appropriate type"""
+    if value.lower() == "true":
+        return True
+    if value.lower() == "false":
+        return False
+    if value == "null":
+        return None
+    try:
+        return eval(value)
+    except:  # noqa E722
+        return value

local_server/common/utils/utils.py

@@ -0,0 +1,118 @@
+import contextlib
+import errno
+import importlib.util
+import logging
+import os
+import pkgutil
+import socket
+from urllib.parse import urlsplit, urljoin
+
+import numpy as np
+from flask import json
+
+from local_server.common.errors import ConfigurationError
+
+
+def find_available_port(host, port=5005):
+    """
+    Helper method to find open port on host. Tries 5000 ports incremented from the specified port
+    """
+    # Takes approx 2 seconds to do a scan of 5000 ports on my laptop
+    num_ports_to_try = 5000
+    for port_to_try in range(port, port + num_ports_to_try):
+        if is_port_available(host, port_to_try):
+            return port_to_try
+    raise socket.error(errno.EADDRINUSE, f"No port in range {port} - {port + num_ports_to_try - 1} available.")
+
+
+def is_port_available(host, port):
+    is_available = False
+    with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
+        try:
+            s.bind((host, port))
+            is_available = True
+        except socket.error:
+            pass
+    return is_available
+
+
+def sort_options(command):
+    """
+    Helper for the click options - will sort options in a command, and can
+    be used as a decorator.
+    """
+    command.params.sort(key=lambda p: p.name)
+    return command
+
+
+def path_join(base, *urls):
+    """
+    this is like urllib.parse.urljoin, except it works around the scheme-specific
+    cleverness in the aforementioned code, ignores anything in the url except the path,
+    and accepts more than one url.
+    """
+    if not base.endswith("/"):
+        base += "/"
+    btpl = urlsplit(base)
+    path = btpl.path
+    for url in urls:
+        utpl = urlsplit(url)
+        if btpl.scheme == "":
+            path = os.path.join(path, utpl.path)
+            path = os.path.normpath(path)
+        else:
+            path = urljoin(path, utpl.path)
+    return btpl._replace(path=path).geturl()
+
+
+class Float32JSONEncoder(json.JSONEncoder):
+    def __init__(self, *args, **kwargs):
+        """
+        NaN/Infinities are illegal in standard JSON.  Python extends JSON with
+        non-standard symbols that most JavaScript JSON parsers do not understand.
+        The `allow_nan` parameter will force Python simplejson to throw an ValueError
+        if it runs into non-finite floating point values which are unsupported by
+        standard JSON.
+        """
+        kwargs["allow_nan"] = False
+        super().__init__(*args, **kwargs)
+
+    def default(self, obj):
+        if isinstance(obj, np.float32):
+            return float(obj)
+        elif isinstance(obj, np.integer):
+            return int(obj)
+        return json.JSONEncoder.default(self, obj)
+
+
+def custom_format_warning(msg, *args, **kwargs):
+    return f"[cellxgene] Warning: {msg} \n"
+
+
+def jsonify_numpy(data):
+    return json.dumps(data, cls=Float32JSONEncoder, allow_nan=False)
+
+
+def import_plugins(plugin_module):
+    """
+    Load optional plugin modules from local_server.common.plugins
+
+    If you would like to customize cellxgene, you can add submodules to server.common.plugins before running the app.
+    This code will import each, loading the code in each. If no plugins are defined, initializing the app continues as
+    normal.
+    """
+    loaded_modules = []
+    try:
+        pkg = importlib.import_module(plugin_module)
+        for loader, name, is_pkg in pkgutil.walk_packages(pkg.__path__):
+            full_name = f"{plugin_module}.{name}"
+            try:
+                module = importlib.import_module(full_name)
+            except Exception as e:
+                raise ConfigurationError(f"Unexpected error while importing plugin: {plugin_module}.{name}: {str(e)}")
+            loaded_modules.append(module)
+    except ModuleNotFoundError as e:
+        #  This exception occurs when the plugin_module does not exist (not an error).
+        logging.debug(f"No plugins found in module: {plugin_module}: {str(e)}")
+
+    return loaded_modules

local_server/compute/diffexp_generic.py

@@ -0,0 +1,134 @@
+import numpy as np
+from scipy import sparse, stats
+
+
+def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
+    """
+    Return differential expression statistics for top N variables.
+
+    Algorithm:
+    - compute log fold change (log2(meanA/meanB))
+    - compute Welch's t-test statistic and pvalue (w/ Bonferroni correction)
+    - return top N abs(logfoldchange) where lfc > diffexp_lfc_cutoff
+
+    If there are not N which meet criteria, augment by removing the logfoldchange
+    threshold requirement.
+
+    Notes on alogrithm:
+    - Welch's ttest provides basic statistics test.
+      https://en.wikipedia.org/wiki/Welch%27s_t-test
+    - p-values adjusted with Bonferroni correction.
+      https://en.wikipedia.org/wiki/Bonferroni_correction
+
+    :param adaptor: DataAdaptor instance
+    :param maskA: observation selection mask for set 1
+    :param maskB: observation selection mask for set 2
+    :param top_n: number of variables to return stats for
+    :param diffexp_lfc_cutoff: minimum
+    :return:  for top N genes, [ varindex, logfoldchange, pval, pval_adj ]
+    """
+
+    dataA = adaptor.get_X_array(maskA, None)
+    dataB = adaptor.get_X_array(maskB, None)
+
+    # mean, variance, N - calculate for both selections
+    meanA, vA, nA = mean_var_n(dataA)
+    meanB, vB, nB = mean_var_n(dataB)
+    res = diffexp_ttest_from_mean_var(meanA, vA, nA, meanB, vB, nB, top_n, diffexp_lfc_cutoff)
+
+    return res
+
+
+def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp_lfc_cutoff):
+    n_var = meanA.shape[0]
+    top_n = min(top_n, n_var)
+
+    # variance / N
+    vnA = varA / min(nA, nB)  # overestimate variance, would normally be nA
+    vnB = varB / min(nA, nB)  # overestimate variance, would normally be nB
+    sum_vn = vnA + vnB
+
+    # degrees of freedom for Welch's t-test
+    with np.errstate(divide="ignore", invalid="ignore"):
+        dof = sum_vn ** 2 / (vnA ** 2 / (nA - 1) + vnB ** 2 / (nB - 1))
+    dof[np.isnan(dof)] = 1
+
+    # Welch's t-test score calculation
+    with np.errstate(divide="ignore", invalid="ignore"):
+        tscores = (meanA - meanB) / np.sqrt(sum_vn)
+    tscores[np.isnan(tscores)] = 0
+
+    # p-value
+    pvals = stats.t.sf(np.abs(tscores), dof) * 2
+    pvals_adj = pvals * n_var
+    pvals_adj[pvals_adj > 1] = 1  # cap adjusted p-value at 1
+
+    # logfoldchanges: log2(meanA / meanB)
+    logfoldchanges = np.log2(np.abs((meanA + 1e-9) / (meanB + 1e-9)))
+
+    # find all with lfc > cutoff
+    lfc_above_cutoff_idx = np.nonzero(np.abs(logfoldchanges) > diffexp_lfc_cutoff)[0]
+    stats_to_sort = np.abs(tscores)
+
+    # derive sort order
+    if lfc_above_cutoff_idx.shape[0] > top_n:
+        # partition top N
+        rel_t_partition = np.argpartition(stats_to_sort[lfc_above_cutoff_idx], -top_n)[-top_n:]
+        t_partition = lfc_above_cutoff_idx[rel_t_partition]
+        # sort the top N partition
+        rel_sort_order = np.argsort(stats_to_sort[t_partition])[::-1]
+        sort_order = t_partition[rel_sort_order]
+    else:
+        # partition and sort top N, ignoring lfc cutoff
+        partition = np.argpartition(stats_to_sort, -top_n)[-top_n:]
+        rel_sort_order = np.argsort(stats_to_sort[partition])[::-1]
+        indices = np.indices(stats_to_sort.shape)[0]
+        sort_order = indices[partition][rel_sort_order]
+
+    # top n slice based upon sort order
+    logfoldchanges_top_n = logfoldchanges[sort_order]
+    pvals_top_n = pvals[sort_order]
+    pvals_adj_top_n = pvals_adj[sort_order]
+
+    # varIndex, logfoldchange, pval, pval_adj
+    result = [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in range(top_n)]
+    return result
+
+
+# Convenience function which handles sparse data
+def mean_var_n(X):
+    """
+    Two-pass variance calculation.  Numerically (more) stable
+    than naive methods (and same method used by numpy.var())
+    https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Two-pass
+    """
+    # fp_err_occurred is a flag indicating that a floating point error
+    # occured somewhere in our compute.  Used to trigger non-finite
+    # number handling.
+    fp_err_occurred = False
+
+    def fp_err_set(err, flag):
+        nonlocal fp_err_occurred
+        fp_err_occurred = True
+
+    with np.errstate(divide="call", invalid="call", call=fp_err_set):
+        n = X.shape[0]
+        if sparse.issparse(X):
+            mean = X.mean(axis=0).A1
+            dfm = X - mean
+            sumsq = np.sum(np.multiply(dfm, dfm), axis=0).A1
+            v = sumsq / (n - 1)
+        else:
+            mean = X.mean(axis=0)
+            dfm = X - mean
+            sumsq = np.sum(np.multiply(dfm, dfm), axis=0)
+            v = sumsq / (n - 1)
+
+    if fp_err_occurred:
+        mean[np.isfinite(mean) == False] = 0  # noqa: E712
+        v[np.isfinite(v) == False] = 0  # noqa: E712
+    else:
+        mean[np.isnan(mean)] = 0
+        v[np.isnan(v)] = 0
+
+    return mean, v, n

local_server/compute/scanpy.py

@@ -0,0 +1,53 @@
+import importlib
+import numpy as np
+
+"""
+Wrapper for various scanpy modules.  Will raise NotImplementedError if the scanpy
+module is not installed/available
+"""
+
+
+def get_scanpy_module():
+    try:
+        sc = importlib.import_module("scanpy")
+        # Future: we could enforce versions here, eg, lookat sc.__version__
+        return sc
+    except ModuleNotFoundError as e:
+        raise NotImplementedError("Please install scanpy to enable UMAP re-embedding") from e
+    except Exception as e:
+        # will capture other ImportError corner cases
+        raise NotImplementedError() from e
+
+
+def scanpy_umap(adata, obs_mask=None, pca_options={}, neighbors_options={}, umap_options={}):
+    """
+    Given adata and an obs mask, return a new embedding for adata[obs_mask, :]
+    as an ndarray of shape (len(obs_mask), N), where N>=2.
+
+    Do NOT mutate adata.
+    """
+
+    # backed mode is incompatible with the current implementation
+    if adata.isbacked:
+        raise NotImplementedError("Backed mode is incompatible with re-embedding")
+
+    # safely get scanpy module, which may not be present.
+    sc = get_scanpy_module()
+
+    # https://github.com/theislab/anndata/issues/311
+    obs_mask = slice(None) if obs_mask is None else obs_mask
+    adata = adata[obs_mask, :].copy()
+
+    for k in list(adata.obsm.keys()):
+        del adata.obsm[k]
+    for k in list(adata.uns.keys()):
+        del adata.uns[k]
+
+    sc.pp.pca(adata, zero_center=None, n_comps=min(adata.n_vars - 1, 50), **pca_options)
+    sc.pp.neighbors(adata, **neighbors_options)
+    sc.tl.umap(adata, **umap_options)
+
+    umap = adata.obsm["X_umap"]
+    result = np.full((obs_mask.shape[0], umap.shape[1]), np.NaN)
+    result[obs_mask] = umap
+    return result

local_server/converters/schema/gene_symbol.py

@@ -0,0 +1,211 @@
+"""Helpers for converting and checking HGNC gene symbols."""
+
+import argparse
+import enum
+import logging
+import os
+import re
+import numpy as np
+import pandas as pd
+
+
+def get_upgraded_var_index(var, hgnc_path=None):
+    """Given an anndata var dataframe, return a new index for the dataframe
+    where human gene symbols have been upgraded to the current HGNC set.
+    """
+
+    if not hgnc_path:
+        hgnc_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "hgnc_complete_set.txt.gz")
+
+    hgnc_symbol_checker = HGNCSymbolChecker.from_hgnc_records(hgnc_path)
+
+    return pd.Index([hgnc_symbol_checker.upgrade_symbol(s) for s in var.index])
+
+
+class SymbolStatus(enum.Enum):
+    """The status of a symbol in the HGNC database.
+
+    APPROVED: Currently a valid symbol
+    WITHDRAWN: A previously approved HGNC symbol for a gene that has since been shown
+               not to exist _unless_ that symbol is also approved
+    AMBIGUOUS: A symbol that is not approved but is an alias or previous symbol for
+               multiple approved symbols
+    UPGRADABLE: A symbol that is not approved but unambiguously maps to an approved
+                symbol
+    UNKNOWN: A symbol that does not appear in HGNC
+    """
+
+    APPROVED = 1
+    WITHDRAWN = 2
+    AMBIGUOUS = 3
+    UPGRADABLE = 4
+    UNKNOWN = 5
+
+
+class HGNCSymbolChecker:
+    """Handle checking and correcting HGNC symbols."""
+
+    def __init__(self, approved_symbols, withdrawn_symbols, ambiguous_symbols, symbol_map):
+        self.approved_symbols = approved_symbols
+        self.withdrawn_symbols = withdrawn_symbols
+        self.ambiguous_symbols = ambiguous_symbols
+        self.symbol_map = symbol_map
+
+    def print_symbol_map(self):
+        """Print out a map from old symbol to new symbol."""
+
+        for symbol_pair in self.symbol_map.items():
+            print("\t".join(symbol_pair))
+
+    def check_symbol(self, symbol):
+        """See if a symbol if approved or something else."""
+        if symbol in self.approved_symbols:
+            return SymbolStatus.APPROVED
+
+        if symbol in self.withdrawn_symbols:
+            return SymbolStatus.WITHDRAWN
+
+        if symbol in self.ambiguous_symbols:
+            return SymbolStatus.AMBIGUOUS
+
+        if symbol in self.symbol_map:
+            return SymbolStatus.UPGRADABLE
+
+        return SymbolStatus.UNKNOWN
+
+    def upgrade_symbol(self, symbol):
+        """Return the approved symbol for the given symbol.
+
+        If the symbol cannot be upgraded, just return the original symbol.
+        """
+
+        fixed_symbol, stripped_symbol = format_symbol(symbol)
+
+        if fixed_symbol in self.approved_symbols:
+            return fixed_symbol
+        elif fixed_symbol in self.symbol_map:
+            return self.symbol_map[fixed_symbol]
+        elif stripped_symbol in self.approved_symbols:
+            return stripped_symbol
+        elif stripped_symbol in self.symbol_map:
+            return self.symbol_map[stripped_symbol]
+
+        return symbol
+
+    @classmethod
+    def from_hgnc_records(cls, hgnc_dataset_path):
+        """Parse a hgnc database download into a HGNCSymbolChecker object."""
+
+        def all_symbols(record):
+            """Get all the symbols associated with an HGNC record including previous, alias,
+            and approved."""
+            yield format_symbol(record["symbol"])[0]
+            for symbol in alias_and_previous_symbols(record):
+                yield symbol
+
+        def alias_and_previous_symbols(record):
+            """Get alias and previous symbols from an HGNC record."""
+            for field in ("alias_symbol", "prev_symbol"):
+                if record[field] is not np.nan:
+                    for symbol in record[field].split("|"):
+                        yield format_symbol(symbol)[0]
+            # Sometimes something like HGNC:1234 appears in datasets, which we
+            # want to fix as well.
+            yield record["hgnc_id"]
+
+        hgnc_records = pd.read_csv(hgnc_dataset_path, sep="\t", header=0, low_memory=False).to_dict("records")
+
+        # Get all symbols that are currently approved.
+        approved_symbols = set()
+        for record in hgnc_records:
+            if record["status"] == "Approved":
+                approved_symbols.add(format_symbol(record["symbol"])[0])
+
+        # Get all symbols that have been withdrawn
+        withdrawn_symbols = set()
+        for record in hgnc_records:
+            if record["status"] == "Entry Withdrawn":
+                for symbol in all_symbols(record):
+                    withdrawn_symbols.add(symbol)
+
+        # If a symbol is both approved and withdrawn, be optimistic and call it approved
+        logging.warning(
+            f"Some symbols are simulaneously withdrawn and approved\n"
+            f"We will treat them at approved:\n"
+            f"{withdrawn_symbols.intersection(approved_symbols)}"
+        )
+        withdrawn_symbols = withdrawn_symbols.difference(approved_symbols)
+
+        # Now try to map from symbols that are not approved but are an alias or previous symbol for an approved symbol
+        alias_previous_to_approved = {}
+        ambiguous_symbols = set()
+
+        for record in hgnc_records:
+            if record["status"] == "Approved":
+
+                # The approved symbol is what we'll map to
+                approved_symbol = format_symbol(record["symbol"])[0]
+
+                for symbol in alias_and_previous_symbols(record):
+
+                    # If the alias or previous symbol is also an approved symbol,
+                    # we'll just leave it alone
+                    if symbol in approved_symbols:
+                        continue
+
+                    # If the alias or previous symbol maps to a different approved symbol, mark it as ambiguous
+                    if symbol in alias_previous_to_approved and alias_previous_to_approved[symbol] != approved_symbol:
+                        ambiguous_symbols.add(symbol)
+                    else:
+                        alias_previous_to_approved[symbol] = approved_symbol
+
+        # Remove all the ambiguous symbols from the map
+        for ambiguous_symbol in ambiguous_symbols:
+            alias_previous_to_approved.pop(ambiguous_symbol)
+
+        return HGNCSymbolChecker(approved_symbols, withdrawn_symbols, ambiguous_symbols, alias_previous_to_approved)
+
+
+def format_symbol(symbol):
+    """HGNC rules say symbols should all be upper case except for C#orf#. However, case is
+    variable in both alias and previous symbols as well as in the symbols we get in
+    submissions. So, upper case everything except for the one situation where mixed-case
+    is allowed, which are the genes like C2orf157.
+
+    Also, seurat and scanpy append ".1" or "-1" to duplicated gene names, and these altered
+    names persist throughout the life of the object. They won't match against the HGNC database
+    and we want to merge them, so we need to strip off the suffix and try matching again.
+
+    This function takes a symbol and returns the symbol with the fixed case and also with the
+    seurat/scanpy suffix stripped off.
+    """
+
+    match = re.match(r"^(C)(\d+)(orf)(\d+)$", symbol, re.IGNORECASE)
+
+    if match:
+        fixed_case = f"C{match.group(2)}orf{match.group(4)}"
+    else:
+        fixed_case = symbol.upper()
+
+    suffix_stripped = re.sub(r"[\.\-]\d+$", "", fixed_case)
+
+    return fixed_case, suffix_stripped
+
+
+def main():
+    """When called as main, parse a given hgnc download and print out a map from old to new
+    symbol.
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "hgnc_dataset", help="HGNC dataset tsv, available from www.genenames.org/download/statistics-and-files/"
+    )
+    args = parser.parse_args()
+
+    hgnc_symbol_checker = HGNCSymbolChecker.from_hgnc_records(args.hgnc_dataset)
+
+    hgnc_symbol_checker.print_symbol_map()
+
+
+if __name__ == "__main__":
+    main()

local_server/converters/schema/ontology.py

@@ -0,0 +1,86 @@
+"""Methods for working with ontologies and the OLS."""
+from urllib.parse import quote_plus
+
+import requests
+
+OLS_API_ROOT = "http://www.ebi.ac.uk/ols/api"
+
+# Curie means something like CL:0000001
+
+
+def _ontology_name(curie):
+    """Get the name of the ontology from the curie, CL or UBERON for example."""
+    return curie.split(":")[0]
+
+
+def _ontology_value(curie):
+    """Get the id component of the curie, 0000001 from CL:0000001 for example."""
+    return curie.split(":")[1]
+
+
+def _double_encode(url):
+    """Double url encode a url. This is required by the OLS API."""
+    return quote_plus(quote_plus(url))
+
+
+def _iri(curie):
+    """Get the iri from a curie. This is a bit hopeful that they all map to purl.obolibrary.org"""
+    if _ontology_name(curie) == "EFO":
+        return f"http://www.ebi.ac.uk/efo/EFO_{_ontology_value(curie)}"
+    return f"http://purl.obolibrary.org/obo/{_ontology_name(curie)}_{_ontology_value(curie)}"
+
+
+class OntologyLookupError(Exception):
+    """Exception for some problem with looking up ontology information."""
+
+
+def _ontology_info_url(curie):
+    """Get the to make a GET to to get information about an ontology term."""
+
+    # If the curie is empty, just return an empty string. This happens when there is no
+    # valid ontology value.
+    if not curie:
+        return ""
+    else:
+        return f"{OLS_API_ROOT}/ontologies/{_ontology_name(curie)}/terms/{_double_encode(_iri(curie))}"
+
+
+def get_ontology_label(curie):
+    """For a given curie like 'CL:1000413', get the label like 'endothelial cell of artery'"""
+
+    url = _ontology_info_url(curie)
+
+    if not url:
+        return ""
+
+    response = requests.get(url)
+
+    if not response.ok:
+        raise OntologyLookupError(
+            f"Curie {curie} lookup failed, got status code {response.status_code}: {response.text}"
+        )
+    return response.json()["label"]
+
+
+def lookup_candidate_term(label, ontology="cl", method="select"):
+    """Lookup candidate terms for a label. This is useful when there is an existing label in a
+    submitted dataset, and you want to find an appropriate ontology term.
+
+    Args:
+      label: the label to find ontology terms for
+      ontology: the ontology to search in, cl or uberon or efo for example
+      method: select or search. search provides much broader results
+
+    Returns:
+      list of (curie, label) tuples returned by OLS
+    """
+    # using OLS REST API [https://www.ebi.ac.uk/ols/docs/api]
+    url = f"{OLS_API_ROOT}/{method}?q={quote_plus(label)}&ontology={ontology.lower()}"
+    response = requests.get(url)
+
+    if not response.ok:
+        raise OntologyLookupError(
+            f"Label {label} lookup failed, got status code {response.status_code}: {response.text}"
+        )
+
+    return [(r["obo_id"], r["label"]) for r in response.json()["response"]["docs"]]

local_server/converters/schema/remix.py

@@ -0,0 +1,264 @@
+import argparse
+import collections
+import json
+import logging
+import math
+import string
+
+import anndata
+import numpy as np
+import pandas as pd
+import yaml
+
+from . import gene_symbol
+from . import ontology
+from . import validate
+
+REPLACE_SUFFIX = "_original"
+ONTOLOGY_SUFFIX = "_ontology_term_id"
+
+
+def is_curie(value):
+    """Return True iff the value is an OBO-id CURIE like EFO:000001"""
+    return (value.count(":")
+            and all(len(part) > 0 for part in value.split(":"))
+            and all(c in string.digits for c in value.split(":")[1]))
+
+
+def is_ontology_field(field_name):
+    """Return True iff the field_name is an ontology field like tissue_ontology_term_id"""
+    return field_name.endswith(ONTOLOGY_SUFFIX)
+
+
+def get_label_field_name(field_name):
+    """Get the associated label field from an ontology field, assay_ontology_term_id --> assay"""
+    return field_name[: -len(ONTOLOGY_SUFFIX)]
+
+
+def split_suffix(maybe_curie):
+    """Split off the (cell culture) or (organoid) suffix."""
+
+    suffixes = [" (cell culture)", " (organoid)"]
+    for suffix in suffixes:
+        if maybe_curie.endswith(suffix):
+            return maybe_curie[:-len(suffix)], suffix
+    return maybe_curie, ""
+
+
+def get_curie_and_label(maybe_curie):
+    """Given a string that might be a curie, return a (curie, label) pair"""
+
+    maybe_curie, suffix = split_suffix(maybe_curie)
+    if not is_curie(maybe_curie):
+        return ("", maybe_curie + suffix)
+    return (maybe_curie + suffix, ontology.get_ontology_label(maybe_curie) + suffix)
+
+
+def safe_add_field(adata_attr, field_name, field_value):
+    """Add a field and value to an AnnData, but don't clobber an exising value."""
+
+    if (
+        isinstance(field_value, list)
+        and field_value
+        and isinstance(field_value[0], dict)
+    ):
+        field_value = json.dumps(field_value)
+    if field_name in adata_attr:
+        adata_attr[field_name + REPLACE_SUFFIX] = adata_attr[field_name]
+    adata_attr[field_name] = field_value
+
+
+def remix_uns(adata, uns_config):
+    """Add fields from the config to adata.uns"""
+    for field_name, field_value in uns_config.items():
+
+        if is_ontology_field(field_name):
+            # If it's an ontology field, look it up
+            label_field_name = get_label_field_name(field_name)
+            ontology_term, ontology_label = get_curie_and_label(field_value)
+            safe_add_field(adata.uns, field_name, ontology_term)
+            safe_add_field(adata.uns, label_field_name, ontology_label)
+        else:
+            safe_add_field(adata.uns, field_name, field_value)
+
+
+def remix_obs(adata, obs_config):
+    """Add fields from the config to adata.obs"""
+
+    for field_name, field_value in obs_config.items():
+
+        if isinstance(field_value, dict):
+            # If the value is a dict, that means we are supposed to map from an
+            # existing column to the new one
+            source_column, column_map = next(iter(field_value.items()))
+            nan_value = None
+            for key in column_map:
+                if isinstance(key, float) and math.isnan(key):
+                    nan_value = column_map[key]
+            if nan_value is not None:
+                column_map["nan"] = nan_value
+
+            for key in column_map:
+                if key not in adata.obs[source_column].unique():
+                    logging.warning(f'Key {key} not in adata.obs["{source_column}"]')
+
+            for value in adata.obs[source_column].unique():
+                if value not in column_map:
+                    logging.warning(f'Value {value} in adata.obs["{source_column}"] not in translation dict')
+
+            if is_ontology_field(field_name):
+                ontology_term_map, ontology_label_map = {}, {}
+                logging.info(f"Looking up labels for {field_name}")
+                for original_value, maybe_curie in column_map.items():
+                    curie, label = get_curie_and_label(maybe_curie)
+                    ontology_term_map[original_value] = curie
+                    ontology_label_map[original_value] = label
+                    logging.info(f"Mapping {original_value} -> {curie} -> {label}")
+
+                ontology_column = adata.obs[source_column].replace(
+                    ontology_term_map, inplace=False
+                )
+                label_column = adata.obs[source_column].replace(
+                    ontology_label_map, inplace=False
+                )
+
+                safe_add_field(adata.obs, field_name, ontology_column)
+                safe_add_field(
+                    adata.obs, get_label_field_name(field_name), label_column
+                )
+            else:
+                label_column = adata.obs[source_column].replace(
+                    column_map, inplace=False
+                )
+                safe_add_field(adata.obs, field_name, label_column)
+
+        else:
+            if is_ontology_field(field_name):
+                # If it's an ontology field, look it up
+                label_field_name = get_label_field_name(field_name)
+                ontology_term, ontology_label = get_curie_and_label(field_value)
+                safe_add_field(adata.obs, field_name, ontology_term)
+                safe_add_field(adata.obs, label_field_name, ontology_label)
+            else:
+                safe_add_field(adata.obs, field_name, field_value)
+
+
+def merge_df(df, domain, index, columns):
+    """
+    Given a dataframe with duplicate column labels, merge and return a dataframe where
+    the duplicates have been merged together, resulting in a dataframe with unique column
+    labels.
+
+    "merge" depends on the value of domain. If the domain is "raw", then duplicate columns
+    can just be summed. If it's "log1p" or "sqrt", it needs to be exp1m'd or squared, then
+    summed, and then logged or sqrt'd again.
+    """
+
+    if not isinstance(df, np.ndarray):
+        to_merge = df.toarray()
+    else:
+        to_merge = df
+    if domain == "raw":
+        merged_df = pd.DataFrame(to_merge, index=index, columns=columns).sum(
+            axis=1, level=0, skipna=False
+        )
+    elif domain == "log1p":
+        merged_df = (
+            pd.DataFrame(np.expm1(to_merge, dtype=np.float128), index=index, columns=columns)
+            .sum(axis=1, level=0, skipna=False)
+        )
+        merged_df = pd.DataFrame(np.log1p(merged_df.to_numpy()), index=merged_df.index, columns=merged_df.columns)
+    elif domain == "sqrt":
+        merged_df = (
+            pd.DataFrame(np.square(to_merge), index=index, columns=columns)
+            .sum(axis=1, level=0, skipna=False)
+        )
+        merged_df = pd.DataFrame(np.sqrt(merged_df.to_numpy()), index=merged_df.index, columns=merged_df.columns)
+
+    return merged_df
+
+
+def fixup_gene_symbols(adata, fixup_config):
+    """Update the var index to hold a consistent set of HGNC gene symbols."""
+
+    upgraded_var_index = gene_symbol.get_upgraded_var_index(adata.var)
+
+    merged_X = merge_df(adata.X, fixup_config["X"], adata.obs.index, upgraded_var_index)
+    fixup_adata = anndata.AnnData(
+        X=merged_X,
+        obs=adata.obs,
+        var=merged_X.columns.to_frame(name="hgnc_gene_symbol"),
+        uns=adata.uns,
+        obsm=adata.obsm,
+    )
+
+    for layer, domain in fixup_config.items():
+        if layer == "X":
+            continue
+        if layer == "raw.X":
+            df = adata.raw.X
+        else:
+            df = adata.layers[layer]
+
+        merged_df = merge_df(df, domain, adata.obs.index, upgraded_var_index)
+        assert merged_df.index.equals(merged_X.index)
+        assert merged_df.columns.equals(merged_X.columns)
+
+        if domain == "raw":
+            fixup_raw = anndata.AnnData(
+                X=merged_df,
+                obs=adata.obs,
+                var=merged_X.columns.to_frame(name="hgnc_gene_symbol"),
+            )
+            fixup_adata.raw = fixup_raw
+        else:
+            fixup_adata.layers[layer] = merged_df
+
+    return fixup_adata
+
+def _strip_version(adata):
+    """Remove version information from the AnnData object."""
+
+    if "version" in adata.uns_keys():
+        del adata.uns["version"]
+
+def apply_schema(source_h5ad, remix_config, output_filename):
+
+    try:
+        import scanpy
+    except ImportError:
+        raise ImportError("scanpy must be installed for cellxgene schema")
+    adata = scanpy.read_h5ad(source_h5ad)
+    config = yaml.load(open(remix_config), Loader=yaml.FullLoader)
+    remix_uns(adata, config["uns"])
+    remix_obs(adata, config["obs"])
+
+    if config.get("fixup_gene_symbols"):
+        adata = fixup_gene_symbols(adata, config["fixup_gene_symbols"])
+
+    if ("version" in adata.uns_keys()
+            and isinstance(adata.uns["version"], collections.Mapping)
+            and "corpora_schema_version" in adata.uns["version"]):
+        schema_version = adata.uns["version"]["corpora_schema_version"]
+        try:
+            validate.get_schema_definition(schema_version)
+        except ValueError:
+            logging.warning(f"Stripping version information out of AnnData because schema "
+                            f"version {schema_version} is unknown.")
+            _strip_version(adata)
+
+        if not validate.validate_adata(adata, shallow=False):
+            logging.warning(f"Stripping version information out of AnnData because it does not "
+                            f"follow schema version {schema_version} .")
+            _strip_version(adata)
+
+    adata.write_h5ad(output_filename, compression="gzip")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--source-h5ad", required=True)
+    parser.add_argument("--remix-config", required=True)
+    parser.add_argument("--output-filename", required=True)
+    args = parser.parse_args()
+    apply_schema(args.source_h5ad, args.remix_config, args.output_filename)

local_server/converters/schema/schema_definitions/1_0_0.yaml

@@ -0,0 +1,95 @@
+title: Corpora schema version 1.0.0
+type: anndata
+components:
+    uns:
+        type: dict
+        keys:
+            version:
+                type: dict
+                keys:
+                    corpora_schema_version: null
+                    corpora_encoding_version: null
+            title:
+                type: string
+            contributors:
+                type: stringified list of dicts
+            layer_descriptions:
+                type: dict
+                keys:
+                    X: null
+            organism:
+                type: string
+                nullable: false
+            organism_ontology_term_id:
+                type: curie
+                prefixes:
+                    - NCBITaxon
+    var:
+        type: dataframe
+        index:
+            type: human-readable string
+            unique: true
+    obs:
+        type: dataframe
+        index:
+            unique: true
+        columns:
+            tissue:
+                type: human-readable string
+                nullable: false
+            tissue_ontology_term_id:
+                type: suffixed curie
+                nullable: true
+                prefixes:
+                    - UBERON
+            assay:
+                type: human-readable string
+                nullable: false
+            assay_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - EFO
+            disease:
+                type: human-readable string
+                nullable: false
+            disease_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - MONDO
+                    - PATO
+            cell_type:
+                type: human-readable string
+                nullable: false
+            cell_type_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - CL
+                    - UBERON
+            sex:
+                type: string
+                enum:
+                    - male
+                    - female
+                    - mixed
+                    - unknown
+                    - other
+            ethnicity:
+                type: human-readable string
+                nullable: false
+            ethnicity_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - HANCESTRO
+            development_stage:
+                type: human-readable string
+                nullable: false
+            development_stage_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - HsapDv
+                    - EFO

local_server/converters/schema/schema_definitions/1_1_0.yaml

@@ -0,0 +1,93 @@
+title: Corpora schema version 1.1.0
+type: anndata
+components:
+    uns:
+        type: dict
+        keys:
+            version:
+                type: dict
+                keys:
+                    corpora_schema_version: null
+                    corpora_encoding_version: null
+            title:
+                type: string
+            layer_descriptions:
+                type: dict
+                keys:
+                    X: null
+            organism:
+                type: string
+                nullable: false
+            organism_ontology_term_id:
+                type: curie
+                prefixes:
+                    - NCBITaxon
+    var:
+        type: dataframe
+        index:
+            type: human-readable string
+            unique: true
+    obs:
+        type: dataframe
+        index:
+            unique: true
+        columns:
+            tissue:
+                type: human-readable string
+                nullable: false
+            tissue_ontology_term_id:
+                type: suffixed curie
+                nullable: true
+                prefixes:
+                    - UBERON
+            assay:
+                type: human-readable string
+                nullable: false
+            assay_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - EFO
+            disease:
+                type: human-readable string
+                nullable: false
+            disease_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - MONDO
+                    - PATO
+            cell_type:
+                type: human-readable string
+                nullable: false
+            cell_type_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - CL
+                    - UBERON
+            sex:
+                type: string
+                enum:
+                    - male
+                    - female
+                    - mixed
+                    - unknown
+                    - other
+            ethnicity:
+                type: human-readable string
+                nullable: false
+            ethnicity_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - HANCESTRO
+            development_stage:
+                type: human-readable string
+                nullable: false
+            development_stage_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - HsapDv
+                    - EFO

local_server/converters/schema/validate.py

@@ -0,0 +1,236 @@
+import json
+import re
+import os
+import sys
+
+import pandas as pd
+import yaml
+
+
+def _is_null(v):
+    """Return True if v is null, for one of the multiple ways a "null" value shows up in an h5ad."""
+    return pd.isnull(v) or (hasattr(v, "__len__") and len(v) == 0)
+
+
+def _validate_stringified_list_of_dicts(s):
+    """Verify that a string can be parsed into a list.
+
+    We have some types that are lists of dicts. Those cannot be stored directly in an h5ad, so we have to
+    json.dumps them. This verifies that we can load them back.
+    """
+
+    try:
+        list_ = json.loads(s)
+        if not isinstance(list_, list):
+            return False
+        for el in list_:
+            if not isinstance(el, dict):
+                return False
+        return True
+    except (json.JSONDecodeError, TypeError):
+        pass
+    return False
+
+
+def _validate_human_readable_string(s):
+    """Verify that a string is human-readable.
+
+    There are parts of the schema where a "human-readable" string is required. "Human-readable" is kind
+    of vague and subjective. I feel like I can read many strings. So here we just check for the main ways
+    that fails: someone puts in an ontology term id or and ensembl gene/transcript id.
+
+    Returns False if s is not a string or is one of those bad string types.
+    """
+
+    return isinstance(s, str) and (not re.match(r"[A-Z]\w+:\d+", s)) and (not re.match(r"ENS[GT]\d+$", s))
+
+
+def _validate_curie(c, prefixes):
+    """Verify that a string is a valid compact URI, like EFO:000001. If prefixes is not empty, make sure the
+    prefix of the curies is in prefixes.
+    """
+
+    if not c:
+        return True
+
+    match = re.match(r"([A-Z]\w+):\d+$", c)
+
+    if prefixes:
+        return match and match.group(1) in prefixes
+    else:
+        return match
+
+
+def _validate_suffixed_curie(c, prefixes):
+    """Verify that a string is a compact URI with an optional suffix like 'EFO:00001 (cell culture)'"""
+
+    # Pull off the suffix
+    suffix = re.findall(r"\ \(.*\)$", c)
+    if suffix:
+        c = c[: -len(suffix[0])]
+    return _validate_curie(c, prefixes)
+
+
+def _validate_column(column, column_name, df_name, schema_def):
+    """Given a schema definition and the column of a dataframe, verify that the column satifies
+    the schema.
+    """
+
+    errors = []
+
+    if schema_def.get("unique"):
+        if column.nunique() != len(column):
+            errors.append(f"Column {column_name} in dataframe {df_name} is not unique.")
+
+    if "nullable" in schema_def and not schema_def["nullable"]:
+        if any(_is_null(v) for v in column):
+            errors.append(f"Column {column_name} in dataframe {df_name} contains empty values.")
+
+    if schema_def.get("type") == "human-readable string":
+        non_readables = [v for v in column if not _validate_human_readable_string(v)]
+        if non_readables:
+            errors.append(
+                f"Column {column_name} in dataframe {df_name} contains non-human-readable "
+                f"values like {non_readables[0]}"
+            )
+
+    if schema_def.get("type") in ("curie", "suffixed curie"):
+        validation_func = _validate_curie if schema_def.get("type") == "curie" else _validate_suffixed_curie
+        non_valid_curies = [v for v in column if not validation_func(v, schema_def.get("prefixes"))]
+        if non_valid_curies:
+            errors.append(
+                f"Column {column_name} in dataframe {df_name} contains invalid ontology values like "
+                f"{non_valid_curies[0]}."
+            )
+            if "prefixes" in schema_def:
+                errors[-1] += f" Values must be curies from one of these ontologies {schema_def['prefixes']}."
+
+    if "enum" in schema_def:
+        bad_enums = [v for v in column if v not in schema_def["enum"]]
+        if bad_enums:
+            errors.append(
+                f"Column {column_name} in dataframe {df_name} contains unpermitted values like "
+                f"{bad_enums[0]}. Values must be one of {schema_def['enum']}."
+            )
+
+    return errors
+
+
+def _validate_dict(dict_, dict_name, schema_def):
+    """Given a schema definition and dict, verify that the dict satifies the schema."""
+
+    errors = []
+
+    for key in schema_def.get("keys", []):
+        if key not in dict_:
+            errors.append(f"{dict_name} is missing key {key}.")
+        elif schema_def["keys"][key]:
+            if schema_def["keys"][key]["type"] == "stringified list of dicts":
+                if not _validate_stringified_list_of_dicts(dict_[key]):
+                    errors.append(
+                        f"Key {key} in {dict_name} should be a JSON-encoded list of dicts, but it is {dict_[key]}"
+                    )
+            elif schema_def["keys"][key]["type"] == "dict":
+                errors.extend(_validate_dict(dict_[key], key, schema_def["keys"][key]))
+            elif schema_def["keys"][key]["type"] == "curie":
+                if not _validate_curie(dict_[key], schema_def["keys"][key]["prefixes"]):
+                    errors.append(f"Key {key} in {dict_name} contains invalid ontology value.")
+            if "nullable" in schema_def["keys"][key] and not schema_def["keys"][key]["nullable"]:
+                if _is_null(dict_[key]):
+                    errors.append(f"Key {key} in dict {dict_name} is an empty value.")
+
+    return errors
+
+
+def _validate_dataframe(df, df_name, schema_def):
+    """Given a dataframe and schema definition, verify that the dataframe follows the schema."""
+
+    errors = []
+
+    if "index" in schema_def:
+        errors.extend(_validate_column(df.index, "index", df_name, schema_def["index"]))
+
+    for column in schema_def.get("columns", []):
+        if column not in df.columns:
+            errors.append(f"Dataframe {df_name} is missing column {column}.")
+        else:
+            errors.extend(_validate_column(df[column], column, df_name, schema_def["columns"][column]))
+
+    return errors
+
+
+def get_schema_definition(version):
+    """Look up and read a schema definition based on a version number like "1.0.0"."""
+
+    path = os.path.join(
+        os.path.dirname(os.path.realpath(__file__)), "schema_definitions", version.replace(".", "_") + ".yaml"
+    )
+
+    if not os.path.isfile(path):
+        raise ValueError(f"No definition for version {version} found.")
+
+    return yaml.load(open(path), Loader=yaml.FullLoader)
+
+
+def deep_check(adata, schema_def):
+    """Perform a "deep" check of the AnnData object using the schema definition.
+
+    This checks all the columns and unstructured metadata rather than just the version.
+
+    Returns a list of error messages. If that list is empty, the object passed validation.
+    """
+
+    errors = []
+
+    for component, component_def in schema_def["components"].items():
+        if component_def["type"] == "dataframe":
+            errors.extend(_validate_dataframe(getattr(adata, component), component, component_def))
+        elif component_def["type"] == "dict":
+            errors.extend(_validate_dict(getattr(adata, component), component, component_def))
+        else:
+            raise ValueError(f"Unexpected component type {component['type']}")
+
+    return errors
+
+
+def validate_adata(adata, shallow):
+    """Validate an AnnData object. If shallow, just check that the required version information is
+    present.
+    """
+
+    # Does it have the version information written into uns?
+    if "version" not in adata.uns_keys() or "corpora_schema_version" not in adata.uns["version"]:
+        print("AnnData file is missing corpora version information")
+        return False
+
+    # We can stop here if it's a "shallow" check, that is, if we're just
+    # checking that version is present.
+    if shallow:
+        return True
+
+    schema_def = get_schema_definition(adata.uns["version"]["corpora_schema_version"])
+
+    errors = deep_check(adata, schema_def)
+
+    for error in errors:
+        print(error)
+
+    return not errors
+
+
+def validate(h5ad_path, shallow=False):
+    """Entry point for validation."""
+
+    try:
+        import scanpy
+    except ImportError:
+        raise ImportError("scanpy must be installed for cellxgene schema")
+
+    try:
+        adata = scanpy.read_h5ad(h5ad_path, backed="r")
+    except (OSError, TypeError):
+        print(f"Unable to open {h5ad_path} with scanpy.")
+        sys.exit(1)
+
+    if not validate_adata(adata, shallow):
+        sys.exit(1)

local_server/data_anndata/anndata_adaptor.py

@@ -0,0 +1,369 @@
+import warnings
+from datetime import datetime
+
+import anndata
+import numpy as np
+from packaging import version
+from pandas.core.dtypes.dtypes import CategoricalDtype
+from scipy import sparse
+from server_timing import Timing as ServerTiming
+
+import local_server.compute.diffexp_generic as diffexp_generic
+from local_server.common.colors import convert_anndata_category_colors_to_cxg_category_colors
+from local_server.common.constants import Axis, MAX_LAYOUTS
+from local_server.common.corpora import corpora_get_props_from_anndata
+from local_server.common.errors import PrepareError, DatasetAccessError, FilterError
+from local_server.common.utils.type_conversion_utils import get_schema_type_hint_of_array
+from local_server.compute.scanpy import scanpy_umap
+from local_server.data_common.data_adaptor import DataAdaptor
+from local_server.data_common.fbs.matrix import encode_matrix_fbs
+
+anndata_version = version.parse(str(anndata.__version__)).release
+
+
+def anndata_version_is_pre_070():
+    major = anndata_version[0]
+    minor = anndata_version[1] if len(anndata_version) > 1 else 0
+    return major == 0 and minor < 7
+
+
+class AnndataAdaptor(DataAdaptor):
+    def __init__(self, data_locator, app_config=None, dataset_config=None):
+        super().__init__(data_locator, app_config, dataset_config)
+        self.data = None
+        self._load_data(data_locator)
+        self._validate_and_initialize()
+
+    def cleanup(self):
+        pass
+
+    @staticmethod
+    def pre_load_validation(data_locator):
+        if data_locator.islocal():
+            # if data locator is local, apply file system conventions and other "cheap"
+            # validation checks.  If a URI, defer until we actually fetch the data and
+            # try to read it.  Many of these tests don't make sense for URIs (eg, extension-
+            # based typing).
+            if not data_locator.exists():
+                raise DatasetAccessError("does not exist")
+            if not data_locator.isfile():
+                raise DatasetAccessError("is not a file")
+
+    @staticmethod
+    def file_size(data_locator):
+        return data_locator.size() if data_locator.islocal() else 0
+
+    @staticmethod
+    def open(data_locator, app_config, dataset_config=None):
+        return AnndataAdaptor(data_locator, app_config, dataset_config)
+
+    def get_corpora_props(self):
+        return corpora_get_props_from_anndata(self.data)
+
+    def get_name(self):
+        return "cellxgene anndata adaptor version"
+
+    def get_library_versions(self):
+        return dict(anndata=str(anndata.__version__))
+
+    @staticmethod
+    def _create_unique_column_name(df, col_name_prefix):
+        """ given the columns of a dataframe, and a name prefix, return a column name which
+            does not exist in the dataframe, AND which is prefixed by `prefix`
+
+            The approach is to append a numeric suffix, starting at zero and increasing by
+            one, until an unused name is found (eg, prefix_0, prefix_1, ...).
+        """
+        suffix = 0
+        while f"{col_name_prefix}{suffix}" in df:
+            suffix += 1
+        return f"{col_name_prefix}{suffix}"
+
+    def _alias_annotation_names(self):
+        """
+        The front-end relies on the existance of a unique, human-readable
+        index for obs & var (eg, var is typically gene name, obs the cell name).
+        The user can specify these via the --obs-names and --var-names config.
+        If they are not specified, use the existing index to create them, giving
+        the resulting column a unique name (eg, "name").
+
+        In both cases, enforce that the result is unique, and communicate the
+        index column name to the front-end via the obs_names and var_names config
+        (which is incorporated into the schema).
+        """
+        self.original_obs_index = self.data.obs.index
+
+        for (ax_name, var_name) in ((Axis.OBS, "obs"), (Axis.VAR, "var")):
+            config_name = f"single_dataset__{var_name}_names"
+            parameter_name = f"{var_name}_names"
+            name = getattr(self.server_config, config_name)
+            df_axis = getattr(self.data, str(ax_name))
+            if name is None:
+                # Default: create unique names from index
+                if not df_axis.index.is_unique:
+                    raise KeyError(
+                        f"Values in {ax_name}.index must be unique. "
+                        "Please prepare data to contain unique index values, or specify an "
+                        "alternative with --{ax_name}-name."
+                    )
+                name = self._create_unique_column_name(df_axis.columns, "name_")
+                self.parameters[parameter_name] = name
+                # reset index to simple range; alias name to point at the
+                # previously specified index.
+                df_axis.rename_axis(name, inplace=True)
+                df_axis.reset_index(inplace=True)
+            elif name in df_axis.columns:
+                # User has specified alternative column for unique names, and it exists
+                if not df_axis[name].is_unique:
+                    raise KeyError(
+                        f"Values in {ax_name}.{name} must be unique. " "Please prepare data to contain unique values."
+                    )
+                df_axis.reset_index(drop=True, inplace=True)
+                self.parameters[parameter_name] = name
+            else:
+                # user specified a non-existent column name
+                raise KeyError(f"Annotation name {name}, specified in --{ax_name}-name does not exist.")
+
+    def _create_schema(self):
+        self.schema = {
+            "dataframe": {"nObs": self.cell_count, "nVar": self.gene_count, "type": str(self.data.X.dtype)},
+            "annotations": {
+                "obs": {"index": self.parameters.get("obs_names"), "columns": []},
+                "var": {"index": self.parameters.get("var_names"), "columns": []},
+            },
+            "layout": {"obs": []},
+        }
+        for ax in Axis:
+            curr_axis = getattr(self.data, str(ax))
+            for ann in curr_axis:
+                ann_schema = {"name": ann, "writable": False}
+                ann_schema.update(get_schema_type_hint_of_array(curr_axis[ann]))
+                self.schema["annotations"][ax]["columns"].append(ann_schema)
+
+        for layout in self.get_embedding_names():
+            layout_schema = {"name": layout, "type": "float32", "dims": [f"{layout}_0", f"{layout}_1"]}
+            self.schema["layout"]["obs"].append(layout_schema)
+
+    def get_schema(self):
+        return self.schema
+
+    def _load_data(self, data_locator):
+        # as of AnnData 0.6.19, backed mode performs initial load fast, but at the
+        # cost of significantly slower access to X data.
+        try:
+            # there is no guarantee data_locator indicates a local file.  The AnnData
+            # API will only consume local file objects.  If we get a non-local object,
+            # make a copy in tmp, and delete it after we load into memory.
+            with data_locator.local_handle() as lh:
+                # as of AnnData 0.6.19, backed mode performs initial load fast, but at the
+                # cost of significantly slower access to X data.
+                backed = "r" if self.server_config.adaptor__anndata_adaptor__backed else None
+                self.data = anndata.read_h5ad(lh, backed=backed)
+
+        except ValueError:
+            raise DatasetAccessError(
+                "File must be in the .h5ad format. Please read "
+                "https://github.com/theislab/scanpy_usage/blob/master/170505_seurat/info_h5ad.md to "
+                "learn more about this format. You may be able to convert your file into this format "
+                "using `cellxgene prepare`, please run `cellxgene prepare --help` for more "
+                "information."
+            )
+        except MemoryError:
+            raise DatasetAccessError("Out of memory - file is too large for available memory.")
+        except Exception:
+            raise DatasetAccessError(
+                "File not found or is inaccessible. File must be an .h5ad object. "
+                "Please check your input and try again."
+            )
+
+    def _validate_and_initialize(self):
+        if anndata_version_is_pre_070():
+            warnings.warn(
+                "Use of anndata versions older than 0.7 will have serious issues. Please update to at "
+                "least anndata 0.7 or later."
+            )
+
+        # var and obs column names must be unique
+        if not self.data.obs.columns.is_unique or not self.data.var.columns.is_unique:
+            raise KeyError("All annotation column names must be unique.")
+
+        self._alias_annotation_names()
+        self._validate_data_types()
+        self.cell_count = self.data.shape[0]
+        self.gene_count = self.data.shape[1]
+        self._create_schema()
+
+        # heuristic
+        n_values = self.data.shape[0] * self.data.shape[1]
+        if (n_values > 1e8 and self.server_config.adaptor__anndata_adaptor__backed is True) or (n_values > 5e8):
+            self.parameters.update({"diffexp_may_be_slow": True})
+
+    def _is_valid_layout(self, arr):
+        """ return True if this layout data is a valid array for front-end presentation:
+            * ndarray, dtype float/int/uint
+            * with shape (n_obs, >= 2)
+            * with all values finite or NaN (no +Inf or -Inf)
+        """
+        is_valid = type(arr) == np.ndarray and arr.dtype.kind in "fiu"
+        is_valid = is_valid and arr.shape[0] == self.data.n_obs and arr.shape[1] >= 2
+        is_valid = is_valid and not np.any(np.isinf(arr)) and not np.all(np.isnan(arr))
+        return is_valid
+
+    def _validate_data_types(self):
+        # The backed API does not support interrogation of the underlying sparsity or sparse matrix type
+        # Fake it by asking for a small subarray and testing it.   NOTE: if the user has ignored our
+        # anndata <= 0.7 warning, opted for the --backed option, and specified a large, sparse dataset,
+        # this "small" indexing request will load the entire X array. This is due to a bug in anndata<=0.7
+        # which will load the entire X matrix to fullfill any slicing request if X is sparse.  See
+        # user warning in _load_data().
+        X0 = self.data.X[0, 0:1]
+        if sparse.isspmatrix(X0) and not sparse.isspmatrix_csc(X0):
+            warnings.warn(
+                "Anndata data matrix is sparse, but not a CSC (columnar) matrix.  "
+                "Performance may be improved by using CSC."
+            )
+        if self.data.X.dtype != "float32":
+            warnings.warn(
+                f"Anndata data matrix is in {self.data.X.dtype} format not float32. " f"Precision may be truncated."
+            )
+        for ax in Axis:
+            curr_axis = getattr(self.data, str(ax))
+            for ann in curr_axis:
+                datatype = curr_axis[ann].dtype
+                downcast_map = {
+                    "int64": "int32",
+                    "uint32": "int32",
+                    "uint64": "int32",
+                    "float64": "float32",
+                }
+                if datatype in downcast_map:
+                    warnings.warn(
+                        f"Anndata annotation {ax}:{ann} is in unsupported format: {datatype}. "
+                        f"Data will be downcast to {downcast_map[datatype]}."
+                    )
+                if isinstance(datatype, CategoricalDtype):
+                    category_num = len(curr_axis[ann].dtype.categories)
+                    if category_num > 500 and category_num > self.dataset_config.presentation__max_categories:
+                        warnings.warn(
+                            f"{str(ax).title()} annotation '{ann}' has {category_num} categories, this may be "
+                            f"cumbersome or slow to display. We recommend setting the "
+                            f"--max-category-items option to 500, this will hide categorical "
+                            f"annotations with more than 500 categories in the UI"
+                        )
+
+    def annotation_to_fbs_matrix(self, axis, fields=None, labels=None):
+        if axis == Axis.OBS:
+            if labels is not None and not labels.empty:
+                df = self.data.obs.join(labels, self.parameters.get("obs_names"))
+            else:
+                df = self.data.obs
+        else:
+            df = self.data.var
+
+        if fields is not None and len(fields) > 0:
+            df = df[fields]
+        return encode_matrix_fbs(df, col_idx=df.columns)
+
+    def get_embedding_names(self):
+        """
+        Return pre-computed embeddings.
+
+        function:
+            a) generate list of default layouts
+            b) validate layouts are legal.  remove/warn on any that are not
+            c) cap total list of layouts at global const MAX_LAYOUTS
+        """
+        # load default layouts from the data.
+        layouts = self.dataset_config.embeddings__names
+
+        if layouts is None or len(layouts) == 0:
+            layouts = [key[2:] for key in self.data.obsm_keys() if type(key) == str and key.startswith("X_")]
+
+        # remove invalid layouts
+        valid_layouts = []
+        obsm_keys = self.data.obsm_keys()
+        for layout in layouts:
+            layout_name = f"X_{layout}"
+            if layout_name not in obsm_keys:
+                warnings.warn(f"Ignoring unknown layout name: {layout}.")
+            elif not self._is_valid_layout(self.data.obsm[layout_name]):
+                warnings.warn(f"Ignoring layout due to malformed shape or data type: {layout}")
+            else:
+                valid_layouts.append(layout)
+
+        if len(valid_layouts) == 0:
+            raise PrepareError("No valid layout data.")
+
+        # cap layouts to MAX_LAYOUTS
+        return valid_layouts[0:MAX_LAYOUTS]
+
+    def get_embedding_array(self, ename, dims=2):
+        full_embedding = self.data.obsm[f"X_{ename}"]
+        return full_embedding[:, 0:dims]
+
+    def compute_embedding(self, method, obsFilter):
+        if Axis.VAR in obsFilter:
+            raise FilterError("Observation filters may not contain variable conditions")
+        if method != "umap":
+            raise NotImplementedError(f"re-embedding method {method} is not available.")
+        try:
+            shape = self.get_shape()
+            obs_mask = self._axis_filter_to_mask(Axis.OBS, obsFilter["obs"], shape[0])
+        except (KeyError, IndexError):
+            raise FilterError("Error parsing filter")
+        with ServerTiming.time("layout.compute"):
+            X_umap = scanpy_umap(self.data, obs_mask)
+
+        # Server picks reemedding name, which must not collide with any other
+        # embedding name generated by this backend.
+        name = f"reembed:{method}_{datetime.now().isoformat(timespec='milliseconds')}"
+        dims = [f"{name}_0", f"{name}_1"]
+        layout_schema = {"name": name, "type": "float32", "dims": dims}
+        self.schema["layout"]["obs"].append(layout_schema)
+        self.data.obsm[f"X_{name}"] = X_umap
+        return layout_schema
+
+    def compute_diffexp_ttest(self, maskA, maskB, top_n=None, lfc_cutoff=None):
+        if top_n is None:
+            top_n = self.dataset_config.diffexp__top_n
+        if lfc_cutoff is None:
+            lfc_cutoff = self.dataset_config.diffexp__lfc_cutoff
+        return diffexp_generic.diffexp_ttest(self, maskA, maskB, top_n, lfc_cutoff)
+
+    def get_colors(self):
+        return convert_anndata_category_colors_to_cxg_category_colors(self.data)
+
+    def get_X_array(self, obs_mask=None, var_mask=None):
+        if obs_mask is None:
+            obs_mask = slice(None)
+        if var_mask is None:
+            var_mask = slice(None)
+        X = self.data.X[obs_mask, var_mask]
+        return X
+
+    def get_shape(self):
+        return self.data.shape
+
+    def query_var_array(self, term_name):
+        return getattr(self.data.var, term_name)
+
+    def query_obs_array(self, term_name):
+        return getattr(self.data.obs, term_name)
+
+    def get_obs_index(self):
+        name = self.server_config.single_dataset__obs_names
+        if name is None:
+            return self.original_obs_index
+        else:
+            return self.data.obs[name]
+
+    def get_obs_columns(self):
+        return self.data.obs.columns
+
+    def get_obs_keys(self):
+        # return list of keys
+        return self.data.obs.keys().to_list()
+
+    def get_var_keys(self):
+        # return list of keys
+        return self.data.var.keys().to_list()

local_server/data_common/data_adaptor.py

@@ -0,0 +1,481 @@
+from abc import ABCMeta, abstractmethod
+from os.path import basename, splitext
+import re
+
+import numpy as np
+import pandas as pd
+from server_timing import Timing as ServerTiming
+
+from local_server.common.config.app_config import AppConfig
+from local_server.common.constants import Axis
+from local_server.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError
+from local_server.common.utils.utils import jsonify_numpy
+from local_server.data_common.fbs.matrix import encode_matrix_fbs
+
+
+class DataAdaptor(metaclass=ABCMeta):
+    """Base class for loading and accessing matrix data"""
+
+    def __init__(self, data_locator, app_config, dataset_config=None):
+        if type(app_config) != AppConfig:
+            raise TypeError("config expected to be of type AppConfig")
+
+        # location to the dataset
+        self.data_locator = data_locator
+
+        # config is the application configuration
+        self.app_config = app_config
+        self.server_config = self.app_config.server_config
+        self.dataset_config = dataset_config or app_config.dataset_config
+
+        # parameters set by this data adaptor based on the data.
+        self.parameters = {}
+
+    @staticmethod
+    @abstractmethod
+    def pre_load_validation(data_locator):
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def open(data_locator, app_config, dataset_config):
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def file_size(data_locator):
+        pass
+
+    @abstractmethod
+    def get_name(self):
+        """return a string name for this data adaptor"""
+        pass
+
+    @abstractmethod
+    def get_library_versions(self):
+        """return a dictionary of library name to library versions"""
+        pass
+
+    @abstractmethod
+    def get_embedding_names(self):
+        """return a list of pre-computed embedding names"""
+        pass
+
+    @abstractmethod
+    def get_embedding_array(self, ename, dims=2):
+        """return an numpy array for the given pre-computed embedding name."""
+        pass
+
+    @abstractmethod
+    def compute_embedding(self, method, filter):
+        """compute a new embedding on the specified obs subset, and return the embedding schema. """
+        pass
+
+    @abstractmethod
+    def get_X_array(self, obs_mask=None, var_mask=None):
+        """return the X array, possibly filtered by obs_mask or var_mask.
+        the return type is either ndarray or scipy.sparse.spmatrix."""
+        pass
+
+    @abstractmethod
+    def get_shape(self):
+        pass
+
+    @abstractmethod
+    def query_var_array(self, term_var):
+        pass
+
+    @abstractmethod
+    def query_obs_array(self, term_var):
+        pass
+
+    @abstractmethod
+    def get_colors(self):
+        pass
+
+    @abstractmethod
+    def get_obs_index(self):
+        pass
+
+    @abstractmethod
+    def get_obs_columns(self):
+        pass
+
+    @abstractmethod
+    def get_obs_keys(self):
+        # return list of keys
+        pass
+
+    @abstractmethod
+    def get_var_keys(self):
+        # return list of keys
+        pass
+
+    @abstractmethod
+    def cleanup(self):
+        pass
+
+    def get_data_locator(self):
+        return self.data_locator
+
+    def get_location(self):
+        return self.data_locator.uri_or_path
+
+    def get_about(self):
+        return None
+
+    def get_title(self):
+        # default to file name
+        location = self.get_location()
+        if location.endswith("/"):
+            location = location[:-1]
+        return splitext(basename(location))[0]
+
+    def get_corpora_props(self):
+        return None
+
+    @abstractmethod
+    def get_schema(self):
+        """
+        Return current schema
+        """
+        pass
+
+    @abstractmethod
+    def annotation_to_fbs_matrix(self, axis, field=None, uid=None):
+        """
+        Gets annotation value for each observation
+        :param axis: string obs or var
+        :param fields: list of keys for annotation to return, returns all annotation values if not set.
+        :return: flatbuffer: in fbs/matrix.fbs encoding
+        """
+        pass
+
+    def update_parameters(self, parameters):
+        parameters.update(self.parameters)
+
+    def _index_filter_to_mask(self, filter, count):
+        mask = np.zeros((count,), dtype=np.bool)
+        for i in filter:
+            if type(i) == list:
+                mask[i[0] : i[1]] = True
+            else:
+                mask[i] = True
+        return mask
+
+    def _axis_filter_to_mask(self, axis, filter, count):
+        mask = np.ones((count,), dtype=np.bool)
+        if "index" in filter:
+            mask = np.logical_and(mask, self._index_filter_to_mask(filter["index"], count))
+        if "annotation_value" in filter:
+            mask = np.logical_and(mask, self._annotation_filter_to_mask(axis, filter["annotation_value"], count))
+
+        return mask
+
+    def _annotation_filter_to_mask(self, axis, filter, count):
+        mask = np.ones((count,), dtype=np.bool)
+        for v in filter:
+            name = v["name"]
+            if axis == Axis.VAR:
+                anno_data = self.query_var_array(name)
+            elif axis == Axis.OBS:
+                anno_data = self.query_obs_array(name)
+
+            if anno_data.dtype.name in ["boolean", "category", "object"]:
+                values = v.get("values", [])
+                key_idx = np.in1d(anno_data, values)
+                mask = np.logical_and(mask, key_idx)
+
+            else:
+                min_ = v.get("min", None)
+                max_ = v.get("max", None)
+                if min_ is not None:
+                    key_idx = (anno_data >= min_).ravel()
+                    mask = np.logical_and(mask, key_idx)
+                if max_ is not None:
+                    key_idx = (anno_data <= max_).ravel()
+                    mask = np.logical_and(mask, key_idx)
+
+        return mask
+
+    def _filter_to_mask(self, filter):
+        """
+        Return the filter as a row and column selection list.
+        No filter on a dimension means 'all'
+        """
+        shape = self.get_shape()
+        var_selector = None
+        obs_selector = None
+        if filter is not None:
+            if Axis.OBS in filter:
+                obs_selector = self._axis_filter_to_mask(Axis.OBS, filter["obs"], shape[0])
+
+            if Axis.VAR in filter:
+                var_selector = self._axis_filter_to_mask(Axis.VAR, filter["var"], shape[1])
+
+        return (obs_selector, var_selector)
+
+    def check_new_labels(self, labels_df):
+        """Check the new annotations labels, then set the labels_df index"""
+        if labels_df is None or labels_df.empty:
+            return
+
+        labels_df.index = self.get_obs_index()
+        if labels_df.index.name is None:
+            labels_df.index.name = "index"
+
+        # all labels must have a name, which must be unique and not used in obs column names
+        if not labels_df.columns.is_unique:
+            raise KeyError("All column names specified in user annotations must be unique.")
+
+        # the label index must be unique, and must have same values the anndata obs index
+        if not labels_df.index.is_unique:
+            raise KeyError("All row index values specified in user annotations must be unique.")
+
+        obs_columns = self.get_obs_columns()
+
+        duplicate_columns = list(set(labels_df.columns) & set(obs_columns))
+        if len(duplicate_columns) > 0:
+            raise KeyError(
+                "Labels file may not contain column names which overlap " f"with h5ad obs columns {duplicate_columns}"
+            )
+
+        # labels must have same count as obs annotations
+        shape = self.get_shape()
+        if labels_df.shape[0] != shape[0]:
+            raise ValueError("Labels file must have same number of rows as data file.")
+
+        # This will convert a float column that contains integer data into an integer type.
+        # This case can occur when a user makes a copy of a category that originally contained integer data.
+        # The client always copies array data to floats, therefore the copy will contain floats instead of integers.
+        # float data is not allowed as a categorical type.
+        if any([np.issubdtype(coltype.type, np.floating) for coltype in labels_df.dtypes]):
+            labels_df = labels_df.convert_dtypes()
+            for col, dtype in zip(labels_df, labels_df.dtypes):
+                if isinstance(dtype, pd.Int32Dtype):
+                    labels_df[col] = labels_df[col].astype("int32")
+                if isinstance(dtype, pd.Int64Dtype):
+                    labels_df[col] = labels_df[col].astype("int64")
+
+        if any([np.issubdtype(coltype.type, np.floating) for coltype in labels_df.dtypes]):
+            raise ValueError("Columns may not have floating point types")
+
+        return labels_df
+
+    def check_new_gene_sets(self, args, context=None):
+        """
+        Check validity of gene sets, return if correct, else raise error.
+        May also modify the gene set for conditions that should be resolved,
+        but which do not warrant a hard error.
+
+        Argument 'args' must be a tuple containing (genesets, tid).  Genesets
+        may be either the REST OTA format (list of dicts) or the internal format
+        (dict of dicts, keyed by the geneset name).
+
+        Rules:
+        0. all geneset names must be unique.
+        1. All geneset names must be legal, meaning:
+            * no leading or trailing white space
+            * no multi-space runs
+            * character set matches: [A-Z][a-z][0-9][ .()-]
+           Generates hard error.
+        2. Gene symbols must be part of the current var_index.  If symbol not in var_index,
+           will generate a warning and the symbol removed.
+        3. Duplicate gene symbols are silently de-duped.
+        """
+        (genesets, tid) = args
+        messagefn = context["messagefn"] if context else (lambda x: None)
+
+        # accept genesets args as either the internal (dict) or REST (list) format,
+        # as they are identical except for the dict being keyed by geneset_name.
+        if type(genesets) not in (dict, list):
+            raise ValueError("Genesets must be either dict or list.")
+        genesets = genesets if type(genesets) == list else genesets.values()
+
+        # 0. check for uniqueness of geneset names
+        geneset_names = [gs["geneset_name"] for gs in genesets]
+        if len(set(geneset_names)) != len(geneset_names):
+            raise KeyError("All geneset names must be unique.")
+
+        # 1. check gene set character set and format
+        legal_name = re.compile(r"^(\w|[ .()-])+$")
+        for name in geneset_names:
+            if type(name) != str or len(name) == 0:
+                raise KeyError("Geneset names must be non-null string.")
+            if name[0] in " \t\n\r" or name[-1] in " \t\n\r" or not legal_name.match(name) or "  " in name:
+                messagefn(
+                    "Error: "
+                    f"Geneset name {name} is not valid. Only alphanumeric and limited special characters (-_.) "
+                    "and space are allowed. Leading, trailing, and multiple spaces within a name are not allowed."
+                )
+                raise KeyError(
+                    "Geneset name is not valid, only alphanumeric and limited special characters (-_.) "
+                    "and space are allowed. Leading, trailing, and multiple spaces within a name are not allowed."
+                )
+
+        # 2. & 3. check for duplicate gene symbols, and those not present in the dataset. They will
+        # generate a warning and be removed.
+        var_names = set(self.query_var_array(self.parameters.get("var_names")))
+        for geneset in genesets:
+            if type(geneset) != dict:
+                raise ValueError("Each geneset must be a dict.")
+            geneset_name = geneset["geneset_name"]
+            genes = geneset["genes"]
+            if type(genes) != list:
+                raise ValueError("Geneset genes field must be a list")
+            gene_symbol_already_seen = set()
+            new_genes = []
+            for gene in genes:
+                gene_symbol = gene["gene_symbol"]
+                if type(gene_symbol) != str or len(gene_symbol) == 0:
+                    raise ValueError("Gene symbol must be non-null string.")
+                if gene_symbol in gene_symbol_already_seen:
+                    # duplicate check
+                    messagefn(
+                        f"Warning: a duplicate of gene {gene_symbol} was found in geneset {geneset_name}, "
+                        "and will be ignored."
+                    )
+                    continue
+
+                if gene_symbol not in var_names:
+                    messagefn(
+                        f"Warning: {gene_symbol}, used in geneset {geneset_name}, "
+                        "was not found in the dataset and will be ignored."
+                    )
+                    continue
+
+                gene_symbol_already_seen.add(gene_symbol)
+                new_genes.append(gene)
+
+            geneset["genes"] = new_genes
+
+        return args
+
+    def data_frame_to_fbs_matrix(self, filter, axis):
+        """
+        Retrieves data 'X' and returns in a flatbuffer Matrix.
+        :param filter: filter: dictionary with filter params
+        :param axis: string obs or var
+        :return: flatbuffer Matrix
+
+        Caveats:
+        * currently only supports access on VAR axis
+        * currently only supports filtering on VAR axis
+        """
+        if axis != Axis.VAR:
+            raise ValueError("Only VAR dimension access is supported")
+
+        try:
+            obs_selector, var_selector = self._filter_to_mask(filter)
+        except (KeyError, IndexError, TypeError, AttributeError):
+            raise FilterError("Error parsing filter")
+
+        if obs_selector is not None:
+            raise FilterError("filtering on obs unsupported")
+
+        num_columns = self.get_shape()[1] if var_selector is None else np.count_nonzero(var_selector)
+        if self.server_config.exceeds_limit("column_request_max", num_columns):
+            raise ExceedsLimitError("Requested dataframe columns exceed column request limit")
+
+        X = self.get_X_array(obs_selector, var_selector)
+        col_idx = np.nonzero([] if var_selector is None else var_selector)[0]
+        return encode_matrix_fbs(X, col_idx=col_idx, row_idx=None)
+
+    def diffexp_topN(self, obsFilterA, obsFilterB, top_n=None):
+        """
+        Computes the top N differentially expressed variables between two observation sets. If mode
+        is "TOP_N", then stats for the top N
+        dataframes
+        contain a subset of variables, then statistics for all variables will be returned, otherwise
+        only the top N vars will be returned.
+        :param obsFilterA: filter: dictionary with filter params for first set of observations
+        :param obsFilterB: filter: dictionary with filter params for second set of observations
+        :param top_n: Limit results to top N (Top var mode only)
+        :return: top N genes and corresponding stats
+        """
+        if Axis.VAR in obsFilterA or Axis.VAR in obsFilterB:
+            raise FilterError("Observation filters may not contain variable conditions")
+        try:
+            shape = self.get_shape()
+            obs_mask_A = self._axis_filter_to_mask(Axis.OBS, obsFilterA["obs"], shape[0])
+            obs_mask_B = self._axis_filter_to_mask(Axis.OBS, obsFilterB["obs"], shape[0])
+        except (KeyError, IndexError):
+            raise FilterError("Error parsing filter")
+        if top_n is None:
+            top_n = self.dataset_config.diffexp__top_n
+
+        if self.server_config.exceeds_limit(
+            "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B)
+        ):
+            raise ExceedsLimitError("Diffexp request exceeds max cell count limit")
+
+        result = self.compute_diffexp_ttest(obs_mask_A, obs_mask_B, top_n, self.dataset_config.diffexp__lfc_cutoff)
+
+        try:
+            return jsonify_numpy(result)
+        except ValueError:
+            raise JSONEncodingValueError("Error encoding differential expression to JSON")
+
+    @abstractmethod
+    def compute_diffexp_ttest(self, maskA, maskB, top_n, lfc_cutoff):
+        pass
+
+    @staticmethod
+    def normalize_embedding(embedding):
+        """Normalize embedding layout to meet client assumptions.
+        Embedding is an ndarray, shape (n_obs, n)., where n is normally 2
+        """
+
+        # scale isotropically
+        try:
+            min = np.nanmin(embedding, axis=0)
+            max = np.nanmax(embedding, axis=0)
+        except RuntimeError:
+            # indicates entire array was NaN, which should propagate
+            min = np.NaN
+            max = np.NaN
+
+        scale = np.amax(max - min)
+        normalized_layout = (embedding - min) / scale
+
+        # translate to center on both axis
+        translate = 0.5 - ((max - min) / scale / 2)
+        normalized_layout = normalized_layout + translate
+
+        normalized_layout = normalized_layout.astype(dtype=np.float32)
+        return normalized_layout
+
+    def layout_to_fbs_matrix(self, fields):
+        """
+        return specified embeddings as a flatbuffer, using the cellxgene matrix fbs encoding.
+
+        * returns only first two dimensions, with name {ename}_0 and {ename}_1,
+          where {ename} is the embedding name.
+        * client assumes each will be individually centered & scaled (isotropically)
+          to a [0, 1] range.
+        * does not support filtering
+
+        """
+        embeddings = self.get_embedding_names() if fields is None or len(fields) == 0 else fields
+        layout_data = []
+        with ServerTiming.time("layout.query"):
+            for ename in embeddings:
+                embedding = self.get_embedding_array(ename, 2)
+                normalized_layout = DataAdaptor.normalize_embedding(embedding)
+                layout_data.append(pd.DataFrame(normalized_layout, columns=[f"{ename}_0", f"{ename}_1"]))
+
+        with ServerTiming.time("layout.encode"):
+            if layout_data:
+                df = pd.concat(layout_data, axis=1, copy=False)
+            else:
+                df = pd.DataFrame()
+            fbs = encode_matrix_fbs(df, col_idx=df.columns, row_idx=None)
+
+        return fbs
+
+    def get_last_mod_time(self):
+        try:
+            lastmod = self.get_data_locator().lastmodtime()
+        except RuntimeError:
+            lastmod = None
+        return lastmod

local_server/data_common/fbs/NetEncoding/Column.py

@@ -0,0 +1,41 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: NetEncoding
+
+import flatbuffers
+
+class Column(object):
+    __slots__ = ['_tab']
+
+    @classmethod
+    def GetRootAsColumn(cls, buf, offset):
+        n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
+        x = Column()
+        x.Init(buf, n + offset)
+        return x
+
+    # Column
+    def Init(self, buf, pos):
+        self._tab = flatbuffers.table.Table(buf, pos)
+
+    # Column
+    def UType(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos)
+        return 0
+
+    # Column
+    def U(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6))
+        if o != 0:
+            from flatbuffers.table import Table
+            obj = Table(bytearray(), 0)
+            self._tab.Union(obj, o)
+            return obj
+        return None
+
+def ColumnStart(builder): builder.StartObject(2)
+def ColumnAddUType(builder, uType): builder.PrependUint8Slot(0, uType, 0)
+def ColumnAddU(builder, u): builder.PrependUOffsetTRelativeSlot(1, flatbuffers.number_types.UOffsetTFlags.py_type(u), 0)
+def ColumnEnd(builder): return builder.EndObject()

local_server/data_common/fbs/NetEncoding/Float32Array.py

@@ -0,0 +1,46 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: NetEncoding
+
+import flatbuffers
+
+class Float32Array(object):
+    __slots__ = ['_tab']
+
+    @classmethod
+    def GetRootAsFloat32Array(cls, buf, offset):
+        n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
+        x = Float32Array()
+        x.Init(buf, n + offset)
+        return x
+
+    # Float32Array
+    def Init(self, buf, pos):
+        self._tab = flatbuffers.table.Table(buf, pos)
+
+    # Float32Array
+    def Data(self, j):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            a = self._tab.Vector(o)
+            return self._tab.Get(flatbuffers.number_types.Float32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
+        return 0
+
+    # Float32Array
+    def DataAsNumpy(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Float32Flags, o)
+        return 0
+
+    # Float32Array
+    def DataLength(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.VectorLen(o)
+        return 0
+
+def Float32ArrayStart(builder): builder.StartObject(1)
+def Float32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
+def Float32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4)
+def Float32ArrayEnd(builder): return builder.EndObject()

local_server/data_common/fbs/NetEncoding/Float64Array.py

@@ -0,0 +1,46 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: NetEncoding
+
+import flatbuffers
+
+class Float64Array(object):
+    __slots__ = ['_tab']
+
+    @classmethod
+    def GetRootAsFloat64Array(cls, buf, offset):
+        n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
+        x = Float64Array()
+        x.Init(buf, n + offset)
+        return x
+
+    # Float64Array
+    def Init(self, buf, pos):
+        self._tab = flatbuffers.table.Table(buf, pos)
+
+    # Float64Array
+    def Data(self, j):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            a = self._tab.Vector(o)
+            return self._tab.Get(flatbuffers.number_types.Float64Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 8))
+        return 0
+
+    # Float64Array
+    def DataAsNumpy(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Float64Flags, o)
+        return 0
+
+    # Float64Array
+    def DataLength(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.VectorLen(o)
+        return 0
+
+def Float64ArrayStart(builder): builder.StartObject(1)
+def Float64ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
+def Float64ArrayStartDataVector(builder, numElems): return builder.StartVector(8, numElems, 8)
+def Float64ArrayEnd(builder): return builder.EndObject()

local_server/data_common/fbs/NetEncoding/Int32Array.py

@@ -0,0 +1,46 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: NetEncoding
+
+import flatbuffers
+
+class Int32Array(object):
+    __slots__ = ['_tab']
+
+    @classmethod
+    def GetRootAsInt32Array(cls, buf, offset):
+        n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
+        x = Int32Array()
+        x.Init(buf, n + offset)
+        return x
+
+    # Int32Array
+    def Init(self, buf, pos):
+        self._tab = flatbuffers.table.Table(buf, pos)
+
+    # Int32Array
+    def Data(self, j):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            a = self._tab.Vector(o)
+            return self._tab.Get(flatbuffers.number_types.Int32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
+        return 0
+
+    # Int32Array
+    def DataAsNumpy(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Int32Flags, o)
+        return 0
+
+    # Int32Array
+    def DataLength(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.VectorLen(o)
+        return 0
+
+def Int32ArrayStart(builder): builder.StartObject(1)
+def Int32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
+def Int32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4)
+def Int32ArrayEnd(builder): return builder.EndObject()

local_server/data_common/fbs/NetEncoding/JSONEncodedArray.py

@@ -0,0 +1,46 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: NetEncoding
+
+import flatbuffers
+
+class JSONEncodedArray(object):
+    __slots__ = ['_tab']
+
+    @classmethod
+    def GetRootAsJSONEncodedArray(cls, buf, offset):
+        n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
+        x = JSONEncodedArray()
+        x.Init(buf, n + offset)
+        return x
+
+    # JSONEncodedArray
+    def Init(self, buf, pos):
+        self._tab = flatbuffers.table.Table(buf, pos)
+
+    # JSONEncodedArray
+    def Data(self, j):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            a = self._tab.Vector(o)
+            return self._tab.Get(flatbuffers.number_types.Uint8Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 1))
+        return 0
+
+    # JSONEncodedArray
+    def DataAsNumpy(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Uint8Flags, o)
+        return 0
+
+    # JSONEncodedArray
+    def DataLength(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.VectorLen(o)
+        return 0
+
+def JSONEncodedArrayStart(builder): builder.StartObject(1)
+def JSONEncodedArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
+def JSONEncodedArrayStartDataVector(builder, numElems): return builder.StartVector(1, numElems, 1)
+def JSONEncodedArrayEnd(builder): return builder.EndObject()

local_server/data_common/fbs/NetEncoding/Matrix.py

@@ -0,0 +1,98 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: NetEncoding
+
+import flatbuffers
+
+class Matrix(object):
+    __slots__ = ['_tab']
+
+    @classmethod
+    def GetRootAsMatrix(cls, buf, offset):
+        n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
+        x = Matrix()
+        x.Init(buf, n + offset)
+        return x
+
+    # Matrix
+    def Init(self, buf, pos):
+        self._tab = flatbuffers.table.Table(buf, pos)
+
+    # Matrix
+    def NRows(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.Get(flatbuffers.number_types.Uint32Flags, o + self._tab.Pos)
+        return 0
+
+    # Matrix
+    def NCols(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6))
+        if o != 0:
+            return self._tab.Get(flatbuffers.number_types.Uint32Flags, o + self._tab.Pos)
+        return 0
+
+    # Matrix
+    def Columns(self, j):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8))
+        if o != 0:
+            x = self._tab.Vector(o)
+            x += flatbuffers.number_types.UOffsetTFlags.py_type(j) * 4
+            x = self._tab.Indirect(x)
+            from .Column import Column
+            obj = Column()
+            obj.Init(self._tab.Bytes, x)
+            return obj
+        return None
+
+    # Matrix
+    def ColumnsLength(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8))
+        if o != 0:
+            return self._tab.VectorLen(o)
+        return 0
+
+    # Matrix
+    def ColIndexType(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10))
+        if o != 0:
+            return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos)
+        return 0
+
+    # Matrix
+    def ColIndex(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12))
+        if o != 0:
+            from flatbuffers.table import Table
+            obj = Table(bytearray(), 0)
+            self._tab.Union(obj, o)
+            return obj
+        return None
+
+    # Matrix
+    def RowIndexType(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14))
+        if o != 0:
+            return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos)
+        return 0
+
+    # Matrix
+    def RowIndex(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(16))
+        if o != 0:
+            from flatbuffers.table import Table
+            obj = Table(bytearray(), 0)
+            self._tab.Union(obj, o)
+            return obj
+        return None
+
+def MatrixStart(builder): builder.StartObject(7)
+def MatrixAddNRows(builder, nRows): builder.PrependUint32Slot(0, nRows, 0)
+def MatrixAddNCols(builder, nCols): builder.PrependUint32Slot(1, nCols, 0)
+def MatrixAddColumns(builder, columns): builder.PrependUOffsetTRelativeSlot(2, flatbuffers.number_types.UOffsetTFlags.py_type(columns), 0)
+def MatrixStartColumnsVector(builder, numElems): return builder.StartVector(4, numElems, 4)
+def MatrixAddColIndexType(builder, colIndexType): builder.PrependUint8Slot(3, colIndexType, 0)
+def MatrixAddColIndex(builder, colIndex): builder.PrependUOffsetTRelativeSlot(4, flatbuffers.number_types.UOffsetTFlags.py_type(colIndex), 0)
+def MatrixAddRowIndexType(builder, rowIndexType): builder.PrependUint8Slot(5, rowIndexType, 0)
+def MatrixAddRowIndex(builder, rowIndex): builder.PrependUOffsetTRelativeSlot(6, flatbuffers.number_types.UOffsetTFlags.py_type(rowIndex), 0)
+def MatrixEnd(builder): return builder.EndObject()

local_server/data_common/fbs/NetEncoding/TypedArray.py

@@ -0,0 +1,12 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: NetEncoding
+
+class TypedArray(object):
+    NONE = 0
+    Float32Array = 1
+    Int32Array = 2
+    Uint32Array = 3
+    Float64Array = 4
+    JSONEncodedArray = 5
+

local_server/data_common/fbs/NetEncoding/Uint32Array.py

@@ -0,0 +1,46 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: NetEncoding
+
+import flatbuffers
+
+class Uint32Array(object):
+    __slots__ = ['_tab']
+
+    @classmethod
+    def GetRootAsUint32Array(cls, buf, offset):
+        n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
+        x = Uint32Array()
+        x.Init(buf, n + offset)
+        return x
+
+    # Uint32Array
+    def Init(self, buf, pos):
+        self._tab = flatbuffers.table.Table(buf, pos)
+
+    # Uint32Array
+    def Data(self, j):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            a = self._tab.Vector(o)
+            return self._tab.Get(flatbuffers.number_types.Uint32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
+        return 0
+
+    # Uint32Array
+    def DataAsNumpy(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Uint32Flags, o)
+        return 0
+
+    # Uint32Array
+    def DataLength(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.VectorLen(o)
+        return 0
+
+def Uint32ArrayStart(builder): builder.StartObject(1)
+def Uint32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
+def Uint32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4)
+def Uint32ArrayEnd(builder): return builder.EndObject()

local_server/data_common/fbs/matrix.py

@@ -0,0 +1,248 @@
+import json
+
+import numpy as np
+import pandas as pd
+from flatbuffers import Builder
+from scipy import sparse
+
+import local_server.data_common.fbs.NetEncoding.Column as Column
+import local_server.data_common.fbs.NetEncoding.Float32Array as Float32Array
+import local_server.data_common.fbs.NetEncoding.Float64Array as Float64Array
+import local_server.data_common.fbs.NetEncoding.Int32Array as Int32Array
+import local_server.data_common.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray
+import local_server.data_common.fbs.NetEncoding.Matrix as Matrix
+import local_server.data_common.fbs.NetEncoding.TypedArray as TypedArray
+import local_server.data_common.fbs.NetEncoding.Uint32Array as Uint32Array
+
+
+# Serialization helper
+def serialize_column(builder, typed_arr):
+    """ Serialize NetEncoding.Column """
+
+    (u_type, u_value) = typed_arr
+    Column.ColumnStart(builder)
+    Column.ColumnAddUType(builder, u_type)
+    Column.ColumnAddU(builder, u_value)
+    return Column.ColumnEnd(builder)
+
+
+# Serialization helper
+def serialize_matrix(builder, n_rows, n_cols, columns, col_idx):
+    """ Serialize NetEncoding.Matrix """
+
+    Matrix.MatrixStart(builder)
+    Matrix.MatrixAddNRows(builder, n_rows)
+    Matrix.MatrixAddNCols(builder, n_cols)
+    Matrix.MatrixAddColumns(builder, columns)
+    if col_idx is not None:
+        (u_type, u_val) = col_idx
+        Matrix.MatrixAddColIndexType(builder, u_type)
+        Matrix.MatrixAddColIndex(builder, u_val)
+    return Matrix.MatrixEnd(builder)
+
+
+# Serialization helper
+def serialize_typed_array(builder, source_array, encoding_info):
+    """
+    Serialize any of the various typed arrays, eg, Float32Array. Specific means of serialization and type conversion
+    are provided by type_info.
+    """
+
+    arr = source_array
+    (array_type, as_type) = encoding_info(source_array)
+
+    if isinstance(arr, pd.Index):
+        arr = arr.to_series()
+
+    # convert to a simple ndarray
+    if as_type == "json":
+        as_json = arr.to_json(orient="records")
+        arr = np.array(bytearray(as_json, "utf-8"))
+    else:
+        if sparse.issparse(arr):
+            arr = arr.toarray()
+        elif isinstance(arr, pd.Series):
+            arr = arr.to_numpy()
+        if arr.dtype != as_type:
+            arr = arr.astype(as_type)
+
+    # serialize the ndarray into a vector
+    if arr.ndim == 2:
+        if arr.shape[0] == 1:
+            arr = arr[0]
+        elif arr.shape[1] == 1:
+            arr = arr.T[0]
+
+    vec = builder.CreateNumpyVector(arr)
+
+    # serialize the typed array table
+    builder.StartObject(1)
+    builder.PrependUOffsetTRelativeSlot(0, vec, 0)
+    array_value = builder.EndObject()
+    return (array_type, array_value)
+
+
+def column_encoding(arr):
+    column_encoding_type_map = {
+        # array protocol string:  ( array_type, as_type )
+        np.dtype(np.float64).str: (TypedArray.TypedArray.Float64Array, np.float64),
+        np.dtype(np.float32).str: (TypedArray.TypedArray.Float32Array, np.float32),
+        np.dtype(np.float16).str: (TypedArray.TypedArray.Float32Array, np.float32),
+        np.dtype(np.int8).str: (TypedArray.TypedArray.Int32Array, np.int32),
+        np.dtype(np.int16).str: (TypedArray.TypedArray.Int32Array, np.int32),
+        np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32),
+        np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32),
+        np.dtype(np.uint8).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
+        np.dtype(np.uint16).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
+        np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
+        np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
+    }
+    column_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, "json")
+
+    return column_encoding_type_map.get(arr.dtype.str, column_encoding_default)
+
+
+def index_encoding(arr):
+    index_encoding_type_map = {
+        # array protocol string:  ( array_type, as_type )
+        np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32),
+        np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32),
+        np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
+        np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
+    }
+    index_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, "json")
+
+    return index_encoding_type_map.get(arr.dtype.str, index_encoding_default)
+
+
+def guess_at_mem_needed(matrix):
+    (n_rows, n_cols) = matrix.shape
+    if isinstance(matrix, np.ndarray) or sparse.issparse(matrix):
+        guess = (n_rows * n_cols * matrix.dtype.itemsize) + 1024
+    elif isinstance(matrix, pd.DataFrame):
+        # XXX TODO - DataFrame type estimate
+        guess = 1
+    else:
+        guess = 1
+
+    # round up to nearest 1024 bytes
+    guess = (guess + 0x400) & (~0x3FF)
+    return guess
+
+
+def encode_matrix_fbs(matrix, row_idx=None, col_idx=None):
+    """
+    Given a 2D DataFrame, ndarray or sparse equivalent, create and return a Matrix flatbuffer.
+
+    :param matrix: 2D DataFrame, ndarray or sparse equivalent
+    :param row_idx: index for row dimension, Index or ndarray
+    :param col_idx: index for col dimension, Index or ndarray
+
+    NOTE: row indices are (currently) unsupported and must be None
+    """
+
+    if row_idx is not None:
+        raise ValueError("row indexing not supported for FBS Matrix")
+    if matrix.ndim != 2:
+        raise ValueError("FBS Matrix must be 2D")
+
+    (n_rows, n_cols) = matrix.shape
+
+    # estimate size needed, so we don't unnecessarily realloc.
+    builder = Builder(guess_at_mem_needed(matrix))
+
+    columns = []
+    for cidx in range(n_cols - 1, -1, -1):
+        # serialize the typed array
+        col = matrix.iloc[:, cidx] if isinstance(matrix, pd.DataFrame) else matrix[:, cidx]
+        typed_arr = serialize_typed_array(builder, col, column_encoding)
+
+        # serialize the Column union
+        columns.append(serialize_column(builder, typed_arr))
+
+    # Serialize Matrix.columns[]
+    Matrix.MatrixStartColumnsVector(builder, n_cols)
+    for c in columns:
+        builder.PrependUOffsetTRelative(c)
+    matrix_column_vec = builder.EndVector(n_cols)
+
+    # serialize the colIndex if provided
+    cidx = None
+    if col_idx is not None:
+        cidx = serialize_typed_array(builder, col_idx, index_encoding)
+
+    # Serialize Matrix
+    matrix = serialize_matrix(builder, n_rows, n_cols, matrix_column_vec, cidx)
+
+    builder.Finish(matrix)
+    return builder.Output()
+
+
+def deserialize_typed_array(tarr):
+    type_map = {
+        TypedArray.TypedArray.NONE: None,
+        TypedArray.TypedArray.Uint32Array: Uint32Array.Uint32Array,
+        TypedArray.TypedArray.Int32Array: Int32Array.Int32Array,
+        TypedArray.TypedArray.Float32Array: Float32Array.Float32Array,
+        TypedArray.TypedArray.Float64Array: Float64Array.Float64Array,
+        TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray,
+    }
+    (u_type, u) = tarr
+    if u_type is TypedArray.TypedArray.NONE:
+        return None
+
+    TarType = type_map.get(u_type, None)
+    if TarType is None:
+        raise TypeError(f"FBS contains unknown data type: {u_type}")
+
+    arr = TarType()
+    arr.Init(u.Bytes, u.Pos)
+    narr = arr.DataAsNumpy()
+    if u_type == TypedArray.TypedArray.JSONEncodedArray:
+        narr = json.loads(narr.tostring().decode("utf-8"))
+    return narr
+
+
+def decode_matrix_fbs(fbs):
+    """
+    Given an FBS-encoded Matrix, return a Pandas DataFrame the contains the data and indices.
+    """
+
+    matrix = Matrix.Matrix.GetRootAsMatrix(fbs, 0)
+    n_rows = matrix.NRows()
+    n_cols = matrix.NCols()
+    if n_rows == 0 or n_cols == 0:
+        return pd.DataFrame()
+
+    if matrix.RowIndexType() is not TypedArray.TypedArray.NONE:
+        raise ValueError("row indexing not supported for FBS Matrix")
+
+    columns_length = matrix.ColumnsLength()
+
+    columns_index = deserialize_typed_array((matrix.ColIndexType(), matrix.ColIndex()))
+    if columns_index is None:
+        columns_index = range(0, n_cols)
+
+    # sanity checks
+    if len(columns_index) != n_cols or columns_length != n_cols:
+        raise ValueError("FBS column count does not match number of columns in underlying matrix")
+
+    columns_data = {}
+    columns_type = {}
+    for col_idx in range(0, columns_length):
+        col = matrix.Columns(col_idx)
+        tarr = (col.UType(), col.U())
+        data = deserialize_typed_array(tarr)
+        columns_data[columns_index[col_idx]] = data
+        if len(data) != n_rows:
+            raise ValueError("FBS column length does not match number of rows")
+        if col.UType() is TypedArray.TypedArray.JSONEncodedArray:
+            columns_type[columns_index[col_idx]] = "category"
+
+    df = pd.DataFrame.from_dict(data=columns_data).astype(columns_type, copy=False)
+
+    # more sanity checks
+    if not df.columns.is_unique or len(df.columns) != n_cols:
+        raise KeyError("FBS column indices are not unique")
+
+    return df

local_server/data_common/matrix_loader.py

@@ -0,0 +1,55 @@
+from enum import Enum
+from local_server.common.errors import DatasetAccessError
+from local_server.common.data_locator import DataLocator
+from http import HTTPStatus
+
+
+class MatrixDataType(Enum):
+    H5AD = "h5ad"
+    UNKNOWN = "unknown"
+
+
+class MatrixDataLoader(object):
+    def __init__(self, location, matrix_data_type=None, app_config=None):
+        """ location can be a string or DataLocator """
+        region_name = None if app_config is None else app_config.server_config.data_locator__s3__region_name
+        self.location = DataLocator(location, region_name=region_name)
+        if not self.location.exists():
+            raise DatasetAccessError("Dataset does not exist.", HTTPStatus.NOT_FOUND)
+
+        # matrix_data_type is an enum value of type MatrixDataType
+        self.matrix_data_type = matrix_data_type
+        # matrix_type is a DataAdaptor type, which corresonds to the matrix_data_type
+        self.matrix_type = None
+
+        if matrix_data_type is None:
+            self.matrix_data_type = self.__matrix_data_type()
+
+        if not self.__matrix_data_type_allowed(app_config):
+            raise DatasetAccessError("Dataset does not have an allowed type.")
+
+        if self.matrix_data_type == MatrixDataType.H5AD:
+            from local_server.data_anndata.anndata_adaptor import AnndataAdaptor
+
+            self.matrix_type = AnndataAdaptor
+
+    def __matrix_data_type(self):
+        if self.location.path.endswith(".h5ad"):
+            return MatrixDataType.H5AD
+        else:
+            return MatrixDataType.UNKNOWN
+
+    def __matrix_data_type_allowed(self, app_config):
+        return self.matrix_data_type != MatrixDataType.UNKNOWN
+
+    def pre_load_validation(self):
+        if self.matrix_data_type == MatrixDataType.UNKNOWN:
+            raise DatasetAccessError("Dataset does not have a recognized type: .h5ad")
+        self.matrix_type.pre_load_validation(self.location)
+
+    def file_size(self):
+        return self.matrix_type.file_size(self.location)
+
+    def open(self, app_config, dataset_config=None):
+        # create and return a DataAdaptor object
+        return self.matrix_type.open(self.location, app_config, dataset_config)

local_server/default_config.py

@@ -0,0 +1,138 @@
+import yaml
+
+default_config = """
+server:
+  app:
+    verbose: false
+    debug: false
+    host: localhost
+    port : null
+    open_browser: false
+    force_https: false
+    flask_secret_key: null
+
+  authentication:
+    # The authentication types may be "none" or "session"
+    # none:  No authentication support, features like user_annotations must not be enabled.
+    # session:  A session based userid is automatically generated. (no params needed)
+    type: session
+    insecure_test_environment: false
+
+  single_dataset:
+    # If datapath is set, then cellxgene with serve a single dataset located at datapath.
+    datapath: null
+    obs_names: null
+    var_names: null
+    about: null
+    title: null
+
+  data_locator:
+    s3:
+      # s3 region name.
+      #   if true, then the s3 location is automatically determined from the datapath or dataroot.
+      #   if false/null, then do not set.
+      #   if a string, then use that value (e.g. us-east-1).
+      region_name: true
+
+  adaptor:
+    anndata_adaptor:
+      backed: false
+
+  limits:
+    column_request_max: 32
+    diffexp_cellcount_max: null
+
+
+dataset:
+  app:
+    # Scripts can be a list of either file names (string) or dicts containing keys src, integrity and crossorigin.
+    # these will be injected into the index template as script tags with these attributes set.
+    scripts: []
+    # Inline scripts are a list of file names, where the contents of the file will be injected into the index.
+    inline_scripts: []
+
+    # allow authentication support
+    authentication_enable: true
+
+  presentation:
+    max_categories: 1000
+    custom_colors: true
+
+  user_annotations:
+    enable: true
+    type: local_file_csv
+    local_file_csv:
+      directory: null
+      file: null            # annotations file name
+      gene_sets_file: null  # gene sets file name
+    ontology:
+      enable: false
+      obo_location: null
+    gene_sets:
+      readonly: false       # gene sets CRUD enabled/disabled
+
+  embeddings:
+    names : []
+    enable_reembedding: false
+
+  diffexp:
+    enable: true
+    lfc_cutoff: 0.01
+    top_n: 10
+
+external:
+  # You can retrieve configuration parameters from this config file, the environment,
+  # the AWS secrets manager, or from the "cellxgene launch" command line arguments.
+  # They are applied in that order, meaning that if a parameter is defined in more
+  # than one location, the last one applied takes effect.
+
+  # environment variables:
+  # This section describes how to map environment variables to configuration parameters.
+  # The format is a list defining an environment variable.
+  # Each entry in the list is a dictionary with three entries:
+  # name:  the name of the environment variable
+  # path: the path within the cellxgene configuration to update.
+  # required: (default=False) a boolean.  If true, then it is an error if the environment variable is not set.
+
+  environment:
+     - name: CXG_SECRET_KEY
+       path: [server, app, flask_secret_key]
+       required: false
+
+  # AWS Secrets Manager
+  # This section describes how to map aws secrets to configuration parameters.
+  # The format is the region for the secrets manager, then a list of secrets.
+  # each secret has a name, and a list of values.
+  # Each entry in the list of values is a dictionary with three entries:
+  # key:  the key of the aws secret.
+  # path: the path within the cellxgene configuration to update.
+  # required: (default=False) a boolean.  If true, then it is an error if the key does not exist in the secret.
+  #
+  # example:
+  # aws_secrets_manager:
+  #   region: us-west-2
+  #    - name: my_first_secret
+  #      values:
+  #      - key: flask_secret_key
+  #        path: [server, app, flask_secret_key]
+  #        required: true
+  #      - key: db_uri
+  #        path: [dataset, user_annotations, db_uri]
+  #        required: true
+  #    - name:  my_auth_secret
+  #      values:
+  #      - key: client_secret
+  #        path: [server, authentication, client_secret]
+  #        required: true
+  #      - key: client_id
+  #        path: [server, authentication, client_id]
+  #        required: true
+
+  aws_secrets_manager:
+    region: null
+    secrets: []
+"""
+
+
+def get_default_config():
+    return yaml.load(default_config, Loader=yaml.Loader)

local_server/requirements-dev.txt

@@ -0,0 +1,10 @@
+Authlib>=0.14.3
+black
+bumpversion>=0.5
+codecov>=2.0.15
+parameterized>=0.7.0
+psycopg2-binary>=2.8.5
+pytest>=3.6.3
+python-jose>=3.2.0
+twine>=1.12.1
+-r requirements.txt