chore: add schema types (#2369)

* remove unused packages from client

* add missing peer dep

.bumpversion.cfg

@@ -1,5 +1,16 @@
 [bumpversion]
-current_version = 0.16.0
+current_version = 0.17.0
+commit = True
+parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?:-(?P<prerel>rc)\.(?P<prerelversion>\d+))?
+serialize = 
+	{major}.{minor}.{patch}-{prerel}.{prerelversion}
+	{major}.{minor}.{patch}
+
+[bumpversion:part:prerel]
+optional_value = release
+values = 
+	rc
+	release
 
 [bumpversion:file:setup.py]
 search = version="{current_version}"
@@ -9,6 +20,6 @@ replace = version="{new_version}"
 search = "version": "{current_version}"
 replace = "version": "{new_version}"
 
-[bumpversion:file:server/__init__.py]
+[bumpversion:file:backend/server/__init__.py]
 search = __version__ = "{current_version}"
 replace = __version__ = "{new_version}"

.dockerignore

@@ -2,4 +2,4 @@ bin
 client
 dist
 docs
-server
+backend

.github/workflows/canary-deploy.yml

@@ -0,0 +1,13 @@
+name: Deploy canary via single cell infra repo
+
+on:
+  push:
+    branches: main-canary
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: repository dispatch
+        run: |
+          curl -XPOST -u czi-sci-single-cell-eng:${{secrets.SCI_GITHUB_TOKEN}} -H "Accept: application/vnd.github.everest-preview+json"  -H "Content-Type: application/json" https://api.github.com/repos/chanzuckerberg/single-cell-infra/dispatches --data '{"event_type": "canary-hook"}'

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,67 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL Scan"
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ main ]
+  schedule:
+    - cron: '0 8 * * *'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'javascript', 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/compatibility_tests.yml

@@ -25,10 +25,11 @@ jobs:
   cellxgene-main-with-python-and-anndata-versions:
     name: python versions x anndata versions
     runs-on: ubuntu-latest
+    continue-on-error: true
     strategy:
       matrix:
         python-version: [3.6, 3.7, 3.8]
-        anndata-version: [0.6.22.post1, 0.7.1]
+        anndata-version: [0.7.6]
         test-suite: [smoke-test, smoke-test-annotations]
     steps:
     - uses: actions/checkout@v2
@@ -40,14 +41,12 @@ jobs:
       run: |
         # 1. only install the dev requirements on top of what is in the cellxgene pip package
         sudo apt-get update && sudo apt-get install -y libhdf5-serial-dev
-        sed -i 's/-r requirements.txt//' server/requirements-dev.txt
-        pip install -r server/requirements-dev.txt
+        sed -i 's/-r requirements.txt//' backend/server/requirements-dev.txt
+        pip install -r backend/server/requirements-dev.txt
         # 2. install cellxgene
         make pydist install-dist
         # 3. install anndata
         pip install anndata==${{ matrix.anndata-version }}
-        # workaround for anndata 0.6.22.post1 bug
-        [[ "0.6.22.post1" = "${{ matrix.anndata-version }}" ]] && pip install h5py==2.9.0 || true
     - name: Tests
       run: make unit-test ${{ matrix.test-suite }}
 
@@ -67,22 +66,17 @@ jobs:
         uses: actions/checkout@v2
         with:
           path: cellxgene
-      - name: Checkout tools repo
-        uses: actions/checkout@v2
-        with:
-          repository: theislab/anndata
-          path: anndata
       - name: Install dependencies
         run: |
           cd cellxgene
           # 1. only install the dev requirements on top of what is in the cellxgene pip package
           make dev-env-client
-          sed -i 's/-r requirements.txt//' server/requirements-dev.txt
-          pip install -r server/requirements-dev.txt
+          sed -i 's/-r requirements.txt//' backend/server/requirements-dev.txt
+          pip install -r backend/server/requirements-dev.txt
           # 2. install cellxgene
           pip install --upgrade cellxgene
           # 3. install anndata
-          cd ../anndata && pip install -e .
+          pip install git+https://github.com/theislab/anndata
       - name: Tests
         run: cd cellxgene && make unit-test ${{ matrix.test-suite }}
 
@@ -102,17 +96,11 @@ jobs:
         uses: actions/checkout@v2
         with:
           path: cellxgene
-      - name: Checkout tools repo
-        uses: actions/checkout@v2
-        with:
-          repository: theislab/anndata
-          path: anndata
       - name: Install dependencies
         run: |
           cd cellxgene
-          sed -i -E 's/^anndata[>=]=[0-9]+.[0-9]+.[0-9]+$/anndata/g' server/requirements.txt
+          sed -i -E 's/^anndata[>=]=[0-9]+.[0-9]+.[0-9]+$/anndata/g' backend/server/requirements.txt
           make pydist install-dist dev-env
-          cd ../anndata
-          pip install -e .
+          pip install git+https://github.com/theislab/anndata
       - name: Tests
         run: cd cellxgene && make unit-test ${{ matrix.test-suite }}

.github/workflows/push_tests.yml

@@ -31,15 +31,16 @@ jobs:
       - name: Install dependencies
         run: |
           pip install flake8
+          pip install black
           cd client
           npm install
-      - name: Lint with flake8
+      - name: Format with black and lint with flake8
         run: |
-          make lint-server
+          make lint-servers
       - name: Lint src with eslint
         working-directory: ./client
         run: |
-          make lint
+          npx eslint src __tests__
 
   unit-test:
     runs-on: ubuntu-latest
@@ -67,12 +68,12 @@ jobs:
         run: make pydist install-dist dev-env-server
       - name: Unit tests
         run: |
-          make unit-test
-          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k server -cF backend,python,unitTest
+          make unit-test-server
+          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k backend/server -cF backend,python,unitTest
           cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest
 
-  smoke-tests:
-    runs-on: macos-latest
+  unit-test-czi-hosted:
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python 3.7
@@ -94,7 +95,38 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-node-
       - name: Install dependencies
-        run: make pydist install-dist
+        run: make pydist-czi-hosted install-dist dev-env-czi-hosted
+      - name: Unit tests
+        run: |
+          make unit-test-czi-hosted
+          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k backend/czi-hosted -cF backend,python,unitTest
+          cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest
+
+  smoke-tests:
+    runs-on: macos-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Python cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Node cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Install dependencies
+        run: make pydist-czi-hosted install-dist
       - name: Smoke tests (without annotations feature)
         run: |
           cd client && make smoke-test
@@ -102,6 +134,7 @@ jobs:
 
   smoke-tests-annotations:
     runs-on: ubuntu-latest
+    timeout-minutes: 20
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python 3.7

.github/workflows/sastisfaction.yml

@@ -0,0 +1,27 @@
+name: Run SASTisfaction
+on:
+  - pull_request
+
+jobs:
+  sastisfaction:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/checkout@v2
+        with:
+          repository: chanzuckerberg/sastisfaction
+          ref: main
+          path: .github/actions/sastisfaction
+          ssh-key: ${{ secrets.SASTISFACTION_READ_KEY }}
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Docker pull
+        run: docker pull ghcr.io/chanzuckerberg/sastisfaction:main
+      - name: Run SASTisfaction
+        uses: ./.github/actions/sastisfaction
+        with:
+          snowflake_private_key: ${{ secrets.SASTISFACTION_RSA_KEY }}

.github/workflows/scale-test.yml

@@ -0,0 +1,30 @@
+name: "Scale test cellxgene APIs for initial loading"
+
+on:
+  schedule:
+    - cron: "0 0 * * Sun"
+
+jobs:
+  locust-build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Install dependencies
+        run: |
+          pip install -r backend/test/test_czi_hosted/locust/requirements-locust.txt
+      - name: Dev Scale Test
+        run: |
+          locust -f backend/test/test_czi_hosted/locust/locustfile.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt
+      - name: Slack success webhook
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+        run: |
+          DEV_STATS=$(tail -n 15 locust_dev_stats.txt)
+          DEV_MSG="\`\`\`CELLXGENE EXPLORER DEV SCALE TEST RESULTS: ${DEV_STATS}\`\`\`"
+          curl -X POST -H 'Content-type: application/json' --data "{'text':'${DEV_MSG}'}" $SLACK_WEBHOOK
+
+

.gitignore

@@ -19,9 +19,13 @@ venv/
 cellxgene/
 
 # client build
-server/common/web/static/*
-server/common/web/templates/
-server/common/web/csp-hashes.json
+backend/server/common/web/static/*
+backend/server/common/web/templates/
+backend/server/common/web/csp-hashes.json
+
+backend/czi_hosted/common/web/static/*
+backend/czi_hosted/common/web/templates/
+backend/czi_hosted/common/web/csp-hashes.json
 
 # eb build
 artifact.dir

Dockerfile

@@ -4,7 +4,8 @@ ENV LC_ALL=C.UTF-8
 ENV LANG=C.UTF-8
 
 RUN apt-get update && \
-    apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests && \
+    apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests python3-aiohttp && \
+    python3 -m pip install --upgrade pip && \
     pip3 install cellxgene
 
 ENTRYPOINT ["cellxgene"]

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2013
+Copyright (c) 2017-2021 Chan Zuckerberg Initiative
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in
@@ -17,4 +17,4 @@ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
 FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
 COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

MANIFEST.in

@@ -1,5 +1,7 @@
-recursive-include server/common/web/templates *
-recursive-include server/common/web/static *
+recursive-include backend/server/common/web/templates *
+recursive-include backend/server/common/web/static *
 
-include server/requirements.txt
-include server/requirements-prepare.txt
+include backend/server/requirements.txt
+include backend/server/requirements-prepare.txt
+include backend/server/converters/schema/hgnc_complete_set.txt.gz
+include backend/server/converters/schema/schema_definitions/*

MANIFEST_hosted.in

@@ -0,0 +1,7 @@
+recursive-include backend/czi_hosted/common/web/templates *
+recursive-include backend/czi_hosted/common/web/static *
+
+include backend/czi_hosted/requirements.txt
+include backend/czi_hosted/requirements-prepare.txt
+include backend/czi_hosted/converters/schema/hgnc_complete_set.txt.gz
+include backend/czi_hosted/converters/schema/schema_definitions/*

Makefile

@@ -2,23 +2,32 @@ include common.mk
 
 BUILDDIR := build
 CLIENTBUILD := $(BUILDDIR)/client
-SERVERBUILD := $(BUILDDIR)/server
+CZIHOSTEDBUILD := $(BUILDDIR)/backend/czi_hosted
+SERVERBUILD := $(BUILDDIR)/backend/server
 CLEANFILES :=  $(BUILDDIR)/ client/build build dist cellxgene.egg-info
 
 PART ?= patch
 
 # CLEANING
 .PHONY: clean
-clean: clean-lite clean-server clean-client
+clean: clean-lite clean-czi-hosted clean-server clean-client
 
 # cleaning the client's node_modules is the longest one, so we avoid that if possible
 .PHONY: clean-lite
 clean-lite:
 	rm -rf $(CLEANFILES)
 
-clean-%:
-	cd $(*) && $(MAKE) clean
+.PHONY: clean-client
+clean-client:
+	cd client && $(MAKE) clean
 
+.PHONY: clean-server
+clean-server:
+	cd backend/server && $(MAKE) clean
+
+.PHONY: clean-czi-hosted
+clean-czi-hosted:
+	cd backend/czi_hosted && $(MAKE) clean
 
 # BUILDING PACKAGE
 
@@ -28,29 +37,71 @@ build-client:
 
 .PHONY: build
 build: clean build-client
-	git ls-files server/ | grep -v 'server/test/' | cpio -pdm $(BUILDDIR)
+	git ls-files backend/server/ | grep -v 'backend/server/test/' | cpio -pdm $(BUILDDIR)
 	cp -r client/build/  $(CLIENTBUILD)
 	$(call copy_client_assets,$(CLIENTBUILD),$(SERVERBUILD))
+	cp backend/__init__.py $(BUILDDIR)
+	cp backend/__init__.py $(BUILDDIR)/backend
+	cp -r backend/common $(BUILDDIR)/backend/common
 	cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR)
 
+.PHONY: build-czi-hosted
+build-czi-hosted: clean build-client
+	git ls-files backend/czi_hosted/ | grep -v 'backend/czi_hosted/test/' | cpio -pdm $(BUILDDIR)
+	cp -r client/build/  $(CLIENTBUILD)
+	$(call copy_client_assets,$(CLIENTBUILD),$(CZIHOSTEDBUILD))
+	cp -r backend/common $(BUILDDIR)/backend/common
+	cp backend/__init__.py $(BUILDDIR)
+	cp backend/__init__.py $(BUILDDIR)/backend
+	cp MANIFEST_hosted.in README.md setup.cfg setup_hosted.py $(BUILDDIR)
+	mv $(BUILDDIR)/setup_hosted.py $(BUILDDIR)/setup.py
+	mv $(BUILDDIR)/MANIFEST_hosted.in $(BUILDDIR)/MANIFEST.in
+
 # If you are actively developing in the server folder use this, dirties the source tree
 .PHONY: build-for-server-dev
 build-for-server-dev: clean-server build-client
-	$(call copy_client_assets,client/build,server)
+	$(call copy_client_assets,client/build,backend/server)
+
+.PHONY: build-for-czi-hosted-dev
+build-for-czi-hosted-dev: clean-czi-hosted build-client
+	$(call copy_client_assets,client/build,backend/czi_hosted)
 
 .PHONY: copy-client-assets
 copy-client-assets:
-	$(call copy_client_assets,client/build,server)
+	$(call copy_client_assets,client/build,backend/server)
+
+.PHONY: copy-client-assets-czi-hosted
+copy-client-assets-czi-hosted:
+	$(call copy_client_assets,client/build,backend/czi_hosted)
 
 # TESTING
 .PHONY: test
 test: unit-test smoke-test
 
 .PHONY: unit-test
-unit-test: unit-test-server unit-test-client
+unit-test: unit-test-server unit-test-client unit-test-common
 
-unit-test-%:
-	cd $(*) && $(MAKE) unit-test
+.PHONY: test-server
+test-server: unit-test-server smoke-test
+
+.PHONY: test-czi-hosted
+test-czi-hosted: unit-test-czi-hosted smoke-test
+
+.PHONY: unit-test-client
+unit-test-client:
+	cd client && $(MAKE) unit-test
+
+.PHONY: unit-test-czi-hosted
+unit-test-czi-hosted:
+	cd backend/czi_hosted && $(MAKE) unit-test
+
+.PHONY: unit-test-server
+unit-test-server:
+	cd backend/server && $(MAKE) unit-test
+
+.PHONY: unit-test-common
+unit-test-common:
+	cd backend/common && $(MAKE) unit-test
 
 .PHONY: smoke-test
 smoke-test:
@@ -62,12 +113,11 @@ smoke-test-annotations:
 
 .PHONY: test-db
 test-db:
-	cd server && $(MAKE) test-db
-
+	cd backend/czi_hosted && $(MAKE) test-db
 
 # FORMATTING CODE
 
-.PHOHY: fmt
+.PHONY: fmt
 fmt: fmt-client fmt-py
 
 .PHONY: fmt-client
@@ -79,11 +129,18 @@ fmt-py:
 	black .
 
 .PHONY: lint
-lint: lint-server lint-client
+lint: lint-servers lint-client
+
+.PHONY: lint-servers
+lint-servers: lint-server lint-czi-hosted-server
 
 .PHONY: lint-server
-lint-server:
-	flake8 server
+lint-server: fmt-py
+	flake8 backend/server --per-file-ignores='backend/test/fixtures/dataset_config_outline.py:F821 backend/test/fixtures/server_config_outline.py:F821 backend/server/test/performance/scale_test_annotations.py:E501'
+
+.PHONY: lint-czi-hosted-server
+lint-czi-hosted-server: fmt-py
+	flake8 backend/czi_hosted --per-file-ignores='backend/test/fixtures/czi_hosted_dataset_config_outline.py:F821 backend/test/fixtures/czi_hosted_server_config_outline.py:F821 backend/test/performance/scale_test_annotations.py:E501'
 
 .PHONY: lint-client
 lint-client:
@@ -96,25 +153,40 @@ pydist: build
 	cd $(BUILDDIR); python setup.py sdist -d ../dist
 	@echo "done"
 
+.PHONY: pydist-czi-hosted
+pydist-czi-hosted: build-czi-hosted
+	cd $(BUILDDIR); python setup.py sdist -d ../dist
+	@echo "done"
+
 
 # RELEASE HELPERS
 
-# create new version to commit to main
-.PHONY: release-stage-1
-release-stage-1: dev-env bump clean-lite gen-package-lock
+# Create new version to commit to main
+.PHONY: create-release-candidate
+create-release-candidate: dev-env bump-version clean-lite gen-package-lock
 	@echo "Version bumped part:$(PART) and client built. Ready to commit and push"
 
-# build dist and release to dev pypi
-.PHONY: release-stage-2
-release-stage-2: dev-env pydist twine
+# Bump the release candidate version if needed (i.e. the previous release candidate had errors).
+.PHONY: recreate-release-candidate
+recreate-release-candidate: dev-env bump-release-candidate clean-lite gen-package-lock
+	@echo "Version bumped part:$(PART) and client built. Ready to commit and push"
+
+# Build dist and release to Test PyPI
+.PHONY: release-candidate-to-test-pypi
+release-candidate-to-test-pypi: dev-env pydist twine
 	@echo "Dist built and uploaded to test.pypi.org"
 	@echo "Test the install:"
 	@echo "    make install-release-test"
-	@echo "Then upload to Pypi prod:"
-	@echo "    make twine-prod"
 
-.PHONY: release-stage-final
-release-stage-final: twine-prod
+# Build final dist (gets rid of the rc tag) and release final candidate to TestPyPI
+.PHONY: release-final-to-test-pypi
+release-final-to-test-pypi: dev-env bump-release clean-lite gen-package-lock pydist twine
+	@echo "Final release dist built and uploaded to test.pypi.org"
+	@echo "Test the install:"
+	@echo "    make install-release-test"
+
+.PHONY: release-final
+release-final: twine-prod
 	@echo "Release uploaded to pypi.org"
 
 # DANGER: releases directly to prod
@@ -134,13 +206,27 @@ dev-env-client:
 
 .PHONY: dev-env-server
 dev-env-server:
-	pip install -r server/requirements-dev.txt
+	pip install -r backend/server/requirements-dev.txt
 
-# give PART=[major, minor, part] as param to make bump
-.PHONY: bump
-bump:
+.PHONY: dev-env-czi-hosted
+dev-env-czi-hosted:
+	pip install -r backend/czi_hosted/requirements-dev.txt
+# Set PART=[major, minor, patch] as param to make bump.
+# This will create a release candidate. (i.e. 0.16.1 -> 0.16.2-rc.0 for a patch bump)
+.PHONY: bump-version
+bump-version:
 	bumpversion --config-file .bumpversion.cfg $(PART)
 
+# Increments the release candidate version (i.e. 0.16.2-rc.1 -> 0.16.2-rc.2)
+.PHONY: bump-release-candidate
+bump-release-candidate:
+	bumpversion --config-file .bumpversion.cfg prerelversion --allow-dirty
+
+# Finalizes the release candidate by removing the release candidate tag (i.e. 0.16.2-rc.2 -> 0.16.2).
+.PHONY: bump-release
+bump-release:
+	bumpversion --config-file .bumpversion.cfg prerel --allow-dirty
+
 .PHONY: twine
 twine:
 	twine upload --repository-url https://test.pypi.org/legacy/ dist/*

PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,11 @@
+#### Reviewers
+**Functional:** 
+
+**Readability:** 
+
+---
+
+## Changes
+- add
+- remove
+- modify

Procfile

@@ -1 +0,0 @@
-web: gunicorn --chdir server/eb app:application --log-file -

README.md

@@ -52,7 +52,6 @@ cellxgene currently supports the following browsers:
 - Google Chrome 61+
 - Edge 15+
 - Firefox 60+
-- Safari 10.1+
 
 Please [file an issue](https://github.com/chanzuckerberg/cellxgene/issues/new/choose) if you would like us to add support for an unsupported browser.
 
@@ -67,37 +66,30 @@ For any errors, [report bugs on Github](https://github.com/chanzuckerberg/cellxg
 
 ### Contributing
 
-We warmly welcome contributions from the community! Please see our [contributing guide](https://chanzuckerberg.github.io/cellxgene/posts/contribute) and don't hesitate to open an issue or send a pull request to improve cellxgene.
+We warmly welcome contributions from the community! Please see our [contributing guide](https://chanzuckerberg.github.io/cellxgene/posts/contribute) and don't hesitate to open an issue or send a pull request to improve cellxgene. Please see the [dev_docs](https://github.com/chanzuckerberg/cellxgene/tree/main/dev_docs) for pull request suggestions, unit test details, local documentation preview, and other development specifics. 
 
 This project adheres to the Contributor Covenant [code of conduct](https://github.com/chanzuckerberg/.github/blob/master/CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to opensource@chanzuckerberg.com.
 
 ### Reuse
 
-This project was started with the sole goal of empowering the scientific community to explore and understand their data. As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from this project, and reach out to us with ideas or questions. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT).
+This project was started with the sole goal of empowering the scientific community to explore and understand their data. 
+As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from 
+this project. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT).
+
+
+Before extending cellxgene, we encourage you to reach out to us with ideas or questions. It might be possible that an 
+extension could be directly contributed, which would make it available for a wider audience, or that it's on our 
+[roadmap](./docs/posts/roadmap.md) and under active development. 
+
+See the [cellxgene extensions](./docs/posts/extensions.md) section of our documentation for examples of community use and cellxgene extensions. 
 
 ### Security
 
 If you believe you have found a security issue, we would appreciate notification. Please send email to <security@chanzuckerberg.com>.
 
-# About
+# Inspiration
 
-### Core team
-
-The current core team:
-
-- Colin Megill, frontend & product design
-- Bruce Martin, software engineer
-- Sidney Bell, computational biologist
-- Lia Prins, designer
-- Severiano Badajoz, software engineer
-
-We would also like to gratefully acknowledge contributions from past core team members:
-
-- Charlotte Weaver, software engineer
-
-### Inspiration
-
-We've been heavily inspired by several other related single-cell visualization projects, including the [UCSC Cell Browswer](http://cells.ucsc.edu/), [Cytoscape](http://www.cytoscape.org/), [Xena](https://xena.ucsc.edu/), [ASAP](https://asap.epfl.ch/), [Gene Pattern](http://genepattern-notebook.org/), and many others. We hope to explore collaborations where useful as this community works together on improving interactive visualization for single-cell data.
+We've been heavily inspired by several other related single-cell visualization projects, including the [UCSC Cell Browser](http://cells.ucsc.edu/), [Cytoscape](http://www.cytoscape.org/), [Xena](https://xena.ucsc.edu/), [ASAP](https://asap.epfl.ch/), [GenePattern](http://genepattern-notebook.org/), and many others. We hope to explore collaborations where useful as this community works together on improving interactive visualization for single-cell data.
 
 We were inspired by Mike Bostock and the [crossfilter](https://github.com/crossfilter) team for the design of our filtering implementation.
 

app.json

@@ -1,29 +0,0 @@
-{
-  "name": "cellxgene",
-  "description": "An interactive explorer for single-cell transcriptomics data",
-  "repository": "https://github.com/chanzuckerberg/cellxgene",
-  "logo": "https://cellxgene-example-data.czi.technology/favicon.png",
-  "keywords": [
-    "scientific",
-    "visualization",
-    "scrna-seq",
-    "transcriptomics",
-    "dataviz"
-  ],
-  "buildpacks": [
-    {
-      "url": "heroku/nodejs"
-    },
-    {
-      "url": "heroku/python"
-    }
-  ],
-  "stack": "heroku-18",
-  "env": {
-    "DATASET": {
-      "description": "Link to dataset",
-      "value": "https://cellxgene-example-data.czi.technology/pbmc3k.h5ad",
-      "required": "true"
-    }
-  }
-}

backend/common/Makefile

@@ -0,0 +1,11 @@
+
+.PHONY: unit-test
+unit-test:
+	PYTHONWARNINGS=ignore:ResourceWarning coverage run \
+		--source=fbs,utils \
+		--omit=.coverage,data_common/fbs/NetEncoding,venv \
+		-m unittest discover \
+		--start-directory ../test/test_common/unit \
+		--top-level-directory ../../ \
+		--verbose; test_result=$$?; \
+	exit $$test_result \

backend/common/colors.py

@@ -1,6 +1,6 @@
 import re
 
-from server.common.errors import ColorFormatException
+from backend.common.errors import ColorFormatException
 
 HEX_COLOR_FORMAT = re.compile("^#[a-fA-F0-9]{6,6}$")
 

backend/common/compute/diffexp_generic.py

@@ -1,5 +1,6 @@
 import numpy as np
 from scipy import sparse, stats
+from backend.common.constants import XApproximateDistribution
 
 
 def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
@@ -7,7 +8,7 @@ def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
     Return differential expression statistics for top N variables.
 
     Algorithm:
-    - compute log fold change (log2(meanA/meanB))
+    - compute fold change
     - compute Welch's t-test statistic and pvalue (w/ Bonferroni correction)
     - return top N abs(logfoldchange) where lfc > diffexp_lfc_cutoff
 
@@ -25,21 +26,25 @@ def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
     :param maskB: observation selection mask for set 2
     :param top_n: number of variables to return stats for
     :param diffexp_lfc_cutoff: minimum
-    :return:  for top N genes, [ varindex, logfoldchange, pval, pval_adj ]
+    absolute value returning [ varindex, logfoldchange, pval, pval_adj ] for top N genes
+    :return:  for top N genes, {"positive": for top N genes, [ varindex, foldchange, pval, pval_adj ], "negative": for top N genes, [ varindex, foldchange, pval, pval_adj ]}
     """
 
+    X_approximate_distribution = adaptor.get_X_approximate_distribution()
     dataA = adaptor.get_X_array(maskA, None)
     dataB = adaptor.get_X_array(maskB, None)
 
     # mean, variance, N - calculate for both selections
-    meanA, vA, nA = mean_var_n(dataA)
-    meanB, vB, nB = mean_var_n(dataB)
+    meanA, vA, nA = mean_var_n(dataA, X_approximate_distribution)
+    meanB, vB, nB = mean_var_n(dataB, X_approximate_distribution)
     res = diffexp_ttest_from_mean_var(meanA, vA, nA, meanB, vB, nB, top_n, diffexp_lfc_cutoff)
 
     return res
 
 
 def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp_lfc_cutoff):
+    # IMPORTANT NOTE: this code assumes the data is normally distributed and/or already logged.
+
     n_var = meanA.shape[0]
     top_n = min(top_n, n_var)
 
@@ -63,27 +68,30 @@ def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp
     pvals_adj = pvals * n_var
     pvals_adj[pvals_adj > 1] = 1  # cap adjusted p-value at 1
 
-    # logfoldchanges: log2(meanA / meanB)
-    logfoldchanges = np.log2(np.abs((meanA + 1e-9) / (meanB + 1e-9)))
+    # log fold change. The data is normally distributed/logged, so just subtract the means.
+    logfoldchanges = meanA - meanB
 
+    stats_to_sort = tscores
     # find all with lfc > cutoff
     lfc_above_cutoff_idx = np.nonzero(np.abs(logfoldchanges) > diffexp_lfc_cutoff)[0]
-    stats_to_sort = np.abs(tscores)
 
     # derive sort order
-    if lfc_above_cutoff_idx.shape[0] > top_n:
+    if lfc_above_cutoff_idx.shape[0] > top_n * 2:
         # partition top N
-        rel_t_partition = np.argpartition(stats_to_sort[lfc_above_cutoff_idx], -top_n)[-top_n:]
-        t_partition = lfc_above_cutoff_idx[rel_t_partition]
+        rel_t_partition = np.argpartition(stats_to_sort[lfc_above_cutoff_idx], (top_n, -top_n))
+        rel_t_partition_top_n = np.concatenate((rel_t_partition[-top_n:], rel_t_partition[:top_n]))
+        t_partition = lfc_above_cutoff_idx[rel_t_partition_top_n]
         # sort the top N partition
         rel_sort_order = np.argsort(stats_to_sort[t_partition])[::-1]
         sort_order = t_partition[rel_sort_order]
     else:
         # partition and sort top N, ignoring lfc cutoff
-        partition = np.argpartition(stats_to_sort, -top_n)[-top_n:]
-        rel_sort_order = np.argsort(stats_to_sort[partition])[::-1]
+        partition = np.argpartition(stats_to_sort, (top_n, -top_n))
+        partition_top_n = np.concatenate((partition[-top_n:], partition[:top_n]))
+
+        rel_sort_order = np.argsort(stats_to_sort[partition_top_n])[::-1]
         indices = np.indices(stats_to_sort.shape)[0]
-        sort_order = indices[partition][rel_sort_order]
+        sort_order = indices[partition_top_n][rel_sort_order]
 
     # top n slice based upon sort order
     logfoldchanges_top_n = logfoldchanges[sort_order]
@@ -91,12 +99,21 @@ def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp
     pvals_adj_top_n = pvals_adj[sort_order]
 
     # varIndex, logfoldchange, pval, pval_adj
-    result = [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in range(top_n)]
+    result = {
+        "positive": [
+            [sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in range(top_n)
+        ],
+        "negative": [
+            [sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]]
+            for i in range(-1, -1 - top_n, -1)
+        ],
+    }
+
     return result
 
 
 # Convenience function which handles sparse data
-def mean_var_n(X):
+def mean_var_n(X, X_approximate_distribution=XApproximateDistribution.NORMAL):
     """
     Two-pass variance calculation.  Numerically (more) stable
     than naive methods (and same method used by numpy.var())
@@ -114,16 +131,27 @@ def mean_var_n(X):
     with np.errstate(divide="call", invalid="call", call=fp_err_set):
         n = X.shape[0]
         if sparse.issparse(X):
+            if X_approximate_distribution == XApproximateDistribution.COUNT:
+                X = X.log1p()
             mean = X.mean(axis=0).A1
             dfm = X - mean
             sumsq = np.sum(np.multiply(dfm, dfm), axis=0).A1
             v = sumsq / (n - 1)
         else:
+            if X_approximate_distribution == XApproximateDistribution.COUNT:
+                X = np.log1p(X)
             mean = X.mean(axis=0)
             dfm = X - mean
             sumsq = np.sum(np.multiply(dfm, dfm), axis=0)
             v = sumsq / (n - 1)
 
+    # AnnData does not guarantee that operations on a view of X will
+    # return an ndarray, so force the cast if it wasn't done for us.
+    if type(mean) is not np.ndarray:
+        mean = mean.toarray()
+    if type(v) is not np.ndarray:
+        v = v.toarray()
+
     if fp_err_occurred:
         mean[np.isfinite(mean) == False] = 0  # noqa: E712
         v[np.isfinite(v) == False] = 0  # noqa: E712

backend/common/compute/estimate_distribution.py

@@ -0,0 +1,87 @@
+import numba
+import concurrent.futures
+import numpy as np
+from scipy import sparse
+from backend.common.constants import XApproximateDistribution
+
+
+@numba.njit(error_model="numpy", nogil=True)
+def min_max(arr: np.ndarray):
+    """Return (min, max) values for the ndarray."""
+
+    # initialize to first finite value in array.  Normally,
+    # this will exit on the first value.
+    for i in range(arr.size):
+        min_val = max_val = arr[i]
+        if np.isfinite(min_val):
+            break
+
+    # now find min/max, unrolled by two
+    odd = arr.size % 2
+    unrolled_loop_limit = arr.size - 1 if odd else arr.size
+    i = 0
+    while i < unrolled_loop_limit:
+        x = arr[i]
+        y = arr[i + 1]
+
+        # ignore non-finites
+        x = x if np.isfinite(x) else min_val
+        y = y if np.isfinite(y) else min_val
+
+        if x > y:
+            x, y = y, x
+        min_val = min(x, min_val)
+        max_val = max(y, max_val)
+        i += 2
+
+    # handle the tail if any
+    if odd:
+        x = arr[arr.size - 1]
+
+        # ignore non-finites
+        x = x if np.isfinite(x) else min_val
+
+        min_val = min(x, min_val)
+        max_val = max(x, max_val)
+
+    return min_val, max_val
+
+
+def estimate_approximate_distribution(X) -> XApproximateDistribution:
+    """
+    Estimate the distribution (normal, count) of the X matrix.
+
+    Currently this is based upon the assumption that scRNA-seq data is
+    exponentially distributed in its raw (count) form, and when logged,
+    any (max-min) range in excess of 24 is implies tens of millions of
+    observations of a single feature and so is extremely unlikely.
+    """
+    if X.dtype.kind not in ["i", "u", "f"]:
+        raise TypeError(f"Unsupported matrix dtype: {X.dtype.name}")
+
+    if X.size == 0:
+        # default for empty array
+        return XApproximateDistribution.NORMAL
+
+    if sparse.isspmatrix_csc(X) or sparse.isspmatrix_csr(X):
+        Xdata = X.data
+    elif type(X) is np.ndarray:
+        Xdata = X.reshape(
+            X.size,
+        )
+    else:
+        raise TypeError(f"Unsupported matrix format: {str(type(X))}")
+
+    CHUNKSIZE = 1 << 24
+    if Xdata.size > CHUNKSIZE:
+        min_val = max_val = Xdata[0]
+        with concurrent.futures.ThreadPoolExecutor() as tp:
+            for (_min, _max) in tp.map(min_max, [Xdata[i : i + CHUNKSIZE] for i in range(0, Xdata.size, CHUNKSIZE)]):
+                min_val = min(_min, min_val)
+                max_val = max(_max, max_val)
+
+    else:
+        min_val, max_val = min_max(Xdata)
+
+    excess_range = (max_val - min_val) > 24
+    return XApproximateDistribution.COUNT if excess_range else XApproximateDistribution.NORMAL

backend/common/constants.py

@@ -24,6 +24,11 @@ class DiffExpMode(AugmentedEnum):
     VAR_FILTER = "varFilter"
 
 
+class XApproximateDistribution(AugmentedEnum):
+    NORMAL = "normal"
+    COUNT = "count"
+
+
 JSON_NaN_to_num_warning_msg = "JSON encoding failure - please verify all data are finite values (no NaN or Infinities)"
 REACTIVE_LIMIT = 1_000_000
 

backend/common/errors.py

@@ -42,10 +42,17 @@ define_request_exception(
 define_request_exception("ExceedsLimitError", "Raised when an HTTP request exceeds a limit/quota")
 define_request_exception("ColorFormatException", "Raised when color helper functions encounter an unknown color format")
 define_request_exception(
-    "AuthenticationError",
-    "Raised when there is an authentication error",
-    default_status_code=HTTPStatus.UNAUTHORIZED)
+    "AuthenticationError", "Raised when there is an authentication error", default_status_code=HTTPStatus.UNAUTHORIZED
+)
+
+define_request_exception(
+    "AnnotationCategoryNameError",
+    "Raised when an annotation category name cant be saved",
+    default_status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
+)
 
-define_exception("OntologyLoadFailure", "Raised when reading the ontology file fails")
 define_exception("ConfigurationError", "Raised when checking configuration errors")
 define_exception("PrepareError", "Raised when data is misprepared")
+define_exception("SecretKeyRetrievalError", "Raised when get_secret_key from AWS fails")
+define_exception("ObsoleteRequest", "Raised when the request is no longer valid.")
+define_exception("UnsupportedSummaryMethod", "Raised when a gene set summary method is unknown or unsupported.")

backend/common/fbs/matrix.py

@@ -5,14 +5,16 @@ import pandas as pd
 from flatbuffers import Builder
 from scipy import sparse
 
-import server.data_common.fbs.NetEncoding.Column as Column
-import server.data_common.fbs.NetEncoding.Float32Array as Float32Array
-import server.data_common.fbs.NetEncoding.Float64Array as Float64Array
-import server.data_common.fbs.NetEncoding.Int32Array as Int32Array
-import server.data_common.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray
-import server.data_common.fbs.NetEncoding.Matrix as Matrix
-import server.data_common.fbs.NetEncoding.TypedArray as TypedArray
-import server.data_common.fbs.NetEncoding.Uint32Array as Uint32Array
+from backend.common.utils.type_conversion_utils import get_encoding_dtype_of_array
+
+import backend.common.fbs.NetEncoding.Column as Column
+import backend.common.fbs.NetEncoding.Float32Array as Float32Array
+import backend.common.fbs.NetEncoding.Float64Array as Float64Array
+import backend.common.fbs.NetEncoding.Int32Array as Int32Array
+import backend.common.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray
+import backend.common.fbs.NetEncoding.Matrix as Matrix
+import backend.common.fbs.NetEncoding.TypedArray as TypedArray
+import backend.common.fbs.NetEncoding.Uint32Array as Uint32Array
 
 
 # Serialization helper
@@ -99,7 +101,8 @@ def column_encoding(arr):
     }
     column_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, "json")
 
-    return column_encoding_type_map.get(arr.dtype.str, column_encoding_default)
+    encoding_dtype = np.dtype(get_encoding_dtype_of_array(arr))
+    return column_encoding_type_map.get(encoding_dtype.str, column_encoding_default)
 
 
 def index_encoding(arr):
@@ -199,7 +202,7 @@ def deserialize_typed_array(tarr):
     arr.Init(u.Bytes, u.Pos)
     narr = arr.DataAsNumpy()
     if u_type == TypedArray.TypedArray.JSONEncodedArray:
-        narr = json.loads(narr.tostring().decode("utf-8"))
+        narr = json.loads(narr.tobytes().decode("utf-8"))
     return narr
 
 

backend/common/genesets.py

@@ -0,0 +1,239 @@
+"""
+Utility code for gene sets handling
+"""
+
+import re
+import csv
+import hashlib
+
+from .errors import AnnotationsError
+
+
+GENESETS_TIDYCSV_HEADER = [
+    "gene_set_name",
+    "gene_set_description",
+    "gene_symbol",
+    "gene_description",
+]
+
+
+def read_gene_sets_tidycsv(gs_locator, context=None):
+    """
+    Read & parse the Tidy CSV format, applying validation checks for mandatory
+    values, and de-duping rules.
+
+    Format is a four-column CSV, with a mandatory header row, and optional "#" prefixed
+    comments.  Format:
+
+        gene_set_name, gene_set_description, gene_symbol, gene_description
+
+    gene_set_name must be non-null; others are optional.
+
+    Returns: a dictionary of the shape (values in angle-brackets vary):
+
+        {
+            <string, a gene set name>: {
+                "geneset_name": <string, a gene set name>,
+                "geneset_description": <a string or None>,
+                "genes": [
+                    {
+                        "gene_symbol": <string, a gene symbol or name>,
+                        "gene_description": <a string or None>
+                    },
+                    ...
+                ]
+            },
+            ...
+        }
+    """
+
+    class myDialect(csv.excel):
+        skipinitialspace = False
+
+    def just(n, seq):
+        it = iter(seq)
+        for _ in range(n - 1):
+            yield next(it, "")
+        yield tuple(it)
+
+    messagefn = context["messagefn"] if context else (lambda x: None)
+
+    gene_sets = {}
+    with gs_locator.local_handle() as fname:
+        with open(fname, newline="") as f:
+            reader = csv.reader(f, dialect=myDialect())
+            haveReadHeader = False
+            lineno = 0
+            for row in reader:
+                lineno += 1
+                # ignore empty rows
+                if len(row) == 0:
+                    continue
+                # if row starts with '#' it is a comment
+                if row[0].startswith("#"):
+                    continue
+                # if this is the first non-comment row, assume it is a header and validate
+                # column names.  OK if the user has extra columns after our initial set.
+                if not haveReadHeader:
+                    if row[0:len(GENESETS_TIDYCSV_HEADER)] != GENESETS_TIDYCSV_HEADER:
+                        raise AnnotationsError("Gene set CSV file missing the required column header.")
+                    haveReadHeader = True
+                    continue
+
+                geneset_name, geneset_description, gene_symbol, gene_description, _ = just(5, row)
+                if not geneset_name:
+                    raise AnnotationsError(f"Gene set CSV missing required gene set name on line {lineno}")
+                if (not gene_symbol) and gene_description:
+                    messagefn(f"Warning: Missing gene name in gene set name {geneset_name} on line {lineno}.")
+
+                if geneset_name in gene_sets:
+                    gs = gene_sets[geneset_name]
+                else:
+                    gs = gene_sets[geneset_name] = {
+                        "geneset_name": geneset_name,
+                        "geneset_description": geneset_description,
+                        "genes": [],
+                    }
+                # Use first geneset_description with a value
+                if not gs["geneset_description"] and geneset_description:
+                    gs["geneset_description"] = geneset_description
+                # add the gene if the gene_symbol is defined
+                if gene_symbol:
+                    gs["genes"].append(
+                        {
+                            "gene_symbol": gene_symbol,
+                            "gene_description": gene_description,
+                        }
+                    )
+
+    return gene_sets
+
+
+def write_gene_sets_tidycsv(f, genesets):
+    """
+    Convert the internal gene sets format (returned by read_gene_set_tidycsv) into
+    the simple Tidy CSV.
+    """
+    writer = csv.writer(f, dialect="excel")
+    writer.writerow(GENESETS_TIDYCSV_HEADER)
+    for geneset in genesets:
+        # genes may be empty, in which case we skip the gene set entirely
+        genes = geneset["genes"]
+        if not genes:
+            writer.writerow([geneset["geneset_name"], geneset.get("geneset_description", ""), "", ""])
+        else:
+            writer.writerows(
+                [
+                    [
+                        geneset["geneset_name"],
+                        geneset.get("geneset_description", ""),
+                        gene["gene_symbol"],
+                        gene.get("gene_description", ""),
+                    ]
+                    for gene in genes
+                ]
+            )
+
+
+def summarizeQueryHash(raw_query):
+    """ generate a cache key (hash) from the raw query string """
+    return hashlib.sha1(raw_query).hexdigest()
+
+
+def validate_gene_sets(genesets, var_names, context=None):
+    """
+    Check validity of gene sets, return if correct, else raise error.
+    May also modify the gene set for conditions that should be resolved,
+    but which do not warrant a hard error.
+
+    Argument gene sets may be either the REST OTA format (list of dicts) or the internal
+    format (dict of dicts, keyed by the gene set name).
+
+    Will return a modified gene sets (eg, remove warnings) of the same type as the
+    provided argument. Ie, dict->dict, list->list
+
+    Rules:
+
+    0. All gene set names must be unique. [error]
+    1. Gene set names must conform to the following: [error]
+        * Names must be comprised of 1 or more ASCII characters 32-126
+        * No leading or trailing spaces (ASCII 32)
+        * No multi-space (ASCII 32) runs
+    2. Gene symbols must be part of the current var_index. [warning]
+       If gene symbol is not in the var_index, generate a warning and remove the symbol
+       from the gene sets.
+    3. Gene symbols must not be duplicated in a gene set.  [warning]
+       Duplications will be silently de-duped.
+
+    Items marked [error] will generate a hard error, causing the validation to fail.
+
+    Items marked [warning] will generate a warning, and will be resolved without failing
+    the validation (typically by removing the offending item from the gene sets).
+    """
+
+    messagefn = context["messagefn"] if context else (lambda x: None)
+
+    # accept genesets args as either the internal (dict) or REST (list) format,
+    # as they are identical except for the dict being keyed by geneset_name.
+    if not isinstance(genesets, dict) and not isinstance(genesets, list):
+        raise ValueError("Gene sets must be either dict or list.")
+    genesets_iterable = genesets if isinstance(genesets, list) else genesets.values()
+
+    # 0. check for uniqueness of geneset names
+    geneset_names = [gs["geneset_name"] for gs in genesets_iterable]
+    if len(set(geneset_names)) != len(geneset_names):
+        raise KeyError("All gene set names must be unique.")
+
+    # 1. check gene set character set and format
+    illegal_name = re.compile(r"^\s|  |[\u0000-\u001F\u007F-\uFFFF]|\s$")
+    for name in geneset_names:
+        if type(name) != str or len(name) == 0:
+            raise KeyError("Gene set names must be non-null string.")
+        if illegal_name.search(name):
+            messagefn(
+                "Error: "
+                f"Gene set name {name} "
+                "is not valid. Leading, trailing, and multiple spaces within a name are not allowed."
+            )
+            raise KeyError(
+                "Gene set name is not valid. Leading, trailing, and multiple spaces within a name are not allowed."
+            )
+
+    # 2. & 3. check for duplicate gene symbols, and those not present in the dataset. They will
+    # generate a warning and be removed.
+    for geneset in genesets_iterable:
+        if not isinstance(geneset, dict):
+            raise ValueError("Each gene set must be a dict.")
+        geneset_name = geneset["geneset_name"]
+        genes = geneset["genes"]
+        if not isinstance(genes, list):
+            raise ValueError("Gene set genes field must be a list")
+        geneset.setdefault("geneset_description", "")
+        gene_symbol_already_seen = set()
+        new_genes = []
+        for gene in genes:
+            gene_symbol = gene["gene_symbol"]
+            if not isinstance(gene_symbol, str) or len(gene_symbol) == 0:
+                raise ValueError("Gene symbol must be non-null string.")
+            if gene_symbol in gene_symbol_already_seen:
+                # duplicate check
+                messagefn(
+                    f"Warning: a duplicate of gene {gene_symbol} was found in gene set {geneset_name}, "
+                    "and will be ignored."
+                )
+                continue
+
+            if gene_symbol not in var_names:
+                messagefn(
+                    f"Warning: {gene_symbol}, used in gene set {geneset_name}, "
+                    "was not found in the dataset and will be ignored."
+                )
+                continue
+
+            gene_symbol_already_seen.add(gene_symbol)
+            gene.setdefault("gene_description", "")
+            new_genes.append(gene)
+
+        geneset["genes"] = new_genes
+
+    return genesets

backend/common/utils/aws_secret_utils.py

@@ -0,0 +1,23 @@
+import logging
+
+import boto3
+from flask import json
+
+from backend.common.errors import SecretKeyRetrievalError
+
+
+def get_secret_key(region_name, secret_name):
+    session = boto3.session.Session()
+    client = session.client(service_name="secretsmanager", region_name=region_name)
+
+    try:
+        get_secret_value_response = client.get_secret_value(SecretId=secret_name)
+        if "SecretString" in get_secret_value_response:
+            var = get_secret_value_response["SecretString"]
+            secret = json.loads(var)
+            return secret
+    except Exception as e:
+        logging.critical(f"Caught exception during get_secret_key, {e}", exc_info=True)
+        raise SecretKeyRetrievalError(str(e))
+
+    return None

backend/common/utils/type_conversion_utils.py

@@ -0,0 +1,191 @@
+from typing import Union, Tuple
+import logging
+
+import numpy as np
+import pandas as pd
+
+"""
+These routines drive all type inference for the schema generation and the
+FBS (REST OTA) encoding. They are also used for CXG generation.
+
+
+H5AD Type                       REST              REST
+(ndarray, Series, Index)        FBS encoding      schema type       ERROR/exceptions
+----------------------------    --------------    ---------------   ----------------------
+bool_/bool                      uint8             boolean
+(u)int8, (u)int16, int32        int32             int32
+uint32, (u)int64                int32             int32             CHECKS value bounds
+float16, float32, float64       float32           float32[0]
+
+categorical[T is numeric[4]]:
+    hasna = False               T                 categorical[1]
+    hasna = True                float32           categorical[1]    CHECKS value bounds
+
+categorical[T not numeric]      JSON/str          categorical[1,2]
+
+(other object)                  JSON/str          string
+
+(all other)                                                         Always an ERROR[3]
+
+
+Notes:
+[0] IEEE format, includes non-finite numbers (NaN, Inf, ...)
+[1] with NO categories enumerated (client side does it to handle rounding)
+[2] NA (undefined) categories are assigned a JSON null value
+[3] Includes all other numpy types:  datetime, complex, etc.
+[4] means float, int, uint (dtype.kind in ['i','u','f'])
+
+"""
+
+
+def get_dtypes_and_schemas_of_dataframe(dataframe: pd.DataFrame):
+    dtypes_by_column_name = {}
+    schema_type_hints_by_column_name = {}
+
+    for column_name, column_values in dataframe.items():
+        (
+            dtypes_by_column_name[column_name],
+            schema_type_hints_by_column_name[column_name],
+        ) = get_dtype_and_schema_of_array(column_values)
+
+    return dtypes_by_column_name, schema_type_hints_by_column_name
+
+
+def get_encoding_dtype_of_array(array: Union[np.ndarray, pd.Series, pd.Index]) -> np.dtype:
+    return _get_type_info(array)[0]
+
+
+def get_schema_type_hint_of_array(array: Union[np.ndarray, pd.Series, pd.Index]) -> dict:
+    return _get_type_info(array)[1]
+
+
+def get_dtype_and_schema_of_array(array: Union[np.ndarray, pd.Series, pd.Index]) -> Tuple[np.dtype, dict]:
+    """Return tuple (encoding_dtype, schema_type_hint)"""
+    return _get_type_info(array)
+
+
+def get_schema_type_hint_from_dtype(dtype) -> dict:
+    res = _get_type_info_from_dtype(dtype)
+    if res is None:
+        raise TypeError(f"Annotations of type {dtype} are unsupported.")
+    else:
+        return res[1]
+
+
+def _get_type_info_from_dtype(dtype) -> Union[Tuple[np.dtype, dict], None]:
+    """
+    Best-effort to determine encoding type and schema hint from a dtype.
+    If this is not possible, or the type is unsupported, return None.
+
+    This should be a subset of the cases which are supported by
+    _get_type_info().  The latter should be preferred if the array (values)
+    are available for typing.
+    """
+    if dtype.kind == "b":
+        return (np.uint8, {"type": "boolean"})
+
+    if dtype.kind == "U":
+        return (np.dtype(str), {"type": "string"})
+
+    if dtype.kind in ["i", "u"]:
+        if np.can_cast(dtype, np.int32):
+            return (np.int32, {"type": "int32"})
+
+    if dtype.kind == "f":
+        _float64_warning(dtype)
+        return (np.float32, {"type": "float32"})
+
+    if dtype.kind == "O" and not dtype.name == "category":
+        return (np.dtype(str), {"type": "string"})
+
+    return None
+
+
+def _get_type_info(array: Union[np.ndarray, pd.Series, pd.Index]) -> Tuple[np.dtype, dict]:
+    """
+    Determine encoding type and schema hint from an array.  This allows more
+    flexible casting than may be possible by using just the dtype, as it can
+    account for category types and array values.
+    """
+    if (
+        not isinstance(array, np.ndarray)
+        and not isinstance(array, pd.Series)
+        and not isinstance(array, pd.Index)
+        and not hasattr(array, "dtype")
+    ):
+        raise TypeError("Unsupported data type.")
+
+    dtype = array.dtype
+
+    res = _get_type_info_from_dtype(dtype)
+    if res is not None:
+        return res
+
+    if dtype.kind == "O":
+        if dtype.name == "category":
+            # Sometimes CategoricalDType can be encoded as int or float without further fuss.
+            # Do not specify the categories in the schema - let the client-side figure it out
+            # on its own.  Utilize Series.to_numpy() to do casting that handles categorical
+            # NA/NaN (missing or undefined) categories.
+            if dtype.categories.dtype.kind in ["f", "i", "u"]:
+                return (
+                    _get_type_info(array.to_numpy())[0],
+                    {"type": "categorical"},
+                )
+            else:
+                return (np.dtype(str), {"type": "categorical", "categories": dtype.categories.to_list()})
+
+        # all other extension types are str-encoded
+        return (np.dtype(str), {"type": "string"})
+
+    if dtype.kind in ["i", "u"] and _can_cast_array_values_to_int32(array):
+        return (np.int32, {"type": "int32"})
+
+    if dtype.kind == "f":
+        _float64_warning(array.dtype)
+        return (np.float32, {"type": "float32"})
+
+    raise TypeError(f"Annotations of type {dtype} are unsupported.")
+
+
+def _float64_warning(dtype):
+    """
+    Warn the user if we are down-casting a float64 to float32, and may potentially lose information.
+    """
+    if dtype.kind == "f" and not np.can_cast(dtype, np.float32):
+        logging.warning(f"Type {dtype.name} will be converted to 32 bit float and may lose precision.")
+
+
+def _can_cast_array_values_to_int32(array: Union[np.ndarray, pd.Series, pd.Index]) -> bool:
+    """
+    Return true if the (U)INT array values can be safely cast to int32.  We allow size reducing
+    casts (ie, int64 to int32) if no actual values require the larger size (ie, actual values
+    can be represented by the smaller type).
+    """
+    assert array.dtype.kind in ["u", "i"]
+
+    if np.can_cast(array.dtype, np.int32):
+        return True
+
+    if array.size == 0:
+        return True
+
+    int32_machine_limits = np.iinfo(np.int32)
+    if array.min() >= int32_machine_limits.min and array.max() <= int32_machine_limits.max:
+        return True
+
+    return False
+
+
+def convert_string_to_value(value: str):
+    """convert a string to value with the most appropriate type"""
+    if value.lower() == "true":
+        return True
+    if value.lower() == "false":
+        return False
+    if value == "null":
+        return None
+    try:
+        return eval(value)
+    except:  # noqa E722
+        return value

backend/common/utils/utils.py

@@ -5,13 +5,12 @@ import logging
 import os
 import pkgutil
 import socket
-import warnings
-
-from flask import json
 from urllib.parse import urlsplit, urljoin
+
 import numpy as np
-import pandas as pd
-from server.common.errors import ConfigurationError
+from flask import json
+
+from backend.common.errors import ConfigurationError
 
 
 def find_available_port(host, port=5005):
@@ -94,61 +93,6 @@ def jsonify_numpy(data):
     return json.dumps(data, cls=Float32JSONEncoder, allow_nan=False)
 
 
-def dtype_to_schema(dtype):
-    schema = {}
-    if dtype == np.float32:
-        schema["type"] = "float32"
-    elif dtype == np.int32:
-        schema["type"] = "int32"
-    elif dtype == np.bool_:
-        schema["type"] = "boolean"
-    elif dtype == np.str:
-        schema["type"] = "string"
-    elif dtype == "category":
-        schema["type"] = "categorical"
-        schema["categories"] = dtype.categories.tolist()
-    else:
-        raise TypeError(f"Annotations of type {dtype} are unsupported.")
-    return schema
-
-
-def can_cast_to_float32(array):
-    if array.dtype.kind == "f":
-        if not np.can_cast(array.dtype, np.float32):
-            warnings.warn(f"Annotation {array.name} will be converted to 32 bit float and may lose precision.")
-        return True
-    return False
-
-
-def can_cast_to_int32(array):
-    if array.dtype.kind in ["i", "u"]:
-        if np.can_cast(array.dtype, np.int32):
-            return True
-        ii32 = np.iinfo(np.int32)
-        if array.min() >= ii32.min and array.max() <= ii32.max:
-            return True
-    return False
-
-
-def series_to_schema(array):
-    assert type(array) == pd.Series
-    try:
-        return dtype_to_schema(array.dtype)
-    except TypeError:
-        dtype = array.dtype
-        data_kind = dtype.kind
-        schema = {}
-        if can_cast_to_float32(array):
-            schema["type"] = "float32"
-        elif can_cast_to_int32(array):
-            schema["type"] = "int32"
-        elif data_kind == "O" and dtype == "object":
-            schema["type"] = "string"
-        else:
-            raise TypeError(f"Annotations of type {dtype} are unsupported.")
-        return schema
-
-
 def import_plugins(plugin_module):
     """
     Load optional plugin modules from server.common.plugins

backend/czi_hosted/Makefile

@@ -0,0 +1,49 @@
+include ../../common.mk
+
+.PHONY: clean
+clean:
+	rm -f common/web/templates/index.html
+	rm -rf common/web/static
+	rm -f common/web/csp-hashes.json
+
+.PHONY: unit-test
+unit-test: create-test-db
+	PYTHONWARNINGS=ignore:ResourceWarning coverage run \
+		--source=app,auth,cli,common,compute,converters,data_anndata,data_common,data_cxg,eb \
+		--omit=.coverage,venv \
+		-m unittest discover \
+		--start-directory ../test/test_czi_hosted/unit \
+		--top-level-directory ../.. \
+		--verbose; test_result=$$?; \
+	$(MAKE) clean-test-db; \
+	exit $$test_result \
+
+
+.PHONY: test-db
+test-db: create-test-db
+	PYTHONWARNINGS=ignore:ResourceWarning coverage run \
+		--source=db \
+		--omit=.coverage,venv \
+		-m unittest discover \
+		--start-directory ../test/test_czi_hosted/test_database \
+		--top-level-directory ../.. \
+		--verbose; test_result=$$?; \
+	$(MAKE) clean-test-db; \
+	exit $$test_result
+
+.PHONY: create-test-db
+create-test-db:
+	-docker run -d -p 5432:5432 --name test_db -e POSTGRES_PASSWORD=test_pw postgres
+
+.PHONY: clean-test-db
+clean-test-db:
+	-docker stop test_db
+	-docker rm test_db
+
+.PHONY: test-annotations-performance
+test-annotations-performance:
+	python ../test/test_czi_hosted/performance/performance_test_annotations_backend.py
+
+.PHONY: test-annotations-scale
+test-annotations-scale:
+	locust -f ../test/test_czi_hosted/performance/scale_test_annotations.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt

backend/czi_hosted/__init__.py

@@ -0,0 +1,15 @@
+import logging
+import sys
+from backend.common.utils.utils import import_plugins
+
+__version__ = "0.16.7"
+
+
+display_version = "cellxgene v" + __version__
+
+try:
+    import_plugins("backend.czi_hosted.plugins")
+except Exception as e:
+    # Make sure to exit in this case, as the server may not be configured as expected.
+    logging.critical(f"Error in import_plugins: {str(e)}")
+    sys.exit(1)

backend/czi_hosted/__main__.py

@@ -0,0 +1,14 @@
+# Work around bug https://github.com/pallets/werkzeug/issues/461
+if __package__ is None:
+    import sys
+    from pathlib import Path
+
+    PKG_PATH = Path(__file__).parent
+    sys.path.insert(0, str(PKG_PATH.parent))
+    import backend.czi_hosted # noqa F401
+    __package__ = PKG_PATH.name
+
+# Main thing
+from .cli.cli import cli  # noqa F402
+
+cli()

backend/czi_hosted/app/app.py

@@ -1,23 +1,33 @@
 import datetime
 import logging
+from functools import wraps
+from http import HTTPStatus
+from urllib.parse import urlparse
+import hashlib
+import os
 
-from flask import Flask, redirect, current_app, make_response, render_template, abort
-from flask import Blueprint, request
+from flask import (
+    Flask,
+    redirect,
+    current_app,
+    make_response,
+    render_template,
+    abort,
+    Blueprint,
+    request,
+    send_from_directory,
+)
 from flask_restful import Api, Resource
 from server_timing import Timing as ServerTiming
 
-from http import HTTPStatus
+import backend.czi_hosted.common.rest as common_rest
+from backend.common.utils.data_locator import DataLocator
+from backend.common.errors import DatasetAccessError, RequestException
+from backend.czi_hosted.common.health import health_check
+from backend.common.utils.utils import path_join, Float32JSONEncoder
+from backend.czi_hosted.data_common.matrix_loader import MatrixDataLoader
 
-import server.common.rest as common_rest
-from server.common.errors import DatasetAccessError, RequestException
-from server.common.utils import path_join, Float32JSONEncoder
-from server.common.data_locator import DataLocator
-from server.common.health import health_check
-from server.data_common.matrix_loader import MatrixDataLoader
-
-from functools import wraps
-
-webbp = Blueprint("webapp", "server.common.web", template_folder="templates")
+webbp = Blueprint("webapp", "backend.czi_hosted.common.web", template_folder="templates")
 
 ONE_WEEK = 7 * 24 * 60 * 60
 
@@ -86,23 +96,15 @@ def dataset_index(url_dataroot=None, dataset=None):
         cache_manager = current_app.matrix_data_cache_manager
         with cache_manager.data_adaptor(url_dataroot, location, app_config) as data_adaptor:
             data_adaptor.set_uri_path(f"{url_dataroot}/{dataset}")
-            dataset_title = app_config.get_title(data_adaptor)
-            return render_template(
-                "index.html", datasetTitle=dataset_title, SCRIPTS=scripts, INLINE_SCRIPTS=inline_scripts
-            )
+            args = {"SCRIPTS": scripts, "INLINE_SCRIPTS": inline_scripts}
+            return render_template("index.html", **args)
+
     except DatasetAccessError as e:
         return common_rest.abort_and_log(
             e.status_code, f"Invalid dataset {dataset}: {e.message}", loglevel=logging.INFO, include_exc_info=True
         )
 
 
-@webbp.route("/health", methods=["GET"])
-@cache_control_always(no_store=True)
-def health():
-    config = current_app.app_config
-    return health_check(config)
-
-
 @webbp.errorhandler(RequestException)
 def handle_request_exception(error):
     return common_rest.abort_and_log(error.status_code, error.message, loglevel=logging.INFO, include_exc_info=True)
@@ -166,7 +168,7 @@ def rest_get_data_adaptor(func):
     return wrapped_function
 
 
-def dataroot_test_index():
+def  dataroot_test_index():
     # the following index page is meant for testing/debugging purposes
     data = '<!doctype html><html lang="en">'
     data += "<head><title>Hosted Cellxgene</title></head>"
@@ -181,9 +183,9 @@ def dataroot_test_index():
             data += f"<p>Logged in as {auth.get_user_id()} / {auth.get_user_name()} / {auth.get_user_email()}</p>"
         if auth.requires_client_login():
             if server_config.auth.is_user_authenticated():
-                data += "<p><a href='/logout'>Logout</a></p>"
+                data += f"<p><a href='{auth.get_logout_url(None)}'>Logout</a></p>"
             else:
-                data += "<p><a href='/login'>Login</a></p>"
+                data += f"<p><a href='{auth.get_login_url(None)}'>Login</a></p>"
 
     datasets = []
     for dataroot_dict in server_config.multi_dataset__dataroot.values():
@@ -203,7 +205,7 @@ def dataroot_test_index():
     data += "<ul>"
     datasets.sort()
     for url_dataroot, dataset in datasets:
-        data += f"<li><a href={url_dataroot}/{dataset}>{dataset}</a></li>"
+        data += f"<li><a href={url_dataroot}/{dataset}/>{dataset}</a></li>"
     data += "</ul>"
     data += "</body></html>"
 
@@ -221,6 +223,13 @@ def dataroot_index():
         return redirect(config.server_config.multi_dataset__index)
 
 
+class HealthAPI(Resource):
+    @cache_control(no_store=True)
+    def get(self):
+        config = current_app.app_config
+        return health_check(config)
+
+
 class DatasetResource(Resource):
     """Base class for all Resources that act on datasets."""
 
@@ -230,6 +239,7 @@ class DatasetResource(Resource):
 
 
 class SchemaAPI(DatasetResource):
+    # TODO @mdunitz separate dataset schema and user schema
     @cache_control(public=True, max_age=ONE_WEEK)
     @rest_get_data_adaptor
     def get(self, data_adaptor):
@@ -243,6 +253,13 @@ class ConfigAPI(DatasetResource):
         return common_rest.config_get(current_app.app_config, data_adaptor)
 
 
+class UserInfoAPI(DatasetResource):
+    @cache_control_always(no_store=True)
+    @rest_get_data_adaptor
+    def get(self, data_adaptor):
+        return common_rest.userinfo_get(current_app.app_config, data_adaptor)
+
+
 class AnnotationsObsAPI(DatasetResource):
     @cache_control(public=True, max_age=ONE_WEEK)
     @rest_get_data_adaptor
@@ -295,14 +312,38 @@ class LayoutObsAPI(DatasetResource):
     def get(self, data_adaptor):
         return common_rest.layout_obs_get(request, data_adaptor)
 
-    @cache_control(no_store=True)
+
+class GenesetsAPI(DatasetResource):
+    @cache_control(public=True, max_age=ONE_WEEK)
     @rest_get_data_adaptor
-    def put(self, data_adaptor):
-        return common_rest.layout_obs_put(request, data_adaptor)
+    def get(self, data_adaptor):
+        return common_rest.genesets_get(request, data_adaptor)
 
 
-def get_api_resources(bp_api, url_dataroot=None):
-    api = Api(bp_api)
+class SummarizeVarAPI(DatasetResource):
+    @rest_get_data_adaptor
+    @cache_control(public=True, max_age=ONE_WEEK)
+    def get(self, data_adaptor):
+        return common_rest.summarize_var_get(request, data_adaptor)
+
+    @rest_get_data_adaptor
+    @cache_control(no_store=True)
+    def post(self, data_adaptor):
+        return common_rest.summarize_var_post(request, data_adaptor)
+
+
+def get_api_base_resources(bp_base):
+    """Add resources that are accessed from the api_base_url"""
+    api = Api(bp_base)
+
+    # Diagnostics routes
+    api.add_resource(HealthAPI, "/health")
+    return api
+
+
+def get_api_dataroot_resources(bp_dataroot, url_dataroot=None):
+    """Add resources that refer to a dataset"""
+    api = Api(bp_dataroot)
 
     def add_resource(resource, url):
         """convenience function to make the outer function less verbose"""
@@ -311,10 +352,13 @@ def get_api_resources(bp_api, url_dataroot=None):
     # Initialization routes
     add_resource(SchemaAPI, "/schema")
     add_resource(ConfigAPI, "/config")
+    add_resource(UserInfoAPI, "/userinfo")
     # Data routes
     add_resource(AnnotationsObsAPI, "/annotations/obs")
     add_resource(AnnotationsVarAPI, "/annotations/var")
     add_resource(DataVarAPI, "/data/var")
+    add_resource(GenesetsAPI, "/genesets")
+    add_resource(SummarizeVarAPI, "/summarize/var")
     # Display routes
     add_resource(ColorsAPI, "/colors")
     # Computation routes
@@ -323,6 +367,25 @@ def get_api_resources(bp_api, url_dataroot=None):
     return api
 
 
+def handle_api_base_url(app, app_config):
+    """If an api_base_url is provided, then an inline script is generated to
+    handle the new API prefix"""
+    api_base_url = app_config.server_config.get_api_base_url()
+    if not api_base_url:
+        return
+
+    sha256 = hashlib.sha256(api_base_url.encode()).hexdigest()
+    script_name = f"api_base_url-{sha256}.js"
+    script_path = os.path.join(app.root_path, "../common/web/templates", script_name)
+    with open(script_path, "w") as fout:
+        fout.write("window.CELLXGENE.API.prefix = `" + api_base_url + "${location.pathname}api/`;\n")
+
+    dataset_configs = [app_config.default_dataset_config] + list(app_config.dataroot_config.values())
+    for dataset_config in dataset_configs:
+        inline_scripts = dataset_config.app__inline_scripts
+        inline_scripts.append(script_name)
+
+
 class Server:
     @staticmethod
     def _before_adding_routes(app, app_config):
@@ -330,7 +393,8 @@ class Server:
         pass
 
     def __init__(self, app_config):
-        self.app = Flask(__name__, static_folder="../common/web/static")
+        self.app = Flask(__name__, static_folder=None)
+        handle_api_base_url(self.app, app_config)
         self._before_adding_routes(self.app, app_config)
         self.app.json_encoder = Float32JSONEncoder
         server_config = app_config.server_config
@@ -347,34 +411,65 @@ class Server:
         self.app.register_blueprint(webbp)
 
         api_version = "/api/v0.2"
+        api_base_url = server_config.get_api_base_url()
+        api_path = "/"
+        if api_base_url:
+            parse = urlparse(api_base_url)
+            api_path = parse.path
+
+        bp_base = Blueprint("bp_base", __name__, url_prefix=api_path)
+        base_resources = get_api_base_resources(bp_base)
+        self.app.register_blueprint(base_resources.blueprint)
+
         if app_config.is_multi_dataset():
             # NOTE:  These routes only allow the dataset to be in the directory
             # of the dataroot, and not a subdirectory.  We may want to change
             # the route format at some point
             for dataroot_dict in server_config.multi_dataset__dataroot.values():
                 url_dataroot = dataroot_dict["base_url"]
-                bp_api = Blueprint(
-                    f"api_dataset_{url_dataroot}", __name__, url_prefix=f"/{url_dataroot}/<dataset>" + api_version
+                bp_dataroot = Blueprint(
+                    f"api_dataset_{url_dataroot}",
+                    __name__,
+                    url_prefix=f"{api_path}/{url_dataroot}/<dataset>" + api_version,
                 )
-                resources = get_api_resources(bp_api, url_dataroot)
-                self.app.register_blueprint(resources.blueprint)
+                dataroot_resources = get_api_dataroot_resources(bp_dataroot, url_dataroot)
+                self.app.register_blueprint(dataroot_resources.blueprint)
+
                 self.app.add_url_rule(
-                    f"/{url_dataroot}/<dataset>/",
+                    f"/{url_dataroot}/<dataset>",
                     f"dataset_index_{url_dataroot}",
                     lambda dataset, url_dataroot=url_dataroot: dataset_index(url_dataroot, dataset),
                     methods=["GET"],
                 )
+                self.app.add_url_rule(
+                    f"/{url_dataroot}/<dataset>/",
+                    f"dataset_index_{url_dataroot}/",
+                    lambda dataset, url_dataroot=url_dataroot: dataset_index(url_dataroot, dataset),
+                    methods=["GET"],
+                )
+                self.app.add_url_rule(
+                    f"/{url_dataroot}/<dataset>/static/<path:filename>",
+                    f"static_assets_{url_dataroot}",
+                    view_func=lambda dataset, filename: send_from_directory("../common/web/static", filename),
+                    methods=["GET"],
+                )
 
         else:
-            bp_api = Blueprint("api", __name__, url_prefix=api_version)
-            resources = get_api_resources(bp_api)
+            bp_api = Blueprint("api", __name__, url_prefix=f"{api_path}{api_version}")
+            resources = get_api_dataroot_resources(bp_api)
             self.app.register_blueprint(resources.blueprint)
+            self.app.add_url_rule(
+                "/static/<path:filename>",
+                "static_assets",
+                view_func=lambda filename: send_from_directory("../common/web/static", filename),
+                methods=["GET"],
+            )
 
         self.app.matrix_data_cache_manager = server_config.matrix_data_cache_manager
         self.app.app_config = app_config
 
         auth = server_config.auth
         self.app.auth = auth
-        if auth.requires_client_login():
+        if auth and auth.requires_client_login():
             auth.add_url_rules(self.app)
         auth.complete_setup(self.app)

backend/czi_hosted/auth/__init__.py

@@ -0,0 +1,6 @@
+# import the built in auth types so they can be registered
+
+import backend.czi_hosted.auth.auth_test  # noqa: F401
+import backend.czi_hosted.auth.auth_session  # noqa: F401
+import backend.czi_hosted.auth.auth_oauth  # noqa: F401
+import backend.czi_hosted.auth.auth_none  # noqa: F401

backend/czi_hosted/auth/auth.py

@@ -43,6 +43,10 @@ class AuthTypeBase(ABC):
         """Return the name of the user (string)"""
         pass
 
+    def get_user_picture(self):
+        """Return the location to the user's picture"""
+        return None
+
 
 class AuthTypeClientBase(AuthTypeBase):
     """Base type for all authentication types that require the client to login"""
@@ -76,7 +80,7 @@ class AuthTypeFactory:
 
     @staticmethod
     def register(name, auth_type):
-        assert(issubclass(auth_type, AuthTypeBase))
+        assert issubclass(auth_type, AuthTypeBase)
         AuthTypeFactory.auth_types[name] = auth_type
 
     @staticmethod

backend/czi_hosted/auth/auth_none.py

@@ -0,0 +1,27 @@
+from backend.czi_hosted.auth.auth import AuthTypeBase, AuthTypeFactory
+
+
+class AuthTypeNone(AuthTypeBase):
+    def __init__(self, app_config):
+        super().__init__()
+
+    def is_valid_authentication_type(self):
+        return False
+
+    def complete_setup(self, app):
+        pass
+
+    def is_user_authenticated(self):
+        return True
+
+    def get_user_id(self):
+        return None
+
+    def get_user_name(self):
+        return None
+
+    def get_user_email(self):
+        return None
+
+
+AuthTypeFactory.register(None, AuthTypeNone)

backend/czi_hosted/auth/auth_oauth.py

@@ -0,0 +1,385 @@
+from flask import session, request, redirect, current_app, after_this_request, has_request_context, g
+from backend.czi_hosted.auth.auth import AuthTypeClientBase, AuthTypeFactory
+from backend.common.errors import AuthenticationError, ConfigurationError
+from urllib.parse import urlencode, urlparse
+import json
+import requests
+import base64
+
+# It is not required to have authlib or jose.
+# However, it is a configuration error to use this auth type if they are not installed.
+missingimport = []
+try:
+    from authlib.integrations.flask_client import OAuth
+except ModuleNotFoundError:
+    missingimport.append("authlib")
+
+try:
+    from jose import jwt
+    from jose.exceptions import ExpiredSignatureError, JWTError, JWTClaimsError
+except ModuleNotFoundError:
+    missingimport.append("jose")
+
+
+class Tokens:
+    """Simple class to represent the tokens that are saved/restored from the cookie"""
+
+    def __init__(self, access_token, id_token, refresh_token, expires_at, **kwargs):
+        self.access_token = access_token
+        self.id_token = id_token
+        self.refresh_token = refresh_token
+        self.expires_at = expires_at
+
+        # expires_at may be None after a token refresh, and so it is not checked here
+        if not (access_token and id_token and refresh_token):
+            raise KeyError(str(self.__dict__))
+
+
+class AuthTypeOAuth(AuthTypeClientBase):
+    """An authentication type for oauth2 logins."""
+
+    CXG_TOKENS = "auth_tokens"
+
+    def __init__(self, server_config):
+        super().__init__()
+        if missingimport:
+            raise ConfigurationError(f"oauth requires these modules: {', '.join(missingimport)}")
+        self.algorithms = ["RS256"]
+        self.oauth_api_base_url = server_config.authentication__params_oauth__oauth_api_base_url
+        self.client_id = server_config.authentication__params_oauth__client_id
+        self.client_secret = server_config.authentication__params_oauth__client_secret
+        self.session_cookie = server_config.authentication__params_oauth__session_cookie
+        self.cookie_params = server_config.authentication__params_oauth__cookie
+        self.jwt_decode_options = server_config.authentication__params_oauth__jwt_decode_options
+
+        self._validate_cookie_params()
+        self._validate_jwt_decode_options()
+
+        self.api_base_url = server_config.get_api_base_url()
+        self.web_base_url = server_config.get_web_base_url()
+        if self.api_base_url is None:
+            raise ConfigurationError("oauth requires the app__api_base_url to be set")
+
+        # set the audience
+        self.audience = self.client_id
+
+        # load the jwks (JSON Web Key Set).
+        # The JSON Web Key Set (JWKS) is a set of keys which contains the public keys used to verify
+        # any JSON Web Token (JWT) issued by the authorization server and signed using the RS256
+        try:
+            jwksloc = f"{self.oauth_api_base_url}/.well-known/jwks.json"
+            jwksurl = requests.get(jwksloc)
+            self.jwks = jwksurl.json()
+        except Exception:
+            raise ConfigurationError(
+                f"error in oauth, api_url_base: {self.oauth_api_base_url}, cannot access {jwksloc}"
+            )
+
+    def _validate_cookie_params(self):
+        """check the cookie_params, and raise a ConfigurationError if there is something wrong"""
+        if self.session_cookie:
+            return
+
+        if not isinstance(self.cookie_params, dict):
+            raise ConfigurationError("either session_cookie or cookie must be set")
+        valid_keys = {"key", "max_age", "expires", "path", "domain", "secure", "httponly", "samesite"}
+        keys = set(self.cookie_params.keys())
+        unknown = keys - valid_keys
+        if unknown:
+            raise ConfigurationError(f"unexpected key in cookie params: {', '.join(unknown)}")
+        if "key" not in keys:
+            raise ConfigurationError("must have a key (name) in the cookie params")
+
+    def _validate_jwt_decode_options(self):
+        """check the jwt_decode_options, and raise a ConfigurationError if there is something wrong"""
+        if self.jwt_decode_options is None:
+            self.jwt_decode_options = {}
+            return
+
+        valid_keys = {
+            "verify_signature",
+            "verify_aud",
+            "verify_iat",
+            "verify_exp",
+            "verify_nbf",
+            "verify_iss",
+            "verify_sub",
+            "verify_jti",
+            "verify_at_hash",
+            "leeway",
+        }
+        keys = set(self.jwt_decode_options.keys())
+        unknown = keys - valid_keys
+        if unknown:
+            raise ConfigurationError(f"unexpected key in jwt_decode_options: {', '.join(unknown)}")
+
+    def is_valid_authentication_type(self):
+        return True
+
+    def requires_client_login(self):
+        return True
+
+    def add_url_rules(self, app):
+        parse = urlparse(self.api_base_url)
+        app.add_url_rule(f"{parse.path}/login", "login", self.login, methods=["GET"])
+        app.add_url_rule(f"{parse.path}/logout", "logout", self.logout, methods=["GET"])
+        app.add_url_rule(f"{parse.path}/logout_redirect", "logout_redirect", self.logout_redirect, methods=["GET"])
+        app.add_url_rule(f"{parse.path}/oauth2/callback", "callback", self.callback, methods=["GET"])
+
+    def complete_setup(self, flask_app):
+        self.oauth = OAuth(flask_app)
+
+        self.client = self.oauth.register(
+            "auth0",
+            client_id=self.client_id,
+            client_secret=self.client_secret,
+            api_base_url=self.oauth_api_base_url,
+            refresh_token_url=f"{self.oauth_api_base_url}/oauth/token",
+            access_token_url=f"{self.oauth_api_base_url}/oauth/token",
+            authorize_url=f"{self.oauth_api_base_url}/authorize",
+            client_kwargs={"scope": "openid profile email offline_access"},
+        )
+
+    def is_user_authenticated(self):
+        payload = self.get_userinfo()
+        return payload is not None
+
+    def get_user_id(self):
+        payload = self.get_userinfo()
+        return payload.get("sub") if payload else None
+
+    def get_user_name(self):
+        payload = self.get_userinfo()
+        return payload.get("name") if payload else None
+
+    def get_user_email(self):
+        payload = self.get_userinfo()
+        return payload.get("email") if payload else None
+
+    def get_user_picture(self):
+        payload = self.get_userinfo()
+        return payload.get("picture") if payload else None
+
+    def update_response(self, response):
+        response.cache_control.update(dict(public=True, max_age=0, no_store=True, no_cache=True, must_revalidate=True))
+
+    def login(self):
+        callbackurl = f"{self.api_base_url}/oauth2/callback"
+        return_path = request.args.get("dataset", "")
+        return_to = f"{self.web_base_url}/{return_path}"
+        # save the return path in the session cookie, accessed in the callback function
+        session["oauth_callback_redirect"] = return_to
+        response = self.client.authorize_redirect(redirect_uri=callbackurl)
+        self.update_response(response)
+        return response
+
+    def logout(self):
+        """
+        We would like for the user to remain on the same dataset after logout.  oauth requires that
+        the redirect `returnTo` path be whitelisted by the oauth server, therefore a level of
+        indirection is used.  We first redirect to a single path "logout_redirect", and logout_redirect
+        will redirect the user's browser back to the current page.
+        """
+        self.remove_tokens()
+        redirect_path = request.args.get("dataset", "")
+        redirect_to = f"{self.web_base_url}/{redirect_path}"
+        session["oauth_logout_redirect"] = redirect_to
+
+        return_to = f"{self.api_base_url}/logout_redirect"
+        params = {"returnTo": return_to, "client_id": self.client_id}
+        response = redirect(self.client.api_base_url + "/v2/logout?" + urlencode(params))
+        self.update_response(response)
+        return response
+
+    def logout_redirect(self):
+        oauth_logout_redirect = session.pop("oauth_logout_redirect", "/")
+        response = redirect(oauth_logout_redirect)
+        self.update_response(response)
+        return response
+
+    def callback(self):
+        data = self.client.authorize_access_token()
+        tokens = Tokens(
+            access_token=data.get("access_token"),
+            id_token=data.get("id_token"),
+            refresh_token=data.get("refresh_token"),
+            expires_at=data.get("expires_at"),
+        )
+        self.save_tokens(tokens)
+        oauth_callback_redirect = session.pop("oauth_callback_redirect", "/")
+        response = redirect(oauth_callback_redirect)
+        self.update_response(response)
+        return response
+
+    def get_tokens(self):
+        """Extract the tokens from the cookie, and store them in the flask global context"""
+        if "tokens" in g:
+            return g.tokens
+
+        try:
+            if self.session_cookie:
+                value = session.get(self.CXG_TOKENS)
+                if value:
+                    g.tokens = Tokens(**value)
+                else:
+                    return None
+            else:
+                value = request.cookies.get(self.cookie_params["key"])
+                if value is None:
+                    return None
+                value = base64.b64decode(value)
+                value = json.loads(value)
+                g.tokens = Tokens(**value)
+
+        except Exception:
+            # there are many types of exceptions that can be raise in the above section.
+            # It is impractical to list all the exceptions here, since that would be brittle.
+            # If an exception occurs, then return None, meaning that no token could be retrieved.
+            current_app.logger.warning(f"auth cookie is in the wrong format: {str(value)}")
+            g.pop("tokens", None)
+            return None
+
+        return g.tokens
+
+    def save_tokens(self, tokens):
+        g.tokens = tokens
+        if self.session_cookie:
+            session[self.CXG_TOKENS] = tokens.__dict__
+        else:
+
+            @after_this_request
+            def set_cookie(response):
+                args = self.cookie_params.copy()
+                value = base64.b64encode(json.dumps(tokens.__dict__).encode("utf-8"))
+                del args["key"]
+                try:
+                    response.set_cookie(self.cookie_params["key"], value, **args)
+                except Exception as e:
+                    raise AuthenticationError(f"unable to set_cookie {self.cookie_params}") from e
+                return response
+
+    def remove_tokens(self):
+        g.pop("tokens", None)
+        if self.session_cookie:
+            if self.CXG_TOKENS in session:
+                del session[self.CXG_TOKENS]
+        else:
+
+            @after_this_request
+            def remove_cookie(response):
+                response.set_cookie(self.cookie_params["key"], "", expires=0)
+                self.update_response(response)
+                return response
+
+    def get_login_url(self, data_adaptor):
+        """Return the url for the login route"""
+        if data_adaptor and current_app.app_config.is_multi_dataset():
+            return f"{self.api_base_url}/login?dataset={data_adaptor.uri_path}/"
+        else:
+            return f"{self.api_base_url}/login"
+
+    def get_logout_url(self, data_adaptor):
+        """Return the url for the logout route"""
+        if data_adaptor and current_app.app_config.is_multi_dataset():
+            return f"{self.api_base_url}/logout?dataset={data_adaptor.uri_path}/"
+        else:
+            return f"{self.api_base_url}/logout"
+
+    def check_jwt_payload(self, id_token):
+        try:
+            unverified_header = jwt.get_unverified_header(id_token)
+        except JWTError:
+            return None
+
+        rsa_key = {}
+        for key in self.jwks["keys"]:
+            if key["kid"] == unverified_header["kid"]:
+                rsa_key = {
+                    "kty": key["kty"],
+                    "kid": key["kid"],
+                    "use": key["use"],
+                    "n": key.get("n"),
+                    "e": key.get("e"),
+                }
+        if rsa_key:
+            try:
+                payload = jwt.decode(
+                    id_token,
+                    rsa_key,
+                    algorithms=self.algorithms,
+                    audience=self.audience,
+                    issuer=self.oauth_api_base_url + "/",
+                    options=self.jwt_decode_options,
+                )
+                return payload
+
+            except ExpiredSignatureError:
+                # This exception is handled in get_userinfo
+                raise
+            except JWTClaimsError as e:
+                raise AuthenticationError(f"invalid claims {str(e)}") from e
+            except JWTError as e:
+                raise AuthenticationError(f"invalid signature: {str(e)}") from e
+
+        raise AuthenticationError("Unable to find the appropriate key")
+
+    def get_userinfo(self):
+        if not has_request_context():
+            return None
+
+        # check if the userinfo has been retrieved already in this request
+        if "userinfo" in g:
+            return g.get("userinfo")
+
+        # if there is no id_token, return None (user is not authenticated)
+        tokens = self.get_tokens()
+
+        if tokens is None or tokens.id_token is None:
+            return None
+
+        try:
+            # check the jwt payload.  This raises an AuthenticationError if the token is not valid.
+            # It the token has expired, we attempt to refresh the token
+            g.userinfo = self.check_jwt_payload(tokens.id_token)
+            return g.userinfo
+
+        except ExpiredSignatureError:
+            tokens = self.refresh_expired_token(tokens.refresh_token)
+            if tokens is None or tokens.id_token is None:
+                return None
+            else:
+                try:
+                    g.userinfo = self.check_jwt_payload(tokens.id_token)
+                    return g.userinfo
+                except JWTError as e:
+                    raise AuthenticationError(f"error during token refresh: {str(e)}") from e
+
+        except AuthenticationError:
+            self.remove_tokens()
+            raise
+
+    def refresh_expired_token(self, refresh_token):
+        params = {
+            "grant_type": "refresh_token",
+            "client_id": self.client_id,
+            "refresh_token": refresh_token,
+            "client_secret": self.client_secret,
+        }
+        headers = {"content-type": "application/x-www-form-urlencoded"}
+        request = requests.post(f"{self.oauth_api_base_url}/oauth/token", urlencode(params), headers=headers)
+        if request.status_code != 200:
+            # unable to refresh the token, log the user out
+            self.remove_tokens()
+            return None
+        data = request.json()
+        tokens = Tokens(
+            access_token=data.get("access_token"),
+            id_token=data.get("id_token"),
+            refresh_token=data.get("refresh_token", refresh_token),
+            expires_at=data.get("expires_at"),
+        )
+        self.save_tokens(tokens)
+        return tokens
+
+
+AuthTypeFactory.register("oauth", AuthTypeOAuth)

backend/czi_hosted/auth/auth_session.py

@@ -0,0 +1,40 @@
+from flask import session
+from uuid import uuid4
+
+from backend.czi_hosted.auth.auth import AuthTypeBase, AuthTypeFactory
+
+
+class AuthTypeSession(AuthTypeBase):
+    """Session based authentication.  The user is always logged.  The user id is a random number
+    associated with the session.  This is a good choice for desktop servers."""
+
+    # key in the session token for userid
+    CXGUID = "cxguid"
+
+    def __init__(self, app_config):
+        super().__init__()
+
+    def is_valid_authentication_type(self):
+        return True
+
+    def complete_setup(self, app):
+        pass
+
+    def is_user_authenticated(self):
+        # always authenticated
+        return True
+
+    def get_user_id(self):
+        if self.CXGUID not in session:
+            session[self.CXGUID] = uuid4().hex
+            session.permanent = True
+        return session[self.CXGUID]
+
+    def get_user_name(self):
+        return "anonymous"
+
+    def get_user_email(self):
+        return None
+
+
+AuthTypeFactory.register("session", AuthTypeSession)

backend/czi_hosted/auth/auth_test.py

@@ -1,6 +1,7 @@
-from server.auth.auth import AuthTypeClientBase, AuthTypeFactory
 from flask import session, request, redirect, current_app
 
+from backend.czi_hosted.auth.auth import AuthTypeClientBase, AuthTypeFactory
+
 
 class AuthTypeTest(AuthTypeClientBase):
     """An authentication type for testing client based logins.  When the login route is accessed
@@ -10,12 +11,14 @@ class AuthTypeTest(AuthTypeClientBase):
     CXGUID = "cxguid_test"
     CXGUNAME = "cxguname_test"
     CXGUEMAIL = "cxguemail_test"
+    CXGUPICTURE = "cxgupicture_test"
 
     def __init__(self, app_config):
         super().__init__()
         self.user_name = "test_account"
         self.user_id = "id0001"
         self.user_email = "test_account@test.com"
+        self.user_picture = None
 
     def is_valid_authentication_type(self):
         return True
@@ -42,11 +45,16 @@ class AuthTypeTest(AuthTypeClientBase):
     def get_user_email(self):
         return session.get(self.CXGUEMAIL)
 
+    def get_user_picture(self):
+        return session.get(self.CXGUPICTURE)
+
     def login(self):
         args = request.args
         return_to = args.get("dataset", "/")
         session[self.CXGUID] = args.get("userid", self.user_id)
         session[self.CXGUNAME] = args.get("username", self.user_name)
+        session[self.CXGUEMAIL] = args.get("email", self.user_email)
+        session[self.CXGUPICTURE] = args.get("picture", self.user_picture)
         return redirect(return_to)
 
     def logout(self):

backend/czi_hosted/cli/cli.py

@@ -0,0 +1,35 @@
+import click
+
+from .convert_to_cxg import convert_to_cxg
+from .launch import launch
+from .prepare import prepare
+from .upgrade import log_upgrade_check
+from .schema import schema_cli
+from .. import __version__
+
+
+@click.group(
+    name="cellxgene",
+    subcommand_metavar="COMMAND <args>",
+    options_metavar="<options>",
+    context_settings=dict(max_content_width=85, help_option_names=["-h", "--help"]),
+)
+@click.help_option("--help", "-h", help="Show this message and exit.")
+@click.version_option(
+    version=__version__,
+    prog_name="cellxgene",
+    message="[%(prog)s] Version %(version)s",
+    help="Show the software version and exit.",
+)
+@click.option(
+    "--upgrade-check/--no-upgrade-check", default=True, show_default=True, help="Check for release upgrades on start.",
+)
+def cli(upgrade_check):
+    if upgrade_check:
+        log_upgrade_check()
+
+
+cli.add_command(launch)
+cli.add_command(prepare)
+cli.add_command(convert_to_cxg)
+cli.add_command(schema_cli)

backend/czi_hosted/cli/convert_to_cxg.py

@@ -0,0 +1,133 @@
+from os import path
+
+import click
+
+from backend.czi_hosted.converters.h5ad_data_file import H5ADDataFile
+
+
+@click.command(
+    name="convert",
+    short_help="Converts an H5AD dataset to the CXG format.",
+    help="Converts an H5AD dataset to the CXG format. The CXG format is a cellxgene-private data format "
+    "that has performance and access characteristics amenable to a multi-dataset, multi-user serving "
+    "environment. You will be able to launch the cellxgene using the `cellxgene launch` command as "
+    "usually with the generated CXG file.",
+)
+@click.argument(
+    "input-file", nargs=1, type=click.Path(exists=True, dir_okay=False),
+)
+@click.option(
+    "-o",
+    "--output-directory",
+    help="Name of the output CXG directory. If not provided, will default to be the input filename with a "
+    "CXG extension.",
+)
+@click.option(
+    "-b",
+    "--backed",
+    help="When true, loads the H5AD in file backed mode. This will cause the conversion to be slower, "
+    "but will use less memory.",
+    default=False,
+    show_default=True,
+    is_flag=True,
+)
+@click.option(
+    "-t",
+    "--title",
+    help="Human readable dataset title that will be included as metadata about the CXG file. If omitted, "
+    "the dataset title will be the filename.",
+)
+@click.option(
+    "-a",
+    "--about",
+    help="A fully qualified URL that provides more information about the dataset and will be included as "
+    "metadata about the CXG file.",
+)
+@click.option(
+    "-s",
+    "--sparse-threshold",
+    help="If the dataset's percent of non-zero values falls belows the specified threshold, then the X "
+    "array of the dataset will be sparse. Since the default value is 0.0, the default will be to "
+    "convert to dense array.",
+    default=0.0,
+    show_default=True,
+)
+@click.option(
+    "--obs-names",
+    help="Name to a column in the obs dataframe that will be used as the index for the dataframe instead of "
+    "the one designated by the dataframe generated-index.",
+)
+@click.option(
+    "--var-names",
+    help="Name to a column in the var dataframe that will be used as the index for the dataframe instead of "
+    "the one designated by the dataframe generated-index.",
+)
+@click.option(
+    "--disable-custom-colors",
+    help="When set, conversion process will not extract scanpy-compatible category colors from the H5AD file.",
+    default=False,
+    show_default=True,
+    is_flag=True,
+)
+@click.option(
+    "--disable-corpora-schema",
+    help="When set, conversion process will neither extract nor store Corpora schema information. See "
+    "https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md for more "
+    "information.",
+    default=False,
+    show_default=True,
+    is_flag=True,
+)
+@click.option(
+    "--overwrite",
+    help="When set to true, will overwrite the output file if the output file already exists.",
+    default=False,
+    show_default=True,
+    is_flag=True,
+)
+@click.help_option("--help", "-h", help="Show this message and exit.")
+def convert_to_cxg(
+    input_file,
+    output_directory,
+    backed,
+    title,
+    about,
+    sparse_threshold,
+    obs_names,
+    var_names,
+    disable_custom_colors,
+    disable_corpora_schema,
+    overwrite,
+):
+    """
+    Convert a dataset file into CXG.
+    """
+
+    h5ad_data_file = H5ADDataFile(
+        input_file, backed, title, about, obs_names, var_names, use_corpora_schema=not disable_corpora_schema
+    )
+
+    # Get the directory that will hold all the CXG files
+    cxg_output_container = get_output_directory(input_file, output_directory, overwrite)
+
+    h5ad_data_file.to_cxg(
+        cxg_output_container, sparse_threshold, convert_anndata_colors_to_cxg_colors=not disable_custom_colors
+    )
+
+
+def get_output_directory(input_filename, output_directory, should_overwrite):
+    """
+    Get the name of the CXG output directory to be created/populated during the dataset conversion.
+    """
+
+    if output_directory and (not path.isdir(output_directory) or (path.isdir(output_directory) and should_overwrite)):
+        if output_directory.endswith(".cxg"):
+            return output_directory
+        return output_directory + ".cxg"
+    if output_directory and path.isdir(output_directory) and not should_overwrite:
+        raise click.BadParameter(
+            f"Output directory {output_directory} already exists. If you'd like to overwrite, then run the command "
+            f"with the --overwrite flag."
+        )
+
+    return path.splitext(input_filename)[0] + ".cxg"

backend/czi_hosted/cli/launch.py

@@ -1,19 +1,19 @@
 import errno
 import functools
 import logging
-from os import devnull
 import sys
 import webbrowser
-
+import os
 import click
 from flask_compress import Compress
 from flask_cors import CORS
 
-from server.common.utils import sort_options
-from server.common.errors import DatasetAccessError, ConfigurationError
-from server.common.app_config import AppConfig
-from server.common.default_config import default_config
-from server.app.app import Server
+from backend.czi_hosted.default_config import default_config
+from backend.czi_hosted.app.app import Server
+from backend.czi_hosted.common.config.app_config import AppConfig
+from backend.common.errors import DatasetAccessError, ConfigurationError
+from backend.common.utils.utils import sort_options
+
 
 DEFAULT_CONFIG = AppConfig()
 
@@ -44,20 +44,6 @@ def annotation_args(func):
         help="Directory of where to save output annotations; filename will be specified in the application. "
         "Incompatible with --annotations-file.",
     )
-    @click.option(
-        "--experimental-annotations-ontology",
-        is_flag=True,
-        default=DEFAULT_CONFIG.default_dataset_config.user_annotations__ontology__enable,
-        show_default=True,
-        help="When creating annotations, optionally autocomplete names from ontology terms.",
-    )
-    @click.option(
-        "--experimental-annotations-ontology-obo",
-        default=DEFAULT_CONFIG.default_dataset_config.user_annotations__ontology__obo_location,
-        show_default=True,
-        metavar="<path or url>",
-        help="Location of OBO file defining cell annotation autosuggest terms.",
-    )
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
         return func(*args, **kwargs)
@@ -104,14 +90,6 @@ def config_args(func):
         metavar="<text>",
         help="Embedding name, eg, 'umap'. Repeat option for multiple embeddings. Defaults to all.",
     )
-    @click.option(
-        "--experimental-enable-reembedding",
-        is_flag=True,
-        default=DEFAULT_CONFIG.default_dataset_config.embeddings__enable_reembedding,
-        show_default=False,
-        hidden=True,
-        help="Enable experimental on-demand re-embedding using UMAP. WARNING: may be very slow.",
-    )
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
         return func(*args, **kwargs)
@@ -328,9 +306,6 @@ def launch(
     annotations_dir,
     backed,
     disable_diffexp,
-    experimental_annotations_ontology,
-    experimental_annotations_ontology_obo,
-    experimental_enable_reembedding,
     config_file,
     dump_default_config,
 ):
@@ -356,7 +331,6 @@ def launch(
     if dump_default_config:
         print(default_config)
         sys.exit(0)
-
     # Startup message
     click.echo("[cellxgene] Starting the CLI...")
 
@@ -390,12 +364,9 @@ def launch(
             user_annotations__enable=not disable_annotations,
             user_annotations__local_file_csv__file=annotations_file,
             user_annotations__local_file_csv__directory=annotations_dir,
-            user_annotations__ontology__enable=experimental_annotations_ontology,
-            user_annotations__ontology__obo_location=experimental_annotations_ontology_obo,
             presentation__max_categories=max_category_items,
             presentation__custom_colors=not disable_custom_colors,
             embeddings__names=embedding,
-            embeddings__enable_reembedding=experimental_enable_reembedding,
             diffexp__enable=not disable_diffexp,
             diffexp__lfc_cutoff=diffexp_lfc_cutoff,
         )
@@ -443,7 +414,7 @@ def launch(
     click.echo("[cellxgene] Type CTRL-C at any time to exit.")
 
     if not server_config.app__verbose:
-        f = open(devnull, "w")
+        f = open(os.devnull, "w")
         sys.stdout = f
 
     try:

backend/czi_hosted/cli/prepare.py

@@ -5,7 +5,7 @@ import pandas as pd
 from numpy import ndarray, unique
 from scipy.sparse.csc import csc_matrix
 
-from server.common.utils import sort_options
+from backend.common.utils.utils import sort_options
 
 
 @sort_options

backend/czi_hosted/cli/schema.py

@@ -0,0 +1,72 @@
+import click
+
+from backend.czi_hosted.converters.schema import remix, validate
+
+
+@click.group(
+    name="schema",
+    subcommand_metavar="COMMAND <args>",
+    short_help="Apply and validate the cellxgene data integration schema to an h5ad file.",
+    context_settings=dict(max_content_width=85, help_option_names=["-h", "--help"]),
+)
+def schema_cli():
+    try:
+        import scanpy  # noqa: F401
+    except ImportError:
+        raise click.ClickException(
+            "[cellxgene] cellxgene schema requires scanpy"
+        )
+
+
+@click.command(
+    name="apply",
+    short_help="(experimental) Apply the cellxgene data integration schema to an h5ad.",
+    help="(experimental) Using a yaml file that describes schema values to insert or convert and in input "
+    "h5ad file, apply the schema changes and create a new, conforming h5ad.",
+)
+@click.option(
+    "--source-h5ad",
+    help="Input h5ad file.",
+    nargs=1,
+    required=True,
+    type=click.Path(exists=True, dir_okay=False),
+)
+@click.option(
+    "--remix-config",
+    help="Config yaml with information on how to apply the schema.",
+    nargs=1,
+    required=True,
+    type=click.Path(exists=True, dir_okay=False),
+)
+@click.option(
+    "--output-filename",
+    help="Filename for the new, schema-conforming h5ad file.",
+    required=True,
+    nargs=1
+)
+def schema_apply(source_h5ad, remix_config, output_filename):
+    remix.apply_schema(source_h5ad, remix_config, output_filename)
+
+
+@click.command(
+    name="validate",
+    short_help="(experimental) Check that an h5ad follows the cellxgene data integration schema.",
+)
+@click.argument(
+    "h5ad",
+    nargs=1,
+    type=click.Path(exists=True, dir_okay=False),
+)
+@click.option(
+    "--shallow",
+    help="When true, just check that the correct version information is present.",
+    default=False,
+    show_default=True,
+    is_flag=True,
+)
+def schema_validate(h5ad, shallow):
+    validate.validate(h5ad, shallow)
+
+
+schema_cli.add_command(schema_apply)
+schema_cli.add_command(schema_validate)

backend/czi_hosted/cli/upgrade.py

@@ -1,16 +1,16 @@
-import click
 import re
-import requests
 
+import click
+import requests
 from requests.exceptions import ConnectionError
+
 from .. import __version__
 
 # Official SemVer regex: https://semver.org/
 SEMVER_FORMAT = re.compile(
-    r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)"
-    + r"(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)"
-    + r"(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?"
-    + r"(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$"
+    r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*["
+    r"a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+("
+    r"?:\.[0-9a-zA-Z-]+)*))?$"
 )
 
 
@@ -23,8 +23,8 @@ def log_upgrade_check():
         release_tag_generator = (r["tag_name"] for r in _request_cellxgene_releases())
         latest_release = next(release_tag_generator, lambda tag_name: validate_version_str(tag_name))
         if version_gt(latest_release, __version__):
-            click.echo(f"There's a new version of cellxgene available ({latest_release})!")
-            click.echo("To upgrade, run the following: pip install --upgrade cellxgene\n")
+            click.echo(f"There's a new version of cellxgene available ({latest_release})!", err=True)
+            click.echo("To upgrade, run the following: pip install --upgrade cellxgene\n", err=True)
     except (ConnectionError, RateLimitException):
         click.echo("Upgrade check failed.\n")
 

backend/czi_hosted/common/annotations/annotations.py

@@ -0,0 +1,110 @@
+import os
+
+from flask import current_app, has_request_context
+
+from backend.common.errors import DisabledFeatureError
+from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
+from backend.common.genesets import write_gene_sets_tidycsv, read_gene_sets_tidycsv, validate_gene_sets
+from backend.common.utils.data_locator import DataLocator
+from backend.common.utils.utils import path_join
+
+
+class Annotations:
+    """baseclass for annotations and genesets"""
+
+    def __init__(self, config={}):
+        self.config = config
+
+    def user_annotations_enabled(self):
+        return self.config.get("user-annotations", False)
+
+    def check_user_annotations_enabled(self):
+        if not self.user_annotations_enabled():
+            raise DisabledFeatureError("User annotations are disabled.")
+
+    def get_schema(self, data_adaptor):
+        schema = []
+        labels = self.read_labels(data_adaptor)
+        if labels is not None and not labels.empty:
+            for col in labels.columns:
+                col_schema = dict(name=col, writable=True)
+                col_schema.update(get_schema_type_hint_of_array(labels[col]))
+                schema.append(col_schema)
+
+        return schema
+
+    def set_collection(self, name):
+        """set or create a new annotation collection"""
+        raise NotImplementedError
+
+    def read_labels(self, data_adaptor):
+        """Return the labels as a pandas.DataFrame"""
+        raise NotImplementedError
+
+    def write_labels(self, df, data_adaptor):
+        """Write the labels (df) to a persistent storage such that it can later be read"""
+        raise NotImplementedError
+
+    def update_parameters(self, parameters, data_adaptor):
+        """Update configuration parameters that describe information about the annotations feature"""
+        params = {}
+        params["annotations_genesets_readonly"] = True
+        params["annotations_genesets_name_is_read_only"] = True
+        parameters.update(params)
+
+    @staticmethod
+    def gene_sets_to_csv(genesets):
+        """
+        Convert the internal genesets format (returned by read_gene_set) into
+        the simple Tidy CSV.
+        """
+        from io import StringIO
+
+        if isinstance(genesets, dict):
+            genesets = genesets.values()
+
+        with StringIO() as sio:
+            write_gene_sets_tidycsv(sio, genesets)
+            return sio.getvalue()
+
+    @staticmethod
+    def gene_sets_to_response(genesets):
+        """
+        Convert the internal genesets format (returned by read_gene_set) into
+        the dict expected by the JSON REST API
+        """
+        return list(genesets.values())
+
+    def read_gene_sets(self, data_adaptor, context=None):
+        if has_request_context():
+            if not current_app.auth.is_user_authenticated():
+                return ({}, 0)
+
+        gene_sets_uri_or_path = dataset_uri_to_geneset_uri(data_adaptor.data_locator.uri_or_path)
+
+        server_config = data_adaptor.server_config
+        region_name = None if server_config is None else server_config.data_locator__s3__region_name
+        gene_sets_locator = DataLocator(gene_sets_uri_or_path, region_name=region_name)
+        if not gene_sets_locator.exists():
+            return ({}, 0)
+
+        gene_sets = read_gene_sets_tidycsv(gene_sets_locator, context)
+        schema = data_adaptor.get_schema()
+        var_index = schema["annotations"]["var"].get("index", "index")
+        var_names = set(data_adaptor.query_var_array(var_index))
+
+        gene_sets = validate_gene_sets(gene_sets, var_names)
+        return (gene_sets, 0)
+
+
+def dataset_uri_to_geneset_uri(data_uri_or_path):
+    """given a dataset URI, return the associated gene set URI"""
+    data_basename = os.path.basename(data_uri_or_path)
+    base, ext = os.path.splitext(data_basename)
+    if ext is not None:  # strip extension, if any
+        data_basename = base
+
+    genesets_basename = f"{data_basename}-genesets.csv"
+    gene_sets_uri_or_path = path_join(data_uri_or_path, "..", genesets_basename)
+
+    return gene_sets_uri_or_path

backend/czi_hosted/common/annotations/hosted_tiledb.py

@@ -0,0 +1,167 @@
+import json
+import os
+import re
+import time
+
+import pandas as pd
+import tiledb
+from flask import current_app
+
+from backend.czi_hosted.common.annotations.annotations import Annotations
+from backend.common.errors import AnnotationCategoryNameError
+from backend.czi_hosted.common.utils.sanitization_utils import sanitize_values_in_list
+from backend.common.utils.type_conversion_utils import get_dtypes_and_schemas_of_dataframe, get_encoding_dtype_of_array
+from backend.czi_hosted.db.cellxgene_orm import Annotation
+
+
+class AnnotationsHostedTileDB(Annotations):
+    CXG_ANNO_COLLECTION = "cxg_anno_collection"
+
+    def __init__(self, config, directory_path, db):
+        super().__init__(config)
+        self.db = db
+        if directory_path[-1] == "/":
+            self.directory_path = directory_path
+        else:
+            self.directory_path = directory_path + "/"
+
+    def check_category_names(self, df):
+        original_category_names = df.keys().to_list()
+        sanitized_category_names = set(sanitize_values_in_list(original_category_names).values())
+        unsanitary_original_category_names = set(original_category_names).difference(sanitized_category_names)
+        if unsanitary_original_category_names:
+            raise AnnotationCategoryNameError(
+                f"{unsanitary_original_category_names} are not valid category names, please resubmit"
+            )
+
+    def get_user_name(self):
+        return current_app.auth.get_user_name()
+
+    def get_user_id(self):
+        return current_app.auth.get_user_id()
+
+    def is_safe_collection_name(self, name):
+        """
+        return true if this is a safe collection name
+        this is ultra conservative. If we want to allow full legal file name syntax,
+        we could look at modules like `pathvalidate`
+        """
+        if name is None:
+            return False
+        return re.match(r"^[\w\-]+$", name) is not None
+
+    def set_collection(self, name):
+        self.CXG_ANNO_COLLECTION = name
+
+    def read_labels(self, data_adaptor):
+        user_id = self.get_user_id()
+        if user_id is None:
+            return
+        dataset_name = data_adaptor.get_location()
+        dataset_id = self.db.get_or_create_dataset(dataset_name)
+
+        annotation_object = self.db.query_for_most_recent(
+            Annotation, [Annotation.user_id == user_id, Annotation.dataset_id == dataset_id]
+        )
+        if annotation_object:
+            if annotation_object.tiledb_uri == "":
+                # this mean the user has removed all the categories.
+                return None
+            try:
+                df = tiledb.open(annotation_object.tiledb_uri)
+            except tiledb.TileDBError:
+                # don't crash if the annotations file is missing or can't be read.
+                current_app.logger.warning(f"Cannot read annotation file: {annotation_object.tiledb_uri}")
+                return None
+            pandas_df = self.convert_to_pandas_df(df, annotation_object.schema_hints)
+            return pandas_df
+        else:
+            return None
+
+    def convert_to_pandas_df(self, tileDBArray, schema_hints):
+        repr_meta = None
+        index_dims = None
+        schema_hints = json.loads(schema_hints)
+
+        if "__pandas_attribute_repr" in tileDBArray.meta:
+            # backwards compatibility... unsure if necessary at this point
+            repr_meta = json.loads(tileDBArray.meta["__pandas_attribute_repr"])
+        if "__pandas_index_dims" in tileDBArray.meta:
+            index_dims = json.loads(tileDBArray.meta["__pandas_index_dims"])
+
+        data = tileDBArray[:]
+        indexes = list()
+
+        for col_name, col_val in data.items():
+            # If the column values are byte literals, decode them
+            if isinstance(col_val[0], bytes):
+                col_val = [value.decode("utf-8") for value in col_val]
+
+            if schema_hints and col_name in schema_hints:
+                type = schema_hints.get(col_name).get("type")
+                if type and type == "categorical":
+                    new_col = pd.Series(col_val, dtype="category")
+                    data[col_name] = new_col
+            elif repr_meta and col_name in repr_meta:
+                new_col = pd.Series(col_val, dtype=repr_meta[col_name])
+                data[col_name] = new_col
+            elif index_dims and col_name in index_dims:
+                new_col = pd.Series(col_val, dtype=index_dims[col_name])
+                data[col_name] = new_col
+                indexes.append(col_name)
+
+        new_df = pd.DataFrame.from_dict(data)
+        if len(indexes) > 0:
+            new_df.set_index(indexes, inplace=True)
+
+        return new_df
+
+    def write_labels(self, df, data_adaptor):
+        auth_user_id = self.get_user_id()
+        user_name = self.get_user_name()
+        timestamp = time.time()
+        dataset_location = data_adaptor.get_location()
+        dataset_id = self.db.get_or_create_dataset(dataset_location)
+        dataset_name = data_adaptor.get_title()
+        user_id = self.db.get_or_create_user(auth_user_id)
+        """
+        NOTE: The uri contains the dataset name, user name and a timestamp as a convenience for debugging purposes.
+        People may have the same name and time.time() can be server dependent.
+        See - https://docs.python.org/2/library/time.html#time.time
+
+        The annotations objects in the database should be used as the source of truth about who an annotation belongs
+        to (for authorization purposes) and what time it was created (for garbage collection).
+        """
+        uri = f"{self.directory_path}{dataset_name}/{user_name}/{timestamp}"
+        if uri.startswith("s3://"):
+            pass
+        else:
+            os.makedirs(uri, exist_ok=True)
+        _, dataframe_schema_type_hints = get_dtypes_and_schemas_of_dataframe(df)
+        if not df.empty:
+            self.check_category_names(df)
+            # convert to tiledb datatypes
+
+            for col in df:
+                df[col] = df[col].astype(get_encoding_dtype_of_array(df[col]))
+            tiledb.from_pandas(uri, df, sparse=True)
+        else:
+            uri = ""
+
+        annotation = Annotation(
+            tiledb_uri=uri,
+            user_id=user_id,
+            dataset_id=str(dataset_id),
+            schema_hints=json.dumps(dataframe_schema_type_hints),
+        )
+        self.db.session.add(annotation)
+        self.db.session.commit()
+
+    def update_parameters(self, parameters, data_adaptor):
+        super().update_parameters(parameters, data_adaptor)
+
+        params = {}
+        params["annotations"] = True
+        params["user_annotation_collection_name_enabled"] = False
+
+        parameters.update(params)

backend/czi_hosted/common/annotations/local_file_csv.py

@@ -1,94 +1,23 @@
-import json
-import uuid
-import time
-from datetime import datetime
-import re
-import os
-import pandas as pd
-from hashlib import blake2b
 import base64
-from server import __version__ as cellxgene_version
+import os
+import re
 import threading
-from server.common.errors import AnnotationsError, OntologyLoadFailure
-from server.common.utils import series_to_schema
-import fsspec
-import fastobo
-from flask import session, current_app, has_request_context
-from abc import ABCMeta, abstractmethod
+from datetime import datetime
+from hashlib import blake2b
 
-from server.db.cellxgene_orm import CellxGeneDataset, Annotation
-from server.db.db_utils import DbUtils
+import pandas as pd
+from flask import session, has_request_context, current_app
 
-
-class Annotations(metaclass=ABCMeta):
-    """ baseclass for annotations, including ontologies"""
-
-    """ our default ontology is the PURL for the Cell Ontology.
-    See http://www.obofoundry.org/ontology/cl.html """
-    DefaultOnotology = "http://purl.obolibrary.org/obo/cl.obo"
-
-    def __init__(self):
-        self.ontology_data = None
-
-    def load_ontology(self, path):
-        """Load and parse ontologies - currently support OBO files only."""
-        if path is None:
-            path = self.DefaultOnotology
-
-        try:
-            with fsspec.open(path) as f:
-                obo = fastobo.iter(f)
-                terms = filter(lambda stanza: type(stanza) is fastobo.term.TermFrame, obo)
-                names = [tag.name for term in terms for tag in term if type(tag) is fastobo.term.NameClause]
-                self.ontology_data = names
-
-        except FileNotFoundError as e:
-            raise OntologyLoadFailure("Unable to find OBO ontology path") from e
-
-        except SyntaxError as e:
-            raise OntologyLoadFailure("Syntax error loading OBO ontology") from e
-
-        except Exception as e:
-            raise OntologyLoadFailure("Error loading OBO file") from e
-
-    def get_schema(self, data_adaptor):
-        schema = []
-        labels = self.read_labels(data_adaptor)
-        if labels is not None and not labels.empty:
-            for col in labels.columns:
-                col_schema = dict(name=col, writable=True)
-                col_schema.update(series_to_schema(labels[col]))
-                schema.append(col_schema)
-
-        return schema
-
-    @abstractmethod
-    def set_collection(self, name):
-        """set or create a new annotation collection"""
-        pass
-
-    @abstractmethod
-    def read_labels(self, data_adaptor):
-        """Return the labels as a pandas.DataFrame"""
-        pass
-
-    @abstractmethod
-    def write_labels(self, df, data_adaptor):
-        """Write the labels (df) to a persistent storage such that it can later be read"""
-        pass
-
-    @abstractmethod
-    def update_parameters(self, parameters, data_adaptor):
-        """Update configuration parameters that describe information about the annotations feature"""
-        pass
+from backend.czi_hosted import __version__ as cellxgene_version
+from backend.czi_hosted.common.annotations.annotations import Annotations
+from backend.common.errors import AnnotationsError
 
 
 class AnnotationsLocalFile(Annotations):
-
     CXG_ANNO_COLLECTION = "cxg_anno_collection"
 
-    def __init__(self, output_dir, output_file):
-        super().__init__()
+    def __init__(self, config, output_dir, output_file):
+        super().__init__(config)
         self.output_dir = output_dir
         self.output_file = output_file
         # lock used to protect label file write ops
@@ -101,7 +30,6 @@ class AnnotationsLocalFile(Annotations):
     def is_safe_collection_name(self, name):
         """
         return true if this is a safe collection name
-
         this is ultra conservative. If we want to allow full legal file name syntax,
         we could look at modules like `pathvalidate`
         """
@@ -187,7 +115,7 @@ class AnnotationsLocalFile(Annotations):
         return os.getcwd()
 
     def _get_filename(self, data_adaptor):
-        """ return the current annotation file name """
+        """return the current annotation file name"""
         if self.output_file:
             return self.output_file
 
@@ -241,14 +169,11 @@ class AnnotationsLocalFile(Annotations):
                 os.remove(os.path.join(backup_dir, bu))
 
     def update_parameters(self, parameters, data_adaptor):
+        super().update_parameters(parameters, data_adaptor)
+
         params = {}
         params["annotations"] = True
-
-        if self.ontology_data:
-            params["annotations_cell_ontology_enabled"] = True
-            params["annotations_cell_ontology_terms"] = self.ontology_data
-        else:
-            params["annotations_cell_ontology_enabled"] = False
+        params["user_annotation_collection_name_enabled"] = True
 
         if self.output_file is not None:
             # user has hard-wired the name of the annotation data collection
@@ -261,59 +186,7 @@ class AnnotationsLocalFile(Annotations):
             collection = self.get_collection()
             if current_app.auth.is_user_authenticated():
                 params["annotations-user-data-idhash"] = self._get_userdata_idhash(data_adaptor)
-                params["annotations-data-collection-is-read-only"] = False
+                params["annotations-data-collection-is-read-only"] = not self.user_annotations_enabled()
                 params["annotations-data-collection-name"] = collection
 
         parameters.update(params)
-
-
-class AnnotationsHostedTileDB(Annotations):
-    def __init__(self, directory_path: str, db: DbUtils):
-        super().__init__()
-        self.db = db
-        self.directory_path = directory_path
-
-    def set_collection(self, name):
-        pass
-
-    def read_labels(self, data_adaptor):
-        uid = current_app.auth.get_user_id()
-        dataset_name = data_adaptor.get_location()
-        dataset = self.db.query(table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == dataset_name])
-        # Todo @madison retrieve latest based on timestamp
-        annotation_object = self.db.query_for_most_recent(  # noqa F841
-            Annotation, [Annotation.user_id == uid, Annotation.dataset == dataset]
-        )
-        # Todo in future pr, retrieve dataframe from tiledb uri
-
-    def write_labels(self, df, data_adaptor):
-        uid = current_app.auth.get_user_id()
-        timestamp = time.time()
-        dataset_name = data_adaptor.get_location()
-        try:
-            dataset_id = self.db.query(
-                table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == dataset_name]
-            )[0].id
-        except IndexError:
-            dataset_id = uuid.uuid4()
-            dataset = CellxGeneDataset(id=dataset_id, name=dataset_name)
-            self.db.session.add(dataset)
-
-        uri = f"{self.directory_path}/{dataset_name}/{uid}/{timestamp}"
-        if "s3" in uri:
-            pass
-        else:
-            os.makedirs(uri, exist_ok=True)
-        schema_hints = {}
-        annotation = Annotation(
-            tiledb_uri=uri,
-            user_id=uid,
-            dataset_id=str(dataset_id),
-            schema_hints=json.dumps(schema_hints)
-        )
-        # todo in future pr -- write df to tiledb, store at uri
-        self.db.session.add(annotation)
-        self.db.session.commit()
-
-    def update_parameters(self, parameters, data_adaptor):
-        pass

backend/czi_hosted/common/config/__init__.py

@@ -0,0 +1,4 @@
+from backend.common.utils.aws_secret_utils import get_secret_key  # noqa F504
+
+DEFAULT_SERVER_PORT = 5005
+BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20  # 100MB

backend/czi_hosted/common/config/app_config.py

@@ -0,0 +1,247 @@
+import yaml
+from flatten_dict import unflatten
+
+from backend.czi_hosted.common.config.external_config import ExternalConfig
+from backend.czi_hosted.common.config.dataset_config import DatasetConfig
+from backend.czi_hosted.common.config.server_config import ServerConfig
+from backend.common.errors import ConfigurationError
+from backend.czi_hosted.default_config import get_default_config
+
+
+class AppConfig(object):
+    """
+    AppConfig stores all the configuration for cellxgene.
+    AppConfig contains one or more DatasetConfig(s) and one ServerConfig.
+    The server_config contains attributes that refer to the server process as a whole.
+    The default_dataset_config refers to attributes that are associated with the features and
+    presentations of a dataset.
+    The dataset config attributes can be overridden depending on the url by which the
+    dataset was accessed.  These are stored in dataroot_config.
+    AppConfig has methods to initialize, modify, and access the configuration.
+    """
+
+    def __init__(self):
+
+        # the default configuration (see default_config.py)
+        # TODO @madison -- if we always read from the default config (hard coded path) can we set those values as
+        #  defaults within the config class?
+        self.default_config = get_default_config()
+        # the server configuration
+        self.server_config = ServerConfig(self, self.default_config["server"])
+        # the dataset config, unless overridden by an entry in dataroot_config
+        self.default_dataset_config = DatasetConfig(None, self, self.default_config["dataset"])
+        # a dictionary of keys to DatasetConfig objects.  Each key must exist in the multi_dataset__dataroot
+        # attribute of the server_config. The default dataset config will apply to all datasets unless a different set
+        # of config vars was passed for a specific dataset under the multidataset config. For example:
+        """
+        per_dataset_config:
+            d1:
+               user_annotations:
+                   enable:  false
+            d2:
+               user_annotations:
+                   enable:  true
+         """
+        #   dataroot config
+        self.dataroot_config = {}
+
+        #  external config
+        self.external_config = ExternalConfig(self, self.default_config["external"])
+
+        # Set to true when config_completed is called
+        self.is_completed = False
+
+    def get_dataset_config(self, dataroot_key):
+        if self.server_config.single_dataset__datapath:
+            return self.default_dataset_config
+        else:
+            return self.dataroot_config.get(dataroot_key, self.default_dataset_config)
+
+    def check_config(self):
+        """Verify all the attributes in the config have been type checked"""
+        if not self.is_completed:
+            raise ConfigurationError("The configuration has not been completed")
+        self.server_config.check_config()
+        self.default_dataset_config.check_config()
+        for dataset_config in self.dataroot_config.values():
+            dataset_config.check_config()
+        self.external_config.check_config()
+
+    def update_server_config(self, **kw):
+        self.server_config.update(**kw)
+        self.is_completed = False
+
+    def update_default_dataset_config(self, **kw):
+        self.default_dataset_config.update(**kw)
+        # update all the other dataset configs, if any
+        for value in self.dataroot_config.values():
+            value.update(**kw)
+        self.is_completed = False
+
+    def update_single_config_from_path_and_value(self, path, value):
+        """Update a single config parameter with the value.
+        Path is a list of string, that gives a path to the config parameter to be updated.
+        For example, path may be ["server","app","port"].
+        """
+        self.is_completed = False
+        if not isinstance(path, list):
+            raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'")
+        for part in path:
+            if not isinstance(part, str):
+                raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'")
+
+        if len(path) < 1 or path[0] not in ("server", "dataset", "per_dataset_config"):
+            raise ConfigurationError("path must start with 'server', 'dataset', or 'per_dataset_config'")
+
+        if path[0] == "server":
+            attr = "__".join(path[1:])
+            try:
+                self.update_server_config(**{attr: value})
+            except ConfigurationError:
+                raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
+        elif path[0] == "dataset":
+            attr = "__".join(path[1:])
+            try:
+                self.update_default_dataset_config(**{attr: value})
+            except ConfigurationError:
+                raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
+
+        elif path[0] == "per_dataset_config":
+            if len(path) < 2:
+                raise ConfigurationError(f"missing dataroot when using per_dataset_config: got '{path}'")
+            dataroot = path[1]
+            if dataroot not in self.dataroot_config:
+                dataroots = str(list(self.dataroot_config.keys()))
+                raise ConfigurationError(
+                    f"unknown dataroot when using per_dataset_config: got '{path}',"
+                    f" dataroots specified in config are {dataroots}"
+                )
+
+            attr = "__".join(path[2:])
+            try:
+                self.dataroot_config[dataroot].update(**{attr: value})
+            except ConfigurationError:
+                raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
+
+    def update_from_config_file(self, config_file):
+        try:
+            with open(config_file) as yml_file:
+                config = yaml.safe_load(yml_file)
+        except yaml.YAMLError as e:
+            raise ConfigurationError(f"The specified config file contained an error: {e}")
+        except OSError as e:
+            raise ConfigurationError(f"Issue retrieving the specified config file: {e}")
+
+        if config.get("server"):
+            self.server_config.update_from_config(config["server"], "server")
+        if config.get("dataset"):
+            self.default_dataset_config.update_from_config(config["dataset"], "dataset")
+
+        per_dataset_config = config.get("per_dataset_config", {})
+        for key, dataroot_config in per_dataset_config.items():
+            # first create and initialize the dataroot with the default config
+            self.add_dataroot_config(key, **config["dataset"])
+            # then apply the per dataset configuration
+            self.dataroot_config[key].update_from_config(dataroot_config, f"per_dataset_config__{key}")
+
+        if config.get("external"):
+            self.external_config.update_from_config(config["external"], "external")
+
+        self.is_completed = False
+
+    def config_to_dict(self):
+        """return the configuration as an unflattened dict"""
+        server = self.server_config.create_mapping(self.server_config.default_config)
+        dataset = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
+        external = self.external_config.create_mapping(self.external_config.default_config)
+        config = dict(server={}, dataset={})
+        for attrname in server.keys():
+            config["server__" + attrname] = getattr(self.server_config, attrname)
+        for attrname in dataset.keys():
+            config["dataset__" + attrname] = getattr(self.default_dataset_config, attrname)
+        if self.dataroot_config:
+            config["per_dataset_config"] = {}
+        for dataroot_tag, dataroot_config in self.dataroot_config.items():
+            dataset = dataroot_config.create_mapping(dataroot_config.default_config)
+            for attrname in dataset.keys():
+                config[f"per_dataset_config__{dataroot_tag}__" + attrname] = getattr(dataroot_config, attrname)
+        for attrname in external.keys():
+            config["external__" + attrname] = getattr(self.external_config, attrname)
+
+        config = unflatten(config, splitter=lambda key: key.split("__"))
+        return config
+
+    def write_config(self, config_file):
+        """output the config to a yaml file"""
+        config = self.config_to_dict()
+        yaml.dump(config, open(config_file, "w"))
+
+    def changes_from_default(self):
+        """Return all the attribute that are different from the default"""
+        diff_server = self.server_config.changes_from_default()
+        diff_dataset = self.default_dataset_config.changes_from_default()
+        diff_external = self.external.changes_from_default()
+        diff = dict(server=diff_server, dataset=diff_dataset, external=diff_external)
+        return diff
+
+    def add_dataroot_config(self, dataroot_tag, **kw):
+        """Create a new dataset config object based on the default dataset config, and kw parameters"""
+        if dataroot_tag in self.dataroot_config:
+            raise ConfigurationError(f"dataroot config already exists: {dataroot_tag}")
+        if type(self.server_config.multi_dataset__dataroot) != dict:
+            raise ConfigurationError("The server__multi_dataset__dataroot must be a dictionary")
+        if dataroot_tag not in self.server_config.multi_dataset__dataroot:
+            raise ConfigurationError(f"The dataroot_tag ({dataroot_tag}) not found in server__multi_dataset__dataroot")
+
+        self.is_completed = False
+        self.dataroot_config[dataroot_tag] = DatasetConfig(dataroot_tag, self, self.default_config["dataset"])
+        flat_config = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
+        config = {key: value[1] for key, value in flat_config.items()}
+        self.dataroot_config[dataroot_tag].update(**config)
+        self.dataroot_config[dataroot_tag].update_from_config(kw, dataroot_tag)
+
+    def complete_config(self, messagefn=None):
+        """The configure options are checked, and any additional setup based on the config
+        parameters is done"""
+
+        if messagefn is None:
+
+            def noop(message):
+                pass
+
+            messagefn = noop
+
+        # TODO: to give better error messages we can add a mapping between where each config
+        # attribute originated (e.g. command line argument or config file), then in the error
+        # messages we can give correct context for attributes with bad value.
+        context = dict(messagefn=messagefn)
+
+        # complete config for external_config first, since this may update values in the other sections
+        self.external_config.complete_config(context)
+        self.server_config.complete_config(context)
+        self.default_dataset_config.complete_config(context)
+        for dataroot_config in self.dataroot_config.values():
+            dataroot_config.complete_config(context)
+
+        self.is_completed = True
+        self.check_config()
+
+    def get_matrix_data_cache_manager(self):
+        return self.server_config.matrix_data_cache_manager
+
+    def is_multi_dataset(self):
+        return self.server_config.multi_dataset__dataroot is not None
+
+    def get_title(self, data_adaptor):
+        return (
+            self.server_config.single_dataset__title
+            if self.server_config.single_dataset__title
+            else data_adaptor.get_title()
+        )
+
+    def get_about(self, data_adaptor):
+        return (
+            self.server_config.single_dataset__about
+            if self.server_config.single_dataset__about
+            else data_adaptor.get_about()
+        )

backend/czi_hosted/common/config/base_config.py

@@ -0,0 +1,132 @@
+import copy
+
+from flatten_dict import flatten
+from backend.common.errors import ConfigurationError
+
+
+class BaseConfig(object):
+    """
+    This class handles the mechanics of updating and checking attributes.
+    Derived classes are expected to store the actual attributes
+    Currently DatasetConfig and ServerConfig both inherit from BaseConfig.
+    """
+
+    def __init__(self, app_config, default_config, dictval_cases={}):
+        # reference back to the app_config
+        self.app_config = app_config
+        # the complete set of attributes and their default values (unflattened)
+        self.default_config = default_config
+        # attributes where the value may be a dict (and therefore are not flattened)
+        self.dictval_cases = dictval_cases
+        # used to make sure every attribute value is checked
+        self.attr_checked = {key_name: False for key_name in self.create_mapping(default_config).keys()}
+
+    def create_mapping(self, config):
+        """
+        Create a dictionary where the keys are the name of attributes (using double underscore convention)
+        For example: authentication__type
+
+        The values are a tuple,
+        - the first item of the tuple is a tuple of path elements (location in config 'tree')
+        - the second item is the value of the config parameter
+
+        For example: (('authentication', 'type'), 'session'))
+        """
+        config_copy = copy.deepcopy(config)
+        mapping = {}
+
+        # special cases where the value could be a dict.
+        # If its value is not None, the entry is added to the mapping, and not included
+        # in the flattening below.
+        for dictval_case in self.dictval_cases:
+            cur = config_copy
+            for part in dictval_case[:-1]:
+                cur = cur.get(part, {})
+            val = cur.get(dictval_case[-1])
+            if val is not None:
+                key = "__".join(dictval_case)
+                mapping[key] = (dictval_case, val)
+                del cur[dictval_case[-1]]
+
+        flat_config = flatten(config_copy)
+        for key, value in flat_config.items():
+            # name of the attribute
+            attr = "__".join(key)
+            mapping[attr] = (key, value)
+
+        return mapping
+
+    def validate_correct_type_of_configuration_attribute(self, attrname, vtype):
+        val = getattr(self, attrname)
+        if type(vtype) in (list, tuple):
+            if type(val) not in vtype:
+                tnames = ",".join([x.__name__ for x in vtype])
+                raise ConfigurationError(
+                    f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}"
+                )
+        else:
+            if type(val) != vtype:
+                raise ConfigurationError(
+                    f"Invalid type for attribute: {attrname}, "
+                    f"expected type {vtype.__name__}, got {type(val).__name__}"
+                )
+
+        self.attr_checked[attrname] = True
+
+    def check_config(self):
+        mapping = self.create_mapping(self.default_config)
+        for key in mapping.keys():
+            if not self.attr_checked[key]:
+                raise ConfigurationError(f"The attr '{key}' has not been checked")
+
+    def update(self, **kw):
+        """Update the attributes defined in kw with their new values."""
+        for key, value in kw.items():
+            if not hasattr(self, key):
+
+                # check if the key is setting into a dictval entry.
+                found_dictval = False
+                for dictval in self.dictval_cases:
+                    dictvalname = "__".join(dictval)
+                    if dictvalname + "__" in key:
+                        dictkey = key[len(dictvalname) + 2 :]
+                        curdictval = getattr(self, dictvalname)
+                        if curdictval is None:
+                            setattr(self, dictvalname, dict(dictkey=value))
+                        else:
+                            curdictval[dictkey] = value
+
+                        found_dictval = True
+                        break
+
+                if found_dictval:
+                    continue
+                raise ConfigurationError(f"unknown config parameter {key}.")
+            try:
+                if type(value) == tuple:
+                    # convert tuple values to list values
+                    value = list(value)
+                setattr(self, key, value)
+            except KeyError:
+                raise ConfigurationError(f"Unable to set config parameter {key}.")
+
+            self.attr_checked[key] = False
+
+    def update_from_config(self, config, prefix):
+        mapping = self.create_mapping(config)
+        for attr, (key, value) in mapping.items():
+            if not hasattr(self, attr):
+                raise ConfigurationError(f"Unknown key from config file: {prefix}__{attr}")
+            setattr(self, attr, value)
+
+            self.attr_checked[attr] = False
+
+    def changes_from_default(self):
+        """Return all the attribute that are different from the default"""
+        mapping = self.create_mapping(self.default_config)
+        diff = []
+        for attrname, (key, defval) in mapping.items():
+            curval = getattr(self, attrname)
+            if curval != defval:
+                diff.append((attrname, curval, defval))
+        return diff

backend/czi_hosted/common/config/client_config.py

@@ -0,0 +1,121 @@
+from backend.czi_hosted import display_version as cellxgene_display_version
+
+
+def get_client_config(app_config, data_adaptor):
+    """
+    Return the configuration as required by the /config REST route
+    """
+
+    server_config = app_config.server_config
+    dataset_config = data_adaptor.dataset_config
+    annotation = dataset_config.user_annotations
+    auth = server_config.auth
+
+    # FIXME The current set of config is not consistently presented:
+    # we have camalCase, hyphen-text, and underscore_text
+
+    # make sure the configuration has been checked.
+    app_config.check_config()
+
+    # display_names
+    title = app_config.get_title(data_adaptor)
+    about = app_config.get_about(data_adaptor)
+
+    display_names = dict(engine=data_adaptor.get_name(), dataset=title)
+
+    # library_versions
+    library_versions = {}
+    library_versions.update(data_adaptor.get_library_versions())
+    library_versions["cellxgene"] = cellxgene_display_version
+
+    # links
+    links = {"about-dataset": about}
+
+    # parameters
+    parameters = {
+        "layout": dataset_config.embeddings__names,
+        "max-category-items": dataset_config.presentation__max_categories,
+        "obs_names": server_config.single_dataset__obs_names,
+        "var_names": server_config.single_dataset__var_names,
+        "diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff,
+        "backed": server_config.adaptor__anndata_adaptor__backed,
+        "disable-diffexp": not dataset_config.diffexp__enable,
+        "annotations": False,
+        "annotations_file": None,
+        "annotations_dir": None,
+        "annotations_genesets": True,  # feature flag
+        "annotations_genesets_readonly": True,
+        "annotations_genesets_summary_methods": ["mean"],
+        "custom_colors": dataset_config.presentation__custom_colors,
+        "diffexp-may-be-slow": False,
+        "about_legal_tos": dataset_config.app__about_legal_tos,
+        "about_legal_privacy": dataset_config.app__about_legal_privacy,
+    }
+
+    # corpora dataset_props
+    # TODO/Note: putting info from the dataset into the /config is not ideal.
+    # However, it is definitely not part of /schema, and we do not have a top-level
+    # route for data properties.  Consider creating one at some point.
+    corpora_props = data_adaptor.get_corpora_props()
+    if corpora_props and "default_embedding" in corpora_props:
+        default_embedding = corpora_props["default_embedding"]
+        if isinstance(default_embedding, str) and default_embedding.startswith("X_"):
+            default_embedding = default_embedding[2:]  # drop X_ prefix
+        if default_embedding in data_adaptor.get_embedding_names():
+            parameters["default_embedding"] = default_embedding
+
+    data_adaptor.update_parameters(parameters)
+    if annotation:
+        annotation.update_parameters(parameters, data_adaptor)
+
+    # gather it all together
+    client_config = {}
+    config = client_config["config"] = {}
+    config["displayNames"] = display_names
+    config["library_versions"] = library_versions
+    config["links"] = links
+    config["parameters"] = parameters
+    config["corpora_props"] = corpora_props
+    config["limits"] = {
+        "column_request_max": server_config.limits__column_request_max,
+        "diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max,
+    }
+
+    if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
+        config["authentication"] = {
+            "requires_client_login": auth.requires_client_login(),
+        }
+        if auth.requires_client_login():
+            config["authentication"].update(
+                {
+                    # Todo why are these stored on the data_adaptor?
+                    "login": auth.get_login_url(data_adaptor),
+                    "logout": auth.get_logout_url(data_adaptor),
+                }
+            )
+
+    return client_config
+
+
+def get_client_userinfo(app_config, data_adaptor):
+    """
+    Return the userinfo as required by the /userinfo REST route
+    """
+
+    server_config = app_config.server_config
+    dataset_config = data_adaptor.dataset_config
+    auth = server_config.auth
+
+    # make sure the configuration has been checked.
+    app_config.check_config()
+
+    if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
+        userinfo = {}
+        userinfo["userinfo"] = {
+            "is_authenticated": auth.is_user_authenticated(),
+            "username": auth.get_user_name(),
+            "user_id": auth.get_user_id(),
+            "email": auth.get_user_email(),
+            "picture": auth.get_user_picture(),
+        }
+        return userinfo

backend/czi_hosted/common/config/dataset_config.py

@@ -0,0 +1,211 @@
+import os
+from os.path import splitext, isdir
+
+from backend.czi_hosted.common.annotations.annotations import Annotations
+from backend.czi_hosted.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
+from backend.czi_hosted.common.annotations.local_file_csv import AnnotationsLocalFile
+from backend.czi_hosted.common.config.base_config import BaseConfig
+from backend.common.errors import ConfigurationError
+from backend.czi_hosted.db.db_utils import DbUtils
+
+
+class DatasetConfig(BaseConfig):
+    """Manages the config attribute associated with a dataset."""
+
+    def __init__(self, tag, app_config, default_config):
+        super().__init__(app_config, default_config)
+        self.tag = tag
+        try:
+            self.app__scripts = default_config["app"]["scripts"]
+            self.app__inline_scripts = default_config["app"]["inline_scripts"]
+            self.app__about_legal_tos = default_config["app"]["about_legal_tos"]
+            self.app__about_legal_privacy = default_config["app"]["about_legal_privacy"]
+            self.app__authentication_enable = default_config["app"]["authentication_enable"]
+
+            self.presentation__max_categories = default_config["presentation"]["max_categories"]
+            self.presentation__custom_colors = default_config["presentation"]["custom_colors"]
+
+            self.user_annotations__enable = default_config["user_annotations"]["enable"]
+            self.user_annotations__type = default_config["user_annotations"]["type"]
+            self.user_annotations__local_file_csv__directory = default_config["user_annotations"]["local_file_csv"][
+                "directory"
+            ]
+            self.user_annotations__local_file_csv__file = default_config["user_annotations"]["local_file_csv"]["file"]
+            self.user_annotations__hosted_tiledb_array__db_uri = default_config["user_annotations"][
+                "hosted_tiledb_array"
+            ]["db_uri"]
+            self.user_annotations__hosted_tiledb_array__hosted_file_directory = default_config["user_annotations"][
+                "hosted_tiledb_array"
+            ]["hosted_file_directory"]
+
+            self.embeddings__names = default_config["embeddings"]["names"]
+
+            self.diffexp__enable = default_config["diffexp"]["enable"]
+            self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
+            self.diffexp__top_n = default_config["diffexp"]["top_n"]
+
+            self.X_approximate_distribution = default_config["X_approximate_distribution"]
+
+        except KeyError as e:
+            raise ConfigurationError(f"Unexpected config: {str(e)}")
+
+        # Create the default annotation, which supports gene set reading without
+        # further configuration. Depending on configuration options, `complete_config`
+        # may create a more specialized annotation object and replace this default.
+        self.user_annotations = Annotations()
+
+    def complete_config(self, context):
+        self.handle_app()
+        self.handle_presentation()
+        self.handle_user_annotations(context)
+        self.handle_embeddings()
+        self.handle_diffexp(context)
+        self.handle_X_approximate_distribution()
+
+    def handle_app(self):
+        self.validate_correct_type_of_configuration_attribute("app__scripts", list)
+        self.validate_correct_type_of_configuration_attribute("app__inline_scripts", list)
+        self.validate_correct_type_of_configuration_attribute("app__about_legal_tos", (type(None), str))
+        self.validate_correct_type_of_configuration_attribute("app__about_legal_privacy", (type(None), str))
+        self.validate_correct_type_of_configuration_attribute("app__authentication_enable", bool)
+
+        # scripts can be string (filename) or dict (attributes). Convert string to dict.
+        scripts = []
+        for script in self.app__scripts:
+            try:
+                if isinstance(script, str):
+                    scripts.append({"src": script})
+                elif isinstance(script, dict) and isinstance(script["src"], str):
+                    scripts.append(script)
+                else:
+                    raise Exception
+            except Exception as e:
+                raise ConfigurationError(f"Scripts must be string or a dict containing an src key: {e}")
+
+        self.app__scripts = scripts
+
+    def handle_presentation(self):
+        self.validate_correct_type_of_configuration_attribute("presentation__max_categories", int)
+        self.validate_correct_type_of_configuration_attribute("presentation__custom_colors", bool)
+
+    def handle_user_annotations(self, context):
+        self.validate_correct_type_of_configuration_attribute("user_annotations__enable", bool)
+        self.validate_correct_type_of_configuration_attribute("user_annotations__type", str)
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__local_file_csv__directory", (type(None), str)
+        )
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__local_file_csv__file", (type(None), str)
+        )
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__hosted_tiledb_array__db_uri", (type(None), str)
+        )
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__hosted_tiledb_array__hosted_file_directory", (type(None), str)
+        )
+        if self.user_annotations__enable:
+            server_config = self.app_config.server_config
+            if not self.app__authentication_enable:
+                raise ConfigurationError("user annotations requires authentication to be enabled")
+            if not server_config.auth.is_valid_authentication_type():
+                auth_type = server_config.authentication__type
+                raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations")
+
+            if self.user_annotations__type == "local_file_csv":
+                self.handle_local_file_csv_annotations()
+            elif self.user_annotations__type == "hosted_tiledb_array":
+                self.handle_hosted_tiledb_annotations()
+            else:
+                raise ConfigurationError('The only annotation type support is "local_file_csv" or "hosted_tiledb_array')
+        else:
+            self.check_annotation_config_vars_not_set(context)
+
+    def handle_local_file_csv_annotations(self):
+        dirname = self.user_annotations__local_file_csv__directory
+        filename = self.user_annotations__local_file_csv__file
+        if filename is not None and dirname is not None:
+            raise ConfigurationError("'annotations-file' and 'annotations-dir' may not be used together.")
+
+        if filename is not None:
+            lf_name, lf_ext = splitext(filename)
+            if lf_ext and lf_ext != ".csv":
+                raise ConfigurationError(f"annotation file type must be .csv: {filename}")
+
+        if dirname is not None and not isdir(dirname):
+            try:
+                os.mkdir(dirname)
+            except OSError:
+                raise ConfigurationError("Unable to create directory specified by --annotations-dir")
+
+        anno_config = {
+            "user-annotations": self.user_annotations__enable,
+            "genesets-save": False,
+        }
+        self.user_annotations = AnnotationsLocalFile(anno_config, dirname, filename)
+
+        # if the user has specified a fixed label file, go ahead and validate it
+        # so that we can remove errors early in the process.
+        server_config = self.app_config.server_config
+        if server_config.single_dataset__datapath and self.user_annotations__local_file_csv__file:
+            with server_config.matrix_data_cache_manager.data_adaptor(
+                self.tag, server_config.single_dataset__datapath, self.app_config
+            ) as data_adaptor:
+                data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor))
+
+    def handle_hosted_tiledb_annotations(self):
+        self.validate_correct_type_of_configuration_attribute("user_annotations__hosted_tiledb_array__db_uri", str)
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__hosted_tiledb_array__hosted_file_directory", str
+        )
+        anno_config = {
+            "user-annotations": self.user_annotations__enable,
+            "genesets-save": False,
+        }
+        self.user_annotations = AnnotationsHostedTileDB(
+            anno_config,
+            directory_path=self.user_annotations__hosted_tiledb_array__hosted_file_directory,
+            db=DbUtils(self.user_annotations__hosted_tiledb_array__db_uri),
+        )
+
+    def check_annotation_config_vars_not_set(self, context):
+        if self.user_annotations__type is not None:
+            dirname = self.user_annotations__local_file_csv__directory
+            filename = self.user_annotations__local_file_csv__file
+            db_uri = self.user_annotations__hosted_tiledb_array__db_uri
+            hosted_file_dirname = self.user_annotations__hosted_tiledb_array__hosted_file_directory
+            if filename is not None:
+                context["messagefn"]("Warning: --annotations-file ignored as annotations are disabled.")
+            if dirname is not None:
+                context["messagefn"]("Warning: --annotations-dir ignored as annotations are disabled.")
+            if db_uri is not None:
+                context["messagefn"]("Warning: db_uri ignored as annotations are disabled.")
+            if hosted_file_dirname is not None:
+                context["messagefn"](
+                    "Warning: hosted_file_directory for hosted_tiledb_array ignored as annotations are disabled."
+                )
+
+    def handle_embeddings(self):
+        self.validate_correct_type_of_configuration_attribute("embeddings__names", list)
+
+    def handle_diffexp(self, context):
+        self.validate_correct_type_of_configuration_attribute("diffexp__enable", bool)
+        self.validate_correct_type_of_configuration_attribute("diffexp__lfc_cutoff", float)
+        self.validate_correct_type_of_configuration_attribute("diffexp__top_n", int)
+
+        server_config = self.app_config.server_config
+        if server_config.single_dataset__datapath:
+            with server_config.matrix_data_cache_manager.data_adaptor(
+                self.tag, server_config.single_dataset__datapath, self.app_config
+            ) as data_adaptor:
+                if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False):
+                    context["messagefn"](
+                        "CAUTION: due to the size of your dataset, "
+                        "running differential expression may take longer or fail."
+                    )
+
+    def handle_X_approximate_distribution(self):
+        self.validate_correct_type_of_configuration_attribute("X_approximate_distribution", str)
+        if self.X_approximate_distribution not in ["normal", "count"]:
+            raise ConfigurationError(
+                "X_approximate_distribution has unknown value -- must be 'normal' or 'count'."
+            )

backend/czi_hosted/common/config/external_config.py

@@ -0,0 +1,95 @@
+import os
+
+from backend.czi_hosted.common.config.base_config import BaseConfig
+from backend.common.errors import ConfigurationError, SecretKeyRetrievalError
+from backend.common.utils.aws_secret_utils import get_secret_key
+from backend.common.utils.type_conversion_utils import convert_string_to_value
+
+
+class ExternalConfig(BaseConfig):
+    """Manages the config attribute associated with external configuration sources, such as
+    environment variables or the AWS Secrets Manager."""
+
+    def __init__(self, app_config, default_config):
+        super().__init__(app_config, default_config)
+        try:
+            self.environment = default_config["environment"]
+            self.aws_secrets_manager__region = default_config["aws_secrets_manager"]["region"]
+            self.aws_secrets_manager__secrets = default_config["aws_secrets_manager"]["secrets"]
+
+        except KeyError as e:
+            raise ConfigurationError(f"Unexpected config: {str(e)}")
+
+    def complete_config(self, context):
+        self.handle_environment(context)
+        self.handle_aws_secrets_manager(context)
+
+    def handle_environment(self, context):
+        """For each environment variable defined, get the value (if it is set),
+        and set the specified config parameter"""
+        self.validate_correct_type_of_configuration_attribute("environment", list)
+        for envdict in self.environment:
+            name = envdict.get("name")
+            if name is None:
+                raise ConfigurationError("environment: 'name' is missing")
+            required = envdict.get("required", False)
+            if type(required) != bool:
+                raise ConfigurationError("environment: 'required' must be a bool")
+            path = envdict.get("path")
+            if path is None:
+                raise ConfigurationError("environment: 'path' is missing")
+
+            value = os.environ.get(name)
+            if value is None:
+                if required:
+                    raise ConfigurationError(f"required environment variable '{name}' not set")
+            else:
+                value = convert_string_to_value(value)
+                self.app_config.update_single_config_from_path_and_value(path, value)
+
+    def handle_aws_secrets_manager(self, context):
+        """For each aws secret defined, get the key/values, and set the specified config parameter"""
+        self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", (type(None), str))
+        self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__secrets", list)
+
+        if not self.aws_secrets_manager__secrets:
+            return
+
+        self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", str)
+
+        for secret in self.aws_secrets_manager__secrets:
+            secret_name = secret.get("name")
+            if secret_name is None:
+                raise ConfigurationError("aws_secrets_manager: 'name' is missing")
+            if not isinstance(secret_name, str):
+                raise ConfigurationError("aws_secrets_manager: 'name' must be a string")
+
+            try:
+                secret_dict = get_secret_key(self.aws_secrets_manager__region, secret_name)
+            except SecretKeyRetrievalError as e:
+                raise ConfigurationError(f"Unable to retrieve secret {secret_name}: {str(e)}")
+
+            values = secret.get("values")
+            if values is None:
+                raise ConfigurationError("aws_secrets_manager: 'values' is missing")
+            if not isinstance(values, list):
+                raise ConfigurationError("aws_secrets_manager: 'values' must be a list")
+
+            for value in values:
+                key = value.get("key")
+                if key is None:
+                    raise ConfigurationError(f"missing 'key' in secret values: {secret_name}")
+                path = value.get("path")
+                if path is None:
+                    raise ConfigurationError(f"missing 'path' in secret values: {secret_name}")
+                required = value.get("required", False)
+                if type(required) != bool:
+                    raise ConfigurationError(f"wrong type for 'required' in secret values: {secret_name}")
+
+                secret_value = secret_dict.get(key)
+                if secret_value is None:
+                    if required:
+                        raise ConfigurationError(f"required secret '{secret_name}:{key}' not set")
+                else:
+                    secret_value = convert_string_to_value(secret_value)
+                    self.app_config.update_single_config_from_path_and_value(path, secret_value)

backend/czi_hosted/common/config/server_config.py

@@ -0,0 +1,387 @@
+import os
+import sys
+import warnings
+from os.path import basename
+from urllib.parse import urlparse, quote_plus
+
+from backend.czi_hosted.auth.auth import AuthTypeFactory
+from backend.czi_hosted.common.config import DEFAULT_SERVER_PORT, BIG_FILE_SIZE_THRESHOLD
+from backend.czi_hosted.common.config.base_config import BaseConfig
+from backend.common.utils.data_locator import discover_s3_region_name
+from backend.common.errors import ConfigurationError, DatasetAccessError
+from backend.common.utils.utils import is_port_available, find_available_port, custom_format_warning
+from backend.czi_hosted.compute import diffexp_cxg as diffexp_tiledb
+from backend.czi_hosted.data_common.matrix_loader import MatrixDataCacheManager, MatrixDataLoader, MatrixDataType
+
+
+class ServerConfig(BaseConfig):
+    """Manages the config attribute associated with the server."""
+
+    def __init__(self, app_config, default_config):
+        dictval_cases = [
+            ("app", "csp_directives"),
+            ("authentication", "params_oauth", "cookie"),
+            ("authentication", "params_oauth", "jwt_decode_options"),
+            ("adaptor", "cxg_adaptor", "tiledb_ctx"),
+            ("multi_dataset", "dataroot"),
+        ]
+        super().__init__(app_config, default_config, dictval_cases)
+
+        try:
+            self.app__verbose = default_config["app"]["verbose"]
+            self.app__debug = default_config["app"]["debug"]
+            self.app__host = default_config["app"]["host"]
+            self.app__port = default_config["app"]["port"]
+            self.app__open_browser = default_config["app"]["open_browser"]
+            self.app__force_https = default_config["app"]["force_https"]
+            self.app__flask_secret_key = default_config["app"]["flask_secret_key"]
+            self.app__generate_cache_control_headers = default_config["app"]["generate_cache_control_headers"]
+            self.app__server_timing_headers = default_config["app"]["server_timing_headers"]
+            self.app__csp_directives = default_config["app"]["csp_directives"]
+            self.app__api_base_url = default_config["app"]["api_base_url"]
+            self.app__web_base_url = default_config["app"]["web_base_url"]
+
+            self.authentication__type = default_config["authentication"]["type"]
+            self.authentication__insecure_test_environment = default_config["authentication"][
+                "insecure_test_environment"
+            ]
+            self.authentication__params_oauth__oauth_api_base_url = default_config["authentication"]["params_oauth"][
+                "oauth_api_base_url"
+            ]
+            self.authentication__params_oauth__client_id = default_config["authentication"]["params_oauth"]["client_id"]
+            self.authentication__params_oauth__client_secret = default_config["authentication"]["params_oauth"][
+                "client_secret"
+            ]
+            self.authentication__params_oauth__jwt_decode_options = default_config["authentication"]["params_oauth"][
+                "jwt_decode_options"
+            ]
+            self.authentication__params_oauth__session_cookie = default_config["authentication"]["params_oauth"][
+                "session_cookie"
+            ]
+            self.authentication__params_oauth__cookie = default_config["authentication"]["params_oauth"]["cookie"]
+
+            self.multi_dataset__dataroot = default_config["multi_dataset"]["dataroot"]
+            self.multi_dataset__index = default_config["multi_dataset"]["index"]
+            self.multi_dataset__allowed_matrix_types = default_config["multi_dataset"]["allowed_matrix_types"]
+            self.multi_dataset__matrix_cache__max_datasets = default_config["multi_dataset"]["matrix_cache"][
+                "max_datasets"
+            ]
+            self.multi_dataset__matrix_cache__timelimit_s = default_config["multi_dataset"]["matrix_cache"][
+                "timelimit_s"
+            ]
+
+            self.single_dataset__datapath = default_config["single_dataset"]["datapath"]
+            self.single_dataset__obs_names = default_config["single_dataset"]["obs_names"]
+            self.single_dataset__var_names = default_config["single_dataset"]["var_names"]
+            self.single_dataset__about = default_config["single_dataset"]["about"]
+            self.single_dataset__title = default_config["single_dataset"]["title"]
+
+            self.diffexp__alg_cxg__max_workers = default_config["diffexp"]["alg_cxg"]["max_workers"]
+            self.diffexp__alg_cxg__cpu_multiplier = default_config["diffexp"]["alg_cxg"]["cpu_multiplier"]
+            self.diffexp__alg_cxg__target_workunit = default_config["diffexp"]["alg_cxg"]["target_workunit"]
+
+            self.data_locator__s3__region_name = default_config["data_locator"]["s3"]["region_name"]
+
+            self.adaptor__cxg_adaptor__tiledb_ctx = default_config["adaptor"]["cxg_adaptor"]["tiledb_ctx"]
+            self.adaptor__anndata_adaptor__backed = default_config["adaptor"]["anndata_adaptor"]["backed"]
+
+            self.limits__diffexp_cellcount_max = default_config["limits"]["diffexp_cellcount_max"]
+            self.limits__column_request_max = default_config["limits"]["column_request_max"]
+
+        except KeyError as e:
+            raise ConfigurationError(f"Unexpected config: {str(e)}")
+
+        # The matrix data cache manager is created during the complete_config and stored here.
+        self.matrix_data_cache_manager = None
+
+        # The authentication object
+        self.auth = None
+
+    def complete_config(self, context):
+        self.handle_app(context)
+        self.handle_data_source()
+        self.handle_authentication()
+        self.handle_data_locator()
+        self.handle_adaptor()  # may depend on data_locator
+        self.handle_single_dataset(context)  # may depend on adaptor
+        self.handle_multi_dataset()  # may depend on adaptor
+        self.handle_diffexp()
+        self.handle_limits()
+
+        self.check_config()
+
+    def handle_app(self, context):
+        self.validate_correct_type_of_configuration_attribute("app__verbose", bool)
+        self.validate_correct_type_of_configuration_attribute("app__debug", bool)
+        self.validate_correct_type_of_configuration_attribute("app__host", str)
+        self.validate_correct_type_of_configuration_attribute("app__port", (type(None), int))
+        self.validate_correct_type_of_configuration_attribute("app__open_browser", bool)
+        self.validate_correct_type_of_configuration_attribute("app__force_https", bool)
+        self.validate_correct_type_of_configuration_attribute("app__flask_secret_key", str)
+        self.validate_correct_type_of_configuration_attribute("app__generate_cache_control_headers", bool)
+        self.validate_correct_type_of_configuration_attribute("app__server_timing_headers", bool)
+        self.validate_correct_type_of_configuration_attribute("app__csp_directives", (type(None), dict))
+        self.validate_correct_type_of_configuration_attribute("app__api_base_url", (type(None), str))
+        self.validate_correct_type_of_configuration_attribute("app__web_base_url", (type(None), str))
+
+        if self.app__port:
+            try:
+                if not is_port_available(self.app__host, self.app__port):
+                    raise ConfigurationError(
+                        f"The port selected {self.app__port} is in use, please configure an open port."
+                    )
+            except OverflowError:
+                raise ConfigurationError(f"Invalid port: {self.app__port}")
+        else:
+            try:
+                default_server_port = int(os.environ.get("CXG_SERVER_PORT", DEFAULT_SERVER_PORT))
+            except ValueError:
+                raise ConfigurationError(
+                    "Invalid port from environment variable CXG_SERVER_PORT: " + os.environ.get("CXG_SERVER_PORT")
+                )
+            try:
+                self.app__port = find_available_port(self.app__host, default_server_port)
+            except OverflowError:
+                raise ConfigurationError(f"Invalid port: {default_server_port}")
+
+        if self.app__debug:
+            context["messagefn"]("in debug mode, setting verbose=True and open_browser=False")
+            self.app__verbose = True
+            self.app__open_browser = False
+        else:
+            warnings.formatwarning = custom_format_warning
+
+        if not self.app__verbose:
+            sys.tracebacklimit = 0
+
+        # CSP Directives are a dict of string: list(string) or string: string
+        if self.app__csp_directives is not None:
+            for k, v in self.app__csp_directives.items():
+                if not isinstance(k, str):
+                    raise ConfigurationError("CSP directive names must be a string.")
+                if isinstance(v, list):
+                    for policy in v:
+                        if not isinstance(policy, str):
+                            raise ConfigurationError("CSP directive value must be a string or list of strings.")
+                elif not isinstance(v, str):
+                    raise ConfigurationError("CSP directive value must be a string or list of strings.")
+
+        if self.app__web_base_url is None:
+            self.app__web_base_url = self.app__api_base_url
+
+    def handle_authentication(self):
+        self.validate_correct_type_of_configuration_attribute("authentication__type", (type(None), str))
+        self.validate_correct_type_of_configuration_attribute("authentication__insecure_test_environment", bool)
+
+        if self.authentication__type == "test" and not self.authentication__insecure_test_environment:
+            raise ConfigurationError("Test auth can only be used in an insecure test environment")
+
+        # oauth
+        ptypes = str if self.authentication__type == "oauth" else (type(None), str)
+        self.validate_correct_type_of_configuration_attribute(
+            "authentication__params_oauth__oauth_api_base_url", ptypes
+        )
+        self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_id", ptypes)
+        self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_secret", ptypes)
+        self.validate_correct_type_of_configuration_attribute(
+            "authentication__params_oauth__jwt_decode_options", (type(None), dict)
+        )
+        self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__session_cookie", bool)
+
+        if self.authentication__params_oauth__session_cookie:
+            self.validate_correct_type_of_configuration_attribute(
+                "authentication__params_oauth__cookie", (type(None), dict)
+            )
+        else:
+            self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__cookie", dict)
+
+        self.auth = AuthTypeFactory.create(self.authentication__type, self)
+        if self.auth is None:
+            raise ConfigurationError(f"Unknown authentication type: {self.authentication__type}")
+
+    def handle_data_locator(self):
+        self.validate_correct_type_of_configuration_attribute("data_locator__s3__region_name", (type(None), bool, str))
+        if self.data_locator__s3__region_name is True:
+            path = self.single_dataset__datapath or self.multi_dataset__dataroot
+
+            if type(path) == dict:
+                # if multi_dataset__dataroot is a dict, then use the first key
+                # that is in s3.   NOTE:  it is not supported to have dataroots
+                # in different regions.
+                paths = [val.get("dataroot") for val in path.values()]
+                for path in paths:
+                    if path.startswith("s3://"):
+                        break
+            if path.startswith("s3://"):
+                region_name = discover_s3_region_name(path)
+                if region_name is None:
+                    raise ConfigurationError(f"Unable to discover s3 region name from {path}")
+            else:
+                region_name = None
+            self.data_locator__s3__region_name = region_name
+
+    def handle_data_source(self):
+        self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None)))
+        self.validate_correct_type_of_configuration_attribute("multi_dataset__dataroot", (type(None), dict, str))
+
+        if self.single_dataset__datapath and self.multi_dataset__dataroot:
+            raise ConfigurationError(
+                "You must supply either a datapath (for single datasets) or a dataroot (for multidatasets). Not both"
+            )
+        if self.single_dataset__datapath is None and self.multi_dataset__dataroot is None:
+            raise ConfigurationError("You must specify a datapath for a single dataset or a dataroot for multidatasets")
+
+    def handle_single_dataset(self, context):
+        self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None)))
+        self.validate_correct_type_of_configuration_attribute("single_dataset__title", (str, type(None)))
+        self.validate_correct_type_of_configuration_attribute("single_dataset__about", (str, type(None)))
+        self.validate_correct_type_of_configuration_attribute("single_dataset__obs_names", (str, type(None)))
+        self.validate_correct_type_of_configuration_attribute("single_dataset__var_names", (str, type(None)))
+
+        if self.single_dataset__datapath is None:
+            return
+
+        # create the matrix data cache manager:
+        if self.matrix_data_cache_manager is None:
+            self.matrix_data_cache_manager = MatrixDataCacheManager(max_cached=1, timelimit_s=None)
+
+        # preload this data set
+        matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath, app_config=self.app_config)
+        try:
+            matrix_data_loader.pre_load_validation()
+        except DatasetAccessError as e:
+            raise ConfigurationError(str(e))
+
+        file_size = matrix_data_loader.file_size()
+        file_basename = basename(self.single_dataset__datapath)
+        if file_size > BIG_FILE_SIZE_THRESHOLD:
+            context["messagefn"](f"Loading data from {file_basename}, this may take a while...")
+        else:
+            context["messagefn"](f"Loading data from {file_basename}.")
+
+        if self.single_dataset__about:
+
+            def url_check(url):
+                try:
+                    result = urlparse(url)
+                    if all([result.scheme, result.netloc]):
+                        return True
+                    else:
+                        return False
+                except ValueError:
+                    return False
+
+            if not url_check(self.single_dataset__about):
+                raise ConfigurationError(
+                    "Must provide an absolute URL for --about. (Example format: http://example.com)"
+                )
+
+    def handle_multi_dataset(self):
+        self.validate_correct_type_of_configuration_attribute("multi_dataset__dataroot", (type(None), dict, str))
+        self.validate_correct_type_of_configuration_attribute("multi_dataset__index", (type(None), bool, str))
+        self.validate_correct_type_of_configuration_attribute("multi_dataset__allowed_matrix_types", list)
+        self.validate_correct_type_of_configuration_attribute("multi_dataset__matrix_cache__max_datasets", int)
+        self.validate_correct_type_of_configuration_attribute(
+            "multi_dataset__matrix_cache__timelimit_s", (type(None), int, float)
+        )
+
+        if self.multi_dataset__dataroot is None:
+            return
+
+        if type(self.multi_dataset__dataroot) == str:
+            default_dict = dict(base_url="d", dataroot=self.multi_dataset__dataroot)
+            self.multi_dataset__dataroot = dict(d=default_dict)
+
+        for tag, dataroot_dict in self.multi_dataset__dataroot.items():
+            if "base_url" not in dataroot_dict:
+                raise ConfigurationError(f"error in multi_dataset__dataroot: missing base_url for tag {tag}")
+            if "dataroot" not in dataroot_dict:
+                raise ConfigurationError(f"error in multi_dataset__dataroot: missing dataroot, for tag {tag}")
+
+            base_url = dataroot_dict["base_url"]
+
+            # sanity check for well formed base urls
+            bad = False
+            if type(base_url) != str:
+                bad = True
+            elif os.path.normpath(base_url) != base_url:
+                bad = True
+            else:
+                base_url_parts = base_url.split("/")
+                if [quote_plus(part) for part in base_url_parts] != base_url_parts:
+                    bad = True
+                if ".." in base_url_parts:
+                    bad = True
+            if bad:
+                raise ConfigurationError(f"error in multi_dataset__dataroot base_url {base_url} for tag {tag}")
+
+        # verify all the base_urls are unique
+        base_urls = [d["base_url"] for d in self.multi_dataset__dataroot.values()]
+        if len(base_urls) > len(set(base_urls)):
+            raise ConfigurationError("error in multi_dataset__dataroot:  base_urls must be unique")
+
+        # error checking
+        for mtype in self.multi_dataset__allowed_matrix_types:
+            try:
+                MatrixDataType(mtype)
+            except ValueError:
+                raise ConfigurationError(f'Invalid matrix type in "allowed_matrix_types": {mtype}')
+
+        # create the matrix data cache manager:
+        if self.matrix_data_cache_manager is None:
+            self.matrix_data_cache_manager = MatrixDataCacheManager(
+                max_cached=self.multi_dataset__matrix_cache__max_datasets,
+                timelimit_s=self.multi_dataset__matrix_cache__timelimit_s,
+            )
+
+    def handle_diffexp(self):
+        self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__max_workers", (str, int))
+        self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__cpu_multiplier", int)
+        self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__target_workunit", int)
+
+        max_workers = self.diffexp__alg_cxg__max_workers
+        cpu_multiplier = self.diffexp__alg_cxg__cpu_multiplier
+        cpu_count = os.cpu_count()
+        max_workers = min(max_workers, cpu_multiplier * cpu_count)
+        diffexp_tiledb.set_config(max_workers, self.diffexp__alg_cxg__target_workunit)
+
+    def handle_adaptor(self):
+        # cxg
+        self.validate_correct_type_of_configuration_attribute("adaptor__cxg_adaptor__tiledb_ctx", dict)
+        regionkey = "vfs.s3.region"
+        if regionkey not in self.adaptor__cxg_adaptor__tiledb_ctx:
+            if type(self.data_locator__s3__region_name) == str:
+                self.adaptor__cxg_adaptor__tiledb_ctx[regionkey] = self.data_locator__s3__region_name
+
+        from backend.czi_hosted.data_cxg.cxg_adaptor import CxgAdaptor
+
+        CxgAdaptor.set_tiledb_context(self.adaptor__cxg_adaptor__tiledb_ctx)
+
+        # anndata
+        self.validate_correct_type_of_configuration_attribute("adaptor__anndata_adaptor__backed", bool)
+
+    def handle_limits(self):
+        self.validate_correct_type_of_configuration_attribute("limits__diffexp_cellcount_max", (type(None), int))
+        self.validate_correct_type_of_configuration_attribute("limits__column_request_max", (type(None), int))
+
+    def exceeds_limit(self, limit_name, value):
+        limit_value = getattr(self, "limits__" + limit_name, None)
+        if limit_value is None:  # disabled
+            return False
+        return value > limit_value
+
+    def get_api_base_url(self):
+        if self.app__api_base_url == "local":
+            return f"http://{self.app__host}:{self.app__port}"
+        if self.app__api_base_url and self.app__api_base_url.endswith("/"):
+            return self.app__api_base_url[:-1]
+        return self.app__api_base_url
+
+    def get_web_base_url(self):
+        if self.app__web_base_url == "local":
+            return f"http://{self.app__host}:{self.app__port}"
+        if self.app__web_base_url is None:
+            return self.get_api_base_url()
+        if self.app__web_base_url.endswith("/"):
+            return self.app__web_base_url[:-1]
+        return self.app__web_base_url

backend/czi_hosted/common/corpora.py

@@ -8,7 +8,8 @@ https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/c
 import collections
 import json
 
-from server.cli.upgrade import validate_version_str
+from backend.czi_hosted.cli.upgrade import validate_version_str
+from backend.czi_hosted.common.utils.corpora_constants import CorporaConstants
 
 
 def corpora_get_versions_from_anndata(adata):
@@ -56,34 +57,21 @@ def corpora_get_props_from_anndata(adata):
     if not version_is_supported:
         raise ValueError("Unsupported Corpora schema version")
 
-    required_simple_fields = [
-        "version",
-        "title",
-        "layer_descriptions",
-        "organism",
-        "organism_ontology_term_id",
-        "project_name",
-        "project_description",
-    ]
-    # Spec says some values encoded as JSON due to the inability of AnnData to store complex types.
-    required_json_fields = ["contributors", "project_links"]
-    optional_simple_fields = ["preprint_doi", "publication_doi", "default_embedding", "default_field", "tags"]
-
     corpora_props = {}
-    for key in required_simple_fields:
+    for key in CorporaConstants.REQUIRED_SIMPLE_METADATA_FIELDS:
         if key not in adata.uns:
             raise KeyError(f"missing Corpora schema field {key}")
         corpora_props[key] = adata.uns[key]
 
-    for key in required_json_fields:
+    for key in CorporaConstants.OPTIONAL_JSON_ENCODED_METADATA_FIELD:
         if key not in adata.uns:
-            raise KeyError(f"missing Corpora schema field {key}")
+            continue
         try:
             corpora_props[key] = json.loads(adata.uns[key])
         except json.JSONDecodeError:
             raise json.JSONDecodeError(f"Corpora schema field {key} is expected to be a valid JSON string")
 
-    for key in optional_simple_fields:
+    for key in CorporaConstants.OPTIONAL_SIMPLE_METADATA_FIELDS:
         if key in adata.uns:
             corpora_props[key] = adata.uns[key]
 

backend/czi_hosted/common/health.py

@@ -1,8 +1,8 @@
 from http import HTTPStatus
 from flask import make_response, jsonify
 
-from server import __version__ as cellxgene_version
-from server.common.data_locator import DataLocator
+from backend.czi_hosted import __version__ as cellxgene_version
+from backend.common.utils.data_locator import DataLocator
 
 
 def _is_accessible(path, config):

backend/czi_hosted/common/rest.py

@@ -2,12 +2,15 @@ import copy
 import logging
 import sys
 from http import HTTPStatus
+import zlib
+import json
 
 from flask import make_response, jsonify, current_app, abort
 from werkzeug.urls import url_unquote
 
-from server.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg
-from server.common.errors import (
+from backend.czi_hosted.common.config.client_config import get_client_config, get_client_userinfo
+from backend.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg
+from backend.common.errors import (
     FilterError,
     JSONEncodingValueError,
     PrepareError,
@@ -15,10 +18,11 @@ from server.common.errors import (
     ExceedsLimitError,
     DatasetAccessError,
     ColorFormatException,
+    AnnotationsError,
+    UnsupportedSummaryMethod,
 )
-
-import json
-from server.data_common.fbs.matrix import decode_matrix_fbs
+from backend.common.genesets import summarizeQueryHash
+from backend.common.fbs.matrix import decode_matrix_fbs
 
 
 def abort_and_log(code, logmsg, loglevel=logging.DEBUG, include_exc_info=False):
@@ -104,7 +108,7 @@ def schema_get_helper(data_adaptor):
 
     # add label obs annotations as needed
     annotations = data_adaptor.dataset_config.user_annotations
-    if annotations is not None:
+    if annotations.user_annotations_enabled():
         label_schema = annotations.get_schema(data_adaptor)
         schema["annotations"]["obs"]["columns"].extend(label_schema)
 
@@ -117,7 +121,12 @@ def schema_get(data_adaptor):
 
 
 def config_get(app_config, data_adaptor):
-    config = app_config.get_client_config(data_adaptor)
+    config = get_client_config(app_config, data_adaptor)
+    return make_response(jsonify(config), HTTPStatus.OK)
+
+
+def userinfo_get(app_config, data_adaptor):
+    config = get_client_userinfo(app_config, data_adaptor)
     return make_response(jsonify(config), HTTPStatus.OK)
 
 
@@ -133,7 +142,7 @@ def annotations_obs_get(request, data_adaptor):
     try:
         labels = None
         annotations = data_adaptor.dataset_config.user_annotations
-        if annotations:
+        if annotations.user_annotations_enabled():
             labels = annotations.read_labels(data_adaptor)
         fbs = data_adaptor.annotation_to_fbs_matrix(Axis.OBS, fields, labels)
         return make_response(fbs, HTTPStatus.OK, {"Content-Type": "application/octet-stream"})
@@ -144,22 +153,26 @@ def annotations_obs_get(request, data_adaptor):
 def annotations_put_fbs_helper(data_adaptor, fbs):
     """helper function to write annotations from fbs"""
     annotations = data_adaptor.dataset_config.user_annotations
-    if annotations is None:
+    if not annotations.user_annotations_enabled():
         raise DisabledFeatureError("Writable annotations are not enabled")
 
     new_label_df = decode_matrix_fbs(fbs)
     if not new_label_df.empty:
-        data_adaptor.check_new_labels(new_label_df)
+        new_label_df = data_adaptor.check_new_labels(new_label_df)
     annotations.write_labels(new_label_df, data_adaptor)
 
 
+def inflate(data):
+    return zlib.decompress(data)
+
+
 def annotations_obs_put(request, data_adaptor):
     annotations = data_adaptor.dataset_config.user_annotations
-    if annotations is None:
+    if not annotations.user_annotations_enabled():
         return abort(HTTPStatus.NOT_IMPLEMENTED)
 
     anno_collection = request.args.get("annotation-collection-name", default=None)
-    fbs = request.get_data()
+    fbs = inflate(request.get_data())
 
     if anno_collection is not None:
         if not annotations.is_safe_collection_name(anno_collection):
@@ -186,7 +199,7 @@ def annotations_var_get(request, data_adaptor):
     try:
         labels = None
         annotations = data_adaptor.dataset_config.user_annotations
-        if annotations is not None:
+        if annotations.user_annotations_enabled():
             labels = annotations.read_labels(data_adaptor)
         return make_response(
             data_adaptor.annotation_to_fbs_matrix(Axis.VAR, fields, labels),
@@ -247,13 +260,14 @@ def diffexp_obs_post(request, data_adaptor):
     try:
         # TODO: implement varfilter mode
         mode = DiffExpMode(args["mode"])
-
         if mode == DiffExpMode.VAR_FILTER or "varFilter" in args:
             return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, "varFilter not enabled")
 
         set1_filter = args.get("set1", {"filter": {}})["filter"]
         set2_filter = args.get("set2", {"filter": {}})["filter"]
-        count = args.get("count", None)
+        # TODO(#1281): When we simplify the config, we should actually use the config to determine this number,
+        #  this will also require an update in the client
+        count = 15
 
         if set1_filter is None or set2_filter is None or count is None:
             return abort_and_log(HTTPStatus.BAD_REQUEST, "missing required parameter")
@@ -300,20 +314,68 @@ def layout_obs_get(request, data_adaptor):
         )
 
 
-def layout_obs_put(request, data_adaptor):
-    if not data_adaptor.dataset_config.embeddings__enable_reembedding:
-        return abort(HTTPStatus.NOT_IMPLEMENTED)
-
-    args = request.get_json()
-    filter = args["filter"] if args else None
-    if not filter:
-        return abort_and_log(HTTPStatus.BAD_REQUEST, "obs filter is required")
-    method = args["method"] if args else "umap"
+def genesets_get(request, data_adaptor):
+    preferred_mimetype = request.accept_mimetypes.best_match(["application/json", "text/csv"])
+    if preferred_mimetype not in ("application/json", "text/csv"):
+        return abort(HTTPStatus.NOT_ACCEPTABLE)
 
     try:
-        schema = data_adaptor.compute_embedding(method, filter)
-        return make_response(jsonify(schema), HTTPStatus.OK, {"Content-Type": "application/json"})
-    except NotImplementedError as e:
-        return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, str(e))
-    except (ValueError, DisabledFeatureError, FilterError) as e:
-        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+        annotations = data_adaptor.dataset_config.user_annotations
+        (genesets, tid) = annotations.read_gene_sets(data_adaptor)
+
+        if preferred_mimetype == "text/csv":
+            return make_response(
+                annotations.gene_sets_to_csv(genesets),
+                HTTPStatus.OK,
+                {
+                    "Content-Type": "text/csv",
+                    "Content-Disposition": "attachment; filename=genesets.csv",
+                },
+            )
+        else:
+            return make_response(
+                jsonify({"genesets": annotations.gene_sets_to_response(genesets), "tid": tid}), HTTPStatus.OK
+            )
+    except (ValueError, KeyError, AnnotationsError) as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e))
+
+
+def summarize_var_helper(request, data_adaptor, key, raw_query):
+    preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
+    if preferred_mimetype != "application/octet-stream":
+        return abort(HTTPStatus.NOT_ACCEPTABLE)
+
+    summary_method = request.values.get("method", default="mean")
+    query_hash = summarizeQueryHash(raw_query)
+    if key and query_hash != key:
+        return abort(HTTPStatus.BAD_REQUEST, description="query key did not match")
+
+    args_filter_only = request.values.copy()
+    args_filter_only.poplist("method")
+    args_filter_only.poplist("key")
+
+    try:
+        filter = _query_parameter_to_filter(args_filter_only)
+        return make_response(
+            data_adaptor.summarize_var(summary_method, filter, query_hash),
+            HTTPStatus.OK,
+            {"Content-Type": "application/octet-stream"},
+        )
+    except (ValueError) as e:
+        return abort(HTTPStatus.NOT_FOUND, description=str(e))
+    except (UnsupportedSummaryMethod, FilterError) as e:
+        return abort(HTTPStatus.BAD_REQUEST, description=str(e))
+
+
+def summarize_var_get(request, data_adaptor):
+    return summarize_var_helper(request, data_adaptor, None, request.query_string)
+
+
+def summarize_var_post(request, data_adaptor):
+    if not request.content_type or "application/x-www-form-urlencoded" not in request.content_type:
+        return abort(HTTPStatus.UNSUPPORTED_MEDIA_TYPE)
+    if request.content_length > 1_000_000:  # just a sanity check to avoid memory exhaustion
+        return abort(HTTPStatus.BAD_REQUEST)
+
+    key = request.args.get("key", default=None)
+    return summarize_var_helper(request, data_adaptor, key, request.get_data())

backend/czi_hosted/common/utils/corpora_constants.py

@@ -0,0 +1,22 @@
+class CorporaConstants(object):
+    REQUIRED_SIMPLE_METADATA_FIELDS = [
+        "version",
+        "title",
+        "layer_descriptions",
+        "organism",
+        "organism_ontology_term_id",
+    ]
+
+    # The Corpora specification requires some values encoded as JSON due to the inability of AnnData to store complex
+    # types.
+    OPTIONAL_JSON_ENCODED_METADATA_FIELD = ["contributors", "project_links"]
+
+    OPTIONAL_SIMPLE_METADATA_FIELDS = [
+        "preprint_doi",
+        "publication_doi",
+        "default_embedding",
+        "default_field",
+        "tags",
+        "project_name",
+        "project_description",
+    ]

backend/czi_hosted/common/utils/cxg_constants.py

@@ -0,0 +1,4 @@
+class CxgConstants(object):
+    # The CXG container version number.  Must be a semver string (major.minor.patch)
+    # DO NOT UPDATE THIS WITHOUT ALSO UPDATING CXG SPECIFICATION.
+    CXG_VERSION = "0.2.0"

backend/czi_hosted/common/utils/cxg_generation_utils.py

@@ -0,0 +1,178 @@
+import json
+
+import numpy as np
+import tiledb
+
+from backend.common.utils.type_conversion_utils import get_encoding_dtype_of_array, get_dtype_and_schema_of_array
+
+
+def convert_dictionary_to_cxg_group(cxg_container, metadata_dict, group_metadata_name="cxg_group_metadata"):
+    """
+    Saves the contents of the dictionary to the CXG output directory specified.
+
+    This function is primarily used to save metadata about a dataset to the CXG directory. At some point, tiledb will
+    have support for metadata on groups at which point the utility of this function should be revisited. Until such
+    feature exists, this function create an empty array and annotate that array.
+
+    For more information, visit https://github.com/TileDB-Inc/TileDB-Py/issues/254.
+    """
+
+    array_name = f"{cxg_container}/{group_metadata_name}"
+
+    # Because TileDB does not allow one to attach metadata directly to a CXG group, we need to have a workaround
+    # where we create an empty array and attached the metadata onto to this empty array. Below we construct this empty
+    # array.
+    tiledb.from_numpy(array_name, np.zeros((1,)))
+
+    with tiledb.DenseArray(array_name, mode="w") as metadata_array:
+        for key, value in metadata_dict.items():
+            metadata_array.meta[key] = value
+
+
+def convert_dataframe_to_cxg_array(cxg_container, dataframe_name, dataframe, index_column_name, ctx):
+    """
+    Saves the contents of the dataframe to the CXG output directory specified.
+
+    Current access patterns are oriented toward reading very large slices of the dataframe, one attribute at a time.
+    Attribute data also tends to be (often) repetitive (bools, categories, strings). Given this, we use a large tile
+    size (1000) and very aggressive compression levels.
+    """
+
+    def create_dataframe_array(array_name, dataframe):
+        tiledb_filter = tiledb.FilterList(
+            [
+                # Attempt aggressive compression as many of these dataframes are very repetitive strings, bools and
+                # other non-float data.
+                tiledb.ZstdFilter(level=22),
+            ]
+        )
+        attrs = [
+            tiledb.Attr(name=column, dtype=get_encoding_dtype_of_array(dataframe[column]), filters=tiledb_filter)
+            for column in dataframe
+        ]
+        domain = tiledb.Domain(
+            tiledb.Dim(domain=(0, dataframe.shape[0] - 1), tile=min(dataframe.shape[0], 1000), dtype=np.uint32)
+        )
+        schema = tiledb.ArraySchema(
+            domain=domain, sparse=False, attrs=attrs, cell_order="row-major", tile_order="row-major"
+        )
+        tiledb.DenseArray.create(array_name, schema)
+
+    array_name = f"{cxg_container}/{dataframe_name}"
+
+    create_dataframe_array(array_name, dataframe)
+
+    with tiledb.DenseArray(array_name, mode="w", ctx=ctx) as array:
+        value = {}
+        schema_hints = {}
+        for column_name, column_values in dataframe.items():
+            dtype, hints = get_dtype_and_schema_of_array(column_values)
+            value[column_name] = column_values.to_numpy(dtype=dtype)
+            if hints:
+                schema_hints.update({column_name: hints})
+
+        schema_hints.update({"index": index_column_name})
+        array[:] = value
+        array.meta["cxg_schema"] = json.dumps(schema_hints)
+
+    tiledb.consolidate(array_name, ctx=ctx)
+
+
+def convert_ndarray_to_cxg_dense_array(ndarray_name, ndarray, ctx):
+    """
+    Saves contents of ndarray to the CXG output directory specified.
+
+    Generally this function is used to convert dataset embeddings. Because embeddings are typically accessed with
+    very large slices (or all of the embedding), they do not benefit from overly aggressive compression due to their
+    format.  Given this, we use a large tile size (1000) but only default compression level.
+    """
+
+    def create_ndarray_array(ndarray_name, ndarray):
+        filters = tiledb.FilterList([tiledb.ZstdFilter()])
+        attrs = [tiledb.Attr(dtype=ndarray.dtype, filters=filters)]
+        dimensions = [
+            tiledb.Dim(
+                domain=(0, ndarray.shape[dimension] - 1), tile=min(ndarray.shape[dimension], 1000), dtype=np.uint32
+            )
+            for dimension in range(ndarray.ndim)
+        ]
+        domain = tiledb.Domain(*dimensions)
+        schema = tiledb.ArraySchema(
+            domain=domain, sparse=False, attrs=attrs, capacity=1_000_000, cell_order="row-major", tile_order="row-major"
+        )
+        tiledb.DenseArray.create(ndarray_name, schema)
+
+    create_ndarray_array(ndarray_name, ndarray)
+
+    with tiledb.DenseArray(ndarray_name, mode="w", ctx=ctx) as array:
+        array[:] = ndarray
+
+    tiledb.consolidate(ndarray_name, ctx=ctx)
+
+
+def convert_matrix_to_cxg_array(
+    matrix_name, matrix, encode_as_sparse_array, ctx, column_shift_for_sparse_encoding=None
+):
+    """
+    Converts a numpy array matrix into a TileDB SparseArray of DenseArray based on whether `encode_as_sparse_array`
+    is true or not. Note that when the matrix is encoded as a SparseArray, it only writes the values that are
+    nonzero. This means that if you count the number of elements in the SparseArray, it will not equal the total
+    number of elements in the matrix, only the number of nonzero elements.
+
+    Furthermore, if the `column_shift_for_sparse_encoding` matrix is not None, this function will subtract the sparse
+    encoding from the original given matrix and as previously stated, only write the nonzero values to the TileDB
+    SparseArray.
+    """
+
+    def create_matrix_array(matrix_name, number_of_rows, number_of_columns, encode_as_sparse_array):
+        filters = tiledb.FilterList([tiledb.ZstdFilter()])
+        attrs = [tiledb.Attr(dtype=np.float32, filters=filters)]
+        if encode_as_sparse_array:
+            domain = tiledb.Domain(
+                tiledb.Dim(name="obs", domain=(0, number_of_rows - 1), tile=min(number_of_rows, 512), dtype=np.uint32),
+                tiledb.Dim(
+                    name="var", domain=(0, number_of_columns - 1), tile=min(number_of_columns, 2048), dtype=np.uint32
+                ),
+            )
+        else:
+            domain = tiledb.Domain(
+                tiledb.Dim(name="obs", domain=(0, number_of_rows - 1), tile=min(number_of_rows, 50), dtype=np.uint32),
+                tiledb.Dim(
+                    name="var", domain=(0, number_of_columns - 1), tile=min(number_of_columns, 100), dtype=np.uint32
+                ),
+            )
+        schema = tiledb.ArraySchema(
+            domain=domain, sparse=encode_as_sparse_array, attrs=attrs, cell_order="row-major", tile_order="col-major"
+        )
+        if encode_as_sparse_array:
+            tiledb.SparseArray.create(matrix_name, schema)
+        else:
+            tiledb.DenseArray.create(matrix_name, schema)
+
+    number_of_rows = matrix.shape[0]
+    number_of_columns = matrix.shape[1]
+    stride = min(int(np.power(10, np.around(np.log10(1e9 / number_of_columns)))), 10_000)
+
+    create_matrix_array(matrix_name, number_of_rows, number_of_columns, encode_as_sparse_array)
+
+    if encode_as_sparse_array:
+        with tiledb.SparseArray(matrix_name, mode="w", ctx=ctx) as array:
+            for start_row_index in range(0, number_of_rows, stride):
+                end_row_index = min(start_row_index + stride, number_of_rows)
+                matrix_subset = matrix[start_row_index:end_row_index, :]
+                if not isinstance(matrix_subset, np.ndarray):
+                    matrix_subset = matrix_subset.toarray()
+                if column_shift_for_sparse_encoding is not None:
+                    matrix_subset = matrix_subset - column_shift_for_sparse_encoding
+                indices = np.nonzero(matrix_subset)
+                trow = indices[0] + start_row_index
+                array[trow, indices[1]] = matrix_subset[indices[0], indices[1]]
+
+    else:
+        with tiledb.DenseArray(matrix_name, mode="w", ctx=ctx) as array:
+            for start_row_index in range(0, number_of_rows, stride):
+                end_row_index = min(start_row_index + stride, number_of_rows)
+                matrix_subset = matrix[start_row_index:end_row_index, :]
+                if not isinstance(matrix_subset, np.ndarray):
+                    matrix_subset = matrix_subset.toarray()
+                array[start_row_index:end_row_index, :] = matrix_subset

backend/czi_hosted/common/utils/matrix_utils.py

@@ -0,0 +1,115 @@
+import logging
+
+import numpy as np
+from scipy.stats import mode
+
+
+def is_matrix_sparse(matrix: np.ndarray, sparse_threshold):
+    """
+    Returns whether `matrix` is sparse or not (i.e. dense). This is determined by figuring out whether the matrix has
+    a sparsity percentage below the sparse_threshold, returning the number of non-zeros encountered and number of
+    elements evaluated.  This function may return before evaluating the whole matrix if it can be determined that matrix
+    is not sparse enough.
+    """
+
+    if sparse_threshold == 100.0:
+        return True
+    if sparse_threshold == 0.0:
+        return False
+
+    total_number_of_rows = matrix.shape[0]
+    total_number_of_columns = matrix.shape[1]
+    total_number_of_matrix_elements = total_number_of_rows * total_number_of_columns
+
+    # For efficiency, we count the number of non-zero elements in chunks of the matrix at a time until we hit the
+    # maximum number of non zero values allowed before the matrix is deemed "dense." This allows the function the
+    # quit early for large dense matrices.
+    row_stride = min(int(np.power(10, np.around(np.log10(1e9 / total_number_of_columns)))), 10_000)
+
+    maximum_number_of_non_zero_elements_in_matrix = int(
+        total_number_of_rows * total_number_of_columns * sparse_threshold / 100
+    )
+    number_of_non_zero_elements = 0
+
+    for start_row_index in range(0, total_number_of_rows, row_stride):
+        end_row_index = min(start_row_index + row_stride, total_number_of_rows)
+
+        matrix_subset = matrix[start_row_index:end_row_index, :]
+        if not isinstance(matrix_subset, np.ndarray):
+            matrix_subset = matrix_subset.toarray()
+
+        number_of_non_zero_elements += np.count_nonzero(matrix_subset)
+        if number_of_non_zero_elements > maximum_number_of_non_zero_elements_in_matrix:
+            if end_row_index != total_number_of_rows:
+                percentage_of_non_zero_elements = (
+                    100 * number_of_non_zero_elements / (end_row_index * total_number_of_columns)
+                )
+                logging.info(
+                    f"Matrix is not sparse. Percentage of non-zero elements (estimate): "
+                    f"{percentage_of_non_zero_elements:6.2f}"
+                )
+            else:
+                percentage_of_non_zero_elements = 100 * number_of_non_zero_elements / total_number_of_matrix_elements
+                logging.info(
+                    f"Matrix is not sparse. Percentage of non-zero elements (exact): "
+                    f"{percentage_of_non_zero_elements:6.2f}"
+                )
+            return False
+
+    is_sparse = (100.0 * number_of_non_zero_elements / total_number_of_matrix_elements) < sparse_threshold
+    return is_sparse
+
+
+def get_column_shift_encode_for_matrix(matrix, sparse_threshold):
+    """
+    Returns a column shift if there is a column shift that allows the given matrix to be considered as sparse. Column
+    shift encoding works by taking the most common value in each column, then subtracting that value from each  element
+    of the column.  If each column mostly contains its most common value, then the resulting matrix can be very sparse.
+
+    This function determines if column shift encoding can be used to transform the matrix into a sparse matrix with a
+    sparsity below the sparse_threshold. If so, returns the array that stores this encoding. This function also returns
+    the number of non-zeros encountered and number of elements evaluated.  This function may return before evaluating
+    the whole matrix if it can be determined that the matrix cannot benefit from column shift encoding.
+    """
+
+    total_number_of_rows = matrix.shape[0]
+    total_number_of_columns = matrix.shape[1]
+    total_number_of_matrix_elements = total_number_of_rows * total_number_of_columns
+
+    stride = max(1, 128_000_000 // total_number_of_rows)
+    column_shift = np.zeros(total_number_of_columns)
+
+    maximum_number_of_non_zero_elements_in_matrix = int(
+        total_number_of_rows * total_number_of_columns * sparse_threshold / 100
+    )
+    number_of_non_zero_elements = 0
+
+    for start_column_index in range(0, total_number_of_columns, stride):
+        end_column_index = min(start_column_index + stride, total_number_of_columns)
+
+        matrix_subset = matrix[:, start_column_index:end_column_index]
+        if not isinstance(matrix_subset, np.ndarray):
+            matrix_subset = matrix_subset.toarray()
+
+        matrix_subset_mode = mode(matrix_subset)
+
+        column_shift[start_column_index:end_column_index] = matrix_subset_mode.mode
+        number_of_non_zero_elements += total_number_of_rows * (end_column_index - start_column_index) - np.sum(
+            matrix_subset_mode.count
+        )
+
+        if number_of_non_zero_elements > maximum_number_of_non_zero_elements_in_matrix:
+            if end_column_index != total_number_of_columns:
+                logging.info(
+                    "Matrix is not sparse even with column shift. Percentage of non-zero elements (estimate): %6.2f"
+                    % (100 * number_of_non_zero_elements / end_column_index * total_number_of_rows)
+                )
+            else:
+                logging.info(
+                    "Matrix is not sparse even with column shift. Percentage of non-zero elements (exact): %6.2f"
+                    % (100 * number_of_non_zero_elements / total_number_of_matrix_elements)
+                )
+            return None
+
+    is_sparse = (100.0 * number_of_non_zero_elements / total_number_of_matrix_elements) < sparse_threshold
+    return column_shift if is_sparse else None

backend/czi_hosted/common/utils/sanitization_utils.py

@@ -0,0 +1,40 @@
+import re
+
+
+def sanitize_values_in_list(list_of_keys: list):
+    """
+    Returns a dictionary mapping of the old keys in the list of `list_of_keys` to its new, clean name that is both
+    safe and unique.
+    """
+
+    if not all([isinstance(key, str) for key in list_of_keys]):
+        raise Exception("List of keys to sanitize must contain all strings.")
+
+    # Mask out [~/.] and anything outside the ASCII range.
+    mask = re.compile(r"[^ -\-0-\[\]-\}]")
+    clean_keys_list = [mask.sub("_", key) for key in list_of_keys]
+
+    # Dedupe the clean keys list
+    deduped_clean_keys_list = []
+    for index, clean_key in enumerate(clean_keys_list):
+        total_occurrences_of_clean_key = clean_keys_list.count(clean_key)
+        total_occurrences_up_until_current_index = clean_keys_list[:index].count(clean_key)
+        deduped_clean_keys_list.append(
+            clean_key + "_" + str(total_occurrences_up_until_current_index + 1)
+            if total_occurrences_of_clean_key > 1
+            else clean_key
+        )
+
+    return dict(zip(list_of_keys, deduped_clean_keys_list))
+
+
+def sanitize_keys_in_dictionary(dict_to_sanitize: dict):
+    """
+    Clean and dedupe the keys in the given dictionary.
+    """
+
+    clean_keys = sanitize_values_in_list(dict_to_sanitize.keys())
+    for original_key, sanitized_key in clean_keys.items():
+        if original_key != sanitized_key:
+            dict_to_sanitize[sanitized_key] = dict_to_sanitize[original_key]
+            del dict_to_sanitize[original_key]

backend/czi_hosted/compute/diffexp_cxg.py

@@ -1,10 +1,11 @@
 import concurrent.futures
 import numpy as np
-from server.compute.diffexp_generic import diffexp_ttest_from_mean_var, mean_var_n
-from server.data_cxg.cxg_util import pack_selector_from_indices
-from server.common.errors import ComputeError
+
 from numba import jit
 
+from backend.czi_hosted.data_cxg.cxg_util import pack_selector_from_indices
+from backend.common.compute.diffexp_generic import diffexp_ttest_from_mean_var, mean_var_n
+from backend.common.errors import ComputeError
 
 """
 See the comments in diffexp_generic for a description of this algorithm
@@ -114,14 +115,14 @@ def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
             meanB += X_col_shift
 
     r = diffexp_ttest_from_mean_var(
-        meanA.astype(dtype),
-        varA.astype(dtype),
-        nA,
-        meanB.astype(dtype),
-        varB.astype(dtype),
-        nB,
-        top_n,
-        diffexp_lfc_cutoff,
+        meanA=meanA.astype(dtype),
+        varA=varA.astype(dtype),
+        nA=nA,
+        meanB=meanB.astype(dtype),
+        varB=varB.astype(dtype),
+        nB=nB,
+        top_n=top_n,
+        diffexp_lfc_cutoff=diffexp_lfc_cutoff
     )
 
     return r

backend/czi_hosted/converters/h5ad_data_file.py

@@ -0,0 +1,250 @@
+import json
+import logging
+from os import path
+
+import anndata
+import numpy as np
+import tiledb
+
+from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
+from backend.czi_hosted.common.corpora import corpora_get_props_from_anndata
+from backend.common.errors import ColorFormatException
+from backend.czi_hosted.common.utils.cxg_constants import CxgConstants
+from backend.czi_hosted.common.utils.cxg_generation_utils import (
+    convert_dictionary_to_cxg_group,
+    convert_dataframe_to_cxg_array,
+    convert_ndarray_to_cxg_dense_array,
+    convert_matrix_to_cxg_array,
+)
+from backend.czi_hosted.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix
+
+
+class H5ADDataFile:
+    """ Class encapsulating required information about an H5AD datafile that ultimately will be transformed into
+    another format (currently just CXG is supported). """
+
+    def __init__(
+        self,
+        input_filename,
+        backed=False,
+        dataset_title=None,
+        dataset_about=None,
+        obs_index_column_name=None,
+        vars_index_column_name=None,
+        use_corpora_schema=True,
+    ):
+        self.input_filename = input_filename
+        self.backed = backed
+        self.dataset_title = dataset_title
+        self.dataset_about = dataset_about
+        self.obs_index_column_name = obs_index_column_name
+        self.vars_index_column_name = vars_index_column_name
+
+        self.use_corpora_schema = use_corpora_schema
+
+        self.validate_input_file_type()
+
+        self.extract_anndata_elements_from_file()
+        self.extract_metadata_about_dataset()
+
+        self.validate_anndata()
+
+    def to_cxg(self, output_cxg_directory, sparse_threshold, convert_anndata_colors_to_cxg_colors=True):
+        """
+        Writes the following attributes of the anndata to CXG: 1) the metadata as metadata attached to an empty
+        DenseArray, 2) the obs DataFrame as a DenseArray, 3) the var DataFrame as a DenseArray, 4) all valid
+        embeddings stored in obsm, each one as a DenseArray, 5) the main X matrix of the anndata as either a
+        SparseArray or DenseArray based on the `sparse_threshold`, and optionally 6) the column shift of the main X
+        matrix that might turn an otherwise Dense matrix into a Sparse matrix.
+        """
+
+        logging.info("Beginning writing to CXG.")
+        ctx = tiledb.Ctx(
+            {
+                "sm.num_reader_threads": 32,
+                "sm.num_writer_threads": 32,
+                "sm.consolidation.buffer_size": 1 * 1024 * 1024 * 1024,
+            }
+        )
+
+        tiledb.group_create(output_cxg_directory, ctx=ctx)
+        logging.info(f"\t...group created, with name {output_cxg_directory}")
+
+        convert_dictionary_to_cxg_group(
+            output_cxg_directory, self.generate_cxg_metadata(convert_anndata_colors_to_cxg_colors)
+        )
+        logging.info("\t...dataset metadata saved")
+
+        convert_dataframe_to_cxg_array(output_cxg_directory, "obs", self.obs, self.obs_index_column_name, ctx)
+        logging.info("\t...dataset obs dataframe saved")
+
+        convert_dataframe_to_cxg_array(output_cxg_directory, "var", self.var, self.var_index_column_name, ctx)
+        logging.info("\t...dataset var dataframe saved")
+
+        self.write_anndata_embeddings_to_cxg(output_cxg_directory, ctx)
+        logging.info("\t...dataset embeddings saved")
+
+        self.write_anndata_x_matrix_to_cxg(output_cxg_directory, ctx, sparse_threshold)
+        logging.info("\t...dataset X matrix saved")
+
+        logging.info("Completed writing to CXG.")
+
+    def write_anndata_x_matrix_to_cxg(self, output_cxg_directory, ctx, sparse_threshold):
+        matrix_container = f"{output_cxg_directory}/X"
+
+        x_matrix_data = self.anndata.X
+        is_sparse = is_matrix_sparse(x_matrix_data, sparse_threshold)
+        if not is_sparse:
+            col_shift = get_column_shift_encode_for_matrix(x_matrix_data, sparse_threshold)
+            is_sparse = col_shift is not None
+        else:
+            col_shift = None
+
+        if col_shift is not None:
+            logging.info("Converting matrix X as sparse matrix with column shift encoding")
+            x_col_shift_name = f"{output_cxg_directory}/X_col_shift"
+            convert_ndarray_to_cxg_dense_array(x_col_shift_name, col_shift, ctx)
+
+        convert_matrix_to_cxg_array(matrix_container, x_matrix_data, is_sparse, ctx, col_shift)
+
+        tiledb.consolidate(matrix_container, ctx=ctx)
+        if hasattr(tiledb, "vacuum"):
+            tiledb.vacuum(matrix_container)
+
+    def write_anndata_embeddings_to_cxg(self, output_cxg_directory, ctx):
+        def is_valid_embedding(adata, embedding_name, embedding_array):
+            """
+            Returns true if this layout data is a valid array for front-end presentation with the following criteria:
+                * ndarray, with shape (n_obs, >= 2), dtype float/int/uint
+                * follows ScanPy embedding naming conventions
+                * with all values finite or NaN (no +Inf or -Inf)
+            """
+
+            is_valid = isinstance(embedding_name, str) and embedding_name.startswith("X_") and len(embedding_name) > 2
+            is_valid = is_valid and isinstance(embedding_array, np.ndarray) and embedding_array.dtype.kind in "fiu"
+            is_valid = is_valid and embedding_array.shape[0] == adata.n_obs and embedding_array.shape[1] >= 2
+            is_valid = is_valid and not np.any(np.isinf(embedding_array)) and not np.all(np.isnan(embedding_array))
+            return is_valid
+
+        embedding_container = f"{output_cxg_directory}/emb"
+        tiledb.group_create(embedding_container, ctx=ctx)
+
+        for embedding_name, embedding_values in self.anndata.obsm.items():
+            if is_valid_embedding(self.anndata, embedding_name, embedding_values):
+                embedding_name = f"{embedding_container}/{embedding_name[2:]}"
+                convert_ndarray_to_cxg_dense_array(embedding_name, embedding_values, ctx)
+                logging.info(f"\t\t...{embedding_name} embedding created")
+
+    def generate_cxg_metadata(self, convert_anndata_colors_to_cxg_colors):
+        """
+        Return a dictionary containing metadata about CXG dataset. This include data about the version as well as
+        Corpora schema properties if they exist, among other pieces of metadata.
+        """
+
+        cxg_group_metadata = {
+            "cxg_version": CxgConstants.CXG_VERSION,
+            "cxg_properties": json.dumps({"title": self.dataset_title, "about": self.dataset_about}),
+        }
+        if self.corpora_properties is not None:
+            cxg_group_metadata["corpora"] = json.dumps(self.corpora_properties)
+
+        if convert_anndata_colors_to_cxg_colors:
+            try:
+                cxg_group_metadata["cxg_category_colors"] = json.dumps(
+                    convert_anndata_category_colors_to_cxg_category_colors(self.anndata)
+                )
+            except ColorFormatException:
+                logging.warning(
+                    "Failed to extract colors from H5AD file! Fix the H5AD file or rerun with "
+                    "--disable-custom-colors. See help for more details."
+                )
+
+        return cxg_group_metadata
+
+    def validate_input_file_type(self):
+        """
+        Validate that the input file is of a type that we can handle. Currently the only valid file type is `.h5ad`.
+        """
+
+        if not self.input_filename.endswith(".h5ad"):
+            raise Exception(f"Cannot process input file {self.input_filename}. File must be an H5AD.")
+
+        if self.dataset_title or self.dataset_about:
+            logging.warning(
+                "If you convert this dataset into CXG and you explicit specify values for the dataset title metadata "
+                "or the dataset about metadata, it will override any metadata that is extracted as part of the "
+                "Corpora schema fields."
+            )
+
+    def validate_anndata(self):
+        if not self.var.index.is_unique:
+            raise ValueError("Variable index in AnnData object is not unique.")
+        if not self.obs.index.is_unique:
+            raise ValueError("Observation index in AnnData object is not unique.")
+
+    def extract_anndata_elements_from_file(self):
+        logging.info(f"Reading in AnnData dataset: {path.basename(self.input_filename)}")
+        self.anndata = anndata.read_h5ad(self.input_filename, backed="r" if self.backed else None)
+        logging.info("Completed reading in AnnData dataset!")
+
+        self.obs = self.transform_dataframe_index_into_column(self.anndata.obs, "obs", self.obs_index_column_name)
+        self.var = self.transform_dataframe_index_into_column(self.anndata.var, "var", self.vars_index_column_name)
+
+    def extract_metadata_about_dataset(self):
+        """
+        Extract metadata information about the dataset that upon conversion will be saved as group metadata with the
+        CXG that is generated. This metadata information includes Corpora schema properties, the dataset title and
+        a link that details more information about the dataset.
+        """
+
+        self.corpora_properties = corpora_get_props_from_anndata(self.anndata) if self.use_corpora_schema else None
+        if self.corpora_properties is None and self.use_corpora_schema:
+            # If the return value is None, this means that we were not able to figure out what version of the Corpora
+            # schema the object is using and therefore cannot extract any properties.
+            raise ValueError("Unknown source file schema version is unsupported.")
+
+        # The title and about properties of the dataset are set by the following order: if they are explicitly defined
+        # then use the explicit value. If the dataset is a Corpora-schema based schema, then extract the title and about
+        # from the corpora_properties. Otherwise, use the input filename (only for title, about will be blank).
+        if self.corpora_properties:
+            corpora_project_links = self.corpora_properties.get("project_links", [])
+            corpora_about_link = next(
+                (link for link in corpora_project_links if (link.get("link_type", None) == "SUMMARY")), {}
+            )
+        else:
+            corpora_about_link = {}
+
+        filename = path.splitext(path.basename(self.input_filename))[0]
+
+        self.dataset_title = self.dataset_title if self.dataset_title else corpora_about_link.get("link_name", filename)
+        self.dataset_about = self.dataset_about if self.dataset_about else corpora_about_link.get("link_url")
+
+    def transform_dataframe_index_into_column(self, dataframe, dataframe_name, index_column_name):
+        """
+        Convert the dataframe's index into another column in the dataframe. If an index_column_name is specified,
+        use that column as the index instead.
+        """
+
+        if index_column_name is None:
+            # Create a unique column name for the index.
+            suffix = 0
+            while f"name_{suffix}" in dataframe.columns:
+                suffix += 1
+            index_column_name = f"name_{suffix}"
+
+            # Turn the index into a normal column
+            dataframe.rename_axis(index_column_name, inplace=True)
+            dataframe.reset_index(inplace=True)
+
+        elif index_column_name in dataframe.columns:
+            # User has specified alternative column for unique names, and it exists
+            if not dataframe[index_column_name].is_unique:
+                raise KeyError(
+                    f"Values in {dataframe_name}.{index_column_name} must be unique. Please prepare data to contain "
+                    f"unique values."
+                )
+        else:
+            raise KeyError(f"Column {index_column_name} does not exist.")
+
+        setattr(self, f"{dataframe_name}_index_column_name", index_column_name)
+        return dataframe

backend/czi_hosted/converters/schema/gene_symbol.py

@@ -0,0 +1,211 @@
+"""Helpers for converting and checking HGNC gene symbols."""
+
+import argparse
+import enum
+import logging
+import os
+import re
+import numpy as np
+import pandas as pd
+
+
+def get_upgraded_var_index(var, hgnc_path=None):
+    """Given an anndata var dataframe, return a new index for the dataframe
+    where human gene symbols have been upgraded to the current HGNC set.
+    """
+
+    if not hgnc_path:
+        hgnc_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "hgnc_complete_set.txt.gz")
+
+    hgnc_symbol_checker = HGNCSymbolChecker.from_hgnc_records(hgnc_path)
+
+    return pd.Index([hgnc_symbol_checker.upgrade_symbol(s) for s in var.index])
+
+
+class SymbolStatus(enum.Enum):
+    """The status of a symbol in the HGNC database.
+
+    APPROVED: Currently a valid symbol
+    WITHDRAWN: A previously approved HGNC symbol for a gene that has since been shown
+               not to exist _unless_ that symbol is also approved
+    AMBIGUOUS: A symbol that is not approved but is an alias or previous symbol for
+               multiple approved symbols
+    UPGRADABLE: A symbol that is not approved but unambiguously maps to an approved
+                symbol
+    UNKNOWN: A symbol that does not appear in HGNC
+    """
+
+    APPROVED = 1
+    WITHDRAWN = 2
+    AMBIGUOUS = 3
+    UPGRADABLE = 4
+    UNKNOWN = 5
+
+
+class HGNCSymbolChecker:
+    """Handle checking and correcting HGNC symbols."""
+
+    def __init__(self, approved_symbols, withdrawn_symbols, ambiguous_symbols, symbol_map):
+        self.approved_symbols = approved_symbols
+        self.withdrawn_symbols = withdrawn_symbols
+        self.ambiguous_symbols = ambiguous_symbols
+        self.symbol_map = symbol_map
+
+    def print_symbol_map(self):
+        """Print out a map from old symbol to new symbol."""
+
+        for symbol_pair in self.symbol_map.items():
+            print("\t".join(symbol_pair))
+
+    def check_symbol(self, symbol):
+        """See if a symbol if approved or something else."""
+        if symbol in self.approved_symbols:
+            return SymbolStatus.APPROVED
+
+        if symbol in self.withdrawn_symbols:
+            return SymbolStatus.WITHDRAWN
+
+        if symbol in self.ambiguous_symbols:
+            return SymbolStatus.AMBIGUOUS
+
+        if symbol in self.symbol_map:
+            return SymbolStatus.UPGRADABLE
+
+        return SymbolStatus.UNKNOWN
+
+    def upgrade_symbol(self, symbol):
+        """Return the approved symbol for the given symbol.
+
+        If the symbol cannot be upgraded, just return the original symbol.
+        """
+
+        fixed_symbol, stripped_symbol = format_symbol(symbol)
+
+        if fixed_symbol in self.approved_symbols:
+            return fixed_symbol
+        elif fixed_symbol in self.symbol_map:
+            return self.symbol_map[fixed_symbol]
+        elif stripped_symbol in self.approved_symbols:
+            return stripped_symbol
+        elif stripped_symbol in self.symbol_map:
+            return self.symbol_map[stripped_symbol]
+
+        return symbol
+
+    @classmethod
+    def from_hgnc_records(cls, hgnc_dataset_path):
+        """Parse a hgnc database download into a HGNCSymbolChecker object."""
+
+        def all_symbols(record):
+            """Get all the symbols associated with an HGNC record including previous, alias,
+            and approved."""
+            yield format_symbol(record["symbol"])[0]
+            for symbol in alias_and_previous_symbols(record):
+                yield symbol
+
+        def alias_and_previous_symbols(record):
+            """Get alias and previous symbols from an HGNC record."""
+            for field in ("alias_symbol", "prev_symbol"):
+                if record[field] is not np.nan:
+                    for symbol in record[field].split("|"):
+                        yield format_symbol(symbol)[0]
+            # Sometimes something like HGNC:1234 appears in datasets, which we
+            # want to fix as well.
+            yield record["hgnc_id"]
+
+        hgnc_records = pd.read_csv(hgnc_dataset_path, sep="\t", header=0, low_memory=False).to_dict("records")
+
+        # Get all symbols that are currently approved.
+        approved_symbols = set()
+        for record in hgnc_records:
+            if record["status"] == "Approved":
+                approved_symbols.add(format_symbol(record["symbol"])[0])
+
+        # Get all symbols that have been withdrawn
+        withdrawn_symbols = set()
+        for record in hgnc_records:
+            if record["status"] == "Entry Withdrawn":
+                for symbol in all_symbols(record):
+                    withdrawn_symbols.add(symbol)
+
+        # If a symbol is both approved and withdrawn, be optimistic and call it approved
+        logging.warning(
+            f"Some symbols are simulaneously withdrawn and approved\n"
+            f"We will treat them at approved:\n"
+            f"{withdrawn_symbols.intersection(approved_symbols)}"
+        )
+        withdrawn_symbols = withdrawn_symbols.difference(approved_symbols)
+
+        # Now try to map from symbols that are not approved but are an alias or previous symbol for an approved symbol
+        alias_previous_to_approved = {}
+        ambiguous_symbols = set()
+
+        for record in hgnc_records:
+            if record["status"] == "Approved":
+
+                # The approved symbol is what we'll map to
+                approved_symbol = format_symbol(record["symbol"])[0]
+
+                for symbol in alias_and_previous_symbols(record):
+
+                    # If the alias or previous symbol is also an approved symbol,
+                    # we'll just leave it alone
+                    if symbol in approved_symbols:
+                        continue
+
+                    # If the alias or previous symbol maps to a different approved symbol, mark it as ambiguous
+                    if symbol in alias_previous_to_approved and alias_previous_to_approved[symbol] != approved_symbol:
+                        ambiguous_symbols.add(symbol)
+                    else:
+                        alias_previous_to_approved[symbol] = approved_symbol
+
+        # Remove all the ambiguous symbols from the map
+        for ambiguous_symbol in ambiguous_symbols:
+            alias_previous_to_approved.pop(ambiguous_symbol)
+
+        return HGNCSymbolChecker(approved_symbols, withdrawn_symbols, ambiguous_symbols, alias_previous_to_approved)
+
+
+def format_symbol(symbol):
+    """HGNC rules say symbols should all be upper case except for C#orf#. However, case is
+    variable in both alias and previous symbols as well as in the symbols we get in
+    submissions. So, upper case everything except for the one situation where mixed-case
+    is allowed, which are the genes like C2orf157.
+
+    Also, seurat and scanpy append ".1" or "-1" to duplicated gene names, and these altered
+    names persist throughout the life of the object. They won't match against the HGNC database
+    and we want to merge them, so we need to strip off the suffix and try matching again.
+
+    This function takes a symbol and returns the symbol with the fixed case and also with the
+    seurat/scanpy suffix stripped off.
+    """
+
+    match = re.match(r"^(C)(\d+)(orf)(\d+)$", symbol, re.IGNORECASE)
+
+    if match:
+        fixed_case = f"C{match.group(2)}orf{match.group(4)}"
+    else:
+        fixed_case = symbol.upper()
+
+    suffix_stripped = re.sub(r"[\.\-]\d+$", "", fixed_case)
+
+    return fixed_case, suffix_stripped
+
+
+def main():
+    """When called as main, parse a given hgnc download and print out a map from old to new
+    symbol.
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "hgnc_dataset", help="HGNC dataset tsv, available from www.genenames.org/download/statistics-and-files/"
+    )
+    args = parser.parse_args()
+
+    hgnc_symbol_checker = HGNCSymbolChecker.from_hgnc_records(args.hgnc_dataset)
+
+    hgnc_symbol_checker.print_symbol_map()
+
+
+if __name__ == "__main__":
+    main()

backend/czi_hosted/converters/schema/ontology.py

@@ -0,0 +1,86 @@
+"""Methods for working with ontologies and the OLS."""
+from urllib.parse import quote_plus
+
+import requests
+
+OLS_API_ROOT = "http://www.ebi.ac.uk/ols/api"
+
+# Curie means something like CL:0000001
+
+
+def _ontology_name(curie):
+    """Get the name of the ontology from the curie, CL or UBERON for example."""
+    return curie.split(":")[0]
+
+
+def _ontology_value(curie):
+    """Get the id component of the curie, 0000001 from CL:0000001 for example."""
+    return curie.split(":")[1]
+
+
+def _double_encode(url):
+    """Double url encode a url. This is required by the OLS API."""
+    return quote_plus(quote_plus(url))
+
+
+def _iri(curie):
+    """Get the iri from a curie. This is a bit hopeful that they all map to purl.obolibrary.org"""
+    if _ontology_name(curie) == "EFO":
+        return f"http://www.ebi.ac.uk/efo/EFO_{_ontology_value(curie)}"
+    return f"http://purl.obolibrary.org/obo/{_ontology_name(curie)}_{_ontology_value(curie)}"
+
+
+class OntologyLookupError(Exception):
+    """Exception for some problem with looking up ontology information."""
+
+
+def _ontology_info_url(curie):
+    """Get the to make a GET to to get information about an ontology term."""
+
+    # If the curie is empty, just return an empty string. This happens when there is no
+    # valid ontology value.
+    if not curie:
+        return ""
+    else:
+        return f"{OLS_API_ROOT}/ontologies/{_ontology_name(curie)}/terms/{_double_encode(_iri(curie))}"
+
+
+def get_ontology_label(curie):
+    """For a given curie like 'CL:1000413', get the label like 'endothelial cell of artery'"""
+
+    url = _ontology_info_url(curie)
+
+    if not url:
+        return ""
+
+    response = requests.get(url)
+
+    if not response.ok:
+        raise OntologyLookupError(
+            f"Curie {curie} lookup failed, got status code {response.status_code}: {response.text}"
+        )
+    return response.json()["label"]
+
+
+def lookup_candidate_term(label, ontology="cl", method="select"):
+    """Lookup candidate terms for a label. This is useful when there is an existing label in a
+    submitted dataset, and you want to find an appropriate ontology term.
+
+    Args:
+      label: the label to find ontology terms for
+      ontology: the ontology to search in, cl or uberon or efo for example
+      method: select or search. search provides much broader results
+
+    Returns:
+      list of (curie, label) tuples returned by OLS
+    """
+    # using OLS REST API [https://www.ebi.ac.uk/ols/docs/api]
+    url = f"{OLS_API_ROOT}/{method}?q={quote_plus(label)}&ontology={ontology.lower()}"
+    response = requests.get(url)
+
+    if not response.ok:
+        raise OntologyLookupError(
+            f"Label {label} lookup failed, got status code {response.status_code}: {response.text}"
+        )
+
+    return [(r["obo_id"], r["label"]) for r in response.json()["response"]["docs"]]

backend/czi_hosted/converters/schema/remix.py

@@ -0,0 +1,264 @@
+import argparse
+import collections
+import json
+import logging
+import math
+import string
+
+import anndata
+import numpy as np
+import pandas as pd
+import yaml
+
+from . import gene_symbol
+from . import ontology
+from . import validate
+
+REPLACE_SUFFIX = "_original"
+ONTOLOGY_SUFFIX = "_ontology_term_id"
+
+
+def is_curie(value):
+    """Return True iff the value is an OBO-id CURIE like EFO:000001"""
+    return (value.count(":")
+            and all(len(part) > 0 for part in value.split(":"))
+            and all(c in string.digits for c in value.split(":")[1]))
+
+
+def is_ontology_field(field_name):
+    """Return True iff the field_name is an ontology field like tissue_ontology_term_id"""
+    return field_name.endswith(ONTOLOGY_SUFFIX)
+
+
+def get_label_field_name(field_name):
+    """Get the associated label field from an ontology field, assay_ontology_term_id --> assay"""
+    return field_name[: -len(ONTOLOGY_SUFFIX)]
+
+
+def split_suffix(maybe_curie):
+    """Split off the (cell culture) or (organoid) suffix."""
+
+    suffixes = [" (cell culture)", " (organoid)"]
+    for suffix in suffixes:
+        if maybe_curie.endswith(suffix):
+            return maybe_curie[:-len(suffix)], suffix
+    return maybe_curie, ""
+
+
+def get_curie_and_label(maybe_curie):
+    """Given a string that might be a curie, return a (curie, label) pair"""
+
+    maybe_curie, suffix = split_suffix(maybe_curie)
+    if not is_curie(maybe_curie):
+        return ("", maybe_curie + suffix)
+    return (maybe_curie + suffix, ontology.get_ontology_label(maybe_curie) + suffix)
+
+
+def safe_add_field(adata_attr, field_name, field_value):
+    """Add a field and value to an AnnData, but don't clobber an exising value."""
+
+    if (
+        isinstance(field_value, list)
+        and field_value
+        and isinstance(field_value[0], dict)
+    ):
+        field_value = json.dumps(field_value)
+    if field_name in adata_attr:
+        adata_attr[field_name + REPLACE_SUFFIX] = adata_attr[field_name]
+    adata_attr[field_name] = field_value
+
+
+def remix_uns(adata, uns_config):
+    """Add fields from the config to adata.uns"""
+    for field_name, field_value in uns_config.items():
+
+        if is_ontology_field(field_name):
+            # If it's an ontology field, look it up
+            label_field_name = get_label_field_name(field_name)
+            ontology_term, ontology_label = get_curie_and_label(field_value)
+            safe_add_field(adata.uns, field_name, ontology_term)
+            safe_add_field(adata.uns, label_field_name, ontology_label)
+        else:
+            safe_add_field(adata.uns, field_name, field_value)
+
+
+def remix_obs(adata, obs_config):
+    """Add fields from the config to adata.obs"""
+
+    for field_name, field_value in obs_config.items():
+
+        if isinstance(field_value, dict):
+            # If the value is a dict, that means we are supposed to map from an
+            # existing column to the new one
+            source_column, column_map = next(iter(field_value.items()))
+            nan_value = None
+            for key in column_map:
+                if isinstance(key, float) and math.isnan(key):
+                    nan_value = column_map[key]
+            if nan_value is not None:
+                column_map["nan"] = nan_value
+
+            for key in column_map:
+                if key not in adata.obs[source_column].unique():
+                    logging.warning(f'Key {key} not in adata.obs["{source_column}"]')
+
+            for value in adata.obs[source_column].unique():
+                if value not in column_map:
+                    logging.warning(f'Value {value} in adata.obs["{source_column}"] not in translation dict')
+
+            if is_ontology_field(field_name):
+                ontology_term_map, ontology_label_map = {}, {}
+                logging.info(f"Looking up labels for {field_name}")
+                for original_value, maybe_curie in column_map.items():
+                    curie, label = get_curie_and_label(maybe_curie)
+                    ontology_term_map[original_value] = curie
+                    ontology_label_map[original_value] = label
+                    logging.info(f"Mapping {original_value} -> {curie} -> {label}")
+
+                ontology_column = adata.obs[source_column].replace(
+                    ontology_term_map, inplace=False
+                )
+                label_column = adata.obs[source_column].replace(
+                    ontology_label_map, inplace=False
+                )
+
+                safe_add_field(adata.obs, field_name, ontology_column)
+                safe_add_field(
+                    adata.obs, get_label_field_name(field_name), label_column
+                )
+            else:
+                label_column = adata.obs[source_column].replace(
+                    column_map, inplace=False
+                )
+                safe_add_field(adata.obs, field_name, label_column)
+
+        else:
+            if is_ontology_field(field_name):
+                # If it's an ontology field, look it up
+                label_field_name = get_label_field_name(field_name)
+                ontology_term, ontology_label = get_curie_and_label(field_value)
+                safe_add_field(adata.obs, field_name, ontology_term)
+                safe_add_field(adata.obs, label_field_name, ontology_label)
+            else:
+                safe_add_field(adata.obs, field_name, field_value)
+
+
+def merge_df(df, domain, index, columns):
+    """
+    Given a dataframe with duplicate column labels, merge and return a dataframe where
+    the duplicates have been merged together, resulting in a dataframe with unique column
+    labels.
+
+    "merge" depends on the value of domain. If the domain is "raw", then duplicate columns
+    can just be summed. If it's "log1p" or "sqrt", it needs to be exp1m'd or squared, then
+    summed, and then logged or sqrt'd again.
+    """
+
+    if not isinstance(df, np.ndarray):
+        to_merge = df.toarray()
+    else:
+        to_merge = df
+    if domain == "raw":
+        merged_df = pd.DataFrame(to_merge, index=index, columns=columns).sum(
+            axis=1, level=0, skipna=False
+        )
+    elif domain == "log1p":
+        merged_df = (
+            pd.DataFrame(np.expm1(to_merge, dtype=np.float128), index=index, columns=columns)
+            .sum(axis=1, level=0, skipna=False)
+        )
+        merged_df = pd.DataFrame(np.log1p(merged_df.to_numpy()), index=merged_df.index, columns=merged_df.columns)
+    elif domain == "sqrt":
+        merged_df = (
+            pd.DataFrame(np.square(to_merge), index=index, columns=columns)
+            .sum(axis=1, level=0, skipna=False)
+        )
+        merged_df = pd.DataFrame(np.sqrt(merged_df.to_numpy()), index=merged_df.index, columns=merged_df.columns)
+
+    return merged_df
+
+
+def fixup_gene_symbols(adata, fixup_config):
+    """Update the var index to hold a consistent set of HGNC gene symbols."""
+
+    upgraded_var_index = gene_symbol.get_upgraded_var_index(adata.var)
+
+    merged_X = merge_df(adata.X, fixup_config["X"], adata.obs.index, upgraded_var_index)
+    fixup_adata = anndata.AnnData(
+        X=merged_X,
+        obs=adata.obs,
+        var=merged_X.columns.to_frame(name="hgnc_gene_symbol"),
+        uns=adata.uns,
+        obsm=adata.obsm,
+    )
+
+    for layer, domain in fixup_config.items():
+        if layer == "X":
+            continue
+        if layer == "raw.X":
+            df = adata.raw.X
+        else:
+            df = adata.layers[layer]
+
+        merged_df = merge_df(df, domain, adata.obs.index, upgraded_var_index)
+        assert merged_df.index.equals(merged_X.index)
+        assert merged_df.columns.equals(merged_X.columns)
+
+        if domain == "raw":
+            fixup_raw = anndata.AnnData(
+                X=merged_df,
+                obs=adata.obs,
+                var=merged_X.columns.to_frame(name="hgnc_gene_symbol"),
+            )
+            fixup_adata.raw = fixup_raw
+        else:
+            fixup_adata.layers[layer] = merged_df
+
+    return fixup_adata
+
+def _strip_version(adata):
+    """Remove version information from the AnnData object."""
+
+    if "version" in adata.uns_keys():
+        del adata.uns["version"]
+
+def apply_schema(source_h5ad, remix_config, output_filename):
+
+    try:
+        import scanpy
+    except ImportError:
+        raise ImportError("scanpy must be installed for cellxgene schema")
+    adata = scanpy.read_h5ad(source_h5ad)
+    config = yaml.load(open(remix_config), Loader=yaml.FullLoader)
+    remix_uns(adata, config["uns"])
+    remix_obs(adata, config["obs"])
+
+    if config.get("fixup_gene_symbols"):
+        adata = fixup_gene_symbols(adata, config["fixup_gene_symbols"])
+
+    if ("version" in adata.uns_keys()
+            and isinstance(adata.uns["version"], collections.Mapping)
+            and "corpora_schema_version" in adata.uns["version"]):
+        schema_version = adata.uns["version"]["corpora_schema_version"]
+        try:
+            validate.get_schema_definition(schema_version)
+        except ValueError:
+            logging.warning(f"Stripping version information out of AnnData because schema "
+                            f"version {schema_version} is unknown.")
+            _strip_version(adata)
+
+        if not validate.validate_adata(adata, shallow=False):
+            logging.warning(f"Stripping version information out of AnnData because it does not "
+                            f"follow schema version {schema_version} .")
+            _strip_version(adata)
+
+    adata.write_h5ad(output_filename, compression="gzip")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--source-h5ad", required=True)
+    parser.add_argument("--remix-config", required=True)
+    parser.add_argument("--output-filename", required=True)
+    args = parser.parse_args()
+    apply_schema(args.source_h5ad, args.remix_config, args.output_filename)

backend/czi_hosted/converters/schema/schema_definitions/1_0_0.yaml

@@ -0,0 +1,95 @@
+title: Corpora schema version 1.0.0
+type: anndata
+components:
+    uns:
+        type: dict
+        keys:
+            version:
+                type: dict
+                keys:
+                    corpora_schema_version: null
+                    corpora_encoding_version: null
+            title:
+                type: string
+            contributors:
+                type: stringified list of dicts
+            layer_descriptions:
+                type: dict
+                keys:
+                    X: null
+            organism:
+                type: string
+                nullable: false
+            organism_ontology_term_id:
+                type: curie
+                prefixes:
+                    - NCBITaxon
+    var:
+        type: dataframe
+        index:
+            type: human-readable string
+            unique: true
+    obs:
+        type: dataframe
+        index:
+            unique: true
+        columns:
+            tissue:
+                type: human-readable string
+                nullable: false
+            tissue_ontology_term_id:
+                type: suffixed curie
+                nullable: true
+                prefixes:
+                    - UBERON
+            assay:
+                type: human-readable string
+                nullable: false
+            assay_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - EFO
+            disease:
+                type: human-readable string
+                nullable: false
+            disease_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - MONDO
+                    - PATO
+            cell_type:
+                type: human-readable string
+                nullable: false
+            cell_type_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - CL
+                    - UBERON
+            sex:
+                type: string
+                enum:
+                    - male
+                    - female
+                    - mixed
+                    - unknown
+                    - other
+            ethnicity:
+                type: human-readable string
+                nullable: false
+            ethnicity_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - HANCESTRO
+            development_stage:
+                type: human-readable string
+                nullable: false
+            development_stage_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - HsapDv
+                    - EFO

backend/czi_hosted/converters/schema/schema_definitions/1_1_0.yaml

@@ -0,0 +1,93 @@
+title: Corpora schema version 1.1.0
+type: anndata
+components:
+    uns:
+        type: dict
+        keys:
+            version:
+                type: dict
+                keys:
+                    corpora_schema_version: null
+                    corpora_encoding_version: null
+            title:
+                type: string
+            layer_descriptions:
+                type: dict
+                keys:
+                    X: null
+            organism:
+                type: string
+                nullable: false
+            organism_ontology_term_id:
+                type: curie
+                prefixes:
+                    - NCBITaxon
+    var:
+        type: dataframe
+        index:
+            type: human-readable string
+            unique: true
+    obs:
+        type: dataframe
+        index:
+            unique: true
+        columns:
+            tissue:
+                type: human-readable string
+                nullable: false
+            tissue_ontology_term_id:
+                type: suffixed curie
+                nullable: true
+                prefixes:
+                    - UBERON
+            assay:
+                type: human-readable string
+                nullable: false
+            assay_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - EFO
+            disease:
+                type: human-readable string
+                nullable: false
+            disease_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - MONDO
+                    - PATO
+            cell_type:
+                type: human-readable string
+                nullable: false
+            cell_type_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - CL
+                    - UBERON
+            sex:
+                type: string
+                enum:
+                    - male
+                    - female
+                    - mixed
+                    - unknown
+                    - other
+            ethnicity:
+                type: human-readable string
+                nullable: false
+            ethnicity_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - HANCESTRO
+            development_stage:
+                type: human-readable string
+                nullable: false
+            development_stage_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - HsapDv
+                    - EFO

backend/czi_hosted/converters/schema/validate.py

@@ -0,0 +1,236 @@
+import json
+import re
+import os
+import sys
+
+import pandas as pd
+import yaml
+
+
+def _is_null(v):
+    """Return True if v is null, for one of the multiple ways a "null" value shows up in an h5ad."""
+    return pd.isnull(v) or (hasattr(v, "__len__") and len(v) == 0)
+
+
+def _validate_stringified_list_of_dicts(s):
+    """Verify that a string can be parsed into a list.
+
+    We have some types that are lists of dicts. Those cannot be stored directly in an h5ad, so we have to
+    json.dumps them. This verifies that we can load them back.
+    """
+
+    try:
+        list_ = json.loads(s)
+        if not isinstance(list_, list):
+            return False
+        for el in list_:
+            if not isinstance(el, dict):
+                return False
+        return True
+    except (json.JSONDecodeError, TypeError):
+        pass
+    return False
+
+
+def _validate_human_readable_string(s):
+    """Verify that a string is human-readable.
+
+    There are parts of the schema where a "human-readable" string is required. "Human-readable" is kind
+    of vague and subjective. I feel like I can read many strings. So here we just check for the main ways
+    that fails: someone puts in an ontology term id or and ensembl gene/transcript id.
+
+    Returns False if s is not a string or is one of those bad string types.
+    """
+
+    return isinstance(s, str) and (not re.match(r"[A-Z]\w+:\d+", s)) and (not re.match(r"ENS[GT]\d+$", s))
+
+
+def _validate_curie(c, prefixes):
+    """Verify that a string is a valid compact URI, like EFO:000001. If prefixes is not empty, make sure the
+    prefix of the curies is in prefixes.
+    """
+
+    if not c:
+        return True
+
+    match = re.match(r"([A-Z]\w+):\d+$", c)
+
+    if prefixes:
+        return match and match.group(1) in prefixes
+    else:
+        return match
+
+
+def _validate_suffixed_curie(c, prefixes):
+    """Verify that a string is a compact URI with an optional suffix like 'EFO:00001 (cell culture)'"""
+
+    # Pull off the suffix
+    suffix = re.findall(r"\ \(.*\)$", c)
+    if suffix:
+        c = c[: -len(suffix[0])]
+    return _validate_curie(c, prefixes)
+
+
+def _validate_column(column, column_name, df_name, schema_def):
+    """Given a schema definition and the column of a dataframe, verify that the column satifies
+    the schema.
+    """
+
+    errors = []
+
+    if schema_def.get("unique"):
+        if column.nunique() != len(column):
+            errors.append(f"Column {column_name} in dataframe {df_name} is not unique.")
+
+    if "nullable" in schema_def and not schema_def["nullable"]:
+        if any(_is_null(v) for v in column):
+            errors.append(f"Column {column_name} in dataframe {df_name} contains empty values.")
+
+    if schema_def.get("type") == "human-readable string":
+        non_readables = [v for v in column if not _validate_human_readable_string(v)]
+        if non_readables:
+            errors.append(
+                f"Column {column_name} in dataframe {df_name} contains non-human-readable "
+                f"values like {non_readables[0]}"
+            )
+
+    if schema_def.get("type") in ("curie", "suffixed curie"):
+        validation_func = _validate_curie if schema_def.get("type") == "curie" else _validate_suffixed_curie
+        non_valid_curies = [v for v in column if not validation_func(v, schema_def.get("prefixes"))]
+        if non_valid_curies:
+            errors.append(
+                f"Column {column_name} in dataframe {df_name} contains invalid ontology values like "
+                f"{non_valid_curies[0]}."
+            )
+            if "prefixes" in schema_def:
+                errors[-1] += f" Values must be curies from one of these ontologies {schema_def['prefixes']}."
+
+    if "enum" in schema_def:
+        bad_enums = [v for v in column if v not in schema_def["enum"]]
+        if bad_enums:
+            errors.append(
+                f"Column {column_name} in dataframe {df_name} contains unpermitted values like "
+                f"{bad_enums[0]}. Values must be one of {schema_def['enum']}."
+            )
+
+    return errors
+
+
+def _validate_dict(dict_, dict_name, schema_def):
+    """Given a schema definition and dict, verify that the dict satifies the schema."""
+
+    errors = []
+
+    for key in schema_def.get("keys", []):
+        if key not in dict_:
+            errors.append(f"{dict_name} is missing key {key}.")
+        elif schema_def["keys"][key]:
+            if schema_def["keys"][key]["type"] == "stringified list of dicts":
+                if not _validate_stringified_list_of_dicts(dict_[key]):
+                    errors.append(
+                        f"Key {key} in {dict_name} should be a JSON-encoded list of dicts, but it is {dict_[key]}"
+                    )
+            elif schema_def["keys"][key]["type"] == "dict":
+                errors.extend(_validate_dict(dict_[key], key, schema_def["keys"][key]))
+            elif schema_def["keys"][key]["type"] == "curie":
+                if not _validate_curie(dict_[key], schema_def["keys"][key]["prefixes"]):
+                    errors.append(f"Key {key} in {dict_name} contains invalid ontology value.")
+            if "nullable" in schema_def["keys"][key] and not schema_def["keys"][key]["nullable"]:
+                if _is_null(dict_[key]):
+                    errors.append(f"Key {key} in dict {dict_name} is an empty value.")
+
+    return errors
+
+
+def _validate_dataframe(df, df_name, schema_def):
+    """Given a dataframe and schema definition, verify that the dataframe follows the schema."""
+
+    errors = []
+
+    if "index" in schema_def:
+        errors.extend(_validate_column(df.index, "index", df_name, schema_def["index"]))
+
+    for column in schema_def.get("columns", []):
+        if column not in df.columns:
+            errors.append(f"Dataframe {df_name} is missing column {column}.")
+        else:
+            errors.extend(_validate_column(df[column], column, df_name, schema_def["columns"][column]))
+
+    return errors
+
+
+def get_schema_definition(version):
+    """Look up and read a schema definition based on a version number like "1.0.0"."""
+
+    path = os.path.join(
+        os.path.dirname(os.path.realpath(__file__)), "schema_definitions", version.replace(".", "_") + ".yaml"
+    )
+
+    if not os.path.isfile(path):
+        raise ValueError(f"No definition for version {version} found.")
+
+    return yaml.load(open(path), Loader=yaml.FullLoader)
+
+
+def deep_check(adata, schema_def):
+    """Perform a "deep" check of the AnnData object using the schema definition.
+
+    This checks all the columns and unstructured metadata rather than just the version.
+
+    Returns a list of error messages. If that list is empty, the object passed validation.
+    """
+
+    errors = []
+
+    for component, component_def in schema_def["components"].items():
+        if component_def["type"] == "dataframe":
+            errors.extend(_validate_dataframe(getattr(adata, component), component, component_def))
+        elif component_def["type"] == "dict":
+            errors.extend(_validate_dict(getattr(adata, component), component, component_def))
+        else:
+            raise ValueError(f"Unexpected component type {component['type']}")
+
+    return errors
+
+
+def validate_adata(adata, shallow):
+    """Validate an AnnData object. If shallow, just check that the required version information is
+    present.
+    """
+
+    # Does it have the version information written into uns?
+    if "version" not in adata.uns_keys() or "corpora_schema_version" not in adata.uns["version"]:
+        print("AnnData file is missing corpora version information")
+        return False
+
+    # We can stop here if it's a "shallow" check, that is, if we're just
+    # checking that version is present.
+    if shallow:
+        return True
+
+    schema_def = get_schema_definition(adata.uns["version"]["corpora_schema_version"])
+
+    errors = deep_check(adata, schema_def)
+
+    for error in errors:
+        print(error)
+
+    return not errors
+
+
+def validate(h5ad_path, shallow=False):
+    """Entry point for validation."""
+
+    try:
+        import scanpy
+    except ImportError:
+        raise ImportError("scanpy must be installed for cellxgene schema")
+
+    try:
+        adata = scanpy.read_h5ad(h5ad_path, backed="r")
+    except (OSError, TypeError):
+        print(f"Unable to open {h5ad_path} with scanpy.")
+        sys.exit(1)
+
+    if not validate_adata(adata, shallow):
+        sys.exit(1)

backend/czi_hosted/converters/to_sparse.py

@@ -2,12 +2,16 @@
 Script to create a sparse dataset in CXG format based on an input dataset in CXG format.
 The input dataset is not modified.
 """
+import argparse
 import os
 import shutil
-import tiledb
-import argparse
 import sys
-import server.converters.cxgtool as cxgtool
+
+import tiledb
+
+from backend.czi_hosted.common.utils.cxg_generation_utils import convert_ndarray_to_cxg_dense_array, \
+    convert_matrix_to_cxg_array
+from backend.czi_hosted.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix
 
 
 def main():
@@ -49,9 +53,25 @@ def main():
     )
 
     with tiledb.DenseArray(os.path.join(args.input, "X"), mode="r", ctx=ctx) as X_in:
-        is_sparse = cxgtool.save_X(args.output, X_in, ctx, args.sparse_threshold, expect_sparse=True)
+        x_matrix_data = X_in[:, :]
+        matrix_container = args.output
 
-    if is_sparse is False:
+        is_sparse = is_matrix_sparse(x_matrix_data, args.sparse_threshold)
+        if not is_sparse:
+            col_shift = get_column_shift_encode_for_matrix(x_matrix_data, args.sparse_threshold)
+            is_sparse = col_shift is not None
+        else:
+            col_shift = None
+
+        if col_shift is not None:
+            x_col_shift_name = f"{args.output}/X_col_shift"
+            convert_ndarray_to_cxg_dense_array(x_col_shift_name, col_shift, ctx)
+            tiledb.consolidate(matrix_container, ctx=ctx)
+        if is_sparse:
+            convert_matrix_to_cxg_array(matrix_container, x_matrix_data, is_sparse, ctx, col_shift)
+            tiledb.consolidate(matrix_container, ctx=ctx)
+
+    if not is_sparse:
         print("The array is not sparse, cleaning up, abort.")
         shutil.rmtree(args.output)
         sys.exit(1)

backend/czi_hosted/data_anndata/anndata_adaptor.py

@@ -1,22 +1,19 @@
 import warnings
 
-import numpy as np
-from pandas.core.dtypes.dtypes import CategoricalDtype
 import anndata
-from scipy import sparse
+import numpy as np
 from packaging import version
-from datetime import datetime
-from server_timing import Timing as ServerTiming
+from pandas.core.dtypes.dtypes import CategoricalDtype
+from scipy import sparse
 
-from server.data_common.data_adaptor import DataAdaptor
-from server.data_common.fbs.matrix import encode_matrix_fbs
-from server.common.utils import series_to_schema
-from server.common.colors import convert_anndata_category_colors_to_cxg_category_colors
-from server.common.constants import Axis, MAX_LAYOUTS
-from server.common.errors import PrepareError, DatasetAccessError, FilterError
-from server.compute.scanpy import scanpy_umap
-import server.compute.diffexp_generic as diffexp_generic
-from server.common.corpora import corpora_get_props_from_anndata
+import backend.common.compute.diffexp_generic as diffexp_generic
+from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
+from backend.common.constants import Axis, MAX_LAYOUTS, XApproximateDistribution
+from backend.czi_hosted.common.corpora import corpora_get_props_from_anndata
+from backend.common.errors import PrepareError, DatasetAccessError, ConfigurationError
+from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
+from backend.czi_hosted.data_common.data_adaptor import DataAdaptor
+from backend.common.fbs.matrix import encode_matrix_fbs
 
 anndata_version = version.parse(str(anndata.__version__)).release
 
@@ -31,6 +28,7 @@ class AnndataAdaptor(DataAdaptor):
     def __init__(self, data_locator, app_config=None, dataset_config=None):
         super().__init__(data_locator, app_config, dataset_config)
         self.data = None
+        self.X_approximate_distribution = None
         self._load_data(data_locator)
         self._validate_and_initialize()
 
@@ -68,11 +66,11 @@ class AnndataAdaptor(DataAdaptor):
 
     @staticmethod
     def _create_unique_column_name(df, col_name_prefix):
-        """ given the columns of a dataframe, and a name prefix, return a column name which
-            does not exist in the dataframe, AND which is prefixed by `prefix`
+        """given the columns of a dataframe, and a name prefix, return a column name which
+        does not exist in the dataframe, AND which is prefixed by `prefix`
 
-            The approach is to append a numeric suffix, starting at zero and increasing by
-            one, until an unused name is found (eg, prefix_0, prefix_1, ...).
+        The approach is to append a numeric suffix, starting at zero and increasing by
+        one, until an unused name is found (eg, prefix_0, prefix_1, ...).
         """
         suffix = 0
         while f"{col_name_prefix}{suffix}" in df:
@@ -126,7 +124,11 @@ class AnndataAdaptor(DataAdaptor):
 
     def _create_schema(self):
         self.schema = {
-            "dataframe": {"nObs": self.cell_count, "nVar": self.gene_count, "type": str(self.data.X.dtype)},
+            "dataframe": {
+                "nObs": self.cell_count,
+                "nVar": self.gene_count,
+                **get_schema_type_hint_of_array(self.data.X),
+            },
             "annotations": {
                 "obs": {"index": self.parameters.get("obs_names"), "columns": []},
                 "var": {"index": self.parameters.get("var_names"), "columns": []},
@@ -137,7 +139,7 @@ class AnndataAdaptor(DataAdaptor):
             curr_axis = getattr(self.data, str(ax))
             for ann in curr_axis:
                 ann_schema = {"name": ann, "writable": False}
-                ann_schema.update(series_to_schema(curr_axis[ann]))
+                ann_schema.update(get_schema_type_hint_of_array(curr_axis[ann]))
                 self.schema["annotations"][ax]["columns"].append(ann_schema)
 
         for layout in self.get_embedding_names():
@@ -177,10 +179,10 @@ class AnndataAdaptor(DataAdaptor):
             )
 
     def _validate_and_initialize(self):
-        if anndata_version_is_pre_070() and self.server_config.adaptor__anndata_adaptor__backed:
+        if anndata_version_is_pre_070():
             warnings.warn(
-                "Use of --backed mode with anndata versions older than 0.7 will have serious "
-                "performance issues. Please update to at least anndata 0.7 or later."
+                "Use of anndata versions older than 0.7 will have serious issues. Please update to at "
+                "least anndata 0.7 or later."
             )
 
         # var and obs column names must be unique
@@ -193,16 +195,20 @@ class AnndataAdaptor(DataAdaptor):
         self.gene_count = self.data.shape[1]
         self._create_schema()
 
+        if self.dataset_config.X_approximate_distribution == "auto":
+            raise ConfigurationError("X-approximate-distribution 'auto' mode unsupported.")
+        self.X_approximate_distribution = self.dataset_config.X_approximate_distribution
+
         # heuristic
         n_values = self.data.shape[0] * self.data.shape[1]
         if (n_values > 1e8 and self.server_config.adaptor__anndata_adaptor__backed is True) or (n_values > 5e8):
             self.parameters.update({"diffexp_may_be_slow": True})
 
     def _is_valid_layout(self, arr):
-        """ return True if this layout data is a valid array for front-end presentation:
-            * ndarray, dtype float/int/uint
-            * with shape (n_obs, >= 2)
-            * with all values finite or NaN (no +Inf or -Inf)
+        """return True if this layout data is a valid array for front-end presentation:
+        * ndarray, dtype float/int/uint
+        * with shape (n_obs, >= 2)
+        * with all values finite or NaN (no +Inf or -Inf)
         """
         is_valid = type(arr) == np.ndarray and arr.dtype.kind in "fiu"
         is_valid = is_valid and arr.shape[0] == self.data.n_obs and arr.shape[1] >= 2
@@ -301,28 +307,6 @@ class AnndataAdaptor(DataAdaptor):
         full_embedding = self.data.obsm[f"X_{ename}"]
         return full_embedding[:, 0:dims]
 
-    def compute_embedding(self, method, obsFilter):
-        if Axis.VAR in obsFilter:
-            raise FilterError("Observation filters may not contain variable conditions")
-        if method != "umap":
-            raise NotImplementedError(f"re-embedding method {method} is not available.")
-        try:
-            shape = self.get_shape()
-            obs_mask = self._axis_filter_to_mask(Axis.OBS, obsFilter["obs"], shape[0])
-        except (KeyError, IndexError):
-            raise FilterError("Error parsing filter")
-        with ServerTiming.time("layout.compute"):
-            X_umap = scanpy_umap(self.data, obs_mask)
-
-        # Server picks reemedding name, which must not collide with any other
-        # embedding name generated by this backend.
-        name = f"reembed:{method}_{datetime.now().isoformat(timespec='milliseconds')}"
-        dims = [f"{name}_0", f"{name}_1"]
-        layout_schema = {"name": name, "type": "float32", "dims": dims}
-        self.schema["layout"]["obs"].append(layout_schema)
-        self.data.obsm[f"X_{name}"] = X_umap
-        return layout_schema
-
     def compute_diffexp_ttest(self, maskA, maskB, top_n=None, lfc_cutoff=None):
         if top_n is None:
             top_n = self.dataset_config.diffexp__top_n
@@ -334,13 +318,22 @@ class AnndataAdaptor(DataAdaptor):
         return convert_anndata_category_colors_to_cxg_category_colors(self.data)
 
     def get_X_array(self, obs_mask=None, var_mask=None):
+        # H5Py does not support boolean indexing (masks), so convert to integer indexing
+        # when backed (ie, when AnnData is using H5Py indexing)
         if obs_mask is None:
             obs_mask = slice(None)
+        elif self.data.isbacked and obs_mask.dtype == bool:
+            obs_mask = obs_mask.nonzero()[0]
         if var_mask is None:
             var_mask = slice(None)
+        elif self.data.isbacked and var_mask.dtype == bool:
+            var_mask = var_mask.nonzero()[0]
         X = self.data.X[obs_mask, var_mask]
         return X
 
+    def get_X_approximate_distribution(self) -> XApproximateDistribution:
+        return self.X_approximate_distribution
+
     def get_shape(self):
         return self.data.shape