docs: release 0.17.0 (#2278)

* Bump version: 0.16.7 → 0.17.0-rc.0

* Bump version: 0.17.0-rc.0 → 0.17.0

.bumpversion.cfg

@@ -1,17 +1,14 @@
 [bumpversion]
-current_version = 0.16.0
+current_version = 0.17.0
 commit = True
-# The below regex details an acceptable version number by naming the groups (major, minor, patch, prerel, and
-# prerelversion) and also specifying the valid values for each group (integers, `\d+`,  for major, minor, patch, and
-# prerelversion and only `rc` as the acceptable value for prerel).
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?:-(?P<prerel>rc)\.(?P<prerelversion>\d+))?
-serialize =
+serialize = 
 	{major}.{minor}.{patch}-{prerel}.{prerelversion}
 	{major}.{minor}.{patch}
 
 [bumpversion:part:prerel]
 optional_value = release
-values =
+values = 
 	rc
 	release
 
@@ -23,6 +20,6 @@ replace = version="{new_version}"
 search = "version": "{current_version}"
 replace = "version": "{new_version}"
 
-[bumpversion:file:server/__init__.py]
+[bumpversion:file:backend/server/__init__.py]
 search = __version__ = "{current_version}"
 replace = __version__ = "{new_version}"

.dockerignore

@@ -2,4 +2,4 @@ bin
 client
 dist
 docs
-server
+backend

.github/workflows/canary-deploy.yml

@@ -0,0 +1,13 @@
+name: Deploy canary via single cell infra repo
+
+on:
+  push:
+    branches: main-canary
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: repository dispatch
+        run: |
+          curl -XPOST -u czi-sci-single-cell-eng:${{secrets.SCI_GITHUB_TOKEN}} -H "Accept: application/vnd.github.everest-preview+json"  -H "Content-Type: application/json" https://api.github.com/repos/chanzuckerberg/single-cell-infra/dispatches --data '{"event_type": "canary-hook"}'

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,67 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL Scan"
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ main ]
+  schedule:
+    - cron: '0 8 * * *'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'javascript', 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/compatibility_tests.yml

@@ -28,8 +28,8 @@ jobs:
     continue-on-error: true
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8]
-        anndata-version: [0.6.22.post1, 0.7.1]
+        python-version: [3.6, 3.7] # As of Oct 2020 Anndata is not compatible with 3.8
+        anndata-version: [0.7.0, 0.7.1, 0.7.2, 0.7.3, 0.7.4, 0.7.5]
         test-suite: [smoke-test, smoke-test-annotations]
     steps:
     - uses: actions/checkout@v2
@@ -41,8 +41,8 @@ jobs:
       run: |
         # 1. only install the dev requirements on top of what is in the cellxgene pip package
         sudo apt-get update && sudo apt-get install -y libhdf5-serial-dev
-        sed -i 's/-r requirements.txt//' server/requirements-dev.txt
-        pip install -r server/requirements-dev.txt
+        sed -i 's/-r requirements.txt//' backend/server/requirements-dev.txt
+        pip install -r backend/server/requirements-dev.txt
         # 2. install cellxgene
         make pydist install-dist
         # 3. install anndata
@@ -73,8 +73,8 @@ jobs:
           cd cellxgene
           # 1. only install the dev requirements on top of what is in the cellxgene pip package
           make dev-env-client
-          sed -i 's/-r requirements.txt//' server/requirements-dev.txt
-          pip install -r server/requirements-dev.txt
+          sed -i 's/-r requirements.txt//' backend/server/requirements-dev.txt
+          pip install -r backend/server/requirements-dev.txt
           # 2. install cellxgene
           pip install --upgrade cellxgene
           # 3. install anndata
@@ -101,7 +101,7 @@ jobs:
       - name: Install dependencies
         run: |
           cd cellxgene
-          sed -i -E 's/^anndata[>=]=[0-9]+.[0-9]+.[0-9]+$/anndata/g' server/requirements.txt
+          sed -i -E 's/^anndata[>=]=[0-9]+.[0-9]+.[0-9]+$/anndata/g' backend/server/requirements.txt
           make pydist install-dist dev-env
           pip install git+https://github.com/theislab/anndata
       - name: Tests

.github/workflows/push_tests.yml

@@ -36,7 +36,7 @@ jobs:
           npm install
       - name: Format with black and lint with flake8
         run: |
-          make lint-server
+          make lint-servers
       - name: Lint src with eslint
         working-directory: ./client
         run: |
@@ -68,8 +68,38 @@ jobs:
         run: make pydist install-dist dev-env-server
       - name: Unit tests
         run: |
-          make unit-test
-          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k server -cF backend,python,unitTest
+          make unit-test-server
+          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k backend/server -cF backend,python,unitTest
+          cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest
+
+  unit-test-czi-hosted:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Python cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Node cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Install dependencies
+        run: make pydist-czi-hosted install-dist dev-env-czi-hosted
+      - name: Unit tests
+        run: |
+          make unit-test-czi-hosted
+          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k backend/czi-hosted -cF backend,python,unitTest
           cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest
 
   smoke-tests:
@@ -96,7 +126,7 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-node-
       - name: Install dependencies
-        run: make pydist install-dist
+        run: make pydist-czi-hosted install-dist
       - name: Smoke tests (without annotations feature)
         run: |
           cd client && make smoke-test

.github/workflows/scale-test.yml

@@ -0,0 +1,30 @@
+name: "Scale test cellxgene APIs for initial loading"
+
+on:
+  schedule:
+    - cron: "0 0 * * Sun"
+
+jobs:
+  locust-build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Install dependencies
+        run: |
+          pip install -r backend/test/test_czi_hosted/locust/requirements-locust.txt
+      - name: Dev Scale Test
+        run: |
+          locust -f backend/test/test_czi_hosted/locust/locustfile.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt
+      - name: Slack success webhook
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+        run: |
+          DEV_STATS=$(tail -n 15 locust_dev_stats.txt)
+          DEV_MSG="\`\`\`CELLXGENE EXPLORER DEV SCALE TEST RESULTS: ${DEV_STATS}\`\`\`"
+          curl -X POST -H 'Content-type: application/json' --data "{'text':'${DEV_MSG}'}" $SLACK_WEBHOOK
+
+

.gitignore

@@ -19,9 +19,13 @@ venv/
 cellxgene/
 
 # client build
-server/common/web/static/*
-server/common/web/templates/
-server/common/web/csp-hashes.json
+backend/server/common/web/static/*
+backend/server/common/web/templates/
+backend/server/common/web/csp-hashes.json
+
+backend/czi_hosted/common/web/static/*
+backend/czi_hosted/common/web/templates/
+backend/czi_hosted/common/web/csp-hashes.json
 
 # eb build
 artifact.dir

MANIFEST.in

@@ -1,5 +1,7 @@
-recursive-include server/common/web/templates *
-recursive-include server/common/web/static *
+recursive-include backend/server/common/web/templates *
+recursive-include backend/server/common/web/static *
 
-include server/requirements.txt
-include server/requirements-prepare.txt
+include backend/server/requirements.txt
+include backend/server/requirements-prepare.txt
+include backend/server/converters/schema/hgnc_complete_set.txt.gz
+include backend/server/converters/schema/schema_definitions/*

MANIFEST_hosted.in

@@ -0,0 +1,7 @@
+recursive-include backend/czi_hosted/common/web/templates *
+recursive-include backend/czi_hosted/common/web/static *
+
+include backend/czi_hosted/requirements.txt
+include backend/czi_hosted/requirements-prepare.txt
+include backend/czi_hosted/converters/schema/hgnc_complete_set.txt.gz
+include backend/czi_hosted/converters/schema/schema_definitions/*

Makefile

@@ -2,23 +2,32 @@ include common.mk
 
 BUILDDIR := build
 CLIENTBUILD := $(BUILDDIR)/client
-SERVERBUILD := $(BUILDDIR)/server
+CZIHOSTEDBUILD := $(BUILDDIR)/backend/czi_hosted
+SERVERBUILD := $(BUILDDIR)/backend/server
 CLEANFILES :=  $(BUILDDIR)/ client/build build dist cellxgene.egg-info
 
 PART ?= patch
 
 # CLEANING
 .PHONY: clean
-clean: clean-lite clean-server clean-client
+clean: clean-lite clean-czi-hosted clean-server clean-client
 
 # cleaning the client's node_modules is the longest one, so we avoid that if possible
 .PHONY: clean-lite
 clean-lite:
 	rm -rf $(CLEANFILES)
 
-clean-%:
-	cd $(*) && $(MAKE) clean
+.PHONY: clean-client
+clean-client:
+	cd client && $(MAKE) clean
 
+.PHONY: clean-server
+clean-server:
+	cd backend/server && $(MAKE) clean
+
+.PHONY: clean-czi-hosted
+clean-czi-hosted:
+	cd backend/czi_hosted && $(MAKE) clean
 
 # BUILDING PACKAGE
 
@@ -28,29 +37,71 @@ build-client:
 
 .PHONY: build
 build: clean build-client
-	git ls-files server/ | grep -v 'server/test/' | cpio -pdm $(BUILDDIR)
+	git ls-files backend/server/ | grep -v 'backend/server/test/' | cpio -pdm $(BUILDDIR)
 	cp -r client/build/  $(CLIENTBUILD)
 	$(call copy_client_assets,$(CLIENTBUILD),$(SERVERBUILD))
+	cp backend/__init__.py $(BUILDDIR)
+	cp backend/__init__.py $(BUILDDIR)/backend
+	cp -r backend/common $(BUILDDIR)/backend/common
 	cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR)
 
+.PHONY: build-czi-hosted
+build-czi-hosted: clean build-client
+	git ls-files backend/czi_hosted/ | grep -v 'backend/czi_hosted/test/' | cpio -pdm $(BUILDDIR)
+	cp -r client/build/  $(CLIENTBUILD)
+	$(call copy_client_assets,$(CLIENTBUILD),$(CZIHOSTEDBUILD))
+	cp -r backend/common $(BUILDDIR)/backend/common
+	cp backend/__init__.py $(BUILDDIR)
+	cp backend/__init__.py $(BUILDDIR)/backend
+	cp MANIFEST_hosted.in README.md setup.cfg setup_hosted.py $(BUILDDIR)
+	mv $(BUILDDIR)/setup_hosted.py $(BUILDDIR)/setup.py
+	mv $(BUILDDIR)/MANIFEST_hosted.in $(BUILDDIR)/MANIFEST.in
+
 # If you are actively developing in the server folder use this, dirties the source tree
 .PHONY: build-for-server-dev
 build-for-server-dev: clean-server build-client
-	$(call copy_client_assets,client/build,server)
+	$(call copy_client_assets,client/build,backend/server)
+
+.PHONY: build-for-czi-hosted-dev
+build-for-czi-hosted-dev: clean-czi-hosted build-client
+	$(call copy_client_assets,client/build,backend/czi_hosted)
 
 .PHONY: copy-client-assets
 copy-client-assets:
-	$(call copy_client_assets,client/build,server)
+	$(call copy_client_assets,client/build,backend/server)
+
+.PHONY: copy-client-assets-czi-hosted
+copy-client-assets-czi-hosted:
+	$(call copy_client_assets,client/build,backend/czi_hosted)
 
 # TESTING
 .PHONY: test
 test: unit-test smoke-test
 
 .PHONY: unit-test
-unit-test: unit-test-server unit-test-client
+unit-test: unit-test-server unit-test-client unit-test-common
 
-unit-test-%:
-	cd $(*) && $(MAKE) unit-test
+.PHONY: test-server
+test-server: unit-test-server smoke-test
+
+.PHONY: test-czi-hosted
+test-czi-hosted: unit-test-czi-hosted smoke-test
+
+.PHONY: unit-test-client
+unit-test-client:
+	cd client && $(MAKE) unit-test
+
+.PHONY: unit-test-czi-hosted
+unit-test-czi-hosted:
+	cd backend/czi_hosted && $(MAKE) unit-test
+
+.PHONY: unit-test-server
+unit-test-server:
+	cd backend/server && $(MAKE) unit-test
+
+.PHONY: unit-test-common
+unit-test-common:
+	cd backend/common && $(MAKE) unit-test
 
 .PHONY: smoke-test
 smoke-test:
@@ -62,12 +113,11 @@ smoke-test-annotations:
 
 .PHONY: test-db
 test-db:
-	cd server && $(MAKE) test-db
-
+	cd backend/czi_hosted && $(MAKE) test-db
 
 # FORMATTING CODE
 
-.PHOHY: fmt
+.PHONY: fmt
 fmt: fmt-client fmt-py
 
 .PHONY: fmt-client
@@ -79,12 +129,18 @@ fmt-py:
 	black .
 
 .PHONY: lint
-lint: lint-server lint-client
+lint: lint-servers lint-client
+
+.PHONY: lint-servers
+lint-servers: lint-server lint-czi-hosted-server
 
 .PHONY: lint-server
 lint-server: fmt-py
-	flake8 server --per-file-ignores='server/test/fixtures/dataset_config_outline.py:F821 server/test/fixtures/server_config_outline.py:F821 server/test/performance/scale_test_annotations.py:E501'
+	flake8 backend/server --per-file-ignores='backend/test/fixtures/dataset_config_outline.py:F821 backend/test/fixtures/server_config_outline.py:F821 backend/server/test/performance/scale_test_annotations.py:E501'
 
+.PHONY: lint-czi-hosted-server
+lint-czi-hosted-server: fmt-py
+	flake8 backend/czi_hosted --per-file-ignores='backend/test/fixtures/czi_hosted_dataset_config_outline.py:F821 backend/test/fixtures/czi_hosted_server_config_outline.py:F821 backend/test/performance/scale_test_annotations.py:E501'
 
 .PHONY: lint-client
 lint-client:
@@ -97,6 +153,11 @@ pydist: build
 	cd $(BUILDDIR); python setup.py sdist -d ../dist
 	@echo "done"
 
+.PHONY: pydist-czi-hosted
+pydist-czi-hosted: build-czi-hosted
+	cd $(BUILDDIR); python setup.py sdist -d ../dist
+	@echo "done"
+
 
 # RELEASE HELPERS
 
@@ -145,8 +206,11 @@ dev-env-client:
 
 .PHONY: dev-env-server
 dev-env-server:
-	pip install -r server/requirements-dev.txt
+	pip install -r backend/server/requirements-dev.txt
 
+.PHONY: dev-env-czi-hosted
+dev-env-czi-hosted:
+	pip install -r backend/czi_hosted/requirements-dev.txt
 # Set PART=[major, minor, patch] as param to make bump.
 # This will create a release candidate. (i.e. 0.16.1 -> 0.16.2-rc.0 for a patch bump)
 .PHONY: bump-version

PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,11 @@
+#### Reviewers
+**Functional:** 
+
+**Readability:** 
+
+---
+
+## Changes
+- add
+- remove
+- modify

README.md

@@ -52,7 +52,6 @@ cellxgene currently supports the following browsers:
 - Google Chrome 61+
 - Edge 15+
 - Firefox 60+
-- Safari 10.1+
 
 Please [file an issue](https://github.com/chanzuckerberg/cellxgene/issues/new/choose) if you would like us to add support for an unsupported browser.
 
@@ -67,13 +66,22 @@ For any errors, [report bugs on Github](https://github.com/chanzuckerberg/cellxg
 
 ### Contributing
 
-We warmly welcome contributions from the community! Please see our [contributing guide](https://chanzuckerberg.github.io/cellxgene/posts/contribute) and don't hesitate to open an issue or send a pull request to improve cellxgene.
+We warmly welcome contributions from the community! Please see our [contributing guide](https://chanzuckerberg.github.io/cellxgene/posts/contribute) and don't hesitate to open an issue or send a pull request to improve cellxgene. Please see the [dev_docs](https://github.com/chanzuckerberg/cellxgene/tree/main/dev_docs) for pull request suggestions, unit test details, local documentation preview, and other development specifics. 
 
 This project adheres to the Contributor Covenant [code of conduct](https://github.com/chanzuckerberg/.github/blob/master/CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to opensource@chanzuckerberg.com.
 
 ### Reuse
 
-This project was started with the sole goal of empowering the scientific community to explore and understand their data. As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from this project, and reach out to us with ideas or questions. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT).
+This project was started with the sole goal of empowering the scientific community to explore and understand their data. 
+As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from 
+this project. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT).
+
+
+Before extending cellxgene, we encourage you to reach out to us with ideas or questions. It might be possible that an 
+extension could be directly contributed, which would make it available for a wider audience, or that it's on our 
+[roadmap](./docs/posts/roadmap.md) and under active development. 
+
+See the [cellxgene extensions](./docs/posts/extensions.md) section of our documentation for examples of community use and cellxgene extensions. 
 
 ### Security
 

backend/common/Makefile

@@ -0,0 +1,11 @@
+
+.PHONY: unit-test
+unit-test:
+	PYTHONWARNINGS=ignore:ResourceWarning coverage run \
+		--source=fbs,utils \
+		--omit=.coverage,data_common/fbs/NetEncoding,venv \
+		-m unittest discover \
+		--start-directory ../test/test_common/unit \
+		--top-level-directory ../../ \
+		--verbose; test_result=$$?; \
+	exit $$test_result \

backend/common/colors.py

@@ -1,6 +1,6 @@
 import re
 
-from server.common.errors import ColorFormatException
+from backend.common.errors import ColorFormatException
 
 HEX_COLOR_FORMAT = re.compile("^#[a-fA-F0-9]{6,6}$")
 

backend/common/compute/diffexp_generic.py

@@ -25,7 +25,8 @@ def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
     :param maskB: observation selection mask for set 2
     :param top_n: number of variables to return stats for
     :param diffexp_lfc_cutoff: minimum
-    :return:  for top N genes, [ varindex, logfoldchange, pval, pval_adj ]
+    absolute value returning [ varindex, logfoldchange, pval, pval_adj ] for top N genes
+    :return:  for top N genes, {"positive": for top N genes, [ varindex, logfoldchange, pval, pval_adj ], "negative": for top N genes, [ varindex, logfoldchange, pval, pval_adj ]}
     """
 
     dataA = adaptor.get_X_array(maskA, None)
@@ -66,24 +67,27 @@ def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp
     # logfoldchanges: log2(meanA / meanB)
     logfoldchanges = np.log2(np.abs((meanA + 1e-9) / (meanB + 1e-9)))
 
+    stats_to_sort = tscores
     # find all with lfc > cutoff
     lfc_above_cutoff_idx = np.nonzero(np.abs(logfoldchanges) > diffexp_lfc_cutoff)[0]
-    stats_to_sort = np.abs(tscores)
 
     # derive sort order
-    if lfc_above_cutoff_idx.shape[0] > top_n:
+    if lfc_above_cutoff_idx.shape[0] > top_n*2:
         # partition top N
-        rel_t_partition = np.argpartition(stats_to_sort[lfc_above_cutoff_idx], -top_n)[-top_n:]
-        t_partition = lfc_above_cutoff_idx[rel_t_partition]
+        rel_t_partition = np.argpartition(stats_to_sort[lfc_above_cutoff_idx], (top_n, -top_n))
+        rel_t_partition_top_n = np.concatenate((rel_t_partition[-top_n:], rel_t_partition[:top_n]))
+        t_partition = lfc_above_cutoff_idx[rel_t_partition_top_n]
         # sort the top N partition
         rel_sort_order = np.argsort(stats_to_sort[t_partition])[::-1]
         sort_order = t_partition[rel_sort_order]
     else:
         # partition and sort top N, ignoring lfc cutoff
-        partition = np.argpartition(stats_to_sort, -top_n)[-top_n:]
-        rel_sort_order = np.argsort(stats_to_sort[partition])[::-1]
+        partition = np.argpartition(stats_to_sort, (top_n, -top_n))
+        partition_top_n = np.concatenate((partition[-top_n:], partition[:top_n]))
+
+        rel_sort_order = np.argsort(stats_to_sort[partition_top_n])[::-1]
         indices = np.indices(stats_to_sort.shape)[0]
-        sort_order = indices[partition][rel_sort_order]
+        sort_order = indices[partition_top_n][rel_sort_order]
 
     # top n slice based upon sort order
     logfoldchanges_top_n = logfoldchanges[sort_order]
@@ -91,7 +95,11 @@ def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp
     pvals_adj_top_n = pvals_adj[sort_order]
 
     # varIndex, logfoldchange, pval, pval_adj
-    result = [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in range(top_n)]
+    result = {"positive": [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in
+                           range(top_n)],
+              "negative": [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in
+                           range(-1, -1 - top_n, -1)], }
+
     return result
 
 

backend/common/errors.py

@@ -55,3 +55,5 @@ define_exception("OntologyLoadFailure", "Raised when reading the ontology file f
 define_exception("ConfigurationError", "Raised when checking configuration errors")
 define_exception("PrepareError", "Raised when data is misprepared")
 define_exception("SecretKeyRetrievalError", "Raised when get_secret_key from AWS fails")
+define_exception("ObsoleteRequest", "Raised when the request is no longer valid.")
+define_exception("UnsupportedSummaryMethod", "Raised when a gene set summary method is unknown or unsupported.")

backend/common/fbs/matrix.py

@@ -5,15 +5,14 @@ import pandas as pd
 from flatbuffers import Builder
 from scipy import sparse
 
-import server.data_common.fbs.NetEncoding.Column as Column
-import server.data_common.fbs.NetEncoding.Float32Array as Float32Array
-import server.data_common.fbs.NetEncoding.Float64Array as Float64Array
-import server.data_common.fbs.NetEncoding.Int32Array as Int32Array
-import server.data_common.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray
-import server.data_common.fbs.NetEncoding.Matrix as Matrix
-import server.data_common.fbs.NetEncoding.TypedArray as TypedArray
-import server.data_common.fbs.NetEncoding.Uint32Array as Uint32Array
-
+import backend.common.fbs.NetEncoding.Column as Column
+import backend.common.fbs.NetEncoding.Float32Array as Float32Array
+import backend.common.fbs.NetEncoding.Float64Array as Float64Array
+import backend.common.fbs.NetEncoding.Int32Array as Int32Array
+import backend.common.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray
+import backend.common.fbs.NetEncoding.Matrix as Matrix
+import backend.common.fbs.NetEncoding.TypedArray as TypedArray
+import backend.common.fbs.NetEncoding.Uint32Array as Uint32Array
 
 # Serialization helper
 def serialize_column(builder, typed_arr):

backend/common/genesets.py

@@ -0,0 +1,239 @@
+"""
+Utility code for gene sets handling
+"""
+
+import re
+import csv
+import hashlib
+
+from .errors import AnnotationsError
+
+
+GENESETS_TIDYCSV_HEADER = [
+    "gene_set_name",
+    "gene_set_description",
+    "gene_symbol",
+    "gene_description",
+]
+
+
+def read_gene_sets_tidycsv(gs_locator, context=None):
+    """
+    Read & parse the Tidy CSV format, applying validation checks for mandatory
+    values, and de-duping rules.
+
+    Format is a four-column CSV, with a mandatory header row, and optional "#" prefixed
+    comments.  Format:
+
+        gene_set_name, gene_set_description, gene_symbol, gene_description
+
+    gene_set_name must be non-null; others are optional.
+
+    Returns: a dictionary of the shape (values in angle-brackets vary):
+
+        {
+            <string, a gene set name>: {
+                "geneset_name": <string, a gene set name>,
+                "geneset_description": <a string or None>,
+                "genes": [
+                    {
+                        "gene_symbol": <string, a gene symbol or name>,
+                        "gene_description": <a string or None>
+                    },
+                    ...
+                ]
+            },
+            ...
+        }
+    """
+
+    class myDialect(csv.excel):
+        skipinitialspace = False
+
+    def just(n, seq):
+        it = iter(seq)
+        for _ in range(n - 1):
+            yield next(it, "")
+        yield tuple(it)
+
+    messagefn = context["messagefn"] if context else (lambda x: None)
+
+    gene_sets = {}
+    with gs_locator.local_handle() as fname:
+        with open(fname, newline="") as f:
+            reader = csv.reader(f, dialect=myDialect())
+            haveReadHeader = False
+            lineno = 0
+            for row in reader:
+                lineno += 1
+                # ignore empty rows
+                if len(row) == 0:
+                    continue
+                # if row starts with '#' it is a comment
+                if row[0].startswith("#"):
+                    continue
+                # if this is the first non-comment row, assume it is a header and validate
+                # column names.  OK if the user has extra columns after our initial set.
+                if not haveReadHeader:
+                    if row[0:len(GENESETS_TIDYCSV_HEADER)] != GENESETS_TIDYCSV_HEADER:
+                        raise AnnotationsError("Gene set CSV file missing the required column header.")
+                    haveReadHeader = True
+                    continue
+
+                geneset_name, geneset_description, gene_symbol, gene_description, _ = just(5, row)
+                if not geneset_name:
+                    raise AnnotationsError(f"Gene set CSV missing required gene set name on line {lineno}")
+                if (not gene_symbol) and gene_description:
+                    messagefn(f"Warning: Missing gene name in gene set name {geneset_name} on line {lineno}.")
+
+                if geneset_name in gene_sets:
+                    gs = gene_sets[geneset_name]
+                else:
+                    gs = gene_sets[geneset_name] = {
+                        "geneset_name": geneset_name,
+                        "geneset_description": geneset_description,
+                        "genes": [],
+                    }
+                # Use first geneset_description with a value
+                if not gs["geneset_description"] and geneset_description:
+                    gs["geneset_description"] = geneset_description
+                # add the gene if the gene_symbol is defined
+                if gene_symbol:
+                    gs["genes"].append(
+                        {
+                            "gene_symbol": gene_symbol,
+                            "gene_description": gene_description,
+                        }
+                    )
+
+    return gene_sets
+
+
+def write_gene_sets_tidycsv(f, genesets):
+    """
+    Convert the internal gene sets format (returned by read_gene_set_tidycsv) into
+    the simple Tidy CSV.
+    """
+    writer = csv.writer(f, dialect="excel")
+    writer.writerow(GENESETS_TIDYCSV_HEADER)
+    for geneset in genesets:
+        # genes may be empty, in which case we skip the gene set entirely
+        genes = geneset["genes"]
+        if not genes:
+            writer.writerow([geneset["geneset_name"], geneset.get("geneset_description", ""), "", ""])
+        else:
+            writer.writerows(
+                [
+                    [
+                        geneset["geneset_name"],
+                        geneset.get("geneset_description", ""),
+                        gene["gene_symbol"],
+                        gene.get("gene_description", ""),
+                    ]
+                    for gene in genes
+                ]
+            )
+
+
+def summarizeQueryHash(raw_query):
+    """ generate a cache key (hash) from the raw query string """
+    return hashlib.sha1(raw_query).hexdigest()
+
+
+def validate_gene_sets(genesets, var_names, context=None):
+    """
+    Check validity of gene sets, return if correct, else raise error.
+    May also modify the gene set for conditions that should be resolved,
+    but which do not warrant a hard error.
+
+    Argument gene sets may be either the REST OTA format (list of dicts) or the internal
+    format (dict of dicts, keyed by the gene set name).
+
+    Will return a modified gene sets (eg, remove warnings) of the same type as the
+    provided argument. Ie, dict->dict, list->list
+
+    Rules:
+
+    0. All gene set names must be unique. [error]
+    1. Gene set names must conform to the following: [error]
+        * Names must be comprised of 1 or more ASCII characters 32-126
+        * No leading or trailing spaces (ASCII 32)
+        * No multi-space (ASCII 32) runs
+    2. Gene symbols must be part of the current var_index. [warning]
+       If gene symbol is not in the var_index, generate a warning and remove the symbol
+       from the gene sets.
+    3. Gene symbols must not be duplicated in a gene set.  [warning]
+       Duplications will be silently de-duped.
+
+    Items marked [error] will generate a hard error, causing the validation to fail.
+
+    Items marked [warning] will generate a warning, and will be resolved without failing
+    the validation (typically by removing the offending item from the gene sets).
+    """
+
+    messagefn = context["messagefn"] if context else (lambda x: None)
+
+    # accept genesets args as either the internal (dict) or REST (list) format,
+    # as they are identical except for the dict being keyed by geneset_name.
+    if not isinstance(genesets, dict) and not isinstance(genesets, list):
+        raise ValueError("Gene sets must be either dict or list.")
+    genesets_iterable = genesets if isinstance(genesets, list) else genesets.values()
+
+    # 0. check for uniqueness of geneset names
+    geneset_names = [gs["geneset_name"] for gs in genesets_iterable]
+    if len(set(geneset_names)) != len(geneset_names):
+        raise KeyError("All gene set names must be unique.")
+
+    # 1. check gene set character set and format
+    illegal_name = re.compile(r"^\s|  |[\u0000-\u001F\u007F-\uFFFF]|\s$")
+    for name in geneset_names:
+        if type(name) != str or len(name) == 0:
+            raise KeyError("Gene set names must be non-null string.")
+        if illegal_name.search(name):
+            messagefn(
+                "Error: "
+                f"Gene set name {name} "
+                "is not valid. Leading, trailing, and multiple spaces within a name are not allowed."
+            )
+            raise KeyError(
+                "Gene set name is not valid. Leading, trailing, and multiple spaces within a name are not allowed."
+            )
+
+    # 2. & 3. check for duplicate gene symbols, and those not present in the dataset. They will
+    # generate a warning and be removed.
+    for geneset in genesets_iterable:
+        if not isinstance(geneset, dict):
+            raise ValueError("Each gene set must be a dict.")
+        geneset_name = geneset["geneset_name"]
+        genes = geneset["genes"]
+        if not isinstance(genes, list):
+            raise ValueError("Gene set genes field must be a list")
+        geneset.setdefault("geneset_description", "")
+        gene_symbol_already_seen = set()
+        new_genes = []
+        for gene in genes:
+            gene_symbol = gene["gene_symbol"]
+            if not isinstance(gene_symbol, str) or len(gene_symbol) == 0:
+                raise ValueError("Gene symbol must be non-null string.")
+            if gene_symbol in gene_symbol_already_seen:
+                # duplicate check
+                messagefn(
+                    f"Warning: a duplicate of gene {gene_symbol} was found in gene set {geneset_name}, "
+                    "and will be ignored."
+                )
+                continue
+
+            if gene_symbol not in var_names:
+                messagefn(
+                    f"Warning: {gene_symbol}, used in gene set {geneset_name}, "
+                    "was not found in the dataset and will be ignored."
+                )
+                continue
+
+            gene_symbol_already_seen.add(gene_symbol)
+            gene.setdefault("gene_description", "")
+            new_genes.append(gene)
+
+        geneset["genes"] = new_genes
+
+    return genesets

backend/common/utils/aws_secret_utils.py

@@ -3,7 +3,7 @@ import logging
 import boto3
 from flask import json
 
-from server.common.errors import SecretKeyRetrievalError
+from backend.common.errors import SecretKeyRetrievalError
 
 
 def get_secret_key(region_name, secret_name):

backend/common/utils/type_conversion_utils.py

@@ -44,7 +44,7 @@ def get_dtype_from_dtype(dtype, array_values=None):
     if dtype_name == "bool":
         return np.uint8
     if dtype_name == "object" and dtype_kind == "O":
-        return np.unicode
+        return str
     if dtype_name == "category":
         return get_dtype_from_dtype(dtype.categories.dtype, array_values)
 
@@ -129,9 +129,9 @@ def can_cast_to_int32(dtype, array_values=None):
             return True
         ii32 = np.iinfo(np.int32)
         if (
-                not ordered_array_values.empty
-                and (ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max)
-                or ordered_array_values.empty
+            not ordered_array_values.empty
+            and (ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max)
+            or ordered_array_values.empty
         ):
             return True
     return False

backend/common/utils/utils.py

@@ -10,7 +10,7 @@ from urllib.parse import urlsplit, urljoin
 import numpy as np
 from flask import json
 
-from server.common.errors import ConfigurationError
+from backend.common.errors import ConfigurationError
 
 
 def find_available_port(host, port=5005):

backend/czi_hosted/Makefile

@@ -1,4 +1,4 @@
-include ../common.mk
+include ../../common.mk
 
 .PHONY: clean
 clean:
@@ -9,11 +9,11 @@ clean:
 .PHONY: unit-test
 unit-test: create-test-db
 	PYTHONWARNINGS=ignore:ResourceWarning coverage run \
-		--source=app,cli,common,compute,converters,data_anndata,data_common,data_cxg \
-		--omit=.coverage,data_common/fbs/NetEncoding,venv \
+		--source=app,auth,cli,common,compute,converters,data_anndata,data_common,data_cxg,eb \
+		--omit=.coverage,venv \
 		-m unittest discover \
-		--start-directory test/ \
-		--top-level-directory ../ \
+		--start-directory ../test/test_czi_hosted/unit \
+		--top-level-directory ../.. \
 		--verbose; test_result=$$?; \
 	$(MAKE) clean-test-db; \
 	exit $$test_result \
@@ -22,11 +22,11 @@ unit-test: create-test-db
 .PHONY: test-db
 test-db: create-test-db
 	PYTHONWARNINGS=ignore:ResourceWarning coverage run \
-		--source=app,cli,common,compute,converters,data_anndata,data_common,data_cxg \
-		--omit=.coverage,data_common/fbs/NetEncoding,venv \
+		--source=db \
+		--omit=.coverage,venv \
 		-m unittest discover \
-		--start-directory test/test_database \
-		--top-level-directory ../ \
+		--start-directory ../test/test_czi_hosted/test_database \
+		--top-level-directory ../.. \
 		--verbose; test_result=$$?; \
 	$(MAKE) clean-test-db; \
 	exit $$test_result
@@ -42,8 +42,8 @@ clean-test-db:
 
 .PHONY: test-annotations-performance
 test-annotations-performance:
-	python test/performance/performance_test_annotations_backend.py
+	python ../test/test_czi_hosted/performance/performance_test_annotations_backend.py
 
 .PHONY: test-annotations-scale
 test-annotations-scale:
-	locust -f test/performance/scale_test_annotations.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt
+	locust -f ../test/test_czi_hosted/performance/scale_test_annotations.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt

backend/czi_hosted/__init__.py

@@ -0,0 +1,15 @@
+import logging
+import sys
+from backend.common.utils.utils import import_plugins
+
+__version__ = "0.16.7"
+
+
+display_version = "cellxgene v" + __version__
+
+try:
+    import_plugins("backend.czi_hosted.plugins")
+except Exception as e:
+    # Make sure to exit in this case, as the server may not be configured as expected.
+    logging.critical(f"Error in import_plugins: {str(e)}")
+    sys.exit(1)

backend/czi_hosted/__main__.py

@@ -0,0 +1,14 @@
+# Work around bug https://github.com/pallets/werkzeug/issues/461
+if __package__ is None:
+    import sys
+    from pathlib import Path
+
+    PKG_PATH = Path(__file__).parent
+    sys.path.insert(0, str(PKG_PATH.parent))
+    import backend.czi_hosted # noqa F401
+    __package__ = PKG_PATH.name
+
+# Main thing
+from .cli.cli import cli  # noqa F402
+
+cli()

backend/czi_hosted/app/app.py

@@ -20,14 +20,14 @@ from flask import (
 from flask_restful import Api, Resource
 from server_timing import Timing as ServerTiming
 
-import server.common.rest as common_rest
-from server.common.data_locator import DataLocator
-from server.common.errors import DatasetAccessError, RequestException
-from server.common.health import health_check
-from server.common.utils.utils import path_join, Float32JSONEncoder
-from server.data_common.matrix_loader import MatrixDataLoader
+import backend.czi_hosted.common.rest as common_rest
+from backend.common.utils.data_locator import DataLocator
+from backend.common.errors import DatasetAccessError, RequestException
+from backend.czi_hosted.common.health import health_check
+from backend.common.utils.utils import path_join, Float32JSONEncoder
+from backend.czi_hosted.data_common.matrix_loader import MatrixDataLoader
 
-webbp = Blueprint("webapp", "server.common.web", template_folder="templates")
+webbp = Blueprint("webapp", "backend.czi_hosted.common.web", template_folder="templates")
 
 ONE_WEEK = 7 * 24 * 60 * 60
 
@@ -105,17 +105,6 @@ def dataset_index(url_dataroot=None, dataset=None):
         )
 
 
-# TODO:  This route will be deprecated, but needs to be left for a short time until all the
-# deployments are upgraded to the new location for the health check (or else the upgrade will
-# fail).  Once the upgrade is complete, the deployments can move to the new health check URL
-# and this route will be removed.
-@webbp.route("/health", methods=["GET"])
-@cache_control_always(no_store=True)
-def health():
-    config = current_app.app_config
-    return health_check(config)
-
-
 @webbp.errorhandler(RequestException)
 def handle_request_exception(error):
     return common_rest.abort_and_log(error.status_code, error.message, loglevel=logging.INFO, include_exc_info=True)
@@ -179,7 +168,7 @@ def rest_get_data_adaptor(func):
     return wrapped_function
 
 
-def dataroot_test_index():
+def  dataroot_test_index():
     # the following index page is meant for testing/debugging purposes
     data = '<!doctype html><html lang="en">'
     data += "<head><title>Hosted Cellxgene</title></head>"
@@ -216,7 +205,7 @@ def dataroot_test_index():
     data += "<ul>"
     datasets.sort()
     for url_dataroot, dataset in datasets:
-        data += f"<li><a href={url_dataroot}/{dataset}>{dataset}</a></li>"
+        data += f"<li><a href={url_dataroot}/{dataset}/>{dataset}</a></li>"
     data += "</ul>"
     data += "</body></html>"
 
@@ -251,7 +240,7 @@ class DatasetResource(Resource):
 
 class SchemaAPI(DatasetResource):
     # TODO @mdunitz separate dataset schema and user schema
-    @cache_control(no_store=True)
+    @cache_control(public=True, max_age=ONE_WEEK)
     @rest_get_data_adaptor
     def get(self, data_adaptor):
         return common_rest.schema_get(data_adaptor)
@@ -272,7 +261,7 @@ class UserInfoAPI(DatasetResource):
 
 
 class AnnotationsObsAPI(DatasetResource):
-    @cache_control(public=True, no_store=True)
+    @cache_control(public=True, max_age=ONE_WEEK)
     @rest_get_data_adaptor
     def get(self, data_adaptor):
         return common_rest.annotations_obs_get(request, data_adaptor)
@@ -329,6 +318,25 @@ class LayoutObsAPI(DatasetResource):
         return common_rest.layout_obs_put(request, data_adaptor)
 
 
+class GenesetsAPI(DatasetResource):
+    @cache_control(public=True, max_age=ONE_WEEK)
+    @rest_get_data_adaptor
+    def get(self, data_adaptor):
+        return common_rest.genesets_get(request, data_adaptor)
+
+
+class SummarizeVarAPI(DatasetResource):
+    @rest_get_data_adaptor
+    @cache_control(public=True, max_age=ONE_WEEK)
+    def get(self, data_adaptor):
+        return common_rest.summarize_var_get(request, data_adaptor)
+
+    @rest_get_data_adaptor
+    @cache_control(no_store=True)
+    def post(self, data_adaptor):
+        return common_rest.summarize_var_post(request, data_adaptor)
+
+
 def get_api_base_resources(bp_base):
     """Add resources that are accessed from the api_base_url"""
     api = Api(bp_base)
@@ -354,6 +362,8 @@ def get_api_dataroot_resources(bp_dataroot, url_dataroot=None):
     add_resource(AnnotationsObsAPI, "/annotations/obs")
     add_resource(AnnotationsVarAPI, "/annotations/var")
     add_resource(DataVarAPI, "/data/var")
+    add_resource(GenesetsAPI, "/genesets")
+    add_resource(SummarizeVarAPI, "/summarize/var")
     # Display routes
     add_resource(ColorsAPI, "/colors")
     # Computation routes
@@ -431,11 +441,17 @@ class Server:
                 self.app.register_blueprint(dataroot_resources.blueprint)
 
                 self.app.add_url_rule(
-                    f"/{url_dataroot}/<dataset>/",
+                    f"/{url_dataroot}/<dataset>",
                     f"dataset_index_{url_dataroot}",
                     lambda dataset, url_dataroot=url_dataroot: dataset_index(url_dataroot, dataset),
                     methods=["GET"],
                 )
+                self.app.add_url_rule(
+                    f"/{url_dataroot}/<dataset>/",
+                    f"dataset_index_{url_dataroot}/",
+                    lambda dataset, url_dataroot=url_dataroot: dataset_index(url_dataroot, dataset),
+                    methods=["GET"],
+                )
                 self.app.add_url_rule(
                     f"/{url_dataroot}/<dataset>/static/<path:filename>",
                     f"static_assets_{url_dataroot}",
@@ -459,6 +475,6 @@ class Server:
 
         auth = server_config.auth
         self.app.auth = auth
-        if auth.requires_client_login():
+        if auth and auth.requires_client_login():
             auth.add_url_rules(self.app)
         auth.complete_setup(self.app)

backend/czi_hosted/auth/__init__.py

@@ -0,0 +1,6 @@
+# import the built in auth types so they can be registered
+
+import backend.czi_hosted.auth.auth_test  # noqa: F401
+import backend.czi_hosted.auth.auth_session  # noqa: F401
+import backend.czi_hosted.auth.auth_oauth  # noqa: F401
+import backend.czi_hosted.auth.auth_none  # noqa: F401

backend/czi_hosted/auth/auth_none.py

@@ -0,0 +1,27 @@
+from backend.czi_hosted.auth.auth import AuthTypeBase, AuthTypeFactory
+
+
+class AuthTypeNone(AuthTypeBase):
+    def __init__(self, app_config):
+        super().__init__()
+
+    def is_valid_authentication_type(self):
+        return False
+
+    def complete_setup(self, app):
+        pass
+
+    def is_user_authenticated(self):
+        return True
+
+    def get_user_id(self):
+        return None
+
+    def get_user_name(self):
+        return None
+
+    def get_user_email(self):
+        return None
+
+
+AuthTypeFactory.register(None, AuthTypeNone)

backend/czi_hosted/auth/auth_oauth.py

@@ -1,6 +1,6 @@
 from flask import session, request, redirect, current_app, after_this_request, has_request_context, g
-from server.auth.auth import AuthTypeClientBase, AuthTypeFactory
-from server.common.errors import AuthenticationError, ConfigurationError
+from backend.czi_hosted.auth.auth import AuthTypeClientBase, AuthTypeFactory
+from backend.common.errors import AuthenticationError, ConfigurationError
 from urllib.parse import urlencode, urlparse
 import json
 import requests
@@ -24,7 +24,7 @@ except ModuleNotFoundError:
 class Tokens:
     """Simple class to represent the tokens that are saved/restored from the cookie"""
 
-    def __init__(self, access_token, id_token, refresh_token, expires_at):
+    def __init__(self, access_token, id_token, refresh_token, expires_at, **kwargs):
         self.access_token = access_token
         self.id_token = id_token
         self.refresh_token = refresh_token
@@ -218,22 +218,24 @@ class AuthTypeOAuth(AuthTypeClientBase):
 
         try:
             if self.session_cookie:
-                tokensdict = session.get(self.CXG_TOKENS)
-                if tokensdict:
-                    g.tokens = Tokens(**tokensdict)
+                value = session.get(self.CXG_TOKENS)
+                if value:
+                    g.tokens = Tokens(**value)
                 else:
                     return None
             else:
                 value = request.cookies.get(self.cookie_params["key"])
-                value = base64.b64decode(value)
-                try:
-                    tokensdict = json.loads(value)
-                    g.tokens = Tokens(**tokensdict)
-                except (TypeError, KeyError, json.decoder.JSONDecodeError):
-                    g.pop("tokens", None)
+                if value is None:
                     return None
+                value = base64.b64decode(value)
+                value = json.loads(value)
+                g.tokens = Tokens(**value)
 
-        except (TypeError, KeyError):
+        except Exception:
+            # there are many types of exceptions that can be raise in the above section.
+            # It is impractical to list all the exceptions here, since that would be brittle.
+            # If an exception occurs, then return None, meaning that no token could be retrieved.
+            current_app.logger.warning(f"auth cookie is in the wrong format: {str(value)}")
             g.pop("tokens", None)
             return None
 
@@ -331,6 +333,7 @@ class AuthTypeOAuth(AuthTypeClientBase):
 
         # if there is no id_token, return None (user is not authenticated)
         tokens = self.get_tokens()
+
         if tokens is None or tokens.id_token is None:
             return None
 

backend/czi_hosted/auth/auth_session.py

@@ -0,0 +1,40 @@
+from flask import session
+from uuid import uuid4
+
+from backend.czi_hosted.auth.auth import AuthTypeBase, AuthTypeFactory
+
+
+class AuthTypeSession(AuthTypeBase):
+    """Session based authentication.  The user is always logged.  The user id is a random number
+    associated with the session.  This is a good choice for desktop servers."""
+
+    # key in the session token for userid
+    CXGUID = "cxguid"
+
+    def __init__(self, app_config):
+        super().__init__()
+
+    def is_valid_authentication_type(self):
+        return True
+
+    def complete_setup(self, app):
+        pass
+
+    def is_user_authenticated(self):
+        # always authenticated
+        return True
+
+    def get_user_id(self):
+        if self.CXGUID not in session:
+            session[self.CXGUID] = uuid4().hex
+            session.permanent = True
+        return session[self.CXGUID]
+
+    def get_user_name(self):
+        return "anonymous"
+
+    def get_user_email(self):
+        return None
+
+
+AuthTypeFactory.register("session", AuthTypeSession)

backend/czi_hosted/auth/auth_test.py

@@ -1,6 +1,7 @@
-from server.auth.auth import AuthTypeClientBase, AuthTypeFactory
 from flask import session, request, redirect, current_app
 
+from backend.czi_hosted.auth.auth import AuthTypeClientBase, AuthTypeFactory
+
 
 class AuthTypeTest(AuthTypeClientBase):
     """An authentication type for testing client based logins.  When the login route is accessed

backend/czi_hosted/cli/cli.py

@@ -4,6 +4,7 @@ from .convert_to_cxg import convert_to_cxg
 from .launch import launch
 from .prepare import prepare
 from .upgrade import log_upgrade_check
+from .schema import schema_cli
 from .. import __version__
 
 
@@ -31,3 +32,4 @@ def cli(upgrade_check):
 cli.add_command(launch)
 cli.add_command(prepare)
 cli.add_command(convert_to_cxg)
+cli.add_command(schema_cli)

backend/czi_hosted/cli/convert_to_cxg.py

@@ -2,7 +2,7 @@ from os import path
 
 import click
 
-from server.converters.h5ad_data_file import H5ADDataFile
+from backend.czi_hosted.converters.h5ad_data_file import H5ADDataFile
 
 
 @click.command(

backend/czi_hosted/cli/launch.py

@@ -8,11 +8,12 @@ import click
 from flask_compress import Compress
 from flask_cors import CORS
 
-from server.default_config import default_config
-from server.app.app import Server
-from server.common.config.app_config import AppConfig
-from server.common.errors import DatasetAccessError, ConfigurationError
-from server.common.utils.utils import sort_options
+from backend.czi_hosted.default_config import default_config
+from backend.czi_hosted.app.app import Server
+from backend.czi_hosted.common.config.app_config import AppConfig
+from backend.common.errors import DatasetAccessError, ConfigurationError
+from backend.common.utils.utils import sort_options
+
 
 DEFAULT_CONFIG = AppConfig()
 
@@ -355,7 +356,6 @@ def launch(
     if dump_default_config:
         print(default_config)
         sys.exit(0)
-
     # Startup message
     click.echo("[cellxgene] Starting the CLI...")
 

backend/czi_hosted/cli/prepare.py

@@ -5,7 +5,7 @@ import pandas as pd
 from numpy import ndarray, unique
 from scipy.sparse.csc import csc_matrix
 
-from server.common.utils.utils import sort_options
+from backend.common.utils.utils import sort_options
 
 
 @sort_options

backend/czi_hosted/cli/schema.py

@@ -0,0 +1,72 @@
+import click
+
+from backend.czi_hosted.converters.schema import remix, validate
+
+
+@click.group(
+    name="schema",
+    subcommand_metavar="COMMAND <args>",
+    short_help="Apply and validate the cellxgene data integration schema to an h5ad file.",
+    context_settings=dict(max_content_width=85, help_option_names=["-h", "--help"]),
+)
+def schema_cli():
+    try:
+        import scanpy  # noqa: F401
+    except ImportError:
+        raise click.ClickException(
+            "[cellxgene] cellxgene schema requires scanpy"
+        )
+
+
+@click.command(
+    name="apply",
+    short_help="(experimental) Apply the cellxgene data integration schema to an h5ad.",
+    help="(experimental) Using a yaml file that describes schema values to insert or convert and in input "
+    "h5ad file, apply the schema changes and create a new, conforming h5ad.",
+)
+@click.option(
+    "--source-h5ad",
+    help="Input h5ad file.",
+    nargs=1,
+    required=True,
+    type=click.Path(exists=True, dir_okay=False),
+)
+@click.option(
+    "--remix-config",
+    help="Config yaml with information on how to apply the schema.",
+    nargs=1,
+    required=True,
+    type=click.Path(exists=True, dir_okay=False),
+)
+@click.option(
+    "--output-filename",
+    help="Filename for the new, schema-conforming h5ad file.",
+    required=True,
+    nargs=1
+)
+def schema_apply(source_h5ad, remix_config, output_filename):
+    remix.apply_schema(source_h5ad, remix_config, output_filename)
+
+
+@click.command(
+    name="validate",
+    short_help="(experimental) Check that an h5ad follows the cellxgene data integration schema.",
+)
+@click.argument(
+    "h5ad",
+    nargs=1,
+    type=click.Path(exists=True, dir_okay=False),
+)
+@click.option(
+    "--shallow",
+    help="When true, just check that the correct version information is present.",
+    default=False,
+    show_default=True,
+    is_flag=True,
+)
+def schema_validate(h5ad, shallow):
+    validate.validate(h5ad, shallow)
+
+
+schema_cli.add_command(schema_apply)
+schema_cli.add_command(schema_validate)

backend/czi_hosted/common/annotations/annotations.py

@@ -0,0 +1,138 @@
+import fastobo
+import fsspec
+import os
+
+from flask import current_app, has_request_context
+
+from backend.common.errors import OntologyLoadFailure, DisabledFeatureError
+from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
+from backend.common.genesets import write_gene_sets_tidycsv, read_gene_sets_tidycsv, validate_gene_sets
+from backend.common.utils.data_locator import DataLocator
+from backend.common.utils.utils import path_join
+
+
+class Annotations:
+    """ baseclass for annotations, including ontologies and genesets """
+
+    """ our default ontology is the PURL for the Cell Ontology.
+    See http://www.obofoundry.org/ontology/cl.html """
+    DefaultOnotology = "http://purl.obolibrary.org/obo/cl.obo"
+
+    def __init__(self, config={}):
+        self.ontology_data = None
+        self.config = config
+
+    def user_annotations_enabled(self):
+        return self.config.get("user-annotations", False)
+
+    def check_user_annotations_enabled(self):
+        if not self.user_annotations_enabled():
+            raise DisabledFeatureError("User annotations are disabled.")
+
+    def load_ontology(self, path):
+        """Load and parse ontologies - currently support OBO files only."""
+        if path is None:
+            path = self.DefaultOnotology
+
+        try:
+            with fsspec.open(path) as f:
+                obo = fastobo.iter(f)
+                terms = filter(lambda stanza: type(stanza) is fastobo.term.TermFrame, obo)
+                names = [tag.name for term in terms for tag in term if type(tag) is fastobo.term.NameClause]
+                self.ontology_data = names
+
+        except FileNotFoundError as e:
+            raise OntologyLoadFailure("Unable to find OBO ontology path") from e
+
+        except SyntaxError as e:
+            raise OntologyLoadFailure(f"{path}:{e.lineno}:{e.offset} OBO syntax error, unable to read ontology") from e
+
+        except Exception as e:
+            raise OntologyLoadFailure(f"{path}:Error loading OBO file") from e
+
+    def get_schema(self, data_adaptor):
+        schema = []
+        labels = self.read_labels(data_adaptor)
+        if labels is not None and not labels.empty:
+            for col in labels.columns:
+                col_schema = dict(name=col, writable=True)
+                col_schema.update(get_schema_type_hint_of_array(labels[col]))
+                schema.append(col_schema)
+
+        return schema
+
+    def set_collection(self, name):
+        """set or create a new annotation collection"""
+        raise NotImplementedError
+
+    def read_labels(self, data_adaptor):
+        """Return the labels as a pandas.DataFrame"""
+        raise NotImplementedError
+
+    def write_labels(self, df, data_adaptor):
+        """Write the labels (df) to a persistent storage such that it can later be read"""
+        raise NotImplementedError
+
+    def update_parameters(self, parameters, data_adaptor):
+        """Update configuration parameters that describe information about the annotations feature"""
+        params = {}
+        params["annotations_genesets_readonly"] = True
+        params["annotations_genesets_name_is_read_only"] = True
+        parameters.update(params)
+
+    @staticmethod
+    def gene_sets_to_csv(genesets):
+        """
+        Convert the internal genesets format (returned by read_gene_set) into
+        the simple Tidy CSV.
+        """
+        from io import StringIO
+
+        if isinstance(genesets, dict):
+            genesets = genesets.values()
+
+        with StringIO() as sio:
+            write_gene_sets_tidycsv(sio, genesets)
+            return sio.getvalue()
+
+    @staticmethod
+    def gene_sets_to_response(genesets):
+        """
+        Convert the internal genesets format (returned by read_gene_set) into
+        the dict expected by the JSON REST API
+        """
+        return list(genesets.values())
+
+    def read_gene_sets(self, data_adaptor, context=None):
+        if has_request_context():
+            if not current_app.auth.is_user_authenticated():
+                return ({}, 0)
+
+        gene_sets_uri_or_path = dataset_uri_to_geneset_uri(data_adaptor.data_locator.uri_or_path)
+
+        server_config = data_adaptor.server_config
+        region_name = None if server_config is None else server_config.data_locator__s3__region_name
+        gene_sets_locator = DataLocator(gene_sets_uri_or_path, region_name=region_name)
+        if not gene_sets_locator.exists():
+            return ({}, 0)
+
+        gene_sets = read_gene_sets_tidycsv(gene_sets_locator, context)
+        schema = data_adaptor.get_schema()
+        var_index = schema["annotations"]["var"].get("index", "index")
+        var_names = set(data_adaptor.query_var_array(var_index))
+
+        gene_sets = validate_gene_sets(gene_sets, var_names)
+        return (gene_sets, 0)
+
+
+def dataset_uri_to_geneset_uri(data_uri_or_path):
+    """ given a dataset URI, return the associated gene set URI """
+    data_basename = os.path.basename(data_uri_or_path)
+    base, ext = os.path.splitext(data_basename)
+    if ext is not None:  # strip extension, if any
+        data_basename = base
+
+    genesets_basename = f"{data_basename}-genesets.csv"
+    gene_sets_uri_or_path = path_join(data_uri_or_path, "..", genesets_basename)
+
+    return gene_sets_uri_or_path

backend/czi_hosted/common/annotations/hosted_tiledb.py

@@ -7,18 +7,18 @@ import pandas as pd
 import tiledb
 from flask import current_app
 
-from server.common.annotations.annotations import Annotations
-from server.common.errors import AnnotationCategoryNameError
-from server.common.utils.sanitization_utils import sanitize_values_in_list
-from server.common.utils.type_conversion_utils import get_dtypes_and_schemas_of_dataframe, get_dtype_of_array
-from server.db.cellxgene_orm import Annotation
+from backend.czi_hosted.common.annotations.annotations import Annotations
+from backend.common.errors import AnnotationCategoryNameError
+from backend.czi_hosted.common.utils.sanitization_utils import sanitize_values_in_list
+from backend.common.utils.type_conversion_utils import get_dtypes_and_schemas_of_dataframe, get_dtype_of_array
+from backend.czi_hosted.db.cellxgene_orm import Annotation
 
 
 class AnnotationsHostedTileDB(Annotations):
     CXG_ANNO_COLLECTION = "cxg_anno_collection"
 
-    def __init__(self, directory_path, db):
-        super().__init__()
+    def __init__(self, config, directory_path, db):
+        super().__init__(config)
         self.db = db
         if directory_path[-1] == "/":
             self.directory_path = directory_path
@@ -34,6 +34,12 @@ class AnnotationsHostedTileDB(Annotations):
                 f"{unsanitary_original_category_names} are not valid category names, please resubmit"
             )
 
+    def get_user_name(self):
+        return current_app.auth.get_user_name()
+
+    def get_user_id(self):
+        return current_app.auth.get_user_id()
+
     def is_safe_collection_name(self, name):
         """
         return true if this is a safe collection name
@@ -48,7 +54,7 @@ class AnnotationsHostedTileDB(Annotations):
         self.CXG_ANNO_COLLECTION = name
 
     def read_labels(self, data_adaptor):
-        user_id = current_app.auth.get_user_id()
+        user_id = self.get_user_id()
         if user_id is None:
             return
         dataset_name = data_adaptor.get_location()
@@ -58,7 +64,15 @@ class AnnotationsHostedTileDB(Annotations):
             Annotation, [Annotation.user_id == user_id, Annotation.dataset_id == dataset_id]
         )
         if annotation_object:
-            df = tiledb.open(annotation_object.tiledb_uri)
+            if annotation_object.tiledb_uri == "":
+                # this mean the user has removed all the categories.
+                return None
+            try:
+                df = tiledb.open(annotation_object.tiledb_uri)
+            except tiledb.TileDBError:
+                # don't crash if the annotations file is missing or can't be read.
+                current_app.logger.warning(f"Cannot read annotation file: {annotation_object.tiledb_uri}")
+                return None
             pandas_df = self.convert_to_pandas_df(df, annotation_object.schema_hints)
             return pandas_df
         else:
@@ -103,8 +117,8 @@ class AnnotationsHostedTileDB(Annotations):
         return new_df
 
     def write_labels(self, df, data_adaptor):
-        auth_user_id = current_app.auth.get_user_id()
-        user_name = current_app.auth.get_user_name()
+        auth_user_id = self.get_user_id()
+        user_name = self.get_user_name()
         timestamp = time.time()
         dataset_location = data_adaptor.get_location()
         dataset_id = self.db.get_or_create_dataset(dataset_location)
@@ -124,24 +138,28 @@ class AnnotationsHostedTileDB(Annotations):
         else:
             os.makedirs(uri, exist_ok=True)
         _, dataframe_schema_type_hints = get_dtypes_and_schemas_of_dataframe(df)
-        annotation = Annotation(
-            tiledb_uri=uri,
-            user_id=user_id,
-            dataset_id=str(dataset_id),
-            schema_hints=json.dumps(dataframe_schema_type_hints),
-        )
         if not df.empty:
             self.check_category_names(df)
             # convert to tiledb datatypes
 
             for col in df:
                 df[col] = df[col].astype(get_dtype_of_array(df[col]))
-            tiledb.from_pandas(uri, df)
+            tiledb.from_pandas(uri, df, sparse=True)
+        else:
+            uri = ""
 
+        annotation = Annotation(
+            tiledb_uri=uri,
+            user_id=user_id,
+            dataset_id=str(dataset_id),
+            schema_hints=json.dumps(dataframe_schema_type_hints),
+        )
         self.db.session.add(annotation)
         self.db.session.commit()
 
     def update_parameters(self, parameters, data_adaptor):
+        super().update_parameters(parameters, data_adaptor)
+
         params = {}
         params["annotations"] = True
         params["user_annotation_collection_name_enabled"] = False

backend/czi_hosted/common/annotations/local_file_csv.py

@@ -8,16 +8,16 @@ from hashlib import blake2b
 import pandas as pd
 from flask import session, has_request_context, current_app
 
-from server import __version__ as cellxgene_version
-from server.common.annotations.annotations import Annotations
-from server.common.errors import AnnotationsError
+from backend.czi_hosted import __version__ as cellxgene_version
+from backend.czi_hosted.common.annotations.annotations import Annotations
+from backend.common.errors import AnnotationsError
 
 
 class AnnotationsLocalFile(Annotations):
     CXG_ANNO_COLLECTION = "cxg_anno_collection"
 
-    def __init__(self, output_dir, output_file):
-        super().__init__()
+    def __init__(self, config, output_dir, output_file):
+        super().__init__(config)
         self.output_dir = output_dir
         self.output_file = output_file
         # lock used to protect label file write ops
@@ -169,6 +169,8 @@ class AnnotationsLocalFile(Annotations):
                 os.remove(os.path.join(backup_dir, bu))
 
     def update_parameters(self, parameters, data_adaptor):
+        super().update_parameters(parameters, data_adaptor)
+
         params = {}
         params["annotations"] = True
         params["user_annotation_collection_name_enabled"] = True
@@ -190,7 +192,7 @@ class AnnotationsLocalFile(Annotations):
             collection = self.get_collection()
             if current_app.auth.is_user_authenticated():
                 params["annotations-user-data-idhash"] = self._get_userdata_idhash(data_adaptor)
-                params["annotations-data-collection-is-read-only"] = False
+                params["annotations-data-collection-is-read-only"] = not self.user_annotations_enabled()
                 params["annotations-data-collection-name"] = collection
 
         parameters.update(params)

backend/czi_hosted/common/config/__init__.py

@@ -0,0 +1,4 @@
+from backend.common.utils.aws_secret_utils import get_secret_key  # noqa F504
+
+DEFAULT_SERVER_PORT = 5005
+BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20  # 100MB

backend/czi_hosted/common/config/app_config.py

@@ -1,11 +1,11 @@
 import yaml
 from flatten_dict import unflatten
 
-from server.default_config import get_default_config
-from server.common.config.dataset_config import DatasetConfig
-from server.common.config.server_config import ServerConfig
-from server.common.config.external_config import ExternalConfig
-from server.common.errors import ConfigurationError
+from backend.czi_hosted.common.config.external_config import ExternalConfig
+from backend.czi_hosted.common.config.dataset_config import DatasetConfig
+from backend.czi_hosted.common.config.server_config import ServerConfig
+from backend.common.errors import ConfigurationError
+from backend.czi_hosted.default_config import get_default_config
 
 
 class AppConfig(object):
@@ -69,21 +69,21 @@ class AppConfig(object):
 
     def update_server_config(self, **kw):
         self.server_config.update(**kw)
-        self.is_complete = False
+        self.is_completed = False
 
     def update_default_dataset_config(self, **kw):
         self.default_dataset_config.update(**kw)
         # update all the other dataset configs, if any
         for value in self.dataroot_config.values():
             value.update(**kw)
-        self.is_complete = False
+        self.is_completed = False
 
     def update_single_config_from_path_and_value(self, path, value):
         """Update a single config parameter with the value.
         Path is a list of string, that gives a path to the config parameter to be updated.
         For example, path may be ["server","app","port"].
         """
-        self.is_complete = False
+        self.is_completed = False
         if not isinstance(path, list):
             raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'")
         for part in path:
@@ -147,7 +147,7 @@ class AppConfig(object):
         if config.get("external"):
             self.external_config.update_from_config(config["external"], "external")
 
-        self.is_complete = False
+        self.is_completed = False
 
     def config_to_dict(self):
         """return the configuration as an unflattened dict"""

backend/czi_hosted/common/config/base_config.py

@@ -1,7 +1,7 @@
 import copy
 
 from flatten_dict import flatten
-from server.common.errors import ConfigurationError
+from backend.common.errors import ConfigurationError
 
 
 class BaseConfig(object):

backend/czi_hosted/common/config/client_config.py

@@ -1,4 +1,4 @@
-from server import display_version as cellxgene_display_version
+from backend.czi_hosted import display_version as cellxgene_display_version
 
 
 def get_client_config(app_config, data_adaptor):
@@ -44,6 +44,9 @@ def get_client_config(app_config, data_adaptor):
         "annotations": False,
         "annotations_file": None,
         "annotations_dir": None,
+        "annotations_genesets": True,  # feature flag
+        "annotations_genesets_readonly": True,
+        "annotations_genesets_summary_methods": ["mean"],
         "annotations_cell_ontology_enabled": False,
         "annotations_cell_ontology_obopath": None,
         "annotations_cell_ontology_terms": None,

backend/czi_hosted/common/config/dataset_config.py

@@ -1,13 +1,14 @@
 import os
 from os.path import splitext, isdir
 
-from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
-from server.common.annotations.local_file_csv import AnnotationsLocalFile
-from server.common.config.base_config import BaseConfig
-from server.common.errors import ConfigurationError, OntologyLoadFailure
-from server.compute.scanpy import get_scanpy_module
-from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataType
-from server.db.db_utils import DbUtils
+from backend.czi_hosted.common.annotations.annotations import Annotations
+from backend.czi_hosted.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
+from backend.czi_hosted.common.annotations.local_file_csv import AnnotationsLocalFile
+from backend.czi_hosted.common.config.base_config import BaseConfig
+from backend.common.errors import ConfigurationError, OntologyLoadFailure
+from backend.czi_hosted.compute.scanpy import get_scanpy_module
+from backend.czi_hosted.data_common.matrix_loader import MatrixDataLoader, MatrixDataType
+from backend.czi_hosted.db.db_utils import DbUtils
 
 
 class DatasetConfig(BaseConfig):
@@ -53,8 +54,10 @@ class DatasetConfig(BaseConfig):
         except KeyError as e:
             raise ConfigurationError(f"Unexpected config: {str(e)}")
 
-        # The annotation object is created during complete_config and stored here.
-        self.user_annotations = None
+        # Create the default annotation, which supports gene set reading without
+        # further configuration. Depending on configuration options, `complete_config` 
+        # may create a more specialized annotation object and replace this default.
+        self.user_annotations = Annotations()
 
     def complete_config(self, context):
         self.handle_app()
@@ -147,7 +150,11 @@ class DatasetConfig(BaseConfig):
             except OSError:
                 raise ConfigurationError("Unable to create directory specified by --annotations-dir")
 
-        self.user_annotations = AnnotationsLocalFile(dirname, filename)
+        anno_config = {
+            "user-annotations": self.user_annotations__enable,
+            "genesets-save": False,
+        }
+        self.user_annotations = AnnotationsLocalFile(anno_config, dirname, filename)
 
         # if the user has specified a fixed label file, go ahead and validate it
         # so that we can remove errors early in the process.
@@ -163,7 +170,12 @@ class DatasetConfig(BaseConfig):
         self.validate_correct_type_of_configuration_attribute(
             "user_annotations__hosted_tiledb_array__hosted_file_directory", str
         )
+        anno_config = {
+            "user-annotations": self.user_annotations__enable,
+            "genesets-save": False,
+        }
         self.user_annotations = AnnotationsHostedTileDB(
+            anno_config,
             directory_path=self.user_annotations__hosted_tiledb_array__hosted_file_directory,
             db=DbUtils(self.user_annotations__hosted_tiledb_array__db_uri),
         )

backend/czi_hosted/common/config/external_config.py

@@ -1,10 +1,9 @@
 import os
 
-from server.common.config.base_config import BaseConfig
-from server.common.errors import ConfigurationError
-from server.common.config import get_secret_key
-from server.common.errors import SecretKeyRetrievalError
-from server.common.utils.type_conversion_utils import convert_string_to_value
+from backend.czi_hosted.common.config.base_config import BaseConfig
+from backend.common.errors import ConfigurationError, SecretKeyRetrievalError
+from backend.common.utils.aws_secret_utils import get_secret_key
+from backend.common.utils.type_conversion_utils import convert_string_to_value
 
 
 class ExternalConfig(BaseConfig):

backend/czi_hosted/common/config/server_config.py

@@ -4,14 +4,14 @@ import warnings
 from os.path import basename
 from urllib.parse import urlparse, quote_plus
 
-from server.auth.auth import AuthTypeFactory
-from server.common.config.base_config import BaseConfig
-from server.common.config import DEFAULT_SERVER_PORT, BIG_FILE_SIZE_THRESHOLD
-from server.common.errors import ConfigurationError, DatasetAccessError
-from server.common.data_locator import discover_s3_region_name
-from server.common.utils.utils import is_port_available, find_available_port, custom_format_warning
-from server.compute import diffexp_cxg as diffexp_tiledb
-from server.data_common.matrix_loader import MatrixDataCacheManager, MatrixDataLoader, MatrixDataType
+from backend.czi_hosted.auth.auth import AuthTypeFactory
+from backend.czi_hosted.common.config import DEFAULT_SERVER_PORT, BIG_FILE_SIZE_THRESHOLD
+from backend.czi_hosted.common.config.base_config import BaseConfig
+from backend.common.utils.data_locator import discover_s3_region_name
+from backend.common.errors import ConfigurationError, DatasetAccessError
+from backend.common.utils.utils import is_port_available, find_available_port, custom_format_warning
+from backend.czi_hosted.compute import diffexp_cxg as diffexp_tiledb
+from backend.czi_hosted.data_common.matrix_loader import MatrixDataCacheManager, MatrixDataLoader, MatrixDataType
 
 
 class ServerConfig(BaseConfig):
@@ -42,6 +42,9 @@ class ServerConfig(BaseConfig):
             self.app__web_base_url = default_config["app"]["web_base_url"]
 
             self.authentication__type = default_config["authentication"]["type"]
+            self.authentication__insecure_test_environment = default_config["authentication"][
+                "insecure_test_environment"
+            ]
             self.authentication__params_oauth__oauth_api_base_url = default_config["authentication"]["params_oauth"][
                 "oauth_api_base_url"
             ]
@@ -168,6 +171,10 @@ class ServerConfig(BaseConfig):
 
     def handle_authentication(self):
         self.validate_correct_type_of_configuration_attribute("authentication__type", (type(None), str))
+        self.validate_correct_type_of_configuration_attribute("authentication__insecure_test_environment", bool)
+
+        if self.authentication__type == "test" and not self.authentication__insecure_test_environment:
+            raise ConfigurationError("Test auth can only be used in an insecure test environment")
 
         # oauth
         ptypes = str if self.authentication__type == "oauth" else (type(None), str)
@@ -346,7 +353,7 @@ class ServerConfig(BaseConfig):
             if type(self.data_locator__s3__region_name) == str:
                 self.adaptor__cxg_adaptor__tiledb_ctx[regionkey] = self.data_locator__s3__region_name
 
-        from server.data_cxg.cxg_adaptor import CxgAdaptor
+        from backend.czi_hosted.data_cxg.cxg_adaptor import CxgAdaptor
 
         CxgAdaptor.set_tiledb_context(self.adaptor__cxg_adaptor__tiledb_ctx)
 

backend/czi_hosted/common/corpora.py

@@ -8,8 +8,8 @@ https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/c
 import collections
 import json
 
-from server.cli.upgrade import validate_version_str
-from server.common.utils.corpora_constants import CorporaConstants
+from backend.czi_hosted.cli.upgrade import validate_version_str
+from backend.czi_hosted.common.utils.corpora_constants import CorporaConstants
 
 
 def corpora_get_versions_from_anndata(adata):
@@ -63,9 +63,9 @@ def corpora_get_props_from_anndata(adata):
             raise KeyError(f"missing Corpora schema field {key}")
         corpora_props[key] = adata.uns[key]
 
-    for key in CorporaConstants.REQUIRED_JSON_ENCODED_METADATA_FIELD:
+    for key in CorporaConstants.OPTIONAL_JSON_ENCODED_METADATA_FIELD:
         if key not in adata.uns:
-            raise KeyError(f"missing Corpora schema field {key}")
+            continue
         try:
             corpora_props[key] = json.loads(adata.uns[key])
         except json.JSONDecodeError:

backend/czi_hosted/common/health.py

@@ -1,8 +1,8 @@
 from http import HTTPStatus
 from flask import make_response, jsonify
 
-from server import __version__ as cellxgene_version
-from server.common.data_locator import DataLocator
+from backend.czi_hosted import __version__ as cellxgene_version
+from backend.common.utils.data_locator import DataLocator
 
 
 def _is_accessible(path, config):

backend/czi_hosted/common/rest.py

@@ -2,13 +2,15 @@ import copy
 import logging
 import sys
 from http import HTTPStatus
+import zlib
+import json
 
 from flask import make_response, jsonify, current_app, abort
 from werkzeug.urls import url_unquote
 
-from server.common.config.client_config import get_client_config, get_client_userinfo
-from server.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg
-from server.common.errors import (
+from backend.czi_hosted.common.config.client_config import get_client_config, get_client_userinfo
+from backend.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg
+from backend.common.errors import (
     FilterError,
     JSONEncodingValueError,
     PrepareError,
@@ -16,10 +18,11 @@ from server.common.errors import (
     ExceedsLimitError,
     DatasetAccessError,
     ColorFormatException,
+    AnnotationsError,
+    UnsupportedSummaryMethod,
 )
-
-import json
-from server.data_common.fbs.matrix import decode_matrix_fbs
+from backend.common.genesets import summarizeQueryHash
+from backend.common.fbs.matrix import decode_matrix_fbs
 
 
 def abort_and_log(code, logmsg, loglevel=logging.DEBUG, include_exc_info=False):
@@ -105,7 +108,7 @@ def schema_get_helper(data_adaptor):
 
     # add label obs annotations as needed
     annotations = data_adaptor.dataset_config.user_annotations
-    if annotations is not None:
+    if annotations.user_annotations_enabled():
         label_schema = annotations.get_schema(data_adaptor)
         schema["annotations"]["obs"]["columns"].extend(label_schema)
 
@@ -139,7 +142,7 @@ def annotations_obs_get(request, data_adaptor):
     try:
         labels = None
         annotations = data_adaptor.dataset_config.user_annotations
-        if annotations:
+        if annotations.user_annotations_enabled():
             labels = annotations.read_labels(data_adaptor)
         fbs = data_adaptor.annotation_to_fbs_matrix(Axis.OBS, fields, labels)
         return make_response(fbs, HTTPStatus.OK, {"Content-Type": "application/octet-stream"})
@@ -150,22 +153,26 @@ def annotations_obs_get(request, data_adaptor):
 def annotations_put_fbs_helper(data_adaptor, fbs):
     """helper function to write annotations from fbs"""
     annotations = data_adaptor.dataset_config.user_annotations
-    if annotations is None:
+    if not annotations.user_annotations_enabled():
         raise DisabledFeatureError("Writable annotations are not enabled")
 
     new_label_df = decode_matrix_fbs(fbs)
     if not new_label_df.empty:
-        data_adaptor.check_new_labels(new_label_df)
+        new_label_df = data_adaptor.check_new_labels(new_label_df)
     annotations.write_labels(new_label_df, data_adaptor)
 
 
+def inflate(data):
+    return zlib.decompress(data)
+
+
 def annotations_obs_put(request, data_adaptor):
     annotations = data_adaptor.dataset_config.user_annotations
-    if annotations is None:
+    if not annotations.user_annotations_enabled():
         return abort(HTTPStatus.NOT_IMPLEMENTED)
 
     anno_collection = request.args.get("annotation-collection-name", default=None)
-    fbs = request.get_data()
+    fbs = inflate(request.get_data())
 
     if anno_collection is not None:
         if not annotations.is_safe_collection_name(anno_collection):
@@ -192,7 +199,7 @@ def annotations_var_get(request, data_adaptor):
     try:
         labels = None
         annotations = data_adaptor.dataset_config.user_annotations
-        if annotations is not None:
+        if annotations.user_annotations_enabled():
             labels = annotations.read_labels(data_adaptor)
         return make_response(
             data_adaptor.annotation_to_fbs_matrix(Axis.VAR, fields, labels),
@@ -253,7 +260,6 @@ def diffexp_obs_post(request, data_adaptor):
     try:
         # TODO: implement varfilter mode
         mode = DiffExpMode(args["mode"])
-
         if mode == DiffExpMode.VAR_FILTER or "varFilter" in args:
             return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, "varFilter not enabled")
 
@@ -323,3 +329,70 @@ def layout_obs_put(request, data_adaptor):
         return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, str(e))
     except (ValueError, DisabledFeatureError, FilterError) as e:
         return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+
+
+def genesets_get(request, data_adaptor):
+    preferred_mimetype = request.accept_mimetypes.best_match(["application/json", "text/csv"])
+    if preferred_mimetype not in ("application/json", "text/csv"):
+        return abort(HTTPStatus.NOT_ACCEPTABLE)
+
+    try:
+        annotations = data_adaptor.dataset_config.user_annotations
+        (genesets, tid) = annotations.read_gene_sets(data_adaptor)
+
+        if preferred_mimetype == "text/csv":
+            return make_response(
+                annotations.gene_sets_to_csv(genesets),
+                HTTPStatus.OK,
+                {
+                    "Content-Type": "text/csv",
+                    "Content-Disposition": "attachment; filename=genesets.csv",
+                },
+            )
+        else:
+            return make_response(
+                jsonify({"genesets": annotations.gene_sets_to_response(genesets), "tid": tid}), HTTPStatus.OK
+            )
+    except (ValueError, KeyError, AnnotationsError) as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e))
+
+
+def summarize_var_helper(request, data_adaptor, key, raw_query):
+    preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
+    if preferred_mimetype != "application/octet-stream":
+        return abort(HTTPStatus.NOT_ACCEPTABLE)
+
+    summary_method = request.values.get("method", default="mean")
+    query_hash = summarizeQueryHash(raw_query)
+    if key and query_hash != key:
+        return abort(HTTPStatus.BAD_REQUEST, description="query key did not match")
+
+    args_filter_only = request.values.copy()
+    args_filter_only.poplist("method")
+    args_filter_only.poplist("key")
+
+    try:
+        filter = _query_parameter_to_filter(args_filter_only)
+        return make_response(
+            data_adaptor.summarize_var(summary_method, filter, query_hash),
+            HTTPStatus.OK,
+            {"Content-Type": "application/octet-stream"},
+        )
+    except (ValueError) as e:
+        return abort(HTTPStatus.NOT_FOUND, description=str(e))
+    except (UnsupportedSummaryMethod, FilterError) as e:
+        return abort(HTTPStatus.BAD_REQUEST, description=str(e))
+
+
+def summarize_var_get(request, data_adaptor):
+    return summarize_var_helper(request, data_adaptor, None, request.query_string)
+
+
+def summarize_var_post(request, data_adaptor):
+    if not request.content_type or "application/x-www-form-urlencoded" not in request.content_type:
+        return abort(HTTPStatus.UNSUPPORTED_MEDIA_TYPE)
+    if request.content_length > 1_000_000:  # just a sanity check to avoid memory exhaustion
+        return abort(HTTPStatus.BAD_REQUEST)
+
+    key = request.args.get("key", default=None)
+    return summarize_var_helper(request, data_adaptor, key, request.get_data())

backend/czi_hosted/common/utils/corpora_constants.py

@@ -5,12 +5,18 @@ class CorporaConstants(object):
         "layer_descriptions",
         "organism",
         "organism_ontology_term_id",
-        "project_name",
-        "project_description",
     ]
 
     # The Corpora specification requires some values encoded as JSON due to the inability of AnnData to store complex
     # types.
-    REQUIRED_JSON_ENCODED_METADATA_FIELD = ["contributors", "project_links"]
+    OPTIONAL_JSON_ENCODED_METADATA_FIELD = ["contributors", "project_links"]
 
-    OPTIONAL_SIMPLE_METADATA_FIELDS = ["preprint_doi", "publication_doi", "default_embedding", "default_field", "tags"]
+    OPTIONAL_SIMPLE_METADATA_FIELDS = [
+        "preprint_doi",
+        "publication_doi",
+        "default_embedding",
+        "default_field",
+        "tags",
+        "project_name",
+        "project_description",
+    ]

backend/czi_hosted/common/utils/cxg_generation_utils.py

@@ -3,7 +3,7 @@ import json
 import numpy as np
 import tiledb
 
-from server.common.utils.type_conversion_utils import get_dtype_of_array, get_dtype_and_schema_of_array
+from backend.common.utils.type_conversion_utils import get_dtype_of_array, get_dtype_and_schema_of_array
 
 
 def convert_dictionary_to_cxg_group(cxg_container, metadata_dict, group_metadata_name="cxg_group_metadata"):

backend/czi_hosted/compute/diffexp_cxg.py

@@ -1,10 +1,11 @@
 import concurrent.futures
 import numpy as np
-from server.compute.diffexp_generic import diffexp_ttest_from_mean_var, mean_var_n
-from server.data_cxg.cxg_util import pack_selector_from_indices
-from server.common.errors import ComputeError
+
 from numba import jit
 
+from backend.czi_hosted.data_cxg.cxg_util import pack_selector_from_indices
+from backend.common.compute.diffexp_generic import diffexp_ttest_from_mean_var, mean_var_n
+from backend.common.errors import ComputeError
 
 """
 See the comments in diffexp_generic for a description of this algorithm
@@ -114,14 +115,14 @@ def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
             meanB += X_col_shift
 
     r = diffexp_ttest_from_mean_var(
-        meanA.astype(dtype),
-        varA.astype(dtype),
-        nA,
-        meanB.astype(dtype),
-        varB.astype(dtype),
-        nB,
-        top_n,
-        diffexp_lfc_cutoff,
+        meanA=meanA.astype(dtype),
+        varA=varA.astype(dtype),
+        nA=nA,
+        meanB=meanB.astype(dtype),
+        varB=varB.astype(dtype),
+        nB=nB,
+        top_n=top_n,
+        diffexp_lfc_cutoff=diffexp_lfc_cutoff
     )
 
     return r

backend/czi_hosted/converters/h5ad_data_file.py

@@ -6,17 +6,17 @@ import anndata
 import numpy as np
 import tiledb
 
-from server.common.colors import convert_anndata_category_colors_to_cxg_category_colors
-from server.common.corpora import corpora_get_props_from_anndata
-from server.common.errors import ColorFormatException
-from server.common.utils.cxg_constants import CxgConstants
-from server.common.utils.cxg_generation_utils import (
+from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
+from backend.czi_hosted.common.corpora import corpora_get_props_from_anndata
+from backend.common.errors import ColorFormatException
+from backend.czi_hosted.common.utils.cxg_constants import CxgConstants
+from backend.czi_hosted.common.utils.cxg_generation_utils import (
     convert_dictionary_to_cxg_group,
     convert_dataframe_to_cxg_array,
     convert_ndarray_to_cxg_dense_array,
     convert_matrix_to_cxg_array,
 )
-from server.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix
+from backend.czi_hosted.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix
 
 
 class H5ADDataFile:

backend/czi_hosted/converters/schema/gene_symbol.py

@@ -0,0 +1,211 @@
+"""Helpers for converting and checking HGNC gene symbols."""
+
+import argparse
+import enum
+import logging
+import os
+import re
+import numpy as np
+import pandas as pd
+
+
+def get_upgraded_var_index(var, hgnc_path=None):
+    """Given an anndata var dataframe, return a new index for the dataframe
+    where human gene symbols have been upgraded to the current HGNC set.
+    """
+
+    if not hgnc_path:
+        hgnc_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "hgnc_complete_set.txt.gz")
+
+    hgnc_symbol_checker = HGNCSymbolChecker.from_hgnc_records(hgnc_path)
+
+    return pd.Index([hgnc_symbol_checker.upgrade_symbol(s) for s in var.index])
+
+
+class SymbolStatus(enum.Enum):
+    """The status of a symbol in the HGNC database.
+
+    APPROVED: Currently a valid symbol
+    WITHDRAWN: A previously approved HGNC symbol for a gene that has since been shown
+               not to exist _unless_ that symbol is also approved
+    AMBIGUOUS: A symbol that is not approved but is an alias or previous symbol for
+               multiple approved symbols
+    UPGRADABLE: A symbol that is not approved but unambiguously maps to an approved
+                symbol
+    UNKNOWN: A symbol that does not appear in HGNC
+    """
+
+    APPROVED = 1
+    WITHDRAWN = 2
+    AMBIGUOUS = 3
+    UPGRADABLE = 4
+    UNKNOWN = 5
+
+
+class HGNCSymbolChecker:
+    """Handle checking and correcting HGNC symbols."""
+
+    def __init__(self, approved_symbols, withdrawn_symbols, ambiguous_symbols, symbol_map):
+        self.approved_symbols = approved_symbols
+        self.withdrawn_symbols = withdrawn_symbols
+        self.ambiguous_symbols = ambiguous_symbols
+        self.symbol_map = symbol_map
+
+    def print_symbol_map(self):
+        """Print out a map from old symbol to new symbol."""
+
+        for symbol_pair in self.symbol_map.items():
+            print("\t".join(symbol_pair))
+
+    def check_symbol(self, symbol):
+        """See if a symbol if approved or something else."""
+        if symbol in self.approved_symbols:
+            return SymbolStatus.APPROVED
+
+        if symbol in self.withdrawn_symbols:
+            return SymbolStatus.WITHDRAWN
+
+        if symbol in self.ambiguous_symbols:
+            return SymbolStatus.AMBIGUOUS
+
+        if symbol in self.symbol_map:
+            return SymbolStatus.UPGRADABLE
+
+        return SymbolStatus.UNKNOWN
+
+    def upgrade_symbol(self, symbol):
+        """Return the approved symbol for the given symbol.
+
+        If the symbol cannot be upgraded, just return the original symbol.
+        """
+
+        fixed_symbol, stripped_symbol = format_symbol(symbol)
+
+        if fixed_symbol in self.approved_symbols:
+            return fixed_symbol
+        elif fixed_symbol in self.symbol_map:
+            return self.symbol_map[fixed_symbol]
+        elif stripped_symbol in self.approved_symbols:
+            return stripped_symbol
+        elif stripped_symbol in self.symbol_map:
+            return self.symbol_map[stripped_symbol]
+
+        return symbol
+
+    @classmethod
+    def from_hgnc_records(cls, hgnc_dataset_path):
+        """Parse a hgnc database download into a HGNCSymbolChecker object."""
+
+        def all_symbols(record):
+            """Get all the symbols associated with an HGNC record including previous, alias,
+            and approved."""
+            yield format_symbol(record["symbol"])[0]
+            for symbol in alias_and_previous_symbols(record):
+                yield symbol
+
+        def alias_and_previous_symbols(record):
+            """Get alias and previous symbols from an HGNC record."""
+            for field in ("alias_symbol", "prev_symbol"):
+                if record[field] is not np.nan:
+                    for symbol in record[field].split("|"):
+                        yield format_symbol(symbol)[0]
+            # Sometimes something like HGNC:1234 appears in datasets, which we
+            # want to fix as well.
+            yield record["hgnc_id"]
+
+        hgnc_records = pd.read_csv(hgnc_dataset_path, sep="\t", header=0, low_memory=False).to_dict("records")
+
+        # Get all symbols that are currently approved.
+        approved_symbols = set()
+        for record in hgnc_records:
+            if record["status"] == "Approved":
+                approved_symbols.add(format_symbol(record["symbol"])[0])
+
+        # Get all symbols that have been withdrawn
+        withdrawn_symbols = set()
+        for record in hgnc_records:
+            if record["status"] == "Entry Withdrawn":
+                for symbol in all_symbols(record):
+                    withdrawn_symbols.add(symbol)
+
+        # If a symbol is both approved and withdrawn, be optimistic and call it approved
+        logging.warning(
+            f"Some symbols are simulaneously withdrawn and approved\n"
+            f"We will treat them at approved:\n"
+            f"{withdrawn_symbols.intersection(approved_symbols)}"
+        )
+        withdrawn_symbols = withdrawn_symbols.difference(approved_symbols)
+
+        # Now try to map from symbols that are not approved but are an alias or previous symbol for an approved symbol
+        alias_previous_to_approved = {}
+        ambiguous_symbols = set()
+
+        for record in hgnc_records:
+            if record["status"] == "Approved":
+
+                # The approved symbol is what we'll map to
+                approved_symbol = format_symbol(record["symbol"])[0]
+
+                for symbol in alias_and_previous_symbols(record):
+
+                    # If the alias or previous symbol is also an approved symbol,
+                    # we'll just leave it alone
+                    if symbol in approved_symbols:
+                        continue
+
+                    # If the alias or previous symbol maps to a different approved symbol, mark it as ambiguous
+                    if symbol in alias_previous_to_approved and alias_previous_to_approved[symbol] != approved_symbol:
+                        ambiguous_symbols.add(symbol)
+                    else:
+                        alias_previous_to_approved[symbol] = approved_symbol
+
+        # Remove all the ambiguous symbols from the map
+        for ambiguous_symbol in ambiguous_symbols:
+            alias_previous_to_approved.pop(ambiguous_symbol)
+
+        return HGNCSymbolChecker(approved_symbols, withdrawn_symbols, ambiguous_symbols, alias_previous_to_approved)
+
+
+def format_symbol(symbol):
+    """HGNC rules say symbols should all be upper case except for C#orf#. However, case is
+    variable in both alias and previous symbols as well as in the symbols we get in
+    submissions. So, upper case everything except for the one situation where mixed-case
+    is allowed, which are the genes like C2orf157.
+
+    Also, seurat and scanpy append ".1" or "-1" to duplicated gene names, and these altered
+    names persist throughout the life of the object. They won't match against the HGNC database
+    and we want to merge them, so we need to strip off the suffix and try matching again.
+
+    This function takes a symbol and returns the symbol with the fixed case and also with the
+    seurat/scanpy suffix stripped off.
+    """
+
+    match = re.match(r"^(C)(\d+)(orf)(\d+)$", symbol, re.IGNORECASE)
+
+    if match:
+        fixed_case = f"C{match.group(2)}orf{match.group(4)}"
+    else:
+        fixed_case = symbol.upper()
+
+    suffix_stripped = re.sub(r"[\.\-]\d+$", "", fixed_case)
+
+    return fixed_case, suffix_stripped
+
+
+def main():
+    """When called as main, parse a given hgnc download and print out a map from old to new
+    symbol.
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "hgnc_dataset", help="HGNC dataset tsv, available from www.genenames.org/download/statistics-and-files/"
+    )
+    args = parser.parse_args()
+
+    hgnc_symbol_checker = HGNCSymbolChecker.from_hgnc_records(args.hgnc_dataset)
+
+    hgnc_symbol_checker.print_symbol_map()
+
+
+if __name__ == "__main__":
+    main()

backend/czi_hosted/converters/schema/ontology.py

@@ -0,0 +1,86 @@
+"""Methods for working with ontologies and the OLS."""
+from urllib.parse import quote_plus
+
+import requests
+
+OLS_API_ROOT = "http://www.ebi.ac.uk/ols/api"
+
+# Curie means something like CL:0000001
+
+
+def _ontology_name(curie):
+    """Get the name of the ontology from the curie, CL or UBERON for example."""
+    return curie.split(":")[0]
+
+
+def _ontology_value(curie):
+    """Get the id component of the curie, 0000001 from CL:0000001 for example."""
+    return curie.split(":")[1]
+
+
+def _double_encode(url):
+    """Double url encode a url. This is required by the OLS API."""
+    return quote_plus(quote_plus(url))
+
+
+def _iri(curie):
+    """Get the iri from a curie. This is a bit hopeful that they all map to purl.obolibrary.org"""
+    if _ontology_name(curie) == "EFO":
+        return f"http://www.ebi.ac.uk/efo/EFO_{_ontology_value(curie)}"
+    return f"http://purl.obolibrary.org/obo/{_ontology_name(curie)}_{_ontology_value(curie)}"
+
+
+class OntologyLookupError(Exception):
+    """Exception for some problem with looking up ontology information."""
+
+
+def _ontology_info_url(curie):
+    """Get the to make a GET to to get information about an ontology term."""
+
+    # If the curie is empty, just return an empty string. This happens when there is no
+    # valid ontology value.
+    if not curie:
+        return ""
+    else:
+        return f"{OLS_API_ROOT}/ontologies/{_ontology_name(curie)}/terms/{_double_encode(_iri(curie))}"
+
+
+def get_ontology_label(curie):
+    """For a given curie like 'CL:1000413', get the label like 'endothelial cell of artery'"""
+
+    url = _ontology_info_url(curie)
+
+    if not url:
+        return ""
+
+    response = requests.get(url)
+
+    if not response.ok:
+        raise OntologyLookupError(
+            f"Curie {curie} lookup failed, got status code {response.status_code}: {response.text}"
+        )
+    return response.json()["label"]
+
+
+def lookup_candidate_term(label, ontology="cl", method="select"):
+    """Lookup candidate terms for a label. This is useful when there is an existing label in a
+    submitted dataset, and you want to find an appropriate ontology term.
+
+    Args:
+      label: the label to find ontology terms for
+      ontology: the ontology to search in, cl or uberon or efo for example
+      method: select or search. search provides much broader results
+
+    Returns:
+      list of (curie, label) tuples returned by OLS
+    """
+    # using OLS REST API [https://www.ebi.ac.uk/ols/docs/api]
+    url = f"{OLS_API_ROOT}/{method}?q={quote_plus(label)}&ontology={ontology.lower()}"
+    response = requests.get(url)
+
+    if not response.ok:
+        raise OntologyLookupError(
+            f"Label {label} lookup failed, got status code {response.status_code}: {response.text}"
+        )
+
+    return [(r["obo_id"], r["label"]) for r in response.json()["response"]["docs"]]

backend/czi_hosted/converters/schema/remix.py

@@ -0,0 +1,264 @@
+import argparse
+import collections
+import json
+import logging
+import math
+import string
+
+import anndata
+import numpy as np
+import pandas as pd
+import yaml
+
+from . import gene_symbol
+from . import ontology
+from . import validate
+
+REPLACE_SUFFIX = "_original"
+ONTOLOGY_SUFFIX = "_ontology_term_id"
+
+
+def is_curie(value):
+    """Return True iff the value is an OBO-id CURIE like EFO:000001"""
+    return (value.count(":")
+            and all(len(part) > 0 for part in value.split(":"))
+            and all(c in string.digits for c in value.split(":")[1]))
+
+
+def is_ontology_field(field_name):
+    """Return True iff the field_name is an ontology field like tissue_ontology_term_id"""
+    return field_name.endswith(ONTOLOGY_SUFFIX)
+
+
+def get_label_field_name(field_name):
+    """Get the associated label field from an ontology field, assay_ontology_term_id --> assay"""
+    return field_name[: -len(ONTOLOGY_SUFFIX)]
+
+
+def split_suffix(maybe_curie):
+    """Split off the (cell culture) or (organoid) suffix."""
+
+    suffixes = [" (cell culture)", " (organoid)"]
+    for suffix in suffixes:
+        if maybe_curie.endswith(suffix):
+            return maybe_curie[:-len(suffix)], suffix
+    return maybe_curie, ""
+
+
+def get_curie_and_label(maybe_curie):
+    """Given a string that might be a curie, return a (curie, label) pair"""
+
+    maybe_curie, suffix = split_suffix(maybe_curie)
+    if not is_curie(maybe_curie):
+        return ("", maybe_curie + suffix)
+    return (maybe_curie + suffix, ontology.get_ontology_label(maybe_curie) + suffix)
+
+
+def safe_add_field(adata_attr, field_name, field_value):
+    """Add a field and value to an AnnData, but don't clobber an exising value."""
+
+    if (
+        isinstance(field_value, list)
+        and field_value
+        and isinstance(field_value[0], dict)
+    ):
+        field_value = json.dumps(field_value)
+    if field_name in adata_attr:
+        adata_attr[field_name + REPLACE_SUFFIX] = adata_attr[field_name]
+    adata_attr[field_name] = field_value
+
+
+def remix_uns(adata, uns_config):
+    """Add fields from the config to adata.uns"""
+    for field_name, field_value in uns_config.items():
+
+        if is_ontology_field(field_name):
+            # If it's an ontology field, look it up
+            label_field_name = get_label_field_name(field_name)
+            ontology_term, ontology_label = get_curie_and_label(field_value)
+            safe_add_field(adata.uns, field_name, ontology_term)
+            safe_add_field(adata.uns, label_field_name, ontology_label)
+        else:
+            safe_add_field(adata.uns, field_name, field_value)
+
+
+def remix_obs(adata, obs_config):
+    """Add fields from the config to adata.obs"""
+
+    for field_name, field_value in obs_config.items():
+
+        if isinstance(field_value, dict):
+            # If the value is a dict, that means we are supposed to map from an
+            # existing column to the new one
+            source_column, column_map = next(iter(field_value.items()))
+            nan_value = None
+            for key in column_map:
+                if isinstance(key, float) and math.isnan(key):
+                    nan_value = column_map[key]
+            if nan_value is not None:
+                column_map["nan"] = nan_value
+
+            for key in column_map:
+                if key not in adata.obs[source_column].unique():
+                    logging.warning(f'Key {key} not in adata.obs["{source_column}"]')
+
+            for value in adata.obs[source_column].unique():
+                if value not in column_map:
+                    logging.warning(f'Value {value} in adata.obs["{source_column}"] not in translation dict')
+
+            if is_ontology_field(field_name):
+                ontology_term_map, ontology_label_map = {}, {}
+                logging.info(f"Looking up labels for {field_name}")
+                for original_value, maybe_curie in column_map.items():
+                    curie, label = get_curie_and_label(maybe_curie)
+                    ontology_term_map[original_value] = curie
+                    ontology_label_map[original_value] = label
+                    logging.info(f"Mapping {original_value} -> {curie} -> {label}")
+
+                ontology_column = adata.obs[source_column].replace(
+                    ontology_term_map, inplace=False
+                )
+                label_column = adata.obs[source_column].replace(
+                    ontology_label_map, inplace=False
+                )
+
+                safe_add_field(adata.obs, field_name, ontology_column)
+                safe_add_field(
+                    adata.obs, get_label_field_name(field_name), label_column
+                )
+            else:
+                label_column = adata.obs[source_column].replace(
+                    column_map, inplace=False
+                )
+                safe_add_field(adata.obs, field_name, label_column)
+
+        else:
+            if is_ontology_field(field_name):
+                # If it's an ontology field, look it up
+                label_field_name = get_label_field_name(field_name)
+                ontology_term, ontology_label = get_curie_and_label(field_value)
+                safe_add_field(adata.obs, field_name, ontology_term)
+                safe_add_field(adata.obs, label_field_name, ontology_label)
+            else:
+                safe_add_field(adata.obs, field_name, field_value)
+
+
+def merge_df(df, domain, index, columns):
+    """
+    Given a dataframe with duplicate column labels, merge and return a dataframe where
+    the duplicates have been merged together, resulting in a dataframe with unique column
+    labels.
+
+    "merge" depends on the value of domain. If the domain is "raw", then duplicate columns
+    can just be summed. If it's "log1p" or "sqrt", it needs to be exp1m'd or squared, then
+    summed, and then logged or sqrt'd again.
+    """
+
+    if not isinstance(df, np.ndarray):
+        to_merge = df.toarray()
+    else:
+        to_merge = df
+    if domain == "raw":
+        merged_df = pd.DataFrame(to_merge, index=index, columns=columns).sum(
+            axis=1, level=0, skipna=False
+        )
+    elif domain == "log1p":
+        merged_df = (
+            pd.DataFrame(np.expm1(to_merge, dtype=np.float128), index=index, columns=columns)
+            .sum(axis=1, level=0, skipna=False)
+        )
+        merged_df = pd.DataFrame(np.log1p(merged_df.to_numpy()), index=merged_df.index, columns=merged_df.columns)
+    elif domain == "sqrt":
+        merged_df = (
+            pd.DataFrame(np.square(to_merge), index=index, columns=columns)
+            .sum(axis=1, level=0, skipna=False)
+        )
+        merged_df = pd.DataFrame(np.sqrt(merged_df.to_numpy()), index=merged_df.index, columns=merged_df.columns)
+
+    return merged_df
+
+
+def fixup_gene_symbols(adata, fixup_config):
+    """Update the var index to hold a consistent set of HGNC gene symbols."""
+
+    upgraded_var_index = gene_symbol.get_upgraded_var_index(adata.var)
+
+    merged_X = merge_df(adata.X, fixup_config["X"], adata.obs.index, upgraded_var_index)
+    fixup_adata = anndata.AnnData(
+        X=merged_X,
+        obs=adata.obs,
+        var=merged_X.columns.to_frame(name="hgnc_gene_symbol"),
+        uns=adata.uns,
+        obsm=adata.obsm,
+    )
+
+    for layer, domain in fixup_config.items():
+        if layer == "X":
+            continue
+        if layer == "raw.X":
+            df = adata.raw.X
+        else:
+            df = adata.layers[layer]
+
+        merged_df = merge_df(df, domain, adata.obs.index, upgraded_var_index)
+        assert merged_df.index.equals(merged_X.index)
+        assert merged_df.columns.equals(merged_X.columns)
+
+        if domain == "raw":
+            fixup_raw = anndata.AnnData(
+                X=merged_df,
+                obs=adata.obs,
+                var=merged_X.columns.to_frame(name="hgnc_gene_symbol"),
+            )
+            fixup_adata.raw = fixup_raw
+        else:
+            fixup_adata.layers[layer] = merged_df
+
+    return fixup_adata
+
+def _strip_version(adata):
+    """Remove version information from the AnnData object."""
+
+    if "version" in adata.uns_keys():
+        del adata.uns["version"]
+
+def apply_schema(source_h5ad, remix_config, output_filename):
+
+    try:
+        import scanpy
+    except ImportError:
+        raise ImportError("scanpy must be installed for cellxgene schema")
+    adata = scanpy.read_h5ad(source_h5ad)
+    config = yaml.load(open(remix_config), Loader=yaml.FullLoader)
+    remix_uns(adata, config["uns"])
+    remix_obs(adata, config["obs"])
+
+    if config.get("fixup_gene_symbols"):
+        adata = fixup_gene_symbols(adata, config["fixup_gene_symbols"])
+
+    if ("version" in adata.uns_keys()
+            and isinstance(adata.uns["version"], collections.Mapping)
+            and "corpora_schema_version" in adata.uns["version"]):
+        schema_version = adata.uns["version"]["corpora_schema_version"]
+        try:
+            validate.get_schema_definition(schema_version)
+        except ValueError:
+            logging.warning(f"Stripping version information out of AnnData because schema "
+                            f"version {schema_version} is unknown.")
+            _strip_version(adata)
+
+        if not validate.validate_adata(adata, shallow=False):
+            logging.warning(f"Stripping version information out of AnnData because it does not "
+                            f"follow schema version {schema_version} .")
+            _strip_version(adata)
+
+    adata.write_h5ad(output_filename, compression="gzip")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--source-h5ad", required=True)
+    parser.add_argument("--remix-config", required=True)
+    parser.add_argument("--output-filename", required=True)
+    args = parser.parse_args()
+    apply_schema(args.source_h5ad, args.remix_config, args.output_filename)

backend/czi_hosted/converters/schema/schema_definitions/1_0_0.yaml

@@ -0,0 +1,95 @@
+title: Corpora schema version 1.0.0
+type: anndata
+components:
+    uns:
+        type: dict
+        keys:
+            version:
+                type: dict
+                keys:
+                    corpora_schema_version: null
+                    corpora_encoding_version: null
+            title:
+                type: string
+            contributors:
+                type: stringified list of dicts
+            layer_descriptions:
+                type: dict
+                keys:
+                    X: null
+            organism:
+                type: string
+                nullable: false
+            organism_ontology_term_id:
+                type: curie
+                prefixes:
+                    - NCBITaxon
+    var:
+        type: dataframe
+        index:
+            type: human-readable string
+            unique: true
+    obs:
+        type: dataframe
+        index:
+            unique: true
+        columns:
+            tissue:
+                type: human-readable string
+                nullable: false
+            tissue_ontology_term_id:
+                type: suffixed curie
+                nullable: true
+                prefixes:
+                    - UBERON
+            assay:
+                type: human-readable string
+                nullable: false
+            assay_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - EFO
+            disease:
+                type: human-readable string
+                nullable: false
+            disease_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - MONDO
+                    - PATO
+            cell_type:
+                type: human-readable string
+                nullable: false
+            cell_type_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - CL
+                    - UBERON
+            sex:
+                type: string
+                enum:
+                    - male
+                    - female
+                    - mixed
+                    - unknown
+                    - other
+            ethnicity:
+                type: human-readable string
+                nullable: false
+            ethnicity_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - HANCESTRO
+            development_stage:
+                type: human-readable string
+                nullable: false
+            development_stage_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - HsapDv
+                    - EFO

backend/czi_hosted/converters/schema/schema_definitions/1_1_0.yaml

@@ -0,0 +1,93 @@
+title: Corpora schema version 1.1.0
+type: anndata
+components:
+    uns:
+        type: dict
+        keys:
+            version:
+                type: dict
+                keys:
+                    corpora_schema_version: null
+                    corpora_encoding_version: null
+            title:
+                type: string
+            layer_descriptions:
+                type: dict
+                keys:
+                    X: null
+            organism:
+                type: string
+                nullable: false
+            organism_ontology_term_id:
+                type: curie
+                prefixes:
+                    - NCBITaxon
+    var:
+        type: dataframe
+        index:
+            type: human-readable string
+            unique: true
+    obs:
+        type: dataframe
+        index:
+            unique: true
+        columns:
+            tissue:
+                type: human-readable string
+                nullable: false
+            tissue_ontology_term_id:
+                type: suffixed curie
+                nullable: true
+                prefixes:
+                    - UBERON
+            assay:
+                type: human-readable string
+                nullable: false
+            assay_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - EFO
+            disease:
+                type: human-readable string
+                nullable: false
+            disease_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - MONDO
+                    - PATO
+            cell_type:
+                type: human-readable string
+                nullable: false
+            cell_type_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - CL
+                    - UBERON
+            sex:
+                type: string
+                enum:
+                    - male
+                    - female
+                    - mixed
+                    - unknown
+                    - other
+            ethnicity:
+                type: human-readable string
+                nullable: false
+            ethnicity_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - HANCESTRO
+            development_stage:
+                type: human-readable string
+                nullable: false
+            development_stage_ontology_term_id:
+                type: curie
+                nullable: true
+                prefixes:
+                    - HsapDv
+                    - EFO

backend/czi_hosted/converters/schema/validate.py

@@ -0,0 +1,236 @@
+import json
+import re
+import os
+import sys
+
+import pandas as pd
+import yaml
+
+
+def _is_null(v):
+    """Return True if v is null, for one of the multiple ways a "null" value shows up in an h5ad."""
+    return pd.isnull(v) or (hasattr(v, "__len__") and len(v) == 0)
+
+
+def _validate_stringified_list_of_dicts(s):
+    """Verify that a string can be parsed into a list.
+
+    We have some types that are lists of dicts. Those cannot be stored directly in an h5ad, so we have to
+    json.dumps them. This verifies that we can load them back.
+    """
+
+    try:
+        list_ = json.loads(s)
+        if not isinstance(list_, list):
+            return False
+        for el in list_:
+            if not isinstance(el, dict):
+                return False
+        return True
+    except (json.JSONDecodeError, TypeError):
+        pass
+    return False
+
+
+def _validate_human_readable_string(s):
+    """Verify that a string is human-readable.
+
+    There are parts of the schema where a "human-readable" string is required. "Human-readable" is kind
+    of vague and subjective. I feel like I can read many strings. So here we just check for the main ways
+    that fails: someone puts in an ontology term id or and ensembl gene/transcript id.
+
+    Returns False if s is not a string or is one of those bad string types.
+    """
+
+    return isinstance(s, str) and (not re.match(r"[A-Z]\w+:\d+", s)) and (not re.match(r"ENS[GT]\d+$", s))
+
+
+def _validate_curie(c, prefixes):
+    """Verify that a string is a valid compact URI, like EFO:000001. If prefixes is not empty, make sure the
+    prefix of the curies is in prefixes.
+    """
+
+    if not c:
+        return True
+
+    match = re.match(r"([A-Z]\w+):\d+$", c)
+
+    if prefixes:
+        return match and match.group(1) in prefixes
+    else:
+        return match
+
+
+def _validate_suffixed_curie(c, prefixes):
+    """Verify that a string is a compact URI with an optional suffix like 'EFO:00001 (cell culture)'"""
+
+    # Pull off the suffix
+    suffix = re.findall(r"\ \(.*\)$", c)
+    if suffix:
+        c = c[: -len(suffix[0])]
+    return _validate_curie(c, prefixes)
+
+
+def _validate_column(column, column_name, df_name, schema_def):
+    """Given a schema definition and the column of a dataframe, verify that the column satifies
+    the schema.
+    """
+
+    errors = []
+
+    if schema_def.get("unique"):
+        if column.nunique() != len(column):
+            errors.append(f"Column {column_name} in dataframe {df_name} is not unique.")
+
+    if "nullable" in schema_def and not schema_def["nullable"]:
+        if any(_is_null(v) for v in column):
+            errors.append(f"Column {column_name} in dataframe {df_name} contains empty values.")
+
+    if schema_def.get("type") == "human-readable string":
+        non_readables = [v for v in column if not _validate_human_readable_string(v)]
+        if non_readables:
+            errors.append(
+                f"Column {column_name} in dataframe {df_name} contains non-human-readable "
+                f"values like {non_readables[0]}"
+            )
+
+    if schema_def.get("type") in ("curie", "suffixed curie"):
+        validation_func = _validate_curie if schema_def.get("type") == "curie" else _validate_suffixed_curie
+        non_valid_curies = [v for v in column if not validation_func(v, schema_def.get("prefixes"))]
+        if non_valid_curies:
+            errors.append(
+                f"Column {column_name} in dataframe {df_name} contains invalid ontology values like "
+                f"{non_valid_curies[0]}."
+            )
+            if "prefixes" in schema_def:
+                errors[-1] += f" Values must be curies from one of these ontologies {schema_def['prefixes']}."
+
+    if "enum" in schema_def:
+        bad_enums = [v for v in column if v not in schema_def["enum"]]
+        if bad_enums:
+            errors.append(
+                f"Column {column_name} in dataframe {df_name} contains unpermitted values like "
+                f"{bad_enums[0]}. Values must be one of {schema_def['enum']}."
+            )
+
+    return errors
+
+
+def _validate_dict(dict_, dict_name, schema_def):
+    """Given a schema definition and dict, verify that the dict satifies the schema."""
+
+    errors = []
+
+    for key in schema_def.get("keys", []):
+        if key not in dict_:
+            errors.append(f"{dict_name} is missing key {key}.")
+        elif schema_def["keys"][key]:
+            if schema_def["keys"][key]["type"] == "stringified list of dicts":
+                if not _validate_stringified_list_of_dicts(dict_[key]):
+                    errors.append(
+                        f"Key {key} in {dict_name} should be a JSON-encoded list of dicts, but it is {dict_[key]}"
+                    )
+            elif schema_def["keys"][key]["type"] == "dict":
+                errors.extend(_validate_dict(dict_[key], key, schema_def["keys"][key]))
+            elif schema_def["keys"][key]["type"] == "curie":
+                if not _validate_curie(dict_[key], schema_def["keys"][key]["prefixes"]):
+                    errors.append(f"Key {key} in {dict_name} contains invalid ontology value.")
+            if "nullable" in schema_def["keys"][key] and not schema_def["keys"][key]["nullable"]:
+                if _is_null(dict_[key]):
+                    errors.append(f"Key {key} in dict {dict_name} is an empty value.")
+
+    return errors
+
+
+def _validate_dataframe(df, df_name, schema_def):
+    """Given a dataframe and schema definition, verify that the dataframe follows the schema."""
+
+    errors = []
+
+    if "index" in schema_def:
+        errors.extend(_validate_column(df.index, "index", df_name, schema_def["index"]))
+
+    for column in schema_def.get("columns", []):
+        if column not in df.columns:
+            errors.append(f"Dataframe {df_name} is missing column {column}.")
+        else:
+            errors.extend(_validate_column(df[column], column, df_name, schema_def["columns"][column]))
+
+    return errors
+
+
+def get_schema_definition(version):
+    """Look up and read a schema definition based on a version number like "1.0.0"."""
+
+    path = os.path.join(
+        os.path.dirname(os.path.realpath(__file__)), "schema_definitions", version.replace(".", "_") + ".yaml"
+    )
+
+    if not os.path.isfile(path):
+        raise ValueError(f"No definition for version {version} found.")
+
+    return yaml.load(open(path), Loader=yaml.FullLoader)
+
+
+def deep_check(adata, schema_def):
+    """Perform a "deep" check of the AnnData object using the schema definition.
+
+    This checks all the columns and unstructured metadata rather than just the version.
+
+    Returns a list of error messages. If that list is empty, the object passed validation.
+    """
+
+    errors = []
+
+    for component, component_def in schema_def["components"].items():
+        if component_def["type"] == "dataframe":
+            errors.extend(_validate_dataframe(getattr(adata, component), component, component_def))
+        elif component_def["type"] == "dict":
+            errors.extend(_validate_dict(getattr(adata, component), component, component_def))
+        else:
+            raise ValueError(f"Unexpected component type {component['type']}")
+
+    return errors
+
+
+def validate_adata(adata, shallow):
+    """Validate an AnnData object. If shallow, just check that the required version information is
+    present.
+    """
+
+    # Does it have the version information written into uns?
+    if "version" not in adata.uns_keys() or "corpora_schema_version" not in adata.uns["version"]:
+        print("AnnData file is missing corpora version information")
+        return False
+
+    # We can stop here if it's a "shallow" check, that is, if we're just
+    # checking that version is present.
+    if shallow:
+        return True
+
+    schema_def = get_schema_definition(adata.uns["version"]["corpora_schema_version"])
+
+    errors = deep_check(adata, schema_def)
+
+    for error in errors:
+        print(error)
+
+    return not errors
+
+
+def validate(h5ad_path, shallow=False):
+    """Entry point for validation."""
+
+    try:
+        import scanpy
+    except ImportError:
+        raise ImportError("scanpy must be installed for cellxgene schema")
+
+    try:
+        adata = scanpy.read_h5ad(h5ad_path, backed="r")
+    except (OSError, TypeError):
+        print(f"Unable to open {h5ad_path} with scanpy.")
+        sys.exit(1)
+
+    if not validate_adata(adata, shallow):
+        sys.exit(1)

backend/czi_hosted/converters/to_sparse.py

@@ -9,8 +9,9 @@ import sys
 
 import tiledb
 
-from server.common.utils.cxg_generation_utils import convert_ndarray_to_cxg_dense_array, convert_matrix_to_cxg_array
-from server.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix
+from backend.czi_hosted.common.utils.cxg_generation_utils import convert_ndarray_to_cxg_dense_array, \
+    convert_matrix_to_cxg_array
+from backend.czi_hosted.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix
 
 
 def main():

backend/czi_hosted/data_anndata/anndata_adaptor.py

@@ -8,15 +8,15 @@ from pandas.core.dtypes.dtypes import CategoricalDtype
 from scipy import sparse
 from server_timing import Timing as ServerTiming
 
-import server.compute.diffexp_generic as diffexp_generic
-from server.common.colors import convert_anndata_category_colors_to_cxg_category_colors
-from server.common.constants import Axis, MAX_LAYOUTS
-from server.common.corpora import corpora_get_props_from_anndata
-from server.common.errors import PrepareError, DatasetAccessError, FilterError
-from server.common.utils.type_conversion_utils import get_schema_type_hint_of_array
-from server.compute.scanpy import scanpy_umap
-from server.data_common.data_adaptor import DataAdaptor
-from server.data_common.fbs.matrix import encode_matrix_fbs
+import backend.common.compute.diffexp_generic as diffexp_generic
+from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
+from backend.common.constants import Axis, MAX_LAYOUTS
+from backend.czi_hosted.common.corpora import corpora_get_props_from_anndata
+from backend.common.errors import PrepareError, DatasetAccessError, FilterError
+from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
+from backend.czi_hosted.compute.scanpy import scanpy_umap
+from backend.czi_hosted.data_common.data_adaptor import DataAdaptor
+from backend.common.fbs.matrix import encode_matrix_fbs
 
 anndata_version = version.parse(str(anndata.__version__)).release
 
@@ -177,10 +177,10 @@ class AnndataAdaptor(DataAdaptor):
             )
 
     def _validate_and_initialize(self):
-        if anndata_version_is_pre_070() and self.server_config.adaptor__anndata_adaptor__backed:
+        if anndata_version_is_pre_070():
             warnings.warn(
-                "Use of --backed mode with anndata versions older than 0.7 will have serious "
-                "performance issues. Please update to at least anndata 0.7 or later."
+                "Use of anndata versions older than 0.7 will have serious issues. Please update to at "
+                "least anndata 0.7 or later."
             )
 
         # var and obs column names must be unique

backend/czi_hosted/data_common/data_adaptor.py

@@ -3,13 +3,14 @@ from os.path import basename, splitext
 
 import numpy as np
 import pandas as pd
+from scipy import sparse
 from server_timing import Timing as ServerTiming
 
-from server.common.config.app_config import AppConfig
-from server.common.constants import Axis
-from server.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError
-from server.common.utils.utils import jsonify_numpy
-from server.data_common.fbs.matrix import encode_matrix_fbs
+from backend.czi_hosted.common.config.app_config import AppConfig
+from backend.common.constants import Axis
+from backend.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError, UnsupportedSummaryMethod, DatasetAccessError
+from backend.common.utils.utils import jsonify_numpy
+from backend.common.fbs.matrix import encode_matrix_fbs
 
 
 class DataAdaptor(metaclass=ABCMeta):
@@ -162,7 +163,7 @@ class DataAdaptor(metaclass=ABCMeta):
         mask = np.zeros((count,), dtype=np.bool)
         for i in filter:
             if type(i) == list:
-                mask[i[0] : i[1]] = True
+                mask[i[0]: i[1]] = True
             else:
                 mask[i] = True
         return mask
@@ -249,6 +250,23 @@ class DataAdaptor(metaclass=ABCMeta):
         if labels_df.shape[0] != shape[0]:
             raise ValueError("Labels file must have same number of rows as data file.")
 
+        # This will convert a float column that contains integer data into an integer type.
+        # This case can occur when a user makes a copy of a category that originally contained integer data.
+        # The client always copies array data to floats, therefore the copy will contain floats instead of integers.
+        # float data is not allowed as a categorical type.
+        if any([np.issubdtype(coltype.type, np.floating) for coltype in labels_df.dtypes]):
+            labels_df = labels_df.convert_dtypes()
+            for col, dtype in zip(labels_df, labels_df.dtypes):
+                if isinstance(dtype, pd.Int32Dtype):
+                    labels_df[col] = labels_df[col].astype("int32")
+                if isinstance(dtype, pd.Int64Dtype):
+                    labels_df[col] = labels_df[col].astype("int64")
+
+        if any([np.issubdtype(coltype.type, np.floating) for coltype in labels_df.dtypes]):
+            raise ValueError("Columns may not have floating point types")
+
+        return labels_df
+
     def data_frame_to_fbs_matrix(self, filter, axis):
         """
         Retrieves data 'X' and returns in a flatbuffer Matrix.
@@ -265,7 +283,7 @@ class DataAdaptor(metaclass=ABCMeta):
 
         try:
             obs_selector, var_selector = self._filter_to_mask(filter)
-        except (KeyError, IndexError, TypeError, AttributeError):
+        except (KeyError, IndexError, TypeError, AttributeError, DatasetAccessError):
             raise FilterError("Error parsing filter")
 
         if obs_selector is not None:
@@ -303,11 +321,12 @@ class DataAdaptor(metaclass=ABCMeta):
             top_n = self.dataset_config.diffexp__top_n
 
         if self.server_config.exceeds_limit(
-            "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B)
+                "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B)
         ):
             raise ExceedsLimitError("Diffexp request exceeds max cell count limit")
 
-        result = self.compute_diffexp_ttest(obs_mask_A, obs_mask_B, top_n, self.dataset_config.diffexp__lfc_cutoff)
+        result = self.compute_diffexp_ttest(
+            maskA=obs_mask_A, maskB=obs_mask_B, top_n=top_n, lfc_cutoff=self.dataset_config.diffexp__lfc_cutoff)
 
         try:
             return jsonify_numpy(result)
@@ -321,7 +340,7 @@ class DataAdaptor(metaclass=ABCMeta):
     @staticmethod
     def normalize_embedding(embedding):
         """Normalize embedding layout to meet client assumptions.
-           Embedding is an ndarray, shape (n_obs, n)., where n is normally 2
+        Embedding is an ndarray, shape (n_obs, n)., where n is normally 2
         """
 
         # scale isotropically
@@ -377,3 +396,26 @@ class DataAdaptor(metaclass=ABCMeta):
         except RuntimeError:
             lastmod = None
         return lastmod
+
+    def summarize_var(self, method, filter, query_hash):
+        if method != "mean":
+            raise UnsupportedSummaryMethod("Unknown gene set summary method.")
+
+        obs_selector, var_selector = self._filter_to_mask(filter)
+        if obs_selector is not None:
+            raise FilterError("filtering on obs unsupported")
+
+        # if no filter, just return zeros.  We don't have a use case
+        # for summarizing the entire X without a filter, and it would
+        # potentially be quite compute / memory intensive.
+        if var_selector is None or np.count_nonzero(var_selector) == 0:
+            mean = np.zeros((self.get_shape()[0], 1), dtype=np.float32)
+        else:
+            X = self.get_X_array(obs_selector, var_selector)
+            if sparse.issparse(X):
+                mean = X.mean(axis=1).A
+            else:
+                mean = X.mean(axis=1, keepdims=True)
+
+        col_idx = pd.Index([query_hash])
+        return encode_matrix_fbs(mean, col_idx=col_idx, row_idx=None)

backend/czi_hosted/data_common/matrix_loader.py

@@ -1,12 +1,14 @@
 from enum import Enum
 import threading
 import time
-from server.data_common.rwlock import RWLock
-from server.common.errors import DatasetAccessError
-from server.common.data_locator import DataLocator
+
+from backend.common.utils.data_locator import DataLocator
+from backend.common.errors import DatasetAccessError
 from contextlib import contextmanager
 from http import HTTPStatus
 
+from backend.czi_hosted.data_common.rwlock import RWLock
+
 
 class MatrixDataCacheItem(object):
     """This class provides access and caching for a dataset.  The first time a dataset is accessed, it is
@@ -224,7 +226,7 @@ class MatrixDataLoader(object):
 
         # matrix_data_type is an enum value of type MatrixDataType
         self.matrix_data_type = matrix_data_type
-        # matrix_type is a DataAdaptor type, which corresonds to the matrix_data_type
+        # matrix_type is a DataAdaptor type, which corresponds to the matrix_data_type
         self.matrix_type = None
 
         if matrix_data_type is None:
@@ -234,11 +236,11 @@ class MatrixDataLoader(object):
             raise DatasetAccessError("Dataset does not have an allowed type.")
 
         if self.matrix_data_type == MatrixDataType.H5AD:
-            from server.data_anndata.anndata_adaptor import AnndataAdaptor
+            from backend.czi_hosted.data_anndata.anndata_adaptor import AnndataAdaptor
 
             self.matrix_type = AnndataAdaptor
         elif self.matrix_data_type == MatrixDataType.CXG:
-            from server.data_cxg.cxg_adaptor import CxgAdaptor
+            from backend.czi_hosted.data_cxg.cxg_adaptor import CxgAdaptor
 
             self.matrix_type = CxgAdaptor