docs: release 0.18.0 (#2413)

* Bump version: 0.17.0 → 0.18.0-rc.0

* Bump version: 0.18.0-rc.0 → 0.18.0

.bumpversion.cfg

@@ -1,17 +1,14 @@
 [bumpversion]
-current_version = 0.16.0
+current_version = 0.18.0
 commit = True
-# The below regex details an acceptable version number by naming the groups (major, minor, patch, prerel, and
-# prerelversion) and also specifying the valid values for each group (integers, `\d+`,  for major, minor, patch, and
-# prerelversion and only `rc` as the acceptable value for prerel).
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?:-(?P<prerel>rc)\.(?P<prerelversion>\d+))?
-serialize =
+serialize = 
 	{major}.{minor}.{patch}-{prerel}.{prerelversion}
 	{major}.{minor}.{patch}
 
 [bumpversion:part:prerel]
 optional_value = release
-values =
+values = 
 	rc
 	release
 
@@ -23,6 +20,6 @@ replace = version="{new_version}"
 search = "version": "{current_version}"
 replace = "version": "{new_version}"
 
-[bumpversion:file:server/__init__.py]
+[bumpversion:file:backend/server/__init__.py]
 search = __version__ = "{current_version}"
 replace = __version__ = "{new_version}"

.dockerignore

@@ -2,4 +2,4 @@ bin
 client
 dist
 docs
-server
+backend

.github/workflows/canary-deploy.yml

@@ -1,8 +1,8 @@
-name: Deploy via single cell infra repo
+name: Deploy canary via single cell infra repo
 
 on:
   push:
-    branches: main
+    branches: main-canary
 
 jobs:
   deploy:
@@ -10,4 +10,4 @@ jobs:
     steps:
       - name: repository dispatch
         run: |
-          curl -XPOST -u czi-sci-single-cell-eng:${{secrets.SCI_GITHUB_TOKEN}} -H "Accept: application/vnd.github.everest-preview+json"  -H "Content-Type: application/json" https://api.github.com/repos/chanzuckerberg/single-cell-infra/dispatches --data '{"event_type": "cellxgene-hook"}'
+          curl -XPOST -u czi-sci-single-cell-eng:${{secrets.SCI_GITHUB_TOKEN}} -H "Accept: application/vnd.github.everest-preview+json"  -H "Content-Type: application/json" https://api.github.com/repos/chanzuckerberg/single-cell-infra/dispatches --data '{"event_type": "canary-hook"}'

.github/workflows/compatibility_tests.yml

@@ -9,6 +9,7 @@ on:
 
 env:
   JEST_ENV: prod
+  CXG_AUTH_TYPE: none
 
 jobs:
   docker-build:
@@ -28,8 +29,8 @@ jobs:
     continue-on-error: true
     strategy:
       matrix:
-        python-version: [3.6, 3.7] # As of Oct 2020 Anndata is not compatible with 3.8
+        python-version: [3.6, 3.7, 3.8]
-        anndata-version: [0.7.0, 0.7.1, 0.7.2, 0.7.3, 0.7.4, 0.7.5]
+        anndata-version: [0.7.6]
         test-suite: [smoke-test, smoke-test-annotations]
     steps:
     - uses: actions/checkout@v2
@@ -41,14 +42,12 @@ jobs:
       run: |
         # 1. only install the dev requirements on top of what is in the cellxgene pip package
         sudo apt-get update && sudo apt-get install -y libhdf5-serial-dev
-        sed -i 's/-r requirements.txt//' server/requirements-dev.txt
+        sed -i 's/-r requirements.txt//' backend/server/requirements-dev.txt
-        pip install -r server/requirements-dev.txt
+        pip install -r backend/server/requirements-dev.txt
         # 2. install cellxgene
         make pydist install-dist
         # 3. install anndata
         pip install anndata==${{ matrix.anndata-version }}
-        # workaround for anndata 0.6.22.post1 bug
-        [[ "0.6.22.post1" = "${{ matrix.anndata-version }}" ]] && pip install h5py==2.9.0 || true
     - name: Tests
       run: make unit-test ${{ matrix.test-suite }}
 
@@ -73,8 +72,8 @@ jobs:
           cd cellxgene
           # 1. only install the dev requirements on top of what is in the cellxgene pip package
           make dev-env-client
-          sed -i 's/-r requirements.txt//' server/requirements-dev.txt
+          sed -i 's/-r requirements.txt//' backend/server/requirements-dev.txt
-          pip install -r server/requirements-dev.txt
+          pip install -r backend/server/requirements-dev.txt
           # 2. install cellxgene
           pip install --upgrade cellxgene
           # 3. install anndata
@@ -101,7 +100,7 @@ jobs:
       - name: Install dependencies
         run: |
           cd cellxgene
-          sed -i -E 's/^anndata[>=]=[0-9]+.[0-9]+.[0-9]+$/anndata/g' server/requirements.txt
+          sed -i -E 's/^anndata[>=]=[0-9]+.[0-9]+.[0-9]+$/anndata/g' backend/server/requirements.txt
           make pydist install-dist dev-env
           pip install git+https://github.com/theislab/anndata
       - name: Tests

.github/workflows/push_tests.yml

@@ -36,11 +36,11 @@ jobs:
           npm install
       - name: Format with black and lint with flake8
         run: |
-          make lint-server
+          make lint-servers
       - name: Lint src with eslint
         working-directory: ./client
         run: |
-          make lint
+          npx eslint src __tests__
 
   unit-test:
     runs-on: ubuntu-latest
@@ -68,8 +68,38 @@ jobs:
         run: make pydist install-dist dev-env-server
       - name: Unit tests
         run: |
-          make unit-test
+          make unit-test-server
-          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k server -cF backend,python,unitTest
+          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k backend/server -cF backend,python,unitTest
+          cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest
+
+  unit-test-czi-hosted:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Python cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Node cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Install dependencies
+        run: make pydist-czi-hosted install-dist dev-env-czi-hosted
+      - name: Unit tests
+        run: |
+          make unit-test-czi-hosted
+          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k backend/czi-hosted -cF backend,python,unitTest
           cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest
 
   smoke-tests:
@@ -96,7 +126,7 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-node-
       - name: Install dependencies
-        run: make pydist install-dist
+        run: make pydist-czi-hosted install-dist
       - name: Smoke tests (without annotations feature)
         run: |
           cd client && make smoke-test

.github/workflows/sastisfaction.yml

@@ -0,0 +1,27 @@
+name: Run SASTisfaction
+on:
+  - pull_request
+
+jobs:
+  sastisfaction:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/checkout@v2
+        with:
+          repository: chanzuckerberg/sastisfaction
+          ref: main
+          path: .github/actions/sastisfaction
+          ssh-key: ${{ secrets.SASTISFACTION_READ_KEY }}
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Docker pull
+        run: docker pull ghcr.io/chanzuckerberg/sastisfaction:main
+      - name: Run SASTisfaction
+        uses: ./.github/actions/sastisfaction
+        with:
+          snowflake_private_key: ${{ secrets.SASTISFACTION_RSA_KEY }}

.github/workflows/scale-test.yml

@@ -15,15 +15,15 @@ jobs:
           python-version: 3.7
       - name: Install dependencies
         run: |
-          pip install -r server/test/locust/requirements-locust.txt
+          pip install -r backend/test/test_czi_hosted/locust/requirements-locust.txt
       - name: Dev Scale Test
         run: |
-          locust -f server/test/locust/locustfile.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt
+          locust -f backend/test/test_czi_hosted/locust/locustfile.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt
       - name: Slack success webhook
         env:
           SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
         run: |
-          DEV_STATS=$(tail -n 61 locust_dev_stats.txt)
+          DEV_STATS=$(tail -n 15 locust_dev_stats.txt)
           DEV_MSG="\`\`\`CELLXGENE EXPLORER DEV SCALE TEST RESULTS: ${DEV_STATS}\`\`\`"
           curl -X POST -H 'Content-type: application/json' --data "{'text':'${DEV_MSG}'}" $SLACK_WEBHOOK
 

.gitignore

@@ -19,9 +19,13 @@ venv/
 cellxgene/
 
 # client build
-server/common/web/static/*
+backend/server/common/web/static/*
-server/common/web/templates/
+backend/server/common/web/templates/
-server/common/web/csp-hashes.json
+backend/server/common/web/csp-hashes.json
+
+backend/czi_hosted/common/web/static/*
+backend/czi_hosted/common/web/templates/
+backend/czi_hosted/common/web/csp-hashes.json
 
 # eb build
 artifact.dir

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2017-2020 Chan Zuckerberg Initiative
+Copyright (c) 2017-2021 Chan Zuckerberg Initiative
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

MANIFEST.in

@@ -1,7 +1,7 @@
-recursive-include server/common/web/templates *
+recursive-include backend/server/common/web/templates *
-recursive-include server/common/web/static *
+recursive-include backend/server/common/web/static *
 
-include server/requirements.txt
+include backend/server/requirements.txt
-include server/requirements-prepare.txt
+include backend/server/requirements-prepare.txt
-include server/converters/schema/hgnc_complete_set.txt.gz
+include backend/server/converters/schema/hgnc_complete_set.txt.gz
-include server/converters/schema/schema_definitions/*
+include backend/server/converters/schema/schema_definitions/*

MANIFEST_hosted.in

@@ -0,0 +1,7 @@
+recursive-include backend/czi_hosted/common/web/templates *
+recursive-include backend/czi_hosted/common/web/static *
+
+include backend/czi_hosted/requirements.txt
+include backend/czi_hosted/requirements-prepare.txt
+include backend/czi_hosted/converters/schema/hgnc_complete_set.txt.gz
+include backend/czi_hosted/converters/schema/schema_definitions/*

Makefile

@@ -2,23 +2,32 @@ include common.mk
 
 BUILDDIR := build
 CLIENTBUILD := $(BUILDDIR)/client
-SERVERBUILD := $(BUILDDIR)/server
+CZIHOSTEDBUILD := $(BUILDDIR)/backend/czi_hosted
+SERVERBUILD := $(BUILDDIR)/backend/server
 CLEANFILES :=  $(BUILDDIR)/ client/build build dist cellxgene.egg-info
 
 PART ?= patch
 
 # CLEANING
 .PHONY: clean
-clean: clean-lite clean-server clean-client
+clean: clean-lite clean-czi-hosted clean-server clean-client
 
 # cleaning the client's node_modules is the longest one, so we avoid that if possible
 .PHONY: clean-lite
 clean-lite:
 	rm -rf $(CLEANFILES)
 
-clean-%:
+.PHONY: clean-client
-	cd $(*) && $(MAKE) clean
+clean-client:
+	cd client && $(MAKE) clean
 
+.PHONY: clean-server
+clean-server:
+	cd backend/server && $(MAKE) clean
+
+.PHONY: clean-czi-hosted
+clean-czi-hosted:
+	cd backend/czi_hosted && $(MAKE) clean
 
 # BUILDING PACKAGE
 
@@ -28,29 +37,71 @@ build-client:
 
 .PHONY: build
 build: clean build-client
-	git ls-files server/ | grep -v 'server/test/' | cpio -pdm $(BUILDDIR)
+	git ls-files backend/server/ | grep -v 'backend/server/test/' | cpio -pdm $(BUILDDIR)
 	cp -r client/build/  $(CLIENTBUILD)
 	$(call copy_client_assets,$(CLIENTBUILD),$(SERVERBUILD))
+	cp backend/__init__.py $(BUILDDIR)
+	cp backend/__init__.py $(BUILDDIR)/backend
+	cp -r backend/common $(BUILDDIR)/backend/common
 	cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR)
 
+.PHONY: build-czi-hosted
+build-czi-hosted: clean build-client
+	git ls-files backend/czi_hosted/ | grep -v 'backend/czi_hosted/test/' | cpio -pdm $(BUILDDIR)
+	cp -r client/build/  $(CLIENTBUILD)
+	$(call copy_client_assets,$(CLIENTBUILD),$(CZIHOSTEDBUILD))
+	cp -r backend/common $(BUILDDIR)/backend/common
+	cp backend/__init__.py $(BUILDDIR)
+	cp backend/__init__.py $(BUILDDIR)/backend
+	cp MANIFEST_hosted.in README.md setup.cfg setup_hosted.py $(BUILDDIR)
+	mv $(BUILDDIR)/setup_hosted.py $(BUILDDIR)/setup.py
+	mv $(BUILDDIR)/MANIFEST_hosted.in $(BUILDDIR)/MANIFEST.in
+
 # If you are actively developing in the server folder use this, dirties the source tree
 .PHONY: build-for-server-dev
 build-for-server-dev: clean-server build-client
-	$(call copy_client_assets,client/build,server)
+	$(call copy_client_assets,client/build,backend/server)
+
+.PHONY: build-for-czi-hosted-dev
+build-for-czi-hosted-dev: clean-czi-hosted build-client
+	$(call copy_client_assets,client/build,backend/czi_hosted)
 
 .PHONY: copy-client-assets
 copy-client-assets:
-	$(call copy_client_assets,client/build,server)
+	$(call copy_client_assets,client/build,backend/server)
+
+.PHONY: copy-client-assets-czi-hosted
+copy-client-assets-czi-hosted:
+	$(call copy_client_assets,client/build,backend/czi_hosted)
 
 # TESTING
 .PHONY: test
 test: unit-test smoke-test
 
 .PHONY: unit-test
-unit-test: unit-test-server unit-test-client
+unit-test: unit-test-server unit-test-client unit-test-common
 
-unit-test-%:
+.PHONY: test-server
-	cd $(*) && $(MAKE) unit-test
+test-server: unit-test-server smoke-test
+
+.PHONY: test-czi-hosted
+test-czi-hosted: unit-test-czi-hosted smoke-test
+
+.PHONY: unit-test-client
+unit-test-client:
+	cd client && $(MAKE) unit-test
+
+.PHONY: unit-test-czi-hosted
+unit-test-czi-hosted:
+	cd backend/czi_hosted && $(MAKE) unit-test
+
+.PHONY: unit-test-server
+unit-test-server:
+	cd backend/server && $(MAKE) unit-test
+
+.PHONY: unit-test-common
+unit-test-common:
+	cd backend/common && $(MAKE) unit-test
 
 .PHONY: smoke-test
 smoke-test:
@@ -62,12 +113,11 @@ smoke-test-annotations:
 
 .PHONY: test-db
 test-db:
-	cd server && $(MAKE) test-db
+	cd backend/czi_hosted && $(MAKE) test-db
-
 
 # FORMATTING CODE
 
-.PHOHY: fmt
+.PHONY: fmt
 fmt: fmt-client fmt-py
 
 .PHONY: fmt-client
@@ -79,12 +129,18 @@ fmt-py:
 	black .
 
 .PHONY: lint
-lint: lint-server lint-client
+lint: lint-servers lint-client
+
+.PHONY: lint-servers
+lint-servers: lint-server lint-czi-hosted-server
 
 .PHONY: lint-server
 lint-server: fmt-py
-	flake8 server --per-file-ignores='server/test/fixtures/dataset_config_outline.py:F821 server/test/fixtures/server_config_outline.py:F821 server/test/performance/scale_test_annotations.py:E501'
+	flake8 backend/server --per-file-ignores='backend/test/fixtures/dataset_config_outline.py:F821 backend/test/fixtures/server_config_outline.py:F821 backend/server/test/performance/scale_test_annotations.py:E501'
 
+.PHONY: lint-czi-hosted-server
+lint-czi-hosted-server: fmt-py
+	flake8 backend/czi_hosted --per-file-ignores='backend/test/fixtures/czi_hosted_dataset_config_outline.py:F821 backend/test/fixtures/czi_hosted_server_config_outline.py:F821 backend/test/performance/scale_test_annotations.py:E501'
 
 .PHONY: lint-client
 lint-client:
@@ -97,6 +153,11 @@ pydist: build
 	cd $(BUILDDIR); python setup.py sdist -d ../dist
 	@echo "done"
 
+.PHONY: pydist-czi-hosted
+pydist-czi-hosted: build-czi-hosted
+	cd $(BUILDDIR); python setup.py sdist -d ../dist
+	@echo "done"
+
 
 # RELEASE HELPERS
 
@@ -145,8 +206,11 @@ dev-env-client:
 
 .PHONY: dev-env-server
 dev-env-server:
-	pip install -r server/requirements-dev.txt
+	pip install -r backend/server/requirements-dev.txt
 
+.PHONY: dev-env-czi-hosted
+dev-env-czi-hosted:
+	pip install -r backend/czi_hosted/requirements-dev.txt
 # Set PART=[major, minor, patch] as param to make bump.
 # This will create a release candidate. (i.e. 0.16.1 -> 0.16.2-rc.0 for a patch bump)
 .PHONY: bump-version

README.md

@@ -52,7 +52,6 @@ cellxgene currently supports the following browsers:
 - Google Chrome 61+
 - Edge 15+
 - Firefox 60+
-- Safari 10.1+
 
 Please [file an issue](https://github.com/chanzuckerberg/cellxgene/issues/new/choose) if you would like us to add support for an unsupported browser.
 

backend/common/Makefile

@@ -0,0 +1,11 @@
+
+.PHONY: unit-test
+unit-test:
+	PYTHONWARNINGS=ignore:ResourceWarning coverage run \
+		--source=fbs,utils \
+		--omit=.coverage,data_common/fbs/NetEncoding,venv \
+		-m unittest discover \
+		--start-directory ../test/test_common/unit \
+		--top-level-directory ../../ \
+		--verbose; test_result=$$?; \
+	exit $$test_result \

backend/common/colors.py

@@ -1,6 +1,6 @@
 import re
 
-from server.common.errors import ColorFormatException
+from backend.common.errors import ColorFormatException
 
 HEX_COLOR_FORMAT = re.compile("^#[a-fA-F0-9]{6,6}$")
 

backend/common/compute/diffexp_generic.py

@@ -1,5 +1,6 @@
 import numpy as np
 from scipy import sparse, stats
+from backend.common.constants import XApproximateDistribution
 
 
 def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
@@ -7,7 +8,7 @@ def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
     Return differential expression statistics for top N variables.
 
     Algorithm:
-    - compute log fold change (log2(meanA/meanB))
+    - compute fold change
     - compute Welch's t-test statistic and pvalue (w/ Bonferroni correction)
     - return top N abs(logfoldchange) where lfc > diffexp_lfc_cutoff
 
@@ -25,21 +26,25 @@ def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
     :param maskB: observation selection mask for set 2
     :param top_n: number of variables to return stats for
     :param diffexp_lfc_cutoff: minimum
-    :return:  for top N genes, [ varindex, logfoldchange, pval, pval_adj ]
+    absolute value returning [ varindex, logfoldchange, pval, pval_adj ] for top N genes
+    :return:  for top N genes, {"positive": for top N genes, [ varindex, foldchange, pval, pval_adj ], "negative": for top N genes, [ varindex, foldchange, pval, pval_adj ]}
     """
 
+    X_approximate_distribution = adaptor.get_X_approximate_distribution()
     dataA = adaptor.get_X_array(maskA, None)
     dataB = adaptor.get_X_array(maskB, None)
 
     # mean, variance, N - calculate for both selections
-    meanA, vA, nA = mean_var_n(dataA)
+    meanA, vA, nA = mean_var_n(dataA, X_approximate_distribution)
-    meanB, vB, nB = mean_var_n(dataB)
+    meanB, vB, nB = mean_var_n(dataB, X_approximate_distribution)
     res = diffexp_ttest_from_mean_var(meanA, vA, nA, meanB, vB, nB, top_n, diffexp_lfc_cutoff)
 
     return res
 
 
 def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp_lfc_cutoff):
+    # IMPORTANT NOTE: this code assumes the data is normally distributed and/or already logged.
+
     n_var = meanA.shape[0]
     top_n = min(top_n, n_var)
 
@@ -63,27 +68,30 @@ def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp
     pvals_adj = pvals * n_var
     pvals_adj[pvals_adj > 1] = 1  # cap adjusted p-value at 1
 
-    # logfoldchanges: log2(meanA / meanB)
+    # log fold change. The data is normally distributed/logged, so just subtract the means.
-    logfoldchanges = np.log2(np.abs((meanA + 1e-9) / (meanB + 1e-9)))
+    logfoldchanges = meanA - meanB
 
+    stats_to_sort = tscores
     # find all with lfc > cutoff
     lfc_above_cutoff_idx = np.nonzero(np.abs(logfoldchanges) > diffexp_lfc_cutoff)[0]
-    stats_to_sort = np.abs(tscores)
 
     # derive sort order
-    if lfc_above_cutoff_idx.shape[0] > top_n:
+    if lfc_above_cutoff_idx.shape[0] > top_n * 2:
         # partition top N
-        rel_t_partition = np.argpartition(stats_to_sort[lfc_above_cutoff_idx], -top_n)[-top_n:]
+        rel_t_partition = np.argpartition(stats_to_sort[lfc_above_cutoff_idx], (top_n, -top_n))
-        t_partition = lfc_above_cutoff_idx[rel_t_partition]
+        rel_t_partition_top_n = np.concatenate((rel_t_partition[-top_n:], rel_t_partition[:top_n]))
+        t_partition = lfc_above_cutoff_idx[rel_t_partition_top_n]
         # sort the top N partition
         rel_sort_order = np.argsort(stats_to_sort[t_partition])[::-1]
         sort_order = t_partition[rel_sort_order]
     else:
         # partition and sort top N, ignoring lfc cutoff
-        partition = np.argpartition(stats_to_sort, -top_n)[-top_n:]
+        partition = np.argpartition(stats_to_sort, (top_n, -top_n))
-        rel_sort_order = np.argsort(stats_to_sort[partition])[::-1]
+        partition_top_n = np.concatenate((partition[-top_n:], partition[:top_n]))
+
+        rel_sort_order = np.argsort(stats_to_sort[partition_top_n])[::-1]
         indices = np.indices(stats_to_sort.shape)[0]
-        sort_order = indices[partition][rel_sort_order]
+        sort_order = indices[partition_top_n][rel_sort_order]
 
     # top n slice based upon sort order
     logfoldchanges_top_n = logfoldchanges[sort_order]
@@ -91,12 +99,21 @@ def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp
     pvals_adj_top_n = pvals_adj[sort_order]
 
     # varIndex, logfoldchange, pval, pval_adj
-    result = [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in range(top_n)]
+    result = {
+        "positive": [
+            [sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in range(top_n)
+        ],
+        "negative": [
+            [sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]]
+            for i in range(-1, -1 - top_n, -1)
+        ],
+    }
+
     return result
 
 
 # Convenience function which handles sparse data
-def mean_var_n(X):
+def mean_var_n(X, X_approximate_distribution=XApproximateDistribution.NORMAL):
     """
     Two-pass variance calculation.  Numerically (more) stable
     than naive methods (and same method used by numpy.var())
@@ -114,16 +131,27 @@ def mean_var_n(X):
     with np.errstate(divide="call", invalid="call", call=fp_err_set):
         n = X.shape[0]
         if sparse.issparse(X):
+            if X_approximate_distribution == XApproximateDistribution.COUNT:
+                X = X.log1p()
             mean = X.mean(axis=0).A1
             dfm = X - mean
             sumsq = np.sum(np.multiply(dfm, dfm), axis=0).A1
             v = sumsq / (n - 1)
         else:
+            if X_approximate_distribution == XApproximateDistribution.COUNT:
+                X = np.log1p(X)
             mean = X.mean(axis=0)
             dfm = X - mean
             sumsq = np.sum(np.multiply(dfm, dfm), axis=0)
             v = sumsq / (n - 1)
 
+    # AnnData does not guarantee that operations on a view of X will
+    # return an ndarray, so force the cast if it wasn't done for us.
+    if type(mean) is not np.ndarray:
+        mean = mean.toarray()
+    if type(v) is not np.ndarray:
+        v = v.toarray()
+
     if fp_err_occurred:
         mean[np.isfinite(mean) == False] = 0  # noqa: E712
         v[np.isfinite(v) == False] = 0  # noqa: E712

backend/common/compute/estimate_distribution.py

@@ -0,0 +1,108 @@
+from typing import Tuple
+import numba
+import concurrent.futures
+import numpy as np
+from scipy import sparse
+from backend.common.constants import XApproximateDistribution
+
+
+@numba.njit(error_model="numpy", nogil=True)
+def min_max_fast(arr: np.ndarray) -> Tuple[float, float]:
+    """Return (min, max) values for the ndarray."""
+
+    # initialize to first finite value in array.  Normally,
+    # this will exit on the first value.
+    for i in range(arr.size):
+        min_val = max_val = arr[i]
+        if np.isfinite(min_val):
+            break
+
+    # now find min/max, unrolled by two
+    odd = arr.size % 2
+    unrolled_loop_limit = arr.size - 1 if odd else arr.size
+    i = 0
+    while i < unrolled_loop_limit:
+        x = arr[i]
+        y = arr[i + 1]
+
+        # ignore non-finites
+        x = x if np.isfinite(x) else min_val
+        y = y if np.isfinite(y) else min_val
+
+        if x > y:
+            x, y = y, x
+        min_val = min(x, min_val)
+        max_val = max(y, max_val)
+        i += 2
+
+    # handle the tail if any
+    if odd:
+        x = arr[arr.size - 1]
+
+        # ignore non-finites
+        x = x if np.isfinite(x) else min_val
+
+        min_val = min(x, min_val)
+        max_val = max(x, max_val)
+
+    return min_val, max_val
+
+
+def min_max_numpy(arr: np.ndarray) -> Tuple[float, float]:
+    return arr.min(), arr.max()
+
+
+def numba_has_support_for_scalar_type(arr: np.ndarray) -> bool:
+    """Numba does not support half-floats, 128 bit floats, ints > 64 bit or non-scalars."""
+    if arr.dtype == np.float32 or arr.dtype == np.float64:
+        return True
+
+    if np.issubdtype(arr.dtype, np.integer) and arr.dtype <= np.int64:
+        return True
+
+    if arr.dtype == np.bool_:
+        return True
+
+    return False
+
+
+def estimate_approximate_distribution(X) -> XApproximateDistribution:
+    """
+    Estimate the distribution (normal, count) of the X matrix.
+
+    Currently this is based upon the assumption that scRNA-seq data is
+    exponentially distributed in its raw (count) form, and when logged,
+    any (max-min) range in excess of 24 is implies tens of millions of
+    observations of a single feature and so is extremely unlikely.
+    """
+    if X.dtype.kind not in ["i", "u", "f"]:
+        raise TypeError(f"Unsupported matrix dtype: {X.dtype.name}")
+
+    if X.size == 0:
+        # default for empty array
+        return XApproximateDistribution.NORMAL
+
+    if sparse.isspmatrix_csc(X) or sparse.isspmatrix_csr(X):
+        Xdata = X.data
+    elif type(X) is np.ndarray:
+        Xdata = X.reshape(
+            X.size,
+        )
+    else:
+        raise TypeError(f"Unsupported matrix format: {str(type(X))}")
+
+    min_max = min_max_fast if numba_has_support_for_scalar_type(Xdata) else min_max_numpy
+
+    CHUNKSIZE = 1 << 24
+    if Xdata.size > CHUNKSIZE:
+        min_val = max_val = Xdata[0]
+        with concurrent.futures.ThreadPoolExecutor() as tp:
+            for (_min, _max) in tp.map(min_max, [Xdata[i : i + CHUNKSIZE] for i in range(0, Xdata.size, CHUNKSIZE)]):
+                min_val = min(_min, min_val)
+                max_val = max(_max, max_val)
+
+    else:
+        min_val, max_val = min_max(Xdata)
+
+    excess_range = (max_val - min_val) > 24
+    return XApproximateDistribution.COUNT if excess_range else XApproximateDistribution.NORMAL

backend/common/constants.py

@@ -24,6 +24,11 @@ class DiffExpMode(AugmentedEnum):
     VAR_FILTER = "varFilter"
 
 
+class XApproximateDistribution(AugmentedEnum):
+    NORMAL = "normal"
+    COUNT = "count"
+
+
 JSON_NaN_to_num_warning_msg = "JSON encoding failure - please verify all data are finite values (no NaN or Infinities)"
 REACTIVE_LIMIT = 1_000_000
 

backend/common/errors.py

@@ -51,7 +51,8 @@ define_request_exception(
     default_status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
 )
 
-define_exception("OntologyLoadFailure", "Raised when reading the ontology file fails")
 define_exception("ConfigurationError", "Raised when checking configuration errors")
 define_exception("PrepareError", "Raised when data is misprepared")
 define_exception("SecretKeyRetrievalError", "Raised when get_secret_key from AWS fails")
+define_exception("ObsoleteRequest", "Raised when the request is no longer valid.")
+define_exception("UnsupportedSummaryMethod", "Raised when a gene set summary method is unknown or unsupported.")

backend/common/fbs/matrix.py

@@ -5,14 +5,16 @@ import pandas as pd
 from flatbuffers import Builder
 from scipy import sparse
 
-import server.data_common.fbs.NetEncoding.Column as Column
+from backend.common.utils.type_conversion_utils import get_encoding_dtype_of_array
-import server.data_common.fbs.NetEncoding.Float32Array as Float32Array
+
-import server.data_common.fbs.NetEncoding.Float64Array as Float64Array
+import backend.common.fbs.NetEncoding.Column as Column
-import server.data_common.fbs.NetEncoding.Int32Array as Int32Array
+import backend.common.fbs.NetEncoding.Float32Array as Float32Array
-import server.data_common.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray
+import backend.common.fbs.NetEncoding.Float64Array as Float64Array
-import server.data_common.fbs.NetEncoding.Matrix as Matrix
+import backend.common.fbs.NetEncoding.Int32Array as Int32Array
-import server.data_common.fbs.NetEncoding.TypedArray as TypedArray
+import backend.common.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray
-import server.data_common.fbs.NetEncoding.Uint32Array as Uint32Array
+import backend.common.fbs.NetEncoding.Matrix as Matrix
+import backend.common.fbs.NetEncoding.TypedArray as TypedArray
+import backend.common.fbs.NetEncoding.Uint32Array as Uint32Array
 
 
 # Serialization helper
@@ -85,7 +87,7 @@ def serialize_typed_array(builder, source_array, encoding_info):
 def column_encoding(arr):
     column_encoding_type_map = {
         # array protocol string:  ( array_type, as_type )
-        np.dtype(np.float64).str: (TypedArray.TypedArray.Float64Array, np.float64),
+        np.dtype(np.float64).str: (TypedArray.TypedArray.Float32Array, np.float32),
         np.dtype(np.float32).str: (TypedArray.TypedArray.Float32Array, np.float32),
         np.dtype(np.float16).str: (TypedArray.TypedArray.Float32Array, np.float32),
         np.dtype(np.int8).str: (TypedArray.TypedArray.Int32Array, np.int32),
@@ -99,7 +101,8 @@ def column_encoding(arr):
     }
     column_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, "json")
 
-    return column_encoding_type_map.get(arr.dtype.str, column_encoding_default)
+    encoding_dtype = np.dtype(get_encoding_dtype_of_array(arr))
+    return column_encoding_type_map.get(encoding_dtype.str, column_encoding_default)
 
 
 def index_encoding(arr):
@@ -199,7 +202,7 @@ def deserialize_typed_array(tarr):
     arr.Init(u.Bytes, u.Pos)
     narr = arr.DataAsNumpy()
     if u_type == TypedArray.TypedArray.JSONEncodedArray:
-        narr = json.loads(narr.tostring().decode("utf-8"))
+        narr = json.loads(narr.tobytes().decode("utf-8"))
     return narr
 
 

backend/common/genesets.py

@@ -0,0 +1,239 @@
+"""
+Utility code for gene sets handling
+"""
+
+import re
+import csv
+import hashlib
+
+from .errors import AnnotationsError
+
+
+GENESETS_TIDYCSV_HEADER = [
+    "gene_set_name",
+    "gene_set_description",
+    "gene_symbol",
+    "gene_description",
+]
+
+
+def read_gene_sets_tidycsv(gs_locator, context=None):
+    """
+    Read & parse the Tidy CSV format, applying validation checks for mandatory
+    values, and de-duping rules.
+
+    Format is a four-column CSV, with a mandatory header row, and optional "#" prefixed
+    comments.  Format:
+
+        gene_set_name, gene_set_description, gene_symbol, gene_description
+
+    gene_set_name must be non-null; others are optional.
+
+    Returns: a dictionary of the shape (values in angle-brackets vary):
+
+        {
+            <string, a gene set name>: {
+                "geneset_name": <string, a gene set name>,
+                "geneset_description": <a string or None>,
+                "genes": [
+                    {
+                        "gene_symbol": <string, a gene symbol or name>,
+                        "gene_description": <a string or None>
+                    },
+                    ...
+                ]
+            },
+            ...
+        }
+    """
+
+    class myDialect(csv.excel):
+        skipinitialspace = False
+
+    def just(n, seq):
+        it = iter(seq)
+        for _ in range(n - 1):
+            yield next(it, "")
+        yield tuple(it)
+
+    messagefn = context["messagefn"] if context else (lambda x: None)
+
+    gene_sets = {}
+    with gs_locator.local_handle() as fname:
+        with open(fname, newline="") as f:
+            reader = csv.reader(f, dialect=myDialect())
+            haveReadHeader = False
+            lineno = 0
+            for row in reader:
+                lineno += 1
+                # ignore empty rows
+                if len(row) == 0:
+                    continue
+                # if row starts with '#' it is a comment
+                if row[0].startswith("#"):
+                    continue
+                # if this is the first non-comment row, assume it is a header and validate
+                # column names.  OK if the user has extra columns after our initial set.
+                if not haveReadHeader:
+                    if row[0:len(GENESETS_TIDYCSV_HEADER)] != GENESETS_TIDYCSV_HEADER:
+                        raise AnnotationsError("Gene set CSV file missing the required column header.")
+                    haveReadHeader = True
+                    continue
+
+                geneset_name, geneset_description, gene_symbol, gene_description, _ = just(5, row)
+                if not geneset_name:
+                    raise AnnotationsError(f"Gene set CSV missing required gene set name on line {lineno}")
+                if (not gene_symbol) and gene_description:
+                    messagefn(f"Warning: Missing gene name in gene set name {geneset_name} on line {lineno}.")
+
+                if geneset_name in gene_sets:
+                    gs = gene_sets[geneset_name]
+                else:
+                    gs = gene_sets[geneset_name] = {
+                        "geneset_name": geneset_name,
+                        "geneset_description": geneset_description,
+                        "genes": [],
+                    }
+                # Use first geneset_description with a value
+                if not gs["geneset_description"] and geneset_description:
+                    gs["geneset_description"] = geneset_description
+                # add the gene if the gene_symbol is defined
+                if gene_symbol:
+                    gs["genes"].append(
+                        {
+                            "gene_symbol": gene_symbol,
+                            "gene_description": gene_description,
+                        }
+                    )
+
+    return gene_sets
+
+
+def write_gene_sets_tidycsv(f, genesets):
+    """
+    Convert the internal gene sets format (returned by read_gene_set_tidycsv) into
+    the simple Tidy CSV.
+    """
+    writer = csv.writer(f, dialect="excel")
+    writer.writerow(GENESETS_TIDYCSV_HEADER)
+    for geneset in genesets:
+        # genes may be empty, in which case we skip the gene set entirely
+        genes = geneset["genes"]
+        if not genes:
+            writer.writerow([geneset["geneset_name"], geneset.get("geneset_description", ""), "", ""])
+        else:
+            writer.writerows(
+                [
+                    [
+                        geneset["geneset_name"],
+                        geneset.get("geneset_description", ""),
+                        gene["gene_symbol"],
+                        gene.get("gene_description", ""),
+                    ]
+                    for gene in genes
+                ]
+            )
+
+
+def summarizeQueryHash(raw_query):
+    """ generate a cache key (hash) from the raw query string """
+    return hashlib.sha1(raw_query).hexdigest()
+
+
+def validate_gene_sets(genesets, var_names, context=None):
+    """
+    Check validity of gene sets, return if correct, else raise error.
+    May also modify the gene set for conditions that should be resolved,
+    but which do not warrant a hard error.
+
+    Argument gene sets may be either the REST OTA format (list of dicts) or the internal
+    format (dict of dicts, keyed by the gene set name).
+
+    Will return a modified gene sets (eg, remove warnings) of the same type as the
+    provided argument. Ie, dict->dict, list->list
+
+    Rules:
+
+    0. All gene set names must be unique. [error]
+    1. Gene set names must conform to the following: [error]
+        * Names must be comprised of 1 or more ASCII characters 32-126
+        * No leading or trailing spaces (ASCII 32)
+        * No multi-space (ASCII 32) runs
+    2. Gene symbols must be part of the current var_index. [warning]
+       If gene symbol is not in the var_index, generate a warning and remove the symbol
+       from the gene sets.
+    3. Gene symbols must not be duplicated in a gene set.  [warning]
+       Duplications will be silently de-duped.
+
+    Items marked [error] will generate a hard error, causing the validation to fail.
+
+    Items marked [warning] will generate a warning, and will be resolved without failing
+    the validation (typically by removing the offending item from the gene sets).
+    """
+
+    messagefn = context["messagefn"] if context else (lambda x: None)
+
+    # accept genesets args as either the internal (dict) or REST (list) format,
+    # as they are identical except for the dict being keyed by geneset_name.
+    if not isinstance(genesets, dict) and not isinstance(genesets, list):
+        raise ValueError("Gene sets must be either dict or list.")
+    genesets_iterable = genesets if isinstance(genesets, list) else genesets.values()
+
+    # 0. check for uniqueness of geneset names
+    geneset_names = [gs["geneset_name"] for gs in genesets_iterable]
+    if len(set(geneset_names)) != len(geneset_names):
+        raise KeyError("All gene set names must be unique.")
+
+    # 1. check gene set character set and format
+    illegal_name = re.compile(r"^\s|  |[\u0000-\u001F\u007F-\uFFFF]|\s$")
+    for name in geneset_names:
+        if type(name) != str or len(name) == 0:
+            raise KeyError("Gene set names must be non-null string.")
+        if illegal_name.search(name):
+            messagefn(
+                "Error: "
+                f"Gene set name {name} "
+                "is not valid. Leading, trailing, and multiple spaces within a name are not allowed."
+            )
+            raise KeyError(
+                "Gene set name is not valid. Leading, trailing, and multiple spaces within a name are not allowed."
+            )
+
+    # 2. & 3. check for duplicate gene symbols, and those not present in the dataset. They will
+    # generate a warning and be removed.
+    for geneset in genesets_iterable:
+        if not isinstance(geneset, dict):
+            raise ValueError("Each gene set must be a dict.")
+        geneset_name = geneset["geneset_name"]
+        genes = geneset["genes"]
+        if not isinstance(genes, list):
+            raise ValueError("Gene set genes field must be a list")
+        geneset.setdefault("geneset_description", "")
+        gene_symbol_already_seen = set()
+        new_genes = []
+        for gene in genes:
+            gene_symbol = gene["gene_symbol"]
+            if not isinstance(gene_symbol, str) or len(gene_symbol) == 0:
+                raise ValueError("Gene symbol must be non-null string.")
+            if gene_symbol in gene_symbol_already_seen:
+                # duplicate check
+                messagefn(
+                    f"Warning: a duplicate of gene {gene_symbol} was found in gene set {geneset_name}, "
+                    "and will be ignored."
+                )
+                continue
+
+            if gene_symbol not in var_names:
+                messagefn(
+                    f"Warning: {gene_symbol}, used in gene set {geneset_name}, "
+                    "was not found in the dataset and will be ignored."
+                )
+                continue
+
+            gene_symbol_already_seen.add(gene_symbol)
+            gene.setdefault("gene_description", "")
+            new_genes.append(gene)
+
+        geneset["genes"] = new_genes
+
+    return genesets

backend/common/utils/aws_secret_utils.py

@@ -3,7 +3,7 @@ import logging
 import boto3
 from flask import json
 
-from server.common.errors import SecretKeyRetrievalError
+from backend.common.errors import SecretKeyRetrievalError
 
 
 def get_secret_key(region_name, secret_name):

backend/common/utils/type_conversion_utils.py

@@ -0,0 +1,191 @@
+from typing import Union, Tuple
+import logging
+
+import numpy as np
+import pandas as pd
+
+"""
+These routines drive all type inference for the schema generation and the
+FBS (REST OTA) encoding. They are also used for CXG generation.
+
+
+H5AD Type                       REST              REST
+(ndarray, Series, Index)        FBS encoding      schema type       ERROR/exceptions
+----------------------------    --------------    ---------------   ----------------------
+bool_/bool                      uint8             boolean
+(u)int8, (u)int16, int32        int32             int32
+uint32, (u)int64                int32             int32             CHECKS value bounds
+float16, float32, float64       float32           float32[0]
+
+categorical[T is numeric[4]]:
+    hasna = False               T                 categorical[1]
+    hasna = True                float32           categorical[1]    CHECKS value bounds
+
+categorical[T not numeric]      JSON/str          categorical[1,2]
+
+(other object)                  JSON/str          string
+
+(all other)                                                         Always an ERROR[3]
+
+
+Notes:
+[0] IEEE format, includes non-finite numbers (NaN, Inf, ...)
+[1] with NO categories enumerated (client side does it to handle rounding)
+[2] NA (undefined) categories are assigned a JSON null value
+[3] Includes all other numpy types:  datetime, complex, etc.
+[4] means float, int, uint (dtype.kind in ['i','u','f'])
+
+"""
+
+
+def get_dtypes_and_schemas_of_dataframe(dataframe: pd.DataFrame):
+    dtypes_by_column_name = {}
+    schema_type_hints_by_column_name = {}
+
+    for column_name, column_values in dataframe.items():
+        (
+            dtypes_by_column_name[column_name],
+            schema_type_hints_by_column_name[column_name],
+        ) = get_dtype_and_schema_of_array(column_values)
+
+    return dtypes_by_column_name, schema_type_hints_by_column_name
+
+
+def get_encoding_dtype_of_array(array: Union[np.ndarray, pd.Series, pd.Index]) -> np.dtype:
+    return _get_type_info(array)[0]
+
+
+def get_schema_type_hint_of_array(array: Union[np.ndarray, pd.Series, pd.Index]) -> dict:
+    return _get_type_info(array)[1]
+
+
+def get_dtype_and_schema_of_array(array: Union[np.ndarray, pd.Series, pd.Index]) -> Tuple[np.dtype, dict]:
+    """Return tuple (encoding_dtype, schema_type_hint)"""
+    return _get_type_info(array)
+
+
+def get_schema_type_hint_from_dtype(dtype) -> dict:
+    res = _get_type_info_from_dtype(dtype)
+    if res is None:
+        raise TypeError(f"Annotations of type {dtype} are unsupported.")
+    else:
+        return res[1]
+
+
+def _get_type_info_from_dtype(dtype) -> Union[Tuple[np.dtype, dict], None]:
+    """
+    Best-effort to determine encoding type and schema hint from a dtype.
+    If this is not possible, or the type is unsupported, return None.
+
+    This should be a subset of the cases which are supported by
+    _get_type_info().  The latter should be preferred if the array (values)
+    are available for typing.
+    """
+    if dtype.kind == "b":
+        return (np.uint8, {"type": "boolean"})
+
+    if dtype.kind == "U":
+        return (np.dtype(str), {"type": "string"})
+
+    if dtype.kind in ["i", "u"]:
+        if np.can_cast(dtype, np.int32):
+            return (np.int32, {"type": "int32"})
+
+    if dtype.kind == "f":
+        _float64_warning(dtype)
+        return (np.float32, {"type": "float32"})
+
+    if dtype.kind == "O" and not dtype.name == "category":
+        return (np.dtype(str), {"type": "string"})
+
+    return None
+
+
+def _get_type_info(array: Union[np.ndarray, pd.Series, pd.Index]) -> Tuple[np.dtype, dict]:
+    """
+    Determine encoding type and schema hint from an array.  This allows more
+    flexible casting than may be possible by using just the dtype, as it can
+    account for category types and array values.
+    """
+    if (
+        not isinstance(array, np.ndarray)
+        and not isinstance(array, pd.Series)
+        and not isinstance(array, pd.Index)
+        and not hasattr(array, "dtype")
+    ):
+        raise TypeError("Unsupported data type.")
+
+    dtype = array.dtype
+
+    res = _get_type_info_from_dtype(dtype)
+    if res is not None:
+        return res
+
+    if dtype.kind == "O":
+        if dtype.name == "category":
+            # Sometimes CategoricalDType can be encoded as int or float without further fuss.
+            # Do not specify the categories in the schema - let the client-side figure it out
+            # on its own.  Utilize Series.to_numpy() to do casting that handles categorical
+            # NA/NaN (missing or undefined) categories.
+            if dtype.categories.dtype.kind in ["f", "i", "u"]:
+                return (
+                    _get_type_info(array.to_numpy())[0],
+                    {"type": "categorical"},
+                )
+            else:
+                return (np.dtype(str), {"type": "categorical", "categories": dtype.categories.to_list()})
+
+        # all other extension types are str-encoded
+        return (np.dtype(str), {"type": "string"})
+
+    if dtype.kind in ["i", "u"] and _can_cast_array_values_to_int32(array):
+        return (np.int32, {"type": "int32"})
+
+    if dtype.kind == "f":
+        _float64_warning(array.dtype)
+        return (np.float32, {"type": "float32"})
+
+    raise TypeError(f"Annotations of type {dtype} are unsupported.")
+
+
+def _float64_warning(dtype):
+    """
+    Warn the user if we are down-casting a float64 to float32, and may potentially lose information.
+    """
+    if dtype.kind == "f" and not np.can_cast(dtype, np.float32):
+        logging.warning(f"Type {dtype.name} will be converted to 32 bit float and may lose precision.")
+
+
+def _can_cast_array_values_to_int32(array: Union[np.ndarray, pd.Series, pd.Index]) -> bool:
+    """
+    Return true if the (U)INT array values can be safely cast to int32.  We allow size reducing
+    casts (ie, int64 to int32) if no actual values require the larger size (ie, actual values
+    can be represented by the smaller type).
+    """
+    assert array.dtype.kind in ["u", "i"]
+
+    if np.can_cast(array.dtype, np.int32):
+        return True
+
+    if array.size == 0:
+        return True
+
+    int32_machine_limits = np.iinfo(np.int32)
+    if array.min() >= int32_machine_limits.min and array.max() <= int32_machine_limits.max:
+        return True
+
+    return False
+
+
+def convert_string_to_value(value: str):
+    """convert a string to value with the most appropriate type"""
+    if value.lower() == "true":
+        return True
+    if value.lower() == "false":
+        return False
+    if value == "null":
+        return None
+    try:
+        return eval(value)
+    except:  # noqa E722
+        return value

backend/common/utils/utils.py

@@ -10,7 +10,7 @@ from urllib.parse import urlsplit, urljoin
 import numpy as np
 from flask import json
 
-from server.common.errors import ConfigurationError
+from backend.common.errors import ConfigurationError
 
 
 def find_available_port(host, port=5005):
@@ -65,7 +65,13 @@ def path_join(base, *urls):
     return btpl._replace(path=path).geturl()
 
 
-class Float32JSONEncoder(json.JSONEncoder):
+class StrictJSONEncoder(json.JSONEncoder):
+    """
+    Custom JSON encoder set-up performing two tasks:
+    1. Strict JSON conformance with non-finite floats (NaN, +/-Inf) via allow_nan=False
+    2. Convert various Numpy types into python types so the encoder will correctly encode.
+    """
+
     def __init__(self, *args, **kwargs):
         """
         NaN/Infinities are illegal in standard JSON.  Python extends JSON with
@@ -78,9 +84,11 @@ class Float32JSONEncoder(json.JSONEncoder):
         super().__init__(*args, **kwargs)
 
     def default(self, obj):
-        if isinstance(obj, np.float32):
+        """This helps us convert types not supported by the native JSON encoder into
+        standard python types, eg, np.int64."""
+        if isinstance(obj, np.floating):
             return float(obj)
-        elif isinstance(obj, np.integer):
+        if isinstance(obj, np.integer):
             return int(obj)
         return json.JSONEncoder.default(self, obj)
 
@@ -89,8 +97,8 @@ def custom_format_warning(msg, *args, **kwargs):
     return f"[cellxgene] Warning: {msg} \n"
 
 
-def jsonify_numpy(data):
+def jsonify_strict(data):
-    return json.dumps(data, cls=Float32JSONEncoder, allow_nan=False)
+    return json.dumps(data, cls=StrictJSONEncoder, allow_nan=False)
 
 
 def import_plugins(plugin_module):

backend/czi_hosted/Makefile

@@ -1,4 +1,4 @@
-include ../common.mk
+include ../../common.mk
 
 .PHONY: clean
 clean:
@@ -9,11 +9,11 @@ clean:
 .PHONY: unit-test
 unit-test: create-test-db
 	PYTHONWARNINGS=ignore:ResourceWarning coverage run \
-		--source=app,cli,common,compute,converters,data_anndata,data_common,data_cxg \
+		--source=app,auth,cli,common,compute,converters,data_anndata,data_common,data_cxg,eb \
-		--omit=.coverage,data_common/fbs/NetEncoding,venv \
+		--omit=.coverage,venv \
 		-m unittest discover \
-		--start-directory test/ \
+		--start-directory ../test/test_czi_hosted/unit \
-		--top-level-directory ../ \
+		--top-level-directory ../.. \
 		--verbose; test_result=$$?; \
 	$(MAKE) clean-test-db; \
 	exit $$test_result \
@@ -22,11 +22,11 @@ unit-test: create-test-db
 .PHONY: test-db
 test-db: create-test-db
 	PYTHONWARNINGS=ignore:ResourceWarning coverage run \
-		--source=app,cli,common,compute,converters,data_anndata,data_common,data_cxg \
+		--source=db \
-		--omit=.coverage,data_common/fbs/NetEncoding,venv \
+		--omit=.coverage,venv \
 		-m unittest discover \
-		--start-directory test/test_database \
+		--start-directory ../test/test_czi_hosted/test_database \
-		--top-level-directory ../ \
+		--top-level-directory ../.. \
 		--verbose; test_result=$$?; \
 	$(MAKE) clean-test-db; \
 	exit $$test_result
@@ -42,8 +42,8 @@ clean-test-db:
 
 .PHONY: test-annotations-performance
 test-annotations-performance:
-	python test/performance/performance_test_annotations_backend.py
+	python ../test/test_czi_hosted/performance/performance_test_annotations_backend.py
 
 .PHONY: test-annotations-scale
 test-annotations-scale:
-	locust -f test/performance/scale_test_annotations.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt
+	locust -f ../test/test_czi_hosted/performance/scale_test_annotations.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt

backend/czi_hosted/__init__.py

@@ -0,0 +1,15 @@
+import logging
+import sys
+from backend.common.utils.utils import import_plugins
+
+__version__ = "0.16.7"
+
+
+display_version = "cellxgene v" + __version__
+
+try:
+    import_plugins("backend.czi_hosted.plugins")
+except Exception as e:
+    # Make sure to exit in this case, as the server may not be configured as expected.
+    logging.critical(f"Error in import_plugins: {str(e)}")
+    sys.exit(1)

backend/czi_hosted/__main__.py

@@ -0,0 +1,14 @@
+# Work around bug https://github.com/pallets/werkzeug/issues/461
+if __package__ is None:
+    import sys
+    from pathlib import Path
+
+    PKG_PATH = Path(__file__).parent
+    sys.path.insert(0, str(PKG_PATH.parent))
+    import backend.czi_hosted # noqa F401
+    __package__ = PKG_PATH.name
+
+# Main thing
+from .cli.cli import cli  # noqa F402
+
+cli()

backend/czi_hosted/app/app.py

@@ -20,14 +20,14 @@ from flask import (
 from flask_restful import Api, Resource
 from server_timing import Timing as ServerTiming
 
-import server.common.rest as common_rest
+import backend.czi_hosted.common.rest as common_rest
-from server.common.data_locator import DataLocator
+from backend.common.utils.data_locator import DataLocator
-from server.common.errors import DatasetAccessError, RequestException
+from backend.common.errors import DatasetAccessError, RequestException
-from server.common.health import health_check
+from backend.czi_hosted.common.health import health_check
-from server.common.utils.utils import path_join, Float32JSONEncoder
+from backend.common.utils.utils import path_join, StrictJSONEncoder
-from server.data_common.matrix_loader import MatrixDataLoader
+from backend.czi_hosted.data_common.matrix_loader import MatrixDataLoader
 
-webbp = Blueprint("webapp", "server.common.web", template_folder="templates")
+webbp = Blueprint("webapp", "backend.czi_hosted.common.web", template_folder="templates")
 
 ONE_WEEK = 7 * 24 * 60 * 60
 
@@ -168,7 +168,7 @@ def rest_get_data_adaptor(func):
     return wrapped_function
 
 
-def dataroot_test_index():
+def  dataroot_test_index():
     # the following index page is meant for testing/debugging purposes
     data = '<!doctype html><html lang="en">'
     data += "<head><title>Hosted Cellxgene</title></head>"
@@ -205,7 +205,7 @@ def dataroot_test_index():
     data += "<ul>"
     datasets.sort()
     for url_dataroot, dataset in datasets:
-        data += f"<li><a href={url_dataroot}/{dataset}>{dataset}</a></li>"
+        data += f"<li><a href={url_dataroot}/{dataset}/>{dataset}</a></li>"
     data += "</ul>"
     data += "</body></html>"
 
@@ -240,7 +240,7 @@ class DatasetResource(Resource):
 
 class SchemaAPI(DatasetResource):
     # TODO @mdunitz separate dataset schema and user schema
-    @cache_control(no_store=True)
+    @cache_control(public=True, max_age=ONE_WEEK)
     @rest_get_data_adaptor
     def get(self, data_adaptor):
         return common_rest.schema_get(data_adaptor)
@@ -261,7 +261,7 @@ class UserInfoAPI(DatasetResource):
 
 
 class AnnotationsObsAPI(DatasetResource):
-    @cache_control(public=True, no_store=True)
+    @cache_control(public=True, max_age=ONE_WEEK)
     @rest_get_data_adaptor
     def get(self, data_adaptor):
         return common_rest.annotations_obs_get(request, data_adaptor)
@@ -312,10 +312,24 @@ class LayoutObsAPI(DatasetResource):
     def get(self, data_adaptor):
         return common_rest.layout_obs_get(request, data_adaptor)
 
-    @cache_control(no_store=True)
+
+class GenesetsAPI(DatasetResource):
+    @cache_control(public=True, max_age=ONE_WEEK)
     @rest_get_data_adaptor
-    def put(self, data_adaptor):
+    def get(self, data_adaptor):
-        return common_rest.layout_obs_put(request, data_adaptor)
+        return common_rest.genesets_get(request, data_adaptor)
+
+
+class SummarizeVarAPI(DatasetResource):
+    @rest_get_data_adaptor
+    @cache_control(public=True, max_age=ONE_WEEK)
+    def get(self, data_adaptor):
+        return common_rest.summarize_var_get(request, data_adaptor)
+
+    @rest_get_data_adaptor
+    @cache_control(no_store=True)
+    def post(self, data_adaptor):
+        return common_rest.summarize_var_post(request, data_adaptor)
 
 
 def get_api_base_resources(bp_base):
@@ -343,6 +357,8 @@ def get_api_dataroot_resources(bp_dataroot, url_dataroot=None):
     add_resource(AnnotationsObsAPI, "/annotations/obs")
     add_resource(AnnotationsVarAPI, "/annotations/var")
     add_resource(DataVarAPI, "/data/var")
+    add_resource(GenesetsAPI, "/genesets")
+    add_resource(SummarizeVarAPI, "/summarize/var")
     # Display routes
     add_resource(ColorsAPI, "/colors")
     # Computation routes
@@ -380,7 +396,7 @@ class Server:
         self.app = Flask(__name__, static_folder=None)
         handle_api_base_url(self.app, app_config)
         self._before_adding_routes(self.app, app_config)
-        self.app.json_encoder = Float32JSONEncoder
+        self.app.json_encoder = StrictJSONEncoder
         server_config = app_config.server_config
         if server_config.app__server_timing_headers:
             ServerTiming(self.app, force_debug=True)
@@ -454,6 +470,6 @@ class Server:
 
         auth = server_config.auth
         self.app.auth = auth
-        if auth.requires_client_login():
+        if auth and auth.requires_client_login():
             auth.add_url_rules(self.app)
         auth.complete_setup(self.app)

backend/czi_hosted/auth/__init__.py

@@ -0,0 +1,6 @@
+# import the built in auth types so they can be registered
+
+import backend.czi_hosted.auth.auth_test  # noqa: F401
+import backend.czi_hosted.auth.auth_session  # noqa: F401
+import backend.czi_hosted.auth.auth_oauth  # noqa: F401
+import backend.czi_hosted.auth.auth_none  # noqa: F401

backend/czi_hosted/auth/auth_none.py

@@ -0,0 +1,27 @@
+from backend.czi_hosted.auth.auth import AuthTypeBase, AuthTypeFactory
+
+
+class AuthTypeNone(AuthTypeBase):
+    def __init__(self, app_config):
+        super().__init__()
+
+    def is_valid_authentication_type(self):
+        return False
+
+    def complete_setup(self, app):
+        pass
+
+    def is_user_authenticated(self):
+        return True
+
+    def get_user_id(self):
+        return None
+
+    def get_user_name(self):
+        return None
+
+    def get_user_email(self):
+        return None
+
+
+AuthTypeFactory.register(None, AuthTypeNone)

backend/czi_hosted/auth/auth_oauth.py

@@ -1,6 +1,6 @@
 from flask import session, request, redirect, current_app, after_this_request, has_request_context, g
-from server.auth.auth import AuthTypeClientBase, AuthTypeFactory
+from backend.czi_hosted.auth.auth import AuthTypeClientBase, AuthTypeFactory
-from server.common.errors import AuthenticationError, ConfigurationError
+from backend.common.errors import AuthenticationError, ConfigurationError
 from urllib.parse import urlencode, urlparse
 import json
 import requests

backend/czi_hosted/auth/auth_session.py

@@ -0,0 +1,40 @@
+from flask import session
+from uuid import uuid4
+
+from backend.czi_hosted.auth.auth import AuthTypeBase, AuthTypeFactory
+
+
+class AuthTypeSession(AuthTypeBase):
+    """Session based authentication.  The user is always logged.  The user id is a random number
+    associated with the session.  This is a good choice for desktop servers."""
+
+    # key in the session token for userid
+    CXGUID = "cxguid"
+
+    def __init__(self, app_config):
+        super().__init__()
+
+    def is_valid_authentication_type(self):
+        return True
+
+    def complete_setup(self, app):
+        pass
+
+    def is_user_authenticated(self):
+        # always authenticated
+        return True
+
+    def get_user_id(self):
+        if self.CXGUID not in session:
+            session[self.CXGUID] = uuid4().hex
+            session.permanent = True
+        return session[self.CXGUID]
+
+    def get_user_name(self):
+        return "anonymous"
+
+    def get_user_email(self):
+        return None
+
+
+AuthTypeFactory.register("session", AuthTypeSession)

backend/czi_hosted/auth/auth_test.py

@@ -1,6 +1,7 @@
-from server.auth.auth import AuthTypeClientBase, AuthTypeFactory
 from flask import session, request, redirect, current_app
 
+from backend.czi_hosted.auth.auth import AuthTypeClientBase, AuthTypeFactory
+
 
 class AuthTypeTest(AuthTypeClientBase):
     """An authentication type for testing client based logins.  When the login route is accessed

backend/czi_hosted/cli/convert_to_cxg.py

@@ -2,7 +2,7 @@ from os import path
 
 import click
 
-from server.converters.h5ad_data_file import H5ADDataFile
+from backend.czi_hosted.converters.h5ad_data_file import H5ADDataFile
 
 
 @click.command(

backend/czi_hosted/cli/launch.py

@@ -8,11 +8,12 @@ import click
 from flask_compress import Compress
 from flask_cors import CORS
 
-from server.default_config import default_config
+from backend.czi_hosted.default_config import default_config
-from server.app.app import Server
+from backend.czi_hosted.app.app import Server
-from server.common.config.app_config import AppConfig
+from backend.czi_hosted.common.config.app_config import AppConfig
-from server.common.errors import DatasetAccessError, ConfigurationError
+from backend.common.errors import DatasetAccessError, ConfigurationError
-from server.common.utils.utils import sort_options
+from backend.common.utils.utils import sort_options
+
 
 DEFAULT_CONFIG = AppConfig()
 
@@ -43,20 +44,6 @@ def annotation_args(func):
         help="Directory of where to save output annotations; filename will be specified in the application. "
         "Incompatible with --annotations-file.",
     )
-    @click.option(
-        "--experimental-annotations-ontology",
-        is_flag=True,
-        default=DEFAULT_CONFIG.default_dataset_config.user_annotations__ontology__enable,
-        show_default=True,
-        help="When creating annotations, optionally autocomplete names from ontology terms.",
-    )
-    @click.option(
-        "--experimental-annotations-ontology-obo",
-        default=DEFAULT_CONFIG.default_dataset_config.user_annotations__ontology__obo_location,
-        show_default=True,
-        metavar="<path or url>",
-        help="Location of OBO file defining cell annotation autosuggest terms.",
-    )
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
         return func(*args, **kwargs)
@@ -103,14 +90,6 @@ def config_args(func):
         metavar="<text>",
         help="Embedding name, eg, 'umap'. Repeat option for multiple embeddings. Defaults to all.",
     )
-    @click.option(
-        "--experimental-enable-reembedding",
-        is_flag=True,
-        default=DEFAULT_CONFIG.default_dataset_config.embeddings__enable_reembedding,
-        show_default=False,
-        hidden=True,
-        help="Enable experimental on-demand re-embedding using UMAP. WARNING: may be very slow.",
-    )
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
         return func(*args, **kwargs)
@@ -327,9 +306,6 @@ def launch(
     annotations_dir,
     backed,
     disable_diffexp,
-    experimental_annotations_ontology,
-    experimental_annotations_ontology_obo,
-    experimental_enable_reembedding,
     config_file,
     dump_default_config,
 ):
@@ -355,7 +331,6 @@ def launch(
     if dump_default_config:
         print(default_config)
         sys.exit(0)
-
     # Startup message
     click.echo("[cellxgene] Starting the CLI...")
 
@@ -389,12 +364,9 @@ def launch(
             user_annotations__enable=not disable_annotations,
             user_annotations__local_file_csv__file=annotations_file,
             user_annotations__local_file_csv__directory=annotations_dir,
-            user_annotations__ontology__enable=experimental_annotations_ontology,
-            user_annotations__ontology__obo_location=experimental_annotations_ontology_obo,
             presentation__max_categories=max_category_items,
             presentation__custom_colors=not disable_custom_colors,
             embeddings__names=embedding,
-            embeddings__enable_reembedding=experimental_enable_reembedding,
             diffexp__enable=not disable_diffexp,
             diffexp__lfc_cutoff=diffexp_lfc_cutoff,
         )

backend/czi_hosted/cli/prepare.py

@@ -5,7 +5,7 @@ import pandas as pd
 from numpy import ndarray, unique
 from scipy.sparse.csc import csc_matrix
 
-from server.common.utils.utils import sort_options
+from backend.common.utils.utils import sort_options
 
 
 @sort_options

backend/czi_hosted/cli/schema.py

@@ -0,0 +1,72 @@
+import click
+
+from backend.czi_hosted.converters.schema import remix, validate
+
+
+@click.group(
+    name="schema",
+    subcommand_metavar="COMMAND <args>",
+    short_help="Apply and validate the cellxgene data integration schema to an h5ad file.",
+    context_settings=dict(max_content_width=85, help_option_names=["-h", "--help"]),
+)
+def schema_cli():
+    try:
+        import scanpy  # noqa: F401
+    except ImportError:
+        raise click.ClickException(
+            "[cellxgene] cellxgene schema requires scanpy"
+        )
+
+
+@click.command(
+    name="apply",
+    short_help="(experimental) Apply the cellxgene data integration schema to an h5ad.",
+    help="(experimental) Using a yaml file that describes schema values to insert or convert and in input "
+    "h5ad file, apply the schema changes and create a new, conforming h5ad.",
+)
+@click.option(
+    "--source-h5ad",
+    help="Input h5ad file.",
+    nargs=1,
+    required=True,
+    type=click.Path(exists=True, dir_okay=False),
+)
+@click.option(
+    "--remix-config",
+    help="Config yaml with information on how to apply the schema.",
+    nargs=1,
+    required=True,
+    type=click.Path(exists=True, dir_okay=False),
+)
+@click.option(
+    "--output-filename",
+    help="Filename for the new, schema-conforming h5ad file.",
+    required=True,
+    nargs=1
+)
+def schema_apply(source_h5ad, remix_config, output_filename):
+    remix.apply_schema(source_h5ad, remix_config, output_filename)
+
+
+@click.command(
+    name="validate",
+    short_help="(experimental) Check that an h5ad follows the cellxgene data integration schema.",
+)
+@click.argument(
+    "h5ad",
+    nargs=1,
+    type=click.Path(exists=True, dir_okay=False),
+)
+@click.option(
+    "--shallow",
+    help="When true, just check that the correct version information is present.",
+    default=False,
+    show_default=True,
+    is_flag=True,
+)
+def schema_validate(h5ad, shallow):
+    validate.validate(h5ad, shallow)
+
+
+schema_cli.add_command(schema_apply)
+schema_cli.add_command(schema_validate)

backend/czi_hosted/common/annotations/annotations.py

@@ -0,0 +1,110 @@
+import os
+
+from flask import current_app, has_request_context
+
+from backend.common.errors import DisabledFeatureError
+from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
+from backend.common.genesets import write_gene_sets_tidycsv, read_gene_sets_tidycsv, validate_gene_sets
+from backend.common.utils.data_locator import DataLocator
+from backend.common.utils.utils import path_join
+
+
+class Annotations:
+    """baseclass for annotations and genesets"""
+
+    def __init__(self, config={}):
+        self.config = config
+
+    def user_annotations_enabled(self):
+        return self.config.get("user-annotations", False)
+
+    def check_user_annotations_enabled(self):
+        if not self.user_annotations_enabled():
+            raise DisabledFeatureError("User annotations are disabled.")
+
+    def get_schema(self, data_adaptor):
+        schema = []
+        labels = self.read_labels(data_adaptor)
+        if labels is not None and not labels.empty:
+            for col in labels.columns:
+                col_schema = dict(name=col, writable=True)
+                col_schema.update(get_schema_type_hint_of_array(labels[col]))
+                schema.append(col_schema)
+
+        return schema
+
+    def set_collection(self, name):
+        """set or create a new annotation collection"""
+        raise NotImplementedError
+
+    def read_labels(self, data_adaptor):
+        """Return the labels as a pandas.DataFrame"""
+        raise NotImplementedError
+
+    def write_labels(self, df, data_adaptor):
+        """Write the labels (df) to a persistent storage such that it can later be read"""
+        raise NotImplementedError
+
+    def update_parameters(self, parameters, data_adaptor):
+        """Update configuration parameters that describe information about the annotations feature"""
+        params = {}
+        params["annotations_genesets_readonly"] = True
+        params["annotations_genesets_name_is_read_only"] = True
+        parameters.update(params)
+
+    @staticmethod
+    def gene_sets_to_csv(genesets):
+        """
+        Convert the internal genesets format (returned by read_gene_set) into
+        the simple Tidy CSV.
+        """
+        from io import StringIO
+
+        if isinstance(genesets, dict):
+            genesets = genesets.values()
+
+        with StringIO() as sio:
+            write_gene_sets_tidycsv(sio, genesets)
+            return sio.getvalue()
+
+    @staticmethod
+    def gene_sets_to_response(genesets):
+        """
+        Convert the internal genesets format (returned by read_gene_set) into
+        the dict expected by the JSON REST API
+        """
+        return list(genesets.values())
+
+    def read_gene_sets(self, data_adaptor, context=None):
+        if has_request_context():
+            if not current_app.auth.is_user_authenticated():
+                return ({}, 0)
+
+        gene_sets_uri_or_path = dataset_uri_to_geneset_uri(data_adaptor.data_locator.uri_or_path)
+
+        server_config = data_adaptor.server_config
+        region_name = None if server_config is None else server_config.data_locator__s3__region_name
+        gene_sets_locator = DataLocator(gene_sets_uri_or_path, region_name=region_name)
+        if not gene_sets_locator.exists():
+            return ({}, 0)
+
+        gene_sets = read_gene_sets_tidycsv(gene_sets_locator, context)
+        schema = data_adaptor.get_schema()
+        var_index = schema["annotations"]["var"].get("index", "index")
+        var_names = set(data_adaptor.query_var_array(var_index))
+
+        gene_sets = validate_gene_sets(gene_sets, var_names)
+        return (gene_sets, 0)
+
+
+def dataset_uri_to_geneset_uri(data_uri_or_path):
+    """given a dataset URI, return the associated gene set URI"""
+    data_basename = os.path.basename(data_uri_or_path)
+    base, ext = os.path.splitext(data_basename)
+    if ext is not None:  # strip extension, if any
+        data_basename = base
+
+    genesets_basename = f"{data_basename}-genesets.csv"
+    gene_sets_uri_or_path = path_join(data_uri_or_path, "..", genesets_basename)
+
+    return gene_sets_uri_or_path

backend/czi_hosted/common/annotations/hosted_tiledb.py

@@ -7,18 +7,18 @@ import pandas as pd
 import tiledb
 from flask import current_app
 
-from server.common.annotations.annotations import Annotations
+from backend.czi_hosted.common.annotations.annotations import Annotations
-from server.common.errors import AnnotationCategoryNameError
+from backend.common.errors import AnnotationCategoryNameError
-from server.common.utils.sanitization_utils import sanitize_values_in_list
+from backend.czi_hosted.common.utils.sanitization_utils import sanitize_values_in_list
-from server.common.utils.type_conversion_utils import get_dtypes_and_schemas_of_dataframe, get_dtype_of_array
+from backend.common.utils.type_conversion_utils import get_dtypes_and_schemas_of_dataframe, get_encoding_dtype_of_array
-from server.db.cellxgene_orm import Annotation
+from backend.czi_hosted.db.cellxgene_orm import Annotation
 
 
 class AnnotationsHostedTileDB(Annotations):
     CXG_ANNO_COLLECTION = "cxg_anno_collection"
 
-    def __init__(self, directory_path, db):
+    def __init__(self, config, directory_path, db):
-        super().__init__()
+        super().__init__(config)
         self.db = db
         if directory_path[-1] == "/":
             self.directory_path = directory_path
@@ -143,7 +143,7 @@ class AnnotationsHostedTileDB(Annotations):
             # convert to tiledb datatypes
 
             for col in df:
-                df[col] = df[col].astype(get_dtype_of_array(df[col]))
+                df[col] = df[col].astype(get_encoding_dtype_of_array(df[col]))
             tiledb.from_pandas(uri, df, sparse=True)
         else:
             uri = ""
@@ -158,14 +158,10 @@ class AnnotationsHostedTileDB(Annotations):
         self.db.session.commit()
 
     def update_parameters(self, parameters, data_adaptor):
+        super().update_parameters(parameters, data_adaptor)
+
         params = {}
         params["annotations"] = True
         params["user_annotation_collection_name_enabled"] = False
 
-        if self.ontology_data:
-            params["annotations_cell_ontology_enabled"] = True
-            params["annotations_cell_ontology_terms"] = self.ontology_data
-        else:
-            params["annotations_cell_ontology_enabled"] = False
-
         parameters.update(params)

backend/czi_hosted/common/annotations/local_file_csv.py

@@ -8,16 +8,16 @@ from hashlib import blake2b
 import pandas as pd
 from flask import session, has_request_context, current_app
 
-from server import __version__ as cellxgene_version
+from backend.czi_hosted import __version__ as cellxgene_version
-from server.common.annotations.annotations import Annotations
+from backend.czi_hosted.common.annotations.annotations import Annotations
-from server.common.errors import AnnotationsError
+from backend.common.errors import AnnotationsError
 
 
 class AnnotationsLocalFile(Annotations):
     CXG_ANNO_COLLECTION = "cxg_anno_collection"
 
-    def __init__(self, output_dir, output_file):
+    def __init__(self, config, output_dir, output_file):
-        super().__init__()
+        super().__init__(config)
         self.output_dir = output_dir
         self.output_file = output_file
         # lock used to protect label file write ops
@@ -115,7 +115,7 @@ class AnnotationsLocalFile(Annotations):
         return os.getcwd()
 
     def _get_filename(self, data_adaptor):
-        """ return the current annotation file name """
+        """return the current annotation file name"""
         if self.output_file:
             return self.output_file
 
@@ -169,16 +169,12 @@ class AnnotationsLocalFile(Annotations):
                 os.remove(os.path.join(backup_dir, bu))
 
     def update_parameters(self, parameters, data_adaptor):
+        super().update_parameters(parameters, data_adaptor)
+
         params = {}
         params["annotations"] = True
         params["user_annotation_collection_name_enabled"] = True
 
-        if self.ontology_data:
-            params["annotations_cell_ontology_enabled"] = True
-            params["annotations_cell_ontology_terms"] = self.ontology_data
-        else:
-            params["annotations_cell_ontology_enabled"] = False
-
         if self.output_file is not None:
             # user has hard-wired the name of the annotation data collection
             fname = os.path.basename(self.output_file)
@@ -190,7 +186,7 @@ class AnnotationsLocalFile(Annotations):
             collection = self.get_collection()
             if current_app.auth.is_user_authenticated():
                 params["annotations-user-data-idhash"] = self._get_userdata_idhash(data_adaptor)
-                params["annotations-data-collection-is-read-only"] = False
+                params["annotations-data-collection-is-read-only"] = not self.user_annotations_enabled()
                 params["annotations-data-collection-name"] = collection
 
         parameters.update(params)

backend/czi_hosted/common/config/__init__.py

@@ -0,0 +1,4 @@
+from backend.common.utils.aws_secret_utils import get_secret_key  # noqa F504
+
+DEFAULT_SERVER_PORT = 5005
+BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20  # 100MB

backend/czi_hosted/common/config/app_config.py

@@ -1,11 +1,11 @@
 import yaml
 from flatten_dict import unflatten
 
-from server.default_config import get_default_config
+from backend.czi_hosted.common.config.external_config import ExternalConfig
-from server.common.config.dataset_config import DatasetConfig
+from backend.czi_hosted.common.config.dataset_config import DatasetConfig
-from server.common.config.server_config import ServerConfig
+from backend.czi_hosted.common.config.server_config import ServerConfig
-from server.common.config.external_config import ExternalConfig
+from backend.common.errors import ConfigurationError
-from server.common.errors import ConfigurationError
+from backend.czi_hosted.default_config import get_default_config
 
 
 class AppConfig(object):
@@ -69,21 +69,21 @@ class AppConfig(object):
 
     def update_server_config(self, **kw):
         self.server_config.update(**kw)
-        self.is_complete = False
+        self.is_completed = False
 
     def update_default_dataset_config(self, **kw):
         self.default_dataset_config.update(**kw)
         # update all the other dataset configs, if any
         for value in self.dataroot_config.values():
             value.update(**kw)
-        self.is_complete = False
+        self.is_completed = False
 
     def update_single_config_from_path_and_value(self, path, value):
         """Update a single config parameter with the value.
         Path is a list of string, that gives a path to the config parameter to be updated.
         For example, path may be ["server","app","port"].
         """
-        self.is_complete = False
+        self.is_completed = False
         if not isinstance(path, list):
             raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'")
         for part in path:
@@ -147,7 +147,7 @@ class AppConfig(object):
         if config.get("external"):
             self.external_config.update_from_config(config["external"], "external")
 
-        self.is_complete = False
+        self.is_completed = False
 
     def config_to_dict(self):
         """return the configuration as an unflattened dict"""

backend/czi_hosted/common/config/base_config.py

@@ -1,7 +1,7 @@
 import copy
 
 from flatten_dict import flatten
-from server.common.errors import ConfigurationError
+from backend.common.errors import ConfigurationError
 
 
 class BaseConfig(object):

backend/czi_hosted/common/config/client_config.py

@@ -1,4 +1,4 @@
-from server import display_version as cellxgene_display_version
+from backend.czi_hosted import display_version as cellxgene_display_version
 
 
 def get_client_config(app_config, data_adaptor):
@@ -40,13 +40,12 @@ def get_client_config(app_config, data_adaptor):
         "diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff,
         "backed": server_config.adaptor__anndata_adaptor__backed,
         "disable-diffexp": not dataset_config.diffexp__enable,
-        "enable-reembedding": dataset_config.embeddings__enable_reembedding,
         "annotations": False,
         "annotations_file": None,
         "annotations_dir": None,
-        "annotations_cell_ontology_enabled": False,
+        "annotations_genesets": True,  # feature flag
-        "annotations_cell_ontology_obopath": None,
+        "annotations_genesets_readonly": True,
-        "annotations_cell_ontology_terms": None,
+        "annotations_genesets_summary_methods": ["mean"],
         "custom_colors": dataset_config.presentation__custom_colors,
         "diffexp-may-be-slow": False,
         "about_legal_tos": dataset_config.app__about_legal_tos,

backend/czi_hosted/common/config/dataset_config.py

@@ -1,13 +1,12 @@
 import os
 from os.path import splitext, isdir
 
-from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
+from backend.czi_hosted.common.annotations.annotations import Annotations
-from server.common.annotations.local_file_csv import AnnotationsLocalFile
+from backend.czi_hosted.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
-from server.common.config.base_config import BaseConfig
+from backend.czi_hosted.common.annotations.local_file_csv import AnnotationsLocalFile
-from server.common.errors import ConfigurationError, OntologyLoadFailure
+from backend.czi_hosted.common.config.base_config import BaseConfig
-from server.compute.scanpy import get_scanpy_module
+from backend.common.errors import ConfigurationError
-from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataType
+from backend.czi_hosted.db.db_utils import DbUtils
-from server.db.db_utils import DbUtils
 
 
 class DatasetConfig(BaseConfig):
@@ -32,10 +31,6 @@ class DatasetConfig(BaseConfig):
                 "directory"
             ]
             self.user_annotations__local_file_csv__file = default_config["user_annotations"]["local_file_csv"]["file"]
-            self.user_annotations__ontology__enable = default_config["user_annotations"]["ontology"]["enable"]
-            self.user_annotations__ontology__obo_location = default_config["user_annotations"]["ontology"][
-                "obo_location"
-            ]
             self.user_annotations__hosted_tiledb_array__db_uri = default_config["user_annotations"][
                 "hosted_tiledb_array"
             ]["db_uri"]
@@ -44,17 +39,20 @@ class DatasetConfig(BaseConfig):
             ]["hosted_file_directory"]
 
             self.embeddings__names = default_config["embeddings"]["names"]
-            self.embeddings__enable_reembedding = default_config["embeddings"]["enable_reembedding"]
 
             self.diffexp__enable = default_config["diffexp"]["enable"]
             self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
             self.diffexp__top_n = default_config["diffexp"]["top_n"]
 
+            self.X_approximate_distribution = default_config["X_approximate_distribution"]
+
         except KeyError as e:
             raise ConfigurationError(f"Unexpected config: {str(e)}")
 
-        # The annotation object is created during complete_config and stored here.
+        # Create the default annotation, which supports gene set reading without
-        self.user_annotations = None
+        # further configuration. Depending on configuration options, `complete_config`
+        # may create a more specialized annotation object and replace this default.
+        self.user_annotations = Annotations()
 
     def complete_config(self, context):
         self.handle_app()
@@ -62,6 +60,7 @@ class DatasetConfig(BaseConfig):
         self.handle_user_annotations(context)
         self.handle_embeddings()
         self.handle_diffexp(context)
+        self.handle_X_approximate_distribution()
 
     def handle_app(self):
         self.validate_correct_type_of_configuration_attribute("app__scripts", list)
@@ -98,10 +97,6 @@ class DatasetConfig(BaseConfig):
         self.validate_correct_type_of_configuration_attribute(
             "user_annotations__local_file_csv__file", (type(None), str)
         )
-        self.validate_correct_type_of_configuration_attribute("user_annotations__ontology__enable", bool)
-        self.validate_correct_type_of_configuration_attribute(
-            "user_annotations__ontology__obo_location", (type(None), str)
-        )
         self.validate_correct_type_of_configuration_attribute(
             "user_annotations__hosted_tiledb_array__db_uri", (type(None), str)
         )
@@ -122,11 +117,6 @@ class DatasetConfig(BaseConfig):
                 self.handle_hosted_tiledb_annotations()
             else:
                 raise ConfigurationError('The only annotation type support is "local_file_csv" or "hosted_tiledb_array')
-            if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location:
-                try:
-                    self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location)
-                except OntologyLoadFailure as e:
-                    raise ConfigurationError("Unable to load ontology terms\n" + str(e))
         else:
             self.check_annotation_config_vars_not_set(context)
 
@@ -147,7 +137,11 @@ class DatasetConfig(BaseConfig):
             except OSError:
                 raise ConfigurationError("Unable to create directory specified by --annotations-dir")
 
-        self.user_annotations = AnnotationsLocalFile(dirname, filename)
+        anno_config = {
+            "user-annotations": self.user_annotations__enable,
+            "genesets-save": False,
+        }
+        self.user_annotations = AnnotationsLocalFile(anno_config, dirname, filename)
 
         # if the user has specified a fixed label file, go ahead and validate it
         # so that we can remove errors early in the process.
@@ -163,7 +157,12 @@ class DatasetConfig(BaseConfig):
         self.validate_correct_type_of_configuration_attribute(
             "user_annotations__hosted_tiledb_array__hosted_file_directory", str
         )
+        anno_config = {
+            "user-annotations": self.user_annotations__enable,
+            "genesets-save": False,
+        }
         self.user_annotations = AnnotationsHostedTileDB(
+            anno_config,
             directory_path=self.user_annotations__hosted_tiledb_array__hosted_file_directory,
             db=DbUtils(self.user_annotations__hosted_tiledb_array__db_uri),
         )
@@ -185,33 +184,8 @@ class DatasetConfig(BaseConfig):
                     "Warning: hosted_file_directory for hosted_tiledb_array ignored as annotations are disabled."
                 )
 
-        if self.user_annotations__ontology__enable:
-            context["messagefn"]("Warning: --experimental-annotations-ontology ignored as annotations are disabled.")
-        if self.user_annotations__ontology__obo_location is not None:
-            context["messagefn"](
-                "Warning: --experimental-annotations-ontology-obo ignored as annotations are disabled."
-            )
-
     def handle_embeddings(self):
         self.validate_correct_type_of_configuration_attribute("embeddings__names", list)
-        self.validate_correct_type_of_configuration_attribute("embeddings__enable_reembedding", bool)
-
-        server_config = self.app_config.server_config
-        if self.embeddings__enable_reembedding:
-            if server_config.single_dataset__datapath:
-                matrix_data_loader = MatrixDataLoader(
-                    server_config.single_dataset__datapath, app_config=self.app_config
-                )
-                if matrix_data_loader.matrix_data_type != MatrixDataType.H5AD:
-                    raise ConfigurationError("enable-reembedding is only supported with H5AD files.")
-                if server_config.adaptor__anndata_adaptor__backed:
-                    raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.")
-
-            try:
-                get_scanpy_module()
-            except NotImplementedError:
-                # Todo add scanpy to requirements.txt and remove this check once re-embeddings is fully supported
-                raise ConfigurationError("Please install scanpy to enable UMAP re-embedding")
 
     def handle_diffexp(self, context):
         self.validate_correct_type_of_configuration_attribute("diffexp__enable", bool)
@@ -228,3 +202,10 @@ class DatasetConfig(BaseConfig):
                         "CAUTION: due to the size of your dataset, "
                         "running differential expression may take longer or fail."
                     )
+
+    def handle_X_approximate_distribution(self):
+        self.validate_correct_type_of_configuration_attribute("X_approximate_distribution", str)
+        if self.X_approximate_distribution not in ["normal", "count"]:
+            raise ConfigurationError(
+                "X_approximate_distribution has unknown value -- must be 'normal' or 'count'."
+            )

backend/czi_hosted/common/config/external_config.py

@@ -1,10 +1,9 @@
 import os
 
-from server.common.config.base_config import BaseConfig
+from backend.czi_hosted.common.config.base_config import BaseConfig
-from server.common.errors import ConfigurationError
+from backend.common.errors import ConfigurationError, SecretKeyRetrievalError
-from server.common.config import get_secret_key
+from backend.common.utils.aws_secret_utils import get_secret_key
-from server.common.errors import SecretKeyRetrievalError
+from backend.common.utils.type_conversion_utils import convert_string_to_value
-from server.common.utils.type_conversion_utils import convert_string_to_value
 
 
 class ExternalConfig(BaseConfig):

backend/czi_hosted/common/config/server_config.py

@@ -4,14 +4,14 @@ import warnings
 from os.path import basename
 from urllib.parse import urlparse, quote_plus
 
-from server.auth.auth import AuthTypeFactory
+from backend.czi_hosted.auth.auth import AuthTypeFactory
-from server.common.config.base_config import BaseConfig
+from backend.czi_hosted.common.config import DEFAULT_SERVER_PORT, BIG_FILE_SIZE_THRESHOLD
-from server.common.config import DEFAULT_SERVER_PORT, BIG_FILE_SIZE_THRESHOLD
+from backend.czi_hosted.common.config.base_config import BaseConfig
-from server.common.errors import ConfigurationError, DatasetAccessError
+from backend.common.utils.data_locator import discover_s3_region_name
-from server.common.data_locator import discover_s3_region_name
+from backend.common.errors import ConfigurationError, DatasetAccessError
-from server.common.utils.utils import is_port_available, find_available_port, custom_format_warning
+from backend.common.utils.utils import is_port_available, find_available_port, custom_format_warning
-from server.compute import diffexp_cxg as diffexp_tiledb
+from backend.czi_hosted.compute import diffexp_cxg as diffexp_tiledb
-from server.data_common.matrix_loader import MatrixDataCacheManager, MatrixDataLoader, MatrixDataType
+from backend.czi_hosted.data_common.matrix_loader import MatrixDataCacheManager, MatrixDataLoader, MatrixDataType
 
 
 class ServerConfig(BaseConfig):
@@ -42,6 +42,9 @@ class ServerConfig(BaseConfig):
             self.app__web_base_url = default_config["app"]["web_base_url"]
 
             self.authentication__type = default_config["authentication"]["type"]
+            self.authentication__insecure_test_environment = default_config["authentication"][
+                "insecure_test_environment"
+            ]
             self.authentication__params_oauth__oauth_api_base_url = default_config["authentication"]["params_oauth"][
                 "oauth_api_base_url"
             ]
@@ -168,6 +171,10 @@ class ServerConfig(BaseConfig):
 
     def handle_authentication(self):
         self.validate_correct_type_of_configuration_attribute("authentication__type", (type(None), str))
+        self.validate_correct_type_of_configuration_attribute("authentication__insecure_test_environment", bool)
+
+        if self.authentication__type == "test" and not self.authentication__insecure_test_environment:
+            raise ConfigurationError("Test auth can only be used in an insecure test environment")
 
         # oauth
         ptypes = str if self.authentication__type == "oauth" else (type(None), str)
@@ -346,7 +353,7 @@ class ServerConfig(BaseConfig):
             if type(self.data_locator__s3__region_name) == str:
                 self.adaptor__cxg_adaptor__tiledb_ctx[regionkey] = self.data_locator__s3__region_name
 
-        from server.data_cxg.cxg_adaptor import CxgAdaptor
+        from backend.czi_hosted.data_cxg.cxg_adaptor import CxgAdaptor
 
         CxgAdaptor.set_tiledb_context(self.adaptor__cxg_adaptor__tiledb_ctx)
 

backend/czi_hosted/common/corpora.py

@@ -0,0 +1,78 @@
+"""
+Corpora schema conventions support.  Helper functions for reading.
+
+https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md
+
+https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema_h5ad_implementation.md
+"""
+import collections
+import json
+
+from backend.czi_hosted.cli.upgrade import validate_version_str
+from backend.czi_hosted.common.utils.corpora_constants import CorporaConstants
+
+
+def corpora_get_versions_from_anndata(adata):
+    """
+    Given an AnnData object, return:
+        * None - if not a Corpora object
+        * [ corpora_schema_version, corpora_encoding_version ] - if a Corpora object
+
+    Implements the identification protocol defined in the specification.
+    """
+
+    # per Corpora AnnData spec, this is a corpora file if the following is true
+    if "version" not in adata.uns_keys():
+        return None
+    version = adata.uns["version"]
+    if not isinstance(version, collections.abc.Mapping) or "corpora_schema_version" not in version:
+        return None
+
+    corpora_schema_version = version.get("corpora_schema_version")
+    corpora_encoding_version = version.get("corpora_encoding_version")
+
+    # TODO: spec says these must be SEMVER values, so check.
+    if validate_version_str(corpora_schema_version) and validate_version_str(corpora_encoding_version):
+        return [corpora_schema_version, corpora_encoding_version]
+
+
+def corpora_is_version_supported(corpora_schema_version, corpora_encoding_version):
+    return (
+        corpora_schema_version
+        and corpora_encoding_version
+        and corpora_schema_version.startswith("1.")
+        and corpora_encoding_version.startswith("0.1.")
+    )
+
+
+def corpora_get_props_from_anndata(adata):
+    """
+    Get Corpora dataset properties from an AnnData
+    """
+    versions = corpora_get_versions_from_anndata(adata)
+    if versions is None:
+        return None
+    [corpora_schema_version, corpora_encoding_version] = versions
+    version_is_supported = corpora_is_version_supported(corpora_schema_version, corpora_encoding_version)
+    if not version_is_supported:
+        raise ValueError("Unsupported Corpora schema version")
+
+    corpora_props = {}
+    for key in CorporaConstants.REQUIRED_SIMPLE_METADATA_FIELDS:
+        if key not in adata.uns:
+            raise KeyError(f"missing Corpora schema field {key}")
+        corpora_props[key] = adata.uns[key]
+
+    for key in CorporaConstants.OPTIONAL_JSON_ENCODED_METADATA_FIELD:
+        if key not in adata.uns:
+            continue
+        try:
+            corpora_props[key] = json.loads(adata.uns[key])
+        except json.JSONDecodeError:
+            raise json.JSONDecodeError(f"Corpora schema field {key} is expected to be a valid JSON string")
+
+    for key in CorporaConstants.OPTIONAL_SIMPLE_METADATA_FIELDS:
+        if key in adata.uns:
+            corpora_props[key] = adata.uns[key]
+
+    return corpora_props

backend/czi_hosted/common/health.py

@@ -1,8 +1,8 @@
 from http import HTTPStatus
 from flask import make_response, jsonify
 
-from server import __version__ as cellxgene_version
+from backend.czi_hosted import __version__ as cellxgene_version
-from server.common.data_locator import DataLocator
+from backend.common.utils.data_locator import DataLocator
 
 
 def _is_accessible(path, config):

backend/czi_hosted/common/rest.py

@@ -3,13 +3,14 @@ import logging
 import sys
 from http import HTTPStatus
 import zlib
+import json
 
 from flask import make_response, jsonify, current_app, abort
 from werkzeug.urls import url_unquote
 
-from server.common.config.client_config import get_client_config, get_client_userinfo
+from backend.czi_hosted.common.config.client_config import get_client_config, get_client_userinfo
-from server.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg
+from backend.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg
-from server.common.errors import (
+from backend.common.errors import (
     FilterError,
     JSONEncodingValueError,
     PrepareError,
@@ -17,10 +18,11 @@ from server.common.errors import (
     ExceedsLimitError,
     DatasetAccessError,
     ColorFormatException,
+    AnnotationsError,
+    UnsupportedSummaryMethod,
 )
-
+from backend.common.genesets import summarizeQueryHash
-import json
+from backend.common.fbs.matrix import decode_matrix_fbs
-from server.data_common.fbs.matrix import decode_matrix_fbs
 
 
 def abort_and_log(code, logmsg, loglevel=logging.DEBUG, include_exc_info=False):
@@ -106,7 +108,7 @@ def schema_get_helper(data_adaptor):
 
     # add label obs annotations as needed
     annotations = data_adaptor.dataset_config.user_annotations
-    if annotations is not None:
+    if annotations.user_annotations_enabled():
         label_schema = annotations.get_schema(data_adaptor)
         schema["annotations"]["obs"]["columns"].extend(label_schema)
 
@@ -140,7 +142,7 @@ def annotations_obs_get(request, data_adaptor):
     try:
         labels = None
         annotations = data_adaptor.dataset_config.user_annotations
-        if annotations:
+        if annotations.user_annotations_enabled():
             labels = annotations.read_labels(data_adaptor)
         fbs = data_adaptor.annotation_to_fbs_matrix(Axis.OBS, fields, labels)
         return make_response(fbs, HTTPStatus.OK, {"Content-Type": "application/octet-stream"})
@@ -151,7 +153,7 @@ def annotations_obs_get(request, data_adaptor):
 def annotations_put_fbs_helper(data_adaptor, fbs):
     """helper function to write annotations from fbs"""
     annotations = data_adaptor.dataset_config.user_annotations
-    if annotations is None:
+    if not annotations.user_annotations_enabled():
         raise DisabledFeatureError("Writable annotations are not enabled")
 
     new_label_df = decode_matrix_fbs(fbs)
@@ -166,7 +168,7 @@ def inflate(data):
 
 def annotations_obs_put(request, data_adaptor):
     annotations = data_adaptor.dataset_config.user_annotations
-    if annotations is None:
+    if not annotations.user_annotations_enabled():
         return abort(HTTPStatus.NOT_IMPLEMENTED)
 
     anno_collection = request.args.get("annotation-collection-name", default=None)
@@ -197,7 +199,7 @@ def annotations_var_get(request, data_adaptor):
     try:
         labels = None
         annotations = data_adaptor.dataset_config.user_annotations
-        if annotations is not None:
+        if annotations.user_annotations_enabled():
             labels = annotations.read_labels(data_adaptor)
         return make_response(
             data_adaptor.annotation_to_fbs_matrix(Axis.VAR, fields, labels),
@@ -258,13 +260,14 @@ def diffexp_obs_post(request, data_adaptor):
     try:
         # TODO: implement varfilter mode
         mode = DiffExpMode(args["mode"])
-
         if mode == DiffExpMode.VAR_FILTER or "varFilter" in args:
             return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, "varFilter not enabled")
 
         set1_filter = args.get("set1", {"filter": {}})["filter"]
         set2_filter = args.get("set2", {"filter": {}})["filter"]
-        count = args.get("count", None)
+        # TODO(#1281): When we simplify the config, we should actually use the config to determine this number,
+        #  this will also require an update in the client
+        count = 15
 
         if set1_filter is None or set2_filter is None or count is None:
             return abort_and_log(HTTPStatus.BAD_REQUEST, "missing required parameter")
@@ -311,20 +314,68 @@ def layout_obs_get(request, data_adaptor):
         )
 
 
-def layout_obs_put(request, data_adaptor):
+def genesets_get(request, data_adaptor):
-    if not data_adaptor.dataset_config.embeddings__enable_reembedding:
+    preferred_mimetype = request.accept_mimetypes.best_match(["application/json", "text/csv"])
-        return abort(HTTPStatus.NOT_IMPLEMENTED)
+    if preferred_mimetype not in ("application/json", "text/csv"):
-
+        return abort(HTTPStatus.NOT_ACCEPTABLE)
-    args = request.get_json()
-    filter = args["filter"] if args else None
-    if not filter:
-        return abort_and_log(HTTPStatus.BAD_REQUEST, "obs filter is required")
-    method = args["method"] if args else "umap"
 
     try:
-        schema = data_adaptor.compute_embedding(method, filter)
+        annotations = data_adaptor.dataset_config.user_annotations
-        return make_response(jsonify(schema), HTTPStatus.OK, {"Content-Type": "application/json"})
+        (genesets, tid) = annotations.read_gene_sets(data_adaptor)
-    except NotImplementedError as e:
+
-        return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, str(e))
+        if preferred_mimetype == "text/csv":
-    except (ValueError, DisabledFeatureError, FilterError) as e:
+            return make_response(
-        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+                annotations.gene_sets_to_csv(genesets),
+                HTTPStatus.OK,
+                {
+                    "Content-Type": "text/csv",
+                    "Content-Disposition": "attachment; filename=genesets.csv",
+                },
+            )
+        else:
+            return make_response(
+                jsonify({"genesets": annotations.gene_sets_to_response(genesets), "tid": tid}), HTTPStatus.OK
+            )
+    except (ValueError, KeyError, AnnotationsError) as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e))
+
+
+def summarize_var_helper(request, data_adaptor, key, raw_query):
+    preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
+    if preferred_mimetype != "application/octet-stream":
+        return abort(HTTPStatus.NOT_ACCEPTABLE)
+
+    summary_method = request.values.get("method", default="mean")
+    query_hash = summarizeQueryHash(raw_query)
+    if key and query_hash != key:
+        return abort(HTTPStatus.BAD_REQUEST, description="query key did not match")
+
+    args_filter_only = request.values.copy()
+    args_filter_only.poplist("method")
+    args_filter_only.poplist("key")
+
+    try:
+        filter = _query_parameter_to_filter(args_filter_only)
+        return make_response(
+            data_adaptor.summarize_var(summary_method, filter, query_hash),
+            HTTPStatus.OK,
+            {"Content-Type": "application/octet-stream"},
+        )
+    except (ValueError) as e:
+        return abort(HTTPStatus.NOT_FOUND, description=str(e))
+    except (UnsupportedSummaryMethod, FilterError) as e:
+        return abort(HTTPStatus.BAD_REQUEST, description=str(e))
+
+
+def summarize_var_get(request, data_adaptor):
+    return summarize_var_helper(request, data_adaptor, None, request.query_string)
+
+
+def summarize_var_post(request, data_adaptor):
+    if not request.content_type or "application/x-www-form-urlencoded" not in request.content_type:
+        return abort(HTTPStatus.UNSUPPORTED_MEDIA_TYPE)
+    if request.content_length > 1_000_000:  # just a sanity check to avoid memory exhaustion
+        return abort(HTTPStatus.BAD_REQUEST)
+
+    key = request.args.get("key", default=None)
+    return summarize_var_helper(request, data_adaptor, key, request.get_data())

backend/czi_hosted/common/utils/cxg_generation_utils.py

@@ -3,7 +3,7 @@ import json
 import numpy as np
 import tiledb
 
-from server.common.utils.type_conversion_utils import get_dtype_of_array, get_dtype_and_schema_of_array
+from backend.common.utils.type_conversion_utils import get_encoding_dtype_of_array, get_dtype_and_schema_of_array
 
 
 def convert_dictionary_to_cxg_group(cxg_container, metadata_dict, group_metadata_name="cxg_group_metadata"):
@@ -47,7 +47,7 @@ def convert_dataframe_to_cxg_array(cxg_container, dataframe_name, dataframe, ind
             ]
         )
         attrs = [
-            tiledb.Attr(name=column, dtype=get_dtype_of_array(dataframe[column]), filters=tiledb_filter)
+            tiledb.Attr(name=column, dtype=get_encoding_dtype_of_array(dataframe[column]), filters=tiledb_filter)
             for column in dataframe
         ]
         domain = tiledb.Domain(

backend/czi_hosted/compute/diffexp_cxg.py

@@ -1,10 +1,11 @@
 import concurrent.futures
 import numpy as np
-from server.compute.diffexp_generic import diffexp_ttest_from_mean_var, mean_var_n
+
-from server.data_cxg.cxg_util import pack_selector_from_indices
-from server.common.errors import ComputeError
 from numba import jit
 
+from backend.czi_hosted.data_cxg.cxg_util import pack_selector_from_indices
+from backend.common.compute.diffexp_generic import diffexp_ttest_from_mean_var, mean_var_n
+from backend.common.errors import ComputeError
 
 """
 See the comments in diffexp_generic for a description of this algorithm
@@ -114,14 +115,14 @@ def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
             meanB += X_col_shift
 
     r = diffexp_ttest_from_mean_var(
-        meanA.astype(dtype),
+        meanA=meanA.astype(dtype),
-        varA.astype(dtype),
+        varA=varA.astype(dtype),
-        nA,
+        nA=nA,
-        meanB.astype(dtype),
+        meanB=meanB.astype(dtype),
-        varB.astype(dtype),
+        varB=varB.astype(dtype),
-        nB,
+        nB=nB,
-        top_n,
+        top_n=top_n,
-        diffexp_lfc_cutoff,
+        diffexp_lfc_cutoff=diffexp_lfc_cutoff
     )
 
     return r

backend/czi_hosted/converters/h5ad_data_file.py

@@ -6,17 +6,17 @@ import anndata
 import numpy as np
 import tiledb
 
-from server.common.colors import convert_anndata_category_colors_to_cxg_category_colors
+from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
-from server.common.corpora import corpora_get_props_from_anndata
+from backend.czi_hosted.common.corpora import corpora_get_props_from_anndata
-from server.common.errors import ColorFormatException
+from backend.common.errors import ColorFormatException
-from server.common.utils.cxg_constants import CxgConstants
+from backend.czi_hosted.common.utils.cxg_constants import CxgConstants
-from server.common.utils.cxg_generation_utils import (
+from backend.czi_hosted.common.utils.cxg_generation_utils import (
     convert_dictionary_to_cxg_group,
     convert_dataframe_to_cxg_array,
     convert_ndarray_to_cxg_dense_array,
     convert_matrix_to_cxg_array,
 )
-from server.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix
+from backend.czi_hosted.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix
 
 
 class H5ADDataFile:

backend/czi_hosted/converters/to_sparse.py

@@ -9,8 +9,9 @@ import sys
 
 import tiledb
 
-from server.common.utils.cxg_generation_utils import convert_ndarray_to_cxg_dense_array, convert_matrix_to_cxg_array
+from backend.czi_hosted.common.utils.cxg_generation_utils import convert_ndarray_to_cxg_dense_array, \
-from server.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix
+    convert_matrix_to_cxg_array
+from backend.czi_hosted.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix
 
 
 def main():

backend/czi_hosted/data_anndata/anndata_adaptor.py

@@ -1,22 +1,19 @@
 import warnings
-from datetime import datetime
 
 import anndata
 import numpy as np
 from packaging import version
 from pandas.core.dtypes.dtypes import CategoricalDtype
 from scipy import sparse
-from server_timing import Timing as ServerTiming
 
-import server.compute.diffexp_generic as diffexp_generic
+import backend.common.compute.diffexp_generic as diffexp_generic
-from server.common.colors import convert_anndata_category_colors_to_cxg_category_colors
+from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
-from server.common.constants import Axis, MAX_LAYOUTS
+from backend.common.constants import Axis, MAX_LAYOUTS, XApproximateDistribution
-from server.common.corpora import corpora_get_props_from_anndata
+from backend.czi_hosted.common.corpora import corpora_get_props_from_anndata
-from server.common.errors import PrepareError, DatasetAccessError, FilterError
+from backend.common.errors import PrepareError, DatasetAccessError, ConfigurationError
-from server.common.utils.type_conversion_utils import get_schema_type_hint_of_array
+from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
-from server.compute.scanpy import scanpy_umap
+from backend.czi_hosted.data_common.data_adaptor import DataAdaptor
-from server.data_common.data_adaptor import DataAdaptor
+from backend.common.fbs.matrix import encode_matrix_fbs
-from server.data_common.fbs.matrix import encode_matrix_fbs
 
 anndata_version = version.parse(str(anndata.__version__)).release
 
@@ -31,6 +28,7 @@ class AnndataAdaptor(DataAdaptor):
     def __init__(self, data_locator, app_config=None, dataset_config=None):
         super().__init__(data_locator, app_config, dataset_config)
         self.data = None
+        self.X_approximate_distribution = None
         self._load_data(data_locator)
         self._validate_and_initialize()
 
@@ -68,11 +66,11 @@ class AnndataAdaptor(DataAdaptor):
 
     @staticmethod
     def _create_unique_column_name(df, col_name_prefix):
-        """ given the columns of a dataframe, and a name prefix, return a column name which
+        """given the columns of a dataframe, and a name prefix, return a column name which
-            does not exist in the dataframe, AND which is prefixed by `prefix`
+        does not exist in the dataframe, AND which is prefixed by `prefix`
 
-            The approach is to append a numeric suffix, starting at zero and increasing by
+        The approach is to append a numeric suffix, starting at zero and increasing by
-            one, until an unused name is found (eg, prefix_0, prefix_1, ...).
+        one, until an unused name is found (eg, prefix_0, prefix_1, ...).
         """
         suffix = 0
         while f"{col_name_prefix}{suffix}" in df:
@@ -126,7 +124,11 @@ class AnndataAdaptor(DataAdaptor):
 
     def _create_schema(self):
         self.schema = {
-            "dataframe": {"nObs": self.cell_count, "nVar": self.gene_count, "type": str(self.data.X.dtype)},
+            "dataframe": {
+                "nObs": self.cell_count,
+                "nVar": self.gene_count,
+                **get_schema_type_hint_of_array(self.data.X),
+            },
             "annotations": {
                 "obs": {"index": self.parameters.get("obs_names"), "columns": []},
                 "var": {"index": self.parameters.get("var_names"), "columns": []},
@@ -193,16 +195,20 @@ class AnndataAdaptor(DataAdaptor):
         self.gene_count = self.data.shape[1]
         self._create_schema()
 
+        if self.dataset_config.X_approximate_distribution == "auto":
+            raise ConfigurationError("X-approximate-distribution 'auto' mode unsupported.")
+        self.X_approximate_distribution = self.dataset_config.X_approximate_distribution
+
         # heuristic
         n_values = self.data.shape[0] * self.data.shape[1]
         if (n_values > 1e8 and self.server_config.adaptor__anndata_adaptor__backed is True) or (n_values > 5e8):
             self.parameters.update({"diffexp_may_be_slow": True})
 
     def _is_valid_layout(self, arr):
-        """ return True if this layout data is a valid array for front-end presentation:
+        """return True if this layout data is a valid array for front-end presentation:
-            * ndarray, dtype float/int/uint
+        * ndarray, dtype float/int/uint
-            * with shape (n_obs, >= 2)
+        * with shape (n_obs, >= 2)
-            * with all values finite or NaN (no +Inf or -Inf)
+        * with all values finite or NaN (no +Inf or -Inf)
         """
         is_valid = type(arr) == np.ndarray and arr.dtype.kind in "fiu"
         is_valid = is_valid and arr.shape[0] == self.data.n_obs and arr.shape[1] >= 2
@@ -301,28 +307,6 @@ class AnndataAdaptor(DataAdaptor):
         full_embedding = self.data.obsm[f"X_{ename}"]
         return full_embedding[:, 0:dims]
 
-    def compute_embedding(self, method, obsFilter):
-        if Axis.VAR in obsFilter:
-            raise FilterError("Observation filters may not contain variable conditions")
-        if method != "umap":
-            raise NotImplementedError(f"re-embedding method {method} is not available.")
-        try:
-            shape = self.get_shape()
-            obs_mask = self._axis_filter_to_mask(Axis.OBS, obsFilter["obs"], shape[0])
-        except (KeyError, IndexError):
-            raise FilterError("Error parsing filter")
-        with ServerTiming.time("layout.compute"):
-            X_umap = scanpy_umap(self.data, obs_mask)
-
-        # Server picks reemedding name, which must not collide with any other
-        # embedding name generated by this backend.
-        name = f"reembed:{method}_{datetime.now().isoformat(timespec='milliseconds')}"
-        dims = [f"{name}_0", f"{name}_1"]
-        layout_schema = {"name": name, "type": "float32", "dims": dims}
-        self.schema["layout"]["obs"].append(layout_schema)
-        self.data.obsm[f"X_{name}"] = X_umap
-        return layout_schema
-
     def compute_diffexp_ttest(self, maskA, maskB, top_n=None, lfc_cutoff=None):
         if top_n is None:
             top_n = self.dataset_config.diffexp__top_n
@@ -334,13 +318,22 @@ class AnndataAdaptor(DataAdaptor):
         return convert_anndata_category_colors_to_cxg_category_colors(self.data)
 
     def get_X_array(self, obs_mask=None, var_mask=None):
+        # H5Py does not support boolean indexing (masks), so convert to integer indexing
+        # when backed (ie, when AnnData is using H5Py indexing)
         if obs_mask is None:
             obs_mask = slice(None)
+        elif self.data.isbacked and obs_mask.dtype == bool:
+            obs_mask = obs_mask.nonzero()[0]
         if var_mask is None:
             var_mask = slice(None)
+        elif self.data.isbacked and var_mask.dtype == bool:
+            var_mask = var_mask.nonzero()[0]
         X = self.data.X[obs_mask, var_mask]
         return X
 
+    def get_X_approximate_distribution(self) -> XApproximateDistribution:
+        return self.X_approximate_distribution
+
     def get_shape(self):
         return self.data.shape
 

backend/czi_hosted/data_common/data_adaptor.py

@@ -3,13 +3,20 @@ from os.path import basename, splitext
 
 import numpy as np
 import pandas as pd
+from scipy import sparse
 from server_timing import Timing as ServerTiming
 
-from server.common.config.app_config import AppConfig
+from backend.czi_hosted.common.config.app_config import AppConfig
-from server.common.constants import Axis
+from backend.common.constants import Axis, XApproximateDistribution
-from server.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError
+from backend.common.errors import (
-from server.common.utils.utils import jsonify_numpy
+    FilterError,
-from server.data_common.fbs.matrix import encode_matrix_fbs
+    JSONEncodingValueError,
+    ExceedsLimitError,
+    UnsupportedSummaryMethod,
+    DatasetAccessError,
+)
+from backend.common.utils.utils import jsonify_strict
+from backend.common.fbs.matrix import encode_matrix_fbs
 
 
 class DataAdaptor(metaclass=ABCMeta):
@@ -70,17 +77,17 @@ class DataAdaptor(metaclass=ABCMeta):
         """return an numpy array for the given pre-computed embedding name."""
         pass
 
-    @abstractmethod
-    def compute_embedding(self, method, filter):
-        """compute a new embedding on the specified obs subset, and return the embedding schema. """
-        pass
-
     @abstractmethod
     def get_X_array(self, obs_mask=None, var_mask=None):
         """return the X array, possibly filtered by obs_mask or var_mask.
         the return type is either ndarray or scipy.sparse.spmatrix."""
         pass
 
+    @abstractmethod
+    def get_X_approximate_distribution(self) -> XApproximateDistribution:
+        """return the approximate distribution of the X matrix."""
+        pass
+
     @abstractmethod
     def get_shape(self):
         pass
@@ -282,7 +289,7 @@ class DataAdaptor(metaclass=ABCMeta):
 
         try:
             obs_selector, var_selector = self._filter_to_mask(filter)
-        except (KeyError, IndexError, TypeError, AttributeError):
+        except (KeyError, IndexError, TypeError, AttributeError, DatasetAccessError):
             raise FilterError("Error parsing filter")
 
         if obs_selector is not None:
@@ -324,10 +331,12 @@ class DataAdaptor(metaclass=ABCMeta):
         ):
             raise ExceedsLimitError("Diffexp request exceeds max cell count limit")
 
-        result = self.compute_diffexp_ttest(obs_mask_A, obs_mask_B, top_n, self.dataset_config.diffexp__lfc_cutoff)
+        result = self.compute_diffexp_ttest(
+            maskA=obs_mask_A, maskB=obs_mask_B, top_n=top_n, lfc_cutoff=self.dataset_config.diffexp__lfc_cutoff
+        )
 
         try:
-            return jsonify_numpy(result)
+            return jsonify_strict(result)
         except ValueError:
             raise JSONEncodingValueError("Error encoding differential expression to JSON")
 
@@ -338,7 +347,7 @@ class DataAdaptor(metaclass=ABCMeta):
     @staticmethod
     def normalize_embedding(embedding):
         """Normalize embedding layout to meet client assumptions.
-           Embedding is an ndarray, shape (n_obs, n)., where n is normally 2
+        Embedding is an ndarray, shape (n_obs, n)., where n is normally 2
         """
 
         # scale isotropically
@@ -394,3 +403,26 @@ class DataAdaptor(metaclass=ABCMeta):
         except RuntimeError:
             lastmod = None
         return lastmod
+
+    def summarize_var(self, method, filter, query_hash):
+        if method != "mean":
+            raise UnsupportedSummaryMethod("Unknown gene set summary method.")
+
+        obs_selector, var_selector = self._filter_to_mask(filter)
+        if obs_selector is not None:
+            raise FilterError("filtering on obs unsupported")
+
+        # if no filter, just return zeros.  We don't have a use case
+        # for summarizing the entire X without a filter, and it would
+        # potentially be quite compute / memory intensive.
+        if var_selector is None or np.count_nonzero(var_selector) == 0:
+            mean = np.zeros((self.get_shape()[0], 1), dtype=np.float32)
+        else:
+            X = self.get_X_array(obs_selector, var_selector)
+            if sparse.issparse(X):
+                mean = X.mean(axis=1).A
+            else:
+                mean = X.mean(axis=1, keepdims=True)
+
+        col_idx = pd.Index([query_hash])
+        return encode_matrix_fbs(mean, col_idx=col_idx, row_idx=None)

backend/czi_hosted/data_common/matrix_loader.py

@@ -1,12 +1,14 @@
 from enum import Enum
 import threading
 import time
-from server.data_common.rwlock import RWLock
+
-from server.common.errors import DatasetAccessError
+from backend.common.utils.data_locator import DataLocator
-from server.common.data_locator import DataLocator
+from backend.common.errors import DatasetAccessError
 from contextlib import contextmanager
 from http import HTTPStatus
 
+from backend.czi_hosted.data_common.rwlock import RWLock
+
 
 class MatrixDataCacheItem(object):
     """This class provides access and caching for a dataset.  The first time a dataset is accessed, it is
@@ -224,7 +226,7 @@ class MatrixDataLoader(object):
 
         # matrix_data_type is an enum value of type MatrixDataType
         self.matrix_data_type = matrix_data_type
-        # matrix_type is a DataAdaptor type, which corresonds to the matrix_data_type
+        # matrix_type is a DataAdaptor type, which corresponds to the matrix_data_type
         self.matrix_type = None
 
         if matrix_data_type is None:
@@ -234,11 +236,11 @@ class MatrixDataLoader(object):
             raise DatasetAccessError("Dataset does not have an allowed type.")
 
         if self.matrix_data_type == MatrixDataType.H5AD:
-            from server.data_anndata.anndata_adaptor import AnndataAdaptor
+            from backend.czi_hosted.data_anndata.anndata_adaptor import AnndataAdaptor
 
             self.matrix_type = AnndataAdaptor
         elif self.matrix_data_type == MatrixDataType.CXG:
-            from server.data_cxg.cxg_adaptor import CxgAdaptor
+            from backend.czi_hosted.data_cxg.cxg_adaptor import CxgAdaptor
 
             self.matrix_type = CxgAdaptor