docs: release 0.17.0 (#2278)

* Bump version: 0.16.7 → 0.17.0-rc.0

* Bump version: 0.17.0-rc.0 → 0.17.0

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.16.7
+current_version = 0.17.0
 commit = True
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?:-(?P<prerel>rc)\.(?P<prerelversion>\d+))?
 serialize = 
@@ -20,6 +20,6 @@ replace = version="{new_version}"
 search = "version": "{current_version}"
 replace = "version": "{new_version}"
 
-[bumpversion:file:server/__init__.py]
+[bumpversion:file:backend/server/__init__.py]
 search = __version__ = "{current_version}"
 replace = __version__ = "{new_version}"

.dockerignore

@@ -2,4 +2,4 @@ bin
 client
 dist
 docs
-server
+backend

.github/workflows/canary-deploy.yml

@@ -0,0 +1,13 @@
+name: Deploy canary via single cell infra repo
+
+on:
+  push:
+    branches: main-canary
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: repository dispatch
+        run: |
+          curl -XPOST -u czi-sci-single-cell-eng:${{secrets.SCI_GITHUB_TOKEN}} -H "Accept: application/vnd.github.everest-preview+json"  -H "Content-Type: application/json" https://api.github.com/repos/chanzuckerberg/single-cell-infra/dispatches --data '{"event_type": "canary-hook"}'

.github/workflows/compatibility_tests.yml

@@ -41,8 +41,8 @@ jobs:
       run: |
         # 1. only install the dev requirements on top of what is in the cellxgene pip package
         sudo apt-get update && sudo apt-get install -y libhdf5-serial-dev
-        sed -i 's/-r requirements.txt//' server/requirements-dev.txt
+        sed -i 's/-r requirements.txt//' backend/server/requirements-dev.txt
-        pip install -r server/requirements-dev.txt
+        pip install -r backend/server/requirements-dev.txt
         # 2. install cellxgene
         make pydist install-dist
         # 3. install anndata
@@ -73,8 +73,8 @@ jobs:
           cd cellxgene
           # 1. only install the dev requirements on top of what is in the cellxgene pip package
           make dev-env-client
-          sed -i 's/-r requirements.txt//' server/requirements-dev.txt
+          sed -i 's/-r requirements.txt//' backend/server/requirements-dev.txt
-          pip install -r server/requirements-dev.txt
+          pip install -r backend/server/requirements-dev.txt
           # 2. install cellxgene
           pip install --upgrade cellxgene
           # 3. install anndata
@@ -101,7 +101,7 @@ jobs:
       - name: Install dependencies
         run: |
           cd cellxgene
-          sed -i -E 's/^anndata[>=]=[0-9]+.[0-9]+.[0-9]+$/anndata/g' server/requirements.txt
+          sed -i -E 's/^anndata[>=]=[0-9]+.[0-9]+.[0-9]+$/anndata/g' backend/server/requirements.txt
           make pydist install-dist dev-env
           pip install git+https://github.com/theislab/anndata
       - name: Tests

.github/workflows/push_tests.yml

@@ -36,7 +36,7 @@ jobs:
           npm install
       - name: Format with black and lint with flake8
         run: |
-          make lint-server
+          make lint-servers
       - name: Lint src with eslint
         working-directory: ./client
         run: |
@@ -68,8 +68,38 @@ jobs:
         run: make pydist install-dist dev-env-server
       - name: Unit tests
         run: |
-          make unit-test
+          make unit-test-server
-          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k server -cF backend,python,unitTest
+          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k backend/server -cF backend,python,unitTest
+          cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest
+
+  unit-test-czi-hosted:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Python cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Node cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Install dependencies
+        run: make pydist-czi-hosted install-dist dev-env-czi-hosted
+      - name: Unit tests
+        run: |
+          make unit-test-czi-hosted
+          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k backend/czi-hosted -cF backend,python,unitTest
           cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest
 
   smoke-tests:
@@ -96,7 +126,7 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-node-
       - name: Install dependencies
-        run: make pydist install-dist
+        run: make pydist-czi-hosted install-dist
       - name: Smoke tests (without annotations feature)
         run: |
           cd client && make smoke-test

.github/workflows/scale-test.yml

@@ -15,15 +15,15 @@ jobs:
           python-version: 3.7
       - name: Install dependencies
         run: |
-          pip install -r server/test/locust/requirements-locust.txt
+          pip install -r backend/test/test_czi_hosted/locust/requirements-locust.txt
       - name: Dev Scale Test
         run: |
-          locust -f server/test/locust/locustfile.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt
+          locust -f backend/test/test_czi_hosted/locust/locustfile.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt
       - name: Slack success webhook
         env:
           SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
         run: |
-          DEV_STATS=$(tail -n 61 locust_dev_stats.txt)
+          DEV_STATS=$(tail -n 15 locust_dev_stats.txt)
           DEV_MSG="\`\`\`CELLXGENE EXPLORER DEV SCALE TEST RESULTS: ${DEV_STATS}\`\`\`"
           curl -X POST -H 'Content-type: application/json' --data "{'text':'${DEV_MSG}'}" $SLACK_WEBHOOK
 

.gitignore

@@ -19,9 +19,13 @@ venv/
 cellxgene/
 
 # client build
-server/common/web/static/*
+backend/server/common/web/static/*
-server/common/web/templates/
+backend/server/common/web/templates/
-server/common/web/csp-hashes.json
+backend/server/common/web/csp-hashes.json
+
+backend/czi_hosted/common/web/static/*
+backend/czi_hosted/common/web/templates/
+backend/czi_hosted/common/web/csp-hashes.json
 
 # eb build
 artifact.dir

MANIFEST.in

@@ -1,7 +1,7 @@
-recursive-include server/common/web/templates *
+recursive-include backend/server/common/web/templates *
-recursive-include server/common/web/static *
+recursive-include backend/server/common/web/static *
 
-include server/requirements.txt
+include backend/server/requirements.txt
-include server/requirements-prepare.txt
+include backend/server/requirements-prepare.txt
-include server/converters/schema/hgnc_complete_set.txt.gz
+include backend/server/converters/schema/hgnc_complete_set.txt.gz
-include server/converters/schema/schema_definitions/*
+include backend/server/converters/schema/schema_definitions/*

MANIFEST_hosted.in

@@ -0,0 +1,7 @@
+recursive-include backend/czi_hosted/common/web/templates *
+recursive-include backend/czi_hosted/common/web/static *
+
+include backend/czi_hosted/requirements.txt
+include backend/czi_hosted/requirements-prepare.txt
+include backend/czi_hosted/converters/schema/hgnc_complete_set.txt.gz
+include backend/czi_hosted/converters/schema/schema_definitions/*

Makefile

@@ -2,23 +2,32 @@ include common.mk
 
 BUILDDIR := build
 CLIENTBUILD := $(BUILDDIR)/client
-SERVERBUILD := $(BUILDDIR)/server
+CZIHOSTEDBUILD := $(BUILDDIR)/backend/czi_hosted
+SERVERBUILD := $(BUILDDIR)/backend/server
 CLEANFILES :=  $(BUILDDIR)/ client/build build dist cellxgene.egg-info
 
 PART ?= patch
 
 # CLEANING
 .PHONY: clean
-clean: clean-lite clean-server clean-client
+clean: clean-lite clean-czi-hosted clean-server clean-client
 
 # cleaning the client's node_modules is the longest one, so we avoid that if possible
 .PHONY: clean-lite
 clean-lite:
 	rm -rf $(CLEANFILES)
 
-clean-%:
+.PHONY: clean-client
-	cd $(*) && $(MAKE) clean
+clean-client:
+	cd client && $(MAKE) clean
 
+.PHONY: clean-server
+clean-server:
+	cd backend/server && $(MAKE) clean
+
+.PHONY: clean-czi-hosted
+clean-czi-hosted:
+	cd backend/czi_hosted && $(MAKE) clean
 
 # BUILDING PACKAGE
 
@@ -28,29 +37,71 @@ build-client:
 
 .PHONY: build
 build: clean build-client
-	git ls-files server/ | grep -v 'server/test/' | cpio -pdm $(BUILDDIR)
+	git ls-files backend/server/ | grep -v 'backend/server/test/' | cpio -pdm $(BUILDDIR)
 	cp -r client/build/  $(CLIENTBUILD)
 	$(call copy_client_assets,$(CLIENTBUILD),$(SERVERBUILD))
+	cp backend/__init__.py $(BUILDDIR)
+	cp backend/__init__.py $(BUILDDIR)/backend
+	cp -r backend/common $(BUILDDIR)/backend/common
 	cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR)
 
+.PHONY: build-czi-hosted
+build-czi-hosted: clean build-client
+	git ls-files backend/czi_hosted/ | grep -v 'backend/czi_hosted/test/' | cpio -pdm $(BUILDDIR)
+	cp -r client/build/  $(CLIENTBUILD)
+	$(call copy_client_assets,$(CLIENTBUILD),$(CZIHOSTEDBUILD))
+	cp -r backend/common $(BUILDDIR)/backend/common
+	cp backend/__init__.py $(BUILDDIR)
+	cp backend/__init__.py $(BUILDDIR)/backend
+	cp MANIFEST_hosted.in README.md setup.cfg setup_hosted.py $(BUILDDIR)
+	mv $(BUILDDIR)/setup_hosted.py $(BUILDDIR)/setup.py
+	mv $(BUILDDIR)/MANIFEST_hosted.in $(BUILDDIR)/MANIFEST.in
+
 # If you are actively developing in the server folder use this, dirties the source tree
 .PHONY: build-for-server-dev
 build-for-server-dev: clean-server build-client
-	$(call copy_client_assets,client/build,server)
+	$(call copy_client_assets,client/build,backend/server)
+
+.PHONY: build-for-czi-hosted-dev
+build-for-czi-hosted-dev: clean-czi-hosted build-client
+	$(call copy_client_assets,client/build,backend/czi_hosted)
 
 .PHONY: copy-client-assets
 copy-client-assets:
-	$(call copy_client_assets,client/build,server)
+	$(call copy_client_assets,client/build,backend/server)
+
+.PHONY: copy-client-assets-czi-hosted
+copy-client-assets-czi-hosted:
+	$(call copy_client_assets,client/build,backend/czi_hosted)
 
 # TESTING
 .PHONY: test
 test: unit-test smoke-test
 
 .PHONY: unit-test
-unit-test: unit-test-server unit-test-client
+unit-test: unit-test-server unit-test-client unit-test-common
 
-unit-test-%:
+.PHONY: test-server
-	cd $(*) && $(MAKE) unit-test
+test-server: unit-test-server smoke-test
+
+.PHONY: test-czi-hosted
+test-czi-hosted: unit-test-czi-hosted smoke-test
+
+.PHONY: unit-test-client
+unit-test-client:
+	cd client && $(MAKE) unit-test
+
+.PHONY: unit-test-czi-hosted
+unit-test-czi-hosted:
+	cd backend/czi_hosted && $(MAKE) unit-test
+
+.PHONY: unit-test-server
+unit-test-server:
+	cd backend/server && $(MAKE) unit-test
+
+.PHONY: unit-test-common
+unit-test-common:
+	cd backend/common && $(MAKE) unit-test
 
 .PHONY: smoke-test
 smoke-test:
@@ -62,12 +113,11 @@ smoke-test-annotations:
 
 .PHONY: test-db
 test-db:
-	cd server && $(MAKE) test-db
+	cd backend/czi_hosted && $(MAKE) test-db
-
 
 # FORMATTING CODE
 
-.PHOHY: fmt
+.PHONY: fmt
 fmt: fmt-client fmt-py
 
 .PHONY: fmt-client
@@ -79,12 +129,18 @@ fmt-py:
 	black .
 
 .PHONY: lint
-lint: lint-server lint-client
+lint: lint-servers lint-client
+
+.PHONY: lint-servers
+lint-servers: lint-server lint-czi-hosted-server
 
 .PHONY: lint-server
 lint-server: fmt-py
-	flake8 server --per-file-ignores='server/test/fixtures/dataset_config_outline.py:F821 server/test/fixtures/server_config_outline.py:F821 server/test/performance/scale_test_annotations.py:E501'
+	flake8 backend/server --per-file-ignores='backend/test/fixtures/dataset_config_outline.py:F821 backend/test/fixtures/server_config_outline.py:F821 backend/server/test/performance/scale_test_annotations.py:E501'
 
+.PHONY: lint-czi-hosted-server
+lint-czi-hosted-server: fmt-py
+	flake8 backend/czi_hosted --per-file-ignores='backend/test/fixtures/czi_hosted_dataset_config_outline.py:F821 backend/test/fixtures/czi_hosted_server_config_outline.py:F821 backend/test/performance/scale_test_annotations.py:E501'
 
 .PHONY: lint-client
 lint-client:
@@ -97,6 +153,11 @@ pydist: build
 	cd $(BUILDDIR); python setup.py sdist -d ../dist
 	@echo "done"
 
+.PHONY: pydist-czi-hosted
+pydist-czi-hosted: build-czi-hosted
+	cd $(BUILDDIR); python setup.py sdist -d ../dist
+	@echo "done"
+
 
 # RELEASE HELPERS
 
@@ -145,8 +206,11 @@ dev-env-client:
 
 .PHONY: dev-env-server
 dev-env-server:
-	pip install -r server/requirements-dev.txt
+	pip install -r backend/server/requirements-dev.txt
 
+.PHONY: dev-env-czi-hosted
+dev-env-czi-hosted:
+	pip install -r backend/czi_hosted/requirements-dev.txt
 # Set PART=[major, minor, patch] as param to make bump.
 # This will create a release candidate. (i.e. 0.16.1 -> 0.16.2-rc.0 for a patch bump)
 .PHONY: bump-version

README.md

@@ -52,7 +52,6 @@ cellxgene currently supports the following browsers:
 - Google Chrome 61+
 - Edge 15+
 - Firefox 60+
-- Safari 10.1+
 
 Please [file an issue](https://github.com/chanzuckerberg/cellxgene/issues/new/choose) if you would like us to add support for an unsupported browser.
 

backend/common/Makefile

@@ -0,0 +1,11 @@
+
+.PHONY: unit-test
+unit-test:
+	PYTHONWARNINGS=ignore:ResourceWarning coverage run \
+		--source=fbs,utils \
+		--omit=.coverage,data_common/fbs/NetEncoding,venv \
+		-m unittest discover \
+		--start-directory ../test/test_common/unit \
+		--top-level-directory ../../ \
+		--verbose; test_result=$$?; \
+	exit $$test_result \

backend/common/colors.py

@@ -1,6 +1,6 @@
 import re
 
-from server.common.errors import ColorFormatException
+from backend.common.errors import ColorFormatException
 
 HEX_COLOR_FORMAT = re.compile("^#[a-fA-F0-9]{6,6}$")
 

backend/common/compute/diffexp_generic.py

@@ -25,7 +25,8 @@ def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
     :param maskB: observation selection mask for set 2
     :param top_n: number of variables to return stats for
     :param diffexp_lfc_cutoff: minimum
-    :return:  for top N genes, [ varindex, logfoldchange, pval, pval_adj ]
+    absolute value returning [ varindex, logfoldchange, pval, pval_adj ] for top N genes
+    :return:  for top N genes, {"positive": for top N genes, [ varindex, logfoldchange, pval, pval_adj ], "negative": for top N genes, [ varindex, logfoldchange, pval, pval_adj ]}
     """
 
     dataA = adaptor.get_X_array(maskA, None)
@@ -66,24 +67,27 @@ def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp
     # logfoldchanges: log2(meanA / meanB)
     logfoldchanges = np.log2(np.abs((meanA + 1e-9) / (meanB + 1e-9)))
 
+    stats_to_sort = tscores
     # find all with lfc > cutoff
     lfc_above_cutoff_idx = np.nonzero(np.abs(logfoldchanges) > diffexp_lfc_cutoff)[0]
-    stats_to_sort = np.abs(tscores)
 
     # derive sort order
-    if lfc_above_cutoff_idx.shape[0] > top_n:
+    if lfc_above_cutoff_idx.shape[0] > top_n*2:
         # partition top N
-        rel_t_partition = np.argpartition(stats_to_sort[lfc_above_cutoff_idx], -top_n)[-top_n:]
+        rel_t_partition = np.argpartition(stats_to_sort[lfc_above_cutoff_idx], (top_n, -top_n))
-        t_partition = lfc_above_cutoff_idx[rel_t_partition]
+        rel_t_partition_top_n = np.concatenate((rel_t_partition[-top_n:], rel_t_partition[:top_n]))
+        t_partition = lfc_above_cutoff_idx[rel_t_partition_top_n]
         # sort the top N partition
         rel_sort_order = np.argsort(stats_to_sort[t_partition])[::-1]
         sort_order = t_partition[rel_sort_order]
     else:
         # partition and sort top N, ignoring lfc cutoff
-        partition = np.argpartition(stats_to_sort, -top_n)[-top_n:]
+        partition = np.argpartition(stats_to_sort, (top_n, -top_n))
-        rel_sort_order = np.argsort(stats_to_sort[partition])[::-1]
+        partition_top_n = np.concatenate((partition[-top_n:], partition[:top_n]))
+
+        rel_sort_order = np.argsort(stats_to_sort[partition_top_n])[::-1]
         indices = np.indices(stats_to_sort.shape)[0]
-        sort_order = indices[partition][rel_sort_order]
+        sort_order = indices[partition_top_n][rel_sort_order]
 
     # top n slice based upon sort order
     logfoldchanges_top_n = logfoldchanges[sort_order]
@@ -91,7 +95,11 @@ def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp
     pvals_adj_top_n = pvals_adj[sort_order]
 
     # varIndex, logfoldchange, pval, pval_adj
-    result = [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in range(top_n)]
+    result = {"positive": [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in
+                           range(top_n)],
+              "negative": [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in
+                           range(-1, -1 - top_n, -1)], }
+
     return result
 
 

backend/common/errors.py

@@ -55,3 +55,5 @@ define_exception("OntologyLoadFailure", "Raised when reading the ontology file f
 define_exception("ConfigurationError", "Raised when checking configuration errors")
 define_exception("PrepareError", "Raised when data is misprepared")
 define_exception("SecretKeyRetrievalError", "Raised when get_secret_key from AWS fails")
+define_exception("ObsoleteRequest", "Raised when the request is no longer valid.")
+define_exception("UnsupportedSummaryMethod", "Raised when a gene set summary method is unknown or unsupported.")

backend/common/fbs/matrix.py

@@ -5,15 +5,14 @@ import pandas as pd
 from flatbuffers import Builder
 from scipy import sparse
 
-import server.data_common.fbs.NetEncoding.Column as Column
+import backend.common.fbs.NetEncoding.Column as Column
-import server.data_common.fbs.NetEncoding.Float32Array as Float32Array
+import backend.common.fbs.NetEncoding.Float32Array as Float32Array
-import server.data_common.fbs.NetEncoding.Float64Array as Float64Array
+import backend.common.fbs.NetEncoding.Float64Array as Float64Array
-import server.data_common.fbs.NetEncoding.Int32Array as Int32Array
+import backend.common.fbs.NetEncoding.Int32Array as Int32Array
-import server.data_common.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray
+import backend.common.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray
-import server.data_common.fbs.NetEncoding.Matrix as Matrix
+import backend.common.fbs.NetEncoding.Matrix as Matrix
-import server.data_common.fbs.NetEncoding.TypedArray as TypedArray
+import backend.common.fbs.NetEncoding.TypedArray as TypedArray
-import server.data_common.fbs.NetEncoding.Uint32Array as Uint32Array
+import backend.common.fbs.NetEncoding.Uint32Array as Uint32Array
-
 
 # Serialization helper
 def serialize_column(builder, typed_arr):

backend/common/genesets.py

@@ -0,0 +1,239 @@
+"""
+Utility code for gene sets handling
+"""
+
+import re
+import csv
+import hashlib
+
+from .errors import AnnotationsError
+
+
+GENESETS_TIDYCSV_HEADER = [
+    "gene_set_name",
+    "gene_set_description",
+    "gene_symbol",
+    "gene_description",
+]
+
+
+def read_gene_sets_tidycsv(gs_locator, context=None):
+    """
+    Read & parse the Tidy CSV format, applying validation checks for mandatory
+    values, and de-duping rules.
+
+    Format is a four-column CSV, with a mandatory header row, and optional "#" prefixed
+    comments.  Format:
+
+        gene_set_name, gene_set_description, gene_symbol, gene_description
+
+    gene_set_name must be non-null; others are optional.
+
+    Returns: a dictionary of the shape (values in angle-brackets vary):
+
+        {
+            <string, a gene set name>: {
+                "geneset_name": <string, a gene set name>,
+                "geneset_description": <a string or None>,
+                "genes": [
+                    {
+                        "gene_symbol": <string, a gene symbol or name>,
+                        "gene_description": <a string or None>
+                    },
+                    ...
+                ]
+            },
+            ...
+        }
+    """
+
+    class myDialect(csv.excel):
+        skipinitialspace = False
+
+    def just(n, seq):
+        it = iter(seq)
+        for _ in range(n - 1):
+            yield next(it, "")
+        yield tuple(it)
+
+    messagefn = context["messagefn"] if context else (lambda x: None)
+
+    gene_sets = {}
+    with gs_locator.local_handle() as fname:
+        with open(fname, newline="") as f:
+            reader = csv.reader(f, dialect=myDialect())
+            haveReadHeader = False
+            lineno = 0
+            for row in reader:
+                lineno += 1
+                # ignore empty rows
+                if len(row) == 0:
+                    continue
+                # if row starts with '#' it is a comment
+                if row[0].startswith("#"):
+                    continue
+                # if this is the first non-comment row, assume it is a header and validate
+                # column names.  OK if the user has extra columns after our initial set.
+                if not haveReadHeader:
+                    if row[0:len(GENESETS_TIDYCSV_HEADER)] != GENESETS_TIDYCSV_HEADER:
+                        raise AnnotationsError("Gene set CSV file missing the required column header.")
+                    haveReadHeader = True
+                    continue
+
+                geneset_name, geneset_description, gene_symbol, gene_description, _ = just(5, row)
+                if not geneset_name:
+                    raise AnnotationsError(f"Gene set CSV missing required gene set name on line {lineno}")
+                if (not gene_symbol) and gene_description:
+                    messagefn(f"Warning: Missing gene name in gene set name {geneset_name} on line {lineno}.")
+
+                if geneset_name in gene_sets:
+                    gs = gene_sets[geneset_name]
+                else:
+                    gs = gene_sets[geneset_name] = {
+                        "geneset_name": geneset_name,
+                        "geneset_description": geneset_description,
+                        "genes": [],
+                    }
+                # Use first geneset_description with a value
+                if not gs["geneset_description"] and geneset_description:
+                    gs["geneset_description"] = geneset_description
+                # add the gene if the gene_symbol is defined
+                if gene_symbol:
+                    gs["genes"].append(
+                        {
+                            "gene_symbol": gene_symbol,
+                            "gene_description": gene_description,
+                        }
+                    )
+
+    return gene_sets
+
+
+def write_gene_sets_tidycsv(f, genesets):
+    """
+    Convert the internal gene sets format (returned by read_gene_set_tidycsv) into
+    the simple Tidy CSV.
+    """
+    writer = csv.writer(f, dialect="excel")
+    writer.writerow(GENESETS_TIDYCSV_HEADER)
+    for geneset in genesets:
+        # genes may be empty, in which case we skip the gene set entirely
+        genes = geneset["genes"]
+        if not genes:
+            writer.writerow([geneset["geneset_name"], geneset.get("geneset_description", ""), "", ""])
+        else:
+            writer.writerows(
+                [
+                    [
+                        geneset["geneset_name"],
+                        geneset.get("geneset_description", ""),
+                        gene["gene_symbol"],
+                        gene.get("gene_description", ""),
+                    ]
+                    for gene in genes
+                ]
+            )
+
+
+def summarizeQueryHash(raw_query):
+    """ generate a cache key (hash) from the raw query string """
+    return hashlib.sha1(raw_query).hexdigest()
+
+
+def validate_gene_sets(genesets, var_names, context=None):
+    """
+    Check validity of gene sets, return if correct, else raise error.
+    May also modify the gene set for conditions that should be resolved,
+    but which do not warrant a hard error.
+
+    Argument gene sets may be either the REST OTA format (list of dicts) or the internal
+    format (dict of dicts, keyed by the gene set name).
+
+    Will return a modified gene sets (eg, remove warnings) of the same type as the
+    provided argument. Ie, dict->dict, list->list
+
+    Rules:
+
+    0. All gene set names must be unique. [error]
+    1. Gene set names must conform to the following: [error]
+        * Names must be comprised of 1 or more ASCII characters 32-126
+        * No leading or trailing spaces (ASCII 32)
+        * No multi-space (ASCII 32) runs
+    2. Gene symbols must be part of the current var_index. [warning]
+       If gene symbol is not in the var_index, generate a warning and remove the symbol
+       from the gene sets.
+    3. Gene symbols must not be duplicated in a gene set.  [warning]
+       Duplications will be silently de-duped.
+
+    Items marked [error] will generate a hard error, causing the validation to fail.
+
+    Items marked [warning] will generate a warning, and will be resolved without failing
+    the validation (typically by removing the offending item from the gene sets).
+    """
+
+    messagefn = context["messagefn"] if context else (lambda x: None)
+
+    # accept genesets args as either the internal (dict) or REST (list) format,
+    # as they are identical except for the dict being keyed by geneset_name.
+    if not isinstance(genesets, dict) and not isinstance(genesets, list):
+        raise ValueError("Gene sets must be either dict or list.")
+    genesets_iterable = genesets if isinstance(genesets, list) else genesets.values()
+
+    # 0. check for uniqueness of geneset names
+    geneset_names = [gs["geneset_name"] for gs in genesets_iterable]
+    if len(set(geneset_names)) != len(geneset_names):
+        raise KeyError("All gene set names must be unique.")
+
+    # 1. check gene set character set and format
+    illegal_name = re.compile(r"^\s|  |[\u0000-\u001F\u007F-\uFFFF]|\s$")
+    for name in geneset_names:
+        if type(name) != str or len(name) == 0:
+            raise KeyError("Gene set names must be non-null string.")
+        if illegal_name.search(name):
+            messagefn(
+                "Error: "
+                f"Gene set name {name} "
+                "is not valid. Leading, trailing, and multiple spaces within a name are not allowed."
+            )
+            raise KeyError(
+                "Gene set name is not valid. Leading, trailing, and multiple spaces within a name are not allowed."
+            )
+
+    # 2. & 3. check for duplicate gene symbols, and those not present in the dataset. They will
+    # generate a warning and be removed.
+    for geneset in genesets_iterable:
+        if not isinstance(geneset, dict):
+            raise ValueError("Each gene set must be a dict.")
+        geneset_name = geneset["geneset_name"]
+        genes = geneset["genes"]
+        if not isinstance(genes, list):
+            raise ValueError("Gene set genes field must be a list")
+        geneset.setdefault("geneset_description", "")
+        gene_symbol_already_seen = set()
+        new_genes = []
+        for gene in genes:
+            gene_symbol = gene["gene_symbol"]
+            if not isinstance(gene_symbol, str) or len(gene_symbol) == 0:
+                raise ValueError("Gene symbol must be non-null string.")
+            if gene_symbol in gene_symbol_already_seen:
+                # duplicate check
+                messagefn(
+                    f"Warning: a duplicate of gene {gene_symbol} was found in gene set {geneset_name}, "
+                    "and will be ignored."
+                )
+                continue
+
+            if gene_symbol not in var_names:
+                messagefn(
+                    f"Warning: {gene_symbol}, used in gene set {geneset_name}, "
+                    "was not found in the dataset and will be ignored."
+                )
+                continue
+
+            gene_symbol_already_seen.add(gene_symbol)
+            gene.setdefault("gene_description", "")
+            new_genes.append(gene)
+
+        geneset["genes"] = new_genes
+
+    return genesets

backend/common/utils/aws_secret_utils.py

@@ -3,7 +3,7 @@ import logging
 import boto3
 from flask import json
 
-from server.common.errors import SecretKeyRetrievalError
+from backend.common.errors import SecretKeyRetrievalError
 
 
 def get_secret_key(region_name, secret_name):

backend/common/utils/utils.py

@@ -10,7 +10,7 @@ from urllib.parse import urlsplit, urljoin
 import numpy as np
 from flask import json
 
-from server.common.errors import ConfigurationError
+from backend.common.errors import ConfigurationError
 
 
 def find_available_port(host, port=5005):

backend/czi_hosted/Makefile

@@ -1,4 +1,4 @@
-include ../common.mk
+include ../../common.mk
 
 .PHONY: clean
 clean:
@@ -9,11 +9,11 @@ clean:
 .PHONY: unit-test
 unit-test: create-test-db
 	PYTHONWARNINGS=ignore:ResourceWarning coverage run \
-		--source=app,cli,common,compute,converters,data_anndata,data_common,data_cxg \
+		--source=app,auth,cli,common,compute,converters,data_anndata,data_common,data_cxg,eb \
-		--omit=.coverage,data_common/fbs/NetEncoding,venv \
+		--omit=.coverage,venv \
 		-m unittest discover \
-		--start-directory test/ \
+		--start-directory ../test/test_czi_hosted/unit \
-		--top-level-directory ../ \
+		--top-level-directory ../.. \
 		--verbose; test_result=$$?; \
 	$(MAKE) clean-test-db; \
 	exit $$test_result \
@@ -22,11 +22,11 @@ unit-test: create-test-db
 .PHONY: test-db
 test-db: create-test-db
 	PYTHONWARNINGS=ignore:ResourceWarning coverage run \
-		--source=app,cli,common,compute,converters,data_anndata,data_common,data_cxg \
+		--source=db \
-		--omit=.coverage,data_common/fbs/NetEncoding,venv \
+		--omit=.coverage,venv \
 		-m unittest discover \
-		--start-directory test/test_database \
+		--start-directory ../test/test_czi_hosted/test_database \
-		--top-level-directory ../ \
+		--top-level-directory ../.. \
 		--verbose; test_result=$$?; \
 	$(MAKE) clean-test-db; \
 	exit $$test_result
@@ -42,8 +42,8 @@ clean-test-db:
 
 .PHONY: test-annotations-performance
 test-annotations-performance:
-	python test/performance/performance_test_annotations_backend.py
+	python ../test/test_czi_hosted/performance/performance_test_annotations_backend.py
 
 .PHONY: test-annotations-scale
 test-annotations-scale:
-	locust -f test/performance/scale_test_annotations.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt
+	locust -f ../test/test_czi_hosted/performance/scale_test_annotations.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt

backend/czi_hosted/__init__.py

@@ -1,12 +1,14 @@
 import logging
 import sys
-from server.common.utils.utils import import_plugins
+from backend.common.utils.utils import import_plugins
 
 __version__ = "0.16.7"
+
+
 display_version = "cellxgene v" + __version__
 
 try:
-    import_plugins("server.plugins")
+    import_plugins("backend.czi_hosted.plugins")
 except Exception as e:
     # Make sure to exit in this case, as the server may not be configured as expected.
     logging.critical(f"Error in import_plugins: {str(e)}")

backend/czi_hosted/__main__.py

@@ -0,0 +1,14 @@
+# Work around bug https://github.com/pallets/werkzeug/issues/461
+if __package__ is None:
+    import sys
+    from pathlib import Path
+
+    PKG_PATH = Path(__file__).parent
+    sys.path.insert(0, str(PKG_PATH.parent))
+    import backend.czi_hosted # noqa F401
+    __package__ = PKG_PATH.name
+
+# Main thing
+from .cli.cli import cli  # noqa F402
+
+cli()

backend/czi_hosted/app/app.py

@@ -20,14 +20,14 @@ from flask import (
 from flask_restful import Api, Resource
 from server_timing import Timing as ServerTiming
 
-import server.common.rest as common_rest
+import backend.czi_hosted.common.rest as common_rest
-from server.common.data_locator import DataLocator
+from backend.common.utils.data_locator import DataLocator
-from server.common.errors import DatasetAccessError, RequestException
+from backend.common.errors import DatasetAccessError, RequestException
-from server.common.health import health_check
+from backend.czi_hosted.common.health import health_check
-from server.common.utils.utils import path_join, Float32JSONEncoder
+from backend.common.utils.utils import path_join, Float32JSONEncoder
-from server.data_common.matrix_loader import MatrixDataLoader
+from backend.czi_hosted.data_common.matrix_loader import MatrixDataLoader
 
-webbp = Blueprint("webapp", "server.common.web", template_folder="templates")
+webbp = Blueprint("webapp", "backend.czi_hosted.common.web", template_folder="templates")
 
 ONE_WEEK = 7 * 24 * 60 * 60
 
@@ -168,7 +168,7 @@ def rest_get_data_adaptor(func):
     return wrapped_function
 
 
-def dataroot_test_index():
+def  dataroot_test_index():
     # the following index page is meant for testing/debugging purposes
     data = '<!doctype html><html lang="en">'
     data += "<head><title>Hosted Cellxgene</title></head>"
@@ -205,7 +205,7 @@ def dataroot_test_index():
     data += "<ul>"
     datasets.sort()
     for url_dataroot, dataset in datasets:
-        data += f"<li><a href={url_dataroot}/{dataset}>{dataset}</a></li>"
+        data += f"<li><a href={url_dataroot}/{dataset}/>{dataset}</a></li>"
     data += "</ul>"
     data += "</body></html>"
 
@@ -240,7 +240,7 @@ class DatasetResource(Resource):
 
 class SchemaAPI(DatasetResource):
     # TODO @mdunitz separate dataset schema and user schema
-    @cache_control(no_store=True)
+    @cache_control(public=True, max_age=ONE_WEEK)
     @rest_get_data_adaptor
     def get(self, data_adaptor):
         return common_rest.schema_get(data_adaptor)
@@ -261,7 +261,7 @@ class UserInfoAPI(DatasetResource):
 
 
 class AnnotationsObsAPI(DatasetResource):
-    @cache_control(public=True, no_store=True)
+    @cache_control(public=True, max_age=ONE_WEEK)
     @rest_get_data_adaptor
     def get(self, data_adaptor):
         return common_rest.annotations_obs_get(request, data_adaptor)
@@ -318,6 +318,25 @@ class LayoutObsAPI(DatasetResource):
         return common_rest.layout_obs_put(request, data_adaptor)
 
 
+class GenesetsAPI(DatasetResource):
+    @cache_control(public=True, max_age=ONE_WEEK)
+    @rest_get_data_adaptor
+    def get(self, data_adaptor):
+        return common_rest.genesets_get(request, data_adaptor)
+
+
+class SummarizeVarAPI(DatasetResource):
+    @rest_get_data_adaptor
+    @cache_control(public=True, max_age=ONE_WEEK)
+    def get(self, data_adaptor):
+        return common_rest.summarize_var_get(request, data_adaptor)
+
+    @rest_get_data_adaptor
+    @cache_control(no_store=True)
+    def post(self, data_adaptor):
+        return common_rest.summarize_var_post(request, data_adaptor)
+
+
 def get_api_base_resources(bp_base):
     """Add resources that are accessed from the api_base_url"""
     api = Api(bp_base)
@@ -343,6 +362,8 @@ def get_api_dataroot_resources(bp_dataroot, url_dataroot=None):
     add_resource(AnnotationsObsAPI, "/annotations/obs")
     add_resource(AnnotationsVarAPI, "/annotations/var")
     add_resource(DataVarAPI, "/data/var")
+    add_resource(GenesetsAPI, "/genesets")
+    add_resource(SummarizeVarAPI, "/summarize/var")
     # Display routes
     add_resource(ColorsAPI, "/colors")
     # Computation routes
@@ -454,6 +475,6 @@ class Server:
 
         auth = server_config.auth
         self.app.auth = auth
-        if auth.requires_client_login():
+        if auth and auth.requires_client_login():
             auth.add_url_rules(self.app)
         auth.complete_setup(self.app)

backend/czi_hosted/auth/__init__.py

@@ -0,0 +1,6 @@
+# import the built in auth types so they can be registered
+
+import backend.czi_hosted.auth.auth_test  # noqa: F401
+import backend.czi_hosted.auth.auth_session  # noqa: F401
+import backend.czi_hosted.auth.auth_oauth  # noqa: F401
+import backend.czi_hosted.auth.auth_none  # noqa: F401

backend/czi_hosted/auth/auth_none.py

@@ -0,0 +1,27 @@
+from backend.czi_hosted.auth.auth import AuthTypeBase, AuthTypeFactory
+
+
+class AuthTypeNone(AuthTypeBase):
+    def __init__(self, app_config):
+        super().__init__()
+
+    def is_valid_authentication_type(self):
+        return False
+
+    def complete_setup(self, app):
+        pass
+
+    def is_user_authenticated(self):
+        return True
+
+    def get_user_id(self):
+        return None
+
+    def get_user_name(self):
+        return None
+
+    def get_user_email(self):
+        return None
+
+
+AuthTypeFactory.register(None, AuthTypeNone)

backend/czi_hosted/auth/auth_oauth.py

@@ -1,6 +1,6 @@
 from flask import session, request, redirect, current_app, after_this_request, has_request_context, g
-from server.auth.auth import AuthTypeClientBase, AuthTypeFactory
+from backend.czi_hosted.auth.auth import AuthTypeClientBase, AuthTypeFactory
-from server.common.errors import AuthenticationError, ConfigurationError
+from backend.common.errors import AuthenticationError, ConfigurationError
 from urllib.parse import urlencode, urlparse
 import json
 import requests

backend/czi_hosted/auth/auth_session.py

@@ -0,0 +1,40 @@
+from flask import session
+from uuid import uuid4
+
+from backend.czi_hosted.auth.auth import AuthTypeBase, AuthTypeFactory
+
+
+class AuthTypeSession(AuthTypeBase):
+    """Session based authentication.  The user is always logged.  The user id is a random number
+    associated with the session.  This is a good choice for desktop servers."""
+
+    # key in the session token for userid
+    CXGUID = "cxguid"
+
+    def __init__(self, app_config):
+        super().__init__()
+
+    def is_valid_authentication_type(self):
+        return True
+
+    def complete_setup(self, app):
+        pass
+
+    def is_user_authenticated(self):
+        # always authenticated
+        return True
+
+    def get_user_id(self):
+        if self.CXGUID not in session:
+            session[self.CXGUID] = uuid4().hex
+            session.permanent = True
+        return session[self.CXGUID]
+
+    def get_user_name(self):
+        return "anonymous"
+
+    def get_user_email(self):
+        return None
+
+
+AuthTypeFactory.register("session", AuthTypeSession)

backend/czi_hosted/auth/auth_test.py

@@ -1,6 +1,7 @@
-from server.auth.auth import AuthTypeClientBase, AuthTypeFactory
 from flask import session, request, redirect, current_app
 
+from backend.czi_hosted.auth.auth import AuthTypeClientBase, AuthTypeFactory
+
 
 class AuthTypeTest(AuthTypeClientBase):
     """An authentication type for testing client based logins.  When the login route is accessed

backend/czi_hosted/cli/convert_to_cxg.py

@@ -2,7 +2,7 @@ from os import path
 
 import click
 
-from server.converters.h5ad_data_file import H5ADDataFile
+from backend.czi_hosted.converters.h5ad_data_file import H5ADDataFile
 
 
 @click.command(

backend/czi_hosted/cli/launch.py

@@ -8,11 +8,12 @@ import click
 from flask_compress import Compress
 from flask_cors import CORS
 
-from server.default_config import default_config
+from backend.czi_hosted.default_config import default_config
-from server.app.app import Server
+from backend.czi_hosted.app.app import Server
-from server.common.config.app_config import AppConfig
+from backend.czi_hosted.common.config.app_config import AppConfig
-from server.common.errors import DatasetAccessError, ConfigurationError
+from backend.common.errors import DatasetAccessError, ConfigurationError
-from server.common.utils.utils import sort_options
+from backend.common.utils.utils import sort_options
+
 
 DEFAULT_CONFIG = AppConfig()
 
@@ -355,7 +356,6 @@ def launch(
     if dump_default_config:
         print(default_config)
         sys.exit(0)
-
     # Startup message
     click.echo("[cellxgene] Starting the CLI...")
 

backend/czi_hosted/cli/prepare.py

@@ -5,7 +5,7 @@ import pandas as pd
 from numpy import ndarray, unique
 from scipy.sparse.csc import csc_matrix
 
-from server.common.utils.utils import sort_options
+from backend.common.utils.utils import sort_options
 
 
 @sort_options

backend/czi_hosted/cli/schema.py

@@ -0,0 +1,72 @@
+import click
+
+from backend.czi_hosted.converters.schema import remix, validate
+
+
+@click.group(
+    name="schema",
+    subcommand_metavar="COMMAND <args>",
+    short_help="Apply and validate the cellxgene data integration schema to an h5ad file.",
+    context_settings=dict(max_content_width=85, help_option_names=["-h", "--help"]),
+)
+def schema_cli():
+    try:
+        import scanpy  # noqa: F401
+    except ImportError:
+        raise click.ClickException(
+            "[cellxgene] cellxgene schema requires scanpy"
+        )
+
+
+@click.command(
+    name="apply",
+    short_help="(experimental) Apply the cellxgene data integration schema to an h5ad.",
+    help="(experimental) Using a yaml file that describes schema values to insert or convert and in input "
+    "h5ad file, apply the schema changes and create a new, conforming h5ad.",
+)
+@click.option(
+    "--source-h5ad",
+    help="Input h5ad file.",
+    nargs=1,
+    required=True,
+    type=click.Path(exists=True, dir_okay=False),
+)
+@click.option(
+    "--remix-config",
+    help="Config yaml with information on how to apply the schema.",
+    nargs=1,
+    required=True,
+    type=click.Path(exists=True, dir_okay=False),
+)
+@click.option(
+    "--output-filename",
+    help="Filename for the new, schema-conforming h5ad file.",
+    required=True,
+    nargs=1
+)
+def schema_apply(source_h5ad, remix_config, output_filename):
+    remix.apply_schema(source_h5ad, remix_config, output_filename)
+
+
+@click.command(
+    name="validate",
+    short_help="(experimental) Check that an h5ad follows the cellxgene data integration schema.",
+)
+@click.argument(
+    "h5ad",
+    nargs=1,
+    type=click.Path(exists=True, dir_okay=False),
+)
+@click.option(
+    "--shallow",
+    help="When true, just check that the correct version information is present.",
+    default=False,
+    show_default=True,
+    is_flag=True,
+)
+def schema_validate(h5ad, shallow):
+    validate.validate(h5ad, shallow)
+
+
+schema_cli.add_command(schema_apply)
+schema_cli.add_command(schema_validate)

backend/czi_hosted/common/annotations/annotations.py

@@ -0,0 +1,138 @@
+import fastobo
+import fsspec
+import os
+
+from flask import current_app, has_request_context
+
+from backend.common.errors import OntologyLoadFailure, DisabledFeatureError
+from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
+from backend.common.genesets import write_gene_sets_tidycsv, read_gene_sets_tidycsv, validate_gene_sets
+from backend.common.utils.data_locator import DataLocator
+from backend.common.utils.utils import path_join
+
+
+class Annotations:
+    """ baseclass for annotations, including ontologies and genesets """
+
+    """ our default ontology is the PURL for the Cell Ontology.
+    See http://www.obofoundry.org/ontology/cl.html """
+    DefaultOnotology = "http://purl.obolibrary.org/obo/cl.obo"
+
+    def __init__(self, config={}):
+        self.ontology_data = None
+        self.config = config
+
+    def user_annotations_enabled(self):
+        return self.config.get("user-annotations", False)
+
+    def check_user_annotations_enabled(self):
+        if not self.user_annotations_enabled():
+            raise DisabledFeatureError("User annotations are disabled.")
+
+    def load_ontology(self, path):
+        """Load and parse ontologies - currently support OBO files only."""
+        if path is None:
+            path = self.DefaultOnotology
+
+        try:
+            with fsspec.open(path) as f:
+                obo = fastobo.iter(f)
+                terms = filter(lambda stanza: type(stanza) is fastobo.term.TermFrame, obo)
+                names = [tag.name for term in terms for tag in term if type(tag) is fastobo.term.NameClause]
+                self.ontology_data = names
+
+        except FileNotFoundError as e:
+            raise OntologyLoadFailure("Unable to find OBO ontology path") from e
+
+        except SyntaxError as e:
+            raise OntologyLoadFailure(f"{path}:{e.lineno}:{e.offset} OBO syntax error, unable to read ontology") from e
+
+        except Exception as e:
+            raise OntologyLoadFailure(f"{path}:Error loading OBO file") from e
+
+    def get_schema(self, data_adaptor):
+        schema = []
+        labels = self.read_labels(data_adaptor)
+        if labels is not None and not labels.empty:
+            for col in labels.columns:
+                col_schema = dict(name=col, writable=True)
+                col_schema.update(get_schema_type_hint_of_array(labels[col]))
+                schema.append(col_schema)
+
+        return schema
+
+    def set_collection(self, name):
+        """set or create a new annotation collection"""
+        raise NotImplementedError
+
+    def read_labels(self, data_adaptor):
+        """Return the labels as a pandas.DataFrame"""
+        raise NotImplementedError
+
+    def write_labels(self, df, data_adaptor):
+        """Write the labels (df) to a persistent storage such that it can later be read"""
+        raise NotImplementedError
+
+    def update_parameters(self, parameters, data_adaptor):
+        """Update configuration parameters that describe information about the annotations feature"""
+        params = {}
+        params["annotations_genesets_readonly"] = True
+        params["annotations_genesets_name_is_read_only"] = True
+        parameters.update(params)
+
+    @staticmethod
+    def gene_sets_to_csv(genesets):
+        """
+        Convert the internal genesets format (returned by read_gene_set) into
+        the simple Tidy CSV.
+        """
+        from io import StringIO
+
+        if isinstance(genesets, dict):
+            genesets = genesets.values()
+
+        with StringIO() as sio:
+            write_gene_sets_tidycsv(sio, genesets)
+            return sio.getvalue()
+
+    @staticmethod
+    def gene_sets_to_response(genesets):
+        """
+        Convert the internal genesets format (returned by read_gene_set) into
+        the dict expected by the JSON REST API
+        """
+        return list(genesets.values())
+
+    def read_gene_sets(self, data_adaptor, context=None):
+        if has_request_context():
+            if not current_app.auth.is_user_authenticated():
+                return ({}, 0)
+
+        gene_sets_uri_or_path = dataset_uri_to_geneset_uri(data_adaptor.data_locator.uri_or_path)
+
+        server_config = data_adaptor.server_config
+        region_name = None if server_config is None else server_config.data_locator__s3__region_name
+        gene_sets_locator = DataLocator(gene_sets_uri_or_path, region_name=region_name)
+        if not gene_sets_locator.exists():
+            return ({}, 0)
+
+        gene_sets = read_gene_sets_tidycsv(gene_sets_locator, context)
+        schema = data_adaptor.get_schema()
+        var_index = schema["annotations"]["var"].get("index", "index")
+        var_names = set(data_adaptor.query_var_array(var_index))
+
+        gene_sets = validate_gene_sets(gene_sets, var_names)
+        return (gene_sets, 0)
+
+
+def dataset_uri_to_geneset_uri(data_uri_or_path):
+    """ given a dataset URI, return the associated gene set URI """
+    data_basename = os.path.basename(data_uri_or_path)
+    base, ext = os.path.splitext(data_basename)
+    if ext is not None:  # strip extension, if any
+        data_basename = base
+
+    genesets_basename = f"{data_basename}-genesets.csv"
+    gene_sets_uri_or_path = path_join(data_uri_or_path, "..", genesets_basename)
+
+    return gene_sets_uri_or_path

backend/czi_hosted/common/annotations/hosted_tiledb.py

@@ -7,18 +7,18 @@ import pandas as pd
 import tiledb
 from flask import current_app
 
-from server.common.annotations.annotations import Annotations
+from backend.czi_hosted.common.annotations.annotations import Annotations
-from server.common.errors import AnnotationCategoryNameError
+from backend.common.errors import AnnotationCategoryNameError
-from server.common.utils.sanitization_utils import sanitize_values_in_list
+from backend.czi_hosted.common.utils.sanitization_utils import sanitize_values_in_list
-from server.common.utils.type_conversion_utils import get_dtypes_and_schemas_of_dataframe, get_dtype_of_array
+from backend.common.utils.type_conversion_utils import get_dtypes_and_schemas_of_dataframe, get_dtype_of_array
-from server.db.cellxgene_orm import Annotation
+from backend.czi_hosted.db.cellxgene_orm import Annotation
 
 
 class AnnotationsHostedTileDB(Annotations):
     CXG_ANNO_COLLECTION = "cxg_anno_collection"
 
-    def __init__(self, directory_path, db):
+    def __init__(self, config, directory_path, db):
-        super().__init__()
+        super().__init__(config)
         self.db = db
         if directory_path[-1] == "/":
             self.directory_path = directory_path
@@ -158,6 +158,8 @@ class AnnotationsHostedTileDB(Annotations):
         self.db.session.commit()
 
     def update_parameters(self, parameters, data_adaptor):
+        super().update_parameters(parameters, data_adaptor)
+
         params = {}
         params["annotations"] = True
         params["user_annotation_collection_name_enabled"] = False

backend/czi_hosted/common/annotations/local_file_csv.py

@@ -8,16 +8,16 @@ from hashlib import blake2b
 import pandas as pd
 from flask import session, has_request_context, current_app
 
-from server import __version__ as cellxgene_version
+from backend.czi_hosted import __version__ as cellxgene_version
-from server.common.annotations.annotations import Annotations
+from backend.czi_hosted.common.annotations.annotations import Annotations
-from server.common.errors import AnnotationsError
+from backend.common.errors import AnnotationsError
 
 
 class AnnotationsLocalFile(Annotations):
     CXG_ANNO_COLLECTION = "cxg_anno_collection"
 
-    def __init__(self, output_dir, output_file):
+    def __init__(self, config, output_dir, output_file):
-        super().__init__()
+        super().__init__(config)
         self.output_dir = output_dir
         self.output_file = output_file
         # lock used to protect label file write ops
@@ -169,6 +169,8 @@ class AnnotationsLocalFile(Annotations):
                 os.remove(os.path.join(backup_dir, bu))
 
     def update_parameters(self, parameters, data_adaptor):
+        super().update_parameters(parameters, data_adaptor)
+
         params = {}
         params["annotations"] = True
         params["user_annotation_collection_name_enabled"] = True
@@ -190,7 +192,7 @@ class AnnotationsLocalFile(Annotations):
             collection = self.get_collection()
             if current_app.auth.is_user_authenticated():
                 params["annotations-user-data-idhash"] = self._get_userdata_idhash(data_adaptor)
-                params["annotations-data-collection-is-read-only"] = False
+                params["annotations-data-collection-is-read-only"] = not self.user_annotations_enabled()
                 params["annotations-data-collection-name"] = collection
 
         parameters.update(params)

backend/czi_hosted/common/config/__init__.py

@@ -0,0 +1,4 @@
+from backend.common.utils.aws_secret_utils import get_secret_key  # noqa F504
+
+DEFAULT_SERVER_PORT = 5005
+BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20  # 100MB

backend/czi_hosted/common/config/app_config.py

@@ -1,11 +1,11 @@
 import yaml
 from flatten_dict import unflatten
 
-from server.default_config import get_default_config
+from backend.czi_hosted.common.config.external_config import ExternalConfig
-from server.common.config.dataset_config import DatasetConfig
+from backend.czi_hosted.common.config.dataset_config import DatasetConfig
-from server.common.config.server_config import ServerConfig
+from backend.czi_hosted.common.config.server_config import ServerConfig
-from server.common.config.external_config import ExternalConfig
+from backend.common.errors import ConfigurationError
-from server.common.errors import ConfigurationError
+from backend.czi_hosted.default_config import get_default_config
 
 
 class AppConfig(object):
@@ -69,21 +69,21 @@ class AppConfig(object):
 
     def update_server_config(self, **kw):
         self.server_config.update(**kw)
-        self.is_complete = False
+        self.is_completed = False
 
     def update_default_dataset_config(self, **kw):
         self.default_dataset_config.update(**kw)
         # update all the other dataset configs, if any
         for value in self.dataroot_config.values():
             value.update(**kw)
-        self.is_complete = False
+        self.is_completed = False
 
     def update_single_config_from_path_and_value(self, path, value):
         """Update a single config parameter with the value.
         Path is a list of string, that gives a path to the config parameter to be updated.
         For example, path may be ["server","app","port"].
         """
-        self.is_complete = False
+        self.is_completed = False
         if not isinstance(path, list):
             raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'")
         for part in path:
@@ -147,7 +147,7 @@ class AppConfig(object):
         if config.get("external"):
             self.external_config.update_from_config(config["external"], "external")
 
-        self.is_complete = False
+        self.is_completed = False
 
     def config_to_dict(self):
         """return the configuration as an unflattened dict"""

backend/czi_hosted/common/config/base_config.py

@@ -1,7 +1,7 @@
 import copy
 
 from flatten_dict import flatten
-from server.common.errors import ConfigurationError
+from backend.common.errors import ConfigurationError
 
 
 class BaseConfig(object):

backend/czi_hosted/common/config/client_config.py

@@ -1,4 +1,4 @@
-from server import display_version as cellxgene_display_version
+from backend.czi_hosted import display_version as cellxgene_display_version
 
 
 def get_client_config(app_config, data_adaptor):
@@ -44,6 +44,9 @@ def get_client_config(app_config, data_adaptor):
         "annotations": False,
         "annotations_file": None,
         "annotations_dir": None,
+        "annotations_genesets": True,  # feature flag
+        "annotations_genesets_readonly": True,
+        "annotations_genesets_summary_methods": ["mean"],
         "annotations_cell_ontology_enabled": False,
         "annotations_cell_ontology_obopath": None,
         "annotations_cell_ontology_terms": None,

backend/czi_hosted/common/config/dataset_config.py

@@ -1,13 +1,14 @@
 import os
 from os.path import splitext, isdir
 
-from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
+from backend.czi_hosted.common.annotations.annotations import Annotations
-from server.common.annotations.local_file_csv import AnnotationsLocalFile
+from backend.czi_hosted.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
-from server.common.config.base_config import BaseConfig
+from backend.czi_hosted.common.annotations.local_file_csv import AnnotationsLocalFile
-from server.common.errors import ConfigurationError, OntologyLoadFailure
+from backend.czi_hosted.common.config.base_config import BaseConfig
-from server.compute.scanpy import get_scanpy_module
+from backend.common.errors import ConfigurationError, OntologyLoadFailure
-from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataType
+from backend.czi_hosted.compute.scanpy import get_scanpy_module
-from server.db.db_utils import DbUtils
+from backend.czi_hosted.data_common.matrix_loader import MatrixDataLoader, MatrixDataType
+from backend.czi_hosted.db.db_utils import DbUtils
 
 
 class DatasetConfig(BaseConfig):
@@ -53,8 +54,10 @@ class DatasetConfig(BaseConfig):
         except KeyError as e:
             raise ConfigurationError(f"Unexpected config: {str(e)}")
 
-        # The annotation object is created during complete_config and stored here.
+        # Create the default annotation, which supports gene set reading without
-        self.user_annotations = None
+        # further configuration. Depending on configuration options, `complete_config` 
+        # may create a more specialized annotation object and replace this default.
+        self.user_annotations = Annotations()
 
     def complete_config(self, context):
         self.handle_app()
@@ -147,7 +150,11 @@ class DatasetConfig(BaseConfig):
             except OSError:
                 raise ConfigurationError("Unable to create directory specified by --annotations-dir")
 
-        self.user_annotations = AnnotationsLocalFile(dirname, filename)
+        anno_config = {
+            "user-annotations": self.user_annotations__enable,
+            "genesets-save": False,
+        }
+        self.user_annotations = AnnotationsLocalFile(anno_config, dirname, filename)
 
         # if the user has specified a fixed label file, go ahead and validate it
         # so that we can remove errors early in the process.
@@ -163,7 +170,12 @@ class DatasetConfig(BaseConfig):
         self.validate_correct_type_of_configuration_attribute(
             "user_annotations__hosted_tiledb_array__hosted_file_directory", str
         )
+        anno_config = {
+            "user-annotations": self.user_annotations__enable,
+            "genesets-save": False,
+        }
         self.user_annotations = AnnotationsHostedTileDB(
+            anno_config,
             directory_path=self.user_annotations__hosted_tiledb_array__hosted_file_directory,
             db=DbUtils(self.user_annotations__hosted_tiledb_array__db_uri),
         )

backend/czi_hosted/common/config/external_config.py

@@ -1,10 +1,9 @@
 import os
 
-from server.common.config.base_config import BaseConfig
+from backend.czi_hosted.common.config.base_config import BaseConfig
-from server.common.errors import ConfigurationError
+from backend.common.errors import ConfigurationError, SecretKeyRetrievalError
-from server.common.config import get_secret_key
+from backend.common.utils.aws_secret_utils import get_secret_key
-from server.common.errors import SecretKeyRetrievalError
+from backend.common.utils.type_conversion_utils import convert_string_to_value
-from server.common.utils.type_conversion_utils import convert_string_to_value
 
 
 class ExternalConfig(BaseConfig):

backend/czi_hosted/common/config/server_config.py

@@ -4,14 +4,14 @@ import warnings
 from os.path import basename
 from urllib.parse import urlparse, quote_plus
 
-from server.auth.auth import AuthTypeFactory
+from backend.czi_hosted.auth.auth import AuthTypeFactory
-from server.common.config.base_config import BaseConfig
+from backend.czi_hosted.common.config import DEFAULT_SERVER_PORT, BIG_FILE_SIZE_THRESHOLD
-from server.common.config import DEFAULT_SERVER_PORT, BIG_FILE_SIZE_THRESHOLD
+from backend.czi_hosted.common.config.base_config import BaseConfig
-from server.common.errors import ConfigurationError, DatasetAccessError
+from backend.common.utils.data_locator import discover_s3_region_name
-from server.common.data_locator import discover_s3_region_name
+from backend.common.errors import ConfigurationError, DatasetAccessError
-from server.common.utils.utils import is_port_available, find_available_port, custom_format_warning
+from backend.common.utils.utils import is_port_available, find_available_port, custom_format_warning
-from server.compute import diffexp_cxg as diffexp_tiledb
+from backend.czi_hosted.compute import diffexp_cxg as diffexp_tiledb
-from server.data_common.matrix_loader import MatrixDataCacheManager, MatrixDataLoader, MatrixDataType
+from backend.czi_hosted.data_common.matrix_loader import MatrixDataCacheManager, MatrixDataLoader, MatrixDataType
 
 
 class ServerConfig(BaseConfig):
@@ -42,6 +42,9 @@ class ServerConfig(BaseConfig):
             self.app__web_base_url = default_config["app"]["web_base_url"]
 
             self.authentication__type = default_config["authentication"]["type"]
+            self.authentication__insecure_test_environment = default_config["authentication"][
+                "insecure_test_environment"
+            ]
             self.authentication__params_oauth__oauth_api_base_url = default_config["authentication"]["params_oauth"][
                 "oauth_api_base_url"
             ]
@@ -168,6 +171,10 @@ class ServerConfig(BaseConfig):
 
     def handle_authentication(self):
         self.validate_correct_type_of_configuration_attribute("authentication__type", (type(None), str))
+        self.validate_correct_type_of_configuration_attribute("authentication__insecure_test_environment", bool)
+
+        if self.authentication__type == "test" and not self.authentication__insecure_test_environment:
+            raise ConfigurationError("Test auth can only be used in an insecure test environment")
 
         # oauth
         ptypes = str if self.authentication__type == "oauth" else (type(None), str)
@@ -346,7 +353,7 @@ class ServerConfig(BaseConfig):
             if type(self.data_locator__s3__region_name) == str:
                 self.adaptor__cxg_adaptor__tiledb_ctx[regionkey] = self.data_locator__s3__region_name
 
-        from server.data_cxg.cxg_adaptor import CxgAdaptor
+        from backend.czi_hosted.data_cxg.cxg_adaptor import CxgAdaptor
 
         CxgAdaptor.set_tiledb_context(self.adaptor__cxg_adaptor__tiledb_ctx)
 

backend/czi_hosted/common/corpora.py

@@ -0,0 +1,78 @@
+"""
+Corpora schema conventions support.  Helper functions for reading.
+
+https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md
+
+https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema_h5ad_implementation.md
+"""
+import collections
+import json
+
+from backend.czi_hosted.cli.upgrade import validate_version_str
+from backend.czi_hosted.common.utils.corpora_constants import CorporaConstants
+
+
+def corpora_get_versions_from_anndata(adata):
+    """
+    Given an AnnData object, return:
+        * None - if not a Corpora object
+        * [ corpora_schema_version, corpora_encoding_version ] - if a Corpora object
+
+    Implements the identification protocol defined in the specification.
+    """
+
+    # per Corpora AnnData spec, this is a corpora file if the following is true
+    if "version" not in adata.uns_keys():
+        return None
+    version = adata.uns["version"]
+    if not isinstance(version, collections.abc.Mapping) or "corpora_schema_version" not in version:
+        return None
+
+    corpora_schema_version = version.get("corpora_schema_version")
+    corpora_encoding_version = version.get("corpora_encoding_version")
+
+    # TODO: spec says these must be SEMVER values, so check.
+    if validate_version_str(corpora_schema_version) and validate_version_str(corpora_encoding_version):
+        return [corpora_schema_version, corpora_encoding_version]
+
+
+def corpora_is_version_supported(corpora_schema_version, corpora_encoding_version):
+    return (
+        corpora_schema_version
+        and corpora_encoding_version
+        and corpora_schema_version.startswith("1.")
+        and corpora_encoding_version.startswith("0.1.")
+    )
+
+
+def corpora_get_props_from_anndata(adata):
+    """
+    Get Corpora dataset properties from an AnnData
+    """
+    versions = corpora_get_versions_from_anndata(adata)
+    if versions is None:
+        return None
+    [corpora_schema_version, corpora_encoding_version] = versions
+    version_is_supported = corpora_is_version_supported(corpora_schema_version, corpora_encoding_version)
+    if not version_is_supported:
+        raise ValueError("Unsupported Corpora schema version")
+
+    corpora_props = {}
+    for key in CorporaConstants.REQUIRED_SIMPLE_METADATA_FIELDS:
+        if key not in adata.uns:
+            raise KeyError(f"missing Corpora schema field {key}")
+        corpora_props[key] = adata.uns[key]
+
+    for key in CorporaConstants.OPTIONAL_JSON_ENCODED_METADATA_FIELD:
+        if key not in adata.uns:
+            continue
+        try:
+            corpora_props[key] = json.loads(adata.uns[key])
+        except json.JSONDecodeError:
+            raise json.JSONDecodeError(f"Corpora schema field {key} is expected to be a valid JSON string")
+
+    for key in CorporaConstants.OPTIONAL_SIMPLE_METADATA_FIELDS:
+        if key in adata.uns:
+            corpora_props[key] = adata.uns[key]
+
+    return corpora_props

backend/czi_hosted/common/health.py

@@ -1,8 +1,8 @@
 from http import HTTPStatus
 from flask import make_response, jsonify
 
-from server import __version__ as cellxgene_version
+from backend.czi_hosted import __version__ as cellxgene_version
-from server.common.data_locator import DataLocator
+from backend.common.utils.data_locator import DataLocator
 
 
 def _is_accessible(path, config):

backend/czi_hosted/common/rest.py

@@ -3,13 +3,14 @@ import logging
 import sys
 from http import HTTPStatus
 import zlib
+import json
 
 from flask import make_response, jsonify, current_app, abort
 from werkzeug.urls import url_unquote
 
-from server.common.config.client_config import get_client_config, get_client_userinfo
+from backend.czi_hosted.common.config.client_config import get_client_config, get_client_userinfo
-from server.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg
+from backend.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg
-from server.common.errors import (
+from backend.common.errors import (
     FilterError,
     JSONEncodingValueError,
     PrepareError,
@@ -17,10 +18,11 @@ from server.common.errors import (
     ExceedsLimitError,
     DatasetAccessError,
     ColorFormatException,
+    AnnotationsError,
+    UnsupportedSummaryMethod,
 )
-
+from backend.common.genesets import summarizeQueryHash
-import json
+from backend.common.fbs.matrix import decode_matrix_fbs
-from server.data_common.fbs.matrix import decode_matrix_fbs
 
 
 def abort_and_log(code, logmsg, loglevel=logging.DEBUG, include_exc_info=False):
@@ -106,7 +108,7 @@ def schema_get_helper(data_adaptor):
 
     # add label obs annotations as needed
     annotations = data_adaptor.dataset_config.user_annotations
-    if annotations is not None:
+    if annotations.user_annotations_enabled():
         label_schema = annotations.get_schema(data_adaptor)
         schema["annotations"]["obs"]["columns"].extend(label_schema)
 
@@ -140,7 +142,7 @@ def annotations_obs_get(request, data_adaptor):
     try:
         labels = None
         annotations = data_adaptor.dataset_config.user_annotations
-        if annotations:
+        if annotations.user_annotations_enabled():
             labels = annotations.read_labels(data_adaptor)
         fbs = data_adaptor.annotation_to_fbs_matrix(Axis.OBS, fields, labels)
         return make_response(fbs, HTTPStatus.OK, {"Content-Type": "application/octet-stream"})
@@ -151,7 +153,7 @@ def annotations_obs_get(request, data_adaptor):
 def annotations_put_fbs_helper(data_adaptor, fbs):
     """helper function to write annotations from fbs"""
     annotations = data_adaptor.dataset_config.user_annotations
-    if annotations is None:
+    if not annotations.user_annotations_enabled():
         raise DisabledFeatureError("Writable annotations are not enabled")
 
     new_label_df = decode_matrix_fbs(fbs)
@@ -166,7 +168,7 @@ def inflate(data):
 
 def annotations_obs_put(request, data_adaptor):
     annotations = data_adaptor.dataset_config.user_annotations
-    if annotations is None:
+    if not annotations.user_annotations_enabled():
         return abort(HTTPStatus.NOT_IMPLEMENTED)
 
     anno_collection = request.args.get("annotation-collection-name", default=None)
@@ -197,7 +199,7 @@ def annotations_var_get(request, data_adaptor):
     try:
         labels = None
         annotations = data_adaptor.dataset_config.user_annotations
-        if annotations is not None:
+        if annotations.user_annotations_enabled():
             labels = annotations.read_labels(data_adaptor)
         return make_response(
             data_adaptor.annotation_to_fbs_matrix(Axis.VAR, fields, labels),
@@ -258,7 +260,6 @@ def diffexp_obs_post(request, data_adaptor):
     try:
         # TODO: implement varfilter mode
         mode = DiffExpMode(args["mode"])
-
         if mode == DiffExpMode.VAR_FILTER or "varFilter" in args:
             return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, "varFilter not enabled")
 
@@ -328,3 +329,70 @@ def layout_obs_put(request, data_adaptor):
         return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, str(e))
     except (ValueError, DisabledFeatureError, FilterError) as e:
         return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+
+
+def genesets_get(request, data_adaptor):
+    preferred_mimetype = request.accept_mimetypes.best_match(["application/json", "text/csv"])
+    if preferred_mimetype not in ("application/json", "text/csv"):
+        return abort(HTTPStatus.NOT_ACCEPTABLE)
+
+    try:
+        annotations = data_adaptor.dataset_config.user_annotations
+        (genesets, tid) = annotations.read_gene_sets(data_adaptor)
+
+        if preferred_mimetype == "text/csv":
+            return make_response(
+                annotations.gene_sets_to_csv(genesets),
+                HTTPStatus.OK,
+                {
+                    "Content-Type": "text/csv",
+                    "Content-Disposition": "attachment; filename=genesets.csv",
+                },
+            )
+        else:
+            return make_response(
+                jsonify({"genesets": annotations.gene_sets_to_response(genesets), "tid": tid}), HTTPStatus.OK
+            )
+    except (ValueError, KeyError, AnnotationsError) as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e))
+
+
+def summarize_var_helper(request, data_adaptor, key, raw_query):
+    preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
+    if preferred_mimetype != "application/octet-stream":
+        return abort(HTTPStatus.NOT_ACCEPTABLE)
+
+    summary_method = request.values.get("method", default="mean")
+    query_hash = summarizeQueryHash(raw_query)
+    if key and query_hash != key:
+        return abort(HTTPStatus.BAD_REQUEST, description="query key did not match")
+
+    args_filter_only = request.values.copy()
+    args_filter_only.poplist("method")
+    args_filter_only.poplist("key")
+
+    try:
+        filter = _query_parameter_to_filter(args_filter_only)
+        return make_response(
+            data_adaptor.summarize_var(summary_method, filter, query_hash),
+            HTTPStatus.OK,
+            {"Content-Type": "application/octet-stream"},
+        )
+    except (ValueError) as e:
+        return abort(HTTPStatus.NOT_FOUND, description=str(e))
+    except (UnsupportedSummaryMethod, FilterError) as e:
+        return abort(HTTPStatus.BAD_REQUEST, description=str(e))
+
+
+def summarize_var_get(request, data_adaptor):
+    return summarize_var_helper(request, data_adaptor, None, request.query_string)
+
+
+def summarize_var_post(request, data_adaptor):
+    if not request.content_type or "application/x-www-form-urlencoded" not in request.content_type:
+        return abort(HTTPStatus.UNSUPPORTED_MEDIA_TYPE)
+    if request.content_length > 1_000_000:  # just a sanity check to avoid memory exhaustion
+        return abort(HTTPStatus.BAD_REQUEST)
+
+    key = request.args.get("key", default=None)
+    return summarize_var_helper(request, data_adaptor, key, request.get_data())

backend/czi_hosted/common/utils/cxg_generation_utils.py

@@ -3,7 +3,7 @@ import json
 import numpy as np
 import tiledb
 
-from server.common.utils.type_conversion_utils import get_dtype_of_array, get_dtype_and_schema_of_array
+from backend.common.utils.type_conversion_utils import get_dtype_of_array, get_dtype_and_schema_of_array
 
 
 def convert_dictionary_to_cxg_group(cxg_container, metadata_dict, group_metadata_name="cxg_group_metadata"):

backend/czi_hosted/compute/diffexp_cxg.py

@@ -1,10 +1,11 @@
 import concurrent.futures
 import numpy as np
-from server.compute.diffexp_generic import diffexp_ttest_from_mean_var, mean_var_n
+
-from server.data_cxg.cxg_util import pack_selector_from_indices
-from server.common.errors import ComputeError
 from numba import jit
 
+from backend.czi_hosted.data_cxg.cxg_util import pack_selector_from_indices
+from backend.common.compute.diffexp_generic import diffexp_ttest_from_mean_var, mean_var_n
+from backend.common.errors import ComputeError
 
 """
 See the comments in diffexp_generic for a description of this algorithm
@@ -114,14 +115,14 @@ def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
             meanB += X_col_shift
 
     r = diffexp_ttest_from_mean_var(
-        meanA.astype(dtype),
+        meanA=meanA.astype(dtype),
-        varA.astype(dtype),
+        varA=varA.astype(dtype),
-        nA,
+        nA=nA,
-        meanB.astype(dtype),
+        meanB=meanB.astype(dtype),
-        varB.astype(dtype),
+        varB=varB.astype(dtype),
-        nB,
+        nB=nB,
-        top_n,
+        top_n=top_n,
-        diffexp_lfc_cutoff,
+        diffexp_lfc_cutoff=diffexp_lfc_cutoff
     )
 
     return r

backend/czi_hosted/converters/h5ad_data_file.py

@@ -6,17 +6,17 @@ import anndata
 import numpy as np
 import tiledb
 
-from server.common.colors import convert_anndata_category_colors_to_cxg_category_colors
+from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
-from server.common.corpora import corpora_get_props_from_anndata
+from backend.czi_hosted.common.corpora import corpora_get_props_from_anndata
-from server.common.errors import ColorFormatException
+from backend.common.errors import ColorFormatException
-from server.common.utils.cxg_constants import CxgConstants
+from backend.czi_hosted.common.utils.cxg_constants import CxgConstants
-from server.common.utils.cxg_generation_utils import (
+from backend.czi_hosted.common.utils.cxg_generation_utils import (
     convert_dictionary_to_cxg_group,
     convert_dataframe_to_cxg_array,
     convert_ndarray_to_cxg_dense_array,
     convert_matrix_to_cxg_array,
 )
-from server.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix
+from backend.czi_hosted.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix
 
 
 class H5ADDataFile:

backend/czi_hosted/converters/to_sparse.py

@@ -9,8 +9,9 @@ import sys
 
 import tiledb
 
-from server.common.utils.cxg_generation_utils import convert_ndarray_to_cxg_dense_array, convert_matrix_to_cxg_array
+from backend.czi_hosted.common.utils.cxg_generation_utils import convert_ndarray_to_cxg_dense_array, \
-from server.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix
+    convert_matrix_to_cxg_array
+from backend.czi_hosted.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix
 
 
 def main():

backend/czi_hosted/data_anndata/anndata_adaptor.py

@@ -8,15 +8,15 @@ from pandas.core.dtypes.dtypes import CategoricalDtype
 from scipy import sparse
 from server_timing import Timing as ServerTiming
 
-import server.compute.diffexp_generic as diffexp_generic
+import backend.common.compute.diffexp_generic as diffexp_generic
-from server.common.colors import convert_anndata_category_colors_to_cxg_category_colors
+from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
-from server.common.constants import Axis, MAX_LAYOUTS
+from backend.common.constants import Axis, MAX_LAYOUTS
-from server.common.corpora import corpora_get_props_from_anndata
+from backend.czi_hosted.common.corpora import corpora_get_props_from_anndata
-from server.common.errors import PrepareError, DatasetAccessError, FilterError
+from backend.common.errors import PrepareError, DatasetAccessError, FilterError
-from server.common.utils.type_conversion_utils import get_schema_type_hint_of_array
+from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
-from server.compute.scanpy import scanpy_umap
+from backend.czi_hosted.compute.scanpy import scanpy_umap
-from server.data_common.data_adaptor import DataAdaptor
+from backend.czi_hosted.data_common.data_adaptor import DataAdaptor
-from server.data_common.fbs.matrix import encode_matrix_fbs
+from backend.common.fbs.matrix import encode_matrix_fbs
 
 anndata_version = version.parse(str(anndata.__version__)).release
 

backend/czi_hosted/data_common/data_adaptor.py

@@ -3,13 +3,14 @@ from os.path import basename, splitext
 
 import numpy as np
 import pandas as pd
+from scipy import sparse
 from server_timing import Timing as ServerTiming
 
-from server.common.config.app_config import AppConfig
+from backend.czi_hosted.common.config.app_config import AppConfig
-from server.common.constants import Axis
+from backend.common.constants import Axis
-from server.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError
+from backend.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError, UnsupportedSummaryMethod, DatasetAccessError
-from server.common.utils.utils import jsonify_numpy
+from backend.common.utils.utils import jsonify_numpy
-from server.data_common.fbs.matrix import encode_matrix_fbs
+from backend.common.fbs.matrix import encode_matrix_fbs
 
 
 class DataAdaptor(metaclass=ABCMeta):
@@ -162,7 +163,7 @@ class DataAdaptor(metaclass=ABCMeta):
         mask = np.zeros((count,), dtype=np.bool)
         for i in filter:
             if type(i) == list:
-                mask[i[0] : i[1]] = True
+                mask[i[0]: i[1]] = True
             else:
                 mask[i] = True
         return mask
@@ -282,7 +283,7 @@ class DataAdaptor(metaclass=ABCMeta):
 
         try:
             obs_selector, var_selector = self._filter_to_mask(filter)
-        except (KeyError, IndexError, TypeError, AttributeError):
+        except (KeyError, IndexError, TypeError, AttributeError, DatasetAccessError):
             raise FilterError("Error parsing filter")
 
         if obs_selector is not None:
@@ -320,11 +321,12 @@ class DataAdaptor(metaclass=ABCMeta):
             top_n = self.dataset_config.diffexp__top_n
 
         if self.server_config.exceeds_limit(
-            "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B)
+                "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B)
         ):
             raise ExceedsLimitError("Diffexp request exceeds max cell count limit")
 
-        result = self.compute_diffexp_ttest(obs_mask_A, obs_mask_B, top_n, self.dataset_config.diffexp__lfc_cutoff)
+        result = self.compute_diffexp_ttest(
+            maskA=obs_mask_A, maskB=obs_mask_B, top_n=top_n, lfc_cutoff=self.dataset_config.diffexp__lfc_cutoff)
 
         try:
             return jsonify_numpy(result)
@@ -338,7 +340,7 @@ class DataAdaptor(metaclass=ABCMeta):
     @staticmethod
     def normalize_embedding(embedding):
         """Normalize embedding layout to meet client assumptions.
-           Embedding is an ndarray, shape (n_obs, n)., where n is normally 2
+        Embedding is an ndarray, shape (n_obs, n)., where n is normally 2
         """
 
         # scale isotropically
@@ -394,3 +396,26 @@ class DataAdaptor(metaclass=ABCMeta):
         except RuntimeError:
             lastmod = None
         return lastmod
+
+    def summarize_var(self, method, filter, query_hash):
+        if method != "mean":
+            raise UnsupportedSummaryMethod("Unknown gene set summary method.")
+
+        obs_selector, var_selector = self._filter_to_mask(filter)
+        if obs_selector is not None:
+            raise FilterError("filtering on obs unsupported")
+
+        # if no filter, just return zeros.  We don't have a use case
+        # for summarizing the entire X without a filter, and it would
+        # potentially be quite compute / memory intensive.
+        if var_selector is None or np.count_nonzero(var_selector) == 0:
+            mean = np.zeros((self.get_shape()[0], 1), dtype=np.float32)
+        else:
+            X = self.get_X_array(obs_selector, var_selector)
+            if sparse.issparse(X):
+                mean = X.mean(axis=1).A
+            else:
+                mean = X.mean(axis=1, keepdims=True)
+
+        col_idx = pd.Index([query_hash])
+        return encode_matrix_fbs(mean, col_idx=col_idx, row_idx=None)

backend/czi_hosted/data_common/matrix_loader.py

@@ -1,12 +1,14 @@
 from enum import Enum
 import threading
 import time
-from server.data_common.rwlock import RWLock
+
-from server.common.errors import DatasetAccessError
+from backend.common.utils.data_locator import DataLocator
-from server.common.data_locator import DataLocator
+from backend.common.errors import DatasetAccessError
 from contextlib import contextmanager
 from http import HTTPStatus
 
+from backend.czi_hosted.data_common.rwlock import RWLock
+
 
 class MatrixDataCacheItem(object):
     """This class provides access and caching for a dataset.  The first time a dataset is accessed, it is
@@ -224,7 +226,7 @@ class MatrixDataLoader(object):
 
         # matrix_data_type is an enum value of type MatrixDataType
         self.matrix_data_type = matrix_data_type
-        # matrix_type is a DataAdaptor type, which corresonds to the matrix_data_type
+        # matrix_type is a DataAdaptor type, which corresponds to the matrix_data_type
         self.matrix_type = None
 
         if matrix_data_type is None:
@@ -234,11 +236,11 @@ class MatrixDataLoader(object):
             raise DatasetAccessError("Dataset does not have an allowed type.")
 
         if self.matrix_data_type == MatrixDataType.H5AD:
-            from server.data_anndata.anndata_adaptor import AnndataAdaptor
+            from backend.czi_hosted.data_anndata.anndata_adaptor import AnndataAdaptor
 
             self.matrix_type = AnndataAdaptor
         elif self.matrix_data_type == MatrixDataType.CXG:
-            from server.data_cxg.cxg_adaptor import CxgAdaptor
+            from backend.czi_hosted.data_cxg.cxg_adaptor import CxgAdaptor
 
             self.matrix_type = CxgAdaptor
 

backend/czi_hosted/data_cxg/cxg_adaptor.py

@@ -8,15 +8,15 @@ import pandas as pd
 import tiledb
 from server_timing import Timing as ServerTiming
 
-import server.compute.diffexp_cxg as diffexp_cxg
+from backend.common.constants import Axis
-from server.common.constants import Axis
+from backend.common.errors import DatasetAccessError, ConfigurationError
-from server.common.errors import DatasetAccessError, ConfigurationError
+from backend.czi_hosted.common.immutable_kvcache import ImmutableKVCache
-from server.common.immutable_kvcache import ImmutableKVCache
+from backend.common.utils.type_conversion_utils import get_schema_type_hint_from_dtype
-from server.common.utils.type_conversion_utils import get_schema_type_hint_from_dtype
+from backend.common.utils.utils import path_join
-from server.common.utils.utils import path_join
+from backend.czi_hosted.compute import diffexp_cxg
-from server.data_common.data_adaptor import DataAdaptor
+from backend.czi_hosted.data_common.data_adaptor import DataAdaptor
-from server.data_common.fbs.matrix import encode_matrix_fbs
+from backend.common.fbs.matrix import encode_matrix_fbs
-from server.data_cxg.cxg_util import pack_selector_from_mask
+from backend.czi_hosted.data_cxg.cxg_util import pack_selector_from_mask
 
 
 class CxgAdaptor(DataAdaptor):
@@ -207,7 +207,8 @@ class CxgAdaptor(DataAdaptor):
             top_n = self.dataset_config.diffexp__top_n
         if lfc_cutoff is None:
             lfc_cutoff = self.dataset_config.diffexp__lfc_cutoff
-        return diffexp_cxg.diffexp_ttest(self, maskA, maskB, top_n, lfc_cutoff)
+        return diffexp_cxg.diffexp_ttest(
+            adaptor=self, maskA=maskA, maskB=maskB, top_n=top_n, diffexp_lfc_cutoff=lfc_cutoff)
 
     def get_colors(self):
         if self.cxg_version == "0.0":