Bump version: 0.16.8-rc.0 → 0.16.8

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.18.0
+current_version = 0.16.8
 commit = True
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?:-(?P<prerel>rc)\.(?P<prerelversion>\d+))?
 serialize = 
@@ -20,6 +20,6 @@ replace = version="{new_version}"
 search = "version": "{current_version}"
 replace = "version": "{new_version}"
 
-[bumpversion:file:backend/server/__init__.py]
+[bumpversion:file:server/__init__.py]
 search = __version__ = "{current_version}"
 replace = __version__ = "{new_version}"

.dockerignore

@@ -2,4 +2,4 @@ bin
 client
 dist
 docs
-backend
+server

.github/workflows/compatibility_tests.yml

@@ -9,7 +9,6 @@ on:
 
 env:
   JEST_ENV: prod
-  CXG_AUTH_TYPE: none
 
 jobs:
   docker-build:
@@ -29,8 +28,8 @@ jobs:
     continue-on-error: true
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8]
+        python-version: [3.6, 3.7] # As of Oct 2020 Anndata is not compatible with 3.8
-        anndata-version: [0.7.6]
+        anndata-version: [0.7.0, 0.7.1, 0.7.2, 0.7.3, 0.7.4, 0.7.5]
         test-suite: [smoke-test, smoke-test-annotations]
     steps:
     - uses: actions/checkout@v2
@@ -42,12 +41,14 @@ jobs:
       run: |
         # 1. only install the dev requirements on top of what is in the cellxgene pip package
         sudo apt-get update && sudo apt-get install -y libhdf5-serial-dev
-        sed -i 's/-r requirements.txt//' backend/server/requirements-dev.txt
+        sed -i 's/-r requirements.txt//' server/requirements-dev.txt
-        pip install -r backend/server/requirements-dev.txt
+        pip install -r server/requirements-dev.txt
         # 2. install cellxgene
         make pydist install-dist
         # 3. install anndata
         pip install anndata==${{ matrix.anndata-version }}
+        # workaround for anndata 0.6.22.post1 bug
+        [[ "0.6.22.post1" = "${{ matrix.anndata-version }}" ]] && pip install h5py==2.9.0 || true
     - name: Tests
       run: make unit-test ${{ matrix.test-suite }}
 
@@ -72,8 +73,8 @@ jobs:
           cd cellxgene
           # 1. only install the dev requirements on top of what is in the cellxgene pip package
           make dev-env-client
-          sed -i 's/-r requirements.txt//' backend/server/requirements-dev.txt
+          sed -i 's/-r requirements.txt//' server/requirements-dev.txt
-          pip install -r backend/server/requirements-dev.txt
+          pip install -r server/requirements-dev.txt
           # 2. install cellxgene
           pip install --upgrade cellxgene
           # 3. install anndata
@@ -100,7 +101,7 @@ jobs:
       - name: Install dependencies
         run: |
           cd cellxgene
-          sed -i -E 's/^anndata[>=]=[0-9]+.[0-9]+.[0-9]+$/anndata/g' backend/server/requirements.txt
+          sed -i -E 's/^anndata[>=]=[0-9]+.[0-9]+.[0-9]+$/anndata/g' server/requirements.txt
           make pydist install-dist dev-env
           pip install git+https://github.com/theislab/anndata
       - name: Tests

.github/workflows/deploy.yml

@@ -1,8 +1,8 @@
-name: Deploy canary via single cell infra repo
+name: Deploy via single cell infra repo
 
 on:
   push:
-    branches: main-canary
+    branches: main
 
 jobs:
   deploy:
@@ -10,4 +10,4 @@ jobs:
     steps:
       - name: repository dispatch
         run: |
-          curl -XPOST -u czi-sci-single-cell-eng:${{secrets.SCI_GITHUB_TOKEN}} -H "Accept: application/vnd.github.everest-preview+json"  -H "Content-Type: application/json" https://api.github.com/repos/chanzuckerberg/single-cell-infra/dispatches --data '{"event_type": "canary-hook"}'
+          curl -XPOST -u czi-sci-single-cell-eng:${{secrets.SCI_GITHUB_TOKEN}} -H "Accept: application/vnd.github.everest-preview+json"  -H "Content-Type: application/json" https://api.github.com/repos/chanzuckerberg/single-cell-infra/dispatches --data '{"event_type": "cellxgene-hook"}'

.github/workflows/push_tests.yml

@@ -36,11 +36,11 @@ jobs:
           npm install
       - name: Format with black and lint with flake8
         run: |
-          make lint-servers
+          make lint-server
       - name: Lint src with eslint
         working-directory: ./client
         run: |
-          npx eslint src __tests__
+          make lint
 
   unit-test:
     runs-on: ubuntu-latest
@@ -68,38 +68,8 @@ jobs:
         run: make pydist install-dist dev-env-server
       - name: Unit tests
         run: |
-          make unit-test-server
+          make unit-test
-          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k backend/server -cF backend,python,unitTest
+          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k server -cF backend,python,unitTest
-          cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest
-
-  unit-test-czi-hosted:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.7
-        uses: actions/setup-python@v1
-        with:
-          python-version: 3.7
-      - name: Python cache
-        uses: actions/cache@v1
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
-      - name: Node cache
-        uses: actions/cache@v1
-        with:
-          path: ~/.npm
-          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
-          restore-keys: |
-            ${{ runner.os }}-node-
-      - name: Install dependencies
-        run: make pydist-czi-hosted install-dist dev-env-czi-hosted
-      - name: Unit tests
-        run: |
-          make unit-test-czi-hosted
-          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k backend/czi-hosted -cF backend,python,unitTest
           cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest
 
   smoke-tests:
@@ -126,7 +96,7 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-node-
       - name: Install dependencies
-        run: make pydist-czi-hosted install-dist
+        run: make pydist install-dist
       - name: Smoke tests (without annotations feature)
         run: |
           cd client && make smoke-test

.github/workflows/sastisfaction.yml

@@ -1,27 +0,0 @@
-name: Run SASTisfaction
-on:
-  - pull_request
-
-jobs:
-  sastisfaction:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/checkout@v2
-        with:
-          repository: chanzuckerberg/sastisfaction
-          ref: main
-          path: .github/actions/sastisfaction
-          ssh-key: ${{ secrets.SASTISFACTION_READ_KEY }}
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Docker pull
-        run: docker pull ghcr.io/chanzuckerberg/sastisfaction:main
-      - name: Run SASTisfaction
-        uses: ./.github/actions/sastisfaction
-        with:
-          snowflake_private_key: ${{ secrets.SASTISFACTION_RSA_KEY }}

.github/workflows/scale-test.yml

@@ -15,15 +15,15 @@ jobs:
           python-version: 3.7
       - name: Install dependencies
         run: |
-          pip install -r backend/test/test_czi_hosted/locust/requirements-locust.txt
+          pip install -r server/test/locust/requirements-locust.txt
       - name: Dev Scale Test
         run: |
-          locust -f backend/test/test_czi_hosted/locust/locustfile.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt
+          locust -f server/test/locust/locustfile.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt
       - name: Slack success webhook
         env:
           SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
         run: |
-          DEV_STATS=$(tail -n 15 locust_dev_stats.txt)
+          DEV_STATS=$(tail -n 61 locust_dev_stats.txt)
           DEV_MSG="\`\`\`CELLXGENE EXPLORER DEV SCALE TEST RESULTS: ${DEV_STATS}\`\`\`"
           curl -X POST -H 'Content-type: application/json' --data "{'text':'${DEV_MSG}'}" $SLACK_WEBHOOK
 

.gitignore

@@ -19,13 +19,9 @@ venv/
 cellxgene/
 
 # client build
-backend/server/common/web/static/*
+server/common/web/static/*
-backend/server/common/web/templates/
+server/common/web/templates/
-backend/server/common/web/csp-hashes.json
+server/common/web/csp-hashes.json
-
-backend/czi_hosted/common/web/static/*
-backend/czi_hosted/common/web/templates/
-backend/czi_hosted/common/web/csp-hashes.json
 
 # eb build
 artifact.dir

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2017-2021 Chan Zuckerberg Initiative
+Copyright (c) 2017-2020 Chan Zuckerberg Initiative
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

MANIFEST.in

@@ -1,7 +1,7 @@
-recursive-include backend/server/common/web/templates *
+recursive-include server/common/web/templates *
-recursive-include backend/server/common/web/static *
+recursive-include server/common/web/static *
 
-include backend/server/requirements.txt
+include server/requirements.txt
-include backend/server/requirements-prepare.txt
+include server/requirements-prepare.txt
-include backend/server/converters/schema/hgnc_complete_set.txt.gz
+include server/converters/schema/hgnc_complete_set.txt.gz
-include backend/server/converters/schema/schema_definitions/*
+include server/converters/schema/schema_definitions/*

MANIFEST_hosted.in

@@ -1,7 +0,0 @@
-recursive-include backend/czi_hosted/common/web/templates *
-recursive-include backend/czi_hosted/common/web/static *
-
-include backend/czi_hosted/requirements.txt
-include backend/czi_hosted/requirements-prepare.txt
-include backend/czi_hosted/converters/schema/hgnc_complete_set.txt.gz
-include backend/czi_hosted/converters/schema/schema_definitions/*

Makefile

@@ -2,32 +2,23 @@ include common.mk
 
 BUILDDIR := build
 CLIENTBUILD := $(BUILDDIR)/client
-CZIHOSTEDBUILD := $(BUILDDIR)/backend/czi_hosted
+SERVERBUILD := $(BUILDDIR)/server
-SERVERBUILD := $(BUILDDIR)/backend/server
 CLEANFILES :=  $(BUILDDIR)/ client/build build dist cellxgene.egg-info
 
 PART ?= patch
 
 # CLEANING
 .PHONY: clean
-clean: clean-lite clean-czi-hosted clean-server clean-client
+clean: clean-lite clean-server clean-client
 
 # cleaning the client's node_modules is the longest one, so we avoid that if possible
 .PHONY: clean-lite
 clean-lite:
 	rm -rf $(CLEANFILES)
 
-.PHONY: clean-client
+clean-%:
-clean-client:
+	cd $(*) && $(MAKE) clean
-	cd client && $(MAKE) clean
 
-.PHONY: clean-server
-clean-server:
-	cd backend/server && $(MAKE) clean
-
-.PHONY: clean-czi-hosted
-clean-czi-hosted:
-	cd backend/czi_hosted && $(MAKE) clean
 
 # BUILDING PACKAGE
 
@@ -37,71 +28,29 @@ build-client:
 
 .PHONY: build
 build: clean build-client
-	git ls-files backend/server/ | grep -v 'backend/server/test/' | cpio -pdm $(BUILDDIR)
+	git ls-files server/ | grep -v 'server/test/' | cpio -pdm $(BUILDDIR)
 	cp -r client/build/  $(CLIENTBUILD)
 	$(call copy_client_assets,$(CLIENTBUILD),$(SERVERBUILD))
-	cp backend/__init__.py $(BUILDDIR)
-	cp backend/__init__.py $(BUILDDIR)/backend
-	cp -r backend/common $(BUILDDIR)/backend/common
 	cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR)
 
-.PHONY: build-czi-hosted
-build-czi-hosted: clean build-client
-	git ls-files backend/czi_hosted/ | grep -v 'backend/czi_hosted/test/' | cpio -pdm $(BUILDDIR)
-	cp -r client/build/  $(CLIENTBUILD)
-	$(call copy_client_assets,$(CLIENTBUILD),$(CZIHOSTEDBUILD))
-	cp -r backend/common $(BUILDDIR)/backend/common
-	cp backend/__init__.py $(BUILDDIR)
-	cp backend/__init__.py $(BUILDDIR)/backend
-	cp MANIFEST_hosted.in README.md setup.cfg setup_hosted.py $(BUILDDIR)
-	mv $(BUILDDIR)/setup_hosted.py $(BUILDDIR)/setup.py
-	mv $(BUILDDIR)/MANIFEST_hosted.in $(BUILDDIR)/MANIFEST.in
-
 # If you are actively developing in the server folder use this, dirties the source tree
 .PHONY: build-for-server-dev
 build-for-server-dev: clean-server build-client
-	$(call copy_client_assets,client/build,backend/server)
+	$(call copy_client_assets,client/build,server)
-
-.PHONY: build-for-czi-hosted-dev
-build-for-czi-hosted-dev: clean-czi-hosted build-client
-	$(call copy_client_assets,client/build,backend/czi_hosted)
 
 .PHONY: copy-client-assets
 copy-client-assets:
-	$(call copy_client_assets,client/build,backend/server)
+	$(call copy_client_assets,client/build,server)
-
-.PHONY: copy-client-assets-czi-hosted
-copy-client-assets-czi-hosted:
-	$(call copy_client_assets,client/build,backend/czi_hosted)
 
 # TESTING
 .PHONY: test
 test: unit-test smoke-test
 
 .PHONY: unit-test
-unit-test: unit-test-server unit-test-client unit-test-common
+unit-test: unit-test-server unit-test-client
 
-.PHONY: test-server
+unit-test-%:
-test-server: unit-test-server smoke-test
+	cd $(*) && $(MAKE) unit-test
-
-.PHONY: test-czi-hosted
-test-czi-hosted: unit-test-czi-hosted smoke-test
-
-.PHONY: unit-test-client
-unit-test-client:
-	cd client && $(MAKE) unit-test
-
-.PHONY: unit-test-czi-hosted
-unit-test-czi-hosted:
-	cd backend/czi_hosted && $(MAKE) unit-test
-
-.PHONY: unit-test-server
-unit-test-server:
-	cd backend/server && $(MAKE) unit-test
-
-.PHONY: unit-test-common
-unit-test-common:
-	cd backend/common && $(MAKE) unit-test
 
 .PHONY: smoke-test
 smoke-test:
@@ -113,11 +62,12 @@ smoke-test-annotations:
 
 .PHONY: test-db
 test-db:
-	cd backend/czi_hosted && $(MAKE) test-db
+	cd server && $(MAKE) test-db
+
 
 # FORMATTING CODE
 
-.PHONY: fmt
+.PHOHY: fmt
 fmt: fmt-client fmt-py
 
 .PHONY: fmt-client
@@ -129,18 +79,12 @@ fmt-py:
 	black .
 
 .PHONY: lint
-lint: lint-servers lint-client
+lint: lint-server lint-client
-
-.PHONY: lint-servers
-lint-servers: lint-server lint-czi-hosted-server
 
 .PHONY: lint-server
 lint-server: fmt-py
-	flake8 backend/server --per-file-ignores='backend/test/fixtures/dataset_config_outline.py:F821 backend/test/fixtures/server_config_outline.py:F821 backend/server/test/performance/scale_test_annotations.py:E501'
+	flake8 server --per-file-ignores='server/test/fixtures/dataset_config_outline.py:F821 server/test/fixtures/server_config_outline.py:F821 server/test/performance/scale_test_annotations.py:E501'
 
-.PHONY: lint-czi-hosted-server
-lint-czi-hosted-server: fmt-py
-	flake8 backend/czi_hosted --per-file-ignores='backend/test/fixtures/czi_hosted_dataset_config_outline.py:F821 backend/test/fixtures/czi_hosted_server_config_outline.py:F821 backend/test/performance/scale_test_annotations.py:E501'
 
 .PHONY: lint-client
 lint-client:
@@ -153,11 +97,6 @@ pydist: build
 	cd $(BUILDDIR); python setup.py sdist -d ../dist
 	@echo "done"
 
-.PHONY: pydist-czi-hosted
-pydist-czi-hosted: build-czi-hosted
-	cd $(BUILDDIR); python setup.py sdist -d ../dist
-	@echo "done"
-
 
 # RELEASE HELPERS
 
@@ -206,11 +145,8 @@ dev-env-client:
 
 .PHONY: dev-env-server
 dev-env-server:
-	pip install -r backend/server/requirements-dev.txt
+	pip install -r server/requirements-dev.txt
 
-.PHONY: dev-env-czi-hosted
-dev-env-czi-hosted:
-	pip install -r backend/czi_hosted/requirements-dev.txt
 # Set PART=[major, minor, patch] as param to make bump.
 # This will create a release candidate. (i.e. 0.16.1 -> 0.16.2-rc.0 for a patch bump)
 .PHONY: bump-version

README.md

@@ -52,6 +52,7 @@ cellxgene currently supports the following browsers:
 - Google Chrome 61+
 - Edge 15+
 - Firefox 60+
+- Safari 10.1+
 
 Please [file an issue](https://github.com/chanzuckerberg/cellxgene/issues/new/choose) if you would like us to add support for an unsupported browser.
 

backend/common/Makefile

@@ -1,11 +0,0 @@
-
-.PHONY: unit-test
-unit-test:
-	PYTHONWARNINGS=ignore:ResourceWarning coverage run \
-		--source=fbs,utils \
-		--omit=.coverage,data_common/fbs/NetEncoding,venv \
-		-m unittest discover \
-		--start-directory ../test/test_common/unit \
-		--top-level-directory ../../ \
-		--verbose; test_result=$$?; \
-	exit $$test_result \

backend/common/compute/estimate_distribution.py

@@ -1,108 +0,0 @@
-from typing import Tuple
-import numba
-import concurrent.futures
-import numpy as np
-from scipy import sparse
-from backend.common.constants import XApproximateDistribution
-
-
-@numba.njit(error_model="numpy", nogil=True)
-def min_max_fast(arr: np.ndarray) -> Tuple[float, float]:
-    """Return (min, max) values for the ndarray."""
-
-    # initialize to first finite value in array.  Normally,
-    # this will exit on the first value.
-    for i in range(arr.size):
-        min_val = max_val = arr[i]
-        if np.isfinite(min_val):
-            break
-
-    # now find min/max, unrolled by two
-    odd = arr.size % 2
-    unrolled_loop_limit = arr.size - 1 if odd else arr.size
-    i = 0
-    while i < unrolled_loop_limit:
-        x = arr[i]
-        y = arr[i + 1]
-
-        # ignore non-finites
-        x = x if np.isfinite(x) else min_val
-        y = y if np.isfinite(y) else min_val
-
-        if x > y:
-            x, y = y, x
-        min_val = min(x, min_val)
-        max_val = max(y, max_val)
-        i += 2
-
-    # handle the tail if any
-    if odd:
-        x = arr[arr.size - 1]
-
-        # ignore non-finites
-        x = x if np.isfinite(x) else min_val
-
-        min_val = min(x, min_val)
-        max_val = max(x, max_val)
-
-    return min_val, max_val
-
-
-def min_max_numpy(arr: np.ndarray) -> Tuple[float, float]:
-    return arr.min(), arr.max()
-
-
-def numba_has_support_for_scalar_type(arr: np.ndarray) -> bool:
-    """Numba does not support half-floats, 128 bit floats, ints > 64 bit or non-scalars."""
-    if arr.dtype == np.float32 or arr.dtype == np.float64:
-        return True
-
-    if np.issubdtype(arr.dtype, np.integer) and arr.dtype <= np.int64:
-        return True
-
-    if arr.dtype == np.bool_:
-        return True
-
-    return False
-
-
-def estimate_approximate_distribution(X) -> XApproximateDistribution:
-    """
-    Estimate the distribution (normal, count) of the X matrix.
-
-    Currently this is based upon the assumption that scRNA-seq data is
-    exponentially distributed in its raw (count) form, and when logged,
-    any (max-min) range in excess of 24 is implies tens of millions of
-    observations of a single feature and so is extremely unlikely.
-    """
-    if X.dtype.kind not in ["i", "u", "f"]:
-        raise TypeError(f"Unsupported matrix dtype: {X.dtype.name}")
-
-    if X.size == 0:
-        # default for empty array
-        return XApproximateDistribution.NORMAL
-
-    if sparse.isspmatrix_csc(X) or sparse.isspmatrix_csr(X):
-        Xdata = X.data
-    elif type(X) is np.ndarray:
-        Xdata = X.reshape(
-            X.size,
-        )
-    else:
-        raise TypeError(f"Unsupported matrix format: {str(type(X))}")
-
-    min_max = min_max_fast if numba_has_support_for_scalar_type(Xdata) else min_max_numpy
-
-    CHUNKSIZE = 1 << 24
-    if Xdata.size > CHUNKSIZE:
-        min_val = max_val = Xdata[0]
-        with concurrent.futures.ThreadPoolExecutor() as tp:
-            for (_min, _max) in tp.map(min_max, [Xdata[i : i + CHUNKSIZE] for i in range(0, Xdata.size, CHUNKSIZE)]):
-                min_val = min(_min, min_val)
-                max_val = max(_max, max_val)
-
-    else:
-        min_val, max_val = min_max(Xdata)
-
-    excess_range = (max_val - min_val) > 24
-    return XApproximateDistribution.COUNT if excess_range else XApproximateDistribution.NORMAL

backend/common/genesets.py

@@ -1,239 +0,0 @@
-"""
-Utility code for gene sets handling
-"""
-
-import re
-import csv
-import hashlib
-
-from .errors import AnnotationsError
-
-
-GENESETS_TIDYCSV_HEADER = [
-    "gene_set_name",
-    "gene_set_description",
-    "gene_symbol",
-    "gene_description",
-]
-
-
-def read_gene_sets_tidycsv(gs_locator, context=None):
-    """
-    Read & parse the Tidy CSV format, applying validation checks for mandatory
-    values, and de-duping rules.
-
-    Format is a four-column CSV, with a mandatory header row, and optional "#" prefixed
-    comments.  Format:
-
-        gene_set_name, gene_set_description, gene_symbol, gene_description
-
-    gene_set_name must be non-null; others are optional.
-
-    Returns: a dictionary of the shape (values in angle-brackets vary):
-
-        {
-            <string, a gene set name>: {
-                "geneset_name": <string, a gene set name>,
-                "geneset_description": <a string or None>,
-                "genes": [
-                    {
-                        "gene_symbol": <string, a gene symbol or name>,
-                        "gene_description": <a string or None>
-                    },
-                    ...
-                ]
-            },
-            ...
-        }
-    """
-
-    class myDialect(csv.excel):
-        skipinitialspace = False
-
-    def just(n, seq):
-        it = iter(seq)
-        for _ in range(n - 1):
-            yield next(it, "")
-        yield tuple(it)
-
-    messagefn = context["messagefn"] if context else (lambda x: None)
-
-    gene_sets = {}
-    with gs_locator.local_handle() as fname:
-        with open(fname, newline="") as f:
-            reader = csv.reader(f, dialect=myDialect())
-            haveReadHeader = False
-            lineno = 0
-            for row in reader:
-                lineno += 1
-                # ignore empty rows
-                if len(row) == 0:
-                    continue
-                # if row starts with '#' it is a comment
-                if row[0].startswith("#"):
-                    continue
-                # if this is the first non-comment row, assume it is a header and validate
-                # column names.  OK if the user has extra columns after our initial set.
-                if not haveReadHeader:
-                    if row[0:len(GENESETS_TIDYCSV_HEADER)] != GENESETS_TIDYCSV_HEADER:
-                        raise AnnotationsError("Gene set CSV file missing the required column header.")
-                    haveReadHeader = True
-                    continue
-
-                geneset_name, geneset_description, gene_symbol, gene_description, _ = just(5, row)
-                if not geneset_name:
-                    raise AnnotationsError(f"Gene set CSV missing required gene set name on line {lineno}")
-                if (not gene_symbol) and gene_description:
-                    messagefn(f"Warning: Missing gene name in gene set name {geneset_name} on line {lineno}.")
-
-                if geneset_name in gene_sets:
-                    gs = gene_sets[geneset_name]
-                else:
-                    gs = gene_sets[geneset_name] = {
-                        "geneset_name": geneset_name,
-                        "geneset_description": geneset_description,
-                        "genes": [],
-                    }
-                # Use first geneset_description with a value
-                if not gs["geneset_description"] and geneset_description:
-                    gs["geneset_description"] = geneset_description
-                # add the gene if the gene_symbol is defined
-                if gene_symbol:
-                    gs["genes"].append(
-                        {
-                            "gene_symbol": gene_symbol,
-                            "gene_description": gene_description,
-                        }
-                    )
-
-    return gene_sets
-
-
-def write_gene_sets_tidycsv(f, genesets):
-    """
-    Convert the internal gene sets format (returned by read_gene_set_tidycsv) into
-    the simple Tidy CSV.
-    """
-    writer = csv.writer(f, dialect="excel")
-    writer.writerow(GENESETS_TIDYCSV_HEADER)
-    for geneset in genesets:
-        # genes may be empty, in which case we skip the gene set entirely
-        genes = geneset["genes"]
-        if not genes:
-            writer.writerow([geneset["geneset_name"], geneset.get("geneset_description", ""), "", ""])
-        else:
-            writer.writerows(
-                [
-                    [
-                        geneset["geneset_name"],
-                        geneset.get("geneset_description", ""),
-                        gene["gene_symbol"],
-                        gene.get("gene_description", ""),
-                    ]
-                    for gene in genes
-                ]
-            )
-
-
-def summarizeQueryHash(raw_query):
-    """ generate a cache key (hash) from the raw query string """
-    return hashlib.sha1(raw_query).hexdigest()
-
-
-def validate_gene_sets(genesets, var_names, context=None):
-    """
-    Check validity of gene sets, return if correct, else raise error.
-    May also modify the gene set for conditions that should be resolved,
-    but which do not warrant a hard error.
-
-    Argument gene sets may be either the REST OTA format (list of dicts) or the internal
-    format (dict of dicts, keyed by the gene set name).
-
-    Will return a modified gene sets (eg, remove warnings) of the same type as the
-    provided argument. Ie, dict->dict, list->list
-
-    Rules:
-
-    0. All gene set names must be unique. [error]
-    1. Gene set names must conform to the following: [error]
-        * Names must be comprised of 1 or more ASCII characters 32-126
-        * No leading or trailing spaces (ASCII 32)
-        * No multi-space (ASCII 32) runs
-    2. Gene symbols must be part of the current var_index. [warning]
-       If gene symbol is not in the var_index, generate a warning and remove the symbol
-       from the gene sets.
-    3. Gene symbols must not be duplicated in a gene set.  [warning]
-       Duplications will be silently de-duped.
-
-    Items marked [error] will generate a hard error, causing the validation to fail.
-
-    Items marked [warning] will generate a warning, and will be resolved without failing
-    the validation (typically by removing the offending item from the gene sets).
-    """
-
-    messagefn = context["messagefn"] if context else (lambda x: None)
-
-    # accept genesets args as either the internal (dict) or REST (list) format,
-    # as they are identical except for the dict being keyed by geneset_name.
-    if not isinstance(genesets, dict) and not isinstance(genesets, list):
-        raise ValueError("Gene sets must be either dict or list.")
-    genesets_iterable = genesets if isinstance(genesets, list) else genesets.values()
-
-    # 0. check for uniqueness of geneset names
-    geneset_names = [gs["geneset_name"] for gs in genesets_iterable]
-    if len(set(geneset_names)) != len(geneset_names):
-        raise KeyError("All gene set names must be unique.")
-
-    # 1. check gene set character set and format
-    illegal_name = re.compile(r"^\s|  |[\u0000-\u001F\u007F-\uFFFF]|\s$")
-    for name in geneset_names:
-        if type(name) != str or len(name) == 0:
-            raise KeyError("Gene set names must be non-null string.")
-        if illegal_name.search(name):
-            messagefn(
-                "Error: "
-                f"Gene set name {name} "
-                "is not valid. Leading, trailing, and multiple spaces within a name are not allowed."
-            )
-            raise KeyError(
-                "Gene set name is not valid. Leading, trailing, and multiple spaces within a name are not allowed."
-            )
-
-    # 2. & 3. check for duplicate gene symbols, and those not present in the dataset. They will
-    # generate a warning and be removed.
-    for geneset in genesets_iterable:
-        if not isinstance(geneset, dict):
-            raise ValueError("Each gene set must be a dict.")
-        geneset_name = geneset["geneset_name"]
-        genes = geneset["genes"]
-        if not isinstance(genes, list):
-            raise ValueError("Gene set genes field must be a list")
-        geneset.setdefault("geneset_description", "")
-        gene_symbol_already_seen = set()
-        new_genes = []
-        for gene in genes:
-            gene_symbol = gene["gene_symbol"]
-            if not isinstance(gene_symbol, str) or len(gene_symbol) == 0:
-                raise ValueError("Gene symbol must be non-null string.")
-            if gene_symbol in gene_symbol_already_seen:
-                # duplicate check
-                messagefn(
-                    f"Warning: a duplicate of gene {gene_symbol} was found in gene set {geneset_name}, "
-                    "and will be ignored."
-                )
-                continue
-
-            if gene_symbol not in var_names:
-                messagefn(
-                    f"Warning: {gene_symbol}, used in gene set {geneset_name}, "
-                    "was not found in the dataset and will be ignored."
-                )
-                continue
-
-            gene_symbol_already_seen.add(gene_symbol)
-            gene.setdefault("gene_description", "")
-            new_genes.append(gene)
-
-        geneset["genes"] = new_genes
-
-    return genesets

backend/common/utils/type_conversion_utils.py

@@ -1,191 +0,0 @@
-from typing import Union, Tuple
-import logging
-
-import numpy as np
-import pandas as pd
-
-"""
-These routines drive all type inference for the schema generation and the
-FBS (REST OTA) encoding. They are also used for CXG generation.
-
-
-H5AD Type                       REST              REST
-(ndarray, Series, Index)        FBS encoding      schema type       ERROR/exceptions
-----------------------------    --------------    ---------------   ----------------------
-bool_/bool                      uint8             boolean
-(u)int8, (u)int16, int32        int32             int32
-uint32, (u)int64                int32             int32             CHECKS value bounds
-float16, float32, float64       float32           float32[0]
-
-categorical[T is numeric[4]]:
-    hasna = False               T                 categorical[1]
-    hasna = True                float32           categorical[1]    CHECKS value bounds
-
-categorical[T not numeric]      JSON/str          categorical[1,2]
-
-(other object)                  JSON/str          string
-
-(all other)                                                         Always an ERROR[3]
-
-
-Notes:
-[0] IEEE format, includes non-finite numbers (NaN, Inf, ...)
-[1] with NO categories enumerated (client side does it to handle rounding)
-[2] NA (undefined) categories are assigned a JSON null value
-[3] Includes all other numpy types:  datetime, complex, etc.
-[4] means float, int, uint (dtype.kind in ['i','u','f'])
-
-"""
-
-
-def get_dtypes_and_schemas_of_dataframe(dataframe: pd.DataFrame):
-    dtypes_by_column_name = {}
-    schema_type_hints_by_column_name = {}
-
-    for column_name, column_values in dataframe.items():
-        (
-            dtypes_by_column_name[column_name],
-            schema_type_hints_by_column_name[column_name],
-        ) = get_dtype_and_schema_of_array(column_values)
-
-    return dtypes_by_column_name, schema_type_hints_by_column_name
-
-
-def get_encoding_dtype_of_array(array: Union[np.ndarray, pd.Series, pd.Index]) -> np.dtype:
-    return _get_type_info(array)[0]
-
-
-def get_schema_type_hint_of_array(array: Union[np.ndarray, pd.Series, pd.Index]) -> dict:
-    return _get_type_info(array)[1]
-
-
-def get_dtype_and_schema_of_array(array: Union[np.ndarray, pd.Series, pd.Index]) -> Tuple[np.dtype, dict]:
-    """Return tuple (encoding_dtype, schema_type_hint)"""
-    return _get_type_info(array)
-
-
-def get_schema_type_hint_from_dtype(dtype) -> dict:
-    res = _get_type_info_from_dtype(dtype)
-    if res is None:
-        raise TypeError(f"Annotations of type {dtype} are unsupported.")
-    else:
-        return res[1]
-
-
-def _get_type_info_from_dtype(dtype) -> Union[Tuple[np.dtype, dict], None]:
-    """
-    Best-effort to determine encoding type and schema hint from a dtype.
-    If this is not possible, or the type is unsupported, return None.
-
-    This should be a subset of the cases which are supported by
-    _get_type_info().  The latter should be preferred if the array (values)
-    are available for typing.
-    """
-    if dtype.kind == "b":
-        return (np.uint8, {"type": "boolean"})
-
-    if dtype.kind == "U":
-        return (np.dtype(str), {"type": "string"})
-
-    if dtype.kind in ["i", "u"]:
-        if np.can_cast(dtype, np.int32):
-            return (np.int32, {"type": "int32"})
-
-    if dtype.kind == "f":
-        _float64_warning(dtype)
-        return (np.float32, {"type": "float32"})
-
-    if dtype.kind == "O" and not dtype.name == "category":
-        return (np.dtype(str), {"type": "string"})
-
-    return None
-
-
-def _get_type_info(array: Union[np.ndarray, pd.Series, pd.Index]) -> Tuple[np.dtype, dict]:
-    """
-    Determine encoding type and schema hint from an array.  This allows more
-    flexible casting than may be possible by using just the dtype, as it can
-    account for category types and array values.
-    """
-    if (
-        not isinstance(array, np.ndarray)
-        and not isinstance(array, pd.Series)
-        and not isinstance(array, pd.Index)
-        and not hasattr(array, "dtype")
-    ):
-        raise TypeError("Unsupported data type.")
-
-    dtype = array.dtype
-
-    res = _get_type_info_from_dtype(dtype)
-    if res is not None:
-        return res
-
-    if dtype.kind == "O":
-        if dtype.name == "category":
-            # Sometimes CategoricalDType can be encoded as int or float without further fuss.
-            # Do not specify the categories in the schema - let the client-side figure it out
-            # on its own.  Utilize Series.to_numpy() to do casting that handles categorical
-            # NA/NaN (missing or undefined) categories.
-            if dtype.categories.dtype.kind in ["f", "i", "u"]:
-                return (
-                    _get_type_info(array.to_numpy())[0],
-                    {"type": "categorical"},
-                )
-            else:
-                return (np.dtype(str), {"type": "categorical", "categories": dtype.categories.to_list()})
-
-        # all other extension types are str-encoded
-        return (np.dtype(str), {"type": "string"})
-
-    if dtype.kind in ["i", "u"] and _can_cast_array_values_to_int32(array):
-        return (np.int32, {"type": "int32"})
-
-    if dtype.kind == "f":
-        _float64_warning(array.dtype)
-        return (np.float32, {"type": "float32"})
-
-    raise TypeError(f"Annotations of type {dtype} are unsupported.")
-
-
-def _float64_warning(dtype):
-    """
-    Warn the user if we are down-casting a float64 to float32, and may potentially lose information.
-    """
-    if dtype.kind == "f" and not np.can_cast(dtype, np.float32):
-        logging.warning(f"Type {dtype.name} will be converted to 32 bit float and may lose precision.")
-
-
-def _can_cast_array_values_to_int32(array: Union[np.ndarray, pd.Series, pd.Index]) -> bool:
-    """
-    Return true if the (U)INT array values can be safely cast to int32.  We allow size reducing
-    casts (ie, int64 to int32) if no actual values require the larger size (ie, actual values
-    can be represented by the smaller type).
-    """
-    assert array.dtype.kind in ["u", "i"]
-
-    if np.can_cast(array.dtype, np.int32):
-        return True
-
-    if array.size == 0:
-        return True
-
-    int32_machine_limits = np.iinfo(np.int32)
-    if array.min() >= int32_machine_limits.min and array.max() <= int32_machine_limits.max:
-        return True
-
-    return False
-
-
-def convert_string_to_value(value: str):
-    """convert a string to value with the most appropriate type"""
-    if value.lower() == "true":
-        return True
-    if value.lower() == "false":
-        return False
-    if value == "null":
-        return None
-    try:
-        return eval(value)
-    except:  # noqa E722
-        return value

backend/czi_hosted/__init__.py

@@ -1,15 +0,0 @@
-import logging
-import sys
-from backend.common.utils.utils import import_plugins
-
-__version__ = "0.16.7"
-
-
-display_version = "cellxgene v" + __version__
-
-try:
-    import_plugins("backend.czi_hosted.plugins")
-except Exception as e:
-    # Make sure to exit in this case, as the server may not be configured as expected.
-    logging.critical(f"Error in import_plugins: {str(e)}")
-    sys.exit(1)

backend/czi_hosted/__main__.py

@@ -1,14 +0,0 @@
-# Work around bug https://github.com/pallets/werkzeug/issues/461
-if __package__ is None:
-    import sys
-    from pathlib import Path
-
-    PKG_PATH = Path(__file__).parent
-    sys.path.insert(0, str(PKG_PATH.parent))
-    import backend.czi_hosted # noqa F401
-    __package__ = PKG_PATH.name
-
-# Main thing
-from .cli.cli import cli  # noqa F402
-
-cli()

backend/czi_hosted/auth/__init__.py

@@ -1,6 +0,0 @@
-# import the built in auth types so they can be registered
-
-import backend.czi_hosted.auth.auth_test  # noqa: F401
-import backend.czi_hosted.auth.auth_session  # noqa: F401
-import backend.czi_hosted.auth.auth_oauth  # noqa: F401
-import backend.czi_hosted.auth.auth_none  # noqa: F401

backend/czi_hosted/auth/auth_none.py

@@ -1,27 +0,0 @@
-from backend.czi_hosted.auth.auth import AuthTypeBase, AuthTypeFactory
-
-
-class AuthTypeNone(AuthTypeBase):
-    def __init__(self, app_config):
-        super().__init__()
-
-    def is_valid_authentication_type(self):
-        return False
-
-    def complete_setup(self, app):
-        pass
-
-    def is_user_authenticated(self):
-        return True
-
-    def get_user_id(self):
-        return None
-
-    def get_user_name(self):
-        return None
-
-    def get_user_email(self):
-        return None
-
-
-AuthTypeFactory.register(None, AuthTypeNone)

backend/czi_hosted/auth/auth_session.py

@@ -1,40 +0,0 @@
-from flask import session
-from uuid import uuid4
-
-from backend.czi_hosted.auth.auth import AuthTypeBase, AuthTypeFactory
-
-
-class AuthTypeSession(AuthTypeBase):
-    """Session based authentication.  The user is always logged.  The user id is a random number
-    associated with the session.  This is a good choice for desktop servers."""
-
-    # key in the session token for userid
-    CXGUID = "cxguid"
-
-    def __init__(self, app_config):
-        super().__init__()
-
-    def is_valid_authentication_type(self):
-        return True
-
-    def complete_setup(self, app):
-        pass
-
-    def is_user_authenticated(self):
-        # always authenticated
-        return True
-
-    def get_user_id(self):
-        if self.CXGUID not in session:
-            session[self.CXGUID] = uuid4().hex
-            session.permanent = True
-        return session[self.CXGUID]
-
-    def get_user_name(self):
-        return "anonymous"
-
-    def get_user_email(self):
-        return None
-
-
-AuthTypeFactory.register("session", AuthTypeSession)

backend/czi_hosted/cli/schema.py

@@ -1,72 +0,0 @@
-import click
-
-from backend.czi_hosted.converters.schema import remix, validate
-
-
-@click.group(
-    name="schema",
-    subcommand_metavar="COMMAND <args>",
-    short_help="Apply and validate the cellxgene data integration schema to an h5ad file.",
-    context_settings=dict(max_content_width=85, help_option_names=["-h", "--help"]),
-)
-def schema_cli():
-    try:
-        import scanpy  # noqa: F401
-    except ImportError:
-        raise click.ClickException(
-            "[cellxgene] cellxgene schema requires scanpy"
-        )
-
-
-@click.command(
-    name="apply",
-    short_help="(experimental) Apply the cellxgene data integration schema to an h5ad.",
-    help="(experimental) Using a yaml file that describes schema values to insert or convert and in input "
-    "h5ad file, apply the schema changes and create a new, conforming h5ad.",
-)
-@click.option(
-    "--source-h5ad",
-    help="Input h5ad file.",
-    nargs=1,
-    required=True,
-    type=click.Path(exists=True, dir_okay=False),
-)
-@click.option(
-    "--remix-config",
-    help="Config yaml with information on how to apply the schema.",
-    nargs=1,
-    required=True,
-    type=click.Path(exists=True, dir_okay=False),
-)
-@click.option(
-    "--output-filename",
-    help="Filename for the new, schema-conforming h5ad file.",
-    required=True,
-    nargs=1
-)
-def schema_apply(source_h5ad, remix_config, output_filename):
-    remix.apply_schema(source_h5ad, remix_config, output_filename)
-
-
-@click.command(
-    name="validate",
-    short_help="(experimental) Check that an h5ad follows the cellxgene data integration schema.",
-)
-@click.argument(
-    "h5ad",
-    nargs=1,
-    type=click.Path(exists=True, dir_okay=False),
-)
-@click.option(
-    "--shallow",
-    help="When true, just check that the correct version information is present.",
-    default=False,
-    show_default=True,
-    is_flag=True,
-)
-def schema_validate(h5ad, shallow):
-    validate.validate(h5ad, shallow)
-
-
-schema_cli.add_command(schema_apply)
-schema_cli.add_command(schema_validate)

backend/czi_hosted/common/annotations/annotations.py

@@ -1,110 +0,0 @@
-import os
-
-from flask import current_app, has_request_context
-
-from backend.common.errors import DisabledFeatureError
-from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
-from backend.common.genesets import write_gene_sets_tidycsv, read_gene_sets_tidycsv, validate_gene_sets
-from backend.common.utils.data_locator import DataLocator
-from backend.common.utils.utils import path_join
-
-
-class Annotations:
-    """baseclass for annotations and genesets"""
-
-    def __init__(self, config={}):
-        self.config = config
-
-    def user_annotations_enabled(self):
-        return self.config.get("user-annotations", False)
-
-    def check_user_annotations_enabled(self):
-        if not self.user_annotations_enabled():
-            raise DisabledFeatureError("User annotations are disabled.")
-
-    def get_schema(self, data_adaptor):
-        schema = []
-        labels = self.read_labels(data_adaptor)
-        if labels is not None and not labels.empty:
-            for col in labels.columns:
-                col_schema = dict(name=col, writable=True)
-                col_schema.update(get_schema_type_hint_of_array(labels[col]))
-                schema.append(col_schema)
-
-        return schema
-
-    def set_collection(self, name):
-        """set or create a new annotation collection"""
-        raise NotImplementedError
-
-    def read_labels(self, data_adaptor):
-        """Return the labels as a pandas.DataFrame"""
-        raise NotImplementedError
-
-    def write_labels(self, df, data_adaptor):
-        """Write the labels (df) to a persistent storage such that it can later be read"""
-        raise NotImplementedError
-
-    def update_parameters(self, parameters, data_adaptor):
-        """Update configuration parameters that describe information about the annotations feature"""
-        params = {}
-        params["annotations_genesets_readonly"] = True
-        params["annotations_genesets_name_is_read_only"] = True
-        parameters.update(params)
-
-    @staticmethod
-    def gene_sets_to_csv(genesets):
-        """
-        Convert the internal genesets format (returned by read_gene_set) into
-        the simple Tidy CSV.
-        """
-        from io import StringIO
-
-        if isinstance(genesets, dict):
-            genesets = genesets.values()
-
-        with StringIO() as sio:
-            write_gene_sets_tidycsv(sio, genesets)
-            return sio.getvalue()
-
-    @staticmethod
-    def gene_sets_to_response(genesets):
-        """
-        Convert the internal genesets format (returned by read_gene_set) into
-        the dict expected by the JSON REST API
-        """
-        return list(genesets.values())
-
-    def read_gene_sets(self, data_adaptor, context=None):
-        if has_request_context():
-            if not current_app.auth.is_user_authenticated():
-                return ({}, 0)
-
-        gene_sets_uri_or_path = dataset_uri_to_geneset_uri(data_adaptor.data_locator.uri_or_path)
-
-        server_config = data_adaptor.server_config
-        region_name = None if server_config is None else server_config.data_locator__s3__region_name
-        gene_sets_locator = DataLocator(gene_sets_uri_or_path, region_name=region_name)
-        if not gene_sets_locator.exists():
-            return ({}, 0)
-
-        gene_sets = read_gene_sets_tidycsv(gene_sets_locator, context)
-        schema = data_adaptor.get_schema()
-        var_index = schema["annotations"]["var"].get("index", "index")
-        var_names = set(data_adaptor.query_var_array(var_index))
-
-        gene_sets = validate_gene_sets(gene_sets, var_names)
-        return (gene_sets, 0)
-
-
-def dataset_uri_to_geneset_uri(data_uri_or_path):
-    """given a dataset URI, return the associated gene set URI"""
-    data_basename = os.path.basename(data_uri_or_path)
-    base, ext = os.path.splitext(data_basename)
-    if ext is not None:  # strip extension, if any
-        data_basename = base
-
-    genesets_basename = f"{data_basename}-genesets.csv"
-    gene_sets_uri_or_path = path_join(data_uri_or_path, "..", genesets_basename)
-
-    return gene_sets_uri_or_path

backend/czi_hosted/common/config/__init__.py

@@ -1,4 +0,0 @@
-from backend.common.utils.aws_secret_utils import get_secret_key  # noqa F504
-
-DEFAULT_SERVER_PORT = 5005
-BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20  # 100MB

backend/czi_hosted/common/corpora.py

@@ -1,78 +0,0 @@
-"""
-Corpora schema conventions support.  Helper functions for reading.
-
-https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md
-
-https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema_h5ad_implementation.md
-"""
-import collections
-import json
-
-from backend.czi_hosted.cli.upgrade import validate_version_str
-from backend.czi_hosted.common.utils.corpora_constants import CorporaConstants
-
-
-def corpora_get_versions_from_anndata(adata):
-    """
-    Given an AnnData object, return:
-        * None - if not a Corpora object
-        * [ corpora_schema_version, corpora_encoding_version ] - if a Corpora object
-
-    Implements the identification protocol defined in the specification.
-    """
-
-    # per Corpora AnnData spec, this is a corpora file if the following is true
-    if "version" not in adata.uns_keys():
-        return None
-    version = adata.uns["version"]
-    if not isinstance(version, collections.abc.Mapping) or "corpora_schema_version" not in version:
-        return None
-
-    corpora_schema_version = version.get("corpora_schema_version")
-    corpora_encoding_version = version.get("corpora_encoding_version")
-
-    # TODO: spec says these must be SEMVER values, so check.
-    if validate_version_str(corpora_schema_version) and validate_version_str(corpora_encoding_version):
-        return [corpora_schema_version, corpora_encoding_version]
-
-
-def corpora_is_version_supported(corpora_schema_version, corpora_encoding_version):
-    return (
-        corpora_schema_version
-        and corpora_encoding_version
-        and corpora_schema_version.startswith("1.")
-        and corpora_encoding_version.startswith("0.1.")
-    )
-
-
-def corpora_get_props_from_anndata(adata):
-    """
-    Get Corpora dataset properties from an AnnData
-    """
-    versions = corpora_get_versions_from_anndata(adata)
-    if versions is None:
-        return None
-    [corpora_schema_version, corpora_encoding_version] = versions
-    version_is_supported = corpora_is_version_supported(corpora_schema_version, corpora_encoding_version)
-    if not version_is_supported:
-        raise ValueError("Unsupported Corpora schema version")
-
-    corpora_props = {}
-    for key in CorporaConstants.REQUIRED_SIMPLE_METADATA_FIELDS:
-        if key not in adata.uns:
-            raise KeyError(f"missing Corpora schema field {key}")
-        corpora_props[key] = adata.uns[key]
-
-    for key in CorporaConstants.OPTIONAL_JSON_ENCODED_METADATA_FIELD:
-        if key not in adata.uns:
-            continue
-        try:
-            corpora_props[key] = json.loads(adata.uns[key])
-        except json.JSONDecodeError:
-            raise json.JSONDecodeError(f"Corpora schema field {key} is expected to be a valid JSON string")
-
-    for key in CorporaConstants.OPTIONAL_SIMPLE_METADATA_FIELDS:
-        if key in adata.uns:
-            corpora_props[key] = adata.uns[key]
-
-    return corpora_props

backend/czi_hosted/requirements-dev.txt

@@ -1,12 +0,0 @@
-Authlib>=0.14.3
-black
-bumpversion>=0.5
-codecov>=2.0.15
-parameterized>=0.7.0
-psycopg2-binary>=2.8.5
-pytest>=3.6.3
-python-jose>=3.2.0
-twine>=1.12.1
--r requirements.txt
--r requirements-prepare.txt
-rsa>=4.7 # not directly required, pinned by Snyk to avoid a vulnerability

backend/czi_hosted/requirements-prepare.txt

@@ -1,4 +0,0 @@
-python-igraph
-louvain>=0.6
-scanpy
-umap-learn<0.5.0 # The pinned version scanpy is not compatible with latest umap-learn

backend/server/Makefile

@@ -1,26 +0,0 @@
-include ../../common.mk
-
-.PHONY: clean
-clean:
-	rm -f common/web/templates/index.html
-	rm -rf common/web/static
-	rm -f common/web/csp-hashes.json
-
-.PHONY: unit-test
-unit-test:
-	PYTHONWARNINGS=ignore:ResourceWarning coverage run \
-		--source=app,auth,cli,common,compute,converters,data_anndata,data_common \
-		--omit=.coverage,venv \
-		-m unittest discover \
-		--start-directory ../test/test_server/unit \
-		--top-level-directory ../../ \
-		--verbose; test_result=$$?; \
-	exit $$test_result \
-
-.PHONY: test-annotations-performance
-test-annotations-performance:
-	python ../test/test_server/performance/performance_test_annotations_backend.py
-
-.PHONY: test-annotations-scale
-test-annotations-scale:
-	locust -f ../test/test_server/performance/scale_test_annotations.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt

backend/server/app/app.py

@@ -1,296 +0,0 @@
-import datetime
-import logging
-from functools import wraps
-from http import HTTPStatus
-
-from flask import (
-    Flask,
-    current_app,
-    make_response,
-    render_template,
-    Blueprint,
-    request,
-    send_from_directory,
-)
-from flask_restful import Api, Resource
-
-import backend.server.common.rest as common_rest
-from backend.common.errors import DatasetAccessError, RequestException
-from backend.server.common.health import health_check
-from backend.common.utils.utils import StrictJSONEncoder
-
-webbp = Blueprint("webapp", "backend.server.common.web", template_folder="templates")
-
-ONE_WEEK = 7 * 24 * 60 * 60
-
-
-def _cache_control(always, **cache_kwargs):
-    """
-    Used to easily manage cache control headers on responses.
-    See Werkzeug for attributes that can be set, eg, no_cache, private, max_age, etc.
-    https://werkzeug.palletsprojects.com/en/1.0.x/datastructures/#werkzeug.datastructures.ResponseCacheControl
-    """
-
-    def inner_cache_control(f):
-        @wraps(f)
-        def wrapper(*args, **kwargs):
-            response = make_response(f(*args, **kwargs))
-            if not always and not current_app.app_config.server_config.app__generate_cache_control_headers:
-                return response
-            if response.status_code >= 400:
-                return response
-            for k, v in cache_kwargs.items():
-                setattr(response.cache_control, k, v)
-            return response
-
-        return wrapper
-
-    return inner_cache_control
-
-
-def cache_control(**cache_kwargs):
-    """ config driven """
-    return _cache_control(False, **cache_kwargs)
-
-
-def cache_control_always(**cache_kwargs):
-    """ always generate headers, regardless of the config """
-    return _cache_control(True, **cache_kwargs)
-
-
-@webbp.route("/", methods=["GET"])
-def dataset_index():
-    app_config = current_app.app_config
-
-    dataset_config = app_config.get_dataset_config()
-    scripts = dataset_config.app__scripts
-    inline_scripts = dataset_config.app__inline_scripts
-
-    try:
-        args = {"SCRIPTS": scripts, "INLINE_SCRIPTS": inline_scripts}
-        return render_template("index.html", **args)
-
-    except DatasetAccessError as e:
-        return common_rest.abort_and_log(
-            e.status_code, f"Invalid dataset: {e.message}", loglevel=logging.INFO, include_exc_info=True
-        )
-
-
-@webbp.errorhandler(RequestException)
-def handle_request_exception(error):
-    return common_rest.abort_and_log(error.status_code, error.message, loglevel=logging.INFO, include_exc_info=True)
-
-
-def requires_authentication(func):
-    @wraps(func)
-    def wrapped_function(self, *args, **kwargs):
-        auth = current_app.auth
-        if auth.is_user_authenticated():
-            return func(self, *args, **kwargs)
-        else:
-            return make_response("not authenticated", HTTPStatus.UNAUTHORIZED)
-
-    return wrapped_function
-
-
-def rest_get_data_adaptor(func):
-    @wraps(func)
-    def wrapped_function(self):
-        try:
-            return func(self, current_app.data_adaptor)
-        except DatasetAccessError as e:
-            return common_rest.abort_and_log(
-                e.status_code, f"Invalid dataset: {e.message}", loglevel=logging.INFO, include_exc_info=True
-            )
-
-    return wrapped_function
-
-
-class HealthAPI(Resource):
-    @cache_control_always(no_store=True)
-    def get(self):
-        config = current_app.app_config
-        return health_check(config)
-
-
-class SchemaAPI(Resource):
-    # TODO @mdunitz separate dataset schema and user schema
-    @cache_control(public=True, max_age=ONE_WEEK)
-    @rest_get_data_adaptor
-    def get(self, data_adaptor):
-        return common_rest.schema_get(data_adaptor)
-
-
-class ConfigAPI(Resource):
-    @cache_control(public=True, max_age=ONE_WEEK)
-    @rest_get_data_adaptor
-    def get(self, data_adaptor):
-        return common_rest.config_get(current_app.app_config, data_adaptor)
-
-
-class UserInfoAPI(Resource):
-    @cache_control_always(no_store=True)
-    @rest_get_data_adaptor
-    def get(self, data_adaptor):
-        return common_rest.userinfo_get(current_app.app_config, data_adaptor)
-
-
-class AnnotationsObsAPI(Resource):
-    @cache_control(public=True, max_age=ONE_WEEK)
-    @rest_get_data_adaptor
-    def get(self, data_adaptor):
-        return common_rest.annotations_obs_get(request, data_adaptor)
-
-    @requires_authentication
-    @cache_control(no_store=True)
-    @rest_get_data_adaptor
-    def put(self, data_adaptor):
-        return common_rest.annotations_obs_put(request, data_adaptor)
-
-
-class AnnotationsVarAPI(Resource):
-    @cache_control(public=True, max_age=ONE_WEEK)
-    @rest_get_data_adaptor
-    def get(self, data_adaptor):
-        return common_rest.annotations_var_get(request, data_adaptor)
-
-
-class DataVarAPI(Resource):
-    @cache_control(no_store=True)
-    @rest_get_data_adaptor
-    def put(self, data_adaptor):
-        return common_rest.data_var_put(request, data_adaptor)
-
-    @cache_control(public=True, max_age=ONE_WEEK)
-    @rest_get_data_adaptor
-    def get(self, data_adaptor):
-        return common_rest.data_var_get(request, data_adaptor)
-
-
-class ColorsAPI(Resource):
-    @cache_control(public=True, max_age=ONE_WEEK)
-    @rest_get_data_adaptor
-    def get(self, data_adaptor):
-        return common_rest.colors_get(data_adaptor)
-
-
-class DiffExpObsAPI(Resource):
-    @cache_control(no_store=True)
-    @rest_get_data_adaptor
-    def post(self, data_adaptor):
-        return common_rest.diffexp_obs_post(request, data_adaptor)
-
-
-class LayoutObsAPI(Resource):
-    @cache_control(public=True, max_age=ONE_WEEK)
-    @rest_get_data_adaptor
-    def get(self, data_adaptor):
-        return common_rest.layout_obs_get(request, data_adaptor)
-
-
-class GenesetsAPI(Resource):
-    @cache_control(public=True, max_age=ONE_WEEK)
-    @rest_get_data_adaptor
-    def get(self, data_adaptor):
-        return common_rest.genesets_get(request, data_adaptor)
-
-    @requires_authentication
-    @cache_control(no_store=True)
-    @rest_get_data_adaptor
-    def put(self, data_adaptor):
-        return common_rest.genesets_put(request, data_adaptor)
-
-
-class SummarizeVarAPI(Resource):
-    @rest_get_data_adaptor
-    @cache_control(public=True, max_age=ONE_WEEK)
-    def get(self, data_adaptor):
-        return common_rest.summarize_var_get(request, data_adaptor)
-
-    @rest_get_data_adaptor
-    @cache_control(no_store=True)
-    def post(self, data_adaptor):
-        return common_rest.summarize_var_post(request, data_adaptor)
-
-
-def get_api_base_resources(bp_base):
-    """Add resources that are accessed from the api url"""
-    api = Api(bp_base)
-
-    # Diagnostics routes
-    api.add_resource(HealthAPI, "/health")
-    return api
-
-
-def get_api_dataroot_resources(bp_dataroot):
-    """Add resources that refer to a dataset"""
-    api = Api(bp_dataroot)
-
-    def add_resource(resource, url):
-        """convenience function to make the outer function less verbose"""
-        api.add_resource(resource, url)
-
-    # Initialization routes
-    add_resource(SchemaAPI, "/schema")
-    add_resource(ConfigAPI, "/config")
-    add_resource(UserInfoAPI, "/userinfo")
-    # Data routes
-    add_resource(AnnotationsObsAPI, "/annotations/obs")
-    add_resource(AnnotationsVarAPI, "/annotations/var")
-    add_resource(DataVarAPI, "/data/var")
-    add_resource(GenesetsAPI, "/genesets")
-    add_resource(SummarizeVarAPI, "/summarize/var")
-    # Display routes
-    add_resource(ColorsAPI, "/colors")
-    # Computation routes
-    add_resource(DiffExpObsAPI, "/diffexp/obs")
-    add_resource(LayoutObsAPI, "/layout/obs")
-    return api
-
-
-class Server:
-    @staticmethod
-    def _before_adding_routes(app, app_config):
-        """ will be called before routes are added, during __init__.  Subclass protocol """
-        pass
-
-    def __init__(self, app_config):
-        self.app = Flask(__name__, static_folder=None)
-        self._before_adding_routes(self.app, app_config)
-        self.app.json_encoder = StrictJSONEncoder
-        server_config = app_config.server_config
-
-        # enable session data
-        self.app.permanent_session_lifetime = datetime.timedelta(days=50 * 365)
-
-        # Config
-        secret_key = server_config.app__flask_secret_key
-        self.app.config.update(SECRET_KEY=secret_key)
-
-        self.app.register_blueprint(webbp)
-
-        api_version = "/api/v0.2"
-        api_path = "/"
-
-        bp_base = Blueprint("bp_base", __name__, url_prefix=api_path)
-        base_resources = get_api_base_resources(bp_base)
-        self.app.register_blueprint(base_resources.blueprint)
-
-        bp_api = Blueprint("api", __name__, url_prefix=f"{api_path}{api_version}")
-        resources = get_api_dataroot_resources(bp_api)
-        self.app.register_blueprint(resources.blueprint)
-        self.app.add_url_rule(
-            "/static/<path:filename>",
-            "static_assets",
-            view_func=lambda filename: send_from_directory("../common/web/static", filename),
-            methods=["GET"],
-        )
-
-        self.app.data_adaptor = server_config.data_adaptor
-        self.app.app_config = app_config
-
-        auth = server_config.auth
-        self.app.auth = auth
-        if auth.requires_client_login():
-            auth.add_url_rules(self.app)
-        auth.complete_setup(self.app)

backend/server/auth/__init__.py

@@ -1,5 +0,0 @@
-# import the built in auth types so they can be registered
-
-import backend.server.auth.auth_none  # noqa: F401
-import backend.server.auth.auth_test  # noqa: F401
-import backend.server.auth.auth_session  # noqa: F401

backend/server/auth/auth.py

@@ -1,91 +0,0 @@
-from abc import ABC, abstractmethod
-
-
-class AuthTypeBase(ABC):
-    """Base type for all authentication types."""
-
-    def __init__(self):
-        super().__init__()
-
-    @abstractmethod
-    def is_valid_authentication_type(self):
-        """Return True if the auth type is valid, e.g. it can return userinfo and username.
-          (AuthTypeNone is the only one type that returns False)"""
-        pass
-
-    def requires_client_login(self):
-        """Return True if the user needs to login from the client (e.g. Login button is shown)"""
-        return False
-
-    @abstractmethod
-    def complete_setup(self, app):
-        """complete any setup that may be needed by this auth type.  The Flask app is passed in.
-        This is the last auth function called before the server starts to run."""
-        pass
-
-    @abstractmethod
-    def is_user_authenticated(self):
-        """Return True if the user is authenticated"""
-        pass
-
-    @abstractmethod
-    def get_user_id(self):
-        """Return the id for this user (string)"""
-        pass
-
-    @abstractmethod
-    def get_user_name(self):
-        """Return the name of the user (string)"""
-        pass
-
-    @abstractmethod
-    def get_user_email(self):
-        """Return the name of the user (string)"""
-        pass
-
-    def get_user_picture(self):
-        """Return the location to the user's picture"""
-        return None
-
-
-class AuthTypeClientBase(AuthTypeBase):
-    """Base type for all authentication types that require the client to login"""
-
-    def __init__(self):
-        super().__init__()
-
-    def requires_client_login(self):
-        return True
-
-    @abstractmethod
-    def add_url_rules(self, selfapp):
-        """Add url rules to the app (like /login, /logout, etc)"""
-        pass
-
-    @abstractmethod
-    def get_login_url(self, data_adaptor):
-        """Return the url for the login route"""
-        pass
-
-    @abstractmethod
-    def get_logout_url(self, data_adaptor):
-        """Return the url for the logout route"""
-        pass
-
-
-class AuthTypeFactory:
-    """Factory class to create an authentication type"""
-
-    auth_types = {}
-
-    @staticmethod
-    def register(name, auth_type):
-        assert issubclass(auth_type, AuthTypeBase)
-        AuthTypeFactory.auth_types[name] = auth_type
-
-    @staticmethod
-    def create(name, app_config):
-        auth_type = AuthTypeFactory.auth_types.get(name)
-        if auth_type is None:
-            return None
-        return auth_type(app_config)

backend/server/auth/auth_test.py

@@ -1,73 +0,0 @@
-from backend.server.auth.auth import AuthTypeClientBase, AuthTypeFactory
-from flask import session, request, redirect
-
-
-class AuthTypeTest(AuthTypeClientBase):
-    """An authentication type for testing client based logins.  When the login route is accessed
-    the user is automatically logged in with a default or configured username"""
-
-    # key in session token with userid and username
-    CXGUID = "cxguid_test"
-    CXGUNAME = "cxguname_test"
-    CXGUEMAIL = "cxguemail_test"
-    CXGUPICTURE = "cxgupicture_test"
-
-    def __init__(self, app_config):
-        super().__init__()
-        self.user_name = "test_account"
-        self.user_id = "id0001"
-        self.user_email = "test_account@test.com"
-        self.user_picture = None
-
-    def is_valid_authentication_type(self):
-        return True
-
-    def requires_client_login(self):
-        return True
-
-    def add_url_rules(self, app):
-        app.add_url_rule("/login", "login", self.login, methods=["GET"])
-        app.add_url_rule("/logout", "logout", self.logout, methods=["GET"])
-
-    def complete_setup(self, app):
-        pass
-
-    def is_user_authenticated(self):
-        return self.CXGUID in session
-
-    def get_user_id(self):
-        return session.get(self.CXGUID)
-
-    def get_user_name(self):
-        return session.get(self.CXGUNAME)
-
-    def get_user_email(self):
-        return session.get(self.CXGUEMAIL)
-
-    def get_user_picture(self):
-        return session.get(self.CXGUPICTURE)
-
-    def login(self):
-        args = request.args
-        return_to = args.get("dataset", "/")
-        session[self.CXGUID] = args.get("userid", self.user_id)
-        session[self.CXGUNAME] = args.get("username", self.user_name)
-        session[self.CXGUEMAIL] = args.get("email", self.user_email)
-        session[self.CXGUPICTURE] = args.get("picture", self.user_picture)
-        return redirect(return_to)
-
-    def logout(self):
-        session.clear()
-        return_to = request.args.get("dataset", "/")
-        return redirect(return_to)
-
-    def get_login_url(self, data_adaptor):
-        """Return the url for the login route"""
-        return "/login"
-
-    def get_logout_url(self, data_adaptor):
-        """Return the url for the logout route"""
-        return "/logout"
-
-
-AuthTypeFactory.register("test", AuthTypeTest)

backend/server/cli/cli.py

@@ -1,33 +0,0 @@
-import click
-
-from .launch import launch
-from .prepare import prepare
-from .upgrade import log_upgrade_check
-from .schema import schema_cli
-from .. import __version__
-
-
-@click.group(
-    name="cellxgene",
-    subcommand_metavar="COMMAND <args>",
-    options_metavar="<options>",
-    context_settings=dict(max_content_width=85, help_option_names=["-h", "--help"]),
-)
-@click.help_option("--help", "-h", help="Show this message and exit.")
-@click.version_option(
-    version=__version__,
-    prog_name="cellxgene",
-    message="[%(prog)s] Version %(version)s",
-    help="Show the software version and exit.",
-)
-@click.option(
-    "--upgrade-check/--no-upgrade-check", default=True, show_default=True, help="Check for release upgrades on start.",
-)
-def cli(upgrade_check):
-    if upgrade_check:
-        log_upgrade_check()
-
-
-cli.add_command(launch)
-cli.add_command(prepare)
-cli.add_command(schema_cli)

backend/server/cli/launch.py

@@ -1,449 +0,0 @@
-import errno
-import functools
-import logging
-import sys
-import webbrowser
-import os
-import click
-from flask_compress import Compress
-from flask_cors import CORS
-
-from backend.server.default_config import default_config
-from backend.server.app.app import Server
-from backend.server.common.config.app_config import AppConfig
-from backend.common.errors import DatasetAccessError, ConfigurationError
-from backend.common.utils.utils import sort_options
-
-DEFAULT_CONFIG = AppConfig()
-
-
-def annotation_args(func):
-    @click.option(
-        "--disable-annotations",
-        is_flag=True,
-        default=not DEFAULT_CONFIG.dataset_config.user_annotations__enable,
-        show_default=True,
-        help="Disable user annotation of data.",
-    )
-    @click.option(
-        "--annotations-file",
-        default=DEFAULT_CONFIG.dataset_config.user_annotations__local_file_csv__file,
-        show_default=True,
-        multiple=False,
-        metavar="<path>",
-        help="CSV file to initialize editing of existing annotations; will be altered in-place. "
-        "Incompatible with --user-generated-data-dir.",
-    )
-    @click.option(
-        "--user-generated-data-dir",
-        "--annotations-dir",
-        default=DEFAULT_CONFIG.dataset_config.user_annotations__local_file_csv__directory,
-        show_default=False,
-        multiple=False,
-        metavar="<directory path>",
-        help="Directory of where to save output annotations; filename will be specified in the application. "
-        "Incompatible with --annotations-file and --gene-sets-file.",
-    )
-    @click.option(
-        "--disable-gene-sets-save",
-        is_flag=True,
-        default=DEFAULT_CONFIG.dataset_config.user_annotations__gene_sets__readonly,
-        show_default=False,
-        help="Disable saving gene sets. If disabled, users will be able to make changes to gene sets but all "
-        "changes will be lost on browser refresh.",
-    )
-    @click.option(
-        "--gene-sets-file",
-        default=DEFAULT_CONFIG.dataset_config.user_annotations__local_file_csv__gene_sets_file,
-        show_default=True,
-        multiple=False,
-        metavar="<path>",
-        help="CSV file to initialize editing of gene sets; will be altered in-place. Incompatible with "
-        "--user-generated-data-dir.",
-    )
-    @functools.wraps(func)
-    def wrapper(*args, **kwargs):
-        return func(*args, **kwargs)
-
-    return wrapper
-
-
-def config_args(func):
-    @click.option(
-        "--max-category-items",
-        default=DEFAULT_CONFIG.dataset_config.presentation__max_categories,
-        metavar="<integer>",
-        show_default=True,
-        help="Will not display categories with more distinct values than specified.",
-    )
-    @click.option(
-        "--disable-custom-colors",
-        is_flag=True,
-        default=False,
-        show_default=False,
-        help="Disable user-defined category-label colors drawn from source data file.",
-    )
-    @click.option(
-        "--diffexp-lfc-cutoff",
-        "-de",
-        default=DEFAULT_CONFIG.dataset_config.diffexp__lfc_cutoff,
-        show_default=True,
-        metavar="<float>",
-        help="Minimum log fold change threshold for differential expression.",
-    )
-    @click.option(
-        "--disable-diffexp",
-        is_flag=True,
-        default=not DEFAULT_CONFIG.dataset_config.diffexp__enable,
-        show_default=False,
-        help="Disable on-demand differential expression.",
-    )
-    @click.option(
-        "--embedding",
-        "-e",
-        default=DEFAULT_CONFIG.dataset_config.embeddings__names,
-        multiple=True,
-        show_default=False,
-        metavar="<text>",
-        help="Embedding name, eg, 'umap'. Repeat option for multiple embeddings. Defaults to all.",
-    )
-    @functools.wraps(func)
-    def wrapper(*args, **kwargs):
-        return func(*args, **kwargs)
-
-    return wrapper
-
-
-def dataset_args(func):
-    @click.option(
-        "--obs-names",
-        "-obs",
-        default=DEFAULT_CONFIG.server_config.single_dataset__obs_names,
-        metavar="<text>",
-        help="Name of annotation field to use for observations. If not specified cellxgene will use the the obs index.",
-    )
-    @click.option(
-        "--var-names",
-        "-var",
-        default=DEFAULT_CONFIG.server_config.single_dataset__var_names,
-        metavar="<text>",
-        help="Name of annotation to use for variables. If not specified cellxgene will use the the var index.",
-    )
-    @click.option(
-        "--backed",
-        "-b",
-        is_flag=True,
-        default=DEFAULT_CONFIG.server_config.adaptor__anndata_adaptor__backed,
-        show_default=False,
-        help="Load anndata in file-backed mode. " "This may save memory, but may result in slower overall performance.",
-    )
-    @click.option(
-        "--title",
-        "-t",
-        default=DEFAULT_CONFIG.server_config.single_dataset__title,
-        metavar="<text>",
-        help="Title to display. If omitted will use file name.",
-    )
-    @click.option(
-        "--about",
-        default=DEFAULT_CONFIG.server_config.single_dataset__about,
-        metavar="<URL>",
-        help="URL providing more information about the dataset (hint: must be a fully specified absolute URL).",
-    )
-    @click.option(
-        "--X-approximate-distribution",
-        default=DEFAULT_CONFIG.dataset_config.X_approximate_distribution,
-        show_default=True,
-        type=click.Choice(["auto", "normal", "count"], case_sensitive=False),
-        help="Specify the approximate distribution of X matrix values. 'auto' will use a heuristic "
-        "to determine the approximate distribution.  Mode 'auto' is incompatible with --backed.",
-    )
-    @functools.wraps(func)
-    def wrapper(*args, **kwargs):
-        return func(*args, **kwargs)
-
-    return wrapper
-
-
-def server_args(func):
-    @click.option(
-        "--debug",
-        "-d",
-        is_flag=True,
-        default=DEFAULT_CONFIG.server_config.app__debug,
-        show_default=True,
-        help="Run in debug mode. This is helpful for cellxgene developers, "
-        "or when you want more information about an error condition.",
-    )
-    @click.option(
-        "--verbose",
-        "-v",
-        is_flag=True,
-        default=DEFAULT_CONFIG.server_config.app__verbose,
-        show_default=True,
-        help="Provide verbose output, including warnings and all server requests.",
-    )
-    @click.option(
-        "--port",
-        "-p",
-        metavar="<port>",
-        default=DEFAULT_CONFIG.server_config.app__port,
-        type=int,
-        show_default=True,
-        help="Port to run server on. If not specified cellxgene will find an available port.",
-    )
-    @click.option(
-        "--host",
-        metavar="<IP address>",
-        default=DEFAULT_CONFIG.server_config.app__host,
-        show_default=False,
-        help="Host IP address. By default cellxgene will use localhost (e.g. 127.0.0.1).",
-    )
-    @click.option(
-        "--scripts",
-        "-s",
-        default=DEFAULT_CONFIG.dataset_config.app__scripts,
-        multiple=True,
-        metavar="<text>",
-        help="Additional script files to include in HTML page. If not specified, "
-        "no additional script files will be included.",
-        show_default=False,
-    )
-    @functools.wraps(func)
-    def wrapper(*args, **kwargs):
-        return func(*args, **kwargs)
-
-    return wrapper
-
-
-def launch_args(func):
-    @annotation_args
-    @config_args
-    @dataset_args
-    @server_args
-    @click.argument("datapath", required=False, metavar="<path to data file>")
-    @click.option(
-        "--open",
-        "-o",
-        "open_browser",
-        is_flag=True,
-        default=DEFAULT_CONFIG.server_config.app__open_browser,
-        show_default=True,
-        help="Open web browser after launch.",
-    )
-    @click.option(
-        "--config-file",
-        "-c",
-        "config_file",
-        default=None,
-        show_default=True,
-        help="Location to yaml file with configuration settings",
-    )
-    @click.option(
-        "--dump-default-config",
-        "dump_default_config",
-        is_flag=True,
-        default=False,
-        show_default=True,
-        help="Print default configuration settings and exit",
-    )
-    @click.help_option("--help", "-h", help="Show this message and exit.")
-    @functools.wraps(func)
-    def wrapper(*args, **kwargs):
-        return func(*args, **kwargs)
-
-    return wrapper
-
-
-def handle_scripts(scripts):
-    if scripts:
-        click.echo(
-            r"""
-    / / /\ \ \__ _ _ __ _ __ (_)_ __   __ _
-    \ \/  \/ / _` | '__| '_ \| | '_ \ / _` |
-     \  /\  / (_| | |  | | | | | | | | (_| |
-      \/  \/ \__,_|_|  |_| |_|_|_| |_|\__, |
-                                      |___/
-    The --scripts flag is intended for developers to include google analytics etc. You could be opening yourself to a
-    security risk by including the --scripts flag. Make sure you trust the scripts that you are including.
-            """
-        )
-        scripts_pretty = ", ".join(scripts)
-        click.confirm(f"Are you sure you want to inject these scripts: {scripts_pretty}?", abort=True)
-
-
-class CliLaunchServer(Server):
-    """
-    the CLI runs a local web server, and needs to enable a few more features.
-    """
-
-    def __init__(self, app_config):
-        super().__init__(app_config)
-
-    @staticmethod
-    def _before_adding_routes(app, app_config):
-        app.config["COMPRESS_MIMETYPES"] = [
-            "text/html",
-            "text/css",
-            "text/xml",
-            "application/json",
-            "application/javascript",
-            "application/octet-stream",
-        ]
-        Compress(app)
-        if app_config.server_config.app__debug:
-            CORS(app, supports_credentials=True)
-
-
-@sort_options
-@click.command(
-    short_help="Launch the cellxgene data viewer. " "Run `cellxgene launch --help` for more information.",
-    options_metavar="<options>",
-)
-@launch_args
-def launch(
-    datapath,
-    verbose,
-    debug,
-    open_browser,
-    port,
-    host,
-    embedding,
-    obs_names,
-    var_names,
-    max_category_items,
-    disable_custom_colors,
-    diffexp_lfc_cutoff,
-    title,
-    scripts,
-    about,
-    disable_annotations,
-    annotations_file,
-    user_generated_data_dir,
-    gene_sets_file,
-    disable_gene_sets_save,
-    backed,
-    disable_diffexp,
-    config_file,
-    dump_default_config,
-    x_approximate_distribution,
-):
-    """Launch the cellxgene data viewer.
-    This web app lets you explore single-cell expression data.
-    Data must be in a format that cellxgene expects.
-    Read the "getting started" guide to learn more:
-    https://chanzuckerberg.github.io/cellxgene/getting-started.html
-
-    Examples:
-
-    > cellxgene launch example-dataset/pbmc3k.h5ad --title pbmc3k
-
-    > cellxgene launch <your data file> --title <your title>
-
-    > cellxgene launch <url>"""
-
-    if dump_default_config:
-        print(default_config)
-        sys.exit(0)
-
-    # Startup message
-    click.echo("[cellxgene] Starting the CLI...")
-
-    # app config
-    app_config = AppConfig()
-    server_config = app_config.server_config
-
-    try:
-        if config_file:
-            app_config.update_from_config_file(config_file)
-
-        # Determine which config options were give on the command line.
-        # Those will override the ones provided in the config file (if provided).
-        cli_config = AppConfig()
-        cli_config.update_server_config(
-            app__verbose=verbose,
-            app__debug=debug,
-            app__host=host,
-            app__port=port,
-            app__open_browser=open_browser,
-            single_dataset__datapath=datapath,
-            single_dataset__title=title,
-            single_dataset__about=about,
-            single_dataset__obs_names=obs_names,
-            single_dataset__var_names=var_names,
-            adaptor__anndata_adaptor__backed=backed,
-        )
-        cli_config.update_dataset_config(
-            app__scripts=scripts,
-            user_annotations__enable=not disable_annotations,
-            user_annotations__local_file_csv__file=annotations_file,
-            user_annotations__local_file_csv__directory=user_generated_data_dir,
-            user_annotations__local_file_csv__gene_sets_file=gene_sets_file,
-            user_annotations__gene_sets__readonly=disable_gene_sets_save,
-            presentation__max_categories=max_category_items,
-            presentation__custom_colors=not disable_custom_colors,
-            embeddings__names=embedding,
-            diffexp__enable=not disable_diffexp,
-            diffexp__lfc_cutoff=diffexp_lfc_cutoff,
-            X_approximate_distribution=x_approximate_distribution,
-        )
-
-        diff = cli_config.server_config.changes_from_default()
-        changes = {key: val for key, val, _ in diff}
-        app_config.update_server_config(**changes)
-
-        diff = cli_config.dataset_config.changes_from_default()
-        changes = {key: val for key, val, _ in diff}
-        app_config.update_dataset_config(**changes)
-
-        # process the configuration
-        #  any errors will be thrown as an exception.
-        #  any info messages will be passed to the messagefn function.
-
-        def messagefn(message):
-            click.echo("[cellxgene] " + message)
-
-        # Use a default secret if one is not provided
-        if not server_config.app__flask_secret_key:
-            app_config.update_server_config(app__flask_secret_key="SparkleAndShine")
-
-        app_config.complete_config(messagefn)
-
-    except (ConfigurationError, DatasetAccessError) as e:
-        raise click.ClickException(e)
-
-    handle_scripts(scripts)
-
-    # create the server
-    server = CliLaunchServer(app_config)
-
-    if not server_config.app__verbose:
-        log = logging.getLogger("werkzeug")
-        log.setLevel(logging.ERROR)
-
-    cellxgene_url = f"http://{app_config.server_config.app__host}:{app_config.server_config.app__port}"
-    if server_config.app__open_browser:
-        click.echo(f"[cellxgene] Launching! Opening your browser to {cellxgene_url} now.")
-        webbrowser.open(cellxgene_url)
-    else:
-        click.echo(f"[cellxgene] Launching! Please go to {cellxgene_url} in your browser.")
-
-    click.echo("[cellxgene] Type CTRL-C at any time to exit.")
-
-    if not server_config.app__verbose:
-        f = open(os.devnull, "w")
-        sys.stdout = f
-
-    try:
-        server.app.run(
-            host=server_config.app__host,
-            debug=server_config.app__debug,
-            port=server_config.app__port,
-            threaded=not server_config.app__debug,
-            use_debugger=False,
-            use_reloader=False,
-        )
-    except OSError as e:
-        if e.errno == errno.EADDRINUSE:
-            raise click.ClickException("Port is in use, please specify an open port using the --port flag.") from e
-        raise

backend/server/cli/prepare.py

@@ -1,274 +0,0 @@
-from os.path import expanduser, isdir, isfile, sep, splitext
-
-import click
-import pandas as pd
-from numpy import ndarray, unique
-from scipy.sparse.csc import csc_matrix
-
-from backend.common.utils.utils import sort_options
-
-
-@sort_options
-@click.command(
-    short_help="Preprocess data for use with cellxgene. " "Run `cellxgene prepare --help` for more information.",
-    options_metavar="<options>",
-)
-@click.argument("data", nargs=1, metavar="<path to data file>", required=True)
-@click.option(
-    "--embedding",
-    "-e",
-    default=["umap", "tsne"],
-    multiple=True,
-    type=click.Choice(["umap", "tsne"]),
-    help="Embedding algorithm(s). Repeat option for multiple embeddings.",
-    show_default=True,
-)
-@click.option(
-    "--recipe", "-r", default="none", type=click.Choice(["none", "seurat", "zheng17"]), show_default=True,
-)
-@click.option("--output", "-o", default="", help="Save a new file to filename.", metavar="<filename>")
-@click.option("--plotting", "-p", default=False, is_flag=True, help="Generate plots.", show_default=True)
-@click.option("--sparse", default=False, is_flag=True, help="Force sparsity.", show_default=True)
-@click.option("--overwrite", default=False, is_flag=True, help="Allow file overwriting.", show_default=True)
-@click.option("--set-obs-names", default="", help="Named field to set as index for obs.", metavar="<name>")
-@click.option("--set-var-names", default="", help="Named field to set as index for var.", metavar="<name>")
-@click.option(
-    "--skip-qc",
-    default=False,
-    is_flag=True,
-    help="Do not run quality control metrics. By default cellxgene runs them "
-    "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).",
-)
-@click.option(
-    "--make-obs-names-unique/--no-make-obs-names-unique",
-    default=True,
-    help="Ensure obs index is unique.",
-    show_default=True,
-)
-@click.option(
-    "--make-var-names-unique/--no-make-var-names-unique",
-    default=True,
-    help="Ensure var index is unique.",
-    show_default=True,
-)
-@click.help_option("--help", "-h", help="Show this message and exit.")
-def prepare(
-    data,
-    embedding,
-    recipe,
-    output,
-    plotting,
-    sparse,
-    overwrite,
-    set_obs_names,
-    set_var_names,
-    skip_qc,
-    make_obs_names_unique,
-    make_var_names_unique,
-):
-    """
-    Preprocess data for use with cellxgene.
-    This tool runs a series of scanpy routines for preparing a dataset for use
-    with cellxgene. It loads data from different formats
-    (h5ad, loom, or a 10x directory), runs dimensionality reduction,
-    computes nearest neighbors, computes an embedding, performs clustering,
-    and saves the results. Includes additional options for naming annotations,
-    ensuring sparsity, and plotting results.
-    """
-
-    # collect slow imports here to make CLI startup more responsive
-    click.echo("[cellxgene] Starting CLI...")
-    try:
-        import matplotlib
-
-        matplotlib.use("Agg")
-        import scanpy as sc
-    except ImportError:
-        raise click.ClickException(
-            "[cellxgene] cellxgene prepare has not been installed. Please run `pip install 'cellxgene[prepare]'` "
-            "to install the necessary requirements."
-        )
-
-    # scanpy settings
-    sc.settings.verbosity = 0
-    sc.settings.autosave = True
-
-    # check args
-    if sparse and not recipe == "none":
-        raise click.UsageError("Cannot use a recipe when forcing sparsity")
-
-    output = expanduser(output)
-
-    if not output:
-        click.echo(
-            "Warning: No file will be saved, to save the results of cellxgene prepare include "
-            "--output <filename> to save output to a new file"
-        )
-    if isfile(output) and not overwrite:
-        raise click.UsageError(f"Cannot overwrite existing file {output}, try using the flag --overwrite")
-
-    def load_data(data):
-        if isfile(data):
-            name, extension = splitext(data)
-            if extension == ".h5ad":
-                adata = sc.read_h5ad(data)
-            elif extension == ".loom":
-                adata = sc.read_loom(data)
-            else:
-                raise click.FileError(data, hint="does not have a valid extension [.h5ad | .loom]")
-        elif isdir(data):
-            if not data.endswith(sep):
-                data += sep
-            adata = sc.read_10x_mtx(data)
-        else:
-            raise click.FileError(data, hint="not a valid file or path")
-
-        if not set_obs_names == "":
-            if set_obs_names not in adata.obs_keys():
-                raise click.UsageError(f"obs {set_obs_names} not found, options are: {adata.obs_keys()}")
-            adata.obs_names = adata.obs[set_obs_names]
-        if not set_var_names == "":
-            if set_var_names not in adata.var_keys():
-                raise click.UsageError(f"var {set_var_names} not found, options are: {adata.var_keys()}")
-            adata.var_names = adata.var[set_var_names]
-        if make_obs_names_unique:
-            adata.obs.index = make_index_unique(adata.obs.index)
-        if make_var_names_unique:
-            adata.var.index = make_index_unique(adata.var.index)
-        if not adata._obs.index.is_unique:
-            click.echo("Warning: obs index is not unique")
-        if not adata._var.index.is_unique:
-            click.echo("Warning: var index is not unique")
-        return adata
-
-    def calculate_qc_metrics(adata):
-        if not skip_qc:
-            sc.pp.calculate_qc_metrics(adata, inplace=True)
-        return adata
-
-    def make_sparse(adata):
-        if (type(adata.X) is ndarray) and sparse:
-            adata.X = csc_matrix(adata.X)
-
-    def run_recipe(adata):
-        if recipe == "seurat":
-            sc.pp.recipe_seurat(adata)
-        elif recipe == "zheng17":
-            sc.pp.recipe_zheng17(adata)
-        else:
-            sc.pp.filter_cells(adata, min_genes=5)
-            sc.pp.filter_genes(adata, min_cells=25)
-            if sparse:
-                sc.pp.scale(adata, zero_center=False)
-            else:
-                sc.pp.scale(adata)
-
-    def run_pca(adata):
-        if sparse:
-            sc.pp.pca(adata, svd_solver="arpack", zero_center=False)
-        else:
-            sc.pp.pca(adata, svd_solver="arpack")
-
-    def run_neighbors(adata):
-        sc.pp.neighbors(adata)
-
-    def run_louvain(adata):
-        sc.tl.louvain(adata)
-
-    def run_embedding(adata):
-        if len(unique(adata.obs["louvain"].values)) < 10:
-            palette = "tab10"
-        else:
-            palette = "tab20"
-
-        if "umap" in embedding:
-            sc.tl.umap(adata)
-            if plotting:
-                sc.pl.umap(adata, color="louvain", palette=palette, save="_louvain")
-
-        if "tsne" in embedding:
-            sc.tl.tsne(adata)
-            if plotting:
-                sc.pl.tsne(adata, color="louvain", palette=palette, save="_louvain")
-
-    def show_step(item):
-        if not skip_qc:
-            qc_name = "Calculating QC metrics"
-        else:
-            qc_name = "Skipping QC"
-        names = {
-            "calculate_qc_metrics": qc_name,
-            "make_sparse": "Ensuring sparsity",
-            "run_recipe": f'Running preprocessing recipe "{recipe}"',
-            "run_pca": "Running PCA",
-            "run_neighbors": "Calculating neighbors",
-            "run_louvain": "Calculating clusters",
-            "run_embedding": "Computing embedding",
-        }
-        if item is not None:
-            return names[item.__name__]
-
-    steps = [calculate_qc_metrics, make_sparse, run_recipe, run_pca, run_neighbors, run_louvain, run_embedding]
-
-    click.echo(f"[cellxgene] Loading data from {data}, please wait...")
-    adata = load_data(data)
-
-    click.echo("[cellxgene] Beginning preprocessing...")
-    with click.progressbar(steps, label="[cellxgene] Progress", show_eta=False, item_show_func=show_step) as bar:
-        for step in bar:
-            step(adata)
-
-    # saving
-    if not output == "":
-        click.echo(f"[cellxgene] Saving results to {output}...")
-        adata.write(output)
-
-    click.echo("[cellxgene] Success!")
-
-
-# TODO (mweiden): remove this once this issue is resolved https://github.com/theislab/anndata/issues/344
-# Note: tentative solution here https://github.com/theislab/anndata/pull/345
-def make_index_unique(index: pd.Index, join: str = "-"):
-    """
-    Makes the index unique by appending a number string to each duplicate index element: '1', '2', etc.
-
-    If a tentative name created by the algorithm already exists in the index, it tries the next integer in the sequence.
-
-    The first occurrence of a non-unique value is ignored.
-    Parameters
-    ----------
-    join
-         The connecting string between name and integer.
-    Examples
-    --------
-    >>> from anndata import AnnData
-    >>> adata1 = AnnData(np.ones((3, 2)), dict(obs_names=['a', 'b', 'c']))
-    >>> adata2 = AnnData(np.zeros((3, 2)), dict(obs_names=['d', 'b', 'b']))
-    >>> adata = adata1.concatenate(adata2)
-    >>> adata.obs_names
-    Index(['a', 'b', 'c', 'd', 'b', 'b'], dtype='object')
-    >>> adata.obs_names_make_unique()
-    >>> adata.obs_names
-    Index(['a', 'b', 'c', 'd', 'b-1', 'b-2'], dtype='object')
-    """
-    if index.is_unique:
-        return index
-    from collections import defaultdict
-
-    values = index.values
-    values_set = set(values)
-    indices_dup = index.duplicated(keep="first")
-    values_dup = values[indices_dup]
-    counter = defaultdict(lambda: 0)
-    for i, v in enumerate(values_dup):
-        while True:
-            counter[v] += 1
-            tentative_new_name = v + join + str(counter[v])
-            if tentative_new_name not in values_set:
-                values_set.add(tentative_new_name)
-                values_dup[i] = tentative_new_name
-                break
-
-    values[indices_dup] = values_dup
-    index = pd.Index(values)
-    return index

backend/server/cli/upgrade.py

@@ -1,85 +0,0 @@
-import re
-
-import click
-import requests
-from requests.exceptions import ConnectionError
-
-from .. import __version__
-
-# Official SemVer regex: https://semver.org/
-SEMVER_FORMAT = re.compile(
-    r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*["
-    r"a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+("
-    r"?:\.[0-9a-zA-Z-]+)*))?$"
-)
-
-
-def log_upgrade_check():
-    # Sanity-check that the CLI version is a properly-formatted SemVer string
-    assert validate_version_str(__version__, release_only=False)
-
-    # Get the current latest release
-    try:
-        release_tag_generator = (r["tag_name"] for r in _request_cellxgene_releases())
-        latest_release = next(release_tag_generator, lambda tag_name: validate_version_str(tag_name))
-        if version_gt(latest_release, __version__):
-            click.echo(f"There's a new version of cellxgene available ({latest_release})!", err=True)
-            click.echo("To upgrade, run the following: pip install --upgrade cellxgene\n", err=True)
-    except (ConnectionError, RateLimitException):
-        click.echo("Upgrade check failed.\n")
-
-
-class RateLimitException(Exception):
-    """
-    Github API Rate Limit Exception
-    """
-
-
-def _request_cellxgene_releases():
-    def raise_on_rate_limit(response):
-        if response.status_code == 403 and res.headers.get("X-RateLimit-Remaining") == "0":
-            raise RateLimitException
-
-    url = "https://api.github.com/repos/chanzuckerberg/cellxgene/releases"
-    res = requests.get(url)
-    raise_on_rate_limit(res)
-    for release in res.json():
-        yield release
-    while "next" in res.links.keys():
-        res = requests.get(res.links["next"]["url"])
-        raise_on_rate_limit(res)
-        for release in res.json():
-            yield release
-
-
-def validate_version_str(version_str, release_only=True):
-    """
-    Test if a string conforms to SemVer format (https://semver.org/)
-    :param version_str: a string to be validated
-    :param release_only: only declare releases (not prereleases) valid
-    :return: True if the version string is of a valid SemVer format else False
-    """
-    match = SEMVER_FORMAT.match(version_str)
-    has_match = match is not None
-    if has_match and release_only:
-        return not match.group("prerelease")
-    return has_match
-
-
-def split_version(version_string):
-    """
-    Split a SemVer-formatted string into its component integers
-    :param version_string: a SemVer string to be split
-    :return: an array of three integers
-    """
-    match = SEMVER_FORMAT.match(version_string)
-    return [int(match.group(group)) for group in ["major", "minor", "patch"]]
-
-
-def version_gt(left_version, right_version):
-    for left, right in zip(split_version(left_version), split_version(right_version)):
-        if left > right:
-            return True
-        elif right > left:
-            return False
-    return False

backend/server/common/annotations/annotations.py

@@ -1,90 +0,0 @@
-from abc import ABCMeta, abstractmethod
-
-from backend.common.errors import DisabledFeatureError
-from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
-from backend.common.genesets import write_gene_sets_tidycsv
-
-
-class Annotations(metaclass=ABCMeta):
-    """baseclass for annotations and gene sets"""
-
-    def __init__(self, config={}):
-        self.config = config
-
-    def user_annotations_enabled(self):
-        return self.config.get("user-annotations", False)
-
-    def gene_sets_save_enabled(self):
-        return self.config.get("genesets-save", False)
-
-    def check_user_annotations_enabled(self):
-        if not self.user_annotations_enabled():
-            raise DisabledFeatureError("User annotations are disabled.")
-
-    def check_gene_sets_save_enabled(self):
-        if not self.gene_sets_save_enabled():
-            raise DisabledFeatureError("User gene sets save is disabled.")
-
-    def get_schema(self, data_adaptor):
-        schema = []
-        labels = self.read_labels(data_adaptor)
-        if labels is not None and not labels.empty:
-            for col in labels.columns:
-                col_schema = dict(name=col, writable=True)
-                col_schema.update(get_schema_type_hint_of_array(labels[col]))
-                schema.append(col_schema)
-
-        return schema
-
-    @abstractmethod
-    def set_collection(self, name):
-        """set or create a new annotation collection"""
-        pass
-
-    @abstractmethod
-    def read_labels(self, data_adaptor):
-        """Return the labels as a pandas.DataFrame"""
-        pass
-
-    @abstractmethod
-    def write_labels(self, df, data_adaptor):
-        """Write the labels (df) to a persistent storage such that it can later be read"""
-        pass
-
-    @abstractmethod
-    def read_gene_sets(self, data_adaptor):
-        """Return the gene sets from persistent storage"""
-        pass
-
-    @abstractmethod
-    def write_gene_sets(self, gs, data_adaptor):
-        """Write the gene sets (gs) to a persistent storage such that it can later be read"""
-        pass
-
-    @abstractmethod
-    def update_parameters(self, parameters, data_adaptor):
-        """Update configuration parameters that describe information about the annotations feature"""
-        pass
-
-    @staticmethod
-    def gene_sets_to_csv(genesets):
-        """
-        Convert the internal gene sets format (returned by read_gene_set) into
-        the simple Tidy CSV.
-        """
-        from io import StringIO
-
-        if isinstance(genesets, dict):
-            genesets = genesets.values()
-
-        with StringIO() as sio:
-            write_gene_sets_tidycsv(sio, genesets)
-            return sio.getvalue()
-
-    @staticmethod
-    def gene_sets_to_response(genesets):
-        """
-        Convert the internal gene sets format (returned by read_gene_set) into
-        the dict expected by the JSON REST API
-        """
-        return list(genesets.values())

backend/server/common/annotations/local_file_csv.py

@@ -1,281 +0,0 @@
-import base64
-import os
-import re
-import threading
-from datetime import datetime
-from hashlib import blake2b
-
-import pandas as pd
-from flask import session, has_request_context, current_app
-
-from backend.server import __version__ as cellxgene_version
-from backend.server.common.annotations.annotations import Annotations
-from backend.common.genesets import read_gene_sets_tidycsv
-from backend.common.errors import AnnotationsError, ObsoleteRequest
-from backend.common.utils.data_locator import DataLocator
-
-
-class AnnotationsLocalFile(Annotations):
-    CXG_ANNO_COLLECTION = "cxg_anno_collection"
-
-    def __init__(self, config, output_dir, label_output_file, gene_sets_output_file):
-        super().__init__(config)
-        self.output_dir = output_dir
-        self.label_output_file = label_output_file
-        self.gene_sets_output_file = gene_sets_output_file
-        # lock used to protect label file write ops
-        self.label_lock = threading.RLock()
-        self.gene_sets_lock = threading.RLock()
-
-        # cache the most recent cell labels/annotations.
-        self.last_label_fname = None
-        self.last_labels = None
-
-        # cache the most recent gene sets.
-        self.last_geneset_fname = None
-        self.last_geneset = None
-
-        # txn ID - used to de-dup geneset writes
-        self.last_geneset_tid = 0
-
-    def is_safe_collection_name(self, name):
-        """
-        return true if this is a safe collection name
-        this is ultra conservative. If we want to allow full legal file name syntax,
-        we could look at modules like `pathvalidate`
-        """
-        if name is None:
-            return False
-        return re.match(r"^[\w\-]+$", name) is not None
-
-    def set_collection(self, name):
-        session[self.CXG_ANNO_COLLECTION] = name
-        session.permanent = True
-
-    def get_collection(self):
-        if session is None:
-            return None
-        return session.get(self.CXG_ANNO_COLLECTION)
-
-    def read_labels(self, data_adaptor):
-        self.check_user_annotations_enabled()  # raises
-
-        if has_request_context():
-            if not current_app.auth.is_user_authenticated():
-                return pd.DataFrame()
-
-        fname = self._get_celllabels_filename(data_adaptor)
-        with self.label_lock:
-            if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0:
-                # returned the cached labels if possible, otherwise read them from the file
-                if fname == self.last_label_fname:
-                    return self.last_labels
-                else:
-                    labels = pd.read_csv(
-                        fname, dtype="category", index_col=0, header=0, comment="#", keep_default_na=False
-                    )
-                    # update the cache
-                    self.last_label_fname = fname
-                    self.last_labels = labels
-                    return labels
-            else:
-                return pd.DataFrame()
-
-    def write_labels(self, df, data_adaptor):
-        self.check_user_annotations_enabled()  # raises
-
-        # update our internal state and save it.  Multi-threading often enabled,
-        # so treat this as a critical section.
-        with self.label_lock:
-            lastmod = data_adaptor.get_last_mod_time()
-            lastmodstr = "'unknown'" if lastmod is None else lastmod.isoformat(timespec="seconds")
-            header = (
-                f"# Annotations generated on {datetime.now().isoformat(timespec='seconds')} "
-                f"using cellxgene version {cellxgene_version}\n"
-                f"# Input data file was {data_adaptor.get_location()}, "
-                f"which was last modified on {lastmodstr}\n"
-            )
-
-            fname = self._get_celllabels_filename(data_adaptor)
-            self._backup(fname)
-            if not df.empty:
-                with open(fname, "w", newline="") as f:
-                    if header is not None:
-                        f.write(header)
-                    df.to_csv(f)
-            else:
-                open(fname, "w").close()
-
-            # update the cache
-            self.last_label_fname = fname
-            self.last_labels = df
-
-    def read_gene_sets(self, data_adaptor, context=None):
-        if has_request_context():
-            if not current_app.auth.is_user_authenticated():
-                return ({}, self.last_geneset_tid)
-
-        fname = self._get_genesets_filename(data_adaptor)
-        gene_sets = {}
-        tid = None
-        with self.gene_sets_lock:
-            tid = self.last_geneset_tid  # inside the critical section
-            if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0:
-                # return the cached genesets if possible, otherwise read from file and validate them
-                if fname == self.last_geneset_fname:
-                    gene_sets = self.last_geneset
-                else:
-                    # read
-                    gene_sets = read_gene_sets_tidycsv(DataLocator(fname), context)
-
-                    # validate
-                    gene_sets = data_adaptor.check_new_gene_sets(gene_sets, context)
-
-                    # update cache
-                    self.last_geneset_fname = fname
-                    self.last_geneset = gene_sets
-
-        return (gene_sets, tid)
-
-    def write_gene_sets(self, gene_sets, tid, data_adaptor):
-        self.check_gene_sets_save_enabled()  # raises
-
-        if type(tid) != int or tid < 0:
-            raise ValueError("tid must be a positive integer")
-
-        # may raise
-        gene_sets = data_adaptor.check_new_gene_sets(gene_sets)
-
-        with self.gene_sets_lock:
-            # skip if the request is stale
-            if tid is not None:
-                if tid <= self.last_geneset_tid:
-                    raise ObsoleteRequest("TID is stale.")
-                self.last_geneset_tid = tid
-
-            lastmod = data_adaptor.get_last_mod_time()
-            lastmodstr = "'unknown'" if lastmod is None else lastmod.isoformat(timespec="seconds")
-            header = (
-                f"# Gene set generated on {datetime.now().isoformat(timespec='seconds')} "
-                f"using cellxgene version {cellxgene_version}\n"
-                f"# Input data file was {data_adaptor.get_location()}, "
-                f"which was last modified on {lastmodstr}\n"
-            )
-
-            fname = self._get_genesets_filename(data_adaptor)
-            self._backup(fname)
-            with open(fname, "w", newline="") as f:
-                f.write(header)
-                f.write(self.gene_sets_to_csv(gene_sets))
-
-            # update the cache
-            self.last_geneset_fname = fname
-            self.last_geneset = gene_sets if type(gene_sets) == dict else {g["geneset_name"]: g for g in gene_sets}
-
-    def _get_userdata_idhash(self, data_adaptor):
-        """
-        Return a short hash that weakly identifies the user and dataset.
-        Used to create safe annotations output file names.
-        """
-        uid = current_app.auth.get_user_id() or ""
-        id = (uid + data_adaptor.get_location()).encode()
-        idhash = base64.b32encode(blake2b(id, digest_size=5).digest()).decode("utf-8")
-        return idhash
-
-    def _get_output_dir(self):
-        if self.output_dir:
-            return self.output_dir
-
-        output_file = self.label_output_file or self.gene_sets_output_file
-        if output_file:
-            return os.path.dirname(os.path.abspath(output_file))
-
-        return os.getcwd()
-
-    def _get_celllabels_filename(self, data_adaptor):
-        """return the current annotation file name"""
-        if self.label_output_file:
-            return self.label_output_file
-
-        return self._get_filename(data_adaptor, "cell-labels")
-
-    def _get_genesets_filename(self, data_adaptor):
-        """return the current gene sets file name"""
-        if self.gene_sets_output_file:
-            return self.gene_sets_output_file
-
-        return self._get_filename(data_adaptor, "gene-sets")
-
-    def _get_filename(self, data_adaptor, anno_name):
-        # we need to generate a file name, which we can only do if we have a UID and collection name
-        if session is None:
-            raise AnnotationsError("unable to determine file name for annotations")
-
-        collection = self.get_collection()
-        if collection is None:
-            return None
-
-        if data_adaptor is None:
-            raise AnnotationsError("unable to determine file name for annotations")
-
-        idhash = self._get_userdata_idhash(data_adaptor)
-        return os.path.join(self._get_output_dir(), f"{collection}-{anno_name}-{idhash}.csv")
-
-    def _backup(self, fname, max_backups=9):
-        """
-        save N backups of file to backup_dir.
-            1. fname -> backup_dir/fname-TIME
-            2. delete excess files in backup_dir
-        """
-        root, ext = os.path.splitext(fname)
-        backup_dir = f"{root}-backups"
-
-        # Make sure there is work to do
-        if not os.path.exists(fname):
-            return
-
-        # Ensure backup_dir exists
-        if not os.path.exists(backup_dir):
-            os.mkdir(backup_dir)
-
-        # Save current file to backup_dir
-        fname_base = os.path.basename(fname)
-        fname_base_root, fname_base_ext = os.path.splitext(fname_base)
-        # don't use ISO standard time format, as it contains characters illegal on some filesytems.
-        nowish = datetime.now().strftime("%Y-%m-%dT%H-%M-%S")
-        backup_fname = os.path.join(backup_dir, f"{fname_base_root}-{nowish}{fname_base_ext}")
-        if os.path.exists(backup_fname):
-            os.remove(backup_fname)
-        os.rename(fname, backup_fname)
-
-        # prune the backup_dir to max number of backup files, keeping the most recent backups
-        backups = list(filter(lambda s: s.startswith(fname_base_root), os.listdir(backup_dir)))
-        excess_count = len(backups) - max_backups
-        if excess_count > 0:
-            backups.sort()
-            for bu in backups[0:excess_count]:
-                os.remove(os.path.join(backup_dir, bu))
-
-    def update_parameters(self, parameters, data_adaptor):
-        params = {}
-        params["annotations"] = self.user_annotations_enabled()
-        params["annotations_genesets_readonly"] = not self.gene_sets_save_enabled()
-        params["annotations_genesets_name_is_read_only"] = self.gene_sets_output_file is not None
-        params["user_annotation_collection_name_enabled"] = True
-
-        if self.label_output_file is not None:
-            # user has hard-wired the name of the annotation cell label data collection
-            fname = os.path.basename(self.label_output_file)
-            collection_fname = os.path.splitext(fname)[0]
-            params["annotations-data-collection-is-read-only"] = True
-            params["annotations-data-collection-name"] = collection_fname
-
-        elif session is not None:
-            collection = self.get_collection()
-            params["annotations-data-collection-is-read-only"] = not self.user_annotations_enabled()
-            params["annotations-data-collection-name"] = collection
-
-        if current_app.auth.is_user_authenticated():
-            params["annotations-user-data-idhash"] = self._get_userdata_idhash(data_adaptor)
-
-        parameters.update(params)

backend/server/common/config/__init__.py

@@ -1,4 +0,0 @@
-from backend.common.utils.aws_secret_utils import get_secret_key  # noqa F504
-
-DEFAULT_SERVER_PORT = 5005
-BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20  # 100MB

backend/server/common/config/app_config.py

@@ -1,171 +0,0 @@
-import yaml
-from flatten_dict import unflatten
-
-from backend.server.default_config import get_default_config
-from backend.server.common.config.dataset_config import DatasetConfig
-from backend.server.common.config.server_config import ServerConfig
-from backend.server.common.config.external_config import ExternalConfig
-from backend.common.errors import ConfigurationError
-
-
-class AppConfig(object):
-    """
-    AppConfig stores all the configuration for cellxgene.
-    AppConfig contains one or more DatasetConfig(s) and one ServerConfig.
-    The server_config contains attributes that refer to the server process as a whole.
-    The dataset_config refers to attributes that are associated with the features and
-    presentations of a dataset.
-    AppConfig has methods to initialize, modify, and access the configuration.
-    """
-
-    def __init__(self):
-
-        # the default configuration (see default_config.py)
-        # TODO @madison -- if we always read from the default config (hard coded path) can we set those values as
-        #  defaults within the config class?
-        self.default_config = get_default_config()
-        # the server configuration
-        self.server_config = ServerConfig(self, self.default_config["server"])
-        # the dataset config
-        self.dataset_config = DatasetConfig(None, self, self.default_config["dataset"])
-        #  external config
-        self.external_config = ExternalConfig(self, self.default_config["external"])
-
-        # Set to true when config_completed is called
-        self.is_completed = False
-
-    def get_dataset_config(self):
-        return self.dataset_config
-
-    def check_config(self):
-        """Verify all the attributes in the config have been type checked"""
-        if not self.is_completed:
-            raise ConfigurationError("The configuration has not been completed")
-        self.server_config.check_config()
-        self.dataset_config.check_config()
-        self.external_config.check_config()
-
-    def update_server_config(self, **kw):
-        self.server_config.update(**kw)
-        self.is_complete = False
-
-    def update_dataset_config(self, **kw):
-        self.dataset_config.update(**kw)
-        self.is_complete = False
-
-    def update_single_config_from_path_and_value(self, path, value):
-        """Update a single config parameter with the value.
-        Path is a list of string, that gives a path to the config parameter to be updated.
-        For example, path may be ["server","app","port"].
-        """
-        self.is_complete = False
-        if not isinstance(path, list):
-            raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'")
-        for part in path:
-            if not isinstance(part, str):
-                raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'")
-
-        if len(path) < 1 or path[0] not in ("server", "dataset"):
-            raise ConfigurationError("path must start with 'server', or 'dataset'")
-
-        if path[0] == "server":
-            attr = "__".join(path[1:])
-            try:
-                self.update_server_config(**{attr: value})
-            except ConfigurationError:
-                raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
-        elif path[0] == "dataset":
-            attr = "__".join(path[1:])
-            try:
-                self.update_dataset_config(**{attr: value})
-            except ConfigurationError:
-                raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
-
-    def update_from_config_file(self, config_file):
-        try:
-            with open(config_file) as yml_file:
-                config = yaml.safe_load(yml_file)
-        except yaml.YAMLError as e:
-            raise ConfigurationError(f"The specified config file contained an error: {e}")
-        except OSError as e:
-            raise ConfigurationError(f"Issue retrieving the specified config file: {e}")
-
-        if config.get("server"):
-            self.server_config.update_from_config(config["server"], "server")
-        if config.get("dataset"):
-            self.dataset_config.update_from_config(config["dataset"], "dataset")
-
-        if config.get("external"):
-            self.external_config.update_from_config(config["external"], "external")
-
-        self.is_complete = False
-
-    def config_to_dict(self):
-        """return the configuration as an unflattened dict"""
-        server = self.server_config.create_mapping(self.server_config.default_config)
-        dataset = self.dataset_config.create_mapping(self.dataset_config.default_config)
-        external = self.external_config.create_mapping(self.external_config.default_config)
-        config = dict(server={}, dataset={})
-        for attrname in server.keys():
-            config["server__" + attrname] = getattr(self.server_config, attrname)
-        for attrname in dataset.keys():
-            config["dataset__" + attrname] = getattr(self.dataset_config, attrname)
-        for attrname in external.keys():
-            config["external__" + attrname] = getattr(self.external_config, attrname)
-
-        config = unflatten(config, splitter=lambda key: key.split("__"))
-        return config
-
-    def write_config(self, config_file):
-        """output the config to a yaml file"""
-        config = self.config_to_dict()
-        yaml.dump(config, open(config_file, "w"))
-
-    def changes_from_default(self):
-        """Return all the attribute that are different from the default"""
-        diff_server = self.server_config.changes_from_default()
-        diff_dataset = self.dataset_config.changes_from_default()
-        diff_external = self.external.changes_from_default()
-        diff = dict(server=diff_server, dataset=diff_dataset, external=diff_external)
-        return diff
-
-    def complete_config(self, messagefn=None):
-        """The configure options are checked, and any additional setup based on the config
-        parameters is done"""
-
-        if messagefn is None:
-
-            def noop(message):
-                pass
-
-            messagefn = noop
-
-        # TODO: to give better error messages we can add a mapping between where each config
-        # attribute originated (e.g. command line argument or config file), then in the error
-        # messages we can give correct context for attributes with bad value.
-        context = dict(messagefn=messagefn)
-
-        # complete config for external_config first, since this may update values in the other sections
-        self.external_config.complete_config(context)
-        self.server_config.complete_config(context)
-        self.dataset_config.complete_config(context)
-
-        self.is_completed = True
-        self.check_config()
-
-    def get_matrix_data_cache_manager(self):
-        return self.server_config.matrix_data_cache_manager
-
-    def get_title(self, data_adaptor):
-        return (
-            self.server_config.single_dataset__title
-            if self.server_config.single_dataset__title
-            else data_adaptor.get_title()
-        )
-
-    def get_about(self, data_adaptor):
-        return (
-            self.server_config.single_dataset__about
-            if self.server_config.single_dataset__about
-            else data_adaptor.get_about()
-        )

backend/server/common/config/base_config.py

@@ -1,99 +0,0 @@
-import copy
-
-from flatten_dict import flatten
-from backend.common.errors import ConfigurationError
-
-
-class BaseConfig(object):
-    """
-    This class handles the mechanics of updating and checking attributes.
-    Derived classes are expected to store the actual attributes
-    Currently DatasetConfig and ServerConfig both inherit from BaseConfig.
-    """
-
-    def __init__(self, app_config, default_config):
-        # reference back to the app_config
-        self.app_config = app_config
-        # the complete set of attributes and their default values (unflattened)
-        self.default_config = default_config
-        # used to make sure every attribute value is checked
-        self.attr_checked = {key_name: False for key_name in self.create_mapping(default_config).keys()}
-
-    def create_mapping(self, config):
-        """
-        Create a dictionary where the keys are the name of attributes (using double underscore convention)
-        For example: authentication__type
-
-        The values are a tuple,
-        - the first item of the tuple is a tuple of path elements (location in config 'tree')
-        - the second item is the value of the config parameter
-
-        For example: (('authentication', 'type'), 'session'))
-        """
-        config_copy = copy.deepcopy(config)
-        mapping = {}
-
-        flat_config = flatten(config_copy)
-        for key, value in flat_config.items():
-            # name of the attribute
-            attr = "__".join(key)
-            mapping[attr] = (key, value)
-
-        return mapping
-
-    def validate_correct_type_of_configuration_attribute(self, attrname, vtype):
-        val = getattr(self, attrname)
-        if type(vtype) in (list, tuple):
-            if type(val) not in vtype:
-                tnames = ",".join([x.__name__ for x in vtype])
-                raise ConfigurationError(
-                    f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}"
-                )
-        else:
-            if type(val) != vtype:
-                raise ConfigurationError(
-                    f"Invalid type for attribute: {attrname}, "
-                    f"expected type {vtype.__name__}, got {type(val).__name__}"
-                )
-
-        self.attr_checked[attrname] = True
-
-    def check_config(self):
-        mapping = self.create_mapping(self.default_config)
-        for key in mapping.keys():
-            if not self.attr_checked[key]:
-                raise ConfigurationError(f"The attr '{key}' has not been checked")
-
-    def update(self, **kw):
-        """Update the attributes defined in kw with their new values."""
-        for key, value in kw.items():
-            if not hasattr(self, key):
-                raise ConfigurationError(f"unknown config parameter {key}.")
-            try:
-                if type(value) == tuple:
-                    # convert tuple values to list values
-                    value = list(value)
-                setattr(self, key, value)
-            except KeyError:
-                raise ConfigurationError(f"Unable to set config parameter {key}.")
-
-            self.attr_checked[key] = False
-
-    def update_from_config(self, config, prefix):
-        mapping = self.create_mapping(config)
-        for attr, (key, value) in mapping.items():
-            if not hasattr(self, attr):
-                raise ConfigurationError(f"Unknown key from config file: {prefix}__{attr}")
-            setattr(self, attr, value)
-
-            self.attr_checked[attr] = False
-
-    def changes_from_default(self):
-        """Return all the attribute that are different from the default"""
-        mapping = self.create_mapping(self.default_config)
-        diff = []
-        for attrname, (key, defval) in mapping.items():
-            curval = getattr(self, attrname)
-            if curval != defval:
-                diff.append((attrname, curval, defval))
-        return diff

backend/server/common/config/client_config.py

@@ -1,119 +0,0 @@
-from backend.server import display_version as cellxgene_display_version
-
-
-def get_client_config(app_config, data_adaptor):
-    """
-    Return the configuration as required by the /config REST route
-    """
-
-    server_config = app_config.server_config
-    dataset_config = data_adaptor.dataset_config
-    annotation = dataset_config.user_annotations
-    auth = server_config.auth
-
-    # FIXME The current set of config is not consistently presented:
-    # we have camalCase, hyphen-text, and underscore_text
-
-    # make sure the configuration has been checked.
-    app_config.check_config()
-
-    # display_names
-    title = app_config.get_title(data_adaptor)
-    about = app_config.get_about(data_adaptor)
-
-    display_names = dict(engine=data_adaptor.get_name(), dataset=title)
-
-    # library_versions
-    library_versions = {}
-    library_versions.update(data_adaptor.get_library_versions())
-    library_versions["cellxgene"] = cellxgene_display_version
-
-    # links
-    links = {"about-dataset": about}
-
-    # parameters
-    parameters = {
-        "layout": dataset_config.embeddings__names,
-        "max-category-items": dataset_config.presentation__max_categories,
-        "obs_names": server_config.single_dataset__obs_names,
-        "var_names": server_config.single_dataset__var_names,
-        "diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff,
-        "backed": server_config.adaptor__anndata_adaptor__backed,
-        "disable-diffexp": not dataset_config.diffexp__enable,
-        "annotations": False,
-        "annotations_file": None,
-        "annotations_dir": None,
-        "annotations_genesets": True,  # feature flag
-        "annotations_genesets_readonly": dataset_config.user_annotations__gene_sets__readonly,
-        "annotations_genesets_summary_methods": ["mean"],
-        "custom_colors": dataset_config.presentation__custom_colors,
-        "diffexp-may-be-slow": False,
-    }
-
-    # corpora dataset_props
-    # TODO/Note: putting info from the dataset into the /config is not ideal.
-    # However, it is definitely not part of /schema, and we do not have a top-level
-    # route for data properties.  Consider creating one at some point.
-    corpora_props = data_adaptor.get_corpora_props()
-    if corpora_props and "default_embedding" in corpora_props:
-        default_embedding = corpora_props["default_embedding"]
-        if isinstance(default_embedding, str) and default_embedding.startswith("X_"):
-            default_embedding = default_embedding[2:]  # drop X_ prefix
-        if default_embedding in data_adaptor.get_embedding_names():
-            parameters["default_embedding"] = default_embedding
-
-    data_adaptor.update_parameters(parameters)
-    if annotation:
-        annotation.update_parameters(parameters, data_adaptor)
-
-    # gather it all together
-    client_config = {}
-    config = client_config["config"] = {}
-    config["displayNames"] = display_names
-    config["library_versions"] = library_versions
-    config["links"] = links
-    config["parameters"] = parameters
-    config["corpora_props"] = corpora_props
-    config["limits"] = {
-        "column_request_max": server_config.limits__column_request_max,
-        "diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max,
-    }
-
-    if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
-        config["authentication"] = {
-            "requires_client_login": auth.requires_client_login(),
-        }
-        if auth.requires_client_login():
-            config["authentication"].update(
-                {
-                    # Todo why are these stored on the data_adaptor?
-                    "login": auth.get_login_url(data_adaptor),
-                    "logout": auth.get_logout_url(data_adaptor),
-                }
-            )
-
-    return client_config
-
-
-def get_client_userinfo(app_config, data_adaptor):
-    """
-    Return the userinfo as required by the /userinfo REST route
-    """
-
-    server_config = app_config.server_config
-    dataset_config = data_adaptor.dataset_config
-    auth = server_config.auth
-
-    # make sure the configuration has been checked.
-    app_config.check_config()
-
-    if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
-        userinfo = {}
-        userinfo["userinfo"] = {
-            "is_authenticated": auth.is_user_authenticated(),
-            "username": auth.get_user_name(),
-            "user_id": auth.get_user_id(),
-            "email": auth.get_user_email(),
-            "picture": auth.get_user_picture(),
-        }
-        return userinfo

backend/server/common/config/dataset_config.py

@@ -1,194 +0,0 @@
-import os
-from os.path import splitext, isdir
-
-from backend.server.common.annotations.local_file_csv import AnnotationsLocalFile
-from backend.server.common.config.base_config import BaseConfig
-from backend.common.errors import ConfigurationError, AnnotationsError
-from backend.server.data_common.matrix_loader import MatrixDataLoader
-
-
-class DatasetConfig(BaseConfig):
-    """Manages the config attribute associated with a dataset."""
-
-    def __init__(self, tag, app_config, default_config):
-        super().__init__(app_config, default_config)
-        self.tag = tag
-        try:
-            self.app__scripts = default_config["app"]["scripts"]
-            self.app__inline_scripts = default_config["app"]["inline_scripts"]
-            self.app__authentication_enable = default_config["app"]["authentication_enable"]
-
-            self.presentation__max_categories = default_config["presentation"]["max_categories"]
-            self.presentation__custom_colors = default_config["presentation"]["custom_colors"]
-
-            self.user_annotations__enable = default_config["user_annotations"]["enable"]
-            self.user_annotations__type = default_config["user_annotations"]["type"]
-            self.user_annotations__local_file_csv__directory = default_config["user_annotations"]["local_file_csv"][
-                "directory"
-            ]
-            self.user_annotations__local_file_csv__file = default_config["user_annotations"]["local_file_csv"]["file"]
-            self.user_annotations__gene_sets__readonly = default_config["user_annotations"]["gene_sets"]["readonly"]
-            self.user_annotations__local_file_csv__gene_sets_file = default_config["user_annotations"][
-                "local_file_csv"
-            ]["gene_sets_file"]
-
-            self.embeddings__names = default_config["embeddings"]["names"]
-
-            self.diffexp__enable = default_config["diffexp"]["enable"]
-            self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
-            self.diffexp__top_n = default_config["diffexp"]["top_n"]
-
-            self.X_approximate_distribution = default_config["X_approximate_distribution"]
-
-        except KeyError as e:
-            raise ConfigurationError(f"Unexpected config: {str(e)}")
-
-        # The annotation object is created during complete_config and stored here.
-        self.user_annotations = None
-
-    def complete_config(self, context):
-        self.handle_app()
-        self.handle_presentation()
-        self.handle_user_annotations(context)
-        self.handle_embeddings()
-        self.handle_diffexp(context)
-        self.handle_X_approximate_distribution()
-
-    def get_data_adaptor(self):
-        server_config = self.app_config.server_config
-        if not server_config.data_adaptor:
-            matrix_data_loader = MatrixDataLoader(server_config.single_dataset__datapath, app_config=self.app_config)
-            server_config.data_adaptor = matrix_data_loader.open(self.app_config)
-
-        return server_config.data_adaptor
-
-    def handle_app(self):
-        self.validate_correct_type_of_configuration_attribute("app__scripts", list)
-        self.validate_correct_type_of_configuration_attribute("app__inline_scripts", list)
-        self.validate_correct_type_of_configuration_attribute("app__authentication_enable", bool)
-
-        # scripts can be string (filename) or dict (attributes). Convert string to dict.
-        scripts = []
-        for script in self.app__scripts:
-            try:
-                if isinstance(script, str):
-                    scripts.append({"src": script})
-                elif isinstance(script, dict) and isinstance(script["src"], str):
-                    scripts.append(script)
-                else:
-                    raise Exception
-            except Exception as e:
-                raise ConfigurationError(f"Scripts must be string or a dict containing an src key: {e}")
-
-        self.app__scripts = scripts
-
-    def handle_presentation(self):
-        self.validate_correct_type_of_configuration_attribute("presentation__max_categories", int)
-        self.validate_correct_type_of_configuration_attribute("presentation__custom_colors", bool)
-
-    def handle_user_annotations(self, context):
-        self.validate_correct_type_of_configuration_attribute("user_annotations__enable", bool)
-        self.validate_correct_type_of_configuration_attribute("user_annotations__type", str)
-        self.validate_correct_type_of_configuration_attribute(
-            "user_annotations__local_file_csv__directory", (type(None), str)
-        )
-        self.validate_correct_type_of_configuration_attribute(
-            "user_annotations__local_file_csv__file", (type(None), str)
-        )
-        self.validate_correct_type_of_configuration_attribute(
-            "user_annotations__local_file_csv__gene_sets_file", (type(None), str)
-        )
-        self.validate_correct_type_of_configuration_attribute("user_annotations__gene_sets__readonly", bool)
-
-        if self.user_annotations__enable or not self.user_annotations__gene_sets__readonly:
-            server_config = self.app_config.server_config
-            if not self.app__authentication_enable:
-                raise ConfigurationError("user annotations requires authentication to be enabled")
-            if not server_config.auth.is_valid_authentication_type():
-                auth_type = server_config.authentication__type
-                raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations")
-
-        # Must always have an annotations instance to support genesets. User annotation (cell labels) are optional
-        # as are writable gene sets
-        if self.user_annotations__type == "local_file_csv":
-            self.handle_local_file_csv_annotations(context)
-        else:
-            raise ConfigurationError('The only annotation type support is "local_file_csv"')
-
-        self.check_annotation_config_vars_not_set(context)
-
-    def handle_local_file_csv_annotations(self, context):
-        dirname = self.user_annotations__local_file_csv__directory
-        filename = self.user_annotations__local_file_csv__file
-        genesets_filename = self.user_annotations__local_file_csv__gene_sets_file
-
-        if dirname is not None and (filename is not None or genesets_filename is not None):
-            raise ConfigurationError(
-                "'user-generated-data-dir' may not be used with 'annotations-file' or 'gene-sets-file'."
-            )
-
-        if filename is not None:
-            lf_name, lf_ext = splitext(filename)
-            if lf_ext and lf_ext != ".csv":
-                raise ConfigurationError(f"annotation file type must be .csv: {filename}")
-
-        if genesets_filename is not None:
-            lf_name, lf_ext = splitext(genesets_filename)
-            if lf_ext and lf_ext != ".csv":
-                raise ConfigurationError(f"genesets file type must be .csv: {genesets_filename}")
-
-        if dirname is not None and not isdir(dirname):
-            try:
-                os.mkdir(dirname)
-            except OSError:
-                raise ConfigurationError("Unable to create directory specified by --user-generated-data-dir")
-
-        anno_config = {
-            "user-annotations": self.user_annotations__enable,
-            "genesets-save": not self.user_annotations__gene_sets__readonly,
-        }
-        self.user_annotations = AnnotationsLocalFile(anno_config, dirname, filename, genesets_filename)
-
-        # if the user has specified a fixed label file, go ahead and validate it
-        # so that we can remove errors early in the process.
-        server_config = self.app_config.server_config
-        if server_config.single_dataset__datapath:
-            data_adaptor = self.get_data_adaptor()
-            if self.user_annotations__local_file_csv__file:
-                self.user_annotations.read_labels(data_adaptor)
-            if self.user_annotations__local_file_csv__gene_sets_file:
-                try:
-                    self.user_annotations.read_gene_sets(data_adaptor, context)
-                except (ValueError, AnnotationsError, KeyError) as e:
-                    raise ConfigurationError(f"Unable to read genesets CSV file: {str(e)}") from e
-
-    def check_annotation_config_vars_not_set(self, context):
-        if self.user_annotations__type is not None:
-            dirname = self.user_annotations__local_file_csv__directory
-            filename = self.user_annotations__local_file_csv__file
-            if not self.user_annotations__enable:
-                if filename is not None:
-                    context["messagefn"]("Warning: --annotations-file ignored as annotations are disabled.")
-                if dirname is not None:
-                    context["messagefn"]("Warning: --user-generated-data-dir ignored as annotations are disabled.")
-
-    def handle_embeddings(self):
-        self.validate_correct_type_of_configuration_attribute("embeddings__names", list)
-
-    def handle_diffexp(self, context):
-        self.validate_correct_type_of_configuration_attribute("diffexp__enable", bool)
-        self.validate_correct_type_of_configuration_attribute("diffexp__lfc_cutoff", float)
-        self.validate_correct_type_of_configuration_attribute("diffexp__top_n", int)
-
-        data_adaptor = self.get_data_adaptor()
-        if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False):
-            context["messagefn"](
-                "CAUTION: due to the size of your dataset, " "running differential expression may take longer or fail."
-            )
-
-    def handle_X_approximate_distribution(self):
-        self.validate_correct_type_of_configuration_attribute("X_approximate_distribution", str)
-        if self.X_approximate_distribution not in ["auto", "normal", "count"]:
-            raise ConfigurationError(
-                "X_approximate_distribution has unknown value -- must be 'auto', 'normal' or 'count'."
-            )

backend/server/common/config/external_config.py

@@ -1,96 +0,0 @@
-import os
-
-from backend.server.common.config.base_config import BaseConfig
-from backend.common.errors import ConfigurationError
-from backend.server.common.config import get_secret_key
-from backend.common.errors import SecretKeyRetrievalError
-from backend.common.utils.type_conversion_utils import convert_string_to_value
-
-
-class ExternalConfig(BaseConfig):
-    """Manages the config attribute associated with external configuration sources, such as
-    environment variables or the AWS Secrets Manager."""
-
-    def __init__(self, app_config, default_config):
-        super().__init__(app_config, default_config)
-        try:
-            self.environment = default_config["environment"]
-            self.aws_secrets_manager__region = default_config["aws_secrets_manager"]["region"]
-            self.aws_secrets_manager__secrets = default_config["aws_secrets_manager"]["secrets"]
-
-        except KeyError as e:
-            raise ConfigurationError(f"Unexpected config: {str(e)}")
-
-    def complete_config(self, context):
-        self.handle_environment(context)
-        self.handle_aws_secrets_manager(context)
-
-    def handle_environment(self, context):
-        """For each environment variable defined, get the value (if it is set),
-        and set the specified config parameter"""
-        self.validate_correct_type_of_configuration_attribute("environment", list)
-        for envdict in self.environment:
-            name = envdict.get("name")
-            if name is None:
-                raise ConfigurationError("environment: 'name' is missing")
-            required = envdict.get("required", False)
-            if type(required) != bool:
-                raise ConfigurationError("environment: 'required' must be a bool")
-            path = envdict.get("path")
-            if path is None:
-                raise ConfigurationError("environment: 'path' is missing")
-
-            value = os.environ.get(name)
-            if value is None:
-                if required:
-                    raise ConfigurationError(f"required environment variable '{name}' not set")
-            else:
-                value = convert_string_to_value(value)
-                self.app_config.update_single_config_from_path_and_value(path, value)
-
-    def handle_aws_secrets_manager(self, context):
-        """For each aws secret defined, get the key/values, and set the specified config parameter"""
-        self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", (type(None), str))
-        self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__secrets", list)
-
-        if not self.aws_secrets_manager__secrets:
-            return
-
-        self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", str)
-
-        for secret in self.aws_secrets_manager__secrets:
-            secret_name = secret.get("name")
-            if secret_name is None:
-                raise ConfigurationError("aws_secrets_manager: 'name' is missing")
-            if not isinstance(secret_name, str):
-                raise ConfigurationError("aws_secrets_manager: 'name' must be a string")
-
-            try:
-                secret_dict = get_secret_key(self.aws_secrets_manager__region, secret_name)
-            except SecretKeyRetrievalError as e:
-                raise ConfigurationError(f"Unable to retrieve secret {secret_name}: {str(e)}")
-
-            values = secret.get("values")
-            if values is None:
-                raise ConfigurationError("aws_secrets_manager: 'values' is missing")
-            if not isinstance(values, list):
-                raise ConfigurationError("aws_secrets_manager: 'values' must be a list")
-
-            for value in values:
-                key = value.get("key")
-                if key is None:
-                    raise ConfigurationError(f"missing 'key' in secret values: {secret_name}")
-                path = value.get("path")
-                if path is None:
-                    raise ConfigurationError(f"missing 'path' in secret values: {secret_name}")
-                required = value.get("required", False)
-                if type(required) != bool:
-                    raise ConfigurationError(f"wrong type for 'required' in secret values: {secret_name}")
-
-                secret_value = secret_dict.get(key)
-                if secret_value is None:
-                    if required:
-                        raise ConfigurationError(f"required secret '{secret_name}:{key}' not set")
-                else:
-                    secret_value = convert_string_to_value(secret_value)
-                    self.app_config.update_single_config_from_path_and_value(path, secret_value)

backend/server/common/config/server_config.py

@@ -1,185 +0,0 @@
-import os
-import sys
-import warnings
-from os.path import basename
-from urllib.parse import urlparse
-
-from backend.server.auth.auth import AuthTypeFactory
-from backend.server.common.config.base_config import BaseConfig
-from backend.server.common.config import DEFAULT_SERVER_PORT, BIG_FILE_SIZE_THRESHOLD
-from backend.common.utils.data_locator import discover_s3_region_name
-from backend.common.errors import ConfigurationError, DatasetAccessError
-from backend.common.utils.utils import is_port_available, find_available_port, custom_format_warning
-from backend.server.data_common.matrix_loader import MatrixDataLoader
-
-
-class ServerConfig(BaseConfig):
-    """Manages the config attribute associated with the server."""
-
-    def __init__(self, app_config, default_config):
-        super().__init__(app_config, default_config)
-
-        try:
-            self.app__verbose = default_config["app"]["verbose"]
-            self.app__debug = default_config["app"]["debug"]
-            self.app__host = default_config["app"]["host"]
-            self.app__port = default_config["app"]["port"]
-            self.app__open_browser = default_config["app"]["open_browser"]
-            self.app__force_https = default_config["app"]["force_https"]
-            self.app__flask_secret_key = default_config["app"]["flask_secret_key"]
-            self.app__generate_cache_control_headers = default_config["app"]["generate_cache_control_headers"]
-
-            self.authentication__type = default_config["authentication"]["type"]
-            self.authentication__insecure_test_environment = default_config["authentication"][
-                "insecure_test_environment"
-            ]
-
-            self.single_dataset__datapath = default_config["single_dataset"]["datapath"]
-            self.single_dataset__obs_names = default_config["single_dataset"]["obs_names"]
-            self.single_dataset__var_names = default_config["single_dataset"]["var_names"]
-            self.single_dataset__about = default_config["single_dataset"]["about"]
-            self.single_dataset__title = default_config["single_dataset"]["title"]
-
-            self.data_locator__s3__region_name = default_config["data_locator"]["s3"]["region_name"]
-
-            self.adaptor__anndata_adaptor__backed = default_config["adaptor"]["anndata_adaptor"]["backed"]
-
-            self.limits__diffexp_cellcount_max = default_config["limits"]["diffexp_cellcount_max"]
-            self.limits__column_request_max = default_config["limits"]["column_request_max"]
-
-        except KeyError as e:
-            raise ConfigurationError(f"Unexpected config: {str(e)}")
-
-        self.data_adaptor = None
-
-        # The authentication object
-        self.auth = None
-
-    def complete_config(self, context):
-        self.handle_app(context)
-        self.handle_data_source()
-        self.handle_authentication()
-        self.handle_data_locator()
-        self.handle_adaptor()  # may depend on data_locator
-        self.handle_single_dataset(context)  # may depend on adaptor
-        self.handle_limits()
-
-        self.check_config()
-
-    def handle_app(self, context):
-        self.validate_correct_type_of_configuration_attribute("app__verbose", bool)
-        self.validate_correct_type_of_configuration_attribute("app__debug", bool)
-        self.validate_correct_type_of_configuration_attribute("app__host", str)
-        self.validate_correct_type_of_configuration_attribute("app__port", (type(None), int))
-        self.validate_correct_type_of_configuration_attribute("app__open_browser", bool)
-        self.validate_correct_type_of_configuration_attribute("app__force_https", bool)
-        self.validate_correct_type_of_configuration_attribute("app__flask_secret_key", str)
-        self.validate_correct_type_of_configuration_attribute("app__generate_cache_control_headers", bool)
-
-        if self.app__port:
-            try:
-                if not is_port_available(self.app__host, self.app__port):
-                    raise ConfigurationError(
-                        f"The port selected {self.app__port} is in use, please configure an open port."
-                    )
-            except OverflowError:
-                raise ConfigurationError(f"Invalid port: {self.app__port}")
-        else:
-            try:
-                default_server_port = int(os.environ.get("CXG_SERVER_PORT", DEFAULT_SERVER_PORT))
-            except ValueError:
-                raise ConfigurationError(
-                    "Invalid port from environment variable CXG_SERVER_PORT: " + os.environ.get("CXG_SERVER_PORT")
-                )
-            try:
-                self.app__port = find_available_port(self.app__host, default_server_port)
-            except OverflowError:
-                raise ConfigurationError(f"Invalid port: {default_server_port}")
-
-        if self.app__debug:
-            context["messagefn"]("in debug mode, setting verbose=True and open_browser=False")
-            self.app__verbose = True
-            self.app__open_browser = False
-        else:
-            warnings.formatwarning = custom_format_warning
-
-        if not self.app__verbose:
-            sys.tracebacklimit = 0
-
-    def handle_authentication(self):
-        self.validate_correct_type_of_configuration_attribute("authentication__type", (type(None), str))
-        self.validate_correct_type_of_configuration_attribute("authentication__insecure_test_environment", bool)
-
-        if self.authentication__type == "test" and not self.authentication__insecure_test_environment:
-            raise ConfigurationError("Test auth can only be used in an insecure test environment")
-
-        self.auth = AuthTypeFactory.create(self.authentication__type, self)
-        if self.auth is None:
-            raise ConfigurationError(f"Unknown authentication type: {self.authentication__type}")
-
-    def handle_data_locator(self):
-        self.validate_correct_type_of_configuration_attribute("data_locator__s3__region_name", (type(None), bool, str))
-        if self.data_locator__s3__region_name is True:
-            path = self.single_dataset__datapath
-
-            if path.startswith("s3://"):
-                region_name = discover_s3_region_name(path)
-                if region_name is None:
-                    raise ConfigurationError(f"Unable to discover s3 region name from {path}")
-            else:
-                region_name = None
-            self.data_locator__s3__region_name = region_name
-
-    def handle_data_source(self):
-        self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", str)
-
-    def handle_single_dataset(self, context):
-        self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None)))
-        self.validate_correct_type_of_configuration_attribute("single_dataset__title", (str, type(None)))
-        self.validate_correct_type_of_configuration_attribute("single_dataset__about", (str, type(None)))
-        self.validate_correct_type_of_configuration_attribute("single_dataset__obs_names", (str, type(None)))
-        self.validate_correct_type_of_configuration_attribute("single_dataset__var_names", (str, type(None)))
-
-        # preload this data set
-        matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath, app_config=self.app_config)
-        try:
-            matrix_data_loader.pre_load_validation()
-        except DatasetAccessError as e:
-            raise ConfigurationError(str(e))
-
-        file_size = matrix_data_loader.file_size()
-        file_basename = basename(self.single_dataset__datapath)
-        if file_size > BIG_FILE_SIZE_THRESHOLD:
-            context["messagefn"](f"Loading data from {file_basename}, this may take a while...")
-        else:
-            context["messagefn"](f"Loading data from {file_basename}.")
-
-        if self.single_dataset__about:
-
-            def url_check(url):
-                try:
-                    result = urlparse(url)
-                    if all([result.scheme, result.netloc]):
-                        return True
-                    else:
-                        return False
-                except ValueError:
-                    return False
-
-            if not url_check(self.single_dataset__about):
-                raise ConfigurationError(
-                    "Must provide an absolute URL for --about. (Example format: http://example.com)"
-                )
-
-    def handle_adaptor(self):
-        self.validate_correct_type_of_configuration_attribute("adaptor__anndata_adaptor__backed", bool)
-
-    def handle_limits(self):
-        self.validate_correct_type_of_configuration_attribute("limits__diffexp_cellcount_max", (type(None), int))
-        self.validate_correct_type_of_configuration_attribute("limits__column_request_max", (type(None), int))
-
-    def exceeds_limit(self, limit_name, value):
-        limit_value = getattr(self, "limits__" + limit_name, None)
-        if limit_value is None:  # disabled
-            return False
-        return value > limit_value

backend/server/common/health.py

@@ -1,33 +0,0 @@
-from http import HTTPStatus
-from flask import make_response, jsonify
-
-from backend.server import __version__ as cellxgene_version
-from backend.common.utils.data_locator import DataLocator
-
-
-def _is_accessible(path, config):
-    if path is None:
-        return True
-
-    try:
-        dl = DataLocator(path, region_name=config.data_locator__s3__region_name)
-        return dl.exists()
-    except RuntimeError:
-        return False
-
-
-def health_check(config):
-    """
-    simple health check - return HTTP response.
-    See https://tools.ietf.org/id/draft-inadarei-api-health-check-01.html
-    """
-    health = {"status": None, "version": "1", "releaseID": cellxgene_version}
-
-    server_config = config.server_config
-    check = _is_accessible(server_config.single_dataset__datapath, server_config)
-
-    health["status"] = "pass" if check else "fail"
-    code = HTTPStatus.OK if health["status"] == "pass" else HTTPStatus.BAD_REQUEST
-    response = make_response(jsonify(health), code)
-    response.headers["Content-Type"] = "application/health+json"
-    return response

backend/server/common/rest.py

@@ -1,404 +0,0 @@
-import copy
-import logging
-import sys
-from http import HTTPStatus
-import zlib
-import json
-
-from flask import make_response, jsonify, current_app, abort
-from werkzeug.urls import url_unquote
-
-from backend.server.common.config.client_config import get_client_config, get_client_userinfo
-from backend.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg
-from backend.common.errors import (
-    FilterError,
-    JSONEncodingValueError,
-    PrepareError,
-    DisabledFeatureError,
-    ExceedsLimitError,
-    DatasetAccessError,
-    ColorFormatException,
-    AnnotationsError,
-    ObsoleteRequest,
-    UnsupportedSummaryMethod,
-)
-from backend.common.genesets import summarizeQueryHash
-from backend.common.fbs.matrix import decode_matrix_fbs
-
-
-def abort_and_log(code, logmsg, loglevel=logging.DEBUG, include_exc_info=False):
-    """
-    Log the message, then abort with HTTP code. If include_exc_info is true,
-    also include current exception via sys.exc_info().
-    """
-    if include_exc_info:
-        exc_info = sys.exc_info()
-    else:
-        exc_info = False
-    current_app.logger.log(loglevel, logmsg, exc_info=exc_info)
-    # Do NOT send log message to HTTP response.
-    return abort(code)
-
-
-def _query_parameter_to_filter(args):
-    """
-    Convert an annotation value filter, if present in the query args,
-    into the standard dict filter format used by internal code.
-
-    Query param filters look like:  <axis>:name=value, where value
-    may be one of:
-        - a range, min,max, where either may be an open range by using an asterisk, eg, 10,*
-        - a value
-    Eg,
-        ...?tissue=lung&obs:tissue=heart&obs:num_reads=1000,*
-    """
-    filters = {
-        "obs": {},
-        "var": {},
-    }
-
-    # args has already been url-unquoted once.  We assume double escaping
-    # on name and value.
-    try:
-        for key, value in args.items(multi=True):
-            axis, name = key.split(":")
-            if axis not in ("obs", "var"):
-                raise FilterError("unknown filter axis")
-            name = url_unquote(name)
-            current = filters[axis].setdefault(name, {"name": name})
-
-            val_split = value.split(",")
-            if len(val_split) == 1:
-                if "min" in current or "max" in current:
-                    raise FilterError("do not mix range and value filters")
-                value = url_unquote(value)
-                values = current.setdefault("values", [])
-                values.append(value)
-
-            elif len(val_split) == 2:
-                if len(current) > 1:
-                    raise FilterError("duplicate range specification")
-                min = url_unquote(val_split[0])
-                max = url_unquote(val_split[1])
-                if min != "*":
-                    current["min"] = float(min)
-                if max != "*":
-                    current["max"] = float(max)
-                if len(current) < 2:
-                    raise FilterError("must specify at least min or max in range filter")
-
-            else:
-                raise FilterError("badly formated filter value")
-
-    except ValueError as e:
-        raise FilterError(str(e))
-
-    result = {}
-    for axis in ("obs", "var"):
-        axis_filter = filters[axis]
-        if len(axis_filter) > 0:
-            result[axis] = {"annotation_value": [val for val in axis_filter.values()]}
-
-    return result
-
-
-def schema_get_helper(data_adaptor):
-    """helper function to gather the schema from the data source and annotations"""
-    schema = data_adaptor.get_schema()
-    schema = copy.deepcopy(schema)
-
-    # add label obs annotations as needed
-    annotations = data_adaptor.dataset_config.user_annotations
-    if annotations.user_annotations_enabled():
-        label_schema = annotations.get_schema(data_adaptor)
-        schema["annotations"]["obs"]["columns"].extend(label_schema)
-
-    return schema
-
-
-def schema_get(data_adaptor):
-    schema = schema_get_helper(data_adaptor)
-    return make_response(jsonify({"schema": schema}), HTTPStatus.OK)
-
-
-def config_get(app_config, data_adaptor):
-    config = get_client_config(app_config, data_adaptor)
-    return make_response(jsonify(config), HTTPStatus.OK)
-
-
-def userinfo_get(app_config, data_adaptor):
-    config = get_client_userinfo(app_config, data_adaptor)
-    return make_response(jsonify(config), HTTPStatus.OK)
-
-
-def annotations_obs_get(request, data_adaptor):
-    fields = request.args.getlist("annotation-name", None)
-    num_columns_requested = len(data_adaptor.get_obs_keys()) if len(fields) == 0 else len(fields)
-    if data_adaptor.server_config.exceeds_limit("column_request_max", num_columns_requested):
-        return abort(HTTPStatus.BAD_REQUEST)
-    preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
-    if preferred_mimetype != "application/octet-stream":
-        return abort(HTTPStatus.NOT_ACCEPTABLE)
-
-    try:
-        labels = None
-        annotations = data_adaptor.dataset_config.user_annotations
-        if annotations.user_annotations_enabled():
-            labels = annotations.read_labels(data_adaptor)
-        fbs = data_adaptor.annotation_to_fbs_matrix(Axis.OBS, fields, labels)
-        return make_response(fbs, HTTPStatus.OK, {"Content-Type": "application/octet-stream"})
-    except KeyError as e:
-        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
-
-
-def annotations_put_fbs_helper(data_adaptor, fbs):
-    """helper function to write annotations from fbs"""
-    annotations = data_adaptor.dataset_config.user_annotations
-    if not annotations.user_annotations_enabled():
-        raise DisabledFeatureError("Writable annotations are not enabled")
-
-    new_label_df = decode_matrix_fbs(fbs)
-    if not new_label_df.empty:
-        new_label_df = data_adaptor.check_new_labels(new_label_df)
-    annotations.write_labels(new_label_df, data_adaptor)
-
-
-def inflate(data):
-    return zlib.decompress(data)
-
-
-def annotations_obs_put(request, data_adaptor):
-    annotations = data_adaptor.dataset_config.user_annotations
-    if not annotations.user_annotations_enabled():
-        return abort(HTTPStatus.NOT_IMPLEMENTED)
-
-    anno_collection = request.args.get("annotation-collection-name", default=None)
-    fbs = inflate(request.get_data())
-
-    if anno_collection is not None:
-        if not annotations.is_safe_collection_name(anno_collection):
-            return abort(HTTPStatus.BAD_REQUEST, "Bad annotation collection name")
-        annotations.set_collection(anno_collection)
-
-    try:
-        annotations_put_fbs_helper(data_adaptor, fbs)
-        res = json.dumps({"status": "OK"})
-        return make_response(res, HTTPStatus.OK, {"Content-Type": "application/json"})
-    except (ValueError, DisabledFeatureError, KeyError) as e:
-        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
-
-
-def annotations_var_get(request, data_adaptor):
-    fields = request.args.getlist("annotation-name", None)
-    num_columns_requested = len(data_adaptor.get_var_keys()) if len(fields) == 0 else len(fields)
-    if data_adaptor.server_config.exceeds_limit("column_request_max", num_columns_requested):
-        return abort(HTTPStatus.BAD_REQUEST)
-    preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
-    if preferred_mimetype != "application/octet-stream":
-        return abort(HTTPStatus.NOT_ACCEPTABLE)
-
-    try:
-        labels = None
-        return make_response(
-            data_adaptor.annotation_to_fbs_matrix(Axis.VAR, fields, labels),
-            HTTPStatus.OK,
-            {"Content-Type": "application/octet-stream"},
-        )
-    except KeyError as e:
-        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
-
-
-def data_var_put(request, data_adaptor):
-    preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
-    if preferred_mimetype != "application/octet-stream":
-        return abort(HTTPStatus.NOT_ACCEPTABLE)
-
-    filter_json = request.get_json()
-    filter = filter_json["filter"] if filter_json else None
-    try:
-        return make_response(
-            data_adaptor.data_frame_to_fbs_matrix(filter, axis=Axis.VAR),
-            HTTPStatus.OK,
-            {"Content-Type": "application/octet-stream"},
-        )
-    except (FilterError, ValueError, ExceedsLimitError) as e:
-        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
-
-
-def data_var_get(request, data_adaptor):
-    preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
-    if preferred_mimetype != "application/octet-stream":
-        return abort(HTTPStatus.NOT_ACCEPTABLE)
-
-    try:
-        filter = _query_parameter_to_filter(request.args)
-        return make_response(
-            data_adaptor.data_frame_to_fbs_matrix(filter, axis=Axis.VAR),
-            HTTPStatus.OK,
-            {"Content-Type": "application/octet-stream"},
-        )
-    except (FilterError, ValueError, ExceedsLimitError) as e:
-        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
-
-
-def colors_get(data_adaptor):
-    if not data_adaptor.dataset_config.presentation__custom_colors:
-        return make_response(jsonify({}), HTTPStatus.OK)
-    try:
-        return make_response(jsonify(data_adaptor.get_colors()), HTTPStatus.OK)
-    except ColorFormatException as e:
-        return abort_and_log(HTTPStatus.NOT_FOUND, str(e), include_exc_info=True)
-
-
-def diffexp_obs_post(request, data_adaptor):
-    if not data_adaptor.dataset_config.diffexp__enable:
-        return abort(HTTPStatus.NOT_IMPLEMENTED)
-
-    args = request.get_json()
-    try:
-        # TODO: implement varfilter mode
-        mode = DiffExpMode(args["mode"])
-
-        if mode == DiffExpMode.VAR_FILTER or "varFilter" in args:
-            return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, "varFilter not enabled")
-
-        set1_filter = args.get("set1", {"filter": {}})["filter"]
-        set2_filter = args.get("set2", {"filter": {}})["filter"]
-        count = args.get("count", None)
-
-        if set1_filter is None or set2_filter is None or count is None:
-            return abort_and_log(HTTPStatus.BAD_REQUEST, "missing required parameter")
-        if Axis.VAR in set1_filter or Axis.VAR in set2_filter:
-            return abort_and_log(HTTPStatus.BAD_REQUEST, "var axis filter not enabled")
-
-    except (KeyError, TypeError) as e:
-        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
-
-    try:
-        diffexp = data_adaptor.diffexp_topN(set1_filter, set2_filter, count)
-        return make_response(diffexp, HTTPStatus.OK, {"Content-Type": "application/json"})
-    except (ValueError, DisabledFeatureError, FilterError, ExceedsLimitError) as e:
-        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
-    except JSONEncodingValueError:
-        # JSON encoding failure, usually due to bad data. Just let it ripple up
-        # to default exception handler.
-        current_app.logger.warning(JSON_NaN_to_num_warning_msg)
-        raise
-
-
-def layout_obs_get(request, data_adaptor):
-    fields = request.args.getlist("layout-name", None)
-    num_columns_requested = len(data_adaptor.get_embedding_names()) if len(fields) == 0 else len(fields)
-    if data_adaptor.server_config.exceeds_limit("column_request_max", num_columns_requested):
-        return abort(HTTPStatus.BAD_REQUEST)
-
-    preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
-    if preferred_mimetype != "application/octet-stream":
-        return abort(HTTPStatus.NOT_ACCEPTABLE)
-
-    try:
-        return make_response(
-            data_adaptor.layout_to_fbs_matrix(fields), HTTPStatus.OK, {"Content-Type": "application/octet-stream"}
-        )
-    except (KeyError, DatasetAccessError) as e:
-        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
-    except PrepareError:
-        return abort_and_log(
-            HTTPStatus.NOT_IMPLEMENTED,
-            f"No embedding available {request.path}",
-            loglevel=logging.ERROR,
-            include_exc_info=True,
-        )
-
-
-def genesets_get(request, data_adaptor):
-    preferred_mimetype = request.accept_mimetypes.best_match(["application/json", "text/csv"])
-    if preferred_mimetype not in ("application/json", "text/csv"):
-        return abort(HTTPStatus.NOT_ACCEPTABLE)
-
-    try:
-        annotations = data_adaptor.dataset_config.user_annotations
-        (genesets, tid) = annotations.read_gene_sets(data_adaptor)
-
-        if preferred_mimetype == "text/csv":
-            return make_response(
-                annotations.gene_sets_to_csv(genesets),
-                HTTPStatus.OK,
-                {
-                    "Content-Type": "text/csv",
-                    "Content-Disposition": "attachment; filename=genesets.csv",
-                },
-            )
-        else:
-            return make_response(
-                jsonify({"genesets": annotations.gene_sets_to_response(genesets), "tid": tid}), HTTPStatus.OK
-            )
-    except (ValueError, KeyError, AnnotationsError) as e:
-        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e))
-
-
-def genesets_put(request, data_adaptor):
-    annotations = data_adaptor.dataset_config.user_annotations
-    if not annotations.gene_sets_save_enabled():
-        return abort(HTTPStatus.NOT_IMPLEMENTED)
-
-    anno_collection = request.args.get("annotation-collection-name", default=None)
-    if anno_collection is not None:
-        if not annotations.is_safe_collection_name(anno_collection):
-            return abort(HTTPStatus.BAD_REQUEST, "Bad annotation collection name")
-        annotations.set_collection(anno_collection)
-
-    args = request.get_json()
-    try:
-        genesets = args.get("genesets", None)
-        tid = args.get("tid", None)
-        if genesets is None:
-            abort(HTTPStatus.BAD_REQUEST)
-
-        annotations.write_gene_sets(genesets, tid, data_adaptor)
-        return make_response(jsonify({"status": "OK"}), HTTPStatus.OK)
-    except (ValueError, DisabledFeatureError, KeyError) as e:
-        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
-    except (ObsoleteRequest, TypeError) as e:
-        return abort(HTTPStatus.NOT_FOUND, description=str(e))
-
-
-def summarize_var_helper(request, data_adaptor, key, raw_query):
-    preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
-    if preferred_mimetype != "application/octet-stream":
-        return abort(HTTPStatus.NOT_ACCEPTABLE)
-
-    summary_method = request.values.get("method", default="mean")
-    query_hash = summarizeQueryHash(raw_query)
-    if key and query_hash != key:
-        return abort(HTTPStatus.BAD_REQUEST, description="query key did not match")
-
-    args_filter_only = request.values.copy()
-    args_filter_only.poplist("method")
-    args_filter_only.poplist("key")
-
-    try:
-        filter = _query_parameter_to_filter(args_filter_only)
-        return make_response(
-            data_adaptor.summarize_var(summary_method, filter, query_hash),
-            HTTPStatus.OK,
-            {"Content-Type": "application/octet-stream"},
-        )
-    except (ValueError) as e:
-        return abort(HTTPStatus.NOT_FOUND, description=str(e))
-    except (UnsupportedSummaryMethod, FilterError) as e:
-        return abort(HTTPStatus.BAD_REQUEST, description=str(e))
-
-
-def summarize_var_get(request, data_adaptor):
-    return summarize_var_helper(request, data_adaptor, None, request.query_string)
-
-
-def summarize_var_post(request, data_adaptor):
-    if not request.content_type or "application/x-www-form-urlencoded" not in request.content_type:
-        return abort(HTTPStatus.UNSUPPORTED_MEDIA_TYPE)
-    if request.content_length > 1_000_000:  # just a sanity check to avoid memory exhaustion
-        return abort(HTTPStatus.BAD_REQUEST)
-
-    key = request.args.get("key", default=None)
-    return summarize_var_helper(request, data_adaptor, key, request.get_data())

backend/server/common/utils/corpora_constants.py

@@ -1,22 +0,0 @@
-class CorporaConstants(object):
-    REQUIRED_SIMPLE_METADATA_FIELDS = [
-        "version",
-        "title",
-        "layer_descriptions",
-        "organism",
-        "organism_ontology_term_id",
-    ]
-
-    # The Corpora specification requires some values encoded as JSON due to the inability of AnnData to store complex
-    # types.
-    OPTIONAL_JSON_ENCODED_METADATA_FIELD = ["contributors", "project_links"]
-
-    OPTIONAL_SIMPLE_METADATA_FIELDS = [
-        "preprint_doi",
-        "publication_doi",
-        "default_embedding",
-        "default_field",
-        "tags",
-        "project_name",
-        "project_description",
-    ]

backend/server/converters/schema/gene_symbol.py

@@ -1,211 +0,0 @@
-"""Helpers for converting and checking HGNC gene symbols."""
-
-import argparse
-import enum
-import logging
-import os
-import re
-import numpy as np
-import pandas as pd
-
-
-def get_upgraded_var_index(var, hgnc_path=None):
-    """Given an anndata var dataframe, return a new index for the dataframe
-    where human gene symbols have been upgraded to the current HGNC set.
-    """
-
-    if not hgnc_path:
-        hgnc_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "hgnc_complete_set.txt.gz")
-
-    hgnc_symbol_checker = HGNCSymbolChecker.from_hgnc_records(hgnc_path)
-
-    return pd.Index([hgnc_symbol_checker.upgrade_symbol(s) for s in var.index])
-
-
-class SymbolStatus(enum.Enum):
-    """The status of a symbol in the HGNC database.
-
-    APPROVED: Currently a valid symbol
-    WITHDRAWN: A previously approved HGNC symbol for a gene that has since been shown
-               not to exist _unless_ that symbol is also approved
-    AMBIGUOUS: A symbol that is not approved but is an alias or previous symbol for
-               multiple approved symbols
-    UPGRADABLE: A symbol that is not approved but unambiguously maps to an approved
-                symbol
-    UNKNOWN: A symbol that does not appear in HGNC
-    """
-
-    APPROVED = 1
-    WITHDRAWN = 2
-    AMBIGUOUS = 3
-    UPGRADABLE = 4
-    UNKNOWN = 5
-
-
-class HGNCSymbolChecker:
-    """Handle checking and correcting HGNC symbols."""
-
-    def __init__(self, approved_symbols, withdrawn_symbols, ambiguous_symbols, symbol_map):
-        self.approved_symbols = approved_symbols
-        self.withdrawn_symbols = withdrawn_symbols
-        self.ambiguous_symbols = ambiguous_symbols
-        self.symbol_map = symbol_map
-
-    def print_symbol_map(self):
-        """Print out a map from old symbol to new symbol."""
-
-        for symbol_pair in self.symbol_map.items():
-            print("\t".join(symbol_pair))
-
-    def check_symbol(self, symbol):
-        """See if a symbol if approved or something else."""
-        if symbol in self.approved_symbols:
-            return SymbolStatus.APPROVED
-
-        if symbol in self.withdrawn_symbols:
-            return SymbolStatus.WITHDRAWN
-
-        if symbol in self.ambiguous_symbols:
-            return SymbolStatus.AMBIGUOUS
-
-        if symbol in self.symbol_map:
-            return SymbolStatus.UPGRADABLE
-
-        return SymbolStatus.UNKNOWN
-
-    def upgrade_symbol(self, symbol):
-        """Return the approved symbol for the given symbol.
-
-        If the symbol cannot be upgraded, just return the original symbol.
-        """
-
-        fixed_symbol, stripped_symbol = format_symbol(symbol)
-
-        if fixed_symbol in self.approved_symbols:
-            return fixed_symbol
-        elif fixed_symbol in self.symbol_map:
-            return self.symbol_map[fixed_symbol]
-        elif stripped_symbol in self.approved_symbols:
-            return stripped_symbol
-        elif stripped_symbol in self.symbol_map:
-            return self.symbol_map[stripped_symbol]
-
-        return symbol
-
-    @classmethod
-    def from_hgnc_records(cls, hgnc_dataset_path):
-        """Parse a hgnc database download into a HGNCSymbolChecker object."""
-
-        def all_symbols(record):
-            """Get all the symbols associated with an HGNC record including previous, alias,
-            and approved."""
-            yield format_symbol(record["symbol"])[0]
-            for symbol in alias_and_previous_symbols(record):
-                yield symbol
-
-        def alias_and_previous_symbols(record):
-            """Get alias and previous symbols from an HGNC record."""
-            for field in ("alias_symbol", "prev_symbol"):
-                if record[field] is not np.nan:
-                    for symbol in record[field].split("|"):
-                        yield format_symbol(symbol)[0]
-            # Sometimes something like HGNC:1234 appears in datasets, which we
-            # want to fix as well.
-            yield record["hgnc_id"]
-
-        hgnc_records = pd.read_csv(hgnc_dataset_path, sep="\t", header=0, low_memory=False).to_dict("records")
-
-        # Get all symbols that are currently approved.
-        approved_symbols = set()
-        for record in hgnc_records:
-            if record["status"] == "Approved":
-                approved_symbols.add(format_symbol(record["symbol"])[0])
-
-        # Get all symbols that have been withdrawn
-        withdrawn_symbols = set()
-        for record in hgnc_records:
-            if record["status"] == "Entry Withdrawn":
-                for symbol in all_symbols(record):
-                    withdrawn_symbols.add(symbol)
-
-        # If a symbol is both approved and withdrawn, be optimistic and call it approved
-        logging.warning(
-            f"Some symbols are simulaneously withdrawn and approved\n"
-            f"We will treat them at approved:\n"
-            f"{withdrawn_symbols.intersection(approved_symbols)}"
-        )
-        withdrawn_symbols = withdrawn_symbols.difference(approved_symbols)
-
-        # Now try to map from symbols that are not approved but are an alias or previous symbol for an approved symbol
-        alias_previous_to_approved = {}
-        ambiguous_symbols = set()
-
-        for record in hgnc_records:
-            if record["status"] == "Approved":
-
-                # The approved symbol is what we'll map to
-                approved_symbol = format_symbol(record["symbol"])[0]
-
-                for symbol in alias_and_previous_symbols(record):
-
-                    # If the alias or previous symbol is also an approved symbol,
-                    # we'll just leave it alone
-                    if symbol in approved_symbols:
-                        continue
-
-                    # If the alias or previous symbol maps to a different approved symbol, mark it as ambiguous
-                    if symbol in alias_previous_to_approved and alias_previous_to_approved[symbol] != approved_symbol:
-                        ambiguous_symbols.add(symbol)
-                    else:
-                        alias_previous_to_approved[symbol] = approved_symbol
-
-        # Remove all the ambiguous symbols from the map
-        for ambiguous_symbol in ambiguous_symbols:
-            alias_previous_to_approved.pop(ambiguous_symbol)
-
-        return HGNCSymbolChecker(approved_symbols, withdrawn_symbols, ambiguous_symbols, alias_previous_to_approved)
-
-
-def format_symbol(symbol):
-    """HGNC rules say symbols should all be upper case except for C#orf#. However, case is
-    variable in both alias and previous symbols as well as in the symbols we get in
-    submissions. So, upper case everything except for the one situation where mixed-case
-    is allowed, which are the genes like C2orf157.
-
-    Also, seurat and scanpy append ".1" or "-1" to duplicated gene names, and these altered
-    names persist throughout the life of the object. They won't match against the HGNC database
-    and we want to merge them, so we need to strip off the suffix and try matching again.
-
-    This function takes a symbol and returns the symbol with the fixed case and also with the
-    seurat/scanpy suffix stripped off.
-    """
-
-    match = re.match(r"^(C)(\d+)(orf)(\d+)$", symbol, re.IGNORECASE)
-
-    if match:
-        fixed_case = f"C{match.group(2)}orf{match.group(4)}"
-    else:
-        fixed_case = symbol.upper()
-
-    suffix_stripped = re.sub(r"[\.\-]\d+$", "", fixed_case)
-
-    return fixed_case, suffix_stripped
-
-
-def main():
-    """When called as main, parse a given hgnc download and print out a map from old to new
-    symbol.
-    """
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "hgnc_dataset", help="HGNC dataset tsv, available from www.genenames.org/download/statistics-and-files/"
-    )
-    args = parser.parse_args()
-
-    hgnc_symbol_checker = HGNCSymbolChecker.from_hgnc_records(args.hgnc_dataset)
-
-    hgnc_symbol_checker.print_symbol_map()
-
-
-if __name__ == "__main__":
-    main()

backend/server/converters/schema/ontology.py

@@ -1,86 +0,0 @@
-"""Methods for working with ontologies and the OLS."""
-from urllib.parse import quote_plus
-
-import requests
-
-OLS_API_ROOT = "http://www.ebi.ac.uk/ols/api"
-
-# Curie means something like CL:0000001
-
-
-def _ontology_name(curie):
-    """Get the name of the ontology from the curie, CL or UBERON for example."""
-    return curie.split(":")[0]
-
-
-def _ontology_value(curie):
-    """Get the id component of the curie, 0000001 from CL:0000001 for example."""
-    return curie.split(":")[1]
-
-
-def _double_encode(url):
-    """Double url encode a url. This is required by the OLS API."""
-    return quote_plus(quote_plus(url))
-
-
-def _iri(curie):
-    """Get the iri from a curie. This is a bit hopeful that they all map to purl.obolibrary.org"""
-    if _ontology_name(curie) == "EFO":
-        return f"http://www.ebi.ac.uk/efo/EFO_{_ontology_value(curie)}"
-    return f"http://purl.obolibrary.org/obo/{_ontology_name(curie)}_{_ontology_value(curie)}"
-
-
-class OntologyLookupError(Exception):
-    """Exception for some problem with looking up ontology information."""
-
-
-def _ontology_info_url(curie):
-    """Get the to make a GET to to get information about an ontology term."""
-
-    # If the curie is empty, just return an empty string. This happens when there is no
-    # valid ontology value.
-    if not curie:
-        return ""
-    else:
-        return f"{OLS_API_ROOT}/ontologies/{_ontology_name(curie)}/terms/{_double_encode(_iri(curie))}"
-
-
-def get_ontology_label(curie):
-    """For a given curie like 'CL:1000413', get the label like 'endothelial cell of artery'"""
-
-    url = _ontology_info_url(curie)
-
-    if not url:
-        return ""
-
-    response = requests.get(url)
-
-    if not response.ok:
-        raise OntologyLookupError(
-            f"Curie {curie} lookup failed, got status code {response.status_code}: {response.text}"
-        )
-    return response.json()["label"]
-
-
-def lookup_candidate_term(label, ontology="cl", method="select"):
-    """Lookup candidate terms for a label. This is useful when there is an existing label in a
-    submitted dataset, and you want to find an appropriate ontology term.
-
-    Args:
-      label: the label to find ontology terms for
-      ontology: the ontology to search in, cl or uberon or efo for example
-      method: select or search. search provides much broader results
-
-    Returns:
-      list of (curie, label) tuples returned by OLS
-    """
-    # using OLS REST API [https://www.ebi.ac.uk/ols/docs/api]
-    url = f"{OLS_API_ROOT}/{method}?q={quote_plus(label)}&ontology={ontology.lower()}"
-    response = requests.get(url)
-
-    if not response.ok:
-        raise OntologyLookupError(
-            f"Label {label} lookup failed, got status code {response.status_code}: {response.text}"
-        )
-
-    return [(r["obo_id"], r["label"]) for r in response.json()["response"]["docs"]]

backend/server/converters/schema/remix.py

@@ -1,264 +0,0 @@
-import argparse
-import collections
-import json
-import logging
-import math
-import string
-
-import anndata
-import numpy as np
-import pandas as pd
-import yaml
-
-from . import gene_symbol
-from . import ontology
-from . import validate
-
-REPLACE_SUFFIX = "_original"
-ONTOLOGY_SUFFIX = "_ontology_term_id"
-
-
-def is_curie(value):
-    """Return True iff the value is an OBO-id CURIE like EFO:000001"""
-    return (value.count(":")
-            and all(len(part) > 0 for part in value.split(":"))
-            and all(c in string.digits for c in value.split(":")[1]))
-
-
-def is_ontology_field(field_name):
-    """Return True iff the field_name is an ontology field like tissue_ontology_term_id"""
-    return field_name.endswith(ONTOLOGY_SUFFIX)
-
-
-def get_label_field_name(field_name):
-    """Get the associated label field from an ontology field, assay_ontology_term_id --> assay"""
-    return field_name[: -len(ONTOLOGY_SUFFIX)]
-
-
-def split_suffix(maybe_curie):
-    """Split off the (cell culture) or (organoid) suffix."""
-
-    suffixes = [" (cell culture)", " (organoid)"]
-    for suffix in suffixes:
-        if maybe_curie.endswith(suffix):
-            return maybe_curie[:-len(suffix)], suffix
-    return maybe_curie, ""
-
-
-def get_curie_and_label(maybe_curie):
-    """Given a string that might be a curie, return a (curie, label) pair"""
-
-    maybe_curie, suffix = split_suffix(maybe_curie)
-    if not is_curie(maybe_curie):
-        return ("", maybe_curie + suffix)
-    return (maybe_curie + suffix, ontology.get_ontology_label(maybe_curie) + suffix)
-
-
-def safe_add_field(adata_attr, field_name, field_value):
-    """Add a field and value to an AnnData, but don't clobber an exising value."""
-
-    if (
-        isinstance(field_value, list)
-        and field_value
-        and isinstance(field_value[0], dict)
-    ):
-        field_value = json.dumps(field_value)
-    if field_name in adata_attr:
-        adata_attr[field_name + REPLACE_SUFFIX] = adata_attr[field_name]
-    adata_attr[field_name] = field_value
-
-
-def remix_uns(adata, uns_config):
-    """Add fields from the config to adata.uns"""
-    for field_name, field_value in uns_config.items():
-
-        if is_ontology_field(field_name):
-            # If it's an ontology field, look it up
-            label_field_name = get_label_field_name(field_name)
-            ontology_term, ontology_label = get_curie_and_label(field_value)
-            safe_add_field(adata.uns, field_name, ontology_term)
-            safe_add_field(adata.uns, label_field_name, ontology_label)
-        else:
-            safe_add_field(adata.uns, field_name, field_value)
-
-
-def remix_obs(adata, obs_config):
-    """Add fields from the config to adata.obs"""
-
-    for field_name, field_value in obs_config.items():
-
-        if isinstance(field_value, dict):
-            # If the value is a dict, that means we are supposed to map from an
-            # existing column to the new one
-            source_column, column_map = next(iter(field_value.items()))
-            nan_value = None
-            for key in column_map:
-                if isinstance(key, float) and math.isnan(key):
-                    nan_value = column_map[key]
-            if nan_value is not None:
-                column_map["nan"] = nan_value
-
-            for key in column_map:
-                if key not in adata.obs[source_column].unique():
-                    logging.warning(f'Key {key} not in adata.obs["{source_column}"]')
-
-            for value in adata.obs[source_column].unique():
-                if value not in column_map:
-                    logging.warning(f'Value {value} in adata.obs["{source_column}"] not in translation dict')
-
-            if is_ontology_field(field_name):
-                ontology_term_map, ontology_label_map = {}, {}
-                logging.info(f"Looking up labels for {field_name}")
-                for original_value, maybe_curie in column_map.items():
-                    curie, label = get_curie_and_label(maybe_curie)
-                    ontology_term_map[original_value] = curie
-                    ontology_label_map[original_value] = label
-                    logging.info(f"Mapping {original_value} -> {curie} -> {label}")
-
-                ontology_column = adata.obs[source_column].replace(
-                    ontology_term_map, inplace=False
-                )
-                label_column = adata.obs[source_column].replace(
-                    ontology_label_map, inplace=False
-                )
-
-                safe_add_field(adata.obs, field_name, ontology_column)
-                safe_add_field(
-                    adata.obs, get_label_field_name(field_name), label_column
-                )
-            else:
-                label_column = adata.obs[source_column].replace(
-                    column_map, inplace=False
-                )
-                safe_add_field(adata.obs, field_name, label_column)
-
-        else:
-            if is_ontology_field(field_name):
-                # If it's an ontology field, look it up
-                label_field_name = get_label_field_name(field_name)
-                ontology_term, ontology_label = get_curie_and_label(field_value)
-                safe_add_field(adata.obs, field_name, ontology_term)
-                safe_add_field(adata.obs, label_field_name, ontology_label)
-            else:
-                safe_add_field(adata.obs, field_name, field_value)
-
-
-def merge_df(df, domain, index, columns):
-    """
-    Given a dataframe with duplicate column labels, merge and return a dataframe where
-    the duplicates have been merged together, resulting in a dataframe with unique column
-    labels.
-
-    "merge" depends on the value of domain. If the domain is "raw", then duplicate columns
-    can just be summed. If it's "log1p" or "sqrt", it needs to be exp1m'd or squared, then
-    summed, and then logged or sqrt'd again.
-    """
-
-    if not isinstance(df, np.ndarray):
-        to_merge = df.toarray()
-    else:
-        to_merge = df
-    if domain == "raw":
-        merged_df = pd.DataFrame(to_merge, index=index, columns=columns).sum(
-            axis=1, level=0, skipna=False
-        )
-    elif domain == "log1p":
-        merged_df = (
-            pd.DataFrame(np.expm1(to_merge, dtype=np.float128), index=index, columns=columns)
-            .sum(axis=1, level=0, skipna=False)
-        )
-        merged_df = pd.DataFrame(np.log1p(merged_df.to_numpy()), index=merged_df.index, columns=merged_df.columns)
-    elif domain == "sqrt":
-        merged_df = (
-            pd.DataFrame(np.square(to_merge), index=index, columns=columns)
-            .sum(axis=1, level=0, skipna=False)
-        )
-        merged_df = pd.DataFrame(np.sqrt(merged_df.to_numpy()), index=merged_df.index, columns=merged_df.columns)
-
-    return merged_df
-
-
-def fixup_gene_symbols(adata, fixup_config):
-    """Update the var index to hold a consistent set of HGNC gene symbols."""
-
-    upgraded_var_index = gene_symbol.get_upgraded_var_index(adata.var)
-
-    merged_X = merge_df(adata.X, fixup_config["X"], adata.obs.index, upgraded_var_index)
-    fixup_adata = anndata.AnnData(
-        X=merged_X,
-        obs=adata.obs,
-        var=merged_X.columns.to_frame(name="hgnc_gene_symbol"),
-        uns=adata.uns,
-        obsm=adata.obsm,
-    )
-
-    for layer, domain in fixup_config.items():
-        if layer == "X":
-            continue
-        if layer == "raw.X":
-            df = adata.raw.X
-        else:
-            df = adata.layers[layer]
-
-        merged_df = merge_df(df, domain, adata.obs.index, upgraded_var_index)
-        assert merged_df.index.equals(merged_X.index)
-        assert merged_df.columns.equals(merged_X.columns)
-
-        if domain == "raw":
-            fixup_raw = anndata.AnnData(
-                X=merged_df,
-                obs=adata.obs,
-                var=merged_X.columns.to_frame(name="hgnc_gene_symbol"),
-            )
-            fixup_adata.raw = fixup_raw
-        else:
-            fixup_adata.layers[layer] = merged_df
-
-    return fixup_adata
-
-def _strip_version(adata):
-    """Remove version information from the AnnData object."""
-
-    if "version" in adata.uns_keys():
-        del adata.uns["version"]
-
-def apply_schema(source_h5ad, remix_config, output_filename):
-
-    try:
-        import scanpy
-    except ImportError:
-        raise ImportError("scanpy must be installed for cellxgene schema")
-    adata = scanpy.read_h5ad(source_h5ad)
-    config = yaml.load(open(remix_config), Loader=yaml.FullLoader)
-    remix_uns(adata, config["uns"])
-    remix_obs(adata, config["obs"])
-
-    if config.get("fixup_gene_symbols"):
-        adata = fixup_gene_symbols(adata, config["fixup_gene_symbols"])
-
-    if ("version" in adata.uns_keys()
-            and isinstance(adata.uns["version"], collections.Mapping)
-            and "corpora_schema_version" in adata.uns["version"]):
-        schema_version = adata.uns["version"]["corpora_schema_version"]
-        try:
-            validate.get_schema_definition(schema_version)
-        except ValueError:
-            logging.warning(f"Stripping version information out of AnnData because schema "
-                            f"version {schema_version} is unknown.")
-            _strip_version(adata)
-
-        if not validate.validate_adata(adata, shallow=False):
-            logging.warning(f"Stripping version information out of AnnData because it does not "
-                            f"follow schema version {schema_version} .")
-            _strip_version(adata)
-
-    adata.write_h5ad(output_filename, compression="gzip")
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--source-h5ad", required=True)
-    parser.add_argument("--remix-config", required=True)
-    parser.add_argument("--output-filename", required=True)
-    args = parser.parse_args()
-    apply_schema(args.source_h5ad, args.remix_config, args.output_filename)

backend/server/converters/schema/schema_definitions/1_0_0.yaml

@@ -1,95 +0,0 @@
-title: Corpora schema version 1.0.0
-type: anndata
-components:
-    uns:
-        type: dict
-        keys:
-            version:
-                type: dict
-                keys:
-                    corpora_schema_version: null
-                    corpora_encoding_version: null
-            title:
-                type: string
-            contributors:
-                type: stringified list of dicts
-            layer_descriptions:
-                type: dict
-                keys:
-                    X: null
-            organism:
-                type: string
-                nullable: false
-            organism_ontology_term_id:
-                type: curie
-                prefixes:
-                    - NCBITaxon
-    var:
-        type: dataframe
-        index:
-            type: human-readable string
-            unique: true
-    obs:
-        type: dataframe
-        index:
-            unique: true
-        columns:
-            tissue:
-                type: human-readable string
-                nullable: false
-            tissue_ontology_term_id:
-                type: suffixed curie
-                nullable: true
-                prefixes:
-                    - UBERON
-            assay:
-                type: human-readable string
-                nullable: false
-            assay_ontology_term_id:
-                type: curie
-                nullable: true
-                prefixes:
-                    - EFO
-            disease:
-                type: human-readable string
-                nullable: false
-            disease_ontology_term_id:
-                type: curie
-                nullable: true
-                prefixes:
-                    - MONDO
-                    - PATO
-            cell_type:
-                type: human-readable string
-                nullable: false
-            cell_type_ontology_term_id:
-                type: curie
-                nullable: true
-                prefixes:
-                    - CL
-                    - UBERON
-            sex:
-                type: string
-                enum:
-                    - male
-                    - female
-                    - mixed
-                    - unknown
-                    - other
-            ethnicity:
-                type: human-readable string
-                nullable: false
-            ethnicity_ontology_term_id:
-                type: curie
-                nullable: true
-                prefixes:
-                    - HANCESTRO
-            development_stage:
-                type: human-readable string
-                nullable: false
-            development_stage_ontology_term_id:
-                type: curie
-                nullable: true
-                prefixes:
-                    - HsapDv
-                    - EFO

backend/server/converters/schema/schema_definitions/1_1_0.yaml

@@ -1,93 +0,0 @@
-title: Corpora schema version 1.1.0
-type: anndata
-components:
-    uns:
-        type: dict
-        keys:
-            version:
-                type: dict
-                keys:
-                    corpora_schema_version: null
-                    corpora_encoding_version: null
-            title:
-                type: string
-            layer_descriptions:
-                type: dict
-                keys:
-                    X: null
-            organism:
-                type: string
-                nullable: false
-            organism_ontology_term_id:
-                type: curie
-                prefixes:
-                    - NCBITaxon
-    var:
-        type: dataframe
-        index:
-            type: human-readable string
-            unique: true
-    obs:
-        type: dataframe
-        index:
-            unique: true
-        columns:
-            tissue:
-                type: human-readable string
-                nullable: false
-            tissue_ontology_term_id:
-                type: suffixed curie
-                nullable: true
-                prefixes:
-                    - UBERON
-            assay:
-                type: human-readable string
-                nullable: false
-            assay_ontology_term_id:
-                type: curie
-                nullable: true
-                prefixes:
-                    - EFO
-            disease:
-                type: human-readable string
-                nullable: false
-            disease_ontology_term_id:
-                type: curie
-                nullable: true
-                prefixes:
-                    - MONDO
-                    - PATO
-            cell_type:
-                type: human-readable string
-                nullable: false
-            cell_type_ontology_term_id:
-                type: curie
-                nullable: true
-                prefixes:
-                    - CL
-                    - UBERON
-            sex:
-                type: string
-                enum:
-                    - male
-                    - female
-                    - mixed
-                    - unknown
-                    - other
-            ethnicity:
-                type: human-readable string
-                nullable: false
-            ethnicity_ontology_term_id:
-                type: curie
-                nullable: true
-                prefixes:
-                    - HANCESTRO
-            development_stage:
-                type: human-readable string
-                nullable: false
-            development_stage_ontology_term_id:
-                type: curie
-                nullable: true
-                prefixes:
-                    - HsapDv
-                    - EFO

backend/server/converters/schema/validate.py

@@ -1,236 +0,0 @@
-import json
-import re
-import os
-import sys
-
-import pandas as pd
-import yaml
-
-
-def _is_null(v):
-    """Return True if v is null, for one of the multiple ways a "null" value shows up in an h5ad."""
-    return pd.isnull(v) or (hasattr(v, "__len__") and len(v) == 0)
-
-
-def _validate_stringified_list_of_dicts(s):
-    """Verify that a string can be parsed into a list.
-
-    We have some types that are lists of dicts. Those cannot be stored directly in an h5ad, so we have to
-    json.dumps them. This verifies that we can load them back.
-    """
-
-    try:
-        list_ = json.loads(s)
-        if not isinstance(list_, list):
-            return False
-        for el in list_:
-            if not isinstance(el, dict):
-                return False
-        return True
-    except (json.JSONDecodeError, TypeError):
-        pass
-    return False
-
-
-def _validate_human_readable_string(s):
-    """Verify that a string is human-readable.
-
-    There are parts of the schema where a "human-readable" string is required. "Human-readable" is kind
-    of vague and subjective. I feel like I can read many strings. So here we just check for the main ways
-    that fails: someone puts in an ontology term id or and ensembl gene/transcript id.
-
-    Returns False if s is not a string or is one of those bad string types.
-    """
-
-    return isinstance(s, str) and (not re.match(r"[A-Z]\w+:\d+", s)) and (not re.match(r"ENS[GT]\d+$", s))
-
-
-def _validate_curie(c, prefixes):
-    """Verify that a string is a valid compact URI, like EFO:000001. If prefixes is not empty, make sure the
-    prefix of the curies is in prefixes.
-    """
-
-    if not c:
-        return True
-
-    match = re.match(r"([A-Z]\w+):\d+$", c)
-
-    if prefixes:
-        return match and match.group(1) in prefixes
-    else:
-        return match
-
-
-def _validate_suffixed_curie(c, prefixes):
-    """Verify that a string is a compact URI with an optional suffix like 'EFO:00001 (cell culture)'"""
-
-    # Pull off the suffix
-    suffix = re.findall(r"\ \(.*\)$", c)
-    if suffix:
-        c = c[: -len(suffix[0])]
-    return _validate_curie(c, prefixes)
-
-
-def _validate_column(column, column_name, df_name, schema_def):
-    """Given a schema definition and the column of a dataframe, verify that the column satifies
-    the schema.
-    """
-
-    errors = []
-
-    if schema_def.get("unique"):
-        if column.nunique() != len(column):
-            errors.append(f"Column {column_name} in dataframe {df_name} is not unique.")
-
-    if "nullable" in schema_def and not schema_def["nullable"]:
-        if any(_is_null(v) for v in column):
-            errors.append(f"Column {column_name} in dataframe {df_name} contains empty values.")
-
-    if schema_def.get("type") == "human-readable string":
-        non_readables = [v for v in column if not _validate_human_readable_string(v)]
-        if non_readables:
-            errors.append(
-                f"Column {column_name} in dataframe {df_name} contains non-human-readable "
-                f"values like {non_readables[0]}"
-            )
-
-    if schema_def.get("type") in ("curie", "suffixed curie"):
-        validation_func = _validate_curie if schema_def.get("type") == "curie" else _validate_suffixed_curie
-        non_valid_curies = [v for v in column if not validation_func(v, schema_def.get("prefixes"))]
-        if non_valid_curies:
-            errors.append(
-                f"Column {column_name} in dataframe {df_name} contains invalid ontology values like "
-                f"{non_valid_curies[0]}."
-            )
-            if "prefixes" in schema_def:
-                errors[-1] += f" Values must be curies from one of these ontologies {schema_def['prefixes']}."
-
-    if "enum" in schema_def:
-        bad_enums = [v for v in column if v not in schema_def["enum"]]
-        if bad_enums:
-            errors.append(
-                f"Column {column_name} in dataframe {df_name} contains unpermitted values like "
-                f"{bad_enums[0]}. Values must be one of {schema_def['enum']}."
-            )
-
-    return errors
-
-
-def _validate_dict(dict_, dict_name, schema_def):
-    """Given a schema definition and dict, verify that the dict satifies the schema."""
-
-    errors = []
-
-    for key in schema_def.get("keys", []):
-        if key not in dict_:
-            errors.append(f"{dict_name} is missing key {key}.")
-        elif schema_def["keys"][key]:
-            if schema_def["keys"][key]["type"] == "stringified list of dicts":
-                if not _validate_stringified_list_of_dicts(dict_[key]):
-                    errors.append(
-                        f"Key {key} in {dict_name} should be a JSON-encoded list of dicts, but it is {dict_[key]}"
-                    )
-            elif schema_def["keys"][key]["type"] == "dict":
-                errors.extend(_validate_dict(dict_[key], key, schema_def["keys"][key]))
-            elif schema_def["keys"][key]["type"] == "curie":
-                if not _validate_curie(dict_[key], schema_def["keys"][key]["prefixes"]):
-                    errors.append(f"Key {key} in {dict_name} contains invalid ontology value.")
-            if "nullable" in schema_def["keys"][key] and not schema_def["keys"][key]["nullable"]:
-                if _is_null(dict_[key]):
-                    errors.append(f"Key {key} in dict {dict_name} is an empty value.")
-
-    return errors
-
-
-def _validate_dataframe(df, df_name, schema_def):
-    """Given a dataframe and schema definition, verify that the dataframe follows the schema."""
-
-    errors = []
-
-    if "index" in schema_def:
-        errors.extend(_validate_column(df.index, "index", df_name, schema_def["index"]))
-
-    for column in schema_def.get("columns", []):
-        if column not in df.columns:
-            errors.append(f"Dataframe {df_name} is missing column {column}.")
-        else:
-            errors.extend(_validate_column(df[column], column, df_name, schema_def["columns"][column]))
-
-    return errors
-
-
-def get_schema_definition(version):
-    """Look up and read a schema definition based on a version number like "1.0.0"."""
-
-    path = os.path.join(
-        os.path.dirname(os.path.realpath(__file__)), "schema_definitions", version.replace(".", "_") + ".yaml"
-    )
-
-    if not os.path.isfile(path):
-        raise ValueError(f"No definition for version {version} found.")
-
-    return yaml.load(open(path), Loader=yaml.FullLoader)
-
-
-def deep_check(adata, schema_def):
-    """Perform a "deep" check of the AnnData object using the schema definition.
-
-    This checks all the columns and unstructured metadata rather than just the version.
-
-    Returns a list of error messages. If that list is empty, the object passed validation.
-    """
-
-    errors = []
-
-    for component, component_def in schema_def["components"].items():
-        if component_def["type"] == "dataframe":
-            errors.extend(_validate_dataframe(getattr(adata, component), component, component_def))
-        elif component_def["type"] == "dict":
-            errors.extend(_validate_dict(getattr(adata, component), component, component_def))
-        else:
-            raise ValueError(f"Unexpected component type {component['type']}")
-
-    return errors
-
-
-def validate_adata(adata, shallow):
-    """Validate an AnnData object. If shallow, just check that the required version information is
-    present.
-    """
-
-    # Does it have the version information written into uns?
-    if "version" not in adata.uns_keys() or "corpora_schema_version" not in adata.uns["version"]:
-        print("AnnData file is missing corpora version information")
-        return False
-
-    # We can stop here if it's a "shallow" check, that is, if we're just
-    # checking that version is present.
-    if shallow:
-        return True
-
-    schema_def = get_schema_definition(adata.uns["version"]["corpora_schema_version"])
-
-    errors = deep_check(adata, schema_def)
-
-    for error in errors:
-        print(error)
-
-    return not errors
-
-
-def validate(h5ad_path, shallow=False):
-    """Entry point for validation."""
-
-    try:
-        import scanpy
-    except ImportError:
-        raise ImportError("scanpy must be installed for cellxgene schema")
-
-    try:
-        adata = scanpy.read_h5ad(h5ad_path, backed="r")
-    except (OSError, TypeError):
-        print(f"Unable to open {h5ad_path} with scanpy.")
-        sys.exit(1)
-
-    if not validate_adata(adata, shallow):
-        sys.exit(1)

backend/server/data_anndata/anndata_adaptor.py

@@ -1,373 +0,0 @@
-import warnings
-
-import anndata
-import numpy as np
-from packaging import version
-from pandas.core.dtypes.dtypes import CategoricalDtype
-from scipy import sparse
-
-import backend.common.compute.diffexp_generic as diffexp_generic
-import backend.common.compute.estimate_distribution as estimate_distribution
-from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
-from backend.common.constants import Axis, MAX_LAYOUTS, XApproximateDistribution
-from backend.server.common.corpora import corpora_get_props_from_anndata
-from backend.common.errors import PrepareError, DatasetAccessError
-from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
-from backend.server.data_common.data_adaptor import DataAdaptor
-from backend.common.fbs.matrix import encode_matrix_fbs
-
-anndata_version = version.parse(str(anndata.__version__)).release
-
-
-def anndata_version_is_pre_070():
-    major = anndata_version[0]
-    minor = anndata_version[1] if len(anndata_version) > 1 else 0
-    return major == 0 and minor < 7
-
-
-class AnndataAdaptor(DataAdaptor):
-    def __init__(self, data_locator, app_config=None, dataset_config=None):
-        super().__init__(data_locator, app_config, dataset_config)
-        self.data = None
-        self.X_approximate_distribution = None
-        self._load_data(data_locator)
-        self._validate_and_initialize()
-
-    def cleanup(self):
-        pass
-
-    @staticmethod
-    def pre_load_validation(data_locator):
-        if data_locator.islocal():
-            # if data locator is local, apply file system conventions and other "cheap"
-            # validation checks.  If a URI, defer until we actually fetch the data and
-            # try to read it.  Many of these tests don't make sense for URIs (eg, extension-
-            # based typing).
-            if not data_locator.exists():
-                raise DatasetAccessError("does not exist")
-            if not data_locator.isfile():
-                raise DatasetAccessError("is not a file")
-
-    @staticmethod
-    def file_size(data_locator):
-        return data_locator.size() if data_locator.islocal() else 0
-
-    @staticmethod
-    def open(data_locator, app_config, dataset_config=None):
-        return AnndataAdaptor(data_locator, app_config, dataset_config)
-
-    def get_corpora_props(self):
-        return corpora_get_props_from_anndata(self.data)
-
-    def get_name(self):
-        return "cellxgene anndata adaptor version"
-
-    def get_library_versions(self):
-        return dict(anndata=str(anndata.__version__))
-
-    @staticmethod
-    def _create_unique_column_name(df, col_name_prefix):
-        """given the columns of a dataframe, and a name prefix, return a column name which
-        does not exist in the dataframe, AND which is prefixed by `prefix`
-
-        The approach is to append a numeric suffix, starting at zero and increasing by
-        one, until an unused name is found (eg, prefix_0, prefix_1, ...).
-        """
-        suffix = 0
-        while f"{col_name_prefix}{suffix}" in df:
-            suffix += 1
-        return f"{col_name_prefix}{suffix}"
-
-    def _alias_annotation_names(self):
-        """
-        The front-end relies on the existance of a unique, human-readable
-        index for obs & var (eg, var is typically gene name, obs the cell name).
-        The user can specify these via the --obs-names and --var-names config.
-        If they are not specified, use the existing index to create them, giving
-        the resulting column a unique name (eg, "name").
-
-        In both cases, enforce that the result is unique, and communicate the
-        index column name to the front-end via the obs_names and var_names config
-        (which is incorporated into the schema).
-        """
-        self.original_obs_index = self.data.obs.index
-
-        for (ax_name, var_name) in ((Axis.OBS, "obs"), (Axis.VAR, "var")):
-            config_name = f"single_dataset__{var_name}_names"
-            parameter_name = f"{var_name}_names"
-            name = getattr(self.server_config, config_name)
-            df_axis = getattr(self.data, str(ax_name))
-            if name is None:
-                # Default: create unique names from index
-                if not df_axis.index.is_unique:
-                    raise KeyError(
-                        f"Values in {ax_name}.index must be unique. "
-                        "Please prepare data to contain unique index values, or specify an "
-                        "alternative with --{ax_name}-name."
-                    )
-                name = self._create_unique_column_name(df_axis.columns, "name_")
-                self.parameters[parameter_name] = name
-                # reset index to simple range; alias name to point at the
-                # previously specified index.
-                df_axis.rename_axis(name, inplace=True)
-                df_axis.reset_index(inplace=True)
-            elif name in df_axis.columns:
-                # User has specified alternative column for unique names, and it exists
-                if not df_axis[name].is_unique:
-                    raise KeyError(
-                        f"Values in {ax_name}.{name} must be unique. " "Please prepare data to contain unique values."
-                    )
-                df_axis.reset_index(drop=True, inplace=True)
-                self.parameters[parameter_name] = name
-            else:
-                # user specified a non-existent column name
-                raise KeyError(f"Annotation name {name}, specified in --{ax_name}-name does not exist.")
-
-    def _create_schema(self):
-        self.schema = {
-            "dataframe": {
-                "nObs": self.cell_count,
-                "nVar": self.gene_count,
-                **get_schema_type_hint_of_array(self.data.X),
-            },
-            "annotations": {
-                "obs": {"index": self.parameters.get("obs_names"), "columns": []},
-                "var": {"index": self.parameters.get("var_names"), "columns": []},
-            },
-            "layout": {"obs": []},
-        }
-        for ax in Axis:
-            curr_axis = getattr(self.data, str(ax))
-            for ann in curr_axis:
-                ann_schema = {"name": ann, "writable": False}
-                ann_schema.update(get_schema_type_hint_of_array(curr_axis[ann]))
-                self.schema["annotations"][ax]["columns"].append(ann_schema)
-
-        for layout in self.get_embedding_names():
-            layout_schema = {"name": layout, "type": "float32", "dims": [f"{layout}_0", f"{layout}_1"]}
-            self.schema["layout"]["obs"].append(layout_schema)
-
-    def get_schema(self):
-        return self.schema
-
-    def _load_data(self, data_locator):
-        # as of AnnData 0.6.19, backed mode performs initial load fast, but at the
-        # cost of significantly slower access to X data.
-        try:
-            # there is no guarantee data_locator indicates a local file.  The AnnData
-            # API will only consume local file objects.  If we get a non-local object,
-            # make a copy in tmp, and delete it after we load into memory.
-            with data_locator.local_handle() as lh:
-                # as of AnnData 0.6.19, backed mode performs initial load fast, but at the
-                # cost of significantly slower access to X data.
-                backed = "r" if self.server_config.adaptor__anndata_adaptor__backed else None
-                self.data = anndata.read_h5ad(lh, backed=backed)
-
-        except ValueError:
-            raise DatasetAccessError(
-                "File must be in the .h5ad format. Please read "
-                "https://github.com/theislab/scanpy_usage/blob/master/170505_seurat/info_h5ad.md to "
-                "learn more about this format. You may be able to convert your file into this format "
-                "using `cellxgene prepare`, please run `cellxgene prepare --help` for more "
-                "information."
-            )
-        except MemoryError:
-            raise DatasetAccessError("Out of memory - file is too large for available memory.")
-        except Exception:
-            raise DatasetAccessError(
-                "File not found or is inaccessible. File must be an .h5ad object. "
-                "Please check your input and try again."
-            )
-
-    def _validate_and_initialize(self):
-        if anndata_version_is_pre_070():
-            warnings.warn(
-                "Use of anndata versions older than 0.7 will have serious issues. Please update to at "
-                "least anndata 0.7 or later."
-            )
-
-        # var and obs column names must be unique
-        if not self.data.obs.columns.is_unique or not self.data.var.columns.is_unique:
-            raise KeyError("All annotation column names must be unique.")
-
-        self._alias_annotation_names()
-        self._validate_data_types()
-        self.cell_count = self.data.shape[0]
-        self.gene_count = self.data.shape[1]
-        self._create_schema()
-
-        if self.dataset_config.X_approximate_distribution == "auto":
-            """Lazy evaluate the heuristic if we are backed."""
-            if not self.data.isbacked:
-                self.X_approximate_distribution = estimate_distribution.estimate_approximate_distribution(self.data.X)
-        else:
-            self.X_approximate_distribution = self.dataset_config.X_approximate_distribution
-
-        # heuristic
-        n_values = self.data.shape[0] * self.data.shape[1]
-        if (n_values > 1e8 and self.server_config.adaptor__anndata_adaptor__backed is True) or (n_values > 5e8):
-            self.parameters.update({"diffexp_may_be_slow": True})
-
-    def _is_valid_layout(self, arr):
-        """return True if this layout data is a valid array for front-end presentation:
-        * ndarray, dtype float/int/uint
-        * with shape (n_obs, >= 2)
-        * with all values finite or NaN (no +Inf or -Inf)
-        """
-        is_valid = type(arr) == np.ndarray and arr.dtype.kind in "fiu"
-        is_valid = is_valid and arr.shape[0] == self.data.n_obs and arr.shape[1] >= 2
-        is_valid = is_valid and not np.any(np.isinf(arr)) and not np.all(np.isnan(arr))
-        return is_valid
-
-    def _validate_data_types(self):
-        # The backed API does not support interrogation of the underlying sparsity or sparse matrix type
-        # Fake it by asking for a small subarray and testing it.   NOTE: if the user has ignored our
-        # anndata <= 0.7 warning, opted for the --backed option, and specified a large, sparse dataset,
-        # this "small" indexing request will load the entire X array. This is due to a bug in anndata<=0.7
-        # which will load the entire X matrix to fullfill any slicing request if X is sparse.  See
-        # user warning in _load_data().
-        X0 = self.data.X[0, 0:1]
-        if sparse.isspmatrix(X0) and not sparse.isspmatrix_csc(X0):
-            warnings.warn(
-                "Anndata data matrix is sparse, but not a CSC (columnar) matrix.  "
-                "Performance may be improved by using CSC."
-            )
-        if self.data.X.dtype > np.dtype(np.float32):
-            warnings.warn(
-                f"Anndata data matrix is in {self.data.X.dtype} format not float32. " f"Precision may be truncated."
-            )
-        for ax in Axis:
-            curr_axis = getattr(self.data, str(ax))
-            for ann in curr_axis:
-                datatype = curr_axis[ann].dtype
-                downcast_map = {
-                    "int64": "int32",
-                    "uint32": "int32",
-                    "uint64": "int32",
-                    "float64": "float32",
-                }
-                if datatype in downcast_map:
-                    warnings.warn(
-                        f"Anndata annotation {ax}:{ann} is in unsupported format: {datatype}. "
-                        f"Data will be downcast to {downcast_map[datatype]}."
-                    )
-                if isinstance(datatype, CategoricalDtype):
-                    category_num = len(curr_axis[ann].dtype.categories)
-                    if category_num > 500 and category_num > self.dataset_config.presentation__max_categories:
-                        warnings.warn(
-                            f"{str(ax).title()} annotation '{ann}' has {category_num} categories, this may be "
-                            f"cumbersome or slow to display. We recommend setting the "
-                            f"--max-category-items option to 500, this will hide categorical "
-                            f"annotations with more than 500 categories in the UI"
-                        )
-
-    def annotation_to_fbs_matrix(self, axis, fields=None, labels=None):
-        if axis == Axis.OBS:
-            if labels is not None and not labels.empty:
-                df = self.data.obs.join(labels, self.parameters.get("obs_names"))
-            else:
-                df = self.data.obs
-        else:
-            df = self.data.var
-
-        if fields is not None and len(fields) > 0:
-            df = df[fields]
-        return encode_matrix_fbs(df, col_idx=df.columns)
-
-    def get_embedding_names(self):
-        """
-        Return pre-computed embeddings.
-
-        function:
-            a) generate list of default layouts
-            b) validate layouts are legal.  remove/warn on any that are not
-            c) cap total list of layouts at global const MAX_LAYOUTS
-        """
-        # load default layouts from the data.
-        layouts = self.dataset_config.embeddings__names
-
-        if layouts is None or len(layouts) == 0:
-            layouts = [key[2:] for key in self.data.obsm_keys() if type(key) == str and key.startswith("X_")]
-
-        # remove invalid layouts
-        valid_layouts = []
-        obsm_keys = self.data.obsm_keys()
-        for layout in layouts:
-            layout_name = f"X_{layout}"
-            if layout_name not in obsm_keys:
-                warnings.warn(f"Ignoring unknown layout name: {layout}.")
-            elif not self._is_valid_layout(self.data.obsm[layout_name]):
-                warnings.warn(f"Ignoring layout due to malformed shape or data type: {layout}")
-            else:
-                valid_layouts.append(layout)
-
-        if len(valid_layouts) == 0:
-            raise PrepareError("No valid layout data.")
-
-        # cap layouts to MAX_LAYOUTS
-        return valid_layouts[0:MAX_LAYOUTS]
-
-    def get_embedding_array(self, ename, dims=2):
-        full_embedding = self.data.obsm[f"X_{ename}"]
-        return full_embedding[:, 0:dims]
-
-    def compute_diffexp_ttest(self, maskA, maskB, top_n=None, lfc_cutoff=None):
-        if top_n is None:
-            top_n = self.dataset_config.diffexp__top_n
-        if lfc_cutoff is None:
-            lfc_cutoff = self.dataset_config.diffexp__lfc_cutoff
-        return diffexp_generic.diffexp_ttest(self, maskA, maskB, top_n, lfc_cutoff)
-
-    def get_colors(self):
-        return convert_anndata_category_colors_to_cxg_category_colors(self.data)
-
-    def get_X_array(self, obs_mask=None, var_mask=None):
-        # H5Py does not support boolean indexing (masks), so convert to integer indexing
-        # when backed (ie, when AnnData is using H5Py indexing)
-        if obs_mask is None:
-            obs_mask = slice(None)
-        elif self.data.isbacked and obs_mask.dtype == bool:
-            obs_mask = obs_mask.nonzero()[0]
-        if var_mask is None:
-            var_mask = slice(None)
-        elif self.data.isbacked and var_mask.dtype == bool:
-            var_mask = var_mask.nonzero()[0]
-        X = self.data.X[obs_mask, var_mask]
-        return X
-
-    def get_X_approximate_distribution(self) -> XApproximateDistribution:
-        """return the approximate distribution of the X matrix."""
-        if self.X_approximate_distribution is None:
-            """Not yet evaluated."""
-            assert self.dataset_config.X_approximate_distribution == "auto"
-            self.data = self.data.to_memory()  # loads data
-            self.X_approximate_distribution = estimate_distribution.estimate_approximate_distribution(self.data.X)
-
-        return self.X_approximate_distribution
-
-    def get_shape(self):
-        return self.data.shape
-
-    def query_var_array(self, term_name):
-        return getattr(self.data.var, term_name)
-
-    def query_obs_array(self, term_name):
-        return getattr(self.data.obs, term_name)
-
-    def get_obs_index(self):
-        name = self.server_config.single_dataset__obs_names
-        if name is None:
-            return self.original_obs_index
-        else:
-            return self.data.obs[name]
-
-    def get_obs_columns(self):
-        return self.data.obs.columns
-
-    def get_obs_keys(self):
-        # return list of keys
-        return self.data.obs.keys().to_list()
-
-    def get_var_keys(self):
-        # return list of keys
-        return self.data.var.keys().to_list()

backend/server/data_common/data_adaptor.py

@@ -1,423 +0,0 @@
-from abc import ABCMeta, abstractmethod
-from os.path import basename, splitext
-import numpy as np
-import pandas as pd
-from scipy import sparse
-from server_timing import Timing as ServerTiming
-
-from backend.server.common.config.app_config import AppConfig
-from backend.common.constants import Axis, XApproximateDistribution
-from backend.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError, UnsupportedSummaryMethod
-from backend.common.utils.utils import jsonify_strict
-from backend.common.fbs.matrix import encode_matrix_fbs
-from backend.common.genesets import validate_gene_sets
-
-
-class DataAdaptor(metaclass=ABCMeta):
-    """Base class for loading and accessing matrix data"""
-
-    def __init__(self, data_locator, app_config, dataset_config=None):
-        if not isinstance(app_config, AppConfig):
-            raise TypeError("config expected to be of type AppConfig")
-
-        # location to the dataset
-        self.data_locator = data_locator
-
-        # config is the application configuration
-        self.app_config = app_config
-        self.server_config = self.app_config.server_config
-        self.dataset_config = dataset_config or app_config.dataset_config
-
-        # parameters set by this data adaptor based on the data.
-        self.parameters = {}
-
-    @staticmethod
-    @abstractmethod
-    def pre_load_validation(data_locator):
-        pass
-
-    @staticmethod
-    @abstractmethod
-    def open(data_locator, app_config, dataset_config):
-        pass
-
-    @staticmethod
-    @abstractmethod
-    def file_size(data_locator):
-        pass
-
-    @abstractmethod
-    def get_name(self):
-        """return a string name for this data adaptor"""
-        pass
-
-    @abstractmethod
-    def get_library_versions(self):
-        """return a dictionary of library name to library versions"""
-        pass
-
-    @abstractmethod
-    def get_embedding_names(self):
-        """return a list of pre-computed embedding names"""
-        pass
-
-    @abstractmethod
-    def get_embedding_array(self, ename, dims=2):
-        """return an numpy array for the given pre-computed embedding name."""
-        pass
-
-    @abstractmethod
-    def get_X_array(self, obs_mask=None, var_mask=None):
-        """return the X array, possibly filtered by obs_mask or var_mask.
-        the return type is either ndarray or scipy.sparse.spmatrix."""
-        pass
-
-    def get_X_approximate_distribution(self) -> XApproximateDistribution:
-        """return the approximate distribution of the X matrix."""
-        return XApproximateDistribution.NORMAL
-
-    @abstractmethod
-    def get_shape(self):
-        pass
-
-    @abstractmethod
-    def query_var_array(self, term_var):
-        pass
-
-    @abstractmethod
-    def query_obs_array(self, term_var):
-        pass
-
-    @abstractmethod
-    def get_colors(self):
-        pass
-
-    @abstractmethod
-    def get_obs_index(self):
-        pass
-
-    @abstractmethod
-    def get_obs_columns(self):
-        pass
-
-    @abstractmethod
-    def get_obs_keys(self):
-        # return list of keys
-        pass
-
-    @abstractmethod
-    def get_var_keys(self):
-        # return list of keys
-        pass
-
-    @abstractmethod
-    def cleanup(self):
-        pass
-
-    def get_data_locator(self):
-        return self.data_locator
-
-    def get_location(self):
-        return self.data_locator.uri_or_path
-
-    def get_about(self):
-        return None
-
-    def get_title(self):
-        # default to file name
-        location = self.get_location()
-        if location.endswith("/"):
-            location = location[:-1]
-        return splitext(basename(location))[0]
-
-    def get_corpora_props(self):
-        return None
-
-    @abstractmethod
-    def get_schema(self):
-        """
-        Return current schema
-        """
-        pass
-
-    @abstractmethod
-    def annotation_to_fbs_matrix(self, axis, field=None, uid=None):
-        """
-        Gets annotation value for each observation
-        :param axis: string obs or var
-        :param fields: list of keys for annotation to return, returns all annotation values if not set.
-        :return: flatbuffer: in fbs/matrix.fbs encoding
-        """
-        pass
-
-    def update_parameters(self, parameters):
-        parameters.update(self.parameters)
-
-    def _index_filter_to_mask(self, filter, count):
-        mask = np.zeros((count,), dtype=np.bool)
-        for i in filter:
-            if isinstance(i, list):
-                mask[i[0] : i[1]] = True
-            else:
-                mask[i] = True
-        return mask
-
-    def _axis_filter_to_mask(self, axis, filter, count):
-        mask = np.ones((count,), dtype=np.bool)
-        if "index" in filter:
-            mask = np.logical_and(mask, self._index_filter_to_mask(filter["index"], count))
-        if "annotation_value" in filter:
-            mask = np.logical_and(mask, self._annotation_filter_to_mask(axis, filter["annotation_value"], count))
-
-        return mask
-
-    def _annotation_filter_to_mask(self, axis, filter, count):
-        mask = np.ones((count,), dtype=np.bool)
-        for v in filter:
-            name = v["name"]
-            if axis == Axis.VAR:
-                anno_data = self.query_var_array(name)
-            elif axis == Axis.OBS:
-                anno_data = self.query_obs_array(name)
-
-            if anno_data.dtype.name in ["boolean", "category", "object"]:
-                values = v.get("values", [])
-                key_idx = np.in1d(anno_data, values)
-                mask = np.logical_and(mask, key_idx)
-
-            else:
-                min_ = v.get("min", None)
-                max_ = v.get("max", None)
-                if min_ is not None:
-                    key_idx = (anno_data >= min_).ravel()
-                    mask = np.logical_and(mask, key_idx)
-                if max_ is not None:
-                    key_idx = (anno_data <= max_).ravel()
-                    mask = np.logical_and(mask, key_idx)
-
-        return mask
-
-    def _filter_to_mask(self, filter):
-        """
-        Return the filter as a row and column selection list.
-        No filter on a dimension means 'all'
-        """
-        shape = self.get_shape()
-        var_selector = None
-        obs_selector = None
-        if filter is not None:
-            if Axis.OBS in filter:
-                obs_selector = self._axis_filter_to_mask(Axis.OBS, filter["obs"], shape[0])
-
-            if Axis.VAR in filter:
-                var_selector = self._axis_filter_to_mask(Axis.VAR, filter["var"], shape[1])
-
-        return (obs_selector, var_selector)
-
-    def check_new_labels(self, labels_df):
-        """Check the new annotations labels, then set the labels_df index"""
-        if labels_df is None or labels_df.empty:
-            return
-
-        labels_df.index = self.get_obs_index()
-        if labels_df.index.name is None:
-            labels_df.index.name = "index"
-
-        # all labels must have a name, which must be unique and not used in obs column names
-        if not labels_df.columns.is_unique:
-            raise KeyError("All column names specified in user annotations must be unique.")
-
-        # the label index must be unique, and must have same values the anndata obs index
-        if not labels_df.index.is_unique:
-            raise KeyError("All row index values specified in user annotations must be unique.")
-
-        obs_columns = self.get_obs_columns()
-
-        duplicate_columns = list(set(labels_df.columns) & set(obs_columns))
-        if len(duplicate_columns) > 0:
-            raise KeyError(
-                "Labels file may not contain column names which overlap " f"with h5ad obs columns {duplicate_columns}"
-            )
-
-        # labels must have same count as obs annotations
-        shape = self.get_shape()
-        if labels_df.shape[0] != shape[0]:
-            raise ValueError("Labels file must have same number of rows as data file.")
-
-        # This will convert a float column that contains integer data into an integer type.
-        # This case can occur when a user makes a copy of a category that originally contained integer data.
-        # The client always copies array data to floats, therefore the copy will contain floats instead of integers.
-        # float data is not allowed as a categorical type.
-        if any([np.issubdtype(coltype.type, np.floating) for coltype in labels_df.dtypes]):
-            labels_df = labels_df.convert_dtypes()
-            for col, dtype in zip(labels_df, labels_df.dtypes):
-                if isinstance(dtype, pd.Int32Dtype):
-                    labels_df[col] = labels_df[col].astype("int32")
-                if isinstance(dtype, pd.Int64Dtype):
-                    labels_df[col] = labels_df[col].astype("int64")
-
-        if any([np.issubdtype(coltype.type, np.floating) for coltype in labels_df.dtypes]):
-            raise ValueError("Columns may not have floating point types")
-
-        return labels_df
-
-    def check_new_gene_sets(self, genesets, context=None):
-        var_names = set(self.query_var_array(self.parameters.get("var_names")))
-        return validate_gene_sets(genesets, var_names)
-
-    def data_frame_to_fbs_matrix(self, filter, axis):
-        """
-        Retrieves data 'X' and returns in a flatbuffer Matrix.
-        :param filter: filter: dictionary with filter params
-        :param axis: string obs or var
-        :return: flatbuffer Matrix
-
-        Caveats:
-        * currently only supports access on VAR axis
-        * currently only supports filtering on VAR axis
-        """
-        if axis != Axis.VAR:
-            raise ValueError("Only VAR dimension access is supported")
-
-        try:
-            obs_selector, var_selector = self._filter_to_mask(filter)
-        except (KeyError, IndexError, TypeError, AttributeError):
-            raise FilterError("Error parsing filter")
-
-        if obs_selector is not None:
-            raise FilterError("filtering on obs unsupported")
-
-        num_columns = self.get_shape()[1] if var_selector is None else np.count_nonzero(var_selector)
-        if self.server_config.exceeds_limit("column_request_max", num_columns):
-            raise ExceedsLimitError("Requested dataframe columns exceed column request limit")
-
-        X = self.get_X_array(obs_selector, var_selector)
-        col_idx = np.nonzero([] if var_selector is None else var_selector)[0]
-        return encode_matrix_fbs(X, col_idx=col_idx, row_idx=None)
-
-    def diffexp_topN(self, obsFilterA, obsFilterB, top_n=None):
-        """
-        Computes the top N differentially expressed variables between two observation sets. If mode
-        is "TOP_N", then stats for the top N
-        dataframes
-        contain a subset of variables, then statistics for all variables will be returned, otherwise
-        only the top N vars will be returned.
-        :param obsFilterA: filter: dictionary with filter params for first set of observations
-        :param obsFilterB: filter: dictionary with filter params for second set of observations
-        :param top_n: Limit results to top N (Top var mode only)
-        :return: top N genes and corresponding stats
-        """
-        if Axis.VAR in obsFilterA or Axis.VAR in obsFilterB:
-            raise FilterError("Observation filters may not contain variable conditions")
-        try:
-            shape = self.get_shape()
-            obs_mask_A = self._axis_filter_to_mask(Axis.OBS, obsFilterA["obs"], shape[0])
-            obs_mask_B = self._axis_filter_to_mask(Axis.OBS, obsFilterB["obs"], shape[0])
-        except (KeyError, IndexError):
-            raise FilterError("Error parsing filter")
-        if top_n is None:
-            top_n = self.dataset_config.diffexp__top_n
-
-        if self.server_config.exceeds_limit(
-            "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B)
-        ):
-            raise ExceedsLimitError("Diffexp request exceeds max cell count limit")
-
-        result = self.compute_diffexp_ttest(
-            maskA=obs_mask_A,
-            maskB=obs_mask_B,
-            top_n=top_n,
-            lfc_cutoff=self.dataset_config.diffexp__lfc_cutoff,
-        )
-
-        try:
-            return jsonify_strict(result)
-        except ValueError:
-            raise JSONEncodingValueError("Error encoding differential expression to JSON")
-
-    @abstractmethod
-    def compute_diffexp_ttest(self, maskA, maskB, top_n, lfc_cutoff):
-        pass
-
-    @staticmethod
-    def normalize_embedding(embedding):
-        """Normalize embedding layout to meet client assumptions.
-        Embedding is an ndarray, shape (n_obs, n)., where n is normally 2
-        """
-
-        # scale isotropically
-        try:
-            min = np.nanmin(embedding, axis=0)
-            max = np.nanmax(embedding, axis=0)
-        except RuntimeError:
-            # indicates entire array was NaN, which should propagate
-            min = np.NaN
-            max = np.NaN
-
-        scale = np.amax(max - min)
-        normalized_layout = (embedding - min) / scale
-
-        # translate to center on both axis
-        translate = 0.5 - ((max - min) / scale / 2)
-        normalized_layout = normalized_layout + translate
-
-        normalized_layout = normalized_layout.astype(dtype=np.float32)
-        return normalized_layout
-
-    def layout_to_fbs_matrix(self, fields):
-        """
-        return specified embeddings as a flatbuffer, using the cellxgene matrix fbs encoding.
-
-        * returns only first two dimensions, with name {ename}_0 and {ename}_1,
-          where {ename} is the embedding name.
-        * client assumes each will be individually centered & scaled (isotropically)
-          to a [0, 1] range.
-        * does not support filtering
-
-        """
-        embeddings = self.get_embedding_names() if fields is None or len(fields) == 0 else fields
-        layout_data = []
-        with ServerTiming.time("layout.query"):
-            for ename in embeddings:
-                embedding = self.get_embedding_array(ename, 2)
-                normalized_layout = DataAdaptor.normalize_embedding(embedding)
-                layout_data.append(pd.DataFrame(normalized_layout, columns=[f"{ename}_0", f"{ename}_1"]))
-
-        with ServerTiming.time("layout.encode"):
-            if layout_data:
-                df = pd.concat(layout_data, axis=1, copy=False)
-            else:
-                df = pd.DataFrame()
-            fbs = encode_matrix_fbs(df, col_idx=df.columns, row_idx=None)
-
-        return fbs
-
-    def get_last_mod_time(self):
-        try:
-            lastmod = self.get_data_locator().lastmodtime()
-        except RuntimeError:
-            lastmod = None
-        return lastmod
-
-    def summarize_var(self, method, filter, query_hash):
-        if method != "mean":
-            raise UnsupportedSummaryMethod("Unknown gene set summary method.")
-
-        obs_selector, var_selector = self._filter_to_mask(filter)
-        if obs_selector is not None:
-            raise FilterError("filtering on obs unsupported")
-
-        # if no filter, just return zeros.  We don't have a use case
-        # for summarizing the entire X without a filter, and it would
-        # potentially be quite compute / memory intensive.
-        if var_selector is None or np.count_nonzero(var_selector) == 0:
-            mean = np.zeros((self.get_shape()[0], 1), dtype=np.float32)
-        else:
-            X = self.get_X_array(obs_selector, var_selector)
-            if sparse.issparse(X):
-                mean = X.mean(axis=1).A
-            else:
-                mean = X.mean(axis=1, keepdims=True)
-
-        col_idx = pd.Index([query_hash])
-        return encode_matrix_fbs(mean, col_idx=col_idx, row_idx=None)

backend/server/data_common/matrix_loader.py

@@ -1,56 +0,0 @@
-from enum import Enum
-
-from backend.common.utils.data_locator import DataLocator
-from backend.common.errors import DatasetAccessError
-from http import HTTPStatus
-
-
-class MatrixDataType(Enum):
-    H5AD = "h5ad"
-    UNKNOWN = "unknown"
-
-
-class MatrixDataLoader(object):
-    def __init__(self, location, matrix_data_type=None, app_config=None):
-        """ location can be a string or DataLocator """
-        region_name = None if app_config is None else app_config.server_config.data_locator__s3__region_name
-        self.location = DataLocator(location, region_name=region_name)
-        if not self.location.exists():
-            raise DatasetAccessError("Dataset does not exist.", HTTPStatus.NOT_FOUND)
-
-        # matrix_data_type is an enum value of type MatrixDataType
-        self.matrix_data_type = matrix_data_type
-        # matrix_type is a DataAdaptor type, which corresonds to the matrix_data_type
-        self.matrix_type = None
-
-        if matrix_data_type is None:
-            self.matrix_data_type = self.__matrix_data_type()
-
-        if not self.__matrix_data_type_allowed(app_config):
-            raise DatasetAccessError("Dataset does not have an allowed type.")
-
-        if self.matrix_data_type == MatrixDataType.H5AD:
-            from backend.server.data_anndata.anndata_adaptor import AnndataAdaptor
-
-            self.matrix_type = AnndataAdaptor
-
-    def __matrix_data_type(self):
-        if self.location.path.endswith(".h5ad"):
-            return MatrixDataType.H5AD
-        else:
-            return MatrixDataType.UNKNOWN
-
-    def __matrix_data_type_allowed(self, app_config):
-        return self.matrix_data_type != MatrixDataType.UNKNOWN
-
-    def pre_load_validation(self):
-        if self.matrix_data_type == MatrixDataType.UNKNOWN:
-            raise DatasetAccessError("Dataset does not have a recognized type: .h5ad")
-        self.matrix_type.pre_load_validation(self.location)
-
-    def file_size(self):
-        return self.matrix_type.file_size(self.location)
-
-    def open(self, app_config, dataset_config=None):
-        # create and return a DataAdaptor object
-        return self.matrix_type.open(self.location, app_config, dataset_config)

backend/server/default_config.py

@@ -1,137 +0,0 @@
-import yaml
-
-default_config = """
-server:
-  app:
-    verbose: false
-    debug: false
-    host: localhost
-    port : null
-    open_browser: false
-    force_https: false
-    flask_secret_key: null
-    generate_cache_control_headers: false
-
-  authentication:
-    # The authentication types may be "none" or "session"
-    # none:  No authentication support, features like user_annotations must not be enabled.
-    # session:  A session based userid is automatically generated. (no params needed)
-    type: session
-    insecure_test_environment: false
-
-  single_dataset:
-    # If datapath is set, then cellxgene with serve a single dataset located at datapath.
-    datapath: null
-    obs_names: null
-    var_names: null
-    about: null
-    title: null
-
-  data_locator:
-    s3:
-      # s3 region name.
-      #   if true, then the s3 location is automatically determined from the datapath or dataroot.
-      #   if false/null, then do not set.
-      #   if a string, then use that value (e.g. us-east-1).
-      region_name: true
-
-  adaptor:
-    anndata_adaptor:
-      backed: false
-
-  limits:
-    column_request_max: 32
-    diffexp_cellcount_max: null
-
-
-dataset:
-  app:
-    # Scripts can be a list of either file names (string) or dicts containing keys src, integrity and crossorigin.
-    # these will be injected into the index template as script tags with these attributes set.
-    scripts: []
-    # Inline scripts are a list of file names, where the contents of the file will be injected into the index.
-    inline_scripts: []
-
-    # allow authentication support
-    authentication_enable: true
-
-  presentation:
-    max_categories: 1000
-    custom_colors: true
-
-  user_annotations:
-    enable: true
-    type: local_file_csv
-    local_file_csv:
-      directory: null
-      file: null            # annotations file name
-      gene_sets_file: null  # gene sets file name
-    gene_sets:
-      readonly: false       # gene sets CRUD enabled/disabled
-
-  embeddings:
-    names : []
-
-  diffexp:
-    enable: true
-    lfc_cutoff: 0.01
-    top_n: 10
-
-  X_approximate_distribution: auto
-
-external:
-  # You can retrieve configuration parameters from this config file, the environment,
-  # the AWS secrets manager, or from the "cellxgene launch" command line arguments.
-  # They are applied in that order, meaning that if a parameter is defined in more
-  # than one location, the last one applied takes effect.
-
-  # environment variables:
-  # This section describes how to map environment variables to configuration parameters.
-  # The format is a list defining an environment variable.
-  # Each entry in the list is a dictionary with three entries:
-  # name:  the name of the environment variable
-  # path: the path within the cellxgene configuration to update.
-  # required: (default=False) a boolean.  If true, then it is an error if the environment variable is not set.
-
-  environment:
-     - name: CXG_SECRET_KEY
-       path: [server, app, flask_secret_key]
-       required: false
-
-  # AWS Secrets Manager
-  # This section describes how to map aws secrets to configuration parameters.
-  # The format is the region for the secrets manager, then a list of secrets.
-  # each secret has a name, and a list of values.
-  # Each entry in the list of values is a dictionary with three entries:
-  # key:  the key of the aws secret.
-  # path: the path within the cellxgene configuration to update.
-  # required: (default=False) a boolean.  If true, then it is an error if the key does not exist in the secret.
-  #
-  # example:
-  # aws_secrets_manager:
-  #   region: us-west-2
-  #    - name: my_first_secret
-  #      values:
-  #      - key: flask_secret_key
-  #        path: [server, app, flask_secret_key]
-  #        required: true
-  #      - key: db_uri
-  #        path: [dataset, user_annotations, db_uri]
-  #        required: true
-  #    - name:  my_auth_secret
-  #      values:
-  #      - key: client_secret
-  #        path: [server, authentication, client_secret]
-  #        required: true
-  #      - key: client_id
-  #        path: [server, authentication, client_id]
-  #        required: true
-
-  aws_secrets_manager:
-    region: null
-    secrets: []
-"""
-
-
-def get_default_config():
-    return yaml.load(default_config, Loader=yaml.Loader)

backend/server/requirements-prepare.txt

@@ -1,4 +0,0 @@
-python-igraph>=0.8
-louvain>=0.6
-scanpy
-umap-learn<0.5.0 # The pinned version scanpy is not compatible with latest umap-learn

backend/server/requirements.txt

@@ -1,23 +0,0 @@
-anndata>=0.7.6 # we need to_memory(), added in 0.7.6
-boto3>=1.12.18
-click>=7.1.2
-Flask>=1.0.2,<2.0.0 # Flask 2.0 is not compatible with the latest version of Flask-RESTful (0.3.8)
-Flask-Compress>=1.4.0
-Flask-Cors>=3.0.9  # CVE-2020-25032
-Flask-RESTful>=0.3.6
-flask-server-timing>=0.1.2
-flask-talisman>=0.7.0
-flatbuffers>=1.11.0,<2.0.0 # cellxgene is not compatible with 2.0.0. Requires migration
-flatten-dict>=0.2.0
-fsspec>=0.4.4,<0.8.0
-gunicorn>=20.0.4
-h5py>=3.0.0
-jinja2>=2.11.3  # Flask sub-dependency. Added due to CVE-2020-28493
-numba>=0.51.2
-numpy>=1.17.5
-packaging>=20.0
-pandas>=1.0,!=1.1  # pandas 1.1 breaks tests, https://github.com/pandas-dev/pandas/issues/35446
-PyYAML>=5.4  # CVE-2020-14343
-scipy>=1.4
-requests>=2.22.0
-s3fs==0.4.2

backend/test/__init__.py

@@ -1,11 +0,0 @@
-import random
-import string
-from os import popen
-
-PROJECT_ROOT = popen("git rev-parse --show-toplevel").read().strip()
-FIXTURES_ROOT = PROJECT_ROOT + "/backend/test/fixtures"
-H5AD_FIXTURE = FIXTURES_ROOT + "/pbmc3k-CSC-gz.h5ad"
-
-
-def random_string(n):
-    return "".join(random.choice(string.ascii_letters) for _ in range(n))

backend/test/fixtures/dataset_config_outline.py

@@ -1,32 +0,0 @@
-f"""
-dataset:
-  app:
-    scripts: {scripts} #list of strs (filenames) or dicts containing keys
-    inline_scripts: {inline_scripts} #list of strs (filenames)
-
-    authentication_enable: {authentication_enable}
-
-  presentation:
-    max_categories: {max_categories}
-    custom_colors: {custom_colors}
-
-  user_annotations:
-    enable: {enable_users_annotations}
-    type: {annotation_type}
-    local_file_csv:
-      directory: {local_file_csv_directory}
-      file: {local_file_csv_file}
-      gene_sets_file: {local_file_csv_gene_sets_file}
-    gene_sets:
-      readonly: {gene_sets_readonly}
-
-  embeddings:
-    names: {embedding_names}
-
-  diffexp:
-    enable: {enable_difexp}
-    lfc_cutoff: {lfc_cutoff}
-    top_n: {top_n}
-
-  X_approximate_distribution: {X_approximate_distribution}
-"""

backend/test/fixtures/pbmc3k-genesets.csv

@@ -1,21 +0,0 @@
-# Test fixture
-gene_set_name,gene_set_description,gene_symbol,gene_description
-first gene set name,,F5, a gene_description
-first gene set name,a description,NO_SUCH_GENE, non-existent gene
-first gene set name,a description,F5, duplicate gene
-first gene set name, a description,SUMO3,
-first gene set name,,SRM,
-second_gene_set,,RER1
-second_gene_set,,SIK1
-third gene set,,NO_SUCH_GENE
-fourth_gene_set,fourth description,,gene intentionally missing
-fifth_dataset,,,
-summary test,,ACD,
-summary test,,AATF,
-summary test,,F5,
-summary test,,PIGU,
-geneset_to_delete,,,
-geneset_to_edit,,,
-fill_this_geneset,,,
-empty_this_geneset,,SIK1,
-brush_this_gene,,SIK1,

backend/test/fixtures/server_config_outline.py

@@ -1,33 +0,0 @@
-f"""server:
-  app:
-    verbose: {verbose}
-    debug: {debug}
-    host: {host}
-    port: {port}
-    open_browser: {open_browser}
-    force_https: {force_https}
-    flask_secret_key: {flask_secret_key}
-    generate_cache_control_headers: {generate_cache_control_headers}
-  authentication:
-    type: {auth_type}
-    insecure_test_environment: {insecure_test_environment}
-
-  single_dataset:
-    datapath: {dataset_datapath}
-    obs_names: {obs_names}
-    var_names: {var_names}
-    about: {about}
-    title: {title}
-
-  data_locator:
-    s3:
-      region_name: {data_locater_region_name}
-
-  adaptor:
-    anndata_adaptor:
-      backed: {anndata_backed}
-
-  limits:
-    column_request_max: {column_request_max}
-    diffexp_cellcount_max: {diffexp_cellcount_max}
-"""

backend/test/test_common/unit/__init__.py

@@ -1 +0,0 @@
-

backend/test/test_common/unit/fbs/test_matrix.py

@@ -1,216 +0,0 @@
-import unittest
-import pandas as pd
-import numpy as np
-from scipy import sparse
-from parameterized import parameterized_class
-import json
-
-from backend.test import decode_fbs
-from backend.common.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs
-from backend.common.utils.type_conversion_utils import get_dtypes_and_schemas_of_dataframe
-import backend.common.fbs as fbs
-
-
-class FbsTests(unittest.TestCase):
-    """Test Case for Matrix FBS data encode/decode"""
-
-    def test_encode_boundary(self):
-        """test various boundary checks"""
-
-        # row indexing is unsupported
-        with self.assertRaises(ValueError):
-            encode_matrix_fbs(matrix=pd.DataFrame(), row_idx=[])
-
-        # matrix must be 2D
-        with self.assertRaises(ValueError):
-            encode_matrix_fbs(matrix=np.zeros((3, 2, 1)))
-        with self.assertRaises(ValueError):
-            encode_matrix_fbs(matrix=np.ones((10,)))
-
-    def fbs_checks(self, fbs, dims, expected_types, expected_column_idx):
-        d = decode_fbs.decode_matrix_FBS(fbs)
-        self.assertEqual(d["n_rows"], dims[0])
-        self.assertEqual(d["n_cols"], dims[1])
-        self.assertIsNone(d["row_idx"])
-        self.assertEqual(len(d["columns"]), dims[1])
-        for i in range(0, len(d["columns"])):
-            self.assertEqual(len(d["columns"][i]), dims[0])
-            self.assertIsInstance(d["columns"][i], expected_types[i][0])
-            if expected_types[i][1] is not None:
-                self.assertEqual(d["columns"][i].dtype, expected_types[i][1])
-        if expected_column_idx is not None:
-            self.assertSetEqual(set(expected_column_idx), set(d["col_idx"]))
-
-    def test_encode_DataFrame(self):
-        df = pd.DataFrame(
-            data={
-                "a": np.zeros((10,), dtype=np.float32),
-                "b": np.ones((10,), dtype=np.int64),
-                "c": np.array([i for i in range(0, 10)], dtype=np.uint16),
-                "d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"),
-            }
-        )
-        expected_types = ((np.ndarray, np.float32), (np.ndarray, np.int32), (np.ndarray, np.int32), (list, None))
-        fbs = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
-        self.fbs_checks(fbs, (10, 4), expected_types, ["a", "b", "c", "d"])
-
-    def test_encode_ndarray(self):
-        arr = np.zeros((3, 2), dtype=np.float32)
-        expected_types = ((np.ndarray, np.float32), (np.ndarray, np.float32), (np.ndarray, np.float32))
-        fbs = encode_matrix_fbs(matrix=arr, row_idx=None, col_idx=None)
-        self.fbs_checks(fbs, (3, 2), expected_types, None)
-
-    def test_encode_sparse(self):
-        csc = sparse.csc_matrix(np.array([[0, 1, 2], [3, 0, 4]]))
-        expected_types = ((np.ndarray, np.int32), (np.ndarray, np.int32), (np.ndarray, np.int32))
-        fbs = encode_matrix_fbs(matrix=csc, row_idx=None, col_idx=None)
-        self.fbs_checks(fbs, (2, 3), expected_types, None)
-
-    def test_roundtrip(self):
-        dfSrc = pd.DataFrame(
-            data={
-                "a": np.zeros((10,), dtype=np.float32),
-                "b": np.ones((10,), dtype=np.int64),
-                "c": np.array([i for i in range(0, 10)], dtype=np.uint16),
-                "d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"),
-            }
-        )
-        dfDst = decode_matrix_fbs(encode_matrix_fbs(matrix=dfSrc, col_idx=dfSrc.columns))
-        self.assertEqual(dfSrc.shape, dfDst.shape)
-        self.assertEqual(set(dfSrc.columns), set(dfDst.columns))
-        for c in dfSrc.columns:
-            self.assertTrue(c in dfDst.columns)
-            if isinstance(dfSrc[c], pd.Series):
-                self.assertTrue(np.all(dfSrc[c] == dfDst[c]))
-            else:
-                self.assertEqual(dfSrc[c], dfDst[c])
-
-
-"""
-Test type consistency between FBS encoding and the underlying schema hint.
-
-Basic assertion:  the FBS type returned by encode_matrix_fbs() will be consistent
-with the schema hint returned by type_conversion_utils (which is in turn used
-to create the client schema).
-
-The following test cases are all dicts which contain the following keys:
-    - dataframe - the dataframe used as input for encode_matrix_fbs
-    - expected_fbs_types - upon success, dict of FBS column types expected (eg, Float32Array)
-    - expected_schema_hints - upon success, dict of schema hint
-All are keyed by column name.
-"""
-
-# simple tests that we convert all ints to int32
-int_dtypes = [np.dtype(d) for d in [np.int8, np.uint8, np.int16, np.uint16, np.int32, np.uint32, np.int64, np.uint64]]
-int_test_cases = [
-    {
-        "dataframe": pd.DataFrame({dtype.name: np.zeros((10,), dtype=dtype) for dtype in int_dtypes}),
-        "expected_fbs_types": dict(
-            [(dtype.name, fbs.NetEncoding.TypedArray.TypedArray.Int32Array) for dtype in int_dtypes]
-        ),
-        "expected_schema_hints": dict([(dtype.name, {"type": "int32"}) for dtype in int_dtypes]),
-    }
-]
-
-# simple tests that we convert all floats to float32
-float_dtypes = [np.dtype(d) for d in [np.float16, np.float32, np.float64]]
-float_test_cases = [
-    {
-        "dataframe": pd.DataFrame({dtype.name: np.zeros((10,), dtype=dtype) for dtype in float_dtypes}),
-        "expected_fbs_types": dict(
-            [(dtype.name, fbs.NetEncoding.TypedArray.TypedArray.Float32Array) for dtype in float_dtypes]
-        ),
-        "expected_schema_hints": dict([(dtype.name, {"type": "float32"}) for dtype in float_dtypes]),
-    }
-]
-
-# boolean - should be encoded as an uint32
-bool_dtypes = [np.dtype(d) for d in [np.bool_, bool]]
-bool_test_cases = [
-    {
-        "dataframe": pd.DataFrame({dtype.name: np.ones((10,), dtype=dtype) for dtype in bool_dtypes}),
-        "expected_fbs_types": dict(
-            [(dtype.name, fbs.NetEncoding.TypedArray.TypedArray.Uint32Array) for dtype in bool_dtypes]
-        ),
-        "expected_schema_hints": dict([(dtype.name, {"type": "boolean"}) for dtype in bool_dtypes]),
-    }
-]
-
-cat_test_cases = [
-    {
-        "dataframe": pd.DataFrame({"a": pd.Series(["a", "b", "c", "a", "b", "c"], dtype="category")}),
-        "expected_fbs_types": {"a": fbs.NetEncoding.TypedArray.TypedArray.JSONEncodedArray},
-        "expected_schema_hints": {"a": {"type": "categorical", "categories": ["a", "b", "c"]}},
-    },
-    {
-        "dataframe": pd.DataFrame(
-            {"a": pd.Series(["a", "b", "c", "a", "b", "c"], dtype="category").cat.remove_categories("b")}
-        ),
-        "expected_fbs_types": {"a": fbs.NetEncoding.TypedArray.TypedArray.JSONEncodedArray},
-        "expected_schema_hints": {"a": {"type": "categorical", "categories": ["a", "c"]}},
-    },
-    {
-        "dataframe": pd.DataFrame({"a": pd.Series(np.arange(0, 10, dtype=np.int64), dtype="category")}),
-        "expected_fbs_types": {"a": fbs.NetEncoding.TypedArray.TypedArray.Int32Array},
-        "expected_schema_hints": {"a": {"type": "categorical"}},
-    },
-    {
-        "dataframe": pd.DataFrame(
-            {"a": pd.Series(np.arange(0, 10, dtype=np.int64), dtype="category").cat.remove_categories(2)}
-        ),
-        "expected_fbs_types": {"a": fbs.NetEncoding.TypedArray.TypedArray.Float32Array},
-        "expected_schema_hints": {"a": {"type": "categorical"}},
-    },
-    {
-        "dataframe": pd.DataFrame({"a": pd.Series(np.arange(0, 10, dtype=np.float64), dtype="category")}),
-        "expected_fbs_types": {"a": fbs.NetEncoding.TypedArray.TypedArray.Float32Array},
-        "expected_schema_hints": {"a": {"type": "categorical"}},
-    },
-    {
-        "dataframe": pd.DataFrame(
-            {"a": pd.Series(np.arange(0, 10, dtype=np.float64), dtype="category").cat.remove_categories(2)}
-        ),
-        "expected_fbs_types": {"a": fbs.NetEncoding.TypedArray.TypedArray.Float32Array},
-        "expected_schema_hints": {"a": {"type": "categorical"}},
-    },
-]
-
-
-test_cases = [
-    *int_test_cases,
-    *float_test_cases,
-    *bool_test_cases,
-    *cat_test_cases,
-]
-
-
-@parameterized_class(test_cases)
-class TestTypeConversionConsistency(unittest.TestCase):
-    def test_type_conversion_consistency(self):
-        self.assertEqual(self.dataframe.shape[1], len(self.expected_fbs_types))
-        self.assertEqual(self.dataframe.shape[1], len(self.expected_schema_hints))
-
-        buf = encode_matrix_fbs(matrix=self.dataframe, col_idx=self.dataframe.columns)
-        encoding_dtypes, schema_hints = get_dtypes_and_schemas_of_dataframe(self.dataframe)
-
-        # check schema hints
-        # print(schema_hints)
-        # print(self.expected_schema_hints)
-        self.assertEqual(schema_hints, self.expected_schema_hints)
-
-        # inspect the FBS types
-        matrix = fbs.NetEncoding.Matrix.Matrix.GetRootAsMatrix(buf, 0)
-        columns_length = matrix.ColumnsLength()
-        self.assertEqual(columns_length, self.dataframe.shape[1])
-
-        self.assertEqual(matrix.ColIndexType(), fbs.NetEncoding.TypedArray.TypedArray.JSONEncodedArray)
-        col_labels_arr = fbs.NetEncoding.JSONEncodedArray.JSONEncodedArray()
-        col_labels_arr.Init(matrix.ColIndex().Bytes, matrix.ColIndex().Pos)
-        col_index_labels = json.loads(col_labels_arr.DataAsNumpy().tobytes().decode("utf-8"))
-        self.assertEqual(len(col_index_labels), self.dataframe.shape[1])
-
-        for col_idx in range(0, columns_length):
-            col_label = col_index_labels[col_idx]
-            col = matrix.Columns(col_idx)
-            col_type = col.UType()
-            self.assertEqual(self.expected_fbs_types[col_label], col_type)

backend/test/test_common/unit/utils/test_jsonify_strict.py

@@ -1,58 +0,0 @@
-import unittest
-import numpy as np
-
-from backend.common.utils.utils import (
-    jsonify_strict,
-)
-
-
-class TestJsonifyStrict(unittest.TestCase):
-    def test_jsonify_numpy_general_cases(self):
-        self.assertEqual(jsonify_strict({}), "{}")
-        self.assertEqual(jsonify_strict({"a": [], "b": "hello", "c": True}), '{"a": [], "b": "hello", "c": true}')
-
-    def test_jsonify_numpy_float_edges(self):
-        with self.assertRaises(ValueError):
-            jsonify_strict({"nan": [np.nan]})
-
-        with self.assertRaises(ValueError):
-            jsonify_strict({"pinf": [np.PINF]})
-
-        with self.assertRaises(ValueError):
-            jsonify_strict({"ninf": [np.NINF]})
-
-    def test_jsonify_numpy_ndarray(self):
-        values = {
-            "integer": [
-                np.int8(0),
-                np.int16(1),
-                np.int32(2),
-                np.int64(3),
-                np.uint8(4),
-                np.uint16(5),
-                np.uint32(6),
-                np.uint64(7),
-            ],
-            "floating": [
-                np.float16(100.0),
-                np.float32(101.0),
-                np.float64(102.0),
-            ],
-        }
-        # these just confirm our test assumptions
-        self.assertTrue(isinstance(values["floating"][0], np.float16))
-        self.assertTrue(isinstance(values["floating"][1], np.float32))
-        self.assertTrue(isinstance(values["floating"][2], np.float64))
-        self.assertTrue(isinstance(values["integer"][0], np.int8))
-        self.assertTrue(isinstance(values["integer"][1], np.int16))
-        self.assertTrue(isinstance(values["integer"][2], np.int32))
-        self.assertTrue(isinstance(values["integer"][3], np.int64))
-        self.assertTrue(isinstance(values["integer"][4], np.uint8))
-        self.assertTrue(isinstance(values["integer"][5], np.uint16))
-        self.assertTrue(isinstance(values["integer"][6], np.uint32))
-        self.assertTrue(isinstance(values["integer"][7], np.uint64))
-        # the actual test!
-        self.assertEqual(
-            jsonify_strict(values),
-            '{"floating": [100.0, 101.0, 102.0], "integer": [0, 1, 2, 3, 4, 5, 6, 7]}',
-        )

backend/test/test_common/unit/utils/test_type_conversion_utils.py

@@ -1,324 +0,0 @@
-import unittest
-import logging
-from parameterized import parameterized_class
-
-import numpy as np
-import pandas as pd
-from pandas import Series, DataFrame
-from scipy import sparse
-
-from backend.common.utils.type_conversion_utils import (
-    get_encoding_dtype_of_array,
-    get_schema_type_hint_of_array,
-    get_dtypes_and_schemas_of_dataframe,
-    get_dtype_and_schema_of_array,
-    get_schema_type_hint_from_dtype,
-)
-
-
-class TestTypeConversionUtils(unittest.TestCase):
-    def test__get_dtypes_and_schemas_of_dataframe__dtype_and_schema_returns_as_expected(self):
-        float_array = Series(data=[1, 2, 3], dtype=np.dtype(np.float64))
-        category_array = Series(data=["a", "b", "b"], dtype="category")
-        dataframe = DataFrame({"float_array": float_array, "category_array": category_array})
-
-        expected_data_types_dict = {"float_array": np.float32, "category_array": str}
-        expected_schema_type_hints_dict = {
-            "float_array": {"type": "float32"},
-            "category_array": {"type": "categorical", "categories": ["a", "b"]},
-        }
-
-        actual_dataframe_data_types, actual_dataframe_schema_type_hints = get_dtypes_and_schemas_of_dataframe(dataframe)
-
-        self.assertEqual(expected_data_types_dict, actual_dataframe_data_types)
-        self.assertEqual(expected_schema_type_hints_dict, actual_dataframe_schema_type_hints)
-
-    def test__get_schema_type_hint_from_dtype(self):
-        self.assertEqual(get_schema_type_hint_from_dtype(np.dtype(np.bool_)), {"type": "boolean"})
-
-        for dtype in [np.int8, np.int8, np.int16, np.uint16, np.int32]:
-            self.assertEqual(get_schema_type_hint_from_dtype(np.dtype(dtype)), {"type": "int32"})
-        for dtype in [np.uint32, np.int64, np.uint64]:
-            with self.assertRaises(TypeError):
-                get_schema_type_hint_from_dtype(np.dtype(dtype))
-
-        for dtype in [np.float16, np.float32, np.float64]:
-            self.assertEqual(get_schema_type_hint_from_dtype(np.dtype(dtype)), {"type": "float32"})
-
-        for dtype in [np.dtype(object), np.dtype(str)]:
-            self.assertEqual(get_schema_type_hint_from_dtype(dtype), {"type": "string"})
-
-
-# Credit: https://stackoverflow.com/questions/35871815/python-3-unit-testing-assert-logger-not-called/64774103#64774103
-class AssertNoLog:
-    def assertNoLogs(self, logger, level):
-        """functions as a context manager.  To be introduced in python 3.10"""
-
-        class AssertNoLogsContext(unittest.TestCase):
-            def __init__(self, logger, level):
-                self.logger = logger
-                self.level = level
-                self.context = self.assertLogs(logger, level)
-
-            def __enter__(self):
-                """enter self.assertLogs as context manager, and log something"""
-                self.initial_logmsg = "sole message"
-                self.cm = self.context.__enter__()
-                self.logger.log(self.level, self.initial_logmsg)
-                return self.cm
-
-            def __exit__(self, exc_type, exc_val, exc_tb):
-                """cleanup logs, and then check nothing extra was logged"""
-                # assertLogs.__exit__ should never fail because of initial msg
-                self.context.__exit__(exc_type, exc_val, exc_tb)
-                if len(self.cm.output) > 1:
-                    """override any exception passed to __exit__"""
-                    self.context._raiseFailure(
-                        "logs of level {} or higher triggered on {} : {}".format(
-                            logging.getLevelName(self.level), self.logger.name, self.cm.output[1:]
-                        )
-                    )
-
-        return AssertNoLogsContext(logger, level)
-
-
-"""
-See table of expected cases in type_conversion_utils.py.
-
-This probes all edge cases. Each case is a dict containing keys:
-    - data - the array to be introspected
-    - throws - if not None, the expected Error (eg, TypeError)
-    - expected_encoding_dtype - upon success
-    - expected_schema_hint - upon success
-    - logs - if not None, specify expected log output
-"""
-
-bool_OK_cases = [
-    {
-        "data": data,
-        "expected_encoding_dtype": np.uint8,
-        "expected_schema_hint": {"type": "boolean"},
-    }
-    for data in [
-        np.array([0, 1, 0, 1], dtype=np.bool_),
-        pd.Series(np.array([0, 1, 0, 1], dtype=np.bool_)),
-        # pd.Index with bools doesn't really make any sense...and becomes dtype=object
-    ]
-]
-
-int_OK_cases = [
-    {
-        "data": data,
-        "expected_encoding_dtype": np.int32,
-        "expected_schema_hint": {"type": "int32"},
-    }
-    for dtype in [np.int8, np.uint8, np.int16, np.uint16, np.int32, np.uint32, np.int64, np.uint64]
-    for data in [
-        np.arange(0, 1000, dtype=dtype),
-        pd.Series(np.arange(0, 1000, dtype=dtype)),
-        pd.Index(np.arange(0, 1000, dtype=dtype)),
-        sparse.csr_matrix((10, 100), dtype=dtype),
-    ]
-]
-
-float_OK_cases = [
-    {
-        "data": data,
-        "expected_encoding_dtype": np.float32,
-        "expected_schema_hint": {"type": "float32"},
-        "logs": None if data.dtype != np.float64 else {"level": logging.WARNING, "output": "may lose precision"},
-    }
-    for dtype in [np.float16, np.float32, np.float64]
-    for data in [
-        np.arange(-128, 1000, dtype=dtype),
-        pd.Series(np.arange(-128, 1000, dtype=dtype)),
-        pd.Index(np.arange(-129, 1000, dtype=dtype)),
-        np.array([-np.nan, np.NINF, -1, np.NZERO, 0, np.PZERO, 1, np.PINF, np.nan], dtype=dtype),
-        np.array([np.finfo(dtype).min, 0, np.finfo(dtype).max], dtype=dtype),
-        sparse.csr_matrix((10, 100), dtype=dtype),
-    ]
-]
-
-
-numeric_ERR_cases = [
-    {
-        "data": data,
-        "throws": TypeError,
-    }
-    for data in [
-        np.array([np.iinfo(np.int64).min, np.iinfo(np.int64).max], dtype=np.int64),
-        np.array([np.iinfo(np.uint64).min, np.iinfo(np.uint64).max], dtype=np.uint64),
-        np.array([np.iinfo(np.uint32).min, np.iinfo(np.uint32).max], dtype=np.uint32),
-    ]
-]
-
-
-string_OK_cases = [
-    {
-        "data": data,
-        "expected_encoding_dtype": np.dtype(str),
-        "expected_schema_hint": {"type": "string"},
-    }
-    for data in [
-        np.array(["a", "b", "c"]),
-        np.array(["a", "b", "c"], dtype="object"),
-        pd.Series(["a", "b", "c"]),
-        pd.Index(["a", "b", "c"]),
-        np.array(["a", [], {}, None, True, False, 383.2], dtype="object"),
-    ]
-]
-
-category_nonnumeric_OK_cases = [
-    {
-        "data": data,
-        "expected_encoding_dtype": np.dtype(str),
-        "expected_schema_hint": {"type": "categorical", "categories": data.dtype.categories.to_list()},
-    }
-    for data in [
-        pd.Series(["a", "b", "c"], dtype="category"),
-        pd.Series(["a", "b", "c", 0, 1, 2], dtype="category"),
-        pd.Series(["a", "b", "c"], dtype="category").cat.remove_categories(["b"]),
-        pd.Series(["a", "b", "c", 0, 1, 2], dtype="category").cat.remove_categories(["b", 0]),
-    ]
-]
-
-category_numeric_OK_cases = [
-    # numeric, no NA/NaN, int
-    *[
-        {
-            "data": data,
-            "expected_encoding_dtype": np.int32,
-            "expected_schema_hint": {"type": "categorical"},
-        }
-        for dtype in [np.int8, np.uint8, np.int16, np.uint16, np.int32, np.uint32, np.int64, np.uint64]
-        for data in [
-            pd.Series(np.array([0, 1, 2], dtype=dtype), dtype="category"),
-        ]
-    ],
-    # numeric, no NA/NaN, float
-    *[
-        {
-            "data": data,
-            "expected_encoding_dtype": np.float32,
-            "expected_schema_hint": {"type": "categorical"},
-            "logs": {"level": logging.WARNING, "output": "may lose precision"},
-        }
-        for dtype in [np.float16, np.float32, np.float64]
-        for data in [
-            pd.Series(np.array([0, 1, 2], dtype=dtype), dtype="category"),
-            pd.Series(np.array([0, 1, 2], dtype=dtype), dtype="category").cat.remove_categories([1]),
-            pd.Categorical(np.array([0, 1, 2], dtype=dtype)),
-        ]
-    ],
-    # numeric, has NA-induced cast to float32
-    *[
-        {
-            "data": data,
-            "expected_encoding_dtype": np.float32,
-            "expected_schema_hint": {"type": "categorical"},
-            "logs": {"level": logging.WARNING, "output": "may lose precision"},
-        }
-        for dtype in [
-            np.int8,
-            np.uint8,
-            np.int16,
-            np.uint16,
-            np.int32,
-            np.uint32,
-            np.int64,
-            np.uint64,
-            np.float16,
-            np.float32,
-            np.float64,
-        ]
-        for data in [
-            pd.Series(np.array([0, 1, 2], dtype=dtype), dtype="category").cat.remove_categories([1]),
-            pd.Categorical(np.array([0, 1, 2], dtype=dtype), categories=np.array([0, 1], dtype=dtype)),
-        ]
-    ],
-]
-
-category_ERR_cases = [
-    # catch expected categorical exceptions for Int64(etc) that have large values
-    {
-        "data": data,
-        "throws": TypeError,
-    }
-    for data in [
-        pd.Categorical(np.array([np.iinfo(np.int64).min, np.iinfo(np.int64).max], dtype=np.int64)),
-        pd.Categorical(np.array([np.iinfo(np.uint64).min, np.iinfo(np.uint64).max], dtype=np.uint64)),
-        pd.Categorical(np.array([np.iinfo(np.uint32).min, np.iinfo(np.uint32).max], dtype=np.uint32)),
-    ]
-]
-
-object_OK_cases = [
-    {
-        "data": data,
-        "expected_encoding_dtype": np.dtype(str),
-        "expected_schema_hint": {"type": "string"},
-    }
-    for data in [
-        np.array(["a", True, 1, [], {}], dtype="object"),
-        pd.Series(["a", True, 1, [], {}], dtype="object"),
-        pd.Index(["a", True, 1, [], {}], dtype="object"),
-    ]
-]
-
-err_cases = [
-    {"data": np.array, "throws": TypeError}
-    for data in [
-        np.ones((10,), dtype=np.complex64),
-        np.ones((10,), dtype=np.complex128),
-        np.array([b"foobar"], dtype=np.bytes_),
-        np.ones((10,), dtype=np.void),
-        np.arange("2005-02", "2005-03", dtype="datetime64[D]"),
-        np.arange("2005-02", "2005-03", dtype="datetime64[D]") - np.datetime64("2008-01-01"),
-        [],
-        {},
-    ]
-]
-
-test_cases = [
-    *bool_OK_cases,
-    *int_OK_cases,
-    *float_OK_cases,
-    *numeric_ERR_cases,
-    *string_OK_cases,
-    *category_nonnumeric_OK_cases,
-    *category_numeric_OK_cases,
-    *category_ERR_cases,
-    *object_OK_cases,
-    *err_cases,
-]
-
-
-@parameterized_class(test_cases)
-class TestTypeInference(unittest.TestCase, AssertNoLog):
-    def test_type_inference(self):
-        throws = getattr(self, "throws", None)
-        if throws:
-            with self.assertRaises(throws):
-                get_dtype_and_schema_of_array(self.data)
-            with self.assertRaises(throws):
-                get_encoding_dtype_of_array(self.data)
-            with self.assertRaises(throws):
-                get_schema_type_hint_of_array(self.data)
-
-        else:
-            logs = getattr(self, "logs", None)
-            if logs is not None:
-                with self.assertLogs(level=logs["level"]) as logger:
-                    encoding_dtype, schema_hint = get_dtype_and_schema_of_array(self.data)
-                    self.assertEqual(encoding_dtype, self.expected_encoding_dtype)
-                    self.assertEqual(schema_hint, self.expected_schema_hint)
-                    self.assertIn(logs["output"], logger.output[0])
-
-            else:
-                with self.assertNoLogs(logging.getLogger(), logging.WARNING):
-                    encoding_dtype, schema_hint = get_dtype_and_schema_of_array(self.data)
-                    self.assertEqual(encoding_dtype, self.expected_encoding_dtype)
-                    self.assertEqual(schema_hint, self.expected_schema_hint)
-
-            # also test the other public API
-            self.assertEqual(get_encoding_dtype_of_array(self.data), self.expected_encoding_dtype)
-            self.assertEqual(get_schema_type_hint_of_array(self.data), self.expected_schema_hint)

backend/test/test_czi_hosted/locust/locustfile.py

@@ -1,165 +0,0 @@
-import json
-import random
-
-import requests
-from config import DataSets
-from locust import HttpUser, SequentialTaskSet, task, between, TaskSet
-from locust.clients import HttpSession
-from requests.packages.urllib3.exceptions import InsecureRequestWarning
-
-import backend.test.decode_fbs as decode_fbs
-
-requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
-
-"""
-Simple locust stress test definition for cellxgene
-"""
-
-API_SUFFIX = "api/v0.2"
-
-
-class CellXGeneTasks(TaskSet):
-    """
-    Simulate use against a single dataset
-    """
-
-    def on_start(self):
-
-        self.client.verify = False
-        self.dataset = random.choice(DataSets)
-
-        with self.client.get(
-            f"{self.dataset}/{API_SUFFIX}/schema", stream=True, catch_response=True
-        ) as schema_response:
-            if schema_response.status_code == 200:
-                self.schema = schema_response.json()["schema"]
-            else:
-                self.schema = None
-
-        with self.client.get(
-            f"{self.dataset}/{API_SUFFIX}/config", stream=True, catch_response=True
-        ) as config_response:
-            if config_response.status_code == 200:
-                self.config = config_response.json()["config"]
-            else:
-                self.config = None
-
-        with self.client.get(
-            f"{self.dataset}/{API_SUFFIX}/annotations/var?annotation-name={self.var_index_name()}",
-            headers={"Accept": "application/octet-stream"},
-            catch_response=True,
-        ) as var_index_response:
-            if var_index_response.status_code == 200:
-                df = decode_fbs.decode_matrix_FBS(var_index_response.content)
-                gene_names_idx = df["col_idx"].index(self.var_index_name())
-                self.gene_names = df["columns"][gene_names_idx]
-            else:
-                self.gene_names = []
-
-    def var_index_name(self):
-        if self.schema is not None:
-            return self.schema["annotations"]["var"]["index"]
-        return None
-
-    def obs_annotation_names(self):
-        if self.schema is not None:
-            return [col["name"] for col in self.schema["annotations"]["obs"]["columns"]]
-        return []
-
-    def layout_names(self):
-        if self.schema is not None:
-            return [layout["name"] for layout in self.schema["layout"]["obs"]]
-        else:
-            return []
-
-    @task(2)
-    class InitializeClient(SequentialTaskSet):
-        """
-        Initial loading of cellxgene - when the user hits the main route.
-
-        Currently this sequence skips some of the static assets, which are quite small and should be served by the
-        HTTP server directly.
-
-        1. Load index.html, etc.
-        2. Concurrently load /config, /schema
-        3. Concurrently load /layout/obs, /annotations/var?annotation-name=<the index>
-        -- Does initial render --
-        4. Concurrently load all /annotations/obs and all /layouts/obs
-        -- Fully initialized --
-        """
-
-        # Users hit all of the init routes as fast as they can, subject to the ordering constraints and network latency.
-        wait_time = between(0.01, 0.1)
-
-        def on_start(self):
-            self.dataset = self.parent.dataset
-            self.client.verify = False
-            self.api_less_client = HttpSession(
-                base_url=self.client.base_url.replace("api.", "").replace("cellxgene/", ""),
-                request_success=self.client.request_success,
-                request_failure=self.client.request_failure,
-            )
-
-        @task
-        def index(self):
-            self.api_less_client.get(f"{self.dataset}", stream=True)
-
-        @task
-        def loadConfigAndSchema(self):
-            self.client.get(f"{self.dataset}/{API_SUFFIX}/schema", stream=True, catch_response=True)
-            self.client.get(f"{self.dataset}/{API_SUFFIX}/config", stream=True, catch_response=True)
-
-        @task
-        def loadBootstrapData(self):
-            self.client.get(
-                f"{self.dataset}/{API_SUFFIX}/layout/obs", headers={"Accept": "application/octet-stream"}, stream=True
-            )
-            self.client.get(
-                f"{self.dataset}/{API_SUFFIX}/annotations/var?annotation-name={self.parent.var_index_name()}",
-                headers={"Accept": "application/octet-stream"},
-                catch_response=True,
-            )
-
-        @task
-        def loadObsAnnotationsAndLayouts(self):
-            obs_names = self.parent.obs_annotation_names()
-            for name in obs_names:
-                self.client.get(
-                    f"{self.dataset}/{API_SUFFIX}/annotations/obs?annotation-name={name}",
-                    headers={"Accept": "application/octet-stream"},
-                    stream=True,
-                )
-
-            layouts = self.parent.layout_names()
-            for name in layouts:
-                self.client.get(
-                    f"{self.dataset}/{API_SUFFIX}/annotations/obs?layout-name={name}",
-                    headers={"Accept": "application/octet-stream"},
-                    stream=True,
-                )
-
-        @task
-        def done(self):
-            self.interrupt()
-
-    @task(1)
-    def load_expression(self):
-        """
-        Simulate user occasionally loading some expression data for a gene
-        """
-
-        gene_name = random.choice(self.gene_names)
-        filter = {"filter": {"var": {"annotation_value": [{"name": self.var_index_name(), "values": [gene_name]}]}}}
-        self.client.put(
-            f"{self.dataset}/{API_SUFFIX}/data/var",
-            data=json.dumps(filter),
-            headers={"Content-Type": "application/json", "Accept": "application/octet-stream"},
-            stream=True,
-        ).close()
-
-
-class CellxgeneUser(HttpUser):
-    tasks = [CellXGeneTasks]
-
-    # Most ops do not require back-end interaction, so slow cadence for users
-    wait_time = between(10, 60)

backend/test/test_czi_hosted/locust/requirements-locust.txt

@@ -1,2 +0,0 @@
-locust
--r ../../../czi_hosted/requirements.txt

backend/test/test_czi_hosted/unit/__init__.py

@@ -1,180 +0,0 @@
-import logging
-import shutil
-import tempfile
-import unittest
-
-from os import path
-
-import pandas as pd
-from flask_compress import Compress
-from flask_cors import CORS
-
-from backend.czi_hosted.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
-from backend.czi_hosted.common.annotations.local_file_csv import AnnotationsLocalFile
-from backend.czi_hosted.common.config.app_config import AppConfig
-from backend.common.utils.data_locator import DataLocator
-from backend.common.fbs.matrix import encode_matrix_fbs
-from backend.czi_hosted.data_common.matrix_loader import MatrixDataType, MatrixDataLoader
-from backend.czi_hosted.db.db_utils import DbUtils
-from backend.czi_hosted.app.app import Server
-from backend.test import PROJECT_ROOT, FIXTURES_ROOT
-
-
-def data_with_tmp_tiledb_annotations(ext: MatrixDataType):
-    tmp_dir = tempfile.mkdtemp()
-    fname = {
-        MatrixDataType.H5AD: f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
-        MatrixDataType.CXG: "test/fixtures/pbmc3k.cxg",
-    }[ext]
-    data_locator = DataLocator(fname)
-    config = AppConfig()
-    config.update_server_config(
-        app__flask_secret_key="secret",
-        multi_dataset__dataroot=data_locator.path,
-        authentication__type="test",
-        authentication__insecure_test_environment=True,
-    )
-    config.update_default_dataset_config(
-        embeddings__names=["umap"],
-        presentation__max_categories=100,
-        diffexp__lfc_cutoff=0.01,
-        user_annotations__type="hosted_tiledb_array",
-        user_annotations__hosted_tiledb_array__db_uri="postgresql://postgres:test_pw@localhost:5432",
-        user_annotations__hosted_tiledb_array__hosted_file_directory=tmp_dir,
-    )
-
-    config.complete_config()
-
-    data = MatrixDataLoader(data_locator.abspath()).open(config)
-    annotations = AnnotationsHostedTileDB(
-        {
-            "user-annotations": True,
-            "genesets-save": False,
-        },
-        tmp_dir,
-        DbUtils("postgresql://postgres:test_pw@localhost:5432"),
-    )
-    return data, tmp_dir, annotations
-
-
-def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False):
-    tmp_dir = tempfile.mkdtemp()
-    annotations_file = path.join(tmp_dir, "test_annotations.csv")
-    if annotations_fixture:
-        shutil.copyfile(f"{FIXTURES_ROOT}/pbmc3k-annotations.csv", annotations_file)
-    fname = {
-        MatrixDataType.H5AD: f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
-        MatrixDataType.CXG: f"{FIXTURES_ROOT}/pbmc3k.cxg",
-    }[ext]
-    data_locator = DataLocator(fname)
-    config = AppConfig()
-    config.update_server_config(
-        app__flask_secret_key="secret",
-        single_dataset__obs_names=None,
-        single_dataset__var_names=None,
-        single_dataset__datapath=data_locator.path,
-    )
-    config.update_default_dataset_config(
-        embeddings__names=["umap"],
-        presentation__max_categories=100,
-        diffexp__lfc_cutoff=0.01,
-    )
-
-    config.complete_config()
-    data = MatrixDataLoader(data_locator.abspath()).open(config)
-    annotations = AnnotationsLocalFile(
-        {
-            "user-annotations": True,
-            "genesets-save": False,
-        },
-        None,
-        annotations_file,
-    )
-    return data, tmp_dir, annotations, config
-
-
-def make_fbs(data):
-    df = pd.DataFrame(data)
-    return encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
-
-
-def skip_if(condition, reason: str):
-    def decorator(f):
-        def wraps(self, *args, **kwargs):
-            if condition(self):
-                self.skipTest(reason)
-            else:
-                f(self, *args, **kwargs)
-
-        return wraps
-
-    return decorator
-
-
-def app_config(data_locator, backed=False, extra_server_config={}, extra_dataset_config={}):
-    config = AppConfig()
-    config.update_server_config(
-        app__flask_secret_key="secret",
-        single_dataset__obs_names=None,
-        single_dataset__var_names=None,
-        adaptor__anndata_adaptor__backed=backed,
-        single_dataset__datapath=data_locator,
-        limits__diffexp_cellcount_max=None,
-        limits__column_request_max=None,
-    )
-    config.update_default_dataset_config(
-        embeddings__names=["umap", "tsne", "pca"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01
-    )
-    config.update_server_config(**extra_server_config)
-    config.update_default_dataset_config(**extra_dataset_config)
-    config.complete_config()
-    return config
-
-
-class TestServer(Server):
-    def __init__(self, app_config):
-        super().__init__(app_config)
-
-    @staticmethod
-    def _before_adding_routes(app, app_config):
-        app.config["COMPRESS_MIMETYPES"] = [
-            "text/html",
-            "text/css",
-            "text/xml",
-            "application/json",
-            "application/javascript",
-            "application/octet-stream",
-        ]
-        Compress(app)
-        if app_config.server_config.app__debug:
-            CORS(app, supports_credentials=True)
-
-
-class BaseTest(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls, app_config=None):
-        cls.TEST_URL_BASE = "/d/pbmc3k.cxg/api/v0.2/"
-        cls.maxDiff = None
-        cls.app = cls.create_app(app_config)
-
-    @classmethod
-    def create_app(cls, app_config=None):
-        if not app_config:
-            app_config = AppConfig()
-            app_config.update_server_config(
-                authentication__type="test",
-                authentication__insecure_test_environment=True,
-                app__flask_secret_key="testing",
-                app__debug=True,
-                multi_dataset__dataroot=f"{FIXTURES_ROOT}",
-                multi_dataset__index=True,
-                multi_dataset__allowed_matrix_types=["cxg"]
-            )
-        app_config.complete_config(logging.info)
-
-        app = TestServer(app_config).app
-
-        app.testing = True
-        app.debug = True
-
-        return app

backend/test/test_czi_hosted/unit/auth/test_auth.py

@@ -1,168 +0,0 @@
-import json
-import unittest
-
-from backend.czi_hosted.common.config.app_config import AppConfig
-from backend.test import FIXTURES_ROOT
-from backend.test.test_czi_hosted.unit import BaseTest
-
-
-class AuthTest(BaseTest):
-    def setUp(self):
-        self.dataset_dataroot = FIXTURES_ROOT
-
-    def test_auth_none(self):
-        app_config = AppConfig()
-        app_config.update_server_config(app__flask_secret_key="secret")
-        app_config.update_server_config(authentication__type=None, multi_dataset__dataroot=self.dataset_dataroot)
-        app_config.update_default_dataset_config(user_annotations__enable=False)
-
-        app_config.complete_config()
-        server= self.create_app(app_config)
-        server.testing = True
-        session = server.test_client()
-        config = json.loads(session.get(f"{self.TEST_URL_BASE}config").data)
-        userinfo = json.loads(session.get(f"{self.TEST_URL_BASE}userinfo").data)
-        self.assertNotIn("authentication", config["config"])
-        self.assertIsNone(userinfo)
-
-    def test_auth_session(self):
-        app_config = AppConfig()
-        app_config.update_server_config(app__flask_secret_key="secret")
-        app_config.update_server_config(authentication__type="session", multi_dataset__dataroot=self.dataset_dataroot)
-        app_config.update_default_dataset_config(user_annotations__enable=True)
-        app_config.complete_config()
-
-        server = self.create_app(app_config)
-        server.auth.is_user_authenticated = lambda: True
-        server.testing = True
-        session = server.test_client()
-        config = json.loads(session.get(f"{self.TEST_URL_BASE}config").data)
-        userinfo = json.loads(session.get(f"{self.TEST_URL_BASE}userinfo").data)
-
-        self.assertFalse(config["config"]["authentication"]["requires_client_login"])
-        self.assertTrue(userinfo["userinfo"]["is_authenticated"])
-        self.assertEqual(userinfo["userinfo"]["username"], "anonymous")
-
-    def test_auth_test(self):
-        app_config = AppConfig()
-        app_config.update_server_config(app__flask_secret_key="secret")
-        app_config.update_server_config(authentication__type="test")
-        app_config.update_server_config(authentication__insecure_test_environment=True)
-        app_config.update_server_config(
-            multi_dataset__dataroot=dict(
-                a1=dict(dataroot=self.dataset_dataroot, base_url="auth"),
-                a2=dict(dataroot=self.dataset_dataroot, base_url="no-auth"),
-            )
-        )
-
-        # specialize the configs
-        app_config.add_dataroot_config("a1", app__authentication_enable=True, user_annotations__enable=True)
-        app_config.add_dataroot_config("a2", app__authentication_enable=False, user_annotations__enable=False)
-
-        app_config.complete_config()
-
-        server=self.create_app(app_config)
-        server.testing = True
-        session = server.test_client()
-
-        # auth datasets
-        config = json.loads(session.get("/auth/pbmc3k.cxg/api/v0.2/config").data)
-        userinfo = json.loads(session.get(f"/auth/pbmc3k.cxg/api/v0.2/userinfo").data)
-
-        self.assertFalse(userinfo["userinfo"]["is_authenticated"])
-        self.assertIsNone(userinfo["userinfo"]["username"])
-        self.assertTrue(config["config"]["authentication"]["requires_client_login"])
-        self.assertTrue(config["config"]["parameters"]["annotations"])
-
-        login_uri = config["config"]["authentication"]["login"]
-        logout_uri = config["config"]["authentication"]["logout"]
-
-        self.assertEqual(login_uri, "/login?dataset=auth/pbmc3k.cxg")
-        self.assertEqual(logout_uri, "/logout?dataset=auth/pbmc3k.cxg")
-
-        response = session.get(login_uri)
-        # check that the login redirect worked
-
-        self.assertEqual(response.status_code, 302)
-        self.assertEqual(response.headers['Location'], 'http://localhost/auth/pbmc3k.cxg')
-        config = json.loads(session.get("/auth/pbmc3k.cxg/api/v0.2/config").data)
-        userinfo = json.loads(session.get("/auth/pbmc3k.cxg/api/v0.2/userinfo").data)
-
-        self.assertTrue(userinfo["userinfo"]["is_authenticated"])
-        self.assertEqual(userinfo["userinfo"]["username"], "test_account")
-        self.assertEqual(userinfo["userinfo"]["picture"], None)
-        self.assertTrue(config["config"]["parameters"]["annotations"])
-
-        response = session.get(logout_uri)
-        # check that the logout redirect worked
-
-        self.assertEqual(response.status_code, 302)
-        config = json.loads(session.get("/auth/pbmc3k.cxg/api/v0.2/config").data)
-        userinfo = json.loads(session.get("/auth/pbmc3k.cxg/api/v0.2/userinfo").data)
-        self.assertFalse(userinfo["userinfo"]["is_authenticated"])
-        self.assertIsNone(userinfo["userinfo"]["username"])
-        self.assertTrue(config["config"]["parameters"]["annotations"])
-
-        # no-auth datasets
-        config = json.loads(session.get("/no-auth/pbmc3k.cxg/api/v0.2/config").data)
-        userinfo = json.loads(session.get("/no-auth/pbmc3k.cxg/api/v0.2/userinfo").data)
-        self.assertIsNone(userinfo)
-        self.assertFalse(config["config"]["parameters"]["annotations"])
-
-        # login with a picture
-        session.get(f"{login_uri}&picture=myimage.png")
-        userinfo = json.loads(session.get("/auth/pbmc3k.cxg/api/v0.2/userinfo").data)
-        self.assertTrue(userinfo["userinfo"]["is_authenticated"])
-        self.assertEqual(userinfo["userinfo"]["picture"], "myimage.png")
-
-    def test_auth_test_single(self):
-        app_config = AppConfig()
-        app_config.update_server_config(app__flask_secret_key="secret")
-        app_config.update_server_config(
-            authentication__type="test", single_dataset__datapath=f"{self.dataset_dataroot}/pbmc3k.cxg"
-        )
-        app_config.update_server_config(authentication__insecure_test_environment=True)
-
-        app_config.complete_config()
-
-        server = self.create_app(app_config)
-        server.testing = True
-        session = server.test_client()
-
-        config = json.loads(session.get("/api/v0.2/config").data)
-        userinfo = json.loads(session.get("/api/v0.2/userinfo").data)
-        self.assertFalse(userinfo["userinfo"]["is_authenticated"])
-        self.assertIsNone(userinfo["userinfo"]["username"])
-        self.assertTrue(config["config"]["authentication"]["requires_client_login"])
-        self.assertTrue(config["config"]["parameters"]["annotations"])
-
-        login_uri = config["config"]["authentication"]["login"]
-        logout_uri = config["config"]["authentication"]["logout"]
-
-        self.assertEqual(login_uri, "/login")
-        self.assertEqual(logout_uri, "/logout")
-
-
-        # check that the login redirect worked
-        with server.test_client() as session:
-            response = session.get(login_uri)
-            self.assertEqual(response.status_code, 302)
-            self.assertEqual(response.headers['Location'], "http://localhost/")
-
-        config = json.loads(session.get("api/v0.2/config").data)
-        userinfo = json.loads(session.get("/api/v0.2/userinfo").data)
-        self.assertTrue(userinfo["userinfo"]["is_authenticated"])
-        self.assertEqual(userinfo["userinfo"]["username"], "test_account")
-        self.assertTrue(config["config"]["parameters"]["annotations"])
-
-        response = session.get(logout_uri)
-        # check that the logout redirect worked
-
-        self.assertEqual(response.status_code, 302)
-        self.assertEqual(response.headers['Location'], "http://localhost/")
-        config = json.loads(session.get("/api/v0.2/config").data)
-
-        userinfo = json.loads(session.get("/api/v0.2/userinfo").data)
-        self.assertFalse(userinfo["userinfo"]["is_authenticated"])
-        self.assertIsNone(userinfo["userinfo"]["username"])
-        self.assertTrue(config["config"]["parameters"]["annotations"])

backend/test/test_czi_hosted/unit/cli/test_launch.py

@@ -1,27 +0,0 @@
-import filecmp
-import os
-import shutil
-import unittest
-
-import yaml
-
-from backend.czi_hosted.default_config import default_config
-from backend.test import FIXTURES_ROOT
-
-
-class CLIPLaunchTests(unittest.TestCase):
-    tmp_dir = os.path.join(FIXTURES_ROOT, "dump_configs")
-
-    @classmethod
-    def setUpClass(cls) -> None:
-        os.mkdir(cls.tmp_dir)
-
-    @classmethod
-    def tearDownClass(cls) -> None:
-        shutil.rmtree(cls.tmp_dir)
-
-    def test_dump_default_config(self):
-        os.system(f"cellxgene launch --dump-default-config > {self.tmp_dir}/test_config_dump.txt")
-        with open(f"{self.tmp_dir}/expected_config_dump.txt", "w") as expected_config:
-            expected_config.write(yaml.dump(default_config))
-        filecmp.cmp(f"{self.tmp_dir}/expected_config_dump.txt", f"{self.tmp_dir}/test_config_dump.txt")

backend/test/test_czi_hosted/unit/cli/test_prepare.py

@@ -1,15 +0,0 @@
-import unittest
-
-import pandas as pd
-
-from backend.czi_hosted.cli.prepare import make_index_unique
-
-
-class CLIPrepareTests(unittest.TestCase):
-    """ Test cases for CLI prepare logic """
-
-    def test_make_index_unique(self):
-        index = pd.Index(["SNORD113", "SNORD113", "SNORD113-1"])
-        result = make_index_unique(index)
-        expected = pd.Index(["SNORD113", "SNORD113-2", "SNORD113-1"])
-        self.assertTrue(all(left == right for left, right in zip(result.values, expected.values)))

backend/test/test_czi_hosted/unit/cli/test_upgrade.py

@@ -1,28 +0,0 @@
-import unittest
-
-from backend.czi_hosted.cli.upgrade import validate_version_str, split_version, version_gt
-
-
-class CLIUpgradeTests(unittest.TestCase):
-    """ Test cases for CLI logic """
-
-    def test_validate_version_str(self):
-        self.assertTrue(validate_version_str("0.1.2"))
-        self.assertTrue(validate_version_str("0.1.2-RC", release_only=False))
-        self.assertFalse(validate_version_str("0.1"))
-        self.assertFalse(validate_version_str("0.1.2.3"))
-        self.assertFalse(validate_version_str("0.1.2-RC"))
-
-    def test_split_version_str(self):
-        self.assertEqual(split_version("0.1.2"), [0, 1, 2])
-        with self.assertRaises(AttributeError):
-            split_version("0.1")
-
-    def test_assert_verstion_gt(self):
-        self.assertTrue(version_gt("1.0.0", "0.1.1"))
-        self.assertTrue(version_gt("0.1.0", "0.0.1"))
-        self.assertTrue(version_gt("0.0.1", "0.0.0"))
-        self.assertFalse(version_gt("0.0.0", "0.0.0"))
-        self.assertFalse(version_gt("0.0.0", "0.0.1"))
-        self.assertFalse(version_gt("0.0.1", "0.1.0"))
-        self.assertFalse(version_gt("0.1.1", "1.0.0"))

backend/test/test_czi_hosted/unit/common/test_api.py

@@ -1,505 +0,0 @@
-import json
-import os
-import time
-from http import HTTPStatus
-import hashlib
-
-import requests
-
-
-from backend.czi_hosted.common.config.app_config import AppConfig
-from backend.test import decode_fbs
-from backend.test.fixtures.fixtures import pbmc3k_colors
-from backend.test.test_czi_hosted.unit import BaseTest, skip_if
-
-BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
-
-class EndPoints(BaseTest):
-    @classmethod
-    def setUpClass(cls, app_config=None):
-        super().setUpClass(app_config)
-        cls.app.testing = True
-        cls.client = cls.app.test_client()
-        os.environ["SKIP_STATIC"] = "True"
-        for i in range(90):
-            try:
-                result = cls.client.get(f"{cls.TEST_URL_BASE}schema")
-                cls.schema = json.loads(result.data)
-            except requests.exceptions.ConnectionError:
-                time.sleep(1)
-
-    def test_initialize(self):
-        endpoint = "schema"
-        url = f"{self.TEST_URL_BASE}{endpoint}"
-        result = self.client.get(url)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/json")
-        result_data = json.loads(result.data)
-        self.assertEqual(result_data["schema"]["dataframe"]["nObs"], 2638)
-        self.assertEqual(len(result_data["schema"]["annotations"]["obs"]), 2)
-        self.assertEqual(
-            len(result_data["schema"]["annotations"]["obs"]["columns"]), 5
-        )
-
-    def test_config(self):
-        endpoint = "config"
-        url = f"{self.TEST_URL_BASE}{endpoint}"
-        result = self.client.get(url)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/json")
-        result_data = json.loads(result.data)
-        self.assertIn("library_versions", result_data["config"])
-        self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k")
-
-    def test_get_layout_fbs(self):
-        endpoint = "layout/obs"
-        url = f"{self.TEST_URL_BASE}{endpoint}"
-        header = {"Accept": "application/octet-stream"}
-        result = self.client.get(url, headers=header)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
-        df = decode_fbs.decode_matrix_FBS(result.data)
-        self.assertEqual(df["n_rows"], 2638)
-        self.assertEqual(df["n_cols"], 8)
-        self.assertIsNotNone(df["columns"])
-        self.assertSetEqual(
-            set(df["col_idx"]),
-            {"pca_0", "pca_1", "tsne_0", "tsne_1", "umap_0", "umap_1", "draw_graph_fr_0", "draw_graph_fr_1"},
-        )
-        self.assertIsNone(df["row_idx"])
-        self.assertEqual(len(df["columns"]), df["n_cols"])
-
-    def test_bad_filter(self):
-        endpoint = "data/var"
-        url = f"{self.TEST_URL_BASE}{endpoint}"
-        header = {"Accept": "application/octet-stream"}
-        result = self.client.put(url, headers=header, json=BAD_FILTER)
-        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
-
-    def test_get_annotations_obs_fbs(self):
-        endpoint = "annotations/obs"
-        url = f"{self.TEST_URL_BASE}{endpoint}"
-        header = {"Accept": "application/octet-stream"}
-        result = self.client.get(url, headers=header)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
-        df = decode_fbs.decode_matrix_FBS(result.data)
-        self.assertEqual(df["n_rows"], 2638)
-        self.assertEqual(df["n_cols"], 5)
-        self.assertIsNotNone(df["columns"])
-        self.assertIsNone(df["row_idx"])
-        self.assertEqual(len(df["columns"]), df["n_cols"])
-        obs_index_col_name = self.schema["schema"]["annotations"]["obs"]["index"]
-        self.assertCountEqual(
-            df["col_idx"],
-            [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"],
-        )
-
-    def test_get_annotations_obs_keys_fbs(self):
-        endpoint = "annotations/obs"
-        query = "annotation-name=n_genes&annotation-name=percent_mito"
-        url = f"{self.TEST_URL_BASE}{endpoint}?{query}"
-        header = {"Accept": "application/octet-stream"}
-        result = self.client.get(url, headers=header)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
-        df = decode_fbs.decode_matrix_FBS(result.data)
-        self.assertEqual(df["n_rows"], 2638)
-        self.assertEqual(df["n_cols"], 2)
-        self.assertIsNotNone(df["columns"])
-        self.assertIsNone(df["row_idx"])
-        self.assertEqual(len(df["columns"]), df["n_cols"])
-        self.assertCountEqual(df["col_idx"], ["n_genes", "percent_mito"])
-
-    def test_get_annotations_obs_error(self):
-        endpoint = "annotations/obs"
-        query = "annotation-name=notakey"
-        url = f"{self.TEST_URL_BASE}{endpoint}?{query}"
-        header = {"Accept": "application/octet-stream"}
-        result = self.client.get(url, headers=header)
-        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
-
-# TEMP: Testing count 15 to match hardcoded values for diffexp
-# TODO(#1281): Switch back to dynamic values
-    def test_diff_exp(self):
-        endpoint = "diffexp/obs"
-        url = f"{self.TEST_URL_BASE}{endpoint}"
-        params = {
-            "mode": "topN",
-            "set1": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["NK cells"]}]}}},
-            "set2": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["CD8 T cells"]}]}}},
-            "count": 15,
-        }
-        result = self.client.post(url, json=params)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/json")
-        result_data = json.loads(result.data)
-        self.assertEqual(len(result_data['positive']), 15)
-        self.assertEqual(len(result_data['negative']), 15)
-
-    def test_diff_exp_indices(self):
-        endpoint = "diffexp/obs"
-        url = f"{self.TEST_URL_BASE}{endpoint}"
-        params = {
-            "mode": "topN",
-            "count": 15,
-            "set1": {"filter": {"obs": {"index": [[0, 500]]}}},
-            "set2": {"filter": {"obs": {"index": [[500, 1000]]}}},
-        }
-        result = self.client.post(url, json=params)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/json")
-        result_data = json.loads(result.data)
-        self.assertEqual(len(result_data['positive']), 15)
-        self.assertEqual(len(result_data['negative']), 15)
-
-    def test_get_annotations_var_fbs(self):
-        endpoint = "annotations/var"
-        url = f"{self.TEST_URL_BASE}{endpoint}"
-        header = {"Accept": "application/octet-stream"}
-        result = self.client.get(url, headers=header)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
-        df = decode_fbs.decode_matrix_FBS(result.data)
-        self.assertEqual(df["n_rows"], 1838)
-        self.assertEqual(df["n_cols"], 2)
-        self.assertIsNotNone(df["columns"])
-        self.assertIsNone(df["row_idx"])
-        self.assertEqual(len(df["columns"]), df["n_cols"])
-        var_index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
-        self.assertCountEqual(df["col_idx"], [var_index_col_name, "n_cells"])
-
-    def test_get_annotations_var_keys_fbs(self):
-        endpoint = "annotations/var"
-        query = "annotation-name=n_cells"
-        url = f"{self.TEST_URL_BASE}{endpoint}?{query}"
-        header = {"Accept": "application/octet-stream"}
-        result = self.client.get(url, headers=header)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
-        df = decode_fbs.decode_matrix_FBS(result.data)
-        self.assertEqual(df["n_rows"], 1838)
-        self.assertEqual(df["n_cols"], 1)
-        self.assertIsNotNone(df["columns"])
-        self.assertIsNone(df["row_idx"])
-        self.assertEqual(len(df["columns"]), df["n_cols"])
-        self.assertCountEqual(df["col_idx"], ["n_cells"])
-
-    def test_get_annotations_var_error(self):
-        endpoint = "annotations/var"
-        query = "annotation-name=notakey"
-        url = f"{self.TEST_URL_BASE}{endpoint}?{query}"
-        header = {"Accept": "application/octet-stream"}
-        result = self.client.get(url, headers=header)
-        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
-
-    def test_data_mimetype_error(self):
-        endpoint = "data/var"
-        header = {"Accept": "xxx"}
-        url = f"{self.TEST_URL_BASE}{endpoint}"
-        result = self.client.put(url, headers=header)
-        self.assertEqual(result.status_code, HTTPStatus.NOT_ACCEPTABLE)
-
-    def test_fbs_default(self):
-        endpoint = "data/var"
-        url = f"{self.TEST_URL_BASE}{endpoint}"
-        headers = {"Accept": "application/octet-stream"}
-        result = self.client.put(url, headers=headers)
-        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
-
-        filter = {"filter": {"var": {"index": [0, 1, 4]}}}
-        result = self.client.put(url, headers=headers, json=filter)
-        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
-
-    def test_data_put_fbs(self):
-        endpoint = "data/var"
-        url = f"{self.TEST_URL_BASE}{endpoint}"
-        header = {"Accept": "application/octet-stream"}
-        result = self.client.put(url, headers=header)
-        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
-
-    def test_data_get_fbs(self):
-        endpoint = "data/var"
-        url = f"{self.TEST_URL_BASE}{endpoint}"
-        header = {"Accept": "application/octet-stream"}
-        result = self.client.get(url, headers=header)
-        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
-
-    def test_data_put_filter_fbs(self):
-        endpoint = "data/var"
-        url = f"{self.TEST_URL_BASE}{endpoint}"
-        header = {"Accept": "application/octet-stream"}
-        filter = {"filter": {"var": {"index": [0, 1, 4]}}}
-        result = self.client.put(url, headers=header, json=filter)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
-        df = decode_fbs.decode_matrix_FBS(result.data)
-        self.assertEqual(df["n_rows"], 2638)
-        self.assertEqual(df["n_cols"], 3)
-        self.assertIsNotNone(df["columns"])
-        self.assertIsNone(df["row_idx"])
-        self.assertEqual(len(df["columns"]), df["n_cols"])
-        self.assertListEqual(df["col_idx"].tolist(), [0, 1, 4])
-
-    def test_data_get_filter_fbs(self):
-        index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
-        endpoint = "data/var"
-        query = f"var:{index_col_name}=SIK1"
-        url = f"{self.TEST_URL_BASE}{endpoint}?{query}"
-        header = {"Accept": "application/octet-stream"}
-        result = self.client.get(url, headers=header)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
-        df = decode_fbs.decode_matrix_FBS(result.data)
-        self.assertEqual(df["n_rows"], 2638)
-        self.assertEqual(df["n_cols"], 1)
-
-    def test_data_get_unknown_filter_fbs(self):
-        index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
-        endpoint = "data/var"
-        query = f"var:{index_col_name}=UNKNOWN"
-        url = f"{self.TEST_URL_BASE}{endpoint}?{query}"
-        header = {"Accept": "application/octet-stream"}
-        result = self.client.get(url, headers=header)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
-        df = decode_fbs.decode_matrix_FBS(result.data)
-        self.assertEqual(df["n_rows"], 2638)
-        self.assertEqual(df["n_cols"], 0)
-
-    def test_data_put_single_var(self):
-        endpoint = "data/var"
-        url = f"{self.TEST_URL_BASE}{endpoint}"
-        header = {"Accept": "application/octet-stream"}
-        index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
-        var_filter = {"filter": {"var": {"annotation_value": [{"name": index_col_name, "values": ["RER1"]}]}}}
-        result = self.client.put(url, headers=header, json=var_filter)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
-        df = decode_fbs.decode_matrix_FBS(result.data)
-        self.assertEqual(df["n_rows"], 2638)
-        self.assertEqual(df["n_cols"], 1)
-
-    def test_colors(self):
-        endpoint = "colors"
-        url = f"{self.TEST_URL_BASE}{endpoint}"
-        result = self.client.get(url)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/json")
-        result_data = json.loads(result.data)
-        self.assertEqual(result_data, pbmc3k_colors)
-
-    @skip_if(lambda x: os.getenv("SKIP_STATIC"), "Skip static test when running locally")
-    def test_static(self):
-        endpoint = "static"
-        file = "assets/favicon.ico"
-        url = f"{endpoint}/{file}"
-        result = self.client.get(url)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-
-    def test_genesets_config(self):
-        result = self.client.get(f"{self.TEST_URL_BASE}config")
-        config_data = json.loads(result.data)
-        params = config_data["config"]["parameters"]
-        annotations_genesets = params["annotations_genesets"]
-        annotations_genesets_readonly = params["annotations_genesets_readonly"]
-        annotations_genesets_summary_methods = params["annotations_genesets_summary_methods"]
-        self.assertTrue(annotations_genesets)
-        self.assertTrue(annotations_genesets_readonly)
-        self.assertEqual(annotations_genesets_summary_methods, ["mean"])
-
-    def test_get_genesets(self):
-        endpoint = "genesets"
-        url = f"{self.TEST_URL_BASE}{endpoint}"
-        result = self.client.get(url, headers={"Accept": "application/json"})
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/json")
-        result_data = json.loads(result.data)
-        self.assertIsNotNone(result_data["genesets"])
-
-    def test_get_summaryvar(self):
-        index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
-        endpoint = "summarize/var"
-
-        # single column
-        filter = f"var:{index_col_name}=F5"
-        query = f"method=mean&{filter}"
-        query_hash = hashlib.sha1(query.encode()).hexdigest()
-        url = f"{self.TEST_URL_BASE}{endpoint}?{query}"
-        header = {"Accept": "application/octet-stream"}
-        result = self.client.get(url, headers=header)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
-        df = decode_fbs.decode_matrix_FBS(result.data)
-        self.assertEqual(df["n_rows"], 2638)
-        self.assertEqual(df["n_cols"], 1)
-        self.assertEqual(df["col_idx"], [query_hash])
-        self.assertAlmostEqual(df["columns"][0][0], -0.110451095)
-
-        # multi-column
-        col_names = ["F5", "BEB3", "SIK1"]
-        filter = "&".join([f"var:{index_col_name}={name}" for name in col_names])
-        query = f"method=mean&{filter}"
-        query_hash = hashlib.sha1(query.encode()).hexdigest()
-        url = f"{self.TEST_URL_BASE}{endpoint}?{query}"
-        header = {"Accept": "application/octet-stream"}
-        result = self.client.get(url, headers=header)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
-        df = decode_fbs.decode_matrix_FBS(result.data)
-        self.assertEqual(df["n_rows"], 2638)
-        self.assertEqual(df["n_cols"], 1)
-        self.assertEqual(df["col_idx"], [query_hash])
-        self.assertAlmostEqual(df["columns"][0][0], -0.16628358)
-
-    def test_post_summaryvar(self):
-        index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
-        endpoint = "summarize/var"
-        headers = {"Content-Type": "application/x-www-form-urlencoded", "Accept": "application/octet-stream"}
-
-        # single column
-        filter = f"var:{index_col_name}=F5"
-        query = f"method=mean&{filter}"
-        query_hash = hashlib.sha1(query.encode()).hexdigest()
-        url = f"{self.TEST_URL_BASE}{endpoint}?key={query_hash}"
-        result = self.client.post(url, headers=headers, data=query)
-
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
-        df = decode_fbs.decode_matrix_FBS(result.data)
-        self.assertEqual(df["n_rows"], 2638)
-        self.assertEqual(df["n_cols"], 1)
-        self.assertEqual(df["col_idx"], [query_hash])
-        self.assertAlmostEqual(df["columns"][0][0], -0.110451095)
-
-        # multi-column
-        col_names = ["F5", "BEB3", "SIK1"]
-        filter = "&".join([f"var:{index_col_name}={name}" for name in col_names])
-        query = f"method=mean&{filter}"
-        query_hash = hashlib.sha1(query.encode()).hexdigest()
-        url = f"{self.TEST_URL_BASE}{endpoint}?key={query_hash}"
-        result = self.client.post(url, headers=headers, data=query)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
-        df = decode_fbs.decode_matrix_FBS(result.data)
-        self.assertEqual(df["n_rows"], 2638)
-        self.assertEqual(df["n_cols"], 1)
-        self.assertEqual(df["col_idx"], [query_hash])
-        self.assertAlmostEqual(df["columns"][0][0], -0.16628358)
-
-
-
-class EndPointsCxg(EndPoints):
-    """Test Case for endpoints"""
-    @classmethod
-    def setUpClass(cls):
-        app_config = AppConfig()
-        app_config.update_default_dataset_config(user_annotations__enable=False)
-
-    def test_get_genesets_json(self):
-        self.app.auth.is_user_authenticated = lambda: True
-        endpoint = "genesets"
-        url = f"{self.TEST_URL_BASE}{endpoint}"
-        result = self.client.get(url, headers={"Accept": "application/json"})
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/json")
-        result_data = json.loads(result.data)
-        self.assertIsNotNone(result_data["genesets"])
-        self.assertIsNotNone(result_data["tid"])
-
-        self.assertEqual(
-            result_data,
-            {
-                "genesets": [
-                    {
-                        "genes": [
-                            {"gene_description": " a gene_description", "gene_symbol": "F5"},
-                            {"gene_description": "", "gene_symbol": "SUMO3"},
-                            {"gene_description": "", "gene_symbol": "SRM"},
-                        ],
-                        "geneset_description": "a description",
-                        "geneset_name": "first gene set name",
-                    },
-                    {
-                        "genes": [
-                            {"gene_description": "", "gene_symbol": "RER1"},
-                            {"gene_description": "", "gene_symbol": "SIK1"},
-                        ],
-                        "geneset_description": "",
-                        "geneset_name": "second_gene_set",
-                    },
-                    {"genes": [], "geneset_description": "", "geneset_name": "third gene set"},
-                    {"genes": [], "geneset_description": "fourth description", "geneset_name": "fourth_gene_set"},
-                    {"genes": [], "geneset_description": "", "geneset_name": "fifth_dataset"},
-                    {
-                        "genes": [
-                            {"gene_description": "", "gene_symbol": "ACD"},
-                            {"gene_description": "", "gene_symbol": "AATF"},
-                            {"gene_description": "", "gene_symbol": "F5"},
-                            {"gene_description": "", "gene_symbol": "PIGU"},
-                        ],
-                        "geneset_description": "",
-                        "geneset_name": "summary test",
-                    },
-                    {'genes': [], 'geneset_description': '', 'geneset_name': 'geneset_to_delete'},
-                    {'genes': [], 'geneset_description': '', 'geneset_name': 'geneset_to_edit'},
-                    {
-                        'genes': [],
-                        'geneset_description': '',
-                        'geneset_name': 'fill_this_geneset'
-                    },
-                    {
-                        'genes': [{'gene_description': '', 'gene_symbol': 'SIK1'}],
-                        'geneset_description': '',
-                        'geneset_name': 'empty_this_geneset'
-                    },
-                    {
-                        'genes': [{'gene_description': '', 'gene_symbol': 'SIK1'}],
-                        'geneset_description': '',
-                        'geneset_name': 'brush_this_gene'
-                    }
-                ],
-                "tid": 0,
-            },
-        )
-
-    def test_get_genesets_csv(self):
-        endpoint = "genesets"
-        url = f"{self.TEST_URL_BASE}{endpoint}"
-        self.app.auth.is_user_authenticated = lambda: True
-        result = self.client.get(url, headers={"Accept": "text/csv"})
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "text/csv")
-        expected_data = """gene_set_name,gene_set_description,gene_symbol,gene_description\r
-first gene set name,a description,F5, a gene_description\r
-first gene set name,a description,SUMO3,\r
-first gene set name,a description,SRM,\r
-second_gene_set,,RER1,\r
-second_gene_set,,SIK1,\r
-third gene set,,,\r
-fourth_gene_set,fourth description,,\r
-fifth_dataset,,,\r
-summary test,,ACD,\r
-summary test,,AATF,\r
-summary test,,F5,\r
-summary test,,PIGU,\r
-geneset_to_delete,,,\r
-geneset_to_edit,,,\r
-fill_this_geneset,,,\r
-empty_this_geneset,,SIK1,\r
-brush_this_gene,,SIK1,\r
-"""
-        self.assertEqual(result.data.decode("utf-8"), expected_data)
-
-    def test_put_genesets(self):
-        endpoint = "genesets"
-        url = f"{self.TEST_URL_BASE}{endpoint}"
-
-        result = self.client.get(url, headers={"Accept": "application/json"})
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-
-        test1 = {"tid": 3, "genesets": []}
-        result = self.client.put(url, json=test1)
-
-        self.assertEqual(result.status_code, HTTPStatus.METHOD_NOT_ALLOWED)
-

backend/test/test_czi_hosted/unit/common/test_corpora.py

@@ -1,168 +0,0 @@
-import json
-import shutil
-import tempfile
-import unittest
-from http import HTTPStatus
-
-import anndata
-
-from backend.czi_hosted.common.config.app_config import AppConfig
-from backend.czi_hosted.common.corpora import (
-    corpora_get_versions_from_anndata,
-    corpora_is_version_supported,
-    corpora_get_props_from_anndata,
-)
-from backend.test.test_czi_hosted.unit import BaseTest
-from backend.test import PROJECT_ROOT
-
-VERSION = "v0.2"
-
-
-class CorporaAPITest(unittest.TestCase):
-    def test_corpora_get_versions_from_anndata(self):
-        adata = self._get_h5ad()
-
-        if "version" in adata.uns:
-            del adata.uns["version"]
-        self.assertIsNone(corpora_get_versions_from_anndata(adata))
-
-        # something bogus
-        adata.uns["version"] = 99
-        self.assertIsNone(corpora_get_versions_from_anndata(adata))
-
-        # something legit
-        adata.uns["version"] = {"corpora_schema_version": "0.0.0", "corpora_encoding_version": "9.9.9"}
-        self.assertEqual(corpora_get_versions_from_anndata(adata), ["0.0.0", "9.9.9"])
-
-    def test_corpora_is_version_supported(self):
-        self.assertTrue(corpora_is_version_supported("1.0.0", "0.1.0"))
-        self.assertFalse(corpora_is_version_supported("0.0.0", "0.1.0"))
-        self.assertFalse(corpora_is_version_supported("1.0.0", "0.0.0"))
-
-    def test_corpora_get_props_from_anndata(self):
-        adata = self._get_h5ad()
-
-        if "version" in adata.uns:
-            del adata.uns["version"]
-        self.assertIsNone(corpora_get_props_from_anndata(adata))
-
-        # something bogus
-        adata.uns["version"] = 99
-        self.assertIsNone(corpora_get_props_from_anndata(adata))
-
-        # unsupported version, but missing required values
-        adata.uns["version"] = {"corpora_schema_version": "99.0.0", "corpora_encoding_version": "32.1.0"}
-        with self.assertRaises(ValueError):
-            corpora_get_props_from_anndata(adata)
-
-        # legit version, but missing required values
-        adata.uns["version"] = {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"}
-        with self.assertRaises(KeyError):
-            corpora_get_props_from_anndata(adata)
-
-        some_fields = {
-            "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"},
-            "title": "title",
-            "layer_descriptions": "layer_descriptions",
-            "organism": "organism",
-            "organism_ontology_term_id": "organism_ontology_term_id",
-            "project_name": "project_name",
-            "project_description": "project_description",
-            "contributors": json.dumps([{"contributors": "contributors"}]),
-            "project_links": json.dumps([{"link_name": "link_name", "link_url": "link_url", "link_type": "SUMMARY"}]),
-        }
-        for k in some_fields:
-            adata.uns[k] = some_fields[k]
-        some_fields["contributors"] = json.loads(some_fields["contributors"])
-        some_fields["project_links"] = json.loads(some_fields["project_links"])
-        self.assertEqual(corpora_get_props_from_anndata(adata), some_fields)
-
-    def test_corpora_get_props_from_anndata_v110(self):
-        adata = self._get_h5ad()
-
-        if "version" in adata.uns:
-            del adata.uns["version"]
-        self.assertIsNone(corpora_get_props_from_anndata(adata))
-
-        # legit version, but missing required values
-        adata.uns["version"] = {"corpora_schema_version": "1.1.0", "corpora_encoding_version": "0.1.0"}
-        with self.assertRaises(KeyError):
-            corpora_get_props_from_anndata(adata)
-
-        # Metadata following schema 1.1.0, which removes some fields relative to 1.1.0
-        some_110_fields = {
-            "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"},
-            "title": "title",
-            "layer_descriptions": "layer_descriptions",
-            "organism": "organism",
-            "organism_ontology_term_id": "organism_ontology_term_id",
-        }
-        for k in some_110_fields:
-            adata.uns[k] = some_110_fields[k]
-        self.assertEqual(corpora_get_props_from_anndata(adata), some_110_fields)
-
-    def _get_h5ad(self):
-        return anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
-
-
-class CorporaRESTAPITest(BaseTest):
-    """ Confirm endpoints reflect Corpora-specific features """
-
-    @classmethod
-    def setCorporaFields(cls, path):
-        adata = anndata.read_h5ad(path)
-        corpora_props = {
-            "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"},
-            "title": "PBMC3K",
-            "contributors": json.dumps([{"name": "name"}]),
-            "layer_descriptions": {"X": "raw counts"},
-            "organism": "human",
-            "organism_ontology_term_id": "unknown",
-            "project_name": "test project",
-            "project_description": "test description",
-            "project_links": json.dumps(
-                [{"link_name": "test link", "link_type": "SUMMARY", "link_url": "https://a.u.r.l/"}]
-            ),
-            "default_embedding": "X_tsne",
-        }
-        adata.uns.update(corpora_props)
-        adata.write(path)
-
-    @classmethod
-    def setUpClass(cls, app_config=None):
-        if not app_config:
-            app_config = AppConfig()
-        cls.tmp_dir = tempfile.TemporaryDirectory()
-        src = f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad"
-        dst = f"{cls.tmp_dir.name}/pbmc3k.h5ad"
-        shutil.copyfile(src, dst)
-        cls.setCorporaFields(dst)
-        app_config.update_server_config(single_dataset__datapath=dst)
-
-        super().setUpClass(app_config)
-        cls.app.testing = True
-        cls.client = cls.app.test_client()
-
-    def setUp(self):
-        self.session = self.client
-        self.url_base = "/api/v0.2/"
-
-    def test_config(self):
-        endpoint = "config"
-        url = f"{self.url_base}{endpoint}"
-        header = {"Content-Type": "application/json"}
-        result = self.session.get(url, headers=header)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/json")
-
-        result_data = json.loads(result.data)
-        self.assertIsInstance(result_data["config"]["corpora_props"], dict)
-        self.assertIsInstance(result_data["config"]["parameters"], dict)
-
-        corpora_props = result_data["config"]["corpora_props"]
-        parameters = result_data["config"]["parameters"]
-
-        self.assertEqual(corpora_props["version"]["corpora_schema_version"], "1.0.0")
-
-        self.assertEqual(corpora_props["organism"], "human")
-        self.assertEqual(parameters["default_embedding"], "tsne")

backend/test/test_czi_hosted/unit/common/test_nan_rest.py

@@ -1,65 +0,0 @@
-from http import HTTPStatus
-import math
-
-import backend.test.decode_fbs as decode_fbs
-
-from backend.czi_hosted.common.config.app_config import AppConfig
-from backend.test import FIXTURES_ROOT
-from backend.test.test_czi_hosted.unit import BaseTest
-
-VERSION = "v0.2"
-BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
-
-
-class WithNaNs(BaseTest):
-    """Test Case for endpoints"""
-
-    @classmethod
-    def setUpClass(cls):
-        app_config = AppConfig()
-        app_config.update_server_config(single_dataset__datapath=f"{FIXTURES_ROOT}/nan.h5ad")
-        app_config.update_default_dataset_config(user_annotations__enable=True)
-        super().setUpClass(app_config)
-        cls.app.testing = True
-        cls.client = cls.app.test_client()
-
-    def setUp(self):
-        self.session = self.client
-        self.url_base = "api/v0.2/"
-
-    def test_initialize(self):
-        endpoint = "schema"
-        url = f"{self.url_base}{endpoint}"
-        result = self.session.get(url)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-
-    def test_data(self):
-        endpoint = "data/var"
-        url = f"{self.url_base}{endpoint}"
-        filter = {"filter": {"var": {"index": [[0, 20]]}}}
-        header = {"Accept": "application/octet-stream"}
-        result = self.session.put(url, headers=header, json=filter)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
-        df = decode_fbs.decode_matrix_FBS(result.data)
-        self.assertTrue(math.isnan(df["columns"][3][3]))
-
-    def test_annotation_obs(self):
-        endpoint = "annotations/obs"
-        url = f"{self.url_base}{endpoint}"
-        header = {"Accept": "application/octet-stream"}
-        result = self.session.get(url, headers=header)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
-        df = decode_fbs.decode_matrix_FBS(result.data)
-        self.assertTrue(math.isnan(df["columns"][2][0]))
-
-    def test_annotation_var(self):
-        endpoint = "annotations/var"
-        url = f"{self.url_base}{endpoint}"
-        header = {"Accept": "application/octet-stream"}
-        result = self.session.get(url, headers=header)
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
-        df = decode_fbs.decode_matrix_FBS(result.data)
-        self.assertTrue(math.isnan(df["columns"][2][0]))

backend/test/test_czi_hosted/unit/common/test_rest.py

@@ -1,79 +0,0 @@
-import unittest
-from urllib.parse import parse_qs
-from werkzeug.datastructures import MultiDict
-from backend.common.errors import FilterError
-from backend.czi_hosted.common.rest import _query_parameter_to_filter
-
-
-def _qsparse(qs):
-    """ emulate what Flask/Werkzeug do to our QS """
-    return MultiDict(parse_qs(qs))
-
-
-class FilterParseTests(unittest.TestCase):
-    """ Test cases for various filter parsing """
-
-    def test_queryparam_to_filter_parse(self):
-        # categories
-        self.assertEqual(
-            _query_parameter_to_filter(_qsparse("obs:foo=bar&var:baz=133&var:baz=A&obs:baz=foo")),
-            {
-                "obs": {"annotation_value": [{"name": "foo", "values": ["bar"]}, {"name": "baz", "values": ["foo"]}]},
-                "var": {"annotation_value": [{"name": "baz", "values": ["133", "A"]}]},
-            },
-        )
-
-        # ranges
-        self.assertEqual(
-            _query_parameter_to_filter(_qsparse("obs:A=1,99&obs:B=*,100&obs:C=0,*")),
-            {
-                "obs": {
-                    "annotation_value": [
-                        {"name": "A", "min": 1, "max": 99.0},
-                        {"name": "B", "max": 100.0},
-                        {"name": "C", "min": 0.0},
-                    ]
-                },
-            },
-        )
-
-        # combo
-        self.assertEqual(
-            _query_parameter_to_filter(_qsparse("var:B=YES&var:A=1,99&var:B=NO")),
-            {
-                "var": {
-                    "annotation_value": [
-                        {"name": "B", "values": ["YES", "NO"]},
-                        {"name": "A", "min": 1.0, "max": 99.0},
-                    ]
-                },
-            },
-        )
-
-    def test_queryparam_to_filter_escaping(self):
-        self.assertEqual(
-            _query_parameter_to_filter(_qsparse("obs:var=%2521%252C%253AOK%253D&obs:A%2521=YO")),
-            {"obs": {"annotation_value": [{"name": "var", "values": ["!,:OK="]}, {"name": "A!", "values": ["YO"]}]}},
-        )
-
-    def test_queryparam_to_filter_errors(self):
-
-        # should raise FilterError
-        filter_errors = [
-            "foo=bar",  # no axis
-            "X=&Y=3",  # no value
-            "X&Y=3",  # no value
-            "moo:foo=bar",  # bad axis
-            "obs:x=1,A",  # non-numeric range
-            "var:X=1,2&var:X=3,4",  # duplicate ranges
-            "var:Y=,",
-            "var:Y=2,",
-            "var:Y=,5",
-            "var:Y=*,",
-            "var:Y=,*",
-            "var:Y=*,*",
-        ]
-
-        for qs in filter_errors:
-            with self.assertRaises(FilterError):
-                _query_parameter_to_filter(_qsparse(qs))

backend/test/test_czi_hosted/unit/common/utils/test_utils.py

@@ -1,34 +0,0 @@
-import os
-import shutil
-import unittest
-
-from backend.common.utils.utils import import_plugins
-from backend.test import PROJECT_ROOT, random_string
-
-
-class TestPlugins(unittest.TestCase):
-    """ Test plugin import functionality """
-
-    plugins_dir = f"{PROJECT_ROOT}/backend/test/test_czi_hosted/unit/plugins"
-    test_plugin_path = f"{plugins_dir}/foo.py"
-    secret = random_string(8)
-
-    @classmethod
-    def setUpClass(cls) -> None:
-        if not os.path.isdir(cls.plugins_dir):
-            os.mkdir(cls.plugins_dir)
-        with open(cls.test_plugin_path, "w") as fh:
-            fh.write(f'SECRET = "{cls.secret}"\n')
-
-    @classmethod
-    def tearDownClass(cls) -> None:
-        if os.path.isdir(cls.plugins_dir):
-            shutil.rmtree(cls.plugins_dir)
-
-    def test_import_plugins(self):
-        self.assertTrue(os.path.isfile(self.test_plugin_path))
-        loaded_modules = import_plugins("backend.test.test_czi_hosted.unit.plugins")
-        # test that import plugins found the file
-        self.assertEqual(["backend.test.test_czi_hosted.unit.plugins.foo"], [ele.__name__ for ele in loaded_modules])
-        # test that the module was properly executed
-        self.assertEqual(self.secret, loaded_modules[0].SECRET)

backend/test/test_czi_hosted/unit/converters/schema/test_gene_symbol.py

@@ -1,61 +0,0 @@
-import os
-import unittest
-
-import pandas as pd
-
-from backend.czi_hosted.converters.schema import gene_symbol
-from backend.test import FIXTURES_ROOT
-
-
-class TestHGNCSymbolChecker(unittest.TestCase):
-
-    def setUp(self):
-        self.test_hgnc_path = os.path.join(FIXTURES_ROOT, "hgnc_example.txt.gz")
-        self.hgnc_checker = gene_symbol.HGNCSymbolChecker.from_hgnc_records(self.test_hgnc_path)
-
-    def test_symbol_upgrade(self):
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("SEPT1"), "SEPTIN1")
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("ADRB2R"), "ADRB2")
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("BAR"), "ADRB2")
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("sept1"), "SEPTIN1")
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("AdRb2R"), "ADRB2")
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("bar"), "ADRB2")
-
-        # Strip off seurat endings when appropriate
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("SEPT1.1"), "SEPTIN1")
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("ADRB2-1"), "ADRB2")
-
-        # DIFF6 is ambiguous so don't upgrade it
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("DIFF6"), "DIFF6")
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("diff6"), "diff6")
-
-        # ARG1 is approved
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("ARG1"), "ARG1")
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("arg1"), "ARG1")
-
-        # HAP1 is both approved and withdrawn
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("HAP1"), "HAP1")
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("hap1"), "HAP1")
-
-        # Leave unknown symbols alone
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("NOTASYMBOL"), "NOTASYMBOL")
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("notasymbol"), "notasymbol")
-
-        # Upgrade HGNC ids unless you can't find it
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("HGNC:286"), "ADRB2")
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("HGNC:4812"), "HAP1")
-        self.assertEqual(self.hgnc_checker.upgrade_symbol("HGNC:123456"), "HGNC:123456")
-
-    def test_check_symbol(self):
-        self.assertEqual(self.hgnc_checker.check_symbol("SEPT1"), gene_symbol.SymbolStatus.UPGRADABLE)
-        self.assertEqual(self.hgnc_checker.check_symbol("DIFF6"), gene_symbol.SymbolStatus.AMBIGUOUS)
-        self.assertEqual(self.hgnc_checker.check_symbol("NOTASYMBOL"), gene_symbol.SymbolStatus.UNKNOWN)
-
-        # HAP1 is one of the approved and withdrawn symbols
-        self.assertEqual(self.hgnc_checker.check_symbol("HAP1"), gene_symbol.SymbolStatus.APPROVED)
-
-    def test_upgrade_index(self):
-        index = pd.Index(["SEPT1", "DIFF6", "NOTASYMBOL", "bar", "SEPTIN1"])
-        var_df = pd.DataFrame([[0] * len(index)], index=index)
-        upgraded_index = gene_symbol.get_upgraded_var_index(var_df, hgnc_path=self.test_hgnc_path)
-        self.assertEqual(upgraded_index.tolist(), ["SEPTIN1", "DIFF6", "NOTASYMBOL", "ADRB2", "SEPTIN1"])

backend/test/test_czi_hosted/unit/converters/schema/test_ontology.py

@@ -1,128 +0,0 @@
-import json
-
-import unittest.mock
-
-from backend.czi_hosted.converters.schema import ontology
-
-
-class TestOntologyParsing(unittest.TestCase):
-    def setUp(self):
-
-        self.curies = ["UBERON:0002048", "HsapDv:0000174", "NCBITaxon:9606", "EFO:0008995"]
-
-        self.names = ["UBERON", "HsapDv", "NCBITaxon", "EFO"]
-
-        self.values = ["0002048", "0000174", "9606", "0008995"]
-
-        self.iris = [
-            "http://purl.obolibrary.org/obo/UBERON_0002048",
-            "http://purl.obolibrary.org/obo/HsapDv_0000174",
-            "http://purl.obolibrary.org/obo/NCBITaxon_9606",
-            "http://www.ebi.ac.uk/efo/EFO_0008995",
-        ]
-
-        URL_ROOT = "http://www.ebi.ac.uk/ols/api/ontologies/"
-        self.urls = [
-            URL_ROOT + "UBERON/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FUBERON_0002048",
-            URL_ROOT + "HsapDv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FHsapDv_0000174",
-            URL_ROOT + "NCBITaxon/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_9606",
-            URL_ROOT + "EFO/terms/http%253A%252F%252Fwww.ebi.ac.uk%252Fefo%252FEFO_0008995",
-        ]
-
-        self.responses = {
-            "UBERON:0002048": {
-                "iri": "http://purl.obolibrary.org/obo/UBERON_0002048",
-                "description": ["Respiration organ that develops as an outpocketing of the esophagus."],
-                "label": "lung",
-            },
-            "HsapDv:0000174": {
-                "iri": "http://purl.obolibrary.org/obo/HsapDv_0000174",
-                "description": ["Infant stage that refers to an infant who is over 1 and under 2 months old."],
-                "label": "1-month-old human stage",
-            },
-            "NCBITaxon:9606": {
-                "iri": "http://purl.obolibrary.org/obo/NCBITaxon_9606",
-                "description": None,
-                "label": "Homo sapiens",
-            },
-            "EFO:0008995": {
-                "iri": "http://www.ebi.ac.uk/efo/EFO_0008995",
-                "description": [
-                    (
-                        '10X is a "synthetic long-read" technology and works by capturing a barcoded oligo-coated '
-                        "gel-bead and 0.3x genome copies into a single emulsion droplet, processing the equivalent "
-                        "of 1 million pipetting steps. Successive versions of the 10x chemistry use different "
-                        "barcode locations to improve the sequencing yield and quality of 10x experiments."
-                    )
-                ],
-                "label": "10X sequencing",
-            },
-        }
-
-    def test_ontololgy_name(self):
-        for curie, expected_name in zip(self.curies, self.names):
-            self.assertEqual(ontology._ontology_name(curie), expected_name)
-
-    def test_ontololgy_value(self):
-        for curie, expected_value in zip(self.curies, self.values):
-            self.assertEqual(ontology._ontology_value(curie), expected_value)
-
-    def test_iri(self):
-        for curie, expected_iri in zip(self.curies, self.iris):
-            self.assertEqual(ontology._iri(curie), expected_iri)
-
-    def test_ontology_info_url(self):
-        for curie, expected_url in zip(self.curies, self.urls):
-            self.assertEqual(ontology._ontology_info_url(curie), expected_url)
-
-    def test_empty_ontology_info_url(self):
-        self.assertEqual(ontology._ontology_info_url(""), "")
-
-
-class TestOntologyLookup(unittest.TestCase):
-    def setUp(self):
-        self.responses = {
-            "UBERON:0002048": {
-                "iri": "http://purl.obolibrary.org/obo/UBERON_0002048",
-                "description": ["Respiration organ that develops as an outpocketing of the esophagus."],
-                "label": "lung",
-            },
-            "HsapDv:0000174": {
-                "iri": "http://purl.obolibrary.org/obo/HsapDv_0000174",
-                "description": ["Infant stage that refers to an infant who is over 1 and under 2 months old."],
-                "label": "1-month-old human stage",
-            },
-            "NCBITaxon:9606": {
-                "iri": "http://purl.obolibrary.org/obo/NCBITaxon_9606",
-                "description": None,
-                "label": "Homo sapiens",
-            },
-            "EFO:0008995": {
-                "iri": "http://www.ebi.ac.uk/efo/EFO_0008995",
-                "description": [
-                    ('10X is a "synthetic long-read" technology and works by capturing a barcoded oligo-coated '
-                     'gel-bead and 0.3x genome copies into a single emulsion droplet, processing the equivalent '
-                     'of 1 million pipetting steps. Successive versions of the 10x chemistry use different barcode '
-                     'locations to improve the sequencing yield and quality of 10x experiments.')
-                ],
-                "label": "10X sequencing",
-            },
-        }
-
-        self.labels = {
-            "UBERON:0002048": "lung",
-            "HsapDv:0000174": "1-month-old human stage",
-            "NCBITaxon:9606": "Homo sapiens",
-            "EFO:0008995": "10X sequencing",
-        }
-
-    @unittest.mock.patch("requests.get")
-    def test_lookup_label(self, mock_get):
-
-        for curie, response in self.responses.items():
-            mock_get.return_value.content = json.dumps(response)
-            mock_get.return_value.json.return_value = response
-            mock_get.return_value.status_code = 200
-
-            label = ontology.get_ontology_label(curie)
-            self.assertEqual(label, self.labels[curie])

backend/test/test_czi_hosted/unit/converters/schema/test_remix.py

@@ -1,257 +0,0 @@
-import json
-import os
-import unittest
-import unittest.mock
-
-import anndata
-import numpy
-import pandas as pd
-import scanpy as sc
-
-from backend.czi_hosted.converters.schema import remix
-
-from backend.test import PROJECT_ROOT, FIXTURES_ROOT
-
-
-class TestApplySchema(unittest.TestCase):
-
-    def setUp(self):
-        self.source_h5ad_path = f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad"
-        self.output_h5ad_path = f"{FIXTURES_ROOT}/test_remix.h5ad"
-        self.config_path = f"{FIXTURES_ROOT}/test_config.yaml"
-        self.bad_config_path = f"{FIXTURES_ROOT}/test_bad_config.yaml"
-
-    def tearDown(self):
-        try:
-            os.remove(self.output_h5ad_path)
-        except OSError:
-            pass
-
-    @unittest.mock.patch("backend.czi_hosted.converters.schema.ontology.get_ontology_label")
-    def test_apply_schema(self, mock_get_ontology_label):
-        mock_get_ontology_label.return_value = "test label"
-        remix.apply_schema(self.source_h5ad_path, self.config_path, self.output_h5ad_path)
-        new_adata = sc.read_h5ad(self.output_h5ad_path)
-
-        self.assertIn("cell_type", new_adata.obs.columns)
-        self.assertListEqual(["test label"], new_adata.obs["cell_type"].unique().tolist())
-        self.assertListEqual(
-            ["CL:00001", "CL:00002", "CL:00003", "CL:00004", "CL:00005", "CL:00006", "CL:00007", "CL:00008"],
-            sorted(new_adata.obs["cell_type_ontology_term_id"].unique().tolist())
-        )
-
-        self.assertIn("version", new_adata.uns_keys())
-
-    @unittest.mock.patch("backend.czi_hosted.converters.schema.ontology.get_ontology_label")
-    def test_apply_bad_schema(self, mock_get_ontology_label):
-        mock_get_ontology_label.return_value = "test label"
-        remix.apply_schema(self.source_h5ad_path, self.bad_config_path, self.output_h5ad_path)
-        new_adata = sc.read_h5ad(self.output_h5ad_path)
-
-        # Should refuse to write the version
-        self.assertNotIn("version", new_adata.uns_keys())
-
-class TestFieldParsing(unittest.TestCase):
-
-    def test_is_curie(self):
-        self.assertTrue(remix.is_curie("EFO:00001"))
-        self.assertTrue(remix.is_curie("UBERON:123456"))
-        self.assertTrue(remix.is_curie("HsapDv:0001"))
-        self.assertFalse(remix.is_curie("UBERON"))
-        self.assertFalse(remix.is_curie("UBERON:"))
-        self.assertFalse(remix.is_curie("123456"))
-
-    def test_is_ontology_field(self):
-        self.assertTrue(remix.is_ontology_field("tissue_ontology_term_id"))
-        self.assertTrue(remix.is_ontology_field("cell_type_ontology_term_id"))
-        self.assertFalse(remix.is_ontology_field("cell_ontology"))
-        self.assertFalse(remix.is_ontology_field("method"))
-
-    def test_get_label_field_name(self):
-        self.assertEqual("tissue", remix.get_label_field_name("tissue_ontology_term_id"))
-        self.assertEqual("cell_type", remix.get_label_field_name("cell_type_ontology_term_id"))
-
-    def test_split_suffix(self):
-        self.assertEqual(("UBERON:1234", " (organoid)"), remix.split_suffix("UBERON:1234 (organoid)"))
-        self.assertEqual(("UBERON:1234", " (cell culture)"), remix.split_suffix("UBERON:1234 (cell culture)"))
-        self.assertEqual(("UBERON:1234", ""), remix.split_suffix("UBERON:1234"))
-        self.assertEqual(("UBERON:1234 (something)", ""), remix.split_suffix("UBERON:1234 (something)"))
-
-    @unittest.mock.patch("backend.czi_hosted.converters.schema.ontology.get_ontology_label")
-    def test_get_curie_and_label(self, mock_get_ontology_label):
-        mock_get_ontology_label.return_value = "test label"
-        self.assertEqual(
-            remix.get_curie_and_label("UBERON:1234"),
-            ("UBERON:1234", "test label")
-        )
-        self.assertEqual(
-            remix.get_curie_and_label("UBERON:1234 (cell culture)"),
-            ("UBERON:1234 (cell culture)", "test label (cell culture)")
-        )
-        self.assertEqual(
-            remix.get_curie_and_label("whatever"),
-            ("", "whatever")
-        )
-
-
-class TestManipulateAnndata(unittest.TestCase):
-
-    def setUp(self):
-
-        self.cell_count = 20
-        self.gene_count = 200
-        X = numpy.random.randint(0, 1000, (self.cell_count, self.gene_count))
-        uns = {"organism": "monkey", "experiment": "monkey experiment"}
-        obs = pd.DataFrame(
-            index=[f"Cell{d}" for d in range(self.cell_count)],
-            columns=["tissue", "CellType"],
-            data=[["lung", "epithelial"]] * (self.cell_count // 2) + [["lung", "endothelial"]] * (self.cell_count // 2)
-        )
-        var = pd.DataFrame(index=[f"SEPT{d}" for d in range(self.gene_count)])
-
-        self.adata = anndata.AnnData(X=X, obs=obs, var=var, uns=uns)
-
-    def test_safe_add_field(self):
-
-        remix.safe_add_field(self.adata.obs, "tissue", ["monkey lung"] * self.cell_count)
-        self.assertEqual(self.adata.obs["tissue_original"].tolist(), ["lung"] * self.cell_count)
-        self.assertEqual(self.adata.obs["tissue"].tolist(), ["monkey lung"] * self.cell_count)
-
-        remix.safe_add_field(self.adata.uns, "contributors", [{"name": "contributor1"}, {"name": "contributor2"}])
-        self.assertEqual(
-            self.adata.uns["contributors"],
-            json.dumps([{"name": "contributor1"}, {"name": "contributor2"}])
-        )
-
-    @unittest.mock.patch("backend.czi_hosted.converters.schema.ontology.get_ontology_label")
-    def test_remix_uns(self, mock_get_ontology_label):
-        mock_get_ontology_label.return_value = "Pan troglodytes"
-        uns_config = {
-            "version": {
-                "corpora_schema_version": "1.0.0",
-                "corpora_encoding_version": "0.1.0"
-            },
-            "organism_ontology_term_id": "NCBITaxon:9598",
-            "contributors": [
-                {
-                    "name": "scientist",
-                    "email": "scientist@science.com"
-                }
-            ]
-        }
-
-        remix.remix_uns(self.adata, uns_config)
-
-        self.assertEqual(
-            sorted(self.adata.uns_keys()),
-            sorted(["organism_original", "organism", "organism_ontology_term_id",
-                    "contributors", "version", "experiment"])
-        )
-
-        self.assertEqual(self.adata.uns['organism'], "Pan troglodytes")
-        self.assertEqual(self.adata.uns['organism_original'], "monkey")
-        self.assertEqual(self.adata.uns['organism_ontology_term_id'], "NCBITaxon:9598")
-        self.assertEqual(self.adata.uns['contributors'],
-                         json.dumps([{"name": "scientist", "email": "scientist@science.com"}]))
-
-    @unittest.mock.patch("backend.czi_hosted.converters.schema.ontology.get_ontology_label")
-    def test_remix_obs(self, mock_get_ontology_label):
-        mock_get_ontology_label.return_value = "lung (in a monkey)"
-        obs_config = {
-            "tissue_ontology_term_id": {
-                "tissue": {
-                    "lung": "UBERON:00000"
-                }
-            },
-            "cell_color": {
-                "CellType": {
-                    "epithelial": "fuschia",
-                    "endothelial": "khaki"
-                }
-            },
-            "sex": "male"
-        }
-
-        remix.remix_obs(self.adata, obs_config)
-        self.assertEqual(
-            sorted(self.adata.obs_keys()),
-            sorted(["tissue", "tissue_ontology_term_id", "tissue_original", "CellType", "cell_color", "sex"])
-        )
-
-        self.assertTrue(all(v == "lung" for v in self.adata.obs.tissue_original))
-        self.assertTrue(all(v == "UBERON:00000" for v in self.adata.obs.tissue_ontology_term_id))
-        self.assertTrue(all(v == "lung (in a monkey)" for v in self.adata.obs.tissue))
-        self.assertTrue(all(v == "male" for v in self.adata.obs.sex))
-        self.assertTrue(all(v in (("epithelial", "fuschia"), ("endothelial", "khaki"))
-                            for v in zip(self.adata.obs.CellType, self.adata.obs.cell_color)))
-
-
-class TestFixupGeneSymbols(unittest.TestCase):
-
-    def setUp(self):
-        self.seurat_path = f"{PROJECT_ROOT}/czi_hosted/test/fixtures/schema_test_data/seurat_tutorial.h5ad"
-        self.seurat_merged_path = f"{PROJECT_ROOT}/czi_hosted/test/fixtures/schema_test_data/seurat_tutorial_merged.h5ad"
-        self.sctransform_path = f"{PROJECT_ROOT}/czi_hosted/test/fixtures/schema_test_data/sctransform.h5ad"
-        self.sctransform_merged_path = f"{PROJECT_ROOT}/czi_hosted/test/fixtures/schema_test_data/sctransform_merged.h5ad"
-
-        # There's lots of MALAT1, but it doesn't collide with any other names,
-        # so it shouldn't change during merging.
-        self.stable_gene = "MALAT1"
-
-    def test_fixup_gene_symbols_seurat(self):
-
-        if not os.path.isfile(self.seurat_path):
-            return unittest.skip(
-                "Skipping gene symbol conversion tests because test h5ads are not present. To create them, "
-                "run czi_hosted/test/fixtures/schema_test_data/generate_test_data.sh"
-            )
-
-        original_adata = sc.read_h5ad(self.seurat_path)
-        merged_adata = sc.read_h5ad(self.seurat_merged_path)
-
-        fixup_config = {"X": "log1p", "counts": "raw", "scale.data": "log1p"}
-
-        fixed_adata = remix.fixup_gene_symbols(original_adata, fixup_config)
-
-        self.assertEqual(
-            merged_adata.layers["counts"][:, merged_adata.var.index == self.stable_gene].sum(),
-            fixed_adata.raw.X[:, fixed_adata.var.index == self.stable_gene].sum()
-        )
-        self.assertAlmostEqual(
-            merged_adata.X[:, merged_adata.var.index == self.stable_gene].sum(),
-            fixed_adata.X[:, fixed_adata.var.index == self.stable_gene].sum()
-        )
-
-        self.assertAlmostEqual(
-            merged_adata.layers["scale.data"][:, merged_adata.var.index == self.stable_gene].sum(),
-            fixed_adata.layers["scale.data"][:, fixed_adata.var.index == self.stable_gene].sum()
-        )
-
-    def test_fixup_gene_symbols_sctransform(self):
-
-        if not os.path.isfile(self.sctransform_path):
-            return unittest.skip(
-                "Skipping gene symbol conversion tests because test h5ads are not present. To create them, "
-                "run czi_hosted/test/fixtures/schema_test_data/generate_test_data.sh"
-            )
-
-        original_adata = sc.read_h5ad(self.sctransform_path)
-        merged_adata = sc.read_h5ad(self.sctransform_merged_path)
-
-        fixup_config = {"X": "log1p", "counts": "raw"}
-
-        fixed_adata = remix.fixup_gene_symbols(original_adata, fixup_config)
-
-        # sctransform does a bunch of stuff, including slightly modifying the
-        # raw counts. So we can't assert for exact equality the way we do with
-        # the vanilla seurat tutorial. But, the results should still be very
-        # close.
-        merged_raw_stable = merged_adata.layers["counts"][:, merged_adata.var.index == self.stable_gene].sum()
-        fixed_raw_stable = fixed_adata.raw.X[:, fixed_adata.var.index == self.stable_gene].sum()
-        self.assertLess(abs(merged_raw_stable - fixed_raw_stable), .001 * merged_raw_stable)
-
-        self.assertAlmostEqual(
-            merged_adata.X[:, merged_adata.var.index == self.stable_gene].sum(),
-            fixed_adata.X[:, fixed_adata.var.index == self.stable_gene].sum(),
-            0
-        )