diff --git a/.github/workflows/canary-deploy.yml b/.github/workflows/canary-deploy.yml
deleted file mode 100644
index f8855b92..00000000
--- a/.github/workflows/canary-deploy.yml
+++ /dev/null
@@ -1,13 +0,0 @@
-name: Deploy canary via single cell infra repo
-
-on:
- push:
- branches: main-canary
-
-jobs:
- deploy:
- runs-on: ubuntu-latest
- steps:
- - name: repository dispatch
- run: |
- curl -XPOST -u czi-sci-single-cell-eng:${{secrets.SCI_GITHUB_TOKEN}} -H "Accept: application/vnd.github.everest-preview+json" -H "Content-Type: application/json" https://api.github.com/repos/chanzuckerberg/single-cell-infra/dispatches --data '{"event_type": "canary-hook"}'
diff --git a/.github/workflows/push_tests.yml b/.github/workflows/push_tests.yml
index a0ab94b7..6887f5e9 100644
--- a/.github/workflows/push_tests.yml
+++ b/.github/workflows/push_tests.yml
@@ -36,7 +36,7 @@ jobs:
npm install
- name: Format with black and lint with flake8
run: |
- make lint-servers
+ make lint-server
- name: Lint src with eslint
working-directory: ./client
run: |
@@ -72,36 +72,6 @@ jobs:
bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k backend/server -cF backend,python,unitTest
cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest
- unit-test-czi-hosted:
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v2
- - name: Set up Python 3.7
- uses: actions/setup-python@v1
- with:
- python-version: 3.7
- - name: Python cache
- uses: actions/cache@v1
- with:
- path: ~/.cache/pip
- key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
- restore-keys: |
- ${{ runner.os }}-pip-
- - name: Node cache
- uses: actions/cache@v1
- with:
- path: ~/.npm
- key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
- restore-keys: |
- ${{ runner.os }}-node-
- - name: Install dependencies
- run: make pydist-czi-hosted install-dist dev-env-czi-hosted
- - name: Unit tests
- run: |
- make unit-test-czi-hosted
- bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k backend/czi-hosted -cF backend,python,unitTest
- cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest
-
smoke-tests:
runs-on: macos-latest
timeout-minutes: 20
@@ -126,7 +96,7 @@ jobs:
restore-keys: |
${{ runner.os }}-node-
- name: Install dependencies
- run: make pydist-czi-hosted install-dist
+ run: make pydist install-dist
- name: Smoke tests (without annotations feature)
run: |
cd client && make smoke-test
diff --git a/.github/workflows/scale-test.yml b/.github/workflows/scale-test.yml
deleted file mode 100644
index a89d93fb..00000000
--- a/.github/workflows/scale-test.yml
+++ /dev/null
@@ -1,30 +0,0 @@
-name: "Scale test cellxgene APIs for initial loading"
-
-on:
- schedule:
- - cron: "0 0 * * Sun"
-
-jobs:
- locust-build:
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v2
- - name: Set up Python 3.7
- uses: actions/setup-python@v1
- with:
- python-version: 3.7
- - name: Install dependencies
- run: |
- pip install -r backend/test/test_czi_hosted/locust/requirements-locust.txt
- - name: Dev Scale Test
- run: |
- locust -f backend/test/test_czi_hosted/locust/locustfile.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt
- - name: Slack success webhook
- env:
- SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
- run: |
- DEV_STATS=$(tail -n 15 locust_dev_stats.txt)
- DEV_MSG="\`\`\`CELLXGENE EXPLORER DEV SCALE TEST RESULTS: ${DEV_STATS}\`\`\`"
- curl -X POST -H 'Content-type: application/json' --data "{'text':'${DEV_MSG}'}" $SLACK_WEBHOOK
-
-
diff --git a/.gitignore b/.gitignore
index 63ff7069..240d90fa 100644
--- a/.gitignore
+++ b/.gitignore
@@ -23,10 +23,6 @@ backend/server/common/web/static/*
backend/server/common/web/templates/
backend/server/common/web/csp-hashes.json
-backend/czi_hosted/common/web/static/*
-backend/czi_hosted/common/web/templates/
-backend/czi_hosted/common/web/csp-hashes.json
-
# eb build
artifact.dir
artifact.zip
diff --git a/MANIFEST_hosted.in b/MANIFEST_hosted.in
deleted file mode 100644
index 6f349b1b..00000000
--- a/MANIFEST_hosted.in
+++ /dev/null
@@ -1,7 +0,0 @@
-recursive-include backend/czi_hosted/common/web/templates *
-recursive-include backend/czi_hosted/common/web/static *
-
-include backend/czi_hosted/requirements.txt
-include backend/czi_hosted/requirements-prepare.txt
-include backend/czi_hosted/converters/schema/hgnc_complete_set.txt.gz
-include backend/czi_hosted/converters/schema/schema_definitions/*
diff --git a/Makefile b/Makefile
index 0241742e..5cf55de6 100644
--- a/Makefile
+++ b/Makefile
@@ -2,7 +2,6 @@ include common.mk
BUILDDIR := build
CLIENTBUILD := $(BUILDDIR)/client
-CZIHOSTEDBUILD := $(BUILDDIR)/backend/czi_hosted
SERVERBUILD := $(BUILDDIR)/backend/server
CLEANFILES := $(BUILDDIR)/ client/build build dist cellxgene.egg-info
@@ -10,7 +9,7 @@ PART ?= patch
# CLEANING
.PHONY: clean
-clean: clean-lite clean-czi-hosted clean-server clean-client
+clean: clean-lite clean-server clean-client
# cleaning the client's node_modules is the longest one, so we avoid that if possible
.PHONY: clean-lite
@@ -25,9 +24,6 @@ clean-client:
clean-server:
cd backend/server && $(MAKE) clean
-.PHONY: clean-czi-hosted
-clean-czi-hosted:
- cd backend/czi_hosted && $(MAKE) clean
# BUILDING PACKAGE
@@ -45,34 +41,18 @@ build: clean build-client
cp -r backend/common $(BUILDDIR)/backend/common
cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR)
-.PHONY: build-czi-hosted
-build-czi-hosted: clean build-client
- git ls-files backend/czi_hosted/ | grep -v 'backend/czi_hosted/test/' | cpio -pdm $(BUILDDIR)
- cp -r client/build/ $(CLIENTBUILD)
- $(call copy_client_assets,$(CLIENTBUILD),$(CZIHOSTEDBUILD))
- cp -r backend/common $(BUILDDIR)/backend/common
- cp backend/__init__.py $(BUILDDIR)
- cp backend/__init__.py $(BUILDDIR)/backend
- cp MANIFEST_hosted.in README.md setup.cfg setup_hosted.py $(BUILDDIR)
- mv $(BUILDDIR)/setup_hosted.py $(BUILDDIR)/setup.py
- mv $(BUILDDIR)/MANIFEST_hosted.in $(BUILDDIR)/MANIFEST.in
-
# If you are actively developing in the server folder use this, dirties the source tree
.PHONY: build-for-server-dev
build-for-server-dev: clean-server build-client
$(call copy_client_assets,client/build,backend/server)
-.PHONY: build-for-czi-hosted-dev
-build-for-czi-hosted-dev: clean-czi-hosted build-client
- $(call copy_client_assets,client/build,backend/czi_hosted)
-
.PHONY: copy-client-assets
copy-client-assets:
$(call copy_client_assets,client/build,backend/server)
.PHONY: copy-client-assets-czi-hosted
copy-client-assets-czi-hosted:
- $(call copy_client_assets,client/build,backend/czi_hosted)
+ $(call copy_client_assets,client/build)
# TESTING
.PHONY: test
@@ -84,17 +64,10 @@ unit-test: unit-test-server unit-test-client unit-test-common
.PHONY: test-server
test-server: unit-test-server smoke-test
-.PHONY: test-czi-hosted
-test-czi-hosted: unit-test-czi-hosted smoke-test
-
.PHONY: unit-test-client
unit-test-client:
cd client && $(MAKE) unit-test
-.PHONY: unit-test-czi-hosted
-unit-test-czi-hosted:
- cd backend/czi_hosted && $(MAKE) unit-test
-
.PHONY: unit-test-server
unit-test-server:
cd backend/server && $(MAKE) unit-test
@@ -111,10 +84,6 @@ smoke-test:
smoke-test-annotations:
cd client && $(MAKE) smoke-test-annotations
-.PHONY: test-db
-test-db:
- cd backend/czi_hosted && $(MAKE) test-db
-
# FORMATTING CODE
.PHONY: fmt
@@ -129,19 +98,13 @@ fmt-py:
black .
.PHONY: lint
-lint: lint-servers lint-client
+lint: lint-server lint-client
-.PHONY: lint-servers
-lint-servers: lint-server lint-czi-hosted-server
.PHONY: lint-server
lint-server: fmt-py
flake8 backend/server --per-file-ignores='backend/test/fixtures/dataset_config_outline.py:F821 backend/test/fixtures/server_config_outline.py:F821 backend/server/test/performance/scale_test_annotations.py:E501'
-.PHONY: lint-czi-hosted-server
-lint-czi-hosted-server: fmt-py
- flake8 backend/czi_hosted --per-file-ignores='backend/test/fixtures/czi_hosted_dataset_config_outline.py:F821 backend/test/fixtures/czi_hosted_server_config_outline.py:F821 backend/test/performance/scale_test_annotations.py:E501'
-
.PHONY: lint-client
lint-client:
cd client && $(MAKE) lint
@@ -153,12 +116,6 @@ pydist: build
cd $(BUILDDIR); python setup.py sdist -d ../dist
@echo "done"
-.PHONY: pydist-czi-hosted
-pydist-czi-hosted: build-czi-hosted
- cd $(BUILDDIR); python setup.py sdist -d ../dist
- @echo "done"
-
-
# RELEASE HELPERS
# Create new version to commit to main
@@ -208,15 +165,6 @@ dev-env-client:
dev-env-server:
pip install -r backend/server/requirements-dev.txt
-.PHONY: dev-env-czi-hosted
-dev-env-czi-hosted:
- pip install -r backend/czi_hosted/requirements-dev.txt
-# Set PART=[major, minor, patch] as param to make bump.
-# This will create a release candidate. (i.e. 0.16.1 -> 0.16.2-rc.0 for a patch bump)
-.PHONY: bump-version
-bump-version:
- bumpversion --config-file .bumpversion.cfg $(PART)
-
# Increments the release candidate version (i.e. 0.16.2-rc.1 -> 0.16.2-rc.2)
.PHONY: bump-release-candidate
bump-release-candidate:
diff --git a/backend/czi_hosted/Makefile b/backend/czi_hosted/Makefile
deleted file mode 100644
index 538c68ff..00000000
--- a/backend/czi_hosted/Makefile
+++ /dev/null
@@ -1,49 +0,0 @@
-include ../../common.mk
-
-.PHONY: clean
-clean:
- rm -f common/web/templates/index.html
- rm -rf common/web/static
- rm -f common/web/csp-hashes.json
-
-.PHONY: unit-test
-unit-test: create-test-db
- PYTHONWARNINGS=ignore:ResourceWarning coverage run \
- --source=app,auth,cli,common,compute,converters,data_anndata,data_common,data_cxg,eb \
- --omit=.coverage,venv \
- -m unittest discover \
- --start-directory ../test/test_czi_hosted/unit \
- --top-level-directory ../.. \
- --verbose; test_result=$$?; \
- $(MAKE) clean-test-db; \
- exit $$test_result \
-
-
-.PHONY: test-db
-test-db: create-test-db
- PYTHONWARNINGS=ignore:ResourceWarning coverage run \
- --source=db \
- --omit=.coverage,venv \
- -m unittest discover \
- --start-directory ../test/test_czi_hosted/test_database \
- --top-level-directory ../.. \
- --verbose; test_result=$$?; \
- $(MAKE) clean-test-db; \
- exit $$test_result
-
-.PHONY: create-test-db
-create-test-db:
- -docker run -d -p 5432:5432 --name test_db -e POSTGRES_PASSWORD=test_pw postgres
-
-.PHONY: clean-test-db
-clean-test-db:
- -docker stop test_db
- -docker rm test_db
-
-.PHONY: test-annotations-performance
-test-annotations-performance:
- python ../test/test_czi_hosted/performance/performance_test_annotations_backend.py
-
-.PHONY: test-annotations-scale
-test-annotations-scale:
- locust -f ../test/test_czi_hosted/performance/scale_test_annotations.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt
diff --git a/backend/czi_hosted/__init__.py b/backend/czi_hosted/__init__.py
deleted file mode 100644
index 053bc062..00000000
--- a/backend/czi_hosted/__init__.py
+++ /dev/null
@@ -1,15 +0,0 @@
-import logging
-import sys
-from backend.common.utils.utils import import_plugins
-
-__version__ = "0.16.7"
-
-
-display_version = "cellxgene v" + __version__
-
-try:
- import_plugins("backend.czi_hosted.plugins")
-except Exception as e:
- # Make sure to exit in this case, as the server may not be configured as expected.
- logging.critical(f"Error in import_plugins: {str(e)}")
- sys.exit(1)
diff --git a/backend/czi_hosted/__main__.py b/backend/czi_hosted/__main__.py
deleted file mode 100644
index 0ffb94d1..00000000
--- a/backend/czi_hosted/__main__.py
+++ /dev/null
@@ -1,14 +0,0 @@
-# Work around bug https://github.com/pallets/werkzeug/issues/461
-if __package__ is None:
- import sys
- from pathlib import Path
-
- PKG_PATH = Path(__file__).parent
- sys.path.insert(0, str(PKG_PATH.parent))
- import backend.czi_hosted # noqa F401
- __package__ = PKG_PATH.name
-
-# Main thing
-from .cli.cli import cli # noqa F402
-
-cli()
diff --git a/backend/czi_hosted/app/__init__.py b/backend/czi_hosted/app/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/backend/czi_hosted/app/app.py b/backend/czi_hosted/app/app.py
deleted file mode 100644
index 32a837c2..00000000
--- a/backend/czi_hosted/app/app.py
+++ /dev/null
@@ -1,475 +0,0 @@
-import datetime
-import logging
-from functools import wraps
-from http import HTTPStatus
-from urllib.parse import urlparse
-import hashlib
-import os
-
-from flask import (
- Flask,
- redirect,
- current_app,
- make_response,
- render_template,
- abort,
- Blueprint,
- request,
- send_from_directory,
-)
-from flask_restful import Api, Resource
-from server_timing import Timing as ServerTiming
-
-import backend.czi_hosted.common.rest as common_rest
-from backend.common.utils.data_locator import DataLocator
-from backend.common.errors import DatasetAccessError, RequestException
-from backend.czi_hosted.common.health import health_check
-from backend.common.utils.utils import path_join, StrictJSONEncoder
-from backend.czi_hosted.data_common.matrix_loader import MatrixDataLoader
-
-webbp = Blueprint("webapp", "backend.czi_hosted.common.web", template_folder="templates")
-
-ONE_WEEK = 7 * 24 * 60 * 60
-
-
-def _cache_control(always, **cache_kwargs):
- """
- Used to easily manage cache control headers on responses.
- See Werkzeug for attributes that can be set, eg, no_cache, private, max_age, etc.
- https://werkzeug.palletsprojects.com/en/1.0.x/datastructures/#werkzeug.datastructures.ResponseCacheControl
- """
-
- def inner_cache_control(f):
- @wraps(f)
- def wrapper(*args, **kwargs):
- response = make_response(f(*args, **kwargs))
- if not always and not current_app.app_config.server_config.app__generate_cache_control_headers:
- return response
- if response.status_code >= 400:
- return response
- for k, v in cache_kwargs.items():
- setattr(response.cache_control, k, v)
- return response
-
- return wrapper
-
- return inner_cache_control
-
-
-def cache_control(**cache_kwargs):
- """ config driven """
- return _cache_control(False, **cache_kwargs)
-
-
-def cache_control_always(**cache_kwargs):
- """ always generate headers, regardless of the config """
- return _cache_control(True, **cache_kwargs)
-
-
-# tell the client not to cache the index.html page so that changes to the app work on redeployment
-# note that the bulk of the data needed by the client (datasets) will still be cached
-@webbp.route("/", methods=["GET"])
-@cache_control_always(public=True, max_age=0, no_store=True, no_cache=True, must_revalidate=True)
-def dataset_index(url_dataroot=None, dataset=None):
- app_config = current_app.app_config
- server_config = app_config.server_config
- if dataset is None:
- if app_config.is_multi_dataset():
- return dataroot_index()
- else:
- location = server_config.single_dataset__datapath
- else:
- dataroot = None
- for key, dataroot_dict in server_config.multi_dataset__dataroot.items():
- if dataroot_dict["base_url"] == url_dataroot:
- dataroot = dataroot_dict["dataroot"]
- break
- if dataroot is None:
- abort(HTTPStatus.NOT_FOUND)
- location = path_join(dataroot, dataset)
-
- dataset_config = app_config.get_dataset_config(url_dataroot)
- scripts = dataset_config.app__scripts
- inline_scripts = dataset_config.app__inline_scripts
-
- try:
- cache_manager = current_app.matrix_data_cache_manager
- with cache_manager.data_adaptor(url_dataroot, location, app_config) as data_adaptor:
- data_adaptor.set_uri_path(f"{url_dataroot}/{dataset}")
- args = {"SCRIPTS": scripts, "INLINE_SCRIPTS": inline_scripts}
- return render_template("index.html", **args)
-
- except DatasetAccessError as e:
- return common_rest.abort_and_log(
- e.status_code, f"Invalid dataset {dataset}: {e.message}", loglevel=logging.INFO, include_exc_info=True
- )
-
-
-@webbp.errorhandler(RequestException)
-def handle_request_exception(error):
- return common_rest.abort_and_log(error.status_code, error.message, loglevel=logging.INFO, include_exc_info=True)
-
-
-def get_data_adaptor(url_dataroot=None, dataset=None):
- config = current_app.app_config
- server_config = config.server_config
- dataset_key = None
-
- if dataset is None:
- datapath = server_config.single_dataset__datapath
- else:
- dataroot = None
- for key, dataroot_dict in server_config.multi_dataset__dataroot.items():
- if dataroot_dict["base_url"] == url_dataroot:
- dataroot = dataroot_dict["dataroot"]
- dataset_key = key
- break
-
- if dataroot is None:
- raise DatasetAccessError(f"Invalid dataset {url_dataroot}/{dataset}")
- datapath = path_join(dataroot, dataset)
- # path_join returns a normalized path. Therefore it is
- # sufficient to check that the datapath starts with the
- # dataroot to determine that the datapath is under the dataroot.
- if not datapath.startswith(dataroot):
- raise DatasetAccessError(f"Invalid dataset {url_dataroot}/{dataset}")
-
- if datapath is None:
- return common_rest.abort_and_log(HTTPStatus.BAD_REQUEST, "Invalid dataset NONE", loglevel=logging.INFO)
-
- cache_manager = current_app.matrix_data_cache_manager
- return cache_manager.data_adaptor(dataset_key, datapath, config)
-
-
-def requires_authentication(func):
- @wraps(func)
- def wrapped_function(self, *args, **kwargs):
- auth = current_app.auth
- if auth.is_user_authenticated():
- return func(self, *args, **kwargs)
- else:
- return make_response("not authenticated", HTTPStatus.UNAUTHORIZED)
-
- return wrapped_function
-
-
-def rest_get_data_adaptor(func):
- @wraps(func)
- def wrapped_function(self, dataset=None):
- try:
- with get_data_adaptor(self.url_dataroot, dataset) as data_adaptor:
- data_adaptor.set_uri_path(f"{self.url_dataroot}/{dataset}")
- return func(self, data_adaptor)
- except DatasetAccessError as e:
- return common_rest.abort_and_log(
- e.status_code, f"Invalid dataset {dataset}: {e.message}", loglevel=logging.INFO, include_exc_info=True
- )
-
- return wrapped_function
-
-
-def dataroot_test_index():
- # the following index page is meant for testing/debugging purposes
- data = ''
- data += "Hosted Cellxgene"
- data += "Welcome to cellxgene
"
-
- config = current_app.app_config
- server_config = config.server_config
-
- auth = server_config.auth
- if auth.is_valid_authentication_type():
- if server_config.auth.is_user_authenticated():
- data += f"Logged in as {auth.get_user_id()} / {auth.get_user_name()} / {auth.get_user_email()}
"
- if auth.requires_client_login():
- if server_config.auth.is_user_authenticated():
- data += f"Logout
"
- else:
- data += f"Login
"
-
- datasets = []
- for dataroot_dict in server_config.multi_dataset__dataroot.values():
- dataroot = dataroot_dict["dataroot"]
- url_dataroot = dataroot_dict["base_url"]
- locator = DataLocator(dataroot, region_name=server_config.data_locator__s3__region_name)
- for fname in locator.ls():
- location = path_join(dataroot, fname)
- try:
- MatrixDataLoader(location, app_config=config)
- datasets.append((url_dataroot, fname))
- except DatasetAccessError:
- # skip over invalid datasets
- pass
-
- data += "
Select one of these datasets...
"
- data += ""
- datasets.sort()
- for url_dataroot, dataset in datasets:
- data += f"- {dataset}
"
- data += "
"
- data += ""
-
- return make_response(data)
-
-
-def dataroot_index():
- # Handle the base url for the cellxgene server when running in multi dataset mode
- config = current_app.app_config
- if not config.server_config.multi_dataset__index:
- abort(HTTPStatus.NOT_FOUND)
- elif config.server_config.multi_dataset__index is True:
- return dataroot_test_index()
- else:
- return redirect(config.server_config.multi_dataset__index)
-
-
-class HealthAPI(Resource):
- @cache_control(no_store=True)
- def get(self):
- config = current_app.app_config
- return health_check(config)
-
-
-class DatasetResource(Resource):
- """Base class for all Resources that act on datasets."""
-
- def __init__(self, url_dataroot):
- super().__init__()
- self.url_dataroot = url_dataroot
-
-
-class SchemaAPI(DatasetResource):
- # TODO @mdunitz separate dataset schema and user schema
- @cache_control(public=True, max_age=ONE_WEEK)
- @rest_get_data_adaptor
- def get(self, data_adaptor):
- return common_rest.schema_get(data_adaptor)
-
-
-class ConfigAPI(DatasetResource):
- @cache_control(public=True, max_age=ONE_WEEK)
- @rest_get_data_adaptor
- def get(self, data_adaptor):
- return common_rest.config_get(current_app.app_config, data_adaptor)
-
-
-class UserInfoAPI(DatasetResource):
- @cache_control_always(no_store=True)
- @rest_get_data_adaptor
- def get(self, data_adaptor):
- return common_rest.userinfo_get(current_app.app_config, data_adaptor)
-
-
-class AnnotationsObsAPI(DatasetResource):
- @cache_control(public=True, max_age=ONE_WEEK)
- @rest_get_data_adaptor
- def get(self, data_adaptor):
- return common_rest.annotations_obs_get(request, data_adaptor)
-
- @requires_authentication
- @cache_control(no_store=True)
- @rest_get_data_adaptor
- def put(self, data_adaptor):
- return common_rest.annotations_obs_put(request, data_adaptor)
-
-
-class AnnotationsVarAPI(DatasetResource):
- @cache_control(public=True, max_age=ONE_WEEK)
- @rest_get_data_adaptor
- def get(self, data_adaptor):
- return common_rest.annotations_var_get(request, data_adaptor)
-
-
-class DataVarAPI(DatasetResource):
- @cache_control(no_store=True)
- @rest_get_data_adaptor
- def put(self, data_adaptor):
- return common_rest.data_var_put(request, data_adaptor)
-
- @cache_control(public=True, max_age=ONE_WEEK)
- @rest_get_data_adaptor
- def get(self, data_adaptor):
- return common_rest.data_var_get(request, data_adaptor)
-
-
-class ColorsAPI(DatasetResource):
- @cache_control(public=True, max_age=ONE_WEEK)
- @rest_get_data_adaptor
- def get(self, data_adaptor):
- return common_rest.colors_get(data_adaptor)
-
-
-class DiffExpObsAPI(DatasetResource):
- @cache_control(no_store=True)
- @rest_get_data_adaptor
- def post(self, data_adaptor):
- return common_rest.diffexp_obs_post(request, data_adaptor)
-
-
-class LayoutObsAPI(DatasetResource):
- @cache_control(public=True, max_age=ONE_WEEK)
- @rest_get_data_adaptor
- def get(self, data_adaptor):
- return common_rest.layout_obs_get(request, data_adaptor)
-
-
-class GenesetsAPI(DatasetResource):
- @cache_control(public=True, max_age=ONE_WEEK)
- @rest_get_data_adaptor
- def get(self, data_adaptor):
- return common_rest.genesets_get(request, data_adaptor)
-
-
-class SummarizeVarAPI(DatasetResource):
- @rest_get_data_adaptor
- @cache_control(public=True, max_age=ONE_WEEK)
- def get(self, data_adaptor):
- return common_rest.summarize_var_get(request, data_adaptor)
-
- @rest_get_data_adaptor
- @cache_control(no_store=True)
- def post(self, data_adaptor):
- return common_rest.summarize_var_post(request, data_adaptor)
-
-
-def get_api_base_resources(bp_base):
- """Add resources that are accessed from the api_base_url"""
- api = Api(bp_base)
-
- # Diagnostics routes
- api.add_resource(HealthAPI, "/health")
- return api
-
-
-def get_api_dataroot_resources(bp_dataroot, url_dataroot=None):
- """Add resources that refer to a dataset"""
- api = Api(bp_dataroot)
-
- def add_resource(resource, url):
- """convenience function to make the outer function less verbose"""
- api.add_resource(resource, url, resource_class_args=(url_dataroot,))
-
- # Initialization routes
- add_resource(SchemaAPI, "/schema")
- add_resource(ConfigAPI, "/config")
- add_resource(UserInfoAPI, "/userinfo")
- # Data routes
- add_resource(AnnotationsObsAPI, "/annotations/obs")
- add_resource(AnnotationsVarAPI, "/annotations/var")
- add_resource(DataVarAPI, "/data/var")
- add_resource(GenesetsAPI, "/genesets")
- add_resource(SummarizeVarAPI, "/summarize/var")
- # Display routes
- add_resource(ColorsAPI, "/colors")
- # Computation routes
- add_resource(DiffExpObsAPI, "/diffexp/obs")
- add_resource(LayoutObsAPI, "/layout/obs")
- return api
-
-
-def handle_api_base_url(app, app_config):
- """If an api_base_url is provided, then an inline script is generated to
- handle the new API prefix"""
- api_base_url = app_config.server_config.get_api_base_url()
- if not api_base_url:
- return
-
- sha256 = hashlib.sha256(api_base_url.encode()).hexdigest()
- script_name = f"api_base_url-{sha256}.js"
- script_path = os.path.join(app.root_path, "../common/web/templates", script_name)
- with open(script_path, "w") as fout:
- fout.write("window.CELLXGENE.API.prefix = `" + api_base_url + "${location.pathname}api/`;\n")
-
- dataset_configs = [app_config.default_dataset_config] + list(app_config.dataroot_config.values())
- for dataset_config in dataset_configs:
- inline_scripts = dataset_config.app__inline_scripts
- inline_scripts.append(script_name)
-
-
-class Server:
- @staticmethod
- def _before_adding_routes(app, app_config):
- """ will be called before routes are added, during __init__. Subclass protocol """
- pass
-
- def __init__(self, app_config):
- self.app = Flask(__name__, static_folder=None)
- handle_api_base_url(self.app, app_config)
- self._before_adding_routes(self.app, app_config)
- self.app.json_encoder = StrictJSONEncoder
- server_config = app_config.server_config
- if server_config.app__server_timing_headers:
- ServerTiming(self.app, force_debug=True)
-
- # enable session data
- self.app.permanent_session_lifetime = datetime.timedelta(days=50 * 365)
-
- # Config
- secret_key = server_config.app__flask_secret_key
- self.app.config.update(SECRET_KEY=secret_key)
-
- self.app.register_blueprint(webbp)
-
- api_version = "/api/v0.2"
- api_base_url = server_config.get_api_base_url()
- api_path = "/"
- if api_base_url:
- parse = urlparse(api_base_url)
- api_path = parse.path
-
- bp_base = Blueprint("bp_base", __name__, url_prefix=api_path)
- base_resources = get_api_base_resources(bp_base)
- self.app.register_blueprint(base_resources.blueprint)
-
- if app_config.is_multi_dataset():
- # NOTE: These routes only allow the dataset to be in the directory
- # of the dataroot, and not a subdirectory. We may want to change
- # the route format at some point
- for dataroot_dict in server_config.multi_dataset__dataroot.values():
- url_dataroot = dataroot_dict["base_url"]
- bp_dataroot = Blueprint(
- f"api_dataset_{url_dataroot}",
- __name__,
- url_prefix=f"{api_path}/{url_dataroot}/" + api_version,
- )
- dataroot_resources = get_api_dataroot_resources(bp_dataroot, url_dataroot)
- self.app.register_blueprint(dataroot_resources.blueprint)
-
- self.app.add_url_rule(
- f"/{url_dataroot}/",
- f"dataset_index_{url_dataroot}",
- lambda dataset, url_dataroot=url_dataroot: dataset_index(url_dataroot, dataset),
- methods=["GET"],
- )
- self.app.add_url_rule(
- f"/{url_dataroot}//",
- f"dataset_index_{url_dataroot}/",
- lambda dataset, url_dataroot=url_dataroot: dataset_index(url_dataroot, dataset),
- methods=["GET"],
- )
- self.app.add_url_rule(
- f"/{url_dataroot}//static/",
- f"static_assets_{url_dataroot}",
- view_func=lambda dataset, filename: send_from_directory("../common/web/static", filename),
- methods=["GET"],
- )
-
- else:
- bp_api = Blueprint("api", __name__, url_prefix=f"{api_path}{api_version}")
- resources = get_api_dataroot_resources(bp_api)
- self.app.register_blueprint(resources.blueprint)
- self.app.add_url_rule(
- "/static/",
- "static_assets",
- view_func=lambda filename: send_from_directory("../common/web/static", filename),
- methods=["GET"],
- )
-
- self.app.matrix_data_cache_manager = server_config.matrix_data_cache_manager
- self.app.app_config = app_config
-
- auth = server_config.auth
- self.app.auth = auth
- if auth and auth.requires_client_login():
- auth.add_url_rules(self.app)
- auth.complete_setup(self.app)
diff --git a/backend/czi_hosted/auth/__init__.py b/backend/czi_hosted/auth/__init__.py
deleted file mode 100644
index 5e24ac8f..00000000
--- a/backend/czi_hosted/auth/__init__.py
+++ /dev/null
@@ -1,6 +0,0 @@
-# import the built in auth types so they can be registered
-
-import backend.czi_hosted.auth.auth_test # noqa: F401
-import backend.czi_hosted.auth.auth_session # noqa: F401
-import backend.czi_hosted.auth.auth_oauth # noqa: F401
-import backend.czi_hosted.auth.auth_none # noqa: F401
diff --git a/backend/czi_hosted/auth/auth.py b/backend/czi_hosted/auth/auth.py
deleted file mode 100644
index 03184e8d..00000000
--- a/backend/czi_hosted/auth/auth.py
+++ /dev/null
@@ -1,91 +0,0 @@
-from abc import ABC, abstractmethod
-
-
-class AuthTypeBase(ABC):
- """Base type for all authentication types."""
-
- def __init__(self):
- super().__init__()
-
- @abstractmethod
- def is_valid_authentication_type(self):
- """Return True if the auth type is valid, e.g. it can return userinfo and username.
- (AuthTypeNone is the only one type that returns False)"""
- pass
-
- def requires_client_login(self):
- """Return True if the user needs to login from the client (e.g. Login button is shown)"""
- return False
-
- @abstractmethod
- def complete_setup(self, app):
- """complete any setup that may be needed by this auth type. The Flask app is passed in.
- This is the last auth function called before the server starts to run."""
- pass
-
- @abstractmethod
- def is_user_authenticated(self):
- """Return True if the user is authenticated"""
- pass
-
- @abstractmethod
- def get_user_id(self):
- """Return the id for this user (string)"""
- pass
-
- @abstractmethod
- def get_user_name(self):
- """Return the name of the user (string)"""
- pass
-
- @abstractmethod
- def get_user_email(self):
- """Return the name of the user (string)"""
- pass
-
- def get_user_picture(self):
- """Return the location to the user's picture"""
- return None
-
-
-class AuthTypeClientBase(AuthTypeBase):
- """Base type for all authentication types that require the client to login"""
-
- def __init__(self):
- super().__init__()
-
- def requires_client_login(self):
- return True
-
- @abstractmethod
- def add_url_rules(self, selfapp):
- """Add url rules to the app (like /login, /logout, etc)"""
- pass
-
- @abstractmethod
- def get_login_url(self, data_adaptor):
- """Return the url for the login route"""
- pass
-
- @abstractmethod
- def get_logout_url(self, data_adaptor):
- """Return the url for the logout route"""
- pass
-
-
-class AuthTypeFactory:
- """Factory class to create an authentication type"""
-
- auth_types = {}
-
- @staticmethod
- def register(name, auth_type):
- assert issubclass(auth_type, AuthTypeBase)
- AuthTypeFactory.auth_types[name] = auth_type
-
- @staticmethod
- def create(name, app_config):
- auth_type = AuthTypeFactory.auth_types.get(name)
- if auth_type is None:
- return None
- return auth_type(app_config)
diff --git a/backend/czi_hosted/auth/auth_none.py b/backend/czi_hosted/auth/auth_none.py
deleted file mode 100644
index 491bed4b..00000000
--- a/backend/czi_hosted/auth/auth_none.py
+++ /dev/null
@@ -1,27 +0,0 @@
-from backend.czi_hosted.auth.auth import AuthTypeBase, AuthTypeFactory
-
-
-class AuthTypeNone(AuthTypeBase):
- def __init__(self, app_config):
- super().__init__()
-
- def is_valid_authentication_type(self):
- return False
-
- def complete_setup(self, app):
- pass
-
- def is_user_authenticated(self):
- return True
-
- def get_user_id(self):
- return None
-
- def get_user_name(self):
- return None
-
- def get_user_email(self):
- return None
-
-
-AuthTypeFactory.register(None, AuthTypeNone)
diff --git a/backend/czi_hosted/auth/auth_oauth.py b/backend/czi_hosted/auth/auth_oauth.py
deleted file mode 100644
index 74d09432..00000000
--- a/backend/czi_hosted/auth/auth_oauth.py
+++ /dev/null
@@ -1,385 +0,0 @@
-from flask import session, request, redirect, current_app, after_this_request, has_request_context, g
-from backend.czi_hosted.auth.auth import AuthTypeClientBase, AuthTypeFactory
-from backend.common.errors import AuthenticationError, ConfigurationError
-from urllib.parse import urlencode, urlparse
-import json
-import requests
-import base64
-
-# It is not required to have authlib or jose.
-# However, it is a configuration error to use this auth type if they are not installed.
-missingimport = []
-try:
- from authlib.integrations.flask_client import OAuth
-except ModuleNotFoundError:
- missingimport.append("authlib")
-
-try:
- from jose import jwt
- from jose.exceptions import ExpiredSignatureError, JWTError, JWTClaimsError
-except ModuleNotFoundError:
- missingimport.append("jose")
-
-
-class Tokens:
- """Simple class to represent the tokens that are saved/restored from the cookie"""
-
- def __init__(self, access_token, id_token, refresh_token, expires_at, **kwargs):
- self.access_token = access_token
- self.id_token = id_token
- self.refresh_token = refresh_token
- self.expires_at = expires_at
-
- # expires_at may be None after a token refresh, and so it is not checked here
- if not (access_token and id_token and refresh_token):
- raise KeyError(str(self.__dict__))
-
-
-class AuthTypeOAuth(AuthTypeClientBase):
- """An authentication type for oauth2 logins."""
-
- CXG_TOKENS = "auth_tokens"
-
- def __init__(self, server_config):
- super().__init__()
- if missingimport:
- raise ConfigurationError(f"oauth requires these modules: {', '.join(missingimport)}")
- self.algorithms = ["RS256"]
- self.oauth_api_base_url = server_config.authentication__params_oauth__oauth_api_base_url
- self.client_id = server_config.authentication__params_oauth__client_id
- self.client_secret = server_config.authentication__params_oauth__client_secret
- self.session_cookie = server_config.authentication__params_oauth__session_cookie
- self.cookie_params = server_config.authentication__params_oauth__cookie
- self.jwt_decode_options = server_config.authentication__params_oauth__jwt_decode_options
-
- self._validate_cookie_params()
- self._validate_jwt_decode_options()
-
- self.api_base_url = server_config.get_api_base_url()
- self.web_base_url = server_config.get_web_base_url()
- if self.api_base_url is None:
- raise ConfigurationError("oauth requires the app__api_base_url to be set")
-
- # set the audience
- self.audience = self.client_id
-
- # load the jwks (JSON Web Key Set).
- # The JSON Web Key Set (JWKS) is a set of keys which contains the public keys used to verify
- # any JSON Web Token (JWT) issued by the authorization server and signed using the RS256
- try:
- jwksloc = f"{self.oauth_api_base_url}/.well-known/jwks.json"
- jwksurl = requests.get(jwksloc)
- self.jwks = jwksurl.json()
- except Exception:
- raise ConfigurationError(
- f"error in oauth, api_url_base: {self.oauth_api_base_url}, cannot access {jwksloc}"
- )
-
- def _validate_cookie_params(self):
- """check the cookie_params, and raise a ConfigurationError if there is something wrong"""
- if self.session_cookie:
- return
-
- if not isinstance(self.cookie_params, dict):
- raise ConfigurationError("either session_cookie or cookie must be set")
- valid_keys = {"key", "max_age", "expires", "path", "domain", "secure", "httponly", "samesite"}
- keys = set(self.cookie_params.keys())
- unknown = keys - valid_keys
- if unknown:
- raise ConfigurationError(f"unexpected key in cookie params: {', '.join(unknown)}")
- if "key" not in keys:
- raise ConfigurationError("must have a key (name) in the cookie params")
-
- def _validate_jwt_decode_options(self):
- """check the jwt_decode_options, and raise a ConfigurationError if there is something wrong"""
- if self.jwt_decode_options is None:
- self.jwt_decode_options = {}
- return
-
- valid_keys = {
- "verify_signature",
- "verify_aud",
- "verify_iat",
- "verify_exp",
- "verify_nbf",
- "verify_iss",
- "verify_sub",
- "verify_jti",
- "verify_at_hash",
- "leeway",
- }
- keys = set(self.jwt_decode_options.keys())
- unknown = keys - valid_keys
- if unknown:
- raise ConfigurationError(f"unexpected key in jwt_decode_options: {', '.join(unknown)}")
-
- def is_valid_authentication_type(self):
- return True
-
- def requires_client_login(self):
- return True
-
- def add_url_rules(self, app):
- parse = urlparse(self.api_base_url)
- app.add_url_rule(f"{parse.path}/login", "login", self.login, methods=["GET"])
- app.add_url_rule(f"{parse.path}/logout", "logout", self.logout, methods=["GET"])
- app.add_url_rule(f"{parse.path}/logout_redirect", "logout_redirect", self.logout_redirect, methods=["GET"])
- app.add_url_rule(f"{parse.path}/oauth2/callback", "callback", self.callback, methods=["GET"])
-
- def complete_setup(self, flask_app):
- self.oauth = OAuth(flask_app)
-
- self.client = self.oauth.register(
- "auth0",
- client_id=self.client_id,
- client_secret=self.client_secret,
- api_base_url=self.oauth_api_base_url,
- refresh_token_url=f"{self.oauth_api_base_url}/oauth/token",
- access_token_url=f"{self.oauth_api_base_url}/oauth/token",
- authorize_url=f"{self.oauth_api_base_url}/authorize",
- client_kwargs={"scope": "openid profile email offline_access"},
- )
-
- def is_user_authenticated(self):
- payload = self.get_userinfo()
- return payload is not None
-
- def get_user_id(self):
- payload = self.get_userinfo()
- return payload.get("sub") if payload else None
-
- def get_user_name(self):
- payload = self.get_userinfo()
- return payload.get("name") if payload else None
-
- def get_user_email(self):
- payload = self.get_userinfo()
- return payload.get("email") if payload else None
-
- def get_user_picture(self):
- payload = self.get_userinfo()
- return payload.get("picture") if payload else None
-
- def update_response(self, response):
- response.cache_control.update(dict(public=True, max_age=0, no_store=True, no_cache=True, must_revalidate=True))
-
- def login(self):
- callbackurl = f"{self.api_base_url}/oauth2/callback"
- return_path = request.args.get("dataset", "")
- return_to = f"{self.web_base_url}/{return_path}"
- # save the return path in the session cookie, accessed in the callback function
- session["oauth_callback_redirect"] = return_to
- response = self.client.authorize_redirect(redirect_uri=callbackurl)
- self.update_response(response)
- return response
-
- def logout(self):
- """
- We would like for the user to remain on the same dataset after logout. oauth requires that
- the redirect `returnTo` path be whitelisted by the oauth server, therefore a level of
- indirection is used. We first redirect to a single path "logout_redirect", and logout_redirect
- will redirect the user's browser back to the current page.
- """
- self.remove_tokens()
- redirect_path = request.args.get("dataset", "")
- redirect_to = f"{self.web_base_url}/{redirect_path}"
- session["oauth_logout_redirect"] = redirect_to
-
- return_to = f"{self.api_base_url}/logout_redirect"
- params = {"returnTo": return_to, "client_id": self.client_id}
- response = redirect(self.client.api_base_url + "/v2/logout?" + urlencode(params))
- self.update_response(response)
- return response
-
- def logout_redirect(self):
- oauth_logout_redirect = session.pop("oauth_logout_redirect", "/")
- response = redirect(oauth_logout_redirect)
- self.update_response(response)
- return response
-
- def callback(self):
- data = self.client.authorize_access_token()
- tokens = Tokens(
- access_token=data.get("access_token"),
- id_token=data.get("id_token"),
- refresh_token=data.get("refresh_token"),
- expires_at=data.get("expires_at"),
- )
- self.save_tokens(tokens)
- oauth_callback_redirect = session.pop("oauth_callback_redirect", "/")
- response = redirect(oauth_callback_redirect)
- self.update_response(response)
- return response
-
- def get_tokens(self):
- """Extract the tokens from the cookie, and store them in the flask global context"""
- if "tokens" in g:
- return g.tokens
-
- try:
- if self.session_cookie:
- value = session.get(self.CXG_TOKENS)
- if value:
- g.tokens = Tokens(**value)
- else:
- return None
- else:
- value = request.cookies.get(self.cookie_params["key"])
- if value is None:
- return None
- value = base64.b64decode(value)
- value = json.loads(value)
- g.tokens = Tokens(**value)
-
- except Exception:
- # there are many types of exceptions that can be raise in the above section.
- # It is impractical to list all the exceptions here, since that would be brittle.
- # If an exception occurs, then return None, meaning that no token could be retrieved.
- current_app.logger.warning(f"auth cookie is in the wrong format: {str(value)}")
- g.pop("tokens", None)
- return None
-
- return g.tokens
-
- def save_tokens(self, tokens):
- g.tokens = tokens
- if self.session_cookie:
- session[self.CXG_TOKENS] = tokens.__dict__
- else:
-
- @after_this_request
- def set_cookie(response):
- args = self.cookie_params.copy()
- value = base64.b64encode(json.dumps(tokens.__dict__).encode("utf-8"))
- del args["key"]
- try:
- response.set_cookie(self.cookie_params["key"], value, **args)
- except Exception as e:
- raise AuthenticationError(f"unable to set_cookie {self.cookie_params}") from e
- return response
-
- def remove_tokens(self):
- g.pop("tokens", None)
- if self.session_cookie:
- if self.CXG_TOKENS in session:
- del session[self.CXG_TOKENS]
- else:
-
- @after_this_request
- def remove_cookie(response):
- response.set_cookie(self.cookie_params["key"], "", expires=0)
- self.update_response(response)
- return response
-
- def get_login_url(self, data_adaptor):
- """Return the url for the login route"""
- if data_adaptor and current_app.app_config.is_multi_dataset():
- return f"{self.api_base_url}/login?dataset={data_adaptor.uri_path}/"
- else:
- return f"{self.api_base_url}/login"
-
- def get_logout_url(self, data_adaptor):
- """Return the url for the logout route"""
- if data_adaptor and current_app.app_config.is_multi_dataset():
- return f"{self.api_base_url}/logout?dataset={data_adaptor.uri_path}/"
- else:
- return f"{self.api_base_url}/logout"
-
- def check_jwt_payload(self, id_token):
- try:
- unverified_header = jwt.get_unverified_header(id_token)
- except JWTError:
- return None
-
- rsa_key = {}
- for key in self.jwks["keys"]:
- if key["kid"] == unverified_header["kid"]:
- rsa_key = {
- "kty": key["kty"],
- "kid": key["kid"],
- "use": key["use"],
- "n": key.get("n"),
- "e": key.get("e"),
- }
- if rsa_key:
- try:
- payload = jwt.decode(
- id_token,
- rsa_key,
- algorithms=self.algorithms,
- audience=self.audience,
- issuer=self.oauth_api_base_url + "/",
- options=self.jwt_decode_options,
- )
- return payload
-
- except ExpiredSignatureError:
- # This exception is handled in get_userinfo
- raise
- except JWTClaimsError as e:
- raise AuthenticationError(f"invalid claims {str(e)}") from e
- except JWTError as e:
- raise AuthenticationError(f"invalid signature: {str(e)}") from e
-
- raise AuthenticationError("Unable to find the appropriate key")
-
- def get_userinfo(self):
- if not has_request_context():
- return None
-
- # check if the userinfo has been retrieved already in this request
- if "userinfo" in g:
- return g.get("userinfo")
-
- # if there is no id_token, return None (user is not authenticated)
- tokens = self.get_tokens()
-
- if tokens is None or tokens.id_token is None:
- return None
-
- try:
- # check the jwt payload. This raises an AuthenticationError if the token is not valid.
- # It the token has expired, we attempt to refresh the token
- g.userinfo = self.check_jwt_payload(tokens.id_token)
- return g.userinfo
-
- except ExpiredSignatureError:
- tokens = self.refresh_expired_token(tokens.refresh_token)
- if tokens is None or tokens.id_token is None:
- return None
- else:
- try:
- g.userinfo = self.check_jwt_payload(tokens.id_token)
- return g.userinfo
- except JWTError as e:
- raise AuthenticationError(f"error during token refresh: {str(e)}") from e
-
- except AuthenticationError:
- self.remove_tokens()
- raise
-
- def refresh_expired_token(self, refresh_token):
- params = {
- "grant_type": "refresh_token",
- "client_id": self.client_id,
- "refresh_token": refresh_token,
- "client_secret": self.client_secret,
- }
- headers = {"content-type": "application/x-www-form-urlencoded"}
- request = requests.post(f"{self.oauth_api_base_url}/oauth/token", urlencode(params), headers=headers)
- if request.status_code != 200:
- # unable to refresh the token, log the user out
- self.remove_tokens()
- return None
- data = request.json()
- tokens = Tokens(
- access_token=data.get("access_token"),
- id_token=data.get("id_token"),
- refresh_token=data.get("refresh_token", refresh_token),
- expires_at=data.get("expires_at"),
- )
- self.save_tokens(tokens)
- return tokens
-
-
-AuthTypeFactory.register("oauth", AuthTypeOAuth)
diff --git a/backend/czi_hosted/auth/auth_session.py b/backend/czi_hosted/auth/auth_session.py
deleted file mode 100644
index 4e9818e4..00000000
--- a/backend/czi_hosted/auth/auth_session.py
+++ /dev/null
@@ -1,40 +0,0 @@
-from flask import session
-from uuid import uuid4
-
-from backend.czi_hosted.auth.auth import AuthTypeBase, AuthTypeFactory
-
-
-class AuthTypeSession(AuthTypeBase):
- """Session based authentication. The user is always logged. The user id is a random number
- associated with the session. This is a good choice for desktop servers."""
-
- # key in the session token for userid
- CXGUID = "cxguid"
-
- def __init__(self, app_config):
- super().__init__()
-
- def is_valid_authentication_type(self):
- return True
-
- def complete_setup(self, app):
- pass
-
- def is_user_authenticated(self):
- # always authenticated
- return True
-
- def get_user_id(self):
- if self.CXGUID not in session:
- session[self.CXGUID] = uuid4().hex
- session.permanent = True
- return session[self.CXGUID]
-
- def get_user_name(self):
- return "anonymous"
-
- def get_user_email(self):
- return None
-
-
-AuthTypeFactory.register("session", AuthTypeSession)
diff --git a/backend/czi_hosted/auth/auth_test.py b/backend/czi_hosted/auth/auth_test.py
deleted file mode 100644
index 40a2af0b..00000000
--- a/backend/czi_hosted/auth/auth_test.py
+++ /dev/null
@@ -1,80 +0,0 @@
-from flask import session, request, redirect, current_app
-
-from backend.czi_hosted.auth.auth import AuthTypeClientBase, AuthTypeFactory
-
-
-class AuthTypeTest(AuthTypeClientBase):
- """An authentication type for testing client based logins. When the login route is accessed
- the user is automatically logged in with a default or configured username"""
-
- # key in session token with userid and username
- CXGUID = "cxguid_test"
- CXGUNAME = "cxguname_test"
- CXGUEMAIL = "cxguemail_test"
- CXGUPICTURE = "cxgupicture_test"
-
- def __init__(self, app_config):
- super().__init__()
- self.user_name = "test_account"
- self.user_id = "id0001"
- self.user_email = "test_account@test.com"
- self.user_picture = None
-
- def is_valid_authentication_type(self):
- return True
-
- def requires_client_login(self):
- return True
-
- def add_url_rules(self, app):
- app.add_url_rule("/login", "login", self.login, methods=["GET"])
- app.add_url_rule("/logout", "logout", self.logout, methods=["GET"])
-
- def complete_setup(self, app):
- pass
-
- def is_user_authenticated(self):
- return self.CXGUID in session
-
- def get_user_id(self):
- return session.get(self.CXGUID)
-
- def get_user_name(self):
- return session.get(self.CXGUNAME)
-
- def get_user_email(self):
- return session.get(self.CXGUEMAIL)
-
- def get_user_picture(self):
- return session.get(self.CXGUPICTURE)
-
- def login(self):
- args = request.args
- return_to = args.get("dataset", "/")
- session[self.CXGUID] = args.get("userid", self.user_id)
- session[self.CXGUNAME] = args.get("username", self.user_name)
- session[self.CXGUEMAIL] = args.get("email", self.user_email)
- session[self.CXGUPICTURE] = args.get("picture", self.user_picture)
- return redirect(return_to)
-
- def logout(self):
- session.clear()
- return_to = request.args.get("dataset", "/")
- return redirect(return_to)
-
- def get_login_url(self, data_adaptor):
- """Return the url for the login route"""
- if current_app.app_config.is_multi_dataset():
- return f"/login?dataset={data_adaptor.uri_path}"
- else:
- return "/login"
-
- def get_logout_url(self, data_adaptor):
- """Return the url for the logout route"""
- if current_app.app_config.is_multi_dataset():
- return f"/logout?dataset={data_adaptor.uri_path}"
- else:
- return "/logout"
-
-
-AuthTypeFactory.register("test", AuthTypeTest)
diff --git a/backend/czi_hosted/cli/__init__.py b/backend/czi_hosted/cli/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/backend/czi_hosted/cli/cli.py b/backend/czi_hosted/cli/cli.py
deleted file mode 100644
index f8f7fde6..00000000
--- a/backend/czi_hosted/cli/cli.py
+++ /dev/null
@@ -1,35 +0,0 @@
-import click
-
-from .convert_to_cxg import convert_to_cxg
-from .launch import launch
-from .prepare import prepare
-from .upgrade import log_upgrade_check
-from .schema import schema_cli
-from .. import __version__
-
-
-@click.group(
- name="cellxgene",
- subcommand_metavar="COMMAND ",
- options_metavar="",
- context_settings=dict(max_content_width=85, help_option_names=["-h", "--help"]),
-)
-@click.help_option("--help", "-h", help="Show this message and exit.")
-@click.version_option(
- version=__version__,
- prog_name="cellxgene",
- message="[%(prog)s] Version %(version)s",
- help="Show the software version and exit.",
-)
-@click.option(
- "--upgrade-check/--no-upgrade-check", default=True, show_default=True, help="Check for release upgrades on start.",
-)
-def cli(upgrade_check):
- if upgrade_check:
- log_upgrade_check()
-
-
-cli.add_command(launch)
-cli.add_command(prepare)
-cli.add_command(convert_to_cxg)
-cli.add_command(schema_cli)
diff --git a/backend/czi_hosted/cli/convert_to_cxg.py b/backend/czi_hosted/cli/convert_to_cxg.py
deleted file mode 100644
index 8c5ce2e6..00000000
--- a/backend/czi_hosted/cli/convert_to_cxg.py
+++ /dev/null
@@ -1,133 +0,0 @@
-from os import path
-
-import click
-
-from backend.czi_hosted.converters.h5ad_data_file import H5ADDataFile
-
-
-@click.command(
- name="convert",
- short_help="Converts an H5AD dataset to the CXG format.",
- help="Converts an H5AD dataset to the CXG format. The CXG format is a cellxgene-private data format "
- "that has performance and access characteristics amenable to a multi-dataset, multi-user serving "
- "environment. You will be able to launch the cellxgene using the `cellxgene launch` command as "
- "usually with the generated CXG file.",
-)
-@click.argument(
- "input-file", nargs=1, type=click.Path(exists=True, dir_okay=False),
-)
-@click.option(
- "-o",
- "--output-directory",
- help="Name of the output CXG directory. If not provided, will default to be the input filename with a "
- "CXG extension.",
-)
-@click.option(
- "-b",
- "--backed",
- help="When true, loads the H5AD in file backed mode. This will cause the conversion to be slower, "
- "but will use less memory.",
- default=False,
- show_default=True,
- is_flag=True,
-)
-@click.option(
- "-t",
- "--title",
- help="Human readable dataset title that will be included as metadata about the CXG file. If omitted, "
- "the dataset title will be the filename.",
-)
-@click.option(
- "-a",
- "--about",
- help="A fully qualified URL that provides more information about the dataset and will be included as "
- "metadata about the CXG file.",
-)
-@click.option(
- "-s",
- "--sparse-threshold",
- help="If the dataset's percent of non-zero values falls belows the specified threshold, then the X "
- "array of the dataset will be sparse. Since the default value is 0.0, the default will be to "
- "convert to dense array.",
- default=0.0,
- show_default=True,
-)
-@click.option(
- "--obs-names",
- help="Name to a column in the obs dataframe that will be used as the index for the dataframe instead of "
- "the one designated by the dataframe generated-index.",
-)
-@click.option(
- "--var-names",
- help="Name to a column in the var dataframe that will be used as the index for the dataframe instead of "
- "the one designated by the dataframe generated-index.",
-)
-@click.option(
- "--disable-custom-colors",
- help="When set, conversion process will not extract scanpy-compatible category colors from the H5AD file.",
- default=False,
- show_default=True,
- is_flag=True,
-)
-@click.option(
- "--disable-corpora-schema",
- help="When set, conversion process will neither extract nor store Corpora schema information. See "
- "https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md for more "
- "information.",
- default=False,
- show_default=True,
- is_flag=True,
-)
-@click.option(
- "--overwrite",
- help="When set to true, will overwrite the output file if the output file already exists.",
- default=False,
- show_default=True,
- is_flag=True,
-)
-@click.help_option("--help", "-h", help="Show this message and exit.")
-def convert_to_cxg(
- input_file,
- output_directory,
- backed,
- title,
- about,
- sparse_threshold,
- obs_names,
- var_names,
- disable_custom_colors,
- disable_corpora_schema,
- overwrite,
-):
- """
- Convert a dataset file into CXG.
- """
-
- h5ad_data_file = H5ADDataFile(
- input_file, backed, title, about, obs_names, var_names, use_corpora_schema=not disable_corpora_schema
- )
-
- # Get the directory that will hold all the CXG files
- cxg_output_container = get_output_directory(input_file, output_directory, overwrite)
-
- h5ad_data_file.to_cxg(
- cxg_output_container, sparse_threshold, convert_anndata_colors_to_cxg_colors=not disable_custom_colors
- )
-
-
-def get_output_directory(input_filename, output_directory, should_overwrite):
- """
- Get the name of the CXG output directory to be created/populated during the dataset conversion.
- """
-
- if output_directory and (not path.isdir(output_directory) or (path.isdir(output_directory) and should_overwrite)):
- if output_directory.endswith(".cxg"):
- return output_directory
- return output_directory + ".cxg"
- if output_directory and path.isdir(output_directory) and not should_overwrite:
- raise click.BadParameter(
- f"Output directory {output_directory} already exists. If you'd like to overwrite, then run the command "
- f"with the --overwrite flag."
- )
-
- return path.splitext(input_filename)[0] + ".cxg"
diff --git a/backend/czi_hosted/cli/launch.py b/backend/czi_hosted/cli/launch.py
deleted file mode 100644
index dd025ffc..00000000
--- a/backend/czi_hosted/cli/launch.py
+++ /dev/null
@@ -1,432 +0,0 @@
-import errno
-import functools
-import logging
-import sys
-import webbrowser
-import os
-import click
-from flask_compress import Compress
-from flask_cors import CORS
-
-from backend.czi_hosted.default_config import default_config
-from backend.czi_hosted.app.app import Server
-from backend.czi_hosted.common.config.app_config import AppConfig
-from backend.common.errors import DatasetAccessError, ConfigurationError
-from backend.common.utils.utils import sort_options
-
-
-DEFAULT_CONFIG = AppConfig()
-
-
-def annotation_args(func):
- @click.option(
- "--disable-annotations",
- is_flag=True,
- default=not DEFAULT_CONFIG.default_dataset_config.user_annotations__enable,
- show_default=True,
- help="Disable user annotation of data.",
- )
- @click.option(
- "--annotations-file",
- default=DEFAULT_CONFIG.default_dataset_config.user_annotations__local_file_csv__file,
- show_default=True,
- multiple=False,
- metavar="",
- help="CSV file to initialize editing of existing annotations; will be altered in-place. "
- "Incompatible with --annotations-dir.",
- )
- @click.option(
- "--annotations-dir",
- default=DEFAULT_CONFIG.default_dataset_config.user_annotations__local_file_csv__directory,
- show_default=False,
- multiple=False,
- metavar="",
- help="Directory of where to save output annotations; filename will be specified in the application. "
- "Incompatible with --annotations-file.",
- )
- @functools.wraps(func)
- def wrapper(*args, **kwargs):
- return func(*args, **kwargs)
-
- return wrapper
-
-
-def config_args(func):
- @click.option(
- "--max-category-items",
- default=DEFAULT_CONFIG.default_dataset_config.presentation__max_categories,
- metavar="",
- show_default=True,
- help="Will not display categories with more distinct values than specified.",
- )
- @click.option(
- "--disable-custom-colors",
- is_flag=True,
- default=False,
- show_default=False,
- help="Disable user-defined category-label colors drawn from source data file.",
- )
- @click.option(
- "--diffexp-lfc-cutoff",
- "-de",
- default=DEFAULT_CONFIG.default_dataset_config.diffexp__lfc_cutoff,
- show_default=True,
- metavar="",
- help="Minimum log fold change threshold for differential expression.",
- )
- @click.option(
- "--disable-diffexp",
- is_flag=True,
- default=not DEFAULT_CONFIG.default_dataset_config.diffexp__enable,
- show_default=False,
- help="Disable on-demand differential expression.",
- )
- @click.option(
- "--embedding",
- "-e",
- default=DEFAULT_CONFIG.default_dataset_config.embeddings__names,
- multiple=True,
- show_default=False,
- metavar="",
- help="Embedding name, eg, 'umap'. Repeat option for multiple embeddings. Defaults to all.",
- )
- @functools.wraps(func)
- def wrapper(*args, **kwargs):
- return func(*args, **kwargs)
-
- return wrapper
-
-
-def dataset_args(func):
- @click.option(
- "--obs-names",
- "-obs",
- default=DEFAULT_CONFIG.server_config.single_dataset__obs_names,
- metavar="",
- help="Name of annotation field to use for observations. If not specified cellxgene will use the the obs index.",
- )
- @click.option(
- "--var-names",
- "-var",
- default=DEFAULT_CONFIG.server_config.single_dataset__var_names,
- metavar="",
- help="Name of annotation to use for variables. If not specified cellxgene will use the the var index.",
- )
- @click.option(
- "--backed",
- "-b",
- is_flag=True,
- default=DEFAULT_CONFIG.server_config.adaptor__anndata_adaptor__backed,
- show_default=False,
- help="Load anndata in file-backed mode. " "This may save memory, but may result in slower overall performance.",
- )
- @click.option(
- "--title",
- "-t",
- default=DEFAULT_CONFIG.server_config.single_dataset__title,
- metavar="",
- help="Title to display. If omitted will use file name.",
- )
- @click.option(
- "--about",
- default=DEFAULT_CONFIG.server_config.single_dataset__about,
- metavar="",
- help="URL providing more information about the dataset (hint: must be a fully specified absolute URL).",
- )
- @functools.wraps(func)
- def wrapper(*args, **kwargs):
- return func(*args, **kwargs)
-
- return wrapper
-
-
-def server_args(func):
- @click.option(
- "--debug",
- "-d",
- is_flag=True,
- default=DEFAULT_CONFIG.server_config.app__debug,
- show_default=True,
- help="Run in debug mode. This is helpful for cellxgene developers, "
- "or when you want more information about an error condition.",
- )
- @click.option(
- "--verbose",
- "-v",
- is_flag=True,
- default=DEFAULT_CONFIG.server_config.app__verbose,
- show_default=True,
- help="Provide verbose output, including warnings and all server requests.",
- )
- @click.option(
- "--port",
- "-p",
- metavar="",
- default=DEFAULT_CONFIG.server_config.app__port,
- type=int,
- show_default=True,
- help="Port to run server on. If not specified cellxgene will find an available port.",
- )
- @click.option(
- "--host",
- metavar="",
- default=DEFAULT_CONFIG.server_config.app__host,
- show_default=False,
- help="Host IP address. By default cellxgene will use localhost (e.g. 127.0.0.1).",
- )
- @click.option(
- "--scripts",
- "-s",
- default=DEFAULT_CONFIG.default_dataset_config.app__scripts,
- multiple=True,
- metavar="",
- help="Additional script files to include in HTML page. If not specified, "
- "no additional script files will be included.",
- show_default=False,
- )
- @functools.wraps(func)
- def wrapper(*args, **kwargs):
- return func(*args, **kwargs)
-
- return wrapper
-
-
-def launch_args(func):
- @annotation_args
- @config_args
- @dataset_args
- @server_args
- @click.option(
- "--dataroot",
- default=DEFAULT_CONFIG.server_config.multi_dataset__dataroot,
- metavar="",
- help="Enable cellxgene to serve multiple files. Supply path (local directory or URL)"
- " to folder containing H5AD and/or CXG datasets.",
- hidden=True,
- ) # TODO, unhide when dataroot is supported)
- @click.argument("datapath", required=False, metavar="")
- @click.option(
- "--open",
- "-o",
- "open_browser",
- is_flag=True,
- default=DEFAULT_CONFIG.server_config.app__open_browser,
- show_default=True,
- help="Open web browser after launch.",
- )
- @click.option(
- "--config-file",
- "-c",
- "config_file",
- default=None,
- show_default=True,
- help="Location to yaml file with configuration settings",
- )
- @click.option(
- "--dump-default-config",
- "dump_default_config",
- is_flag=True,
- default=False,
- show_default=True,
- help="Print default configuration settings and exit",
- )
- @click.help_option("--help", "-h", help="Show this message and exit.")
- @functools.wraps(func)
- def wrapper(*args, **kwargs):
- return func(*args, **kwargs)
-
- return wrapper
-
-
-def handle_scripts(scripts):
- if scripts:
- click.echo(
- r"""
- / / /\ \ \__ _ _ __ _ __ (_)_ __ __ _
- \ \/ \/ / _` | '__| '_ \| | '_ \ / _` |
- \ /\ / (_| | | | | | | | | | | (_| |
- \/ \/ \__,_|_| |_| |_|_|_| |_|\__, |
- |___/
- The --scripts flag is intended for developers to include google analytics etc. You could be opening yourself to a
- security risk by including the --scripts flag. Make sure you trust the scripts that you are including.
- """
- )
- scripts_pretty = ", ".join(scripts)
- click.confirm(f"Are you sure you want to inject these scripts: {scripts_pretty}?", abort=True)
-
-
-class CliLaunchServer(Server):
- """
- the CLI runs a local web server, and needs to enable a few more features.
- """
-
- def __init__(self, app_config):
- super().__init__(app_config)
-
- @staticmethod
- def _before_adding_routes(app, app_config):
- app.config["COMPRESS_MIMETYPES"] = [
- "text/html",
- "text/css",
- "text/xml",
- "application/json",
- "application/javascript",
- "application/octet-stream",
- ]
- Compress(app)
- if app_config.server_config.app__debug:
- CORS(app, supports_credentials=True)
-
-
-@sort_options
-@click.command(
- short_help="Launch the cellxgene data viewer. " "Run `cellxgene launch --help` for more information.",
- options_metavar="",
-)
-@launch_args
-def launch(
- datapath,
- dataroot,
- verbose,
- debug,
- open_browser,
- port,
- host,
- embedding,
- obs_names,
- var_names,
- max_category_items,
- disable_custom_colors,
- diffexp_lfc_cutoff,
- title,
- scripts,
- about,
- disable_annotations,
- annotations_file,
- annotations_dir,
- backed,
- disable_diffexp,
- config_file,
- dump_default_config,
-):
- """Launch the cellxgene data viewer.
- This web app lets you explore single-cell expression data.
- Data must be in a format that cellxgene expects.
- Read the "getting started" guide to learn more:
- https://chanzuckerberg.github.io/cellxgene/getting-started.html
-
- Examples:
-
- > cellxgene launch example-dataset/pbmc3k.h5ad --title pbmc3k
-
- > cellxgene launch --title
-
- > cellxgene launch """
-
- # TODO Examples to provide when "--dataroot" is unhidden
- # > cellxgene launch --dataroot example-dataset/
- #
- # > cellxgene launch --dataroot
-
- if dump_default_config:
- print(default_config)
- sys.exit(0)
- # Startup message
- click.echo("[cellxgene] Starting the CLI...")
-
- # app config
- app_config = AppConfig()
- server_config = app_config.server_config
-
- try:
- if config_file:
- app_config.update_from_config_file(config_file)
-
- # Determine which config options were give on the command line.
- # Those will override the ones provided in the config file (if provided).
- cli_config = AppConfig()
- cli_config.update_server_config(
- app__verbose=verbose,
- app__debug=debug,
- app__host=host,
- app__port=port,
- app__open_browser=open_browser,
- single_dataset__datapath=datapath,
- single_dataset__title=title,
- single_dataset__about=about,
- single_dataset__obs_names=obs_names,
- single_dataset__var_names=var_names,
- multi_dataset__dataroot=dataroot,
- adaptor__anndata_adaptor__backed=backed,
- )
- cli_config.update_default_dataset_config(
- app__scripts=scripts,
- user_annotations__enable=not disable_annotations,
- user_annotations__local_file_csv__file=annotations_file,
- user_annotations__local_file_csv__directory=annotations_dir,
- presentation__max_categories=max_category_items,
- presentation__custom_colors=not disable_custom_colors,
- embeddings__names=embedding,
- diffexp__enable=not disable_diffexp,
- diffexp__lfc_cutoff=diffexp_lfc_cutoff,
- )
-
- diff = cli_config.server_config.changes_from_default()
- changes = {key: val for key, val, _ in diff}
- app_config.update_server_config(**changes)
-
- diff = cli_config.default_dataset_config.changes_from_default()
- changes = {key: val for key, val, _ in diff}
- app_config.update_default_dataset_config(**changes)
-
- # process the configuration
- # any errors will be thrown as an exception.
- # any info messages will be passed to the messagefn function.
-
- def messagefn(message):
- click.echo("[cellxgene] " + message)
-
- # Use a default secret if one is not provided
- if not server_config.app__flask_secret_key:
- app_config.update_server_config(app__flask_secret_key="SparkleAndShine")
-
- app_config.complete_config(messagefn)
-
- except (ConfigurationError, DatasetAccessError) as e:
- raise click.ClickException(e)
-
- handle_scripts(scripts)
-
- # create the server
- server = CliLaunchServer(app_config)
-
- if not server_config.app__verbose:
- log = logging.getLogger("werkzeug")
- log.setLevel(logging.ERROR)
-
- cellxgene_url = f"http://{app_config.server_config.app__host}:{app_config.server_config.app__port}"
- if server_config.app__open_browser:
- click.echo(f"[cellxgene] Launching! Opening your browser to {cellxgene_url} now.")
- webbrowser.open(cellxgene_url)
- else:
- click.echo(f"[cellxgene] Launching! Please go to {cellxgene_url} in your browser.")
-
- click.echo("[cellxgene] Type CTRL-C at any time to exit.")
-
- if not server_config.app__verbose:
- f = open(os.devnull, "w")
- sys.stdout = f
-
- try:
- server.app.run(
- host=server_config.app__host,
- debug=server_config.app__debug,
- port=server_config.app__port,
- threaded=not server_config.app__debug,
- use_debugger=False,
- use_reloader=False,
- )
- except OSError as e:
- if e.errno == errno.EADDRINUSE:
- raise click.ClickException("Port is in use, please specify an open port using the --port flag.") from e
- raise
diff --git a/backend/czi_hosted/cli/prepare.py b/backend/czi_hosted/cli/prepare.py
deleted file mode 100644
index 5b6b467e..00000000
--- a/backend/czi_hosted/cli/prepare.py
+++ /dev/null
@@ -1,274 +0,0 @@
-from os.path import expanduser, isdir, isfile, sep, splitext
-
-import click
-import pandas as pd
-from numpy import ndarray, unique
-from scipy.sparse.csc import csc_matrix
-
-from backend.common.utils.utils import sort_options
-
-
-@sort_options
-@click.command(
- short_help="Preprocess data for use with cellxgene. " "Run `cellxgene prepare --help` for more information.",
- options_metavar="",
-)
-@click.argument("data", nargs=1, metavar="", required=True)
-@click.option(
- "--embedding",
- "-e",
- default=["umap", "tsne"],
- multiple=True,
- type=click.Choice(["umap", "tsne"]),
- help="Embedding algorithm(s). Repeat option for multiple embeddings.",
- show_default=True,
-)
-@click.option(
- "--recipe", "-r", default="none", type=click.Choice(["none", "seurat", "zheng17"]), show_default=True,
-)
-@click.option("--output", "-o", default="", help="Save a new file to filename.", metavar="")
-@click.option("--plotting", "-p", default=False, is_flag=True, help="Generate plots.", show_default=True)
-@click.option("--sparse", default=False, is_flag=True, help="Force sparsity.", show_default=True)
-@click.option("--overwrite", default=False, is_flag=True, help="Allow file overwriting.", show_default=True)
-@click.option("--set-obs-names", default="", help="Named field to set as index for obs.", metavar="")
-@click.option("--set-var-names", default="", help="Named field to set as index for var.", metavar="")
-@click.option(
- "--skip-qc",
- default=False,
- is_flag=True,
- help="Do not run quality control metrics. By default cellxgene runs them "
- "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).",
-)
-@click.option(
- "--make-obs-names-unique/--no-make-obs-names-unique",
- default=True,
- help="Ensure obs index is unique.",
- show_default=True,
-)
-@click.option(
- "--make-var-names-unique/--no-make-var-names-unique",
- default=True,
- help="Ensure var index is unique.",
- show_default=True,
-)
-@click.help_option("--help", "-h", help="Show this message and exit.")
-def prepare(
- data,
- embedding,
- recipe,
- output,
- plotting,
- sparse,
- overwrite,
- set_obs_names,
- set_var_names,
- skip_qc,
- make_obs_names_unique,
- make_var_names_unique,
-):
- """
- Preprocess data for use with cellxgene.
- This tool runs a series of scanpy routines for preparing a dataset for use
- with cellxgene. It loads data from different formats
- (h5ad, loom, or a 10x directory), runs dimensionality reduction,
- computes nearest neighbors, computes an embedding, performs clustering,
- and saves the results. Includes additional options for naming annotations,
- ensuring sparsity, and plotting results.
- """
-
- # collect slow imports here to make CLI startup more responsive
- click.echo("[cellxgene] Starting CLI...")
- try:
- import matplotlib
-
- matplotlib.use("Agg")
- import scanpy as sc
- except ImportError:
- raise click.ClickException(
- "[cellxgene] cellxgene prepare has not been installed. Please run `pip install 'cellxgene[prepare]'` "
- "to install the necessary requirements."
- )
-
- # scanpy settings
- sc.settings.verbosity = 0
- sc.settings.autosave = True
-
- # check args
- if sparse and not recipe == "none":
- raise click.UsageError("Cannot use a recipe when forcing sparsity")
-
- output = expanduser(output)
-
- if not output:
- click.echo(
- "Warning: No file will be saved, to save the results of cellxgene prepare include "
- "--output to save output to a new file"
- )
- if isfile(output) and not overwrite:
- raise click.UsageError(f"Cannot overwrite existing file {output}, try using the flag --overwrite")
-
- def load_data(data):
- if isfile(data):
- name, extension = splitext(data)
- if extension == ".h5ad":
- adata = sc.read_h5ad(data)
- elif extension == ".loom":
- adata = sc.read_loom(data)
- else:
- raise click.FileError(data, hint="does not have a valid extension [.h5ad | .loom]")
- elif isdir(data):
- if not data.endswith(sep):
- data += sep
- adata = sc.read_10x_mtx(data)
- else:
- raise click.FileError(data, hint="not a valid file or path")
-
- if not set_obs_names == "":
- if set_obs_names not in adata.obs_keys():
- raise click.UsageError(f"obs {set_obs_names} not found, options are: {adata.obs_keys()}")
- adata.obs_names = adata.obs[set_obs_names]
- if not set_var_names == "":
- if set_var_names not in adata.var_keys():
- raise click.UsageError(f"var {set_var_names} not found, options are: {adata.var_keys()}")
- adata.var_names = adata.var[set_var_names]
- if make_obs_names_unique:
- adata.obs.index = make_index_unique(adata.obs.index)
- if make_var_names_unique:
- adata.var.index = make_index_unique(adata.var.index)
- if not adata._obs.index.is_unique:
- click.echo("Warning: obs index is not unique")
- if not adata._var.index.is_unique:
- click.echo("Warning: var index is not unique")
- return adata
-
- def calculate_qc_metrics(adata):
- if not skip_qc:
- sc.pp.calculate_qc_metrics(adata, inplace=True)
- return adata
-
- def make_sparse(adata):
- if (type(adata.X) is ndarray) and sparse:
- adata.X = csc_matrix(adata.X)
-
- def run_recipe(adata):
- if recipe == "seurat":
- sc.pp.recipe_seurat(adata)
- elif recipe == "zheng17":
- sc.pp.recipe_zheng17(adata)
- else:
- sc.pp.filter_cells(adata, min_genes=5)
- sc.pp.filter_genes(adata, min_cells=25)
- if sparse:
- sc.pp.scale(adata, zero_center=False)
- else:
- sc.pp.scale(adata)
-
- def run_pca(adata):
- if sparse:
- sc.pp.pca(adata, svd_solver="arpack", zero_center=False)
- else:
- sc.pp.pca(adata, svd_solver="arpack")
-
- def run_neighbors(adata):
- sc.pp.neighbors(adata)
-
- def run_louvain(adata):
- sc.tl.louvain(adata)
-
- def run_embedding(adata):
- if len(unique(adata.obs["louvain"].values)) < 10:
- palette = "tab10"
- else:
- palette = "tab20"
-
- if "umap" in embedding:
- sc.tl.umap(adata)
- if plotting:
- sc.pl.umap(adata, color="louvain", palette=palette, save="_louvain")
-
- if "tsne" in embedding:
- sc.tl.tsne(adata)
- if plotting:
- sc.pl.tsne(adata, color="louvain", palette=palette, save="_louvain")
-
- def show_step(item):
- if not skip_qc:
- qc_name = "Calculating QC metrics"
- else:
- qc_name = "Skipping QC"
- names = {
- "calculate_qc_metrics": qc_name,
- "make_sparse": "Ensuring sparsity",
- "run_recipe": f'Running preprocessing recipe "{recipe}"',
- "run_pca": "Running PCA",
- "run_neighbors": "Calculating neighbors",
- "run_louvain": "Calculating clusters",
- "run_embedding": "Computing embedding",
- }
- if item is not None:
- return names[item.__name__]
-
- steps = [calculate_qc_metrics, make_sparse, run_recipe, run_pca, run_neighbors, run_louvain, run_embedding]
-
- click.echo(f"[cellxgene] Loading data from {data}, please wait...")
- adata = load_data(data)
-
- click.echo("[cellxgene] Beginning preprocessing...")
- with click.progressbar(steps, label="[cellxgene] Progress", show_eta=False, item_show_func=show_step) as bar:
- for step in bar:
- step(adata)
-
- # saving
- if not output == "":
- click.echo(f"[cellxgene] Saving results to {output}...")
- adata.write(output)
-
- click.echo("[cellxgene] Success!")
-
-
-# TODO (mweiden): remove this once this issue is resolved https://github.com/theislab/anndata/issues/344
-# Note: tentative solution here https://github.com/theislab/anndata/pull/345
-def make_index_unique(index: pd.Index, join: str = "-"):
- """
- Makes the index unique by appending a number string to each duplicate index element: '1', '2', etc.
-
- If a tentative name created by the algorithm already exists in the index, it tries the next integer in the sequence.
-
- The first occurrence of a non-unique value is ignored.
- Parameters
- ----------
- join
- The connecting string between name and integer.
- Examples
- --------
- >>> from anndata import AnnData
- >>> adata1 = AnnData(np.ones((3, 2)), dict(obs_names=['a', 'b', 'c']))
- >>> adata2 = AnnData(np.zeros((3, 2)), dict(obs_names=['d', 'b', 'b']))
- >>> adata = adata1.concatenate(adata2)
- >>> adata.obs_names
- Index(['a', 'b', 'c', 'd', 'b', 'b'], dtype='object')
- >>> adata.obs_names_make_unique()
- >>> adata.obs_names
- Index(['a', 'b', 'c', 'd', 'b-1', 'b-2'], dtype='object')
- """
- if index.is_unique:
- return index
- from collections import defaultdict
-
- values = index.values
- values_set = set(values)
- indices_dup = index.duplicated(keep="first")
- values_dup = values[indices_dup]
- counter = defaultdict(lambda: 0)
- for i, v in enumerate(values_dup):
- while True:
- counter[v] += 1
- tentative_new_name = v + join + str(counter[v])
- if tentative_new_name not in values_set:
- values_set.add(tentative_new_name)
- values_dup[i] = tentative_new_name
- break
-
- values[indices_dup] = values_dup
- index = pd.Index(values)
- return index
diff --git a/backend/czi_hosted/cli/schema.py b/backend/czi_hosted/cli/schema.py
deleted file mode 100644
index 8e26ff5a..00000000
--- a/backend/czi_hosted/cli/schema.py
+++ /dev/null
@@ -1,72 +0,0 @@
-import click
-
-from backend.czi_hosted.converters.schema import remix, validate
-
-
-@click.group(
- name="schema",
- subcommand_metavar="COMMAND ",
- short_help="Apply and validate the cellxgene data integration schema to an h5ad file.",
- context_settings=dict(max_content_width=85, help_option_names=["-h", "--help"]),
-)
-def schema_cli():
- try:
- import scanpy # noqa: F401
- except ImportError:
- raise click.ClickException(
- "[cellxgene] cellxgene schema requires scanpy"
- )
-
-
-@click.command(
- name="apply",
- short_help="(experimental) Apply the cellxgene data integration schema to an h5ad.",
- help="(experimental) Using a yaml file that describes schema values to insert or convert and in input "
- "h5ad file, apply the schema changes and create a new, conforming h5ad.",
-)
-@click.option(
- "--source-h5ad",
- help="Input h5ad file.",
- nargs=1,
- required=True,
- type=click.Path(exists=True, dir_okay=False),
-)
-@click.option(
- "--remix-config",
- help="Config yaml with information on how to apply the schema.",
- nargs=1,
- required=True,
- type=click.Path(exists=True, dir_okay=False),
-)
-@click.option(
- "--output-filename",
- help="Filename for the new, schema-conforming h5ad file.",
- required=True,
- nargs=1
-)
-def schema_apply(source_h5ad, remix_config, output_filename):
- remix.apply_schema(source_h5ad, remix_config, output_filename)
-
-
-@click.command(
- name="validate",
- short_help="(experimental) Check that an h5ad follows the cellxgene data integration schema.",
-)
-@click.argument(
- "h5ad",
- nargs=1,
- type=click.Path(exists=True, dir_okay=False),
-)
-@click.option(
- "--shallow",
- help="When true, just check that the correct version information is present.",
- default=False,
- show_default=True,
- is_flag=True,
-)
-def schema_validate(h5ad, shallow):
- validate.validate(h5ad, shallow)
-
-
-schema_cli.add_command(schema_apply)
-schema_cli.add_command(schema_validate)
diff --git a/backend/czi_hosted/cli/upgrade.py b/backend/czi_hosted/cli/upgrade.py
deleted file mode 100644
index 222d7e81..00000000
--- a/backend/czi_hosted/cli/upgrade.py
+++ /dev/null
@@ -1,85 +0,0 @@
-import re
-
-import click
-import requests
-from requests.exceptions import ConnectionError
-
-from .. import __version__
-
-# Official SemVer regex: https://semver.org/
-SEMVER_FORMAT = re.compile(
- r"^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*["
- r"a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+("
- r"?:\.[0-9a-zA-Z-]+)*))?$"
-)
-
-
-def log_upgrade_check():
- # Sanity-check that the CLI version is a properly-formatted SemVer string
- assert validate_version_str(__version__, release_only=False)
-
- # Get the current latest release
- try:
- release_tag_generator = (r["tag_name"] for r in _request_cellxgene_releases())
- latest_release = next(release_tag_generator, lambda tag_name: validate_version_str(tag_name))
- if version_gt(latest_release, __version__):
- click.echo(f"There's a new version of cellxgene available ({latest_release})!", err=True)
- click.echo("To upgrade, run the following: pip install --upgrade cellxgene\n", err=True)
- except (ConnectionError, RateLimitException):
- click.echo("Upgrade check failed.\n")
-
-
-class RateLimitException(Exception):
- """
- Github API Rate Limit Exception
- """
-
-
-def _request_cellxgene_releases():
- def raise_on_rate_limit(response):
- if response.status_code == 403 and res.headers.get("X-RateLimit-Remaining") == "0":
- raise RateLimitException
-
- url = "https://api.github.com/repos/chanzuckerberg/cellxgene/releases"
- res = requests.get(url)
- raise_on_rate_limit(res)
- for release in res.json():
- yield release
- while "next" in res.links.keys():
- res = requests.get(res.links["next"]["url"])
- raise_on_rate_limit(res)
- for release in res.json():
- yield release
-
-
-def validate_version_str(version_str, release_only=True):
- """
- Test if a string conforms to SemVer format (https://semver.org/)
- :param version_str: a string to be validated
- :param release_only: only declare releases (not prereleases) valid
- :return: True if the version string is of a valid SemVer format else False
- """
- match = SEMVER_FORMAT.match(version_str)
- has_match = match is not None
- if has_match and release_only:
- return not match.group("prerelease")
- return has_match
-
-
-def split_version(version_string):
- """
- Split a SemVer-formatted string into its component integers
- :param version_string: a SemVer string to be split
- :return: an array of three integers
- """
- match = SEMVER_FORMAT.match(version_string)
- return [int(match.group(group)) for group in ["major", "minor", "patch"]]
-
-
-def version_gt(left_version, right_version):
- for left, right in zip(split_version(left_version), split_version(right_version)):
- if left > right:
- return True
- elif right > left:
- return False
- return False
diff --git a/backend/czi_hosted/common/__init__.py b/backend/czi_hosted/common/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/backend/czi_hosted/common/annotations/__init__.py b/backend/czi_hosted/common/annotations/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/backend/czi_hosted/common/annotations/annotations.py b/backend/czi_hosted/common/annotations/annotations.py
deleted file mode 100644
index 87851f03..00000000
--- a/backend/czi_hosted/common/annotations/annotations.py
+++ /dev/null
@@ -1,110 +0,0 @@
-import os
-
-from flask import current_app, has_request_context
-
-from backend.common.errors import DisabledFeatureError
-from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
-from backend.common.genesets import write_gene_sets_tidycsv, read_gene_sets_tidycsv, validate_gene_sets
-from backend.common.utils.data_locator import DataLocator
-from backend.common.utils.utils import path_join
-
-
-class Annotations:
- """baseclass for annotations and genesets"""
-
- def __init__(self, config={}):
- self.config = config
-
- def user_annotations_enabled(self):
- return self.config.get("user-annotations", False)
-
- def check_user_annotations_enabled(self):
- if not self.user_annotations_enabled():
- raise DisabledFeatureError("User annotations are disabled.")
-
- def get_schema(self, data_adaptor):
- schema = []
- labels = self.read_labels(data_adaptor)
- if labels is not None and not labels.empty:
- for col in labels.columns:
- col_schema = dict(name=col, writable=True)
- col_schema.update(get_schema_type_hint_of_array(labels[col]))
- schema.append(col_schema)
-
- return schema
-
- def set_collection(self, name):
- """set or create a new annotation collection"""
- raise NotImplementedError
-
- def read_labels(self, data_adaptor):
- """Return the labels as a pandas.DataFrame"""
- raise NotImplementedError
-
- def write_labels(self, df, data_adaptor):
- """Write the labels (df) to a persistent storage such that it can later be read"""
- raise NotImplementedError
-
- def update_parameters(self, parameters, data_adaptor):
- """Update configuration parameters that describe information about the annotations feature"""
- params = {}
- params["annotations_genesets_readonly"] = True
- params["annotations_genesets_name_is_read_only"] = True
- parameters.update(params)
-
- @staticmethod
- def gene_sets_to_csv(genesets):
- """
- Convert the internal genesets format (returned by read_gene_set) into
- the simple Tidy CSV.
- """
- from io import StringIO
-
- if isinstance(genesets, dict):
- genesets = genesets.values()
-
- with StringIO() as sio:
- write_gene_sets_tidycsv(sio, genesets)
- return sio.getvalue()
-
- @staticmethod
- def gene_sets_to_response(genesets):
- """
- Convert the internal genesets format (returned by read_gene_set) into
- the dict expected by the JSON REST API
- """
- return list(genesets.values())
-
- def read_gene_sets(self, data_adaptor, context=None):
- if has_request_context():
- if not current_app.auth.is_user_authenticated():
- return ({}, 0)
-
- gene_sets_uri_or_path = dataset_uri_to_geneset_uri(data_adaptor.data_locator.uri_or_path)
-
- server_config = data_adaptor.server_config
- region_name = None if server_config is None else server_config.data_locator__s3__region_name
- gene_sets_locator = DataLocator(gene_sets_uri_or_path, region_name=region_name)
- if not gene_sets_locator.exists():
- return ({}, 0)
-
- gene_sets = read_gene_sets_tidycsv(gene_sets_locator, context)
- schema = data_adaptor.get_schema()
- var_index = schema["annotations"]["var"].get("index", "index")
- var_names = set(data_adaptor.query_var_array(var_index))
-
- gene_sets = validate_gene_sets(gene_sets, var_names)
- return (gene_sets, 0)
-
-
-def dataset_uri_to_geneset_uri(data_uri_or_path):
- """given a dataset URI, return the associated gene set URI"""
- data_basename = os.path.basename(data_uri_or_path)
- base, ext = os.path.splitext(data_basename)
- if ext is not None: # strip extension, if any
- data_basename = base
-
- genesets_basename = f"{data_basename}-genesets.csv"
- gene_sets_uri_or_path = path_join(data_uri_or_path, "..", genesets_basename)
-
- return gene_sets_uri_or_path
diff --git a/backend/czi_hosted/common/annotations/hosted_tiledb.py b/backend/czi_hosted/common/annotations/hosted_tiledb.py
deleted file mode 100644
index 733185a3..00000000
--- a/backend/czi_hosted/common/annotations/hosted_tiledb.py
+++ /dev/null
@@ -1,167 +0,0 @@
-import json
-import os
-import re
-import time
-
-import pandas as pd
-import tiledb
-from flask import current_app
-
-from backend.czi_hosted.common.annotations.annotations import Annotations
-from backend.common.errors import AnnotationCategoryNameError
-from backend.czi_hosted.common.utils.sanitization_utils import sanitize_values_in_list
-from backend.common.utils.type_conversion_utils import get_dtypes_and_schemas_of_dataframe, get_encoding_dtype_of_array
-from backend.czi_hosted.db.cellxgene_orm import Annotation
-
-
-class AnnotationsHostedTileDB(Annotations):
- CXG_ANNO_COLLECTION = "cxg_anno_collection"
-
- def __init__(self, config, directory_path, db):
- super().__init__(config)
- self.db = db
- if directory_path[-1] == "/":
- self.directory_path = directory_path
- else:
- self.directory_path = directory_path + "/"
-
- def check_category_names(self, df):
- original_category_names = df.keys().to_list()
- sanitized_category_names = set(sanitize_values_in_list(original_category_names).values())
- unsanitary_original_category_names = set(original_category_names).difference(sanitized_category_names)
- if unsanitary_original_category_names:
- raise AnnotationCategoryNameError(
- f"{unsanitary_original_category_names} are not valid category names, please resubmit"
- )
-
- def get_user_name(self):
- return current_app.auth.get_user_name()
-
- def get_user_id(self):
- return current_app.auth.get_user_id()
-
- def is_safe_collection_name(self, name):
- """
- return true if this is a safe collection name
- this is ultra conservative. If we want to allow full legal file name syntax,
- we could look at modules like `pathvalidate`
- """
- if name is None:
- return False
- return re.match(r"^[\w\-]+$", name) is not None
-
- def set_collection(self, name):
- self.CXG_ANNO_COLLECTION = name
-
- def read_labels(self, data_adaptor):
- user_id = self.get_user_id()
- if user_id is None:
- return
- dataset_name = data_adaptor.get_location()
- dataset_id = self.db.get_or_create_dataset(dataset_name)
-
- annotation_object = self.db.query_for_most_recent(
- Annotation, [Annotation.user_id == user_id, Annotation.dataset_id == dataset_id]
- )
- if annotation_object:
- if annotation_object.tiledb_uri == "":
- # this mean the user has removed all the categories.
- return None
- try:
- df = tiledb.open(annotation_object.tiledb_uri)
- except tiledb.TileDBError:
- # don't crash if the annotations file is missing or can't be read.
- current_app.logger.warning(f"Cannot read annotation file: {annotation_object.tiledb_uri}")
- return None
- pandas_df = self.convert_to_pandas_df(df, annotation_object.schema_hints)
- return pandas_df
- else:
- return None
-
- def convert_to_pandas_df(self, tileDBArray, schema_hints):
- repr_meta = None
- index_dims = None
- schema_hints = json.loads(schema_hints)
-
- if "__pandas_attribute_repr" in tileDBArray.meta:
- # backwards compatibility... unsure if necessary at this point
- repr_meta = json.loads(tileDBArray.meta["__pandas_attribute_repr"])
- if "__pandas_index_dims" in tileDBArray.meta:
- index_dims = json.loads(tileDBArray.meta["__pandas_index_dims"])
-
- data = tileDBArray[:]
- indexes = list()
-
- for col_name, col_val in data.items():
- # If the column values are byte literals, decode them
- if isinstance(col_val[0], bytes):
- col_val = [value.decode("utf-8") for value in col_val]
-
- if schema_hints and col_name in schema_hints:
- type = schema_hints.get(col_name).get("type")
- if type and type == "categorical":
- new_col = pd.Series(col_val, dtype="category")
- data[col_name] = new_col
- elif repr_meta and col_name in repr_meta:
- new_col = pd.Series(col_val, dtype=repr_meta[col_name])
- data[col_name] = new_col
- elif index_dims and col_name in index_dims:
- new_col = pd.Series(col_val, dtype=index_dims[col_name])
- data[col_name] = new_col
- indexes.append(col_name)
-
- new_df = pd.DataFrame.from_dict(data)
- if len(indexes) > 0:
- new_df.set_index(indexes, inplace=True)
-
- return new_df
-
- def write_labels(self, df, data_adaptor):
- auth_user_id = self.get_user_id()
- user_name = self.get_user_name()
- timestamp = time.time()
- dataset_location = data_adaptor.get_location()
- dataset_id = self.db.get_or_create_dataset(dataset_location)
- dataset_name = data_adaptor.get_title()
- user_id = self.db.get_or_create_user(auth_user_id)
- """
- NOTE: The uri contains the dataset name, user name and a timestamp as a convenience for debugging purposes.
- People may have the same name and time.time() can be server dependent.
- See - https://docs.python.org/2/library/time.html#time.time
-
- The annotations objects in the database should be used as the source of truth about who an annotation belongs
- to (for authorization purposes) and what time it was created (for garbage collection).
- """
- uri = f"{self.directory_path}{dataset_name}/{user_name}/{timestamp}"
- if uri.startswith("s3://"):
- pass
- else:
- os.makedirs(uri, exist_ok=True)
- _, dataframe_schema_type_hints = get_dtypes_and_schemas_of_dataframe(df)
- if not df.empty:
- self.check_category_names(df)
- # convert to tiledb datatypes
-
- for col in df:
- df[col] = df[col].astype(get_encoding_dtype_of_array(df[col]))
- tiledb.from_pandas(uri, df, sparse=True)
- else:
- uri = ""
-
- annotation = Annotation(
- tiledb_uri=uri,
- user_id=user_id,
- dataset_id=str(dataset_id),
- schema_hints=json.dumps(dataframe_schema_type_hints),
- )
- self.db.session.add(annotation)
- self.db.session.commit()
-
- def update_parameters(self, parameters, data_adaptor):
- super().update_parameters(parameters, data_adaptor)
-
- params = {}
- params["annotations"] = True
- params["user_annotation_collection_name_enabled"] = False
-
- parameters.update(params)
diff --git a/backend/czi_hosted/common/annotations/local_file_csv.py b/backend/czi_hosted/common/annotations/local_file_csv.py
deleted file mode 100644
index 6a5d3a16..00000000
--- a/backend/czi_hosted/common/annotations/local_file_csv.py
+++ /dev/null
@@ -1,192 +0,0 @@
-import base64
-import os
-import re
-import threading
-from datetime import datetime
-from hashlib import blake2b
-
-import pandas as pd
-from flask import session, has_request_context, current_app
-
-from backend.czi_hosted import __version__ as cellxgene_version
-from backend.czi_hosted.common.annotations.annotations import Annotations
-from backend.common.errors import AnnotationsError
-
-
-class AnnotationsLocalFile(Annotations):
- CXG_ANNO_COLLECTION = "cxg_anno_collection"
-
- def __init__(self, config, output_dir, output_file):
- super().__init__(config)
- self.output_dir = output_dir
- self.output_file = output_file
- # lock used to protect label file write ops
- self.label_lock = threading.RLock()
-
- # cache the most recent annotations
- self.last_fname = None
- self.last_labels = None
-
- def is_safe_collection_name(self, name):
- """
- return true if this is a safe collection name
- this is ultra conservative. If we want to allow full legal file name syntax,
- we could look at modules like `pathvalidate`
- """
- if name is None:
- return False
- return re.match(r"^[\w\-]+$", name) is not None
-
- def set_collection(self, name):
- session[self.CXG_ANNO_COLLECTION] = name
- session.permanent = True
-
- def get_collection(self):
- if session is None:
- return None
- return session.get(self.CXG_ANNO_COLLECTION)
-
- def read_labels(self, data_adaptor):
- if has_request_context():
- if not current_app.auth.is_user_authenticated():
- return pd.DataFrame()
-
- fname = self._get_filename(data_adaptor)
- with self.label_lock:
- if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0:
- # returned the cached labels if possible, otherwise read them from the file
- if fname == self.last_fname:
- return self.last_labels
- else:
- labels = pd.read_csv(
- fname, dtype="category", index_col=0, header=0, comment="#", keep_default_na=False
- )
- # update the cache
- self.last_fname = fname
- self.last_labels = labels
- return labels
- else:
- return pd.DataFrame()
-
- def write_labels(self, df, data_adaptor):
- # update our internal state and save it. Multi-threading often enabled,
- # so treat this as a critical section.
- with self.label_lock:
- lastmod = data_adaptor.get_last_mod_time()
- lastmodstr = "'unknown'" if lastmod is None else lastmod.isoformat(timespec="seconds")
- header = (
- f"# Annotations generated on {datetime.now().isoformat(timespec='seconds')} "
- f"using cellxgene version {cellxgene_version}\n"
- f"# Input data file was {data_adaptor.get_location()}, "
- f"which was last modified on {lastmodstr}\n"
- )
-
- fname = self._get_filename(data_adaptor)
- self._backup(fname)
- if not df.empty:
- with open(fname, "w", newline="") as f:
- if header is not None:
- f.write(header)
- df.to_csv(f)
- else:
- open(fname, "w").close()
-
- # update the cache
- self.last_fname = fname
- self.last_labels = df
-
- def _get_userdata_idhash(self, data_adaptor):
- """
- Return a short hash that weakly identifies the user and dataset.
- Used to create safe annotations output file names.
- """
- uid = current_app.auth.get_user_id()
- id = (uid + data_adaptor.get_location()).encode()
- idhash = base64.b32encode(blake2b(id, digest_size=5).digest()).decode("utf-8")
- return idhash
-
- def _get_output_dir(self):
- if self.output_dir:
- return self.output_dir
-
- if self.output_file:
- return os.path.dirname(self.path.abspath(self.output_dir))
-
- return os.getcwd()
-
- def _get_filename(self, data_adaptor):
- """return the current annotation file name"""
- if self.output_file:
- return self.output_file
-
- # we need to generate a file name, which we can only do if we have a UID and collection name
- if session is None:
- raise AnnotationsError("unable to determine file name for annotations")
-
- collection = self.get_collection()
- if collection is None:
- return None
-
- if data_adaptor is None:
- raise AnnotationsError("unable to determine file name for annotations")
-
- idhash = self._get_userdata_idhash(data_adaptor)
- return os.path.join(self._get_output_dir(), f"{collection}-{idhash}.csv")
-
- def _backup(self, fname, max_backups=9):
- """
- save N backups of file to backup_dir.
- 1. fname -> backup_dir/fname-TIME
- 2. delete excess files in backup_dir
- """
- root, ext = os.path.splitext(fname)
- backup_dir = f"{root}-backups"
-
- # Make sure there is work to do
- if not os.path.exists(fname):
- return
-
- # Ensure backup_dir exists
- if not os.path.exists(backup_dir):
- os.mkdir(backup_dir)
-
- # Save current file to backup_dir
- fname_base = os.path.basename(fname)
- fname_base_root, fname_base_ext = os.path.splitext(fname_base)
- # don't use ISO standard time format, as it contains characters illegal on some filesytems.
- nowish = datetime.now().strftime("%Y-%m-%dT%H-%M-%S")
- backup_fname = os.path.join(backup_dir, f"{fname_base_root}-{nowish}{fname_base_ext}")
- if os.path.exists(backup_fname):
- os.remove(backup_fname)
- os.rename(fname, backup_fname)
-
- # prune the backup_dir to max number of backup files, keeping the most recent backups
- backups = list(filter(lambda s: s.startswith(fname_base_root), os.listdir(backup_dir)))
- excess_count = len(backups) - max_backups
- if excess_count > 0:
- backups.sort()
- for bu in backups[0:excess_count]:
- os.remove(os.path.join(backup_dir, bu))
-
- def update_parameters(self, parameters, data_adaptor):
- super().update_parameters(parameters, data_adaptor)
-
- params = {}
- params["annotations"] = True
- params["user_annotation_collection_name_enabled"] = True
-
- if self.output_file is not None:
- # user has hard-wired the name of the annotation data collection
- fname = os.path.basename(self.output_file)
- collection_fname = os.path.splitext(fname)[0]
- params["annotations-data-collection-is-read-only"] = True
- params["annotations-data-collection-name"] = collection_fname
-
- elif session is not None:
- collection = self.get_collection()
- if current_app.auth.is_user_authenticated():
- params["annotations-user-data-idhash"] = self._get_userdata_idhash(data_adaptor)
- params["annotations-data-collection-is-read-only"] = not self.user_annotations_enabled()
- params["annotations-data-collection-name"] = collection
-
- parameters.update(params)
diff --git a/backend/czi_hosted/common/config/__init__.py b/backend/czi_hosted/common/config/__init__.py
deleted file mode 100644
index 8b12f32a..00000000
--- a/backend/czi_hosted/common/config/__init__.py
+++ /dev/null
@@ -1,4 +0,0 @@
-from backend.common.utils.aws_secret_utils import get_secret_key # noqa F504
-
-DEFAULT_SERVER_PORT = 5005
-BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20 # 100MB
diff --git a/backend/czi_hosted/common/config/app_config.py b/backend/czi_hosted/common/config/app_config.py
deleted file mode 100644
index 517491b2..00000000
--- a/backend/czi_hosted/common/config/app_config.py
+++ /dev/null
@@ -1,247 +0,0 @@
-import yaml
-from flatten_dict import unflatten
-
-from backend.czi_hosted.common.config.external_config import ExternalConfig
-from backend.czi_hosted.common.config.dataset_config import DatasetConfig
-from backend.czi_hosted.common.config.server_config import ServerConfig
-from backend.common.errors import ConfigurationError
-from backend.czi_hosted.default_config import get_default_config
-
-
-class AppConfig(object):
- """
- AppConfig stores all the configuration for cellxgene.
- AppConfig contains one or more DatasetConfig(s) and one ServerConfig.
- The server_config contains attributes that refer to the server process as a whole.
- The default_dataset_config refers to attributes that are associated with the features and
- presentations of a dataset.
- The dataset config attributes can be overridden depending on the url by which the
- dataset was accessed. These are stored in dataroot_config.
- AppConfig has methods to initialize, modify, and access the configuration.
- """
-
- def __init__(self):
-
- # the default configuration (see default_config.py)
- # TODO @madison -- if we always read from the default config (hard coded path) can we set those values as
- # defaults within the config class?
- self.default_config = get_default_config()
- # the server configuration
- self.server_config = ServerConfig(self, self.default_config["server"])
- # the dataset config, unless overridden by an entry in dataroot_config
- self.default_dataset_config = DatasetConfig(None, self, self.default_config["dataset"])
- # a dictionary of keys to DatasetConfig objects. Each key must exist in the multi_dataset__dataroot
- # attribute of the server_config. The default dataset config will apply to all datasets unless a different set
- # of config vars was passed for a specific dataset under the multidataset config. For example:
- """
- per_dataset_config:
- d1:
- user_annotations:
- enable: false
- d2:
- user_annotations:
- enable: true
- """
- # dataroot config
- self.dataroot_config = {}
-
- # external config
- self.external_config = ExternalConfig(self, self.default_config["external"])
-
- # Set to true when config_completed is called
- self.is_completed = False
-
- def get_dataset_config(self, dataroot_key):
- if self.server_config.single_dataset__datapath:
- return self.default_dataset_config
- else:
- return self.dataroot_config.get(dataroot_key, self.default_dataset_config)
-
- def check_config(self):
- """Verify all the attributes in the config have been type checked"""
- if not self.is_completed:
- raise ConfigurationError("The configuration has not been completed")
- self.server_config.check_config()
- self.default_dataset_config.check_config()
- for dataset_config in self.dataroot_config.values():
- dataset_config.check_config()
- self.external_config.check_config()
-
- def update_server_config(self, **kw):
- self.server_config.update(**kw)
- self.is_completed = False
-
- def update_default_dataset_config(self, **kw):
- self.default_dataset_config.update(**kw)
- # update all the other dataset configs, if any
- for value in self.dataroot_config.values():
- value.update(**kw)
- self.is_completed = False
-
- def update_single_config_from_path_and_value(self, path, value):
- """Update a single config parameter with the value.
- Path is a list of string, that gives a path to the config parameter to be updated.
- For example, path may be ["server","app","port"].
- """
- self.is_completed = False
- if not isinstance(path, list):
- raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'")
- for part in path:
- if not isinstance(part, str):
- raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'")
-
- if len(path) < 1 or path[0] not in ("server", "dataset", "per_dataset_config"):
- raise ConfigurationError("path must start with 'server', 'dataset', or 'per_dataset_config'")
-
- if path[0] == "server":
- attr = "__".join(path[1:])
- try:
- self.update_server_config(**{attr: value})
- except ConfigurationError:
- raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
- elif path[0] == "dataset":
- attr = "__".join(path[1:])
- try:
- self.update_default_dataset_config(**{attr: value})
- except ConfigurationError:
- raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
-
- elif path[0] == "per_dataset_config":
- if len(path) < 2:
- raise ConfigurationError(f"missing dataroot when using per_dataset_config: got '{path}'")
- dataroot = path[1]
- if dataroot not in self.dataroot_config:
- dataroots = str(list(self.dataroot_config.keys()))
- raise ConfigurationError(
- f"unknown dataroot when using per_dataset_config: got '{path}',"
- f" dataroots specified in config are {dataroots}"
- )
-
- attr = "__".join(path[2:])
- try:
- self.dataroot_config[dataroot].update(**{attr: value})
- except ConfigurationError:
- raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
-
- def update_from_config_file(self, config_file):
- try:
- with open(config_file) as yml_file:
- config = yaml.safe_load(yml_file)
- except yaml.YAMLError as e:
- raise ConfigurationError(f"The specified config file contained an error: {e}")
- except OSError as e:
- raise ConfigurationError(f"Issue retrieving the specified config file: {e}")
-
- if config.get("server"):
- self.server_config.update_from_config(config["server"], "server")
- if config.get("dataset"):
- self.default_dataset_config.update_from_config(config["dataset"], "dataset")
-
- per_dataset_config = config.get("per_dataset_config", {})
- for key, dataroot_config in per_dataset_config.items():
- # first create and initialize the dataroot with the default config
- self.add_dataroot_config(key, **config["dataset"])
- # then apply the per dataset configuration
- self.dataroot_config[key].update_from_config(dataroot_config, f"per_dataset_config__{key}")
-
- if config.get("external"):
- self.external_config.update_from_config(config["external"], "external")
-
- self.is_completed = False
-
- def config_to_dict(self):
- """return the configuration as an unflattened dict"""
- server = self.server_config.create_mapping(self.server_config.default_config)
- dataset = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
- external = self.external_config.create_mapping(self.external_config.default_config)
- config = dict(server={}, dataset={})
- for attrname in server.keys():
- config["server__" + attrname] = getattr(self.server_config, attrname)
- for attrname in dataset.keys():
- config["dataset__" + attrname] = getattr(self.default_dataset_config, attrname)
- if self.dataroot_config:
- config["per_dataset_config"] = {}
- for dataroot_tag, dataroot_config in self.dataroot_config.items():
- dataset = dataroot_config.create_mapping(dataroot_config.default_config)
- for attrname in dataset.keys():
- config[f"per_dataset_config__{dataroot_tag}__" + attrname] = getattr(dataroot_config, attrname)
- for attrname in external.keys():
- config["external__" + attrname] = getattr(self.external_config, attrname)
-
- config = unflatten(config, splitter=lambda key: key.split("__"))
- return config
-
- def write_config(self, config_file):
- """output the config to a yaml file"""
- config = self.config_to_dict()
- yaml.dump(config, open(config_file, "w"))
-
- def changes_from_default(self):
- """Return all the attribute that are different from the default"""
- diff_server = self.server_config.changes_from_default()
- diff_dataset = self.default_dataset_config.changes_from_default()
- diff_external = self.external.changes_from_default()
- diff = dict(server=diff_server, dataset=diff_dataset, external=diff_external)
- return diff
-
- def add_dataroot_config(self, dataroot_tag, **kw):
- """Create a new dataset config object based on the default dataset config, and kw parameters"""
- if dataroot_tag in self.dataroot_config:
- raise ConfigurationError(f"dataroot config already exists: {dataroot_tag}")
- if type(self.server_config.multi_dataset__dataroot) != dict:
- raise ConfigurationError("The server__multi_dataset__dataroot must be a dictionary")
- if dataroot_tag not in self.server_config.multi_dataset__dataroot:
- raise ConfigurationError(f"The dataroot_tag ({dataroot_tag}) not found in server__multi_dataset__dataroot")
-
- self.is_completed = False
- self.dataroot_config[dataroot_tag] = DatasetConfig(dataroot_tag, self, self.default_config["dataset"])
- flat_config = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
- config = {key: value[1] for key, value in flat_config.items()}
- self.dataroot_config[dataroot_tag].update(**config)
- self.dataroot_config[dataroot_tag].update_from_config(kw, dataroot_tag)
-
- def complete_config(self, messagefn=None):
- """The configure options are checked, and any additional setup based on the config
- parameters is done"""
-
- if messagefn is None:
-
- def noop(message):
- pass
-
- messagefn = noop
-
- # TODO: to give better error messages we can add a mapping between where each config
- # attribute originated (e.g. command line argument or config file), then in the error
- # messages we can give correct context for attributes with bad value.
- context = dict(messagefn=messagefn)
-
- # complete config for external_config first, since this may update values in the other sections
- self.external_config.complete_config(context)
- self.server_config.complete_config(context)
- self.default_dataset_config.complete_config(context)
- for dataroot_config in self.dataroot_config.values():
- dataroot_config.complete_config(context)
-
- self.is_completed = True
- self.check_config()
-
- def get_matrix_data_cache_manager(self):
- return self.server_config.matrix_data_cache_manager
-
- def is_multi_dataset(self):
- return self.server_config.multi_dataset__dataroot is not None
-
- def get_title(self, data_adaptor):
- return (
- self.server_config.single_dataset__title
- if self.server_config.single_dataset__title
- else data_adaptor.get_title()
- )
-
- def get_about(self, data_adaptor):
- return (
- self.server_config.single_dataset__about
- if self.server_config.single_dataset__about
- else data_adaptor.get_about()
- )
diff --git a/backend/czi_hosted/common/config/base_config.py b/backend/czi_hosted/common/config/base_config.py
deleted file mode 100644
index 7c4f98bf..00000000
--- a/backend/czi_hosted/common/config/base_config.py
+++ /dev/null
@@ -1,132 +0,0 @@
-import copy
-
-from flatten_dict import flatten
-from backend.common.errors import ConfigurationError
-
-
-class BaseConfig(object):
- """
- This class handles the mechanics of updating and checking attributes.
- Derived classes are expected to store the actual attributes
- Currently DatasetConfig and ServerConfig both inherit from BaseConfig.
- """
-
- def __init__(self, app_config, default_config, dictval_cases={}):
- # reference back to the app_config
- self.app_config = app_config
- # the complete set of attributes and their default values (unflattened)
- self.default_config = default_config
- # attributes where the value may be a dict (and therefore are not flattened)
- self.dictval_cases = dictval_cases
- # used to make sure every attribute value is checked
- self.attr_checked = {key_name: False for key_name in self.create_mapping(default_config).keys()}
-
- def create_mapping(self, config):
- """
- Create a dictionary where the keys are the name of attributes (using double underscore convention)
- For example: authentication__type
-
- The values are a tuple,
- - the first item of the tuple is a tuple of path elements (location in config 'tree')
- - the second item is the value of the config parameter
-
- For example: (('authentication', 'type'), 'session'))
- """
- config_copy = copy.deepcopy(config)
- mapping = {}
-
- # special cases where the value could be a dict.
- # If its value is not None, the entry is added to the mapping, and not included
- # in the flattening below.
- for dictval_case in self.dictval_cases:
- cur = config_copy
- for part in dictval_case[:-1]:
- cur = cur.get(part, {})
- val = cur.get(dictval_case[-1])
- if val is not None:
- key = "__".join(dictval_case)
- mapping[key] = (dictval_case, val)
- del cur[dictval_case[-1]]
-
- flat_config = flatten(config_copy)
- for key, value in flat_config.items():
- # name of the attribute
- attr = "__".join(key)
- mapping[attr] = (key, value)
-
- return mapping
-
- def validate_correct_type_of_configuration_attribute(self, attrname, vtype):
- val = getattr(self, attrname)
- if type(vtype) in (list, tuple):
- if type(val) not in vtype:
- tnames = ",".join([x.__name__ for x in vtype])
- raise ConfigurationError(
- f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}"
- )
- else:
- if type(val) != vtype:
- raise ConfigurationError(
- f"Invalid type for attribute: {attrname}, "
- f"expected type {vtype.__name__}, got {type(val).__name__}"
- )
-
- self.attr_checked[attrname] = True
-
- def check_config(self):
- mapping = self.create_mapping(self.default_config)
- for key in mapping.keys():
- if not self.attr_checked[key]:
- raise ConfigurationError(f"The attr '{key}' has not been checked")
-
- def update(self, **kw):
- """Update the attributes defined in kw with their new values."""
- for key, value in kw.items():
- if not hasattr(self, key):
-
- # check if the key is setting into a dictval entry.
- found_dictval = False
- for dictval in self.dictval_cases:
- dictvalname = "__".join(dictval)
- if dictvalname + "__" in key:
- dictkey = key[len(dictvalname) + 2 :]
- curdictval = getattr(self, dictvalname)
- if curdictval is None:
- setattr(self, dictvalname, dict(dictkey=value))
- else:
- curdictval[dictkey] = value
-
- found_dictval = True
- break
-
- if found_dictval:
- continue
- raise ConfigurationError(f"unknown config parameter {key}.")
- try:
- if type(value) == tuple:
- # convert tuple values to list values
- value = list(value)
- setattr(self, key, value)
- except KeyError:
- raise ConfigurationError(f"Unable to set config parameter {key}.")
-
- self.attr_checked[key] = False
-
- def update_from_config(self, config, prefix):
- mapping = self.create_mapping(config)
- for attr, (key, value) in mapping.items():
- if not hasattr(self, attr):
- raise ConfigurationError(f"Unknown key from config file: {prefix}__{attr}")
- setattr(self, attr, value)
-
- self.attr_checked[attr] = False
-
- def changes_from_default(self):
- """Return all the attribute that are different from the default"""
- mapping = self.create_mapping(self.default_config)
- diff = []
- for attrname, (key, defval) in mapping.items():
- curval = getattr(self, attrname)
- if curval != defval:
- diff.append((attrname, curval, defval))
- return diff
diff --git a/backend/czi_hosted/common/config/client_config.py b/backend/czi_hosted/common/config/client_config.py
deleted file mode 100644
index c4aa5fb8..00000000
--- a/backend/czi_hosted/common/config/client_config.py
+++ /dev/null
@@ -1,121 +0,0 @@
-from backend.czi_hosted import display_version as cellxgene_display_version
-
-
-def get_client_config(app_config, data_adaptor):
- """
- Return the configuration as required by the /config REST route
- """
-
- server_config = app_config.server_config
- dataset_config = data_adaptor.dataset_config
- annotation = dataset_config.user_annotations
- auth = server_config.auth
-
- # FIXME The current set of config is not consistently presented:
- # we have camalCase, hyphen-text, and underscore_text
-
- # make sure the configuration has been checked.
- app_config.check_config()
-
- # display_names
- title = app_config.get_title(data_adaptor)
- about = app_config.get_about(data_adaptor)
-
- display_names = dict(engine=data_adaptor.get_name(), dataset=title)
-
- # library_versions
- library_versions = {}
- library_versions.update(data_adaptor.get_library_versions())
- library_versions["cellxgene"] = cellxgene_display_version
-
- # links
- links = {"about-dataset": about}
-
- # parameters
- parameters = {
- "layout": dataset_config.embeddings__names,
- "max-category-items": dataset_config.presentation__max_categories,
- "obs_names": server_config.single_dataset__obs_names,
- "var_names": server_config.single_dataset__var_names,
- "diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff,
- "backed": server_config.adaptor__anndata_adaptor__backed,
- "disable-diffexp": not dataset_config.diffexp__enable,
- "annotations": False,
- "annotations_file": None,
- "annotations_dir": None,
- "annotations_genesets": True, # feature flag
- "annotations_genesets_readonly": True,
- "annotations_genesets_summary_methods": ["mean"],
- "custom_colors": dataset_config.presentation__custom_colors,
- "diffexp-may-be-slow": False,
- "about_legal_tos": dataset_config.app__about_legal_tos,
- "about_legal_privacy": dataset_config.app__about_legal_privacy,
- }
-
- # corpora dataset_props
- # TODO/Note: putting info from the dataset into the /config is not ideal.
- # However, it is definitely not part of /schema, and we do not have a top-level
- # route for data properties. Consider creating one at some point.
- corpora_props = data_adaptor.get_corpora_props()
- if corpora_props and "default_embedding" in corpora_props:
- default_embedding = corpora_props["default_embedding"]
- if isinstance(default_embedding, str) and default_embedding.startswith("X_"):
- default_embedding = default_embedding[2:] # drop X_ prefix
- if default_embedding in data_adaptor.get_embedding_names():
- parameters["default_embedding"] = default_embedding
-
- data_adaptor.update_parameters(parameters)
- if annotation:
- annotation.update_parameters(parameters, data_adaptor)
-
- # gather it all together
- client_config = {}
- config = client_config["config"] = {}
- config["displayNames"] = display_names
- config["library_versions"] = library_versions
- config["links"] = links
- config["parameters"] = parameters
- config["corpora_props"] = corpora_props
- config["limits"] = {
- "column_request_max": server_config.limits__column_request_max,
- "diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max,
- }
-
- if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
- config["authentication"] = {
- "requires_client_login": auth.requires_client_login(),
- }
- if auth.requires_client_login():
- config["authentication"].update(
- {
- # Todo why are these stored on the data_adaptor?
- "login": auth.get_login_url(data_adaptor),
- "logout": auth.get_logout_url(data_adaptor),
- }
- )
-
- return client_config
-
-
-def get_client_userinfo(app_config, data_adaptor):
- """
- Return the userinfo as required by the /userinfo REST route
- """
-
- server_config = app_config.server_config
- dataset_config = data_adaptor.dataset_config
- auth = server_config.auth
-
- # make sure the configuration has been checked.
- app_config.check_config()
-
- if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
- userinfo = {}
- userinfo["userinfo"] = {
- "is_authenticated": auth.is_user_authenticated(),
- "username": auth.get_user_name(),
- "user_id": auth.get_user_id(),
- "email": auth.get_user_email(),
- "picture": auth.get_user_picture(),
- }
- return userinfo
diff --git a/backend/czi_hosted/common/config/dataset_config.py b/backend/czi_hosted/common/config/dataset_config.py
deleted file mode 100644
index 50d96787..00000000
--- a/backend/czi_hosted/common/config/dataset_config.py
+++ /dev/null
@@ -1,211 +0,0 @@
-import os
-from os.path import splitext, isdir
-
-from backend.czi_hosted.common.annotations.annotations import Annotations
-from backend.czi_hosted.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
-from backend.czi_hosted.common.annotations.local_file_csv import AnnotationsLocalFile
-from backend.czi_hosted.common.config.base_config import BaseConfig
-from backend.common.errors import ConfigurationError
-from backend.czi_hosted.db.db_utils import DbUtils
-
-
-class DatasetConfig(BaseConfig):
- """Manages the config attribute associated with a dataset."""
-
- def __init__(self, tag, app_config, default_config):
- super().__init__(app_config, default_config)
- self.tag = tag
- try:
- self.app__scripts = default_config["app"]["scripts"]
- self.app__inline_scripts = default_config["app"]["inline_scripts"]
- self.app__about_legal_tos = default_config["app"]["about_legal_tos"]
- self.app__about_legal_privacy = default_config["app"]["about_legal_privacy"]
- self.app__authentication_enable = default_config["app"]["authentication_enable"]
-
- self.presentation__max_categories = default_config["presentation"]["max_categories"]
- self.presentation__custom_colors = default_config["presentation"]["custom_colors"]
-
- self.user_annotations__enable = default_config["user_annotations"]["enable"]
- self.user_annotations__type = default_config["user_annotations"]["type"]
- self.user_annotations__local_file_csv__directory = default_config["user_annotations"]["local_file_csv"][
- "directory"
- ]
- self.user_annotations__local_file_csv__file = default_config["user_annotations"]["local_file_csv"]["file"]
- self.user_annotations__hosted_tiledb_array__db_uri = default_config["user_annotations"][
- "hosted_tiledb_array"
- ]["db_uri"]
- self.user_annotations__hosted_tiledb_array__hosted_file_directory = default_config["user_annotations"][
- "hosted_tiledb_array"
- ]["hosted_file_directory"]
-
- self.embeddings__names = default_config["embeddings"]["names"]
-
- self.diffexp__enable = default_config["diffexp"]["enable"]
- self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
- self.diffexp__top_n = default_config["diffexp"]["top_n"]
-
- self.X_approximate_distribution = default_config["X_approximate_distribution"]
-
- except KeyError as e:
- raise ConfigurationError(f"Unexpected config: {str(e)}")
-
- # Create the default annotation, which supports gene set reading without
- # further configuration. Depending on configuration options, `complete_config`
- # may create a more specialized annotation object and replace this default.
- self.user_annotations = Annotations()
-
- def complete_config(self, context):
- self.handle_app()
- self.handle_presentation()
- self.handle_user_annotations(context)
- self.handle_embeddings()
- self.handle_diffexp(context)
- self.handle_X_approximate_distribution()
-
- def handle_app(self):
- self.validate_correct_type_of_configuration_attribute("app__scripts", list)
- self.validate_correct_type_of_configuration_attribute("app__inline_scripts", list)
- self.validate_correct_type_of_configuration_attribute("app__about_legal_tos", (type(None), str))
- self.validate_correct_type_of_configuration_attribute("app__about_legal_privacy", (type(None), str))
- self.validate_correct_type_of_configuration_attribute("app__authentication_enable", bool)
-
- # scripts can be string (filename) or dict (attributes). Convert string to dict.
- scripts = []
- for script in self.app__scripts:
- try:
- if isinstance(script, str):
- scripts.append({"src": script})
- elif isinstance(script, dict) and isinstance(script["src"], str):
- scripts.append(script)
- else:
- raise Exception
- except Exception as e:
- raise ConfigurationError(f"Scripts must be string or a dict containing an src key: {e}")
-
- self.app__scripts = scripts
-
- def handle_presentation(self):
- self.validate_correct_type_of_configuration_attribute("presentation__max_categories", int)
- self.validate_correct_type_of_configuration_attribute("presentation__custom_colors", bool)
-
- def handle_user_annotations(self, context):
- self.validate_correct_type_of_configuration_attribute("user_annotations__enable", bool)
- self.validate_correct_type_of_configuration_attribute("user_annotations__type", str)
- self.validate_correct_type_of_configuration_attribute(
- "user_annotations__local_file_csv__directory", (type(None), str)
- )
- self.validate_correct_type_of_configuration_attribute(
- "user_annotations__local_file_csv__file", (type(None), str)
- )
- self.validate_correct_type_of_configuration_attribute(
- "user_annotations__hosted_tiledb_array__db_uri", (type(None), str)
- )
- self.validate_correct_type_of_configuration_attribute(
- "user_annotations__hosted_tiledb_array__hosted_file_directory", (type(None), str)
- )
- if self.user_annotations__enable:
- server_config = self.app_config.server_config
- if not self.app__authentication_enable:
- raise ConfigurationError("user annotations requires authentication to be enabled")
- if not server_config.auth.is_valid_authentication_type():
- auth_type = server_config.authentication__type
- raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations")
-
- if self.user_annotations__type == "local_file_csv":
- self.handle_local_file_csv_annotations()
- elif self.user_annotations__type == "hosted_tiledb_array":
- self.handle_hosted_tiledb_annotations()
- else:
- raise ConfigurationError('The only annotation type support is "local_file_csv" or "hosted_tiledb_array')
- else:
- self.check_annotation_config_vars_not_set(context)
-
- def handle_local_file_csv_annotations(self):
- dirname = self.user_annotations__local_file_csv__directory
- filename = self.user_annotations__local_file_csv__file
- if filename is not None and dirname is not None:
- raise ConfigurationError("'annotations-file' and 'annotations-dir' may not be used together.")
-
- if filename is not None:
- lf_name, lf_ext = splitext(filename)
- if lf_ext and lf_ext != ".csv":
- raise ConfigurationError(f"annotation file type must be .csv: {filename}")
-
- if dirname is not None and not isdir(dirname):
- try:
- os.mkdir(dirname)
- except OSError:
- raise ConfigurationError("Unable to create directory specified by --annotations-dir")
-
- anno_config = {
- "user-annotations": self.user_annotations__enable,
- "genesets-save": False,
- }
- self.user_annotations = AnnotationsLocalFile(anno_config, dirname, filename)
-
- # if the user has specified a fixed label file, go ahead and validate it
- # so that we can remove errors early in the process.
- server_config = self.app_config.server_config
- if server_config.single_dataset__datapath and self.user_annotations__local_file_csv__file:
- with server_config.matrix_data_cache_manager.data_adaptor(
- self.tag, server_config.single_dataset__datapath, self.app_config
- ) as data_adaptor:
- data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor))
-
- def handle_hosted_tiledb_annotations(self):
- self.validate_correct_type_of_configuration_attribute("user_annotations__hosted_tiledb_array__db_uri", str)
- self.validate_correct_type_of_configuration_attribute(
- "user_annotations__hosted_tiledb_array__hosted_file_directory", str
- )
- anno_config = {
- "user-annotations": self.user_annotations__enable,
- "genesets-save": False,
- }
- self.user_annotations = AnnotationsHostedTileDB(
- anno_config,
- directory_path=self.user_annotations__hosted_tiledb_array__hosted_file_directory,
- db=DbUtils(self.user_annotations__hosted_tiledb_array__db_uri),
- )
-
- def check_annotation_config_vars_not_set(self, context):
- if self.user_annotations__type is not None:
- dirname = self.user_annotations__local_file_csv__directory
- filename = self.user_annotations__local_file_csv__file
- db_uri = self.user_annotations__hosted_tiledb_array__db_uri
- hosted_file_dirname = self.user_annotations__hosted_tiledb_array__hosted_file_directory
- if filename is not None:
- context["messagefn"]("Warning: --annotations-file ignored as annotations are disabled.")
- if dirname is not None:
- context["messagefn"]("Warning: --annotations-dir ignored as annotations are disabled.")
- if db_uri is not None:
- context["messagefn"]("Warning: db_uri ignored as annotations are disabled.")
- if hosted_file_dirname is not None:
- context["messagefn"](
- "Warning: hosted_file_directory for hosted_tiledb_array ignored as annotations are disabled."
- )
-
- def handle_embeddings(self):
- self.validate_correct_type_of_configuration_attribute("embeddings__names", list)
-
- def handle_diffexp(self, context):
- self.validate_correct_type_of_configuration_attribute("diffexp__enable", bool)
- self.validate_correct_type_of_configuration_attribute("diffexp__lfc_cutoff", float)
- self.validate_correct_type_of_configuration_attribute("diffexp__top_n", int)
-
- server_config = self.app_config.server_config
- if server_config.single_dataset__datapath:
- with server_config.matrix_data_cache_manager.data_adaptor(
- self.tag, server_config.single_dataset__datapath, self.app_config
- ) as data_adaptor:
- if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False):
- context["messagefn"](
- "CAUTION: due to the size of your dataset, "
- "running differential expression may take longer or fail."
- )
-
- def handle_X_approximate_distribution(self):
- self.validate_correct_type_of_configuration_attribute("X_approximate_distribution", str)
- if self.X_approximate_distribution not in ["normal", "count"]:
- raise ConfigurationError(
- "X_approximate_distribution has unknown value -- must be 'normal' or 'count'."
- )
diff --git a/backend/czi_hosted/common/config/external_config.py b/backend/czi_hosted/common/config/external_config.py
deleted file mode 100644
index 915486e1..00000000
--- a/backend/czi_hosted/common/config/external_config.py
+++ /dev/null
@@ -1,95 +0,0 @@
-import os
-
-from backend.czi_hosted.common.config.base_config import BaseConfig
-from backend.common.errors import ConfigurationError, SecretKeyRetrievalError
-from backend.common.utils.aws_secret_utils import get_secret_key
-from backend.common.utils.type_conversion_utils import convert_string_to_value
-
-
-class ExternalConfig(BaseConfig):
- """Manages the config attribute associated with external configuration sources, such as
- environment variables or the AWS Secrets Manager."""
-
- def __init__(self, app_config, default_config):
- super().__init__(app_config, default_config)
- try:
- self.environment = default_config["environment"]
- self.aws_secrets_manager__region = default_config["aws_secrets_manager"]["region"]
- self.aws_secrets_manager__secrets = default_config["aws_secrets_manager"]["secrets"]
-
- except KeyError as e:
- raise ConfigurationError(f"Unexpected config: {str(e)}")
-
- def complete_config(self, context):
- self.handle_environment(context)
- self.handle_aws_secrets_manager(context)
-
- def handle_environment(self, context):
- """For each environment variable defined, get the value (if it is set),
- and set the specified config parameter"""
- self.validate_correct_type_of_configuration_attribute("environment", list)
- for envdict in self.environment:
- name = envdict.get("name")
- if name is None:
- raise ConfigurationError("environment: 'name' is missing")
- required = envdict.get("required", False)
- if type(required) != bool:
- raise ConfigurationError("environment: 'required' must be a bool")
- path = envdict.get("path")
- if path is None:
- raise ConfigurationError("environment: 'path' is missing")
-
- value = os.environ.get(name)
- if value is None:
- if required:
- raise ConfigurationError(f"required environment variable '{name}' not set")
- else:
- value = convert_string_to_value(value)
- self.app_config.update_single_config_from_path_and_value(path, value)
-
- def handle_aws_secrets_manager(self, context):
- """For each aws secret defined, get the key/values, and set the specified config parameter"""
- self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", (type(None), str))
- self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__secrets", list)
-
- if not self.aws_secrets_manager__secrets:
- return
-
- self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", str)
-
- for secret in self.aws_secrets_manager__secrets:
- secret_name = secret.get("name")
- if secret_name is None:
- raise ConfigurationError("aws_secrets_manager: 'name' is missing")
- if not isinstance(secret_name, str):
- raise ConfigurationError("aws_secrets_manager: 'name' must be a string")
-
- try:
- secret_dict = get_secret_key(self.aws_secrets_manager__region, secret_name)
- except SecretKeyRetrievalError as e:
- raise ConfigurationError(f"Unable to retrieve secret {secret_name}: {str(e)}")
-
- values = secret.get("values")
- if values is None:
- raise ConfigurationError("aws_secrets_manager: 'values' is missing")
- if not isinstance(values, list):
- raise ConfigurationError("aws_secrets_manager: 'values' must be a list")
-
- for value in values:
- key = value.get("key")
- if key is None:
- raise ConfigurationError(f"missing 'key' in secret values: {secret_name}")
- path = value.get("path")
- if path is None:
- raise ConfigurationError(f"missing 'path' in secret values: {secret_name}")
- required = value.get("required", False)
- if type(required) != bool:
- raise ConfigurationError(f"wrong type for 'required' in secret values: {secret_name}")
-
- secret_value = secret_dict.get(key)
- if secret_value is None:
- if required:
- raise ConfigurationError(f"required secret '{secret_name}:{key}' not set")
- else:
- secret_value = convert_string_to_value(secret_value)
- self.app_config.update_single_config_from_path_and_value(path, secret_value)
diff --git a/backend/czi_hosted/common/config/server_config.py b/backend/czi_hosted/common/config/server_config.py
deleted file mode 100644
index afc76b44..00000000
--- a/backend/czi_hosted/common/config/server_config.py
+++ /dev/null
@@ -1,387 +0,0 @@
-import os
-import sys
-import warnings
-from os.path import basename
-from urllib.parse import urlparse, quote_plus
-
-from backend.czi_hosted.auth.auth import AuthTypeFactory
-from backend.czi_hosted.common.config import DEFAULT_SERVER_PORT, BIG_FILE_SIZE_THRESHOLD
-from backend.czi_hosted.common.config.base_config import BaseConfig
-from backend.common.utils.data_locator import discover_s3_region_name
-from backend.common.errors import ConfigurationError, DatasetAccessError
-from backend.common.utils.utils import is_port_available, find_available_port, custom_format_warning
-from backend.czi_hosted.compute import diffexp_cxg as diffexp_tiledb
-from backend.czi_hosted.data_common.matrix_loader import MatrixDataCacheManager, MatrixDataLoader, MatrixDataType
-
-
-class ServerConfig(BaseConfig):
- """Manages the config attribute associated with the server."""
-
- def __init__(self, app_config, default_config):
- dictval_cases = [
- ("app", "csp_directives"),
- ("authentication", "params_oauth", "cookie"),
- ("authentication", "params_oauth", "jwt_decode_options"),
- ("adaptor", "cxg_adaptor", "tiledb_ctx"),
- ("multi_dataset", "dataroot"),
- ]
- super().__init__(app_config, default_config, dictval_cases)
-
- try:
- self.app__verbose = default_config["app"]["verbose"]
- self.app__debug = default_config["app"]["debug"]
- self.app__host = default_config["app"]["host"]
- self.app__port = default_config["app"]["port"]
- self.app__open_browser = default_config["app"]["open_browser"]
- self.app__force_https = default_config["app"]["force_https"]
- self.app__flask_secret_key = default_config["app"]["flask_secret_key"]
- self.app__generate_cache_control_headers = default_config["app"]["generate_cache_control_headers"]
- self.app__server_timing_headers = default_config["app"]["server_timing_headers"]
- self.app__csp_directives = default_config["app"]["csp_directives"]
- self.app__api_base_url = default_config["app"]["api_base_url"]
- self.app__web_base_url = default_config["app"]["web_base_url"]
-
- self.authentication__type = default_config["authentication"]["type"]
- self.authentication__insecure_test_environment = default_config["authentication"][
- "insecure_test_environment"
- ]
- self.authentication__params_oauth__oauth_api_base_url = default_config["authentication"]["params_oauth"][
- "oauth_api_base_url"
- ]
- self.authentication__params_oauth__client_id = default_config["authentication"]["params_oauth"]["client_id"]
- self.authentication__params_oauth__client_secret = default_config["authentication"]["params_oauth"][
- "client_secret"
- ]
- self.authentication__params_oauth__jwt_decode_options = default_config["authentication"]["params_oauth"][
- "jwt_decode_options"
- ]
- self.authentication__params_oauth__session_cookie = default_config["authentication"]["params_oauth"][
- "session_cookie"
- ]
- self.authentication__params_oauth__cookie = default_config["authentication"]["params_oauth"]["cookie"]
-
- self.multi_dataset__dataroot = default_config["multi_dataset"]["dataroot"]
- self.multi_dataset__index = default_config["multi_dataset"]["index"]
- self.multi_dataset__allowed_matrix_types = default_config["multi_dataset"]["allowed_matrix_types"]
- self.multi_dataset__matrix_cache__max_datasets = default_config["multi_dataset"]["matrix_cache"][
- "max_datasets"
- ]
- self.multi_dataset__matrix_cache__timelimit_s = default_config["multi_dataset"]["matrix_cache"][
- "timelimit_s"
- ]
-
- self.single_dataset__datapath = default_config["single_dataset"]["datapath"]
- self.single_dataset__obs_names = default_config["single_dataset"]["obs_names"]
- self.single_dataset__var_names = default_config["single_dataset"]["var_names"]
- self.single_dataset__about = default_config["single_dataset"]["about"]
- self.single_dataset__title = default_config["single_dataset"]["title"]
-
- self.diffexp__alg_cxg__max_workers = default_config["diffexp"]["alg_cxg"]["max_workers"]
- self.diffexp__alg_cxg__cpu_multiplier = default_config["diffexp"]["alg_cxg"]["cpu_multiplier"]
- self.diffexp__alg_cxg__target_workunit = default_config["diffexp"]["alg_cxg"]["target_workunit"]
-
- self.data_locator__s3__region_name = default_config["data_locator"]["s3"]["region_name"]
-
- self.adaptor__cxg_adaptor__tiledb_ctx = default_config["adaptor"]["cxg_adaptor"]["tiledb_ctx"]
- self.adaptor__anndata_adaptor__backed = default_config["adaptor"]["anndata_adaptor"]["backed"]
-
- self.limits__diffexp_cellcount_max = default_config["limits"]["diffexp_cellcount_max"]
- self.limits__column_request_max = default_config["limits"]["column_request_max"]
-
- except KeyError as e:
- raise ConfigurationError(f"Unexpected config: {str(e)}")
-
- # The matrix data cache manager is created during the complete_config and stored here.
- self.matrix_data_cache_manager = None
-
- # The authentication object
- self.auth = None
-
- def complete_config(self, context):
- self.handle_app(context)
- self.handle_data_source()
- self.handle_authentication()
- self.handle_data_locator()
- self.handle_adaptor() # may depend on data_locator
- self.handle_single_dataset(context) # may depend on adaptor
- self.handle_multi_dataset() # may depend on adaptor
- self.handle_diffexp()
- self.handle_limits()
-
- self.check_config()
-
- def handle_app(self, context):
- self.validate_correct_type_of_configuration_attribute("app__verbose", bool)
- self.validate_correct_type_of_configuration_attribute("app__debug", bool)
- self.validate_correct_type_of_configuration_attribute("app__host", str)
- self.validate_correct_type_of_configuration_attribute("app__port", (type(None), int))
- self.validate_correct_type_of_configuration_attribute("app__open_browser", bool)
- self.validate_correct_type_of_configuration_attribute("app__force_https", bool)
- self.validate_correct_type_of_configuration_attribute("app__flask_secret_key", str)
- self.validate_correct_type_of_configuration_attribute("app__generate_cache_control_headers", bool)
- self.validate_correct_type_of_configuration_attribute("app__server_timing_headers", bool)
- self.validate_correct_type_of_configuration_attribute("app__csp_directives", (type(None), dict))
- self.validate_correct_type_of_configuration_attribute("app__api_base_url", (type(None), str))
- self.validate_correct_type_of_configuration_attribute("app__web_base_url", (type(None), str))
-
- if self.app__port:
- try:
- if not is_port_available(self.app__host, self.app__port):
- raise ConfigurationError(
- f"The port selected {self.app__port} is in use, please configure an open port."
- )
- except OverflowError:
- raise ConfigurationError(f"Invalid port: {self.app__port}")
- else:
- try:
- default_server_port = int(os.environ.get("CXG_SERVER_PORT", DEFAULT_SERVER_PORT))
- except ValueError:
- raise ConfigurationError(
- "Invalid port from environment variable CXG_SERVER_PORT: " + os.environ.get("CXG_SERVER_PORT")
- )
- try:
- self.app__port = find_available_port(self.app__host, default_server_port)
- except OverflowError:
- raise ConfigurationError(f"Invalid port: {default_server_port}")
-
- if self.app__debug:
- context["messagefn"]("in debug mode, setting verbose=True and open_browser=False")
- self.app__verbose = True
- self.app__open_browser = False
- else:
- warnings.formatwarning = custom_format_warning
-
- if not self.app__verbose:
- sys.tracebacklimit = 0
-
- # CSP Directives are a dict of string: list(string) or string: string
- if self.app__csp_directives is not None:
- for k, v in self.app__csp_directives.items():
- if not isinstance(k, str):
- raise ConfigurationError("CSP directive names must be a string.")
- if isinstance(v, list):
- for policy in v:
- if not isinstance(policy, str):
- raise ConfigurationError("CSP directive value must be a string or list of strings.")
- elif not isinstance(v, str):
- raise ConfigurationError("CSP directive value must be a string or list of strings.")
-
- if self.app__web_base_url is None:
- self.app__web_base_url = self.app__api_base_url
-
- def handle_authentication(self):
- self.validate_correct_type_of_configuration_attribute("authentication__type", (type(None), str))
- self.validate_correct_type_of_configuration_attribute("authentication__insecure_test_environment", bool)
-
- if self.authentication__type == "test" and not self.authentication__insecure_test_environment:
- raise ConfigurationError("Test auth can only be used in an insecure test environment")
-
- # oauth
- ptypes = str if self.authentication__type == "oauth" else (type(None), str)
- self.validate_correct_type_of_configuration_attribute(
- "authentication__params_oauth__oauth_api_base_url", ptypes
- )
- self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_id", ptypes)
- self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_secret", ptypes)
- self.validate_correct_type_of_configuration_attribute(
- "authentication__params_oauth__jwt_decode_options", (type(None), dict)
- )
- self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__session_cookie", bool)
-
- if self.authentication__params_oauth__session_cookie:
- self.validate_correct_type_of_configuration_attribute(
- "authentication__params_oauth__cookie", (type(None), dict)
- )
- else:
- self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__cookie", dict)
-
- self.auth = AuthTypeFactory.create(self.authentication__type, self)
- if self.auth is None:
- raise ConfigurationError(f"Unknown authentication type: {self.authentication__type}")
-
- def handle_data_locator(self):
- self.validate_correct_type_of_configuration_attribute("data_locator__s3__region_name", (type(None), bool, str))
- if self.data_locator__s3__region_name is True:
- path = self.single_dataset__datapath or self.multi_dataset__dataroot
-
- if type(path) == dict:
- # if multi_dataset__dataroot is a dict, then use the first key
- # that is in s3. NOTE: it is not supported to have dataroots
- # in different regions.
- paths = [val.get("dataroot") for val in path.values()]
- for path in paths:
- if path.startswith("s3://"):
- break
- if path.startswith("s3://"):
- region_name = discover_s3_region_name(path)
- if region_name is None:
- raise ConfigurationError(f"Unable to discover s3 region name from {path}")
- else:
- region_name = None
- self.data_locator__s3__region_name = region_name
-
- def handle_data_source(self):
- self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None)))
- self.validate_correct_type_of_configuration_attribute("multi_dataset__dataroot", (type(None), dict, str))
-
- if self.single_dataset__datapath and self.multi_dataset__dataroot:
- raise ConfigurationError(
- "You must supply either a datapath (for single datasets) or a dataroot (for multidatasets). Not both"
- )
- if self.single_dataset__datapath is None and self.multi_dataset__dataroot is None:
- raise ConfigurationError("You must specify a datapath for a single dataset or a dataroot for multidatasets")
-
- def handle_single_dataset(self, context):
- self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None)))
- self.validate_correct_type_of_configuration_attribute("single_dataset__title", (str, type(None)))
- self.validate_correct_type_of_configuration_attribute("single_dataset__about", (str, type(None)))
- self.validate_correct_type_of_configuration_attribute("single_dataset__obs_names", (str, type(None)))
- self.validate_correct_type_of_configuration_attribute("single_dataset__var_names", (str, type(None)))
-
- if self.single_dataset__datapath is None:
- return
-
- # create the matrix data cache manager:
- if self.matrix_data_cache_manager is None:
- self.matrix_data_cache_manager = MatrixDataCacheManager(max_cached=1, timelimit_s=None)
-
- # preload this data set
- matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath, app_config=self.app_config)
- try:
- matrix_data_loader.pre_load_validation()
- except DatasetAccessError as e:
- raise ConfigurationError(str(e))
-
- file_size = matrix_data_loader.file_size()
- file_basename = basename(self.single_dataset__datapath)
- if file_size > BIG_FILE_SIZE_THRESHOLD:
- context["messagefn"](f"Loading data from {file_basename}, this may take a while...")
- else:
- context["messagefn"](f"Loading data from {file_basename}.")
-
- if self.single_dataset__about:
-
- def url_check(url):
- try:
- result = urlparse(url)
- if all([result.scheme, result.netloc]):
- return True
- else:
- return False
- except ValueError:
- return False
-
- if not url_check(self.single_dataset__about):
- raise ConfigurationError(
- "Must provide an absolute URL for --about. (Example format: http://example.com)"
- )
-
- def handle_multi_dataset(self):
- self.validate_correct_type_of_configuration_attribute("multi_dataset__dataroot", (type(None), dict, str))
- self.validate_correct_type_of_configuration_attribute("multi_dataset__index", (type(None), bool, str))
- self.validate_correct_type_of_configuration_attribute("multi_dataset__allowed_matrix_types", list)
- self.validate_correct_type_of_configuration_attribute("multi_dataset__matrix_cache__max_datasets", int)
- self.validate_correct_type_of_configuration_attribute(
- "multi_dataset__matrix_cache__timelimit_s", (type(None), int, float)
- )
-
- if self.multi_dataset__dataroot is None:
- return
-
- if type(self.multi_dataset__dataroot) == str:
- default_dict = dict(base_url="d", dataroot=self.multi_dataset__dataroot)
- self.multi_dataset__dataroot = dict(d=default_dict)
-
- for tag, dataroot_dict in self.multi_dataset__dataroot.items():
- if "base_url" not in dataroot_dict:
- raise ConfigurationError(f"error in multi_dataset__dataroot: missing base_url for tag {tag}")
- if "dataroot" not in dataroot_dict:
- raise ConfigurationError(f"error in multi_dataset__dataroot: missing dataroot, for tag {tag}")
-
- base_url = dataroot_dict["base_url"]
-
- # sanity check for well formed base urls
- bad = False
- if type(base_url) != str:
- bad = True
- elif os.path.normpath(base_url) != base_url:
- bad = True
- else:
- base_url_parts = base_url.split("/")
- if [quote_plus(part) for part in base_url_parts] != base_url_parts:
- bad = True
- if ".." in base_url_parts:
- bad = True
- if bad:
- raise ConfigurationError(f"error in multi_dataset__dataroot base_url {base_url} for tag {tag}")
-
- # verify all the base_urls are unique
- base_urls = [d["base_url"] for d in self.multi_dataset__dataroot.values()]
- if len(base_urls) > len(set(base_urls)):
- raise ConfigurationError("error in multi_dataset__dataroot: base_urls must be unique")
-
- # error checking
- for mtype in self.multi_dataset__allowed_matrix_types:
- try:
- MatrixDataType(mtype)
- except ValueError:
- raise ConfigurationError(f'Invalid matrix type in "allowed_matrix_types": {mtype}')
-
- # create the matrix data cache manager:
- if self.matrix_data_cache_manager is None:
- self.matrix_data_cache_manager = MatrixDataCacheManager(
- max_cached=self.multi_dataset__matrix_cache__max_datasets,
- timelimit_s=self.multi_dataset__matrix_cache__timelimit_s,
- )
-
- def handle_diffexp(self):
- self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__max_workers", (str, int))
- self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__cpu_multiplier", int)
- self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__target_workunit", int)
-
- max_workers = self.diffexp__alg_cxg__max_workers
- cpu_multiplier = self.diffexp__alg_cxg__cpu_multiplier
- cpu_count = os.cpu_count()
- max_workers = min(max_workers, cpu_multiplier * cpu_count)
- diffexp_tiledb.set_config(max_workers, self.diffexp__alg_cxg__target_workunit)
-
- def handle_adaptor(self):
- # cxg
- self.validate_correct_type_of_configuration_attribute("adaptor__cxg_adaptor__tiledb_ctx", dict)
- regionkey = "vfs.s3.region"
- if regionkey not in self.adaptor__cxg_adaptor__tiledb_ctx:
- if type(self.data_locator__s3__region_name) == str:
- self.adaptor__cxg_adaptor__tiledb_ctx[regionkey] = self.data_locator__s3__region_name
-
- from backend.czi_hosted.data_cxg.cxg_adaptor import CxgAdaptor
-
- CxgAdaptor.set_tiledb_context(self.adaptor__cxg_adaptor__tiledb_ctx)
-
- # anndata
- self.validate_correct_type_of_configuration_attribute("adaptor__anndata_adaptor__backed", bool)
-
- def handle_limits(self):
- self.validate_correct_type_of_configuration_attribute("limits__diffexp_cellcount_max", (type(None), int))
- self.validate_correct_type_of_configuration_attribute("limits__column_request_max", (type(None), int))
-
- def exceeds_limit(self, limit_name, value):
- limit_value = getattr(self, "limits__" + limit_name, None)
- if limit_value is None: # disabled
- return False
- return value > limit_value
-
- def get_api_base_url(self):
- if self.app__api_base_url == "local":
- return f"http://{self.app__host}:{self.app__port}"
- if self.app__api_base_url and self.app__api_base_url.endswith("/"):
- return self.app__api_base_url[:-1]
- return self.app__api_base_url
-
- def get_web_base_url(self):
- if self.app__web_base_url == "local":
- return f"http://{self.app__host}:{self.app__port}"
- if self.app__web_base_url is None:
- return self.get_api_base_url()
- if self.app__web_base_url.endswith("/"):
- return self.app__web_base_url[:-1]
- return self.app__web_base_url
diff --git a/backend/czi_hosted/common/corpora.py b/backend/czi_hosted/common/corpora.py
deleted file mode 100644
index 29f5a62f..00000000
--- a/backend/czi_hosted/common/corpora.py
+++ /dev/null
@@ -1,78 +0,0 @@
-"""
-Corpora schema conventions support. Helper functions for reading.
-
-https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md
-
-https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema_h5ad_implementation.md
-"""
-import collections
-import json
-
-from backend.czi_hosted.cli.upgrade import validate_version_str
-from backend.czi_hosted.common.utils.corpora_constants import CorporaConstants
-
-
-def corpora_get_versions_from_anndata(adata):
- """
- Given an AnnData object, return:
- * None - if not a Corpora object
- * [ corpora_schema_version, corpora_encoding_version ] - if a Corpora object
-
- Implements the identification protocol defined in the specification.
- """
-
- # per Corpora AnnData spec, this is a corpora file if the following is true
- if "version" not in adata.uns_keys():
- return None
- version = adata.uns["version"]
- if not isinstance(version, collections.abc.Mapping) or "corpora_schema_version" not in version:
- return None
-
- corpora_schema_version = version.get("corpora_schema_version")
- corpora_encoding_version = version.get("corpora_encoding_version")
-
- # TODO: spec says these must be SEMVER values, so check.
- if validate_version_str(corpora_schema_version) and validate_version_str(corpora_encoding_version):
- return [corpora_schema_version, corpora_encoding_version]
-
-
-def corpora_is_version_supported(corpora_schema_version, corpora_encoding_version):
- return (
- corpora_schema_version
- and corpora_encoding_version
- and corpora_schema_version.startswith("1.")
- and corpora_encoding_version.startswith("0.1.")
- )
-
-
-def corpora_get_props_from_anndata(adata):
- """
- Get Corpora dataset properties from an AnnData
- """
- versions = corpora_get_versions_from_anndata(adata)
- if versions is None:
- return None
- [corpora_schema_version, corpora_encoding_version] = versions
- version_is_supported = corpora_is_version_supported(corpora_schema_version, corpora_encoding_version)
- if not version_is_supported:
- raise ValueError("Unsupported Corpora schema version")
-
- corpora_props = {}
- for key in CorporaConstants.REQUIRED_SIMPLE_METADATA_FIELDS:
- if key not in adata.uns:
- raise KeyError(f"missing Corpora schema field {key}")
- corpora_props[key] = adata.uns[key]
-
- for key in CorporaConstants.OPTIONAL_JSON_ENCODED_METADATA_FIELD:
- if key not in adata.uns:
- continue
- try:
- corpora_props[key] = json.loads(adata.uns[key])
- except json.JSONDecodeError:
- raise json.JSONDecodeError(f"Corpora schema field {key} is expected to be a valid JSON string")
-
- for key in CorporaConstants.OPTIONAL_SIMPLE_METADATA_FIELDS:
- if key in adata.uns:
- corpora_props[key] = adata.uns[key]
-
- return corpora_props
diff --git a/backend/czi_hosted/common/health.py b/backend/czi_hosted/common/health.py
deleted file mode 100644
index db13630d..00000000
--- a/backend/czi_hosted/common/health.py
+++ /dev/null
@@ -1,38 +0,0 @@
-from http import HTTPStatus
-from flask import make_response, jsonify
-
-from backend.czi_hosted import __version__ as cellxgene_version
-from backend.common.utils.data_locator import DataLocator
-
-
-def _is_accessible(path, config):
- if path is None:
- return True
-
- try:
- dl = DataLocator(path, region_name=config.data_locator__s3__region_name)
- return dl.exists()
- except RuntimeError:
- return False
-
-
-def health_check(config):
- """
- simple health check - return HTTP response.
- See https://tools.ietf.org/id/draft-inadarei-api-health-check-01.html
- """
- health = {"status": None, "version": "1", "releaseID": cellxgene_version}
-
- checks = False
- server_config = config.server_config
- if config.is_multi_dataset():
- dataroots = [datapath_dict["dataroot"] for datapath_dict in server_config.multi_dataset__dataroot.values()]
- checks = all([_is_accessible(dataroot, server_config) for dataroot in dataroots])
- else:
- checks = _is_accessible(server_config.single_dataset__datapath, server_config)
-
- health["status"] = "pass" if checks else "fail"
- code = HTTPStatus.OK if health["status"] == "pass" else HTTPStatus.BAD_REQUEST
- response = make_response(jsonify(health), code)
- response.headers["Content-Type"] = "application/health+json"
- return response
diff --git a/backend/czi_hosted/common/immutable_kvcache.py b/backend/czi_hosted/common/immutable_kvcache.py
deleted file mode 100644
index dddfe879..00000000
--- a/backend/czi_hosted/common/immutable_kvcache.py
+++ /dev/null
@@ -1,71 +0,0 @@
-import threading
-from collections.abc import MutableMapping
-
-
-class ImmutableKVCache(MutableMapping):
- """
- Guarantees that the factory will be called for each key once, and
- only once.
- """
-
- def __init__(self, factory):
- self.factory = factory # user-provided factory function
- self.lock = threading.Lock() # guards factory_calls
- self.factory_calls = {} # per-key factory condition variables
- self.cache = {} # result cache, indexed by key
- super().__init__()
-
- def __getitem__(self, key):
- if key in self.cache:
- return self.cache[key]
-
- # we need to call factory. First grab the main lock and the per-key CV.
- factory_calls = None
- creation_thr = False
- with self.lock:
- if key in self.cache:
- return self.cache[key]
- if key not in self.factory_calls:
- creation_thr = True
- self.factory_calls[key] = {"cv": threading.Condition(), "is_done": False, "error": None}
- factory_calls = self.factory_calls[key]
-
- # with the CV, create the value (or wait for it to be created)
- cv = factory_calls["cv"]
- with cv:
- if creation_thr:
- try:
- self.cache[key] = self.factory(key)
- except Exception as e:
- factory_calls["error"] = e
-
- factory_calls["is_done"] = True
- cv.notify_all()
- else:
- """ wait for the value to be available """
- while not factory_calls["is_done"]:
- cv.wait()
-
- with self.lock:
- if key in self.factory_calls:
- del self.factory_calls[key]
-
- return self.cache[key]
-
- def __iter__(self):
- """ weak iter, don't call factory """
- return self.cache.__iter__()
-
- def __len__(self):
- return self.cache.__len__()
-
- def __contains__(self, key):
- """ weak contain - don't call factory """
- return self.cache.__contains__(key)
-
- def __delitem__(self, key):
- del self.cache[key]
-
- def __setitem__(self, key, value):
- """ unsupported """
- raise NotImplementedError
diff --git a/backend/czi_hosted/common/rest.py b/backend/czi_hosted/common/rest.py
deleted file mode 100644
index 5ca17d1c..00000000
--- a/backend/czi_hosted/common/rest.py
+++ /dev/null
@@ -1,381 +0,0 @@
-import copy
-import logging
-import sys
-from http import HTTPStatus
-import zlib
-import json
-
-from flask import make_response, jsonify, current_app, abort
-from werkzeug.urls import url_unquote
-
-from backend.czi_hosted.common.config.client_config import get_client_config, get_client_userinfo
-from backend.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg
-from backend.common.errors import (
- FilterError,
- JSONEncodingValueError,
- PrepareError,
- DisabledFeatureError,
- ExceedsLimitError,
- DatasetAccessError,
- ColorFormatException,
- AnnotationsError,
- UnsupportedSummaryMethod,
-)
-from backend.common.genesets import summarizeQueryHash
-from backend.common.fbs.matrix import decode_matrix_fbs
-
-
-def abort_and_log(code, logmsg, loglevel=logging.DEBUG, include_exc_info=False):
- """
- Log the message, then abort with HTTP code. If include_exc_info is true,
- also include current exception via sys.exc_info().
- """
- if include_exc_info:
- exc_info = sys.exc_info()
- else:
- exc_info = False
- current_app.logger.log(loglevel, logmsg, exc_info=exc_info)
- # Do NOT send log message to HTTP response.
- return abort(code)
-
-
-def _query_parameter_to_filter(args):
- """
- Convert an annotation value filter, if present in the query args,
- into the standard dict filter format used by internal code.
-
- Query param filters look like: :name=value, where value
- may be one of:
- - a range, min,max, where either may be an open range by using an asterisc, eg, 10,*
- - a value
- Eg,
- ...?tissue=lung&obs:tissue=heart&obs:num_reads=1000,*
- """
- filters = {
- "obs": {},
- "var": {},
- }
-
- # args has already been url-unquoted once. We assume double escaping
- # on name and value.
- try:
- for key, value in args.items(multi=True):
- axis, name = key.split(":")
- if axis not in ("obs", "var"):
- raise FilterError("unknown filter axis")
- name = url_unquote(name)
- current = filters[axis].setdefault(name, {"name": name})
-
- val_split = value.split(",")
- if len(val_split) == 1:
- if "min" in current or "max" in current:
- raise FilterError("do not mix range and value filters")
- value = url_unquote(value)
- values = current.setdefault("values", [])
- values.append(value)
-
- elif len(val_split) == 2:
- if len(current) > 1:
- raise FilterError("duplicate range specification")
- min = url_unquote(val_split[0])
- max = url_unquote(val_split[1])
- if min != "*":
- current["min"] = float(min)
- if max != "*":
- current["max"] = float(max)
- if len(current) < 2:
- raise FilterError("must specify at least min or max in range filter")
-
- else:
- raise FilterError("badly formated filter value")
-
- except ValueError as e:
- raise FilterError(str(e))
-
- result = {}
- for axis in ("obs", "var"):
- axis_filter = filters[axis]
- if len(axis_filter) > 0:
- result[axis] = {"annotation_value": [val for val in axis_filter.values()]}
-
- return result
-
-
-def schema_get_helper(data_adaptor):
- """helper function to gather the schema from the data source and annotations"""
- schema = data_adaptor.get_schema()
- schema = copy.deepcopy(schema)
-
- # add label obs annotations as needed
- annotations = data_adaptor.dataset_config.user_annotations
- if annotations.user_annotations_enabled():
- label_schema = annotations.get_schema(data_adaptor)
- schema["annotations"]["obs"]["columns"].extend(label_schema)
-
- return schema
-
-
-def schema_get(data_adaptor):
- schema = schema_get_helper(data_adaptor)
- return make_response(jsonify({"schema": schema}), HTTPStatus.OK)
-
-
-def config_get(app_config, data_adaptor):
- config = get_client_config(app_config, data_adaptor)
- return make_response(jsonify(config), HTTPStatus.OK)
-
-
-def userinfo_get(app_config, data_adaptor):
- config = get_client_userinfo(app_config, data_adaptor)
- return make_response(jsonify(config), HTTPStatus.OK)
-
-
-def annotations_obs_get(request, data_adaptor):
- fields = request.args.getlist("annotation-name", None)
- num_columns_requested = len(data_adaptor.get_obs_keys()) if len(fields) == 0 else len(fields)
- if data_adaptor.server_config.exceeds_limit("column_request_max", num_columns_requested):
- return abort(HTTPStatus.BAD_REQUEST)
- preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
- if preferred_mimetype != "application/octet-stream":
- return abort(HTTPStatus.NOT_ACCEPTABLE)
-
- try:
- labels = None
- annotations = data_adaptor.dataset_config.user_annotations
- if annotations.user_annotations_enabled():
- labels = annotations.read_labels(data_adaptor)
- fbs = data_adaptor.annotation_to_fbs_matrix(Axis.OBS, fields, labels)
- return make_response(fbs, HTTPStatus.OK, {"Content-Type": "application/octet-stream"})
- except KeyError as e:
- return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
-
-
-def annotations_put_fbs_helper(data_adaptor, fbs):
- """helper function to write annotations from fbs"""
- annotations = data_adaptor.dataset_config.user_annotations
- if not annotations.user_annotations_enabled():
- raise DisabledFeatureError("Writable annotations are not enabled")
-
- new_label_df = decode_matrix_fbs(fbs)
- if not new_label_df.empty:
- new_label_df = data_adaptor.check_new_labels(new_label_df)
- annotations.write_labels(new_label_df, data_adaptor)
-
-
-def inflate(data):
- return zlib.decompress(data)
-
-
-def annotations_obs_put(request, data_adaptor):
- annotations = data_adaptor.dataset_config.user_annotations
- if not annotations.user_annotations_enabled():
- return abort(HTTPStatus.NOT_IMPLEMENTED)
-
- anno_collection = request.args.get("annotation-collection-name", default=None)
- fbs = inflate(request.get_data())
-
- if anno_collection is not None:
- if not annotations.is_safe_collection_name(anno_collection):
- return abort(HTTPStatus.BAD_REQUEST, "Bad annotation collection name")
- annotations.set_collection(anno_collection)
-
- try:
- annotations_put_fbs_helper(data_adaptor, fbs)
- res = json.dumps({"status": "OK"})
- return make_response(res, HTTPStatus.OK, {"Content-Type": "application/json"})
- except (ValueError, DisabledFeatureError, KeyError) as e:
- return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
-
-
-def annotations_var_get(request, data_adaptor):
- fields = request.args.getlist("annotation-name", None)
- num_columns_requested = len(data_adaptor.get_var_keys()) if len(fields) == 0 else len(fields)
- if data_adaptor.server_config.exceeds_limit("column_request_max", num_columns_requested):
- return abort(HTTPStatus.BAD_REQUEST)
- preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
- if preferred_mimetype != "application/octet-stream":
- return abort(HTTPStatus.NOT_ACCEPTABLE)
-
- try:
- labels = None
- annotations = data_adaptor.dataset_config.user_annotations
- if annotations.user_annotations_enabled():
- labels = annotations.read_labels(data_adaptor)
- return make_response(
- data_adaptor.annotation_to_fbs_matrix(Axis.VAR, fields, labels),
- HTTPStatus.OK,
- {"Content-Type": "application/octet-stream"},
- )
- except KeyError as e:
- return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
-
-
-def data_var_put(request, data_adaptor):
- preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
- if preferred_mimetype != "application/octet-stream":
- return abort(HTTPStatus.NOT_ACCEPTABLE)
-
- filter_json = request.get_json()
- filter = filter_json["filter"] if filter_json else None
- try:
- return make_response(
- data_adaptor.data_frame_to_fbs_matrix(filter, axis=Axis.VAR),
- HTTPStatus.OK,
- {"Content-Type": "application/octet-stream"},
- )
- except (FilterError, ValueError, ExceedsLimitError) as e:
- return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
-
-
-def data_var_get(request, data_adaptor):
- preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
- if preferred_mimetype != "application/octet-stream":
- return abort(HTTPStatus.NOT_ACCEPTABLE)
-
- try:
- filter = _query_parameter_to_filter(request.args)
- return make_response(
- data_adaptor.data_frame_to_fbs_matrix(filter, axis=Axis.VAR),
- HTTPStatus.OK,
- {"Content-Type": "application/octet-stream"},
- )
- except (FilterError, ValueError, ExceedsLimitError) as e:
- return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
-
-
-def colors_get(data_adaptor):
- if not data_adaptor.dataset_config.presentation__custom_colors:
- return make_response(jsonify({}), HTTPStatus.OK)
- try:
- return make_response(jsonify(data_adaptor.get_colors()), HTTPStatus.OK)
- except ColorFormatException as e:
- return abort_and_log(HTTPStatus.NOT_FOUND, str(e), include_exc_info=True)
-
-
-def diffexp_obs_post(request, data_adaptor):
- if not data_adaptor.dataset_config.diffexp__enable:
- return abort(HTTPStatus.NOT_IMPLEMENTED)
-
- args = request.get_json()
- try:
- # TODO: implement varfilter mode
- mode = DiffExpMode(args["mode"])
- if mode == DiffExpMode.VAR_FILTER or "varFilter" in args:
- return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, "varFilter not enabled")
-
- set1_filter = args.get("set1", {"filter": {}})["filter"]
- set2_filter = args.get("set2", {"filter": {}})["filter"]
- # TODO(#1281): When we simplify the config, we should actually use the config to determine this number,
- # this will also require an update in the client
- count = 15
-
- if set1_filter is None or set2_filter is None or count is None:
- return abort_and_log(HTTPStatus.BAD_REQUEST, "missing required parameter")
- if Axis.VAR in set1_filter or Axis.VAR in set2_filter:
- return abort_and_log(HTTPStatus.BAD_REQUEST, "var axis filter not enabled")
-
- except (KeyError, TypeError) as e:
- return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
-
- try:
- diffexp = data_adaptor.diffexp_topN(set1_filter, set2_filter, count)
- return make_response(diffexp, HTTPStatus.OK, {"Content-Type": "application/json"})
- except (ValueError, DisabledFeatureError, FilterError, ExceedsLimitError) as e:
- return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
- except JSONEncodingValueError:
- # JSON encoding failure, usually due to bad data. Just let it ripple up
- # to default exception handler.
- current_app.logger.warning(JSON_NaN_to_num_warning_msg)
- raise
-
-
-def layout_obs_get(request, data_adaptor):
- fields = request.args.getlist("layout-name", None)
- num_columns_requested = len(data_adaptor.get_embedding_names()) if len(fields) == 0 else len(fields)
- if data_adaptor.server_config.exceeds_limit("column_request_max", num_columns_requested):
- return abort(HTTPStatus.BAD_REQUEST)
-
- preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
- if preferred_mimetype != "application/octet-stream":
- return abort(HTTPStatus.NOT_ACCEPTABLE)
-
- try:
- return make_response(
- data_adaptor.layout_to_fbs_matrix(fields), HTTPStatus.OK, {"Content-Type": "application/octet-stream"}
- )
- except (KeyError, DatasetAccessError) as e:
- return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
- except PrepareError:
- return abort_and_log(
- HTTPStatus.NOT_IMPLEMENTED,
- f"No embedding available {request.path}",
- loglevel=logging.ERROR,
- include_exc_info=True,
- )
-
-
-def genesets_get(request, data_adaptor):
- preferred_mimetype = request.accept_mimetypes.best_match(["application/json", "text/csv"])
- if preferred_mimetype not in ("application/json", "text/csv"):
- return abort(HTTPStatus.NOT_ACCEPTABLE)
-
- try:
- annotations = data_adaptor.dataset_config.user_annotations
- (genesets, tid) = annotations.read_gene_sets(data_adaptor)
-
- if preferred_mimetype == "text/csv":
- return make_response(
- annotations.gene_sets_to_csv(genesets),
- HTTPStatus.OK,
- {
- "Content-Type": "text/csv",
- "Content-Disposition": "attachment; filename=genesets.csv",
- },
- )
- else:
- return make_response(
- jsonify({"genesets": annotations.gene_sets_to_response(genesets), "tid": tid}), HTTPStatus.OK
- )
- except (ValueError, KeyError, AnnotationsError) as e:
- return abort_and_log(HTTPStatus.BAD_REQUEST, str(e))
-
-
-def summarize_var_helper(request, data_adaptor, key, raw_query):
- preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
- if preferred_mimetype != "application/octet-stream":
- return abort(HTTPStatus.NOT_ACCEPTABLE)
-
- summary_method = request.values.get("method", default="mean")
- query_hash = summarizeQueryHash(raw_query)
- if key and query_hash != key:
- return abort(HTTPStatus.BAD_REQUEST, description="query key did not match")
-
- args_filter_only = request.values.copy()
- args_filter_only.poplist("method")
- args_filter_only.poplist("key")
-
- try:
- filter = _query_parameter_to_filter(args_filter_only)
- return make_response(
- data_adaptor.summarize_var(summary_method, filter, query_hash),
- HTTPStatus.OK,
- {"Content-Type": "application/octet-stream"},
- )
- except (ValueError) as e:
- return abort(HTTPStatus.NOT_FOUND, description=str(e))
- except (UnsupportedSummaryMethod, FilterError) as e:
- return abort(HTTPStatus.BAD_REQUEST, description=str(e))
-
-
-def summarize_var_get(request, data_adaptor):
- return summarize_var_helper(request, data_adaptor, None, request.query_string)
-
-
-def summarize_var_post(request, data_adaptor):
- if not request.content_type or "application/x-www-form-urlencoded" not in request.content_type:
- return abort(HTTPStatus.UNSUPPORTED_MEDIA_TYPE)
- if request.content_length > 1_000_000: # just a sanity check to avoid memory exhaustion
- return abort(HTTPStatus.BAD_REQUEST)
-
- key = request.args.get("key", default=None)
- return summarize_var_helper(request, data_adaptor, key, request.get_data())
diff --git a/backend/czi_hosted/common/utils/__init__.py b/backend/czi_hosted/common/utils/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/backend/czi_hosted/common/utils/corpora_constants.py b/backend/czi_hosted/common/utils/corpora_constants.py
deleted file mode 100644
index fa1641d5..00000000
--- a/backend/czi_hosted/common/utils/corpora_constants.py
+++ /dev/null
@@ -1,22 +0,0 @@
-class CorporaConstants(object):
- REQUIRED_SIMPLE_METADATA_FIELDS = [
- "version",
- "title",
- "layer_descriptions",
- "organism",
- "organism_ontology_term_id",
- ]
-
- # The Corpora specification requires some values encoded as JSON due to the inability of AnnData to store complex
- # types.
- OPTIONAL_JSON_ENCODED_METADATA_FIELD = ["contributors", "project_links"]
-
- OPTIONAL_SIMPLE_METADATA_FIELDS = [
- "preprint_doi",
- "publication_doi",
- "default_embedding",
- "default_field",
- "tags",
- "project_name",
- "project_description",
- ]
diff --git a/backend/czi_hosted/common/utils/cxg_constants.py b/backend/czi_hosted/common/utils/cxg_constants.py
deleted file mode 100644
index 5982ec15..00000000
--- a/backend/czi_hosted/common/utils/cxg_constants.py
+++ /dev/null
@@ -1,4 +0,0 @@
-class CxgConstants(object):
- # The CXG container version number. Must be a semver string (major.minor.patch)
- # DO NOT UPDATE THIS WITHOUT ALSO UPDATING CXG SPECIFICATION.
- CXG_VERSION = "0.2.0"
diff --git a/backend/czi_hosted/common/utils/cxg_generation_utils.py b/backend/czi_hosted/common/utils/cxg_generation_utils.py
deleted file mode 100644
index c3c1c867..00000000
--- a/backend/czi_hosted/common/utils/cxg_generation_utils.py
+++ /dev/null
@@ -1,178 +0,0 @@
-import json
-
-import numpy as np
-import tiledb
-
-from backend.common.utils.type_conversion_utils import get_encoding_dtype_of_array, get_dtype_and_schema_of_array
-
-
-def convert_dictionary_to_cxg_group(cxg_container, metadata_dict, group_metadata_name="cxg_group_metadata"):
- """
- Saves the contents of the dictionary to the CXG output directory specified.
-
- This function is primarily used to save metadata about a dataset to the CXG directory. At some point, tiledb will
- have support for metadata on groups at which point the utility of this function should be revisited. Until such
- feature exists, this function create an empty array and annotate that array.
-
- For more information, visit https://github.com/TileDB-Inc/TileDB-Py/issues/254.
- """
-
- array_name = f"{cxg_container}/{group_metadata_name}"
-
- # Because TileDB does not allow one to attach metadata directly to a CXG group, we need to have a workaround
- # where we create an empty array and attached the metadata onto to this empty array. Below we construct this empty
- # array.
- tiledb.from_numpy(array_name, np.zeros((1,)))
-
- with tiledb.DenseArray(array_name, mode="w") as metadata_array:
- for key, value in metadata_dict.items():
- metadata_array.meta[key] = value
-
-
-def convert_dataframe_to_cxg_array(cxg_container, dataframe_name, dataframe, index_column_name, ctx):
- """
- Saves the contents of the dataframe to the CXG output directory specified.
-
- Current access patterns are oriented toward reading very large slices of the dataframe, one attribute at a time.
- Attribute data also tends to be (often) repetitive (bools, categories, strings). Given this, we use a large tile
- size (1000) and very aggressive compression levels.
- """
-
- def create_dataframe_array(array_name, dataframe):
- tiledb_filter = tiledb.FilterList(
- [
- # Attempt aggressive compression as many of these dataframes are very repetitive strings, bools and
- # other non-float data.
- tiledb.ZstdFilter(level=22),
- ]
- )
- attrs = [
- tiledb.Attr(name=column, dtype=get_encoding_dtype_of_array(dataframe[column]), filters=tiledb_filter)
- for column in dataframe
- ]
- domain = tiledb.Domain(
- tiledb.Dim(domain=(0, dataframe.shape[0] - 1), tile=min(dataframe.shape[0], 1000), dtype=np.uint32)
- )
- schema = tiledb.ArraySchema(
- domain=domain, sparse=False, attrs=attrs, cell_order="row-major", tile_order="row-major"
- )
- tiledb.DenseArray.create(array_name, schema)
-
- array_name = f"{cxg_container}/{dataframe_name}"
-
- create_dataframe_array(array_name, dataframe)
-
- with tiledb.DenseArray(array_name, mode="w", ctx=ctx) as array:
- value = {}
- schema_hints = {}
- for column_name, column_values in dataframe.items():
- dtype, hints = get_dtype_and_schema_of_array(column_values)
- value[column_name] = column_values.to_numpy(dtype=dtype)
- if hints:
- schema_hints.update({column_name: hints})
-
- schema_hints.update({"index": index_column_name})
- array[:] = value
- array.meta["cxg_schema"] = json.dumps(schema_hints)
-
- tiledb.consolidate(array_name, ctx=ctx)
-
-
-def convert_ndarray_to_cxg_dense_array(ndarray_name, ndarray, ctx):
- """
- Saves contents of ndarray to the CXG output directory specified.
-
- Generally this function is used to convert dataset embeddings. Because embeddings are typically accessed with
- very large slices (or all of the embedding), they do not benefit from overly aggressive compression due to their
- format. Given this, we use a large tile size (1000) but only default compression level.
- """
-
- def create_ndarray_array(ndarray_name, ndarray):
- filters = tiledb.FilterList([tiledb.ZstdFilter()])
- attrs = [tiledb.Attr(dtype=ndarray.dtype, filters=filters)]
- dimensions = [
- tiledb.Dim(
- domain=(0, ndarray.shape[dimension] - 1), tile=min(ndarray.shape[dimension], 1000), dtype=np.uint32
- )
- for dimension in range(ndarray.ndim)
- ]
- domain = tiledb.Domain(*dimensions)
- schema = tiledb.ArraySchema(
- domain=domain, sparse=False, attrs=attrs, capacity=1_000_000, cell_order="row-major", tile_order="row-major"
- )
- tiledb.DenseArray.create(ndarray_name, schema)
-
- create_ndarray_array(ndarray_name, ndarray)
-
- with tiledb.DenseArray(ndarray_name, mode="w", ctx=ctx) as array:
- array[:] = ndarray
-
- tiledb.consolidate(ndarray_name, ctx=ctx)
-
-
-def convert_matrix_to_cxg_array(
- matrix_name, matrix, encode_as_sparse_array, ctx, column_shift_for_sparse_encoding=None
-):
- """
- Converts a numpy array matrix into a TileDB SparseArray of DenseArray based on whether `encode_as_sparse_array`
- is true or not. Note that when the matrix is encoded as a SparseArray, it only writes the values that are
- nonzero. This means that if you count the number of elements in the SparseArray, it will not equal the total
- number of elements in the matrix, only the number of nonzero elements.
-
- Furthermore, if the `column_shift_for_sparse_encoding` matrix is not None, this function will subtract the sparse
- encoding from the original given matrix and as previously stated, only write the nonzero values to the TileDB
- SparseArray.
- """
-
- def create_matrix_array(matrix_name, number_of_rows, number_of_columns, encode_as_sparse_array):
- filters = tiledb.FilterList([tiledb.ZstdFilter()])
- attrs = [tiledb.Attr(dtype=np.float32, filters=filters)]
- if encode_as_sparse_array:
- domain = tiledb.Domain(
- tiledb.Dim(name="obs", domain=(0, number_of_rows - 1), tile=min(number_of_rows, 512), dtype=np.uint32),
- tiledb.Dim(
- name="var", domain=(0, number_of_columns - 1), tile=min(number_of_columns, 2048), dtype=np.uint32
- ),
- )
- else:
- domain = tiledb.Domain(
- tiledb.Dim(name="obs", domain=(0, number_of_rows - 1), tile=min(number_of_rows, 50), dtype=np.uint32),
- tiledb.Dim(
- name="var", domain=(0, number_of_columns - 1), tile=min(number_of_columns, 100), dtype=np.uint32
- ),
- )
- schema = tiledb.ArraySchema(
- domain=domain, sparse=encode_as_sparse_array, attrs=attrs, cell_order="row-major", tile_order="col-major"
- )
- if encode_as_sparse_array:
- tiledb.SparseArray.create(matrix_name, schema)
- else:
- tiledb.DenseArray.create(matrix_name, schema)
-
- number_of_rows = matrix.shape[0]
- number_of_columns = matrix.shape[1]
- stride = min(int(np.power(10, np.around(np.log10(1e9 / number_of_columns)))), 10_000)
-
- create_matrix_array(matrix_name, number_of_rows, number_of_columns, encode_as_sparse_array)
-
- if encode_as_sparse_array:
- with tiledb.SparseArray(matrix_name, mode="w", ctx=ctx) as array:
- for start_row_index in range(0, number_of_rows, stride):
- end_row_index = min(start_row_index + stride, number_of_rows)
- matrix_subset = matrix[start_row_index:end_row_index, :]
- if not isinstance(matrix_subset, np.ndarray):
- matrix_subset = matrix_subset.toarray()
- if column_shift_for_sparse_encoding is not None:
- matrix_subset = matrix_subset - column_shift_for_sparse_encoding
- indices = np.nonzero(matrix_subset)
- trow = indices[0] + start_row_index
- array[trow, indices[1]] = matrix_subset[indices[0], indices[1]]
-
- else:
- with tiledb.DenseArray(matrix_name, mode="w", ctx=ctx) as array:
- for start_row_index in range(0, number_of_rows, stride):
- end_row_index = min(start_row_index + stride, number_of_rows)
- matrix_subset = matrix[start_row_index:end_row_index, :]
- if not isinstance(matrix_subset, np.ndarray):
- matrix_subset = matrix_subset.toarray()
- array[start_row_index:end_row_index, :] = matrix_subset
diff --git a/backend/czi_hosted/common/utils/matrix_utils.py b/backend/czi_hosted/common/utils/matrix_utils.py
deleted file mode 100644
index 60dfb19b..00000000
--- a/backend/czi_hosted/common/utils/matrix_utils.py
+++ /dev/null
@@ -1,115 +0,0 @@
-import logging
-
-import numpy as np
-from scipy.stats import mode
-
-
-def is_matrix_sparse(matrix: np.ndarray, sparse_threshold):
- """
- Returns whether `matrix` is sparse or not (i.e. dense). This is determined by figuring out whether the matrix has
- a sparsity percentage below the sparse_threshold, returning the number of non-zeros encountered and number of
- elements evaluated. This function may return before evaluating the whole matrix if it can be determined that matrix
- is not sparse enough.
- """
-
- if sparse_threshold == 100.0:
- return True
- if sparse_threshold == 0.0:
- return False
-
- total_number_of_rows = matrix.shape[0]
- total_number_of_columns = matrix.shape[1]
- total_number_of_matrix_elements = total_number_of_rows * total_number_of_columns
-
- # For efficiency, we count the number of non-zero elements in chunks of the matrix at a time until we hit the
- # maximum number of non zero values allowed before the matrix is deemed "dense." This allows the function the
- # quit early for large dense matrices.
- row_stride = min(int(np.power(10, np.around(np.log10(1e9 / total_number_of_columns)))), 10_000)
-
- maximum_number_of_non_zero_elements_in_matrix = int(
- total_number_of_rows * total_number_of_columns * sparse_threshold / 100
- )
- number_of_non_zero_elements = 0
-
- for start_row_index in range(0, total_number_of_rows, row_stride):
- end_row_index = min(start_row_index + row_stride, total_number_of_rows)
-
- matrix_subset = matrix[start_row_index:end_row_index, :]
- if not isinstance(matrix_subset, np.ndarray):
- matrix_subset = matrix_subset.toarray()
-
- number_of_non_zero_elements += np.count_nonzero(matrix_subset)
- if number_of_non_zero_elements > maximum_number_of_non_zero_elements_in_matrix:
- if end_row_index != total_number_of_rows:
- percentage_of_non_zero_elements = (
- 100 * number_of_non_zero_elements / (end_row_index * total_number_of_columns)
- )
- logging.info(
- f"Matrix is not sparse. Percentage of non-zero elements (estimate): "
- f"{percentage_of_non_zero_elements:6.2f}"
- )
- else:
- percentage_of_non_zero_elements = 100 * number_of_non_zero_elements / total_number_of_matrix_elements
- logging.info(
- f"Matrix is not sparse. Percentage of non-zero elements (exact): "
- f"{percentage_of_non_zero_elements:6.2f}"
- )
- return False
-
- is_sparse = (100.0 * number_of_non_zero_elements / total_number_of_matrix_elements) < sparse_threshold
- return is_sparse
-
-
-def get_column_shift_encode_for_matrix(matrix, sparse_threshold):
- """
- Returns a column shift if there is a column shift that allows the given matrix to be considered as sparse. Column
- shift encoding works by taking the most common value in each column, then subtracting that value from each element
- of the column. If each column mostly contains its most common value, then the resulting matrix can be very sparse.
-
- This function determines if column shift encoding can be used to transform the matrix into a sparse matrix with a
- sparsity below the sparse_threshold. If so, returns the array that stores this encoding. This function also returns
- the number of non-zeros encountered and number of elements evaluated. This function may return before evaluating
- the whole matrix if it can be determined that the matrix cannot benefit from column shift encoding.
- """
-
- total_number_of_rows = matrix.shape[0]
- total_number_of_columns = matrix.shape[1]
- total_number_of_matrix_elements = total_number_of_rows * total_number_of_columns
-
- stride = max(1, 128_000_000 // total_number_of_rows)
- column_shift = np.zeros(total_number_of_columns)
-
- maximum_number_of_non_zero_elements_in_matrix = int(
- total_number_of_rows * total_number_of_columns * sparse_threshold / 100
- )
- number_of_non_zero_elements = 0
-
- for start_column_index in range(0, total_number_of_columns, stride):
- end_column_index = min(start_column_index + stride, total_number_of_columns)
-
- matrix_subset = matrix[:, start_column_index:end_column_index]
- if not isinstance(matrix_subset, np.ndarray):
- matrix_subset = matrix_subset.toarray()
-
- matrix_subset_mode = mode(matrix_subset)
-
- column_shift[start_column_index:end_column_index] = matrix_subset_mode.mode
- number_of_non_zero_elements += total_number_of_rows * (end_column_index - start_column_index) - np.sum(
- matrix_subset_mode.count
- )
-
- if number_of_non_zero_elements > maximum_number_of_non_zero_elements_in_matrix:
- if end_column_index != total_number_of_columns:
- logging.info(
- "Matrix is not sparse even with column shift. Percentage of non-zero elements (estimate): %6.2f"
- % (100 * number_of_non_zero_elements / end_column_index * total_number_of_rows)
- )
- else:
- logging.info(
- "Matrix is not sparse even with column shift. Percentage of non-zero elements (exact): %6.2f"
- % (100 * number_of_non_zero_elements / total_number_of_matrix_elements)
- )
- return None
-
- is_sparse = (100.0 * number_of_non_zero_elements / total_number_of_matrix_elements) < sparse_threshold
- return column_shift if is_sparse else None
diff --git a/backend/czi_hosted/common/utils/sanitization_utils.py b/backend/czi_hosted/common/utils/sanitization_utils.py
deleted file mode 100644
index af6f299f..00000000
--- a/backend/czi_hosted/common/utils/sanitization_utils.py
+++ /dev/null
@@ -1,40 +0,0 @@
-import re
-
-
-def sanitize_values_in_list(list_of_keys: list):
- """
- Returns a dictionary mapping of the old keys in the list of `list_of_keys` to its new, clean name that is both
- safe and unique.
- """
-
- if not all([isinstance(key, str) for key in list_of_keys]):
- raise Exception("List of keys to sanitize must contain all strings.")
-
- # Mask out [~/.] and anything outside the ASCII range.
- mask = re.compile(r"[^ -\-0-\[\]-\}]")
- clean_keys_list = [mask.sub("_", key) for key in list_of_keys]
-
- # Dedupe the clean keys list
- deduped_clean_keys_list = []
- for index, clean_key in enumerate(clean_keys_list):
- total_occurrences_of_clean_key = clean_keys_list.count(clean_key)
- total_occurrences_up_until_current_index = clean_keys_list[:index].count(clean_key)
- deduped_clean_keys_list.append(
- clean_key + "_" + str(total_occurrences_up_until_current_index + 1)
- if total_occurrences_of_clean_key > 1
- else clean_key
- )
-
- return dict(zip(list_of_keys, deduped_clean_keys_list))
-
-
-def sanitize_keys_in_dictionary(dict_to_sanitize: dict):
- """
- Clean and dedupe the keys in the given dictionary.
- """
-
- clean_keys = sanitize_values_in_list(dict_to_sanitize.keys())
- for original_key, sanitized_key in clean_keys.items():
- if original_key != sanitized_key:
- dict_to_sanitize[sanitized_key] = dict_to_sanitize[original_key]
- del dict_to_sanitize[original_key]
diff --git a/backend/czi_hosted/common/web/__init__.py b/backend/czi_hosted/common/web/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/backend/czi_hosted/compute/__init__.py b/backend/czi_hosted/compute/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/backend/czi_hosted/compute/diffexp_cxg.py b/backend/czi_hosted/compute/diffexp_cxg.py
deleted file mode 100644
index d37d866c..00000000
--- a/backend/czi_hosted/compute/diffexp_cxg.py
+++ /dev/null
@@ -1,199 +0,0 @@
-import concurrent.futures
-import numpy as np
-
-from numba import jit
-
-from backend.czi_hosted.data_cxg.cxg_util import pack_selector_from_indices
-from backend.common.compute.diffexp_generic import diffexp_ttest_from_mean_var, mean_var_n
-from backend.common.errors import ComputeError
-
-"""
-See the comments in diffexp_generic for a description of this algorithm
-
-This implementation runs directly in-process. It is multi- threaded, but not particularly scalable.
-Longer term, will likely move to a distributed framework for this.
-
-There are currently no global throttles on simultaneous workers.
-"""
-
-diffexp_thread_executor = None
-max_workers = None
-target_workunit = None
-
-
-def set_config(config_max_workers, config_target_workunit):
- global max_workers
- global target_workunit
- max_workers = config_max_workers
- target_workunit = config_target_workunit
-
-
-def get_thread_executor():
- global diffexp_thread_executor
- if diffexp_thread_executor is None:
- diffexp_thread_executor = concurrent.futures.ThreadPoolExecutor(max_workers=max_workers)
- return diffexp_thread_executor
-
-
-def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
-
- matrix = adaptor.open_array("X")
- row_selector_A = np.where(maskA)[0]
- row_selector_B = np.where(maskB)[0]
- nA = len(row_selector_A)
- nB = len(row_selector_B)
-
- dtype = matrix.dtype
- cols = matrix.shape[1]
- tile_extent = [dim.tile for dim in matrix.schema.domain]
-
- is_sparse = matrix.schema.sparse
-
- if is_sparse:
- row_selector_A = pack_selector_from_indices(row_selector_A)
- row_selector_B = pack_selector_from_indices(row_selector_B)
- else:
- # The rows from both row_selector_A and row_selector_B are gathered at the
- # same time, then the mean and variance are computed by subsetting on that
- # combined submatrix. Combining the gather reduces number of requests/bandwidth
- # to the data source.
- row_selector_AB = np.union1d(row_selector_A, row_selector_B)
- row_selector_A_in_AB = np.in1d(row_selector_AB, row_selector_A, assume_unique=True)
- row_selector_B_in_AB = np.in1d(row_selector_AB, row_selector_B, assume_unique=True)
- row_selector_AB = pack_selector_from_indices(row_selector_AB)
-
- # because all IO is done per-tile, and we are always col-major,
- # use the tile column size as the unit of partition. Possibly access
- # more than one column tile at a time based on the target_workunit.
- # Revisit partitioning if we change the X layout, or start using a non-local execution environment
- # which may have other constraints.
-
- # TODO: If the number of row selections is large enough, then the cells_per_coltile will exceed
- # the target_workunit. A potential improvement would be to partition by both columns and rows.
- # However partitioning the rows is slightly more complex due to the arbitrary distribution
- # of row selections that are passed into this algorithm.
-
- cells_per_coltile = (nA + nB) * tile_extent[1]
- cols_per_partition = max(1, int(target_workunit / cells_per_coltile)) * tile_extent[1]
- col_partitions = [(c, min(c + cols_per_partition, cols)) for c in range(0, cols, cols_per_partition)]
-
- meanA = np.zeros((cols,), dtype=np.float64)
- varA = np.zeros((cols,), dtype=np.float64)
- meanB = np.zeros((cols,), dtype=np.float64)
- varB = np.zeros((cols,), dtype=np.float64)
-
- executor = get_thread_executor()
- futures = []
-
- if is_sparse:
- for cols in col_partitions:
- futures.append(executor.submit(_mean_var_sparse_ab, matrix, row_selector_A, nA, row_selector_B, nB, cols))
- else:
- for cols in col_partitions:
- futures.append(
- executor.submit(_mean_var_ab, matrix, row_selector_AB, row_selector_A_in_AB, row_selector_B_in_AB, cols)
- )
-
- for future in futures:
- # returns tuple: (meanA, varA, meanB, varB, cols)
- try:
- result = future.result()
- part_meanA, part_varA, part_meanB, part_varB, cols = result
- meanA[cols[0] : cols[1]] += part_meanA
- varA[cols[0] : cols[1]] += part_varA
- meanB[cols[0] : cols[1]] += part_meanB
- varB[cols[0] : cols[1]] += part_varB
- except Exception as e:
- for future in futures:
- future.cancel()
- raise ComputeError(str(e))
-
- if is_sparse:
- if adaptor.has_array("X_col_shift"):
- X_col_shift = adaptor.open_array("X_col_shift")[:]
- meanA += X_col_shift
- meanB += X_col_shift
-
- r = diffexp_ttest_from_mean_var(
- meanA=meanA.astype(dtype),
- varA=varA.astype(dtype),
- nA=nA,
- meanB=meanB.astype(dtype),
- varB=varB.astype(dtype),
- nB=nB,
- top_n=top_n,
- diffexp_lfc_cutoff=diffexp_lfc_cutoff
- )
-
- return r
-
-
-def _mean_var_ab(matrix, row_selector_AB, row_selector_A_in_AB, row_selector_B_in_AB, col_range):
- X = matrix.multi_index[row_selector_AB, col_range[0] : col_range[1] - 1][""]
- meanA, varA, n = mean_var_n(X[row_selector_A_in_AB])
- meanB, varB, n = mean_var_n(X[row_selector_B_in_AB])
- return (meanA, varA, meanB, varB, col_range)
-
-
-def _mean_var_sparse_ab(matrix, row_selector_A, nrows_A, row_selector_B, nrows_B, col_range):
- meanA, varA = _mean_var_sparse(matrix, row_selector_A, nrows_A, col_range)
- meanB, varB = _mean_var_sparse(matrix, row_selector_B, nrows_B, col_range)
- return (meanA, varA, meanB, varB, col_range)
-
-
-@jit(nopython=True)
-def _mean_var_sparse_numba(x, var, nrows, ncols):
- """Kernel to compute the mean and variance. It was not clear if this function
- could be written using numpy, thus avoiding the loops. Therefore numba is
- used here to speed things up. With numba, this function takes a negligible amount
- of time compared to reading in the sparse matrix"""
- mean = np.zeros((ncols,), dtype=np.float64)
- for col, val in zip(var, x):
- mean[col] += val
- mean /= nrows
-
- # optimize the sumsq computation.
- # since most entries in a sparse matrix are 0, then start by assuming
- # all values are 0, so fill the sumsq array with nrows * (0 - mean)**2.
- # as non-zero values are encountered, subtract off the (mean*mean) value
- # and replace with (val-mean)**2. Simplifying the expression
- # gives the following code.
- sumsq = nrows * np.multiply(mean, mean)
- for col, val in zip(var, x):
- sumsq[col] += val * (val - 2 * mean[col])
- v = sumsq / (nrows - 1)
- return mean, v
-
-
-def _mean_var_sparse(matrix, selector, nrows, col_range):
- data = matrix.multi_index[selector, col_range[0] : col_range[1] - 1]
- x = data[""]
-
- # tiledb < 0.6.0 and >= 0.6.0 have slightly different interfaces.
- # the following takes care of both cases:
- # older: data["coords]["var"]
- # newer: data["var"]
- var = data.get("coords", data)["var"]
-
- # shift the column indices to start at 0, this
- # will become the index into the mean and var arrays.
- var -= col_range[0]
-
- fp_err_occurred = False
-
- def fp_err_set(err, flag):
- nonlocal fp_err_occurred
- fp_err_occurred = True
-
- ncols = col_range[1] - col_range[0]
- with np.errstate(divide="call", invalid="call", call=fp_err_set):
- mean, v = _mean_var_sparse_numba(x, var, nrows, ncols)
-
- if fp_err_occurred:
- mean[np.isfinite(mean) == False] = 0 # noqa: E712
- v[np.isfinite(v) == False] = 0 # noqa: E712
- else:
- mean[np.isnan(mean)] = 0
- v[np.isnan(v)] = 0
-
- return mean, v
diff --git a/backend/czi_hosted/converters/__init__.py b/backend/czi_hosted/converters/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/backend/czi_hosted/converters/h5ad_data_file.py b/backend/czi_hosted/converters/h5ad_data_file.py
deleted file mode 100644
index 7b09030b..00000000
--- a/backend/czi_hosted/converters/h5ad_data_file.py
+++ /dev/null
@@ -1,250 +0,0 @@
-import json
-import logging
-from os import path
-
-import anndata
-import numpy as np
-import tiledb
-
-from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
-from backend.czi_hosted.common.corpora import corpora_get_props_from_anndata
-from backend.common.errors import ColorFormatException
-from backend.czi_hosted.common.utils.cxg_constants import CxgConstants
-from backend.czi_hosted.common.utils.cxg_generation_utils import (
- convert_dictionary_to_cxg_group,
- convert_dataframe_to_cxg_array,
- convert_ndarray_to_cxg_dense_array,
- convert_matrix_to_cxg_array,
-)
-from backend.czi_hosted.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix
-
-
-class H5ADDataFile:
- """ Class encapsulating required information about an H5AD datafile that ultimately will be transformed into
- another format (currently just CXG is supported). """
-
- def __init__(
- self,
- input_filename,
- backed=False,
- dataset_title=None,
- dataset_about=None,
- obs_index_column_name=None,
- vars_index_column_name=None,
- use_corpora_schema=True,
- ):
- self.input_filename = input_filename
- self.backed = backed
- self.dataset_title = dataset_title
- self.dataset_about = dataset_about
- self.obs_index_column_name = obs_index_column_name
- self.vars_index_column_name = vars_index_column_name
-
- self.use_corpora_schema = use_corpora_schema
-
- self.validate_input_file_type()
-
- self.extract_anndata_elements_from_file()
- self.extract_metadata_about_dataset()
-
- self.validate_anndata()
-
- def to_cxg(self, output_cxg_directory, sparse_threshold, convert_anndata_colors_to_cxg_colors=True):
- """
- Writes the following attributes of the anndata to CXG: 1) the metadata as metadata attached to an empty
- DenseArray, 2) the obs DataFrame as a DenseArray, 3) the var DataFrame as a DenseArray, 4) all valid
- embeddings stored in obsm, each one as a DenseArray, 5) the main X matrix of the anndata as either a
- SparseArray or DenseArray based on the `sparse_threshold`, and optionally 6) the column shift of the main X
- matrix that might turn an otherwise Dense matrix into a Sparse matrix.
- """
-
- logging.info("Beginning writing to CXG.")
- ctx = tiledb.Ctx(
- {
- "sm.num_reader_threads": 32,
- "sm.num_writer_threads": 32,
- "sm.consolidation.buffer_size": 1 * 1024 * 1024 * 1024,
- }
- )
-
- tiledb.group_create(output_cxg_directory, ctx=ctx)
- logging.info(f"\t...group created, with name {output_cxg_directory}")
-
- convert_dictionary_to_cxg_group(
- output_cxg_directory, self.generate_cxg_metadata(convert_anndata_colors_to_cxg_colors)
- )
- logging.info("\t...dataset metadata saved")
-
- convert_dataframe_to_cxg_array(output_cxg_directory, "obs", self.obs, self.obs_index_column_name, ctx)
- logging.info("\t...dataset obs dataframe saved")
-
- convert_dataframe_to_cxg_array(output_cxg_directory, "var", self.var, self.var_index_column_name, ctx)
- logging.info("\t...dataset var dataframe saved")
-
- self.write_anndata_embeddings_to_cxg(output_cxg_directory, ctx)
- logging.info("\t...dataset embeddings saved")
-
- self.write_anndata_x_matrix_to_cxg(output_cxg_directory, ctx, sparse_threshold)
- logging.info("\t...dataset X matrix saved")
-
- logging.info("Completed writing to CXG.")
-
- def write_anndata_x_matrix_to_cxg(self, output_cxg_directory, ctx, sparse_threshold):
- matrix_container = f"{output_cxg_directory}/X"
-
- x_matrix_data = self.anndata.X
- is_sparse = is_matrix_sparse(x_matrix_data, sparse_threshold)
- if not is_sparse:
- col_shift = get_column_shift_encode_for_matrix(x_matrix_data, sparse_threshold)
- is_sparse = col_shift is not None
- else:
- col_shift = None
-
- if col_shift is not None:
- logging.info("Converting matrix X as sparse matrix with column shift encoding")
- x_col_shift_name = f"{output_cxg_directory}/X_col_shift"
- convert_ndarray_to_cxg_dense_array(x_col_shift_name, col_shift, ctx)
-
- convert_matrix_to_cxg_array(matrix_container, x_matrix_data, is_sparse, ctx, col_shift)
-
- tiledb.consolidate(matrix_container, ctx=ctx)
- if hasattr(tiledb, "vacuum"):
- tiledb.vacuum(matrix_container)
-
- def write_anndata_embeddings_to_cxg(self, output_cxg_directory, ctx):
- def is_valid_embedding(adata, embedding_name, embedding_array):
- """
- Returns true if this layout data is a valid array for front-end presentation with the following criteria:
- * ndarray, with shape (n_obs, >= 2), dtype float/int/uint
- * follows ScanPy embedding naming conventions
- * with all values finite or NaN (no +Inf or -Inf)
- """
-
- is_valid = isinstance(embedding_name, str) and embedding_name.startswith("X_") and len(embedding_name) > 2
- is_valid = is_valid and isinstance(embedding_array, np.ndarray) and embedding_array.dtype.kind in "fiu"
- is_valid = is_valid and embedding_array.shape[0] == adata.n_obs and embedding_array.shape[1] >= 2
- is_valid = is_valid and not np.any(np.isinf(embedding_array)) and not np.all(np.isnan(embedding_array))
- return is_valid
-
- embedding_container = f"{output_cxg_directory}/emb"
- tiledb.group_create(embedding_container, ctx=ctx)
-
- for embedding_name, embedding_values in self.anndata.obsm.items():
- if is_valid_embedding(self.anndata, embedding_name, embedding_values):
- embedding_name = f"{embedding_container}/{embedding_name[2:]}"
- convert_ndarray_to_cxg_dense_array(embedding_name, embedding_values, ctx)
- logging.info(f"\t\t...{embedding_name} embedding created")
-
- def generate_cxg_metadata(self, convert_anndata_colors_to_cxg_colors):
- """
- Return a dictionary containing metadata about CXG dataset. This include data about the version as well as
- Corpora schema properties if they exist, among other pieces of metadata.
- """
-
- cxg_group_metadata = {
- "cxg_version": CxgConstants.CXG_VERSION,
- "cxg_properties": json.dumps({"title": self.dataset_title, "about": self.dataset_about}),
- }
- if self.corpora_properties is not None:
- cxg_group_metadata["corpora"] = json.dumps(self.corpora_properties)
-
- if convert_anndata_colors_to_cxg_colors:
- try:
- cxg_group_metadata["cxg_category_colors"] = json.dumps(
- convert_anndata_category_colors_to_cxg_category_colors(self.anndata)
- )
- except ColorFormatException:
- logging.warning(
- "Failed to extract colors from H5AD file! Fix the H5AD file or rerun with "
- "--disable-custom-colors. See help for more details."
- )
-
- return cxg_group_metadata
-
- def validate_input_file_type(self):
- """
- Validate that the input file is of a type that we can handle. Currently the only valid file type is `.h5ad`.
- """
-
- if not self.input_filename.endswith(".h5ad"):
- raise Exception(f"Cannot process input file {self.input_filename}. File must be an H5AD.")
-
- if self.dataset_title or self.dataset_about:
- logging.warning(
- "If you convert this dataset into CXG and you explicit specify values for the dataset title metadata "
- "or the dataset about metadata, it will override any metadata that is extracted as part of the "
- "Corpora schema fields."
- )
-
- def validate_anndata(self):
- if not self.var.index.is_unique:
- raise ValueError("Variable index in AnnData object is not unique.")
- if not self.obs.index.is_unique:
- raise ValueError("Observation index in AnnData object is not unique.")
-
- def extract_anndata_elements_from_file(self):
- logging.info(f"Reading in AnnData dataset: {path.basename(self.input_filename)}")
- self.anndata = anndata.read_h5ad(self.input_filename, backed="r" if self.backed else None)
- logging.info("Completed reading in AnnData dataset!")
-
- self.obs = self.transform_dataframe_index_into_column(self.anndata.obs, "obs", self.obs_index_column_name)
- self.var = self.transform_dataframe_index_into_column(self.anndata.var, "var", self.vars_index_column_name)
-
- def extract_metadata_about_dataset(self):
- """
- Extract metadata information about the dataset that upon conversion will be saved as group metadata with the
- CXG that is generated. This metadata information includes Corpora schema properties, the dataset title and
- a link that details more information about the dataset.
- """
-
- self.corpora_properties = corpora_get_props_from_anndata(self.anndata) if self.use_corpora_schema else None
- if self.corpora_properties is None and self.use_corpora_schema:
- # If the return value is None, this means that we were not able to figure out what version of the Corpora
- # schema the object is using and therefore cannot extract any properties.
- raise ValueError("Unknown source file schema version is unsupported.")
-
- # The title and about properties of the dataset are set by the following order: if they are explicitly defined
- # then use the explicit value. If the dataset is a Corpora-schema based schema, then extract the title and about
- # from the corpora_properties. Otherwise, use the input filename (only for title, about will be blank).
- if self.corpora_properties:
- corpora_project_links = self.corpora_properties.get("project_links", [])
- corpora_about_link = next(
- (link for link in corpora_project_links if (link.get("link_type", None) == "SUMMARY")), {}
- )
- else:
- corpora_about_link = {}
-
- filename = path.splitext(path.basename(self.input_filename))[0]
-
- self.dataset_title = self.dataset_title if self.dataset_title else corpora_about_link.get("link_name", filename)
- self.dataset_about = self.dataset_about if self.dataset_about else corpora_about_link.get("link_url")
-
- def transform_dataframe_index_into_column(self, dataframe, dataframe_name, index_column_name):
- """
- Convert the dataframe's index into another column in the dataframe. If an index_column_name is specified,
- use that column as the index instead.
- """
-
- if index_column_name is None:
- # Create a unique column name for the index.
- suffix = 0
- while f"name_{suffix}" in dataframe.columns:
- suffix += 1
- index_column_name = f"name_{suffix}"
-
- # Turn the index into a normal column
- dataframe.rename_axis(index_column_name, inplace=True)
- dataframe.reset_index(inplace=True)
-
- elif index_column_name in dataframe.columns:
- # User has specified alternative column for unique names, and it exists
- if not dataframe[index_column_name].is_unique:
- raise KeyError(
- f"Values in {dataframe_name}.{index_column_name} must be unique. Please prepare data to contain "
- f"unique values."
- )
- else:
- raise KeyError(f"Column {index_column_name} does not exist.")
-
- setattr(self, f"{dataframe_name}_index_column_name", index_column_name)
- return dataframe
diff --git a/backend/czi_hosted/converters/schema/__init__.py b/backend/czi_hosted/converters/schema/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/backend/czi_hosted/converters/schema/gene_symbol.py b/backend/czi_hosted/converters/schema/gene_symbol.py
deleted file mode 100644
index 2d0de5a7..00000000
--- a/backend/czi_hosted/converters/schema/gene_symbol.py
+++ /dev/null
@@ -1,211 +0,0 @@
-"""Helpers for converting and checking HGNC gene symbols."""
-
-import argparse
-import enum
-import logging
-import os
-import re
-import numpy as np
-import pandas as pd
-
-
-def get_upgraded_var_index(var, hgnc_path=None):
- """Given an anndata var dataframe, return a new index for the dataframe
- where human gene symbols have been upgraded to the current HGNC set.
- """
-
- if not hgnc_path:
- hgnc_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "hgnc_complete_set.txt.gz")
-
- hgnc_symbol_checker = HGNCSymbolChecker.from_hgnc_records(hgnc_path)
-
- return pd.Index([hgnc_symbol_checker.upgrade_symbol(s) for s in var.index])
-
-
-class SymbolStatus(enum.Enum):
- """The status of a symbol in the HGNC database.
-
- APPROVED: Currently a valid symbol
- WITHDRAWN: A previously approved HGNC symbol for a gene that has since been shown
- not to exist _unless_ that symbol is also approved
- AMBIGUOUS: A symbol that is not approved but is an alias or previous symbol for
- multiple approved symbols
- UPGRADABLE: A symbol that is not approved but unambiguously maps to an approved
- symbol
- UNKNOWN: A symbol that does not appear in HGNC
- """
-
- APPROVED = 1
- WITHDRAWN = 2
- AMBIGUOUS = 3
- UPGRADABLE = 4
- UNKNOWN = 5
-
-
-class HGNCSymbolChecker:
- """Handle checking and correcting HGNC symbols."""
-
- def __init__(self, approved_symbols, withdrawn_symbols, ambiguous_symbols, symbol_map):
- self.approved_symbols = approved_symbols
- self.withdrawn_symbols = withdrawn_symbols
- self.ambiguous_symbols = ambiguous_symbols
- self.symbol_map = symbol_map
-
- def print_symbol_map(self):
- """Print out a map from old symbol to new symbol."""
-
- for symbol_pair in self.symbol_map.items():
- print("\t".join(symbol_pair))
-
- def check_symbol(self, symbol):
- """See if a symbol if approved or something else."""
- if symbol in self.approved_symbols:
- return SymbolStatus.APPROVED
-
- if symbol in self.withdrawn_symbols:
- return SymbolStatus.WITHDRAWN
-
- if symbol in self.ambiguous_symbols:
- return SymbolStatus.AMBIGUOUS
-
- if symbol in self.symbol_map:
- return SymbolStatus.UPGRADABLE
-
- return SymbolStatus.UNKNOWN
-
- def upgrade_symbol(self, symbol):
- """Return the approved symbol for the given symbol.
-
- If the symbol cannot be upgraded, just return the original symbol.
- """
-
- fixed_symbol, stripped_symbol = format_symbol(symbol)
-
- if fixed_symbol in self.approved_symbols:
- return fixed_symbol
- elif fixed_symbol in self.symbol_map:
- return self.symbol_map[fixed_symbol]
- elif stripped_symbol in self.approved_symbols:
- return stripped_symbol
- elif stripped_symbol in self.symbol_map:
- return self.symbol_map[stripped_symbol]
-
- return symbol
-
- @classmethod
- def from_hgnc_records(cls, hgnc_dataset_path):
- """Parse a hgnc database download into a HGNCSymbolChecker object."""
-
- def all_symbols(record):
- """Get all the symbols associated with an HGNC record including previous, alias,
- and approved."""
- yield format_symbol(record["symbol"])[0]
- for symbol in alias_and_previous_symbols(record):
- yield symbol
-
- def alias_and_previous_symbols(record):
- """Get alias and previous symbols from an HGNC record."""
- for field in ("alias_symbol", "prev_symbol"):
- if record[field] is not np.nan:
- for symbol in record[field].split("|"):
- yield format_symbol(symbol)[0]
- # Sometimes something like HGNC:1234 appears in datasets, which we
- # want to fix as well.
- yield record["hgnc_id"]
-
- hgnc_records = pd.read_csv(hgnc_dataset_path, sep="\t", header=0, low_memory=False).to_dict("records")
-
- # Get all symbols that are currently approved.
- approved_symbols = set()
- for record in hgnc_records:
- if record["status"] == "Approved":
- approved_symbols.add(format_symbol(record["symbol"])[0])
-
- # Get all symbols that have been withdrawn
- withdrawn_symbols = set()
- for record in hgnc_records:
- if record["status"] == "Entry Withdrawn":
- for symbol in all_symbols(record):
- withdrawn_symbols.add(symbol)
-
- # If a symbol is both approved and withdrawn, be optimistic and call it approved
- logging.warning(
- f"Some symbols are simulaneously withdrawn and approved\n"
- f"We will treat them at approved:\n"
- f"{withdrawn_symbols.intersection(approved_symbols)}"
- )
- withdrawn_symbols = withdrawn_symbols.difference(approved_symbols)
-
- # Now try to map from symbols that are not approved but are an alias or previous symbol for an approved symbol
- alias_previous_to_approved = {}
- ambiguous_symbols = set()
-
- for record in hgnc_records:
- if record["status"] == "Approved":
-
- # The approved symbol is what we'll map to
- approved_symbol = format_symbol(record["symbol"])[0]
-
- for symbol in alias_and_previous_symbols(record):
-
- # If the alias or previous symbol is also an approved symbol,
- # we'll just leave it alone
- if symbol in approved_symbols:
- continue
-
- # If the alias or previous symbol maps to a different approved symbol, mark it as ambiguous
- if symbol in alias_previous_to_approved and alias_previous_to_approved[symbol] != approved_symbol:
- ambiguous_symbols.add(symbol)
- else:
- alias_previous_to_approved[symbol] = approved_symbol
-
- # Remove all the ambiguous symbols from the map
- for ambiguous_symbol in ambiguous_symbols:
- alias_previous_to_approved.pop(ambiguous_symbol)
-
- return HGNCSymbolChecker(approved_symbols, withdrawn_symbols, ambiguous_symbols, alias_previous_to_approved)
-
-
-def format_symbol(symbol):
- """HGNC rules say symbols should all be upper case except for C#orf#. However, case is
- variable in both alias and previous symbols as well as in the symbols we get in
- submissions. So, upper case everything except for the one situation where mixed-case
- is allowed, which are the genes like C2orf157.
-
- Also, seurat and scanpy append ".1" or "-1" to duplicated gene names, and these altered
- names persist throughout the life of the object. They won't match against the HGNC database
- and we want to merge them, so we need to strip off the suffix and try matching again.
-
- This function takes a symbol and returns the symbol with the fixed case and also with the
- seurat/scanpy suffix stripped off.
- """
-
- match = re.match(r"^(C)(\d+)(orf)(\d+)$", symbol, re.IGNORECASE)
-
- if match:
- fixed_case = f"C{match.group(2)}orf{match.group(4)}"
- else:
- fixed_case = symbol.upper()
-
- suffix_stripped = re.sub(r"[\.\-]\d+$", "", fixed_case)
-
- return fixed_case, suffix_stripped
-
-
-def main():
- """When called as main, parse a given hgnc download and print out a map from old to new
- symbol.
- """
- parser = argparse.ArgumentParser()
- parser.add_argument(
- "hgnc_dataset", help="HGNC dataset tsv, available from www.genenames.org/download/statistics-and-files/"
- )
- args = parser.parse_args()
-
- hgnc_symbol_checker = HGNCSymbolChecker.from_hgnc_records(args.hgnc_dataset)
-
- hgnc_symbol_checker.print_symbol_map()
-
-
-if __name__ == "__main__":
- main()
diff --git a/backend/czi_hosted/converters/schema/hgnc_complete_set.txt.gz b/backend/czi_hosted/converters/schema/hgnc_complete_set.txt.gz
deleted file mode 100644
index 29c3c7a9..00000000
Binary files a/backend/czi_hosted/converters/schema/hgnc_complete_set.txt.gz and /dev/null differ
diff --git a/backend/czi_hosted/converters/schema/ontology.py b/backend/czi_hosted/converters/schema/ontology.py
deleted file mode 100644
index 8a524402..00000000
--- a/backend/czi_hosted/converters/schema/ontology.py
+++ /dev/null
@@ -1,86 +0,0 @@
-"""Methods for working with ontologies and the OLS."""
-from urllib.parse import quote_plus
-
-import requests
-
-OLS_API_ROOT = "http://www.ebi.ac.uk/ols/api"
-
-# Curie means something like CL:0000001
-
-
-def _ontology_name(curie):
- """Get the name of the ontology from the curie, CL or UBERON for example."""
- return curie.split(":")[0]
-
-
-def _ontology_value(curie):
- """Get the id component of the curie, 0000001 from CL:0000001 for example."""
- return curie.split(":")[1]
-
-
-def _double_encode(url):
- """Double url encode a url. This is required by the OLS API."""
- return quote_plus(quote_plus(url))
-
-
-def _iri(curie):
- """Get the iri from a curie. This is a bit hopeful that they all map to purl.obolibrary.org"""
- if _ontology_name(curie) == "EFO":
- return f"http://www.ebi.ac.uk/efo/EFO_{_ontology_value(curie)}"
- return f"http://purl.obolibrary.org/obo/{_ontology_name(curie)}_{_ontology_value(curie)}"
-
-
-class OntologyLookupError(Exception):
- """Exception for some problem with looking up ontology information."""
-
-
-def _ontology_info_url(curie):
- """Get the to make a GET to to get information about an ontology term."""
-
- # If the curie is empty, just return an empty string. This happens when there is no
- # valid ontology value.
- if not curie:
- return ""
- else:
- return f"{OLS_API_ROOT}/ontologies/{_ontology_name(curie)}/terms/{_double_encode(_iri(curie))}"
-
-
-def get_ontology_label(curie):
- """For a given curie like 'CL:1000413', get the label like 'endothelial cell of artery'"""
-
- url = _ontology_info_url(curie)
-
- if not url:
- return ""
-
- response = requests.get(url)
-
- if not response.ok:
- raise OntologyLookupError(
- f"Curie {curie} lookup failed, got status code {response.status_code}: {response.text}"
- )
- return response.json()["label"]
-
-
-def lookup_candidate_term(label, ontology="cl", method="select"):
- """Lookup candidate terms for a label. This is useful when there is an existing label in a
- submitted dataset, and you want to find an appropriate ontology term.
-
- Args:
- label: the label to find ontology terms for
- ontology: the ontology to search in, cl or uberon or efo for example
- method: select or search. search provides much broader results
-
- Returns:
- list of (curie, label) tuples returned by OLS
- """
- # using OLS REST API [https://www.ebi.ac.uk/ols/docs/api]
- url = f"{OLS_API_ROOT}/{method}?q={quote_plus(label)}&ontology={ontology.lower()}"
- response = requests.get(url)
-
- if not response.ok:
- raise OntologyLookupError(
- f"Label {label} lookup failed, got status code {response.status_code}: {response.text}"
- )
-
- return [(r["obo_id"], r["label"]) for r in response.json()["response"]["docs"]]
diff --git a/backend/czi_hosted/converters/schema/remix.py b/backend/czi_hosted/converters/schema/remix.py
deleted file mode 100644
index 5cc60746..00000000
--- a/backend/czi_hosted/converters/schema/remix.py
+++ /dev/null
@@ -1,264 +0,0 @@
-import argparse
-import collections
-import json
-import logging
-import math
-import string
-
-import anndata
-import numpy as np
-import pandas as pd
-import yaml
-
-from . import gene_symbol
-from . import ontology
-from . import validate
-
-REPLACE_SUFFIX = "_original"
-ONTOLOGY_SUFFIX = "_ontology_term_id"
-
-
-def is_curie(value):
- """Return True iff the value is an OBO-id CURIE like EFO:000001"""
- return (value.count(":")
- and all(len(part) > 0 for part in value.split(":"))
- and all(c in string.digits for c in value.split(":")[1]))
-
-
-def is_ontology_field(field_name):
- """Return True iff the field_name is an ontology field like tissue_ontology_term_id"""
- return field_name.endswith(ONTOLOGY_SUFFIX)
-
-
-def get_label_field_name(field_name):
- """Get the associated label field from an ontology field, assay_ontology_term_id --> assay"""
- return field_name[: -len(ONTOLOGY_SUFFIX)]
-
-
-def split_suffix(maybe_curie):
- """Split off the (cell culture) or (organoid) suffix."""
-
- suffixes = [" (cell culture)", " (organoid)"]
- for suffix in suffixes:
- if maybe_curie.endswith(suffix):
- return maybe_curie[:-len(suffix)], suffix
- return maybe_curie, ""
-
-
-def get_curie_and_label(maybe_curie):
- """Given a string that might be a curie, return a (curie, label) pair"""
-
- maybe_curie, suffix = split_suffix(maybe_curie)
- if not is_curie(maybe_curie):
- return ("", maybe_curie + suffix)
- return (maybe_curie + suffix, ontology.get_ontology_label(maybe_curie) + suffix)
-
-
-def safe_add_field(adata_attr, field_name, field_value):
- """Add a field and value to an AnnData, but don't clobber an exising value."""
-
- if (
- isinstance(field_value, list)
- and field_value
- and isinstance(field_value[0], dict)
- ):
- field_value = json.dumps(field_value)
- if field_name in adata_attr:
- adata_attr[field_name + REPLACE_SUFFIX] = adata_attr[field_name]
- adata_attr[field_name] = field_value
-
-
-def remix_uns(adata, uns_config):
- """Add fields from the config to adata.uns"""
- for field_name, field_value in uns_config.items():
-
- if is_ontology_field(field_name):
- # If it's an ontology field, look it up
- label_field_name = get_label_field_name(field_name)
- ontology_term, ontology_label = get_curie_and_label(field_value)
- safe_add_field(adata.uns, field_name, ontology_term)
- safe_add_field(adata.uns, label_field_name, ontology_label)
- else:
- safe_add_field(adata.uns, field_name, field_value)
-
-
-def remix_obs(adata, obs_config):
- """Add fields from the config to adata.obs"""
-
- for field_name, field_value in obs_config.items():
-
- if isinstance(field_value, dict):
- # If the value is a dict, that means we are supposed to map from an
- # existing column to the new one
- source_column, column_map = next(iter(field_value.items()))
- nan_value = None
- for key in column_map:
- if isinstance(key, float) and math.isnan(key):
- nan_value = column_map[key]
- if nan_value is not None:
- column_map["nan"] = nan_value
-
- for key in column_map:
- if key not in adata.obs[source_column].unique():
- logging.warning(f'Key {key} not in adata.obs["{source_column}"]')
-
- for value in adata.obs[source_column].unique():
- if value not in column_map:
- logging.warning(f'Value {value} in adata.obs["{source_column}"] not in translation dict')
-
- if is_ontology_field(field_name):
- ontology_term_map, ontology_label_map = {}, {}
- logging.info(f"Looking up labels for {field_name}")
- for original_value, maybe_curie in column_map.items():
- curie, label = get_curie_and_label(maybe_curie)
- ontology_term_map[original_value] = curie
- ontology_label_map[original_value] = label
- logging.info(f"Mapping {original_value} -> {curie} -> {label}")
-
- ontology_column = adata.obs[source_column].replace(
- ontology_term_map, inplace=False
- )
- label_column = adata.obs[source_column].replace(
- ontology_label_map, inplace=False
- )
-
- safe_add_field(adata.obs, field_name, ontology_column)
- safe_add_field(
- adata.obs, get_label_field_name(field_name), label_column
- )
- else:
- label_column = adata.obs[source_column].replace(
- column_map, inplace=False
- )
- safe_add_field(adata.obs, field_name, label_column)
-
- else:
- if is_ontology_field(field_name):
- # If it's an ontology field, look it up
- label_field_name = get_label_field_name(field_name)
- ontology_term, ontology_label = get_curie_and_label(field_value)
- safe_add_field(adata.obs, field_name, ontology_term)
- safe_add_field(adata.obs, label_field_name, ontology_label)
- else:
- safe_add_field(adata.obs, field_name, field_value)
-
-
-def merge_df(df, domain, index, columns):
- """
- Given a dataframe with duplicate column labels, merge and return a dataframe where
- the duplicates have been merged together, resulting in a dataframe with unique column
- labels.
-
- "merge" depends on the value of domain. If the domain is "raw", then duplicate columns
- can just be summed. If it's "log1p" or "sqrt", it needs to be exp1m'd or squared, then
- summed, and then logged or sqrt'd again.
- """
-
- if not isinstance(df, np.ndarray):
- to_merge = df.toarray()
- else:
- to_merge = df
- if domain == "raw":
- merged_df = pd.DataFrame(to_merge, index=index, columns=columns).sum(
- axis=1, level=0, skipna=False
- )
- elif domain == "log1p":
- merged_df = (
- pd.DataFrame(np.expm1(to_merge, dtype=np.float128), index=index, columns=columns)
- .sum(axis=1, level=0, skipna=False)
- )
- merged_df = pd.DataFrame(np.log1p(merged_df.to_numpy()), index=merged_df.index, columns=merged_df.columns)
- elif domain == "sqrt":
- merged_df = (
- pd.DataFrame(np.square(to_merge), index=index, columns=columns)
- .sum(axis=1, level=0, skipna=False)
- )
- merged_df = pd.DataFrame(np.sqrt(merged_df.to_numpy()), index=merged_df.index, columns=merged_df.columns)
-
- return merged_df
-
-
-def fixup_gene_symbols(adata, fixup_config):
- """Update the var index to hold a consistent set of HGNC gene symbols."""
-
- upgraded_var_index = gene_symbol.get_upgraded_var_index(adata.var)
-
- merged_X = merge_df(adata.X, fixup_config["X"], adata.obs.index, upgraded_var_index)
- fixup_adata = anndata.AnnData(
- X=merged_X,
- obs=adata.obs,
- var=merged_X.columns.to_frame(name="hgnc_gene_symbol"),
- uns=adata.uns,
- obsm=adata.obsm,
- )
-
- for layer, domain in fixup_config.items():
- if layer == "X":
- continue
- if layer == "raw.X":
- df = adata.raw.X
- else:
- df = adata.layers[layer]
-
- merged_df = merge_df(df, domain, adata.obs.index, upgraded_var_index)
- assert merged_df.index.equals(merged_X.index)
- assert merged_df.columns.equals(merged_X.columns)
-
- if domain == "raw":
- fixup_raw = anndata.AnnData(
- X=merged_df,
- obs=adata.obs,
- var=merged_X.columns.to_frame(name="hgnc_gene_symbol"),
- )
- fixup_adata.raw = fixup_raw
- else:
- fixup_adata.layers[layer] = merged_df
-
- return fixup_adata
-
-def _strip_version(adata):
- """Remove version information from the AnnData object."""
-
- if "version" in adata.uns_keys():
- del adata.uns["version"]
-
-def apply_schema(source_h5ad, remix_config, output_filename):
-
- try:
- import scanpy
- except ImportError:
- raise ImportError("scanpy must be installed for cellxgene schema")
- adata = scanpy.read_h5ad(source_h5ad)
- config = yaml.load(open(remix_config), Loader=yaml.FullLoader)
- remix_uns(adata, config["uns"])
- remix_obs(adata, config["obs"])
-
- if config.get("fixup_gene_symbols"):
- adata = fixup_gene_symbols(adata, config["fixup_gene_symbols"])
-
- if ("version" in adata.uns_keys()
- and isinstance(adata.uns["version"], collections.Mapping)
- and "corpora_schema_version" in adata.uns["version"]):
- schema_version = adata.uns["version"]["corpora_schema_version"]
- try:
- validate.get_schema_definition(schema_version)
- except ValueError:
- logging.warning(f"Stripping version information out of AnnData because schema "
- f"version {schema_version} is unknown.")
- _strip_version(adata)
-
- if not validate.validate_adata(adata, shallow=False):
- logging.warning(f"Stripping version information out of AnnData because it does not "
- f"follow schema version {schema_version} .")
- _strip_version(adata)
-
- adata.write_h5ad(output_filename, compression="gzip")
-
-
-if __name__ == "__main__":
- parser = argparse.ArgumentParser()
- parser.add_argument("--source-h5ad", required=True)
- parser.add_argument("--remix-config", required=True)
- parser.add_argument("--output-filename", required=True)
- args = parser.parse_args()
- apply_schema(args.source_h5ad, args.remix_config, args.output_filename)
diff --git a/backend/czi_hosted/converters/schema/schema_definitions/1_0_0.yaml b/backend/czi_hosted/converters/schema/schema_definitions/1_0_0.yaml
deleted file mode 100644
index acff2238..00000000
--- a/backend/czi_hosted/converters/schema/schema_definitions/1_0_0.yaml
+++ /dev/null
@@ -1,95 +0,0 @@
-title: Corpora schema version 1.0.0
-type: anndata
-components:
- uns:
- type: dict
- keys:
- version:
- type: dict
- keys:
- corpora_schema_version: null
- corpora_encoding_version: null
- title:
- type: string
- contributors:
- type: stringified list of dicts
- layer_descriptions:
- type: dict
- keys:
- X: null
- organism:
- type: string
- nullable: false
- organism_ontology_term_id:
- type: curie
- prefixes:
- - NCBITaxon
- var:
- type: dataframe
- index:
- type: human-readable string
- unique: true
- obs:
- type: dataframe
- index:
- unique: true
- columns:
- tissue:
- type: human-readable string
- nullable: false
- tissue_ontology_term_id:
- type: suffixed curie
- nullable: true
- prefixes:
- - UBERON
- assay:
- type: human-readable string
- nullable: false
- assay_ontology_term_id:
- type: curie
- nullable: true
- prefixes:
- - EFO
- disease:
- type: human-readable string
- nullable: false
- disease_ontology_term_id:
- type: curie
- nullable: true
- prefixes:
- - MONDO
- - PATO
- cell_type:
- type: human-readable string
- nullable: false
- cell_type_ontology_term_id:
- type: curie
- nullable: true
- prefixes:
- - CL
- - UBERON
- sex:
- type: string
- enum:
- - male
- - female
- - mixed
- - unknown
- - other
- ethnicity:
- type: human-readable string
- nullable: false
- ethnicity_ontology_term_id:
- type: curie
- nullable: true
- prefixes:
- - HANCESTRO
- development_stage:
- type: human-readable string
- nullable: false
- development_stage_ontology_term_id:
- type: curie
- nullable: true
- prefixes:
- - HsapDv
- - EFO
diff --git a/backend/czi_hosted/converters/schema/schema_definitions/1_1_0.yaml b/backend/czi_hosted/converters/schema/schema_definitions/1_1_0.yaml
deleted file mode 100644
index 7531c72e..00000000
--- a/backend/czi_hosted/converters/schema/schema_definitions/1_1_0.yaml
+++ /dev/null
@@ -1,93 +0,0 @@
-title: Corpora schema version 1.1.0
-type: anndata
-components:
- uns:
- type: dict
- keys:
- version:
- type: dict
- keys:
- corpora_schema_version: null
- corpora_encoding_version: null
- title:
- type: string
- layer_descriptions:
- type: dict
- keys:
- X: null
- organism:
- type: string
- nullable: false
- organism_ontology_term_id:
- type: curie
- prefixes:
- - NCBITaxon
- var:
- type: dataframe
- index:
- type: human-readable string
- unique: true
- obs:
- type: dataframe
- index:
- unique: true
- columns:
- tissue:
- type: human-readable string
- nullable: false
- tissue_ontology_term_id:
- type: suffixed curie
- nullable: true
- prefixes:
- - UBERON
- assay:
- type: human-readable string
- nullable: false
- assay_ontology_term_id:
- type: curie
- nullable: true
- prefixes:
- - EFO
- disease:
- type: human-readable string
- nullable: false
- disease_ontology_term_id:
- type: curie
- nullable: true
- prefixes:
- - MONDO
- - PATO
- cell_type:
- type: human-readable string
- nullable: false
- cell_type_ontology_term_id:
- type: curie
- nullable: true
- prefixes:
- - CL
- - UBERON
- sex:
- type: string
- enum:
- - male
- - female
- - mixed
- - unknown
- - other
- ethnicity:
- type: human-readable string
- nullable: false
- ethnicity_ontology_term_id:
- type: curie
- nullable: true
- prefixes:
- - HANCESTRO
- development_stage:
- type: human-readable string
- nullable: false
- development_stage_ontology_term_id:
- type: curie
- nullable: true
- prefixes:
- - HsapDv
- - EFO
diff --git a/backend/czi_hosted/converters/schema/validate.py b/backend/czi_hosted/converters/schema/validate.py
deleted file mode 100644
index ec463dd5..00000000
--- a/backend/czi_hosted/converters/schema/validate.py
+++ /dev/null
@@ -1,236 +0,0 @@
-import json
-import re
-import os
-import sys
-
-import pandas as pd
-import yaml
-
-
-def _is_null(v):
- """Return True if v is null, for one of the multiple ways a "null" value shows up in an h5ad."""
- return pd.isnull(v) or (hasattr(v, "__len__") and len(v) == 0)
-
-
-def _validate_stringified_list_of_dicts(s):
- """Verify that a string can be parsed into a list.
-
- We have some types that are lists of dicts. Those cannot be stored directly in an h5ad, so we have to
- json.dumps them. This verifies that we can load them back.
- """
-
- try:
- list_ = json.loads(s)
- if not isinstance(list_, list):
- return False
- for el in list_:
- if not isinstance(el, dict):
- return False
- return True
- except (json.JSONDecodeError, TypeError):
- pass
- return False
-
-
-def _validate_human_readable_string(s):
- """Verify that a string is human-readable.
-
- There are parts of the schema where a "human-readable" string is required. "Human-readable" is kind
- of vague and subjective. I feel like I can read many strings. So here we just check for the main ways
- that fails: someone puts in an ontology term id or and ensembl gene/transcript id.
-
- Returns False if s is not a string or is one of those bad string types.
- """
-
- return isinstance(s, str) and (not re.match(r"[A-Z]\w+:\d+", s)) and (not re.match(r"ENS[GT]\d+$", s))
-
-
-def _validate_curie(c, prefixes):
- """Verify that a string is a valid compact URI, like EFO:000001. If prefixes is not empty, make sure the
- prefix of the curies is in prefixes.
- """
-
- if not c:
- return True
-
- match = re.match(r"([A-Z]\w+):\d+$", c)
-
- if prefixes:
- return match and match.group(1) in prefixes
- else:
- return match
-
-
-def _validate_suffixed_curie(c, prefixes):
- """Verify that a string is a compact URI with an optional suffix like 'EFO:00001 (cell culture)'"""
-
- # Pull off the suffix
- suffix = re.findall(r"\ \(.*\)$", c)
- if suffix:
- c = c[: -len(suffix[0])]
- return _validate_curie(c, prefixes)
-
-
-def _validate_column(column, column_name, df_name, schema_def):
- """Given a schema definition and the column of a dataframe, verify that the column satifies
- the schema.
- """
-
- errors = []
-
- if schema_def.get("unique"):
- if column.nunique() != len(column):
- errors.append(f"Column {column_name} in dataframe {df_name} is not unique.")
-
- if "nullable" in schema_def and not schema_def["nullable"]:
- if any(_is_null(v) for v in column):
- errors.append(f"Column {column_name} in dataframe {df_name} contains empty values.")
-
- if schema_def.get("type") == "human-readable string":
- non_readables = [v for v in column if not _validate_human_readable_string(v)]
- if non_readables:
- errors.append(
- f"Column {column_name} in dataframe {df_name} contains non-human-readable "
- f"values like {non_readables[0]}"
- )
-
- if schema_def.get("type") in ("curie", "suffixed curie"):
- validation_func = _validate_curie if schema_def.get("type") == "curie" else _validate_suffixed_curie
- non_valid_curies = [v for v in column if not validation_func(v, schema_def.get("prefixes"))]
- if non_valid_curies:
- errors.append(
- f"Column {column_name} in dataframe {df_name} contains invalid ontology values like "
- f"{non_valid_curies[0]}."
- )
- if "prefixes" in schema_def:
- errors[-1] += f" Values must be curies from one of these ontologies {schema_def['prefixes']}."
-
- if "enum" in schema_def:
- bad_enums = [v for v in column if v not in schema_def["enum"]]
- if bad_enums:
- errors.append(
- f"Column {column_name} in dataframe {df_name} contains unpermitted values like "
- f"{bad_enums[0]}. Values must be one of {schema_def['enum']}."
- )
-
- return errors
-
-
-def _validate_dict(dict_, dict_name, schema_def):
- """Given a schema definition and dict, verify that the dict satifies the schema."""
-
- errors = []
-
- for key in schema_def.get("keys", []):
- if key not in dict_:
- errors.append(f"{dict_name} is missing key {key}.")
- elif schema_def["keys"][key]:
- if schema_def["keys"][key]["type"] == "stringified list of dicts":
- if not _validate_stringified_list_of_dicts(dict_[key]):
- errors.append(
- f"Key {key} in {dict_name} should be a JSON-encoded list of dicts, but it is {dict_[key]}"
- )
- elif schema_def["keys"][key]["type"] == "dict":
- errors.extend(_validate_dict(dict_[key], key, schema_def["keys"][key]))
- elif schema_def["keys"][key]["type"] == "curie":
- if not _validate_curie(dict_[key], schema_def["keys"][key]["prefixes"]):
- errors.append(f"Key {key} in {dict_name} contains invalid ontology value.")
- if "nullable" in schema_def["keys"][key] and not schema_def["keys"][key]["nullable"]:
- if _is_null(dict_[key]):
- errors.append(f"Key {key} in dict {dict_name} is an empty value.")
-
- return errors
-
-
-def _validate_dataframe(df, df_name, schema_def):
- """Given a dataframe and schema definition, verify that the dataframe follows the schema."""
-
- errors = []
-
- if "index" in schema_def:
- errors.extend(_validate_column(df.index, "index", df_name, schema_def["index"]))
-
- for column in schema_def.get("columns", []):
- if column not in df.columns:
- errors.append(f"Dataframe {df_name} is missing column {column}.")
- else:
- errors.extend(_validate_column(df[column], column, df_name, schema_def["columns"][column]))
-
- return errors
-
-
-def get_schema_definition(version):
- """Look up and read a schema definition based on a version number like "1.0.0"."""
-
- path = os.path.join(
- os.path.dirname(os.path.realpath(__file__)), "schema_definitions", version.replace(".", "_") + ".yaml"
- )
-
- if not os.path.isfile(path):
- raise ValueError(f"No definition for version {version} found.")
-
- return yaml.load(open(path), Loader=yaml.FullLoader)
-
-
-def deep_check(adata, schema_def):
- """Perform a "deep" check of the AnnData object using the schema definition.
-
- This checks all the columns and unstructured metadata rather than just the version.
-
- Returns a list of error messages. If that list is empty, the object passed validation.
- """
-
- errors = []
-
- for component, component_def in schema_def["components"].items():
- if component_def["type"] == "dataframe":
- errors.extend(_validate_dataframe(getattr(adata, component), component, component_def))
- elif component_def["type"] == "dict":
- errors.extend(_validate_dict(getattr(adata, component), component, component_def))
- else:
- raise ValueError(f"Unexpected component type {component['type']}")
-
- return errors
-
-
-def validate_adata(adata, shallow):
- """Validate an AnnData object. If shallow, just check that the required version information is
- present.
- """
-
- # Does it have the version information written into uns?
- if "version" not in adata.uns_keys() or "corpora_schema_version" not in adata.uns["version"]:
- print("AnnData file is missing corpora version information")
- return False
-
- # We can stop here if it's a "shallow" check, that is, if we're just
- # checking that version is present.
- if shallow:
- return True
-
- schema_def = get_schema_definition(adata.uns["version"]["corpora_schema_version"])
-
- errors = deep_check(adata, schema_def)
-
- for error in errors:
- print(error)
-
- return not errors
-
-
-def validate(h5ad_path, shallow=False):
- """Entry point for validation."""
-
- try:
- import scanpy
- except ImportError:
- raise ImportError("scanpy must be installed for cellxgene schema")
-
- try:
- adata = scanpy.read_h5ad(h5ad_path, backed="r")
- except (OSError, TypeError):
- print(f"Unable to open {h5ad_path} with scanpy.")
- sys.exit(1)
-
- if not validate_adata(adata, shallow):
- sys.exit(1)
diff --git a/backend/czi_hosted/converters/to_sparse.py b/backend/czi_hosted/converters/to_sparse.py
deleted file mode 100644
index daaa4c43..00000000
--- a/backend/czi_hosted/converters/to_sparse.py
+++ /dev/null
@@ -1,81 +0,0 @@
-"""
-Script to create a sparse dataset in CXG format based on an input dataset in CXG format.
-The input dataset is not modified.
-"""
-import argparse
-import os
-import shutil
-import sys
-
-import tiledb
-
-from backend.czi_hosted.common.utils.cxg_generation_utils import convert_ndarray_to_cxg_dense_array, \
- convert_matrix_to_cxg_array
-from backend.czi_hosted.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix
-
-
-def main():
- parser = argparse.ArgumentParser()
- parser.add_argument("input", help="input cxg directory")
- parser.add_argument("output", help="output cxg directory")
- parser.add_argument("--overwrite", action="store_true", help="replace output cxg directory")
- parser.add_argument("--verbose", "-v", action="count", default=0, help="verbose output")
- parser.add_argument(
- "--sparse-threshold",
- "-s",
- type=float,
- default=5.0, # default is 5% non-zero values
- help="The X array will be sparse if the percent of non-zeros falls below this value",
- )
- args = parser.parse_args()
-
- if os.path.exists(args.output):
- print("output dir exists:", args.output)
- if args.overwrite:
- print("output dir removed:", args.output)
- shutil.rmtree(args.output)
- else:
- print("use the overwrite option to remove the output directory")
- sys.exit(1)
-
- if not os.path.isdir(args.input):
- print("input is not a directory", args.input)
- sys.exit(1)
-
- shutil.copytree(args.input, args.output, ignore=shutil.ignore_patterns("X", "X_col_shift"))
-
- ctx = tiledb.Ctx(
- {
- "sm.num_reader_threads": 32,
- "sm.num_writer_threads": 32,
- "sm.consolidation.buffer_size": 1 * 1024 * 1024 * 1024,
- }
- )
-
- with tiledb.DenseArray(os.path.join(args.input, "X"), mode="r", ctx=ctx) as X_in:
- x_matrix_data = X_in[:, :]
- matrix_container = args.output
-
- is_sparse = is_matrix_sparse(x_matrix_data, args.sparse_threshold)
- if not is_sparse:
- col_shift = get_column_shift_encode_for_matrix(x_matrix_data, args.sparse_threshold)
- is_sparse = col_shift is not None
- else:
- col_shift = None
-
- if col_shift is not None:
- x_col_shift_name = f"{args.output}/X_col_shift"
- convert_ndarray_to_cxg_dense_array(x_col_shift_name, col_shift, ctx)
- tiledb.consolidate(matrix_container, ctx=ctx)
- if is_sparse:
- convert_matrix_to_cxg_array(matrix_container, x_matrix_data, is_sparse, ctx, col_shift)
- tiledb.consolidate(matrix_container, ctx=ctx)
-
- if not is_sparse:
- print("The array is not sparse, cleaning up, abort.")
- shutil.rmtree(args.output)
- sys.exit(1)
-
-
-if __name__ == "__main__":
- main()
diff --git a/backend/czi_hosted/data_anndata/__init__.py b/backend/czi_hosted/data_anndata/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/backend/czi_hosted/data_anndata/anndata_adaptor.py b/backend/czi_hosted/data_anndata/anndata_adaptor.py
deleted file mode 100644
index 8badd8de..00000000
--- a/backend/czi_hosted/data_anndata/anndata_adaptor.py
+++ /dev/null
@@ -1,362 +0,0 @@
-import warnings
-
-import anndata
-import numpy as np
-from packaging import version
-from pandas.core.dtypes.dtypes import CategoricalDtype
-from scipy import sparse
-
-import backend.common.compute.diffexp_generic as diffexp_generic
-from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
-from backend.common.constants import Axis, MAX_LAYOUTS, XApproximateDistribution
-from backend.czi_hosted.common.corpora import corpora_get_props_from_anndata
-from backend.common.errors import PrepareError, DatasetAccessError, ConfigurationError
-from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
-from backend.czi_hosted.data_common.data_adaptor import DataAdaptor
-from backend.common.fbs.matrix import encode_matrix_fbs
-
-anndata_version = version.parse(str(anndata.__version__)).release
-
-
-def anndata_version_is_pre_070():
- major = anndata_version[0]
- minor = anndata_version[1] if len(anndata_version) > 1 else 0
- return major == 0 and minor < 7
-
-
-class AnndataAdaptor(DataAdaptor):
- def __init__(self, data_locator, app_config=None, dataset_config=None):
- super().__init__(data_locator, app_config, dataset_config)
- self.data = None
- self.X_approximate_distribution = None
- self._load_data(data_locator)
- self._validate_and_initialize()
-
- def cleanup(self):
- pass
-
- @staticmethod
- def pre_load_validation(data_locator):
- if data_locator.islocal():
- # if data locator is local, apply file system conventions and other "cheap"
- # validation checks. If a URI, defer until we actually fetch the data and
- # try to read it. Many of these tests don't make sense for URIs (eg, extension-
- # based typing).
- if not data_locator.exists():
- raise DatasetAccessError("does not exist")
- if not data_locator.isfile():
- raise DatasetAccessError("is not a file")
-
- @staticmethod
- def file_size(data_locator):
- return data_locator.size() if data_locator.islocal() else 0
-
- @staticmethod
- def open(data_locator, app_config, dataset_config=None):
- return AnndataAdaptor(data_locator, app_config, dataset_config)
-
- def get_corpora_props(self):
- return corpora_get_props_from_anndata(self.data)
-
- def get_name(self):
- return "cellxgene anndata adaptor version"
-
- def get_library_versions(self):
- return dict(anndata=str(anndata.__version__))
-
- @staticmethod
- def _create_unique_column_name(df, col_name_prefix):
- """given the columns of a dataframe, and a name prefix, return a column name which
- does not exist in the dataframe, AND which is prefixed by `prefix`
-
- The approach is to append a numeric suffix, starting at zero and increasing by
- one, until an unused name is found (eg, prefix_0, prefix_1, ...).
- """
- suffix = 0
- while f"{col_name_prefix}{suffix}" in df:
- suffix += 1
- return f"{col_name_prefix}{suffix}"
-
- def _alias_annotation_names(self):
- """
- The front-end relies on the existance of a unique, human-readable
- index for obs & var (eg, var is typically gene name, obs the cell name).
- The user can specify these via the --obs-names and --var-names config.
- If they are not specified, use the existing index to create them, giving
- the resulting column a unique name (eg, "name").
-
- In both cases, enforce that the result is unique, and communicate the
- index column name to the front-end via the obs_names and var_names config
- (which is incorporated into the schema).
- """
- self.original_obs_index = self.data.obs.index
-
- for (ax_name, var_name) in ((Axis.OBS, "obs"), (Axis.VAR, "var")):
- config_name = f"single_dataset__{var_name}_names"
- parameter_name = f"{var_name}_names"
- name = getattr(self.server_config, config_name)
- df_axis = getattr(self.data, str(ax_name))
- if name is None:
- # Default: create unique names from index
- if not df_axis.index.is_unique:
- raise KeyError(
- f"Values in {ax_name}.index must be unique. "
- "Please prepare data to contain unique index values, or specify an "
- "alternative with --{ax_name}-name."
- )
- name = self._create_unique_column_name(df_axis.columns, "name_")
- self.parameters[parameter_name] = name
- # reset index to simple range; alias name to point at the
- # previously specified index.
- df_axis.rename_axis(name, inplace=True)
- df_axis.reset_index(inplace=True)
- elif name in df_axis.columns:
- # User has specified alternative column for unique names, and it exists
- if not df_axis[name].is_unique:
- raise KeyError(
- f"Values in {ax_name}.{name} must be unique. " "Please prepare data to contain unique values."
- )
- df_axis.reset_index(drop=True, inplace=True)
- self.parameters[parameter_name] = name
- else:
- # user specified a non-existent column name
- raise KeyError(f"Annotation name {name}, specified in --{ax_name}-name does not exist.")
-
- def _create_schema(self):
- self.schema = {
- "dataframe": {
- "nObs": self.cell_count,
- "nVar": self.gene_count,
- **get_schema_type_hint_of_array(self.data.X),
- },
- "annotations": {
- "obs": {"index": self.parameters.get("obs_names"), "columns": []},
- "var": {"index": self.parameters.get("var_names"), "columns": []},
- },
- "layout": {"obs": []},
- }
- for ax in Axis:
- curr_axis = getattr(self.data, str(ax))
- for ann in curr_axis:
- ann_schema = {"name": ann, "writable": False}
- ann_schema.update(get_schema_type_hint_of_array(curr_axis[ann]))
- self.schema["annotations"][ax]["columns"].append(ann_schema)
-
- for layout in self.get_embedding_names():
- layout_schema = {"name": layout, "type": "float32", "dims": [f"{layout}_0", f"{layout}_1"]}
- self.schema["layout"]["obs"].append(layout_schema)
-
- def get_schema(self):
- return self.schema
-
- def _load_data(self, data_locator):
- # as of AnnData 0.6.19, backed mode performs initial load fast, but at the
- # cost of significantly slower access to X data.
- try:
- # there is no guarantee data_locator indicates a local file. The AnnData
- # API will only consume local file objects. If we get a non-local object,
- # make a copy in tmp, and delete it after we load into memory.
- with data_locator.local_handle() as lh:
- # as of AnnData 0.6.19, backed mode performs initial load fast, but at the
- # cost of significantly slower access to X data.
- backed = "r" if self.server_config.adaptor__anndata_adaptor__backed else None
- self.data = anndata.read_h5ad(lh, backed=backed)
-
- except ValueError:
- raise DatasetAccessError(
- "File must be in the .h5ad format. Please read "
- "https://github.com/theislab/scanpy_usage/blob/master/170505_seurat/info_h5ad.md to "
- "learn more about this format. You may be able to convert your file into this format "
- "using `cellxgene prepare`, please run `cellxgene prepare --help` for more "
- "information."
- )
- except MemoryError:
- raise DatasetAccessError("Out of memory - file is too large for available memory.")
- except Exception:
- raise DatasetAccessError(
- "File not found or is inaccessible. File must be an .h5ad object. "
- "Please check your input and try again."
- )
-
- def _validate_and_initialize(self):
- if anndata_version_is_pre_070():
- warnings.warn(
- "Use of anndata versions older than 0.7 will have serious issues. Please update to at "
- "least anndata 0.7 or later."
- )
-
- # var and obs column names must be unique
- if not self.data.obs.columns.is_unique or not self.data.var.columns.is_unique:
- raise KeyError("All annotation column names must be unique.")
-
- self._alias_annotation_names()
- self._validate_data_types()
- self.cell_count = self.data.shape[0]
- self.gene_count = self.data.shape[1]
- self._create_schema()
-
- if self.dataset_config.X_approximate_distribution == "auto":
- raise ConfigurationError("X-approximate-distribution 'auto' mode unsupported.")
- self.X_approximate_distribution = self.dataset_config.X_approximate_distribution
-
- # heuristic
- n_values = self.data.shape[0] * self.data.shape[1]
- if (n_values > 1e8 and self.server_config.adaptor__anndata_adaptor__backed is True) or (n_values > 5e8):
- self.parameters.update({"diffexp_may_be_slow": True})
-
- def _is_valid_layout(self, arr):
- """return True if this layout data is a valid array for front-end presentation:
- * ndarray, dtype float/int/uint
- * with shape (n_obs, >= 2)
- * with all values finite or NaN (no +Inf or -Inf)
- """
- is_valid = type(arr) == np.ndarray and arr.dtype.kind in "fiu"
- is_valid = is_valid and arr.shape[0] == self.data.n_obs and arr.shape[1] >= 2
- is_valid = is_valid and not np.any(np.isinf(arr)) and not np.all(np.isnan(arr))
- return is_valid
-
- def _validate_data_types(self):
- # The backed API does not support interrogation of the underlying sparsity or sparse matrix type
- # Fake it by asking for a small subarray and testing it. NOTE: if the user has ignored our
- # anndata <= 0.7 warning, opted for the --backed option, and specified a large, sparse dataset,
- # this "small" indexing request will load the entire X array. This is due to a bug in anndata<=0.7
- # which will load the entire X matrix to fullfill any slicing request if X is sparse. See
- # user warning in _load_data().
- X0 = self.data.X[0, 0:1]
- if sparse.isspmatrix(X0) and not sparse.isspmatrix_csc(X0):
- warnings.warn(
- "Anndata data matrix is sparse, but not a CSC (columnar) matrix. "
- "Performance may be improved by using CSC."
- )
- if self.data.X.dtype != "float32":
- warnings.warn(
- f"Anndata data matrix is in {self.data.X.dtype} format not float32. " f"Precision may be truncated."
- )
- for ax in Axis:
- curr_axis = getattr(self.data, str(ax))
- for ann in curr_axis:
- datatype = curr_axis[ann].dtype
- downcast_map = {
- "int64": "int32",
- "uint32": "int32",
- "uint64": "int32",
- "float64": "float32",
- }
- if datatype in downcast_map:
- warnings.warn(
- f"Anndata annotation {ax}:{ann} is in unsupported format: {datatype}. "
- f"Data will be downcast to {downcast_map[datatype]}."
- )
- if isinstance(datatype, CategoricalDtype):
- category_num = len(curr_axis[ann].dtype.categories)
- if category_num > 500 and category_num > self.dataset_config.presentation__max_categories:
- warnings.warn(
- f"{str(ax).title()} annotation '{ann}' has {category_num} categories, this may be "
- f"cumbersome or slow to display. We recommend setting the "
- f"--max-category-items option to 500, this will hide categorical "
- f"annotations with more than 500 categories in the UI"
- )
-
- def annotation_to_fbs_matrix(self, axis, fields=None, labels=None):
- if axis == Axis.OBS:
- if labels is not None and not labels.empty:
- df = self.data.obs.join(labels, self.parameters.get("obs_names"))
- else:
- df = self.data.obs
- else:
- df = self.data.var
-
- if fields is not None and len(fields) > 0:
- df = df[fields]
- return encode_matrix_fbs(df, col_idx=df.columns)
-
- def get_embedding_names(self):
- """
- Return pre-computed embeddings.
-
- function:
- a) generate list of default layouts
- b) validate layouts are legal. remove/warn on any that are not
- c) cap total list of layouts at global const MAX_LAYOUTS
- """
- # load default layouts from the data.
- layouts = self.dataset_config.embeddings__names
-
- if layouts is None or len(layouts) == 0:
- layouts = [key[2:] for key in self.data.obsm_keys() if type(key) == str and key.startswith("X_")]
-
- # remove invalid layouts
- valid_layouts = []
- obsm_keys = self.data.obsm_keys()
- for layout in layouts:
- layout_name = f"X_{layout}"
- if layout_name not in obsm_keys:
- warnings.warn(f"Ignoring unknown layout name: {layout}.")
- elif not self._is_valid_layout(self.data.obsm[layout_name]):
- warnings.warn(f"Ignoring layout due to malformed shape or data type: {layout}")
- else:
- valid_layouts.append(layout)
-
- if len(valid_layouts) == 0:
- raise PrepareError("No valid layout data.")
-
- # cap layouts to MAX_LAYOUTS
- return valid_layouts[0:MAX_LAYOUTS]
-
- def get_embedding_array(self, ename, dims=2):
- full_embedding = self.data.obsm[f"X_{ename}"]
- return full_embedding[:, 0:dims]
-
- def compute_diffexp_ttest(self, maskA, maskB, top_n=None, lfc_cutoff=None):
- if top_n is None:
- top_n = self.dataset_config.diffexp__top_n
- if lfc_cutoff is None:
- lfc_cutoff = self.dataset_config.diffexp__lfc_cutoff
- return diffexp_generic.diffexp_ttest(self, maskA, maskB, top_n, lfc_cutoff)
-
- def get_colors(self):
- return convert_anndata_category_colors_to_cxg_category_colors(self.data)
-
- def get_X_array(self, obs_mask=None, var_mask=None):
- # H5Py does not support boolean indexing (masks), so convert to integer indexing
- # when backed (ie, when AnnData is using H5Py indexing)
- if obs_mask is None:
- obs_mask = slice(None)
- elif self.data.isbacked and obs_mask.dtype == bool:
- obs_mask = obs_mask.nonzero()[0]
- if var_mask is None:
- var_mask = slice(None)
- elif self.data.isbacked and var_mask.dtype == bool:
- var_mask = var_mask.nonzero()[0]
- X = self.data.X[obs_mask, var_mask]
- return X
-
- def get_X_approximate_distribution(self) -> XApproximateDistribution:
- return self.X_approximate_distribution
-
- def get_shape(self):
- return self.data.shape
-
- def query_var_array(self, term_name):
- return getattr(self.data.var, term_name)
-
- def query_obs_array(self, term_name):
- return getattr(self.data.obs, term_name)
-
- def get_obs_index(self):
- name = self.server_config.single_dataset__obs_names
- if name is None:
- return self.original_obs_index
- else:
- return self.data.obs[name]
-
- def get_obs_columns(self):
- return self.data.obs.columns
-
- def get_obs_keys(self):
- # return list of keys
- return self.data.obs.keys().to_list()
-
- def get_var_keys(self):
- # return list of keys
- return self.data.var.keys().to_list()
diff --git a/backend/czi_hosted/data_common/__init__.py b/backend/czi_hosted/data_common/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/backend/czi_hosted/data_common/data_adaptor.py b/backend/czi_hosted/data_common/data_adaptor.py
deleted file mode 100644
index 3bbdc6b1..00000000
--- a/backend/czi_hosted/data_common/data_adaptor.py
+++ /dev/null
@@ -1,428 +0,0 @@
-from abc import ABCMeta, abstractmethod
-from os.path import basename, splitext
-
-import numpy as np
-import pandas as pd
-from scipy import sparse
-from server_timing import Timing as ServerTiming
-
-from backend.czi_hosted.common.config.app_config import AppConfig
-from backend.common.constants import Axis, XApproximateDistribution
-from backend.common.errors import (
- FilterError,
- JSONEncodingValueError,
- ExceedsLimitError,
- UnsupportedSummaryMethod,
- DatasetAccessError,
-)
-from backend.common.utils.utils import jsonify_strict
-from backend.common.fbs.matrix import encode_matrix_fbs
-
-
-class DataAdaptor(metaclass=ABCMeta):
- """Base class for loading and accessing matrix data"""
-
- def __init__(self, data_locator, app_config, dataset_config=None):
- if type(app_config) != AppConfig:
- raise TypeError("config expected to be of type AppConfig")
-
- # location to the dataset
- self.data_locator = data_locator
-
- # config is the application configuration
- self.app_config = app_config
- self.server_config = self.app_config.server_config
- self.dataset_config = dataset_config or app_config.default_dataset_config
-
- # parameters set by this data adaptor based on the data.
- self.parameters = {}
- self.uri_path = None
-
- def set_uri_path(self, path):
- # uri path to the dataset, e.g. /d/
- self.uri_path = path
-
- @staticmethod
- @abstractmethod
- def pre_load_validation(data_locator):
- pass
-
- @staticmethod
- @abstractmethod
- def open(data_locator, app_config, dataset_config):
- pass
-
- @staticmethod
- @abstractmethod
- def file_size(data_locator):
- pass
-
- @abstractmethod
- def get_name(self):
- """return a string name for this data adaptor"""
- pass
-
- @abstractmethod
- def get_library_versions(self):
- """return a dictionary of library name to library versions"""
- pass
-
- @abstractmethod
- def get_embedding_names(self):
- """return a list of pre-computed embedding names"""
- pass
-
- @abstractmethod
- def get_embedding_array(self, ename, dims=2):
- """return an numpy array for the given pre-computed embedding name."""
- pass
-
- @abstractmethod
- def get_X_array(self, obs_mask=None, var_mask=None):
- """return the X array, possibly filtered by obs_mask or var_mask.
- the return type is either ndarray or scipy.sparse.spmatrix."""
- pass
-
- @abstractmethod
- def get_X_approximate_distribution(self) -> XApproximateDistribution:
- """return the approximate distribution of the X matrix."""
- pass
-
- @abstractmethod
- def get_shape(self):
- pass
-
- @abstractmethod
- def query_var_array(self, term_var):
- pass
-
- @abstractmethod
- def query_obs_array(self, term_var):
- pass
-
- @abstractmethod
- def get_colors(self):
- pass
-
- @abstractmethod
- def get_obs_index(self):
- pass
-
- @abstractmethod
- def get_obs_columns(self):
- pass
-
- @abstractmethod
- def get_obs_keys(self):
- # return list of keys
- pass
-
- @abstractmethod
- def get_var_keys(self):
- # return list of keys
- pass
-
- @abstractmethod
- def cleanup(self):
- pass
-
- def get_data_locator(self):
- return self.data_locator
-
- def get_location(self):
- return self.data_locator.uri_or_path
-
- def get_about(self):
- return None
-
- def get_title(self):
- # default to file name
- location = self.get_location()
- if location.endswith("/"):
- location = location[:-1]
- return splitext(basename(location))[0]
-
- def get_corpora_props(self):
- return None
-
- @abstractmethod
- def get_schema(self):
- """
- Return current schema
- """
- pass
-
- @abstractmethod
- def annotation_to_fbs_matrix(self, axis, field=None, uid=None):
- """
- Gets annotation value for each observation
- :param axis: string obs or var
- :param fields: list of keys for annotation to return, returns all annotation values if not set.
- :return: flatbuffer: in fbs/matrix.fbs encoding
- """
- pass
-
- def update_parameters(self, parameters):
- parameters.update(self.parameters)
-
- def _index_filter_to_mask(self, filter, count):
- mask = np.zeros((count,), dtype=np.bool)
- for i in filter:
- if type(i) == list:
- mask[i[0] : i[1]] = True
- else:
- mask[i] = True
- return mask
-
- def _axis_filter_to_mask(self, axis, filter, count):
- mask = np.ones((count,), dtype=np.bool)
- if "index" in filter:
- mask = np.logical_and(mask, self._index_filter_to_mask(filter["index"], count))
- if "annotation_value" in filter:
- mask = np.logical_and(mask, self._annotation_filter_to_mask(axis, filter["annotation_value"], count))
-
- return mask
-
- def _annotation_filter_to_mask(self, axis, filter, count):
- mask = np.ones((count,), dtype=np.bool)
- for v in filter:
- name = v["name"]
- if axis == Axis.VAR:
- anno_data = self.query_var_array(name)
- elif axis == Axis.OBS:
- anno_data = self.query_obs_array(name)
-
- if anno_data.dtype.name in ["boolean", "category", "object"]:
- values = v.get("values", [])
- key_idx = np.in1d(anno_data, values)
- mask = np.logical_and(mask, key_idx)
-
- else:
- min_ = v.get("min", None)
- max_ = v.get("max", None)
- if min_ is not None:
- key_idx = (anno_data >= min_).ravel()
- mask = np.logical_and(mask, key_idx)
- if max_ is not None:
- key_idx = (anno_data <= max_).ravel()
- mask = np.logical_and(mask, key_idx)
-
- return mask
-
- def _filter_to_mask(self, filter):
- """
- Return the filter as a row and column selection list.
- No filter on a dimension means 'all'
- """
- shape = self.get_shape()
- var_selector = None
- obs_selector = None
- if filter is not None:
- if Axis.OBS in filter:
- obs_selector = self._axis_filter_to_mask(Axis.OBS, filter["obs"], shape[0])
-
- if Axis.VAR in filter:
- var_selector = self._axis_filter_to_mask(Axis.VAR, filter["var"], shape[1])
-
- return (obs_selector, var_selector)
-
- def check_new_labels(self, labels_df):
- """Check the new annotations labels, then set the labels_df index"""
- if labels_df is None or labels_df.empty:
- return
-
- labels_df.index = self.get_obs_index()
- if labels_df.index.name is None:
- labels_df.index.name = "index"
-
- # all labels must have a name, which must be unique and not used in obs column names
- if not labels_df.columns.is_unique:
- raise KeyError("All column names specified in user annotations must be unique.")
-
- # the label index must be unique, and must have same values the anndata obs index
- if not labels_df.index.is_unique:
- raise KeyError("All row index values specified in user annotations must be unique.")
-
- obs_columns = self.get_obs_columns()
-
- duplicate_columns = list(set(labels_df.columns) & set(obs_columns))
- if len(duplicate_columns) > 0:
- raise KeyError(
- "Labels file may not contain column names which overlap " f"with h5ad obs columns {duplicate_columns}"
- )
-
- # labels must have same count as obs annotations
- shape = self.get_shape()
- if labels_df.shape[0] != shape[0]:
- raise ValueError("Labels file must have same number of rows as data file.")
-
- # This will convert a float column that contains integer data into an integer type.
- # This case can occur when a user makes a copy of a category that originally contained integer data.
- # The client always copies array data to floats, therefore the copy will contain floats instead of integers.
- # float data is not allowed as a categorical type.
- if any([np.issubdtype(coltype.type, np.floating) for coltype in labels_df.dtypes]):
- labels_df = labels_df.convert_dtypes()
- for col, dtype in zip(labels_df, labels_df.dtypes):
- if isinstance(dtype, pd.Int32Dtype):
- labels_df[col] = labels_df[col].astype("int32")
- if isinstance(dtype, pd.Int64Dtype):
- labels_df[col] = labels_df[col].astype("int64")
-
- if any([np.issubdtype(coltype.type, np.floating) for coltype in labels_df.dtypes]):
- raise ValueError("Columns may not have floating point types")
-
- return labels_df
-
- def data_frame_to_fbs_matrix(self, filter, axis):
- """
- Retrieves data 'X' and returns in a flatbuffer Matrix.
- :param filter: filter: dictionary with filter params
- :param axis: string obs or var
- :return: flatbuffer Matrix
-
- Caveats:
- * currently only supports access on VAR axis
- * currently only supports filtering on VAR axis
- """
- if axis != Axis.VAR:
- raise ValueError("Only VAR dimension access is supported")
-
- try:
- obs_selector, var_selector = self._filter_to_mask(filter)
- except (KeyError, IndexError, TypeError, AttributeError, DatasetAccessError):
- raise FilterError("Error parsing filter")
-
- if obs_selector is not None:
- raise FilterError("filtering on obs unsupported")
-
- num_columns = self.get_shape()[1] if var_selector is None else np.count_nonzero(var_selector)
- if self.server_config.exceeds_limit("column_request_max", num_columns):
- raise ExceedsLimitError("Requested dataframe columns exceed column request limit")
-
- X = self.get_X_array(obs_selector, var_selector)
- col_idx = np.nonzero([] if var_selector is None else var_selector)[0]
- return encode_matrix_fbs(X, col_idx=col_idx, row_idx=None)
-
- def diffexp_topN(self, obsFilterA, obsFilterB, top_n=None):
- """
- Computes the top N differentially expressed variables between two observation sets. If mode
- is "TOP_N", then stats for the top N
- dataframes
- contain a subset of variables, then statistics for all variables will be returned, otherwise
- only the top N vars will be returned.
- :param obsFilterA: filter: dictionary with filter params for first set of observations
- :param obsFilterB: filter: dictionary with filter params for second set of observations
- :param top_n: Limit results to top N (Top var mode only)
- :return: top N genes and corresponding stats
- """
- if Axis.VAR in obsFilterA or Axis.VAR in obsFilterB:
- raise FilterError("Observation filters may not contain variable conditions")
- try:
- shape = self.get_shape()
- obs_mask_A = self._axis_filter_to_mask(Axis.OBS, obsFilterA["obs"], shape[0])
- obs_mask_B = self._axis_filter_to_mask(Axis.OBS, obsFilterB["obs"], shape[0])
- except (KeyError, IndexError):
- raise FilterError("Error parsing filter")
- if top_n is None:
- top_n = self.dataset_config.diffexp__top_n
-
- if self.server_config.exceeds_limit(
- "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B)
- ):
- raise ExceedsLimitError("Diffexp request exceeds max cell count limit")
-
- result = self.compute_diffexp_ttest(
- maskA=obs_mask_A, maskB=obs_mask_B, top_n=top_n, lfc_cutoff=self.dataset_config.diffexp__lfc_cutoff
- )
-
- try:
- return jsonify_strict(result)
- except ValueError:
- raise JSONEncodingValueError("Error encoding differential expression to JSON")
-
- @abstractmethod
- def compute_diffexp_ttest(self, maskA, maskB, top_n, lfc_cutoff):
- pass
-
- @staticmethod
- def normalize_embedding(embedding):
- """Normalize embedding layout to meet client assumptions.
- Embedding is an ndarray, shape (n_obs, n)., where n is normally 2
- """
-
- # scale isotropically
- try:
- min = np.nanmin(embedding, axis=0)
- max = np.nanmax(embedding, axis=0)
- except RuntimeError:
- # indicates entire array was NaN, which should propagate
- min = np.NaN
- max = np.NaN
-
- scale = np.amax(max - min)
- normalized_layout = (embedding - min) / scale
-
- # translate to center on both axis
- translate = 0.5 - ((max - min) / scale / 2)
- normalized_layout = normalized_layout + translate
-
- normalized_layout = normalized_layout.astype(dtype=np.float32)
- return normalized_layout
-
- def layout_to_fbs_matrix(self, fields):
- """
- return specified embeddings as a flatbuffer, using the cellxgene matrix fbs encoding.
-
- * returns only first two dimensions, with name {ename}_0 and {ename}_1,
- where {ename} is the embedding name.
- * client assumes each will be individually centered & scaled (isotropically)
- to a [0, 1] range.
- * does not support filtering
-
- """
- embeddings = self.get_embedding_names() if fields is None or len(fields) == 0 else fields
- layout_data = []
- with ServerTiming.time("layout.query"):
- for ename in embeddings:
- embedding = self.get_embedding_array(ename, 2)
- normalized_layout = DataAdaptor.normalize_embedding(embedding)
- layout_data.append(pd.DataFrame(normalized_layout, columns=[f"{ename}_0", f"{ename}_1"]))
-
- with ServerTiming.time("layout.encode"):
- if layout_data:
- df = pd.concat(layout_data, axis=1, copy=False)
- else:
- df = pd.DataFrame()
- fbs = encode_matrix_fbs(df, col_idx=df.columns, row_idx=None)
-
- return fbs
-
- def get_last_mod_time(self):
- try:
- lastmod = self.get_data_locator().lastmodtime()
- except RuntimeError:
- lastmod = None
- return lastmod
-
- def summarize_var(self, method, filter, query_hash):
- if method != "mean":
- raise UnsupportedSummaryMethod("Unknown gene set summary method.")
-
- obs_selector, var_selector = self._filter_to_mask(filter)
- if obs_selector is not None:
- raise FilterError("filtering on obs unsupported")
-
- # if no filter, just return zeros. We don't have a use case
- # for summarizing the entire X without a filter, and it would
- # potentially be quite compute / memory intensive.
- if var_selector is None or np.count_nonzero(var_selector) == 0:
- mean = np.zeros((self.get_shape()[0], 1), dtype=np.float32)
- else:
- X = self.get_X_array(obs_selector, var_selector)
- if sparse.issparse(X):
- mean = X.mean(axis=1).A
- else:
- mean = X.mean(axis=1, keepdims=True)
-
- col_idx = pd.Index([query_hash])
- return encode_matrix_fbs(mean, col_idx=col_idx, row_idx=None)
diff --git a/backend/czi_hosted/data_common/matrix_loader.py b/backend/czi_hosted/data_common/matrix_loader.py
deleted file mode 100644
index 53fd44b4..00000000
--- a/backend/czi_hosted/data_common/matrix_loader.py
+++ /dev/null
@@ -1,288 +0,0 @@
-from enum import Enum
-import threading
-import time
-
-from backend.common.utils.data_locator import DataLocator
-from backend.common.errors import DatasetAccessError
-from contextlib import contextmanager
-from http import HTTPStatus
-
-from backend.czi_hosted.data_common.rwlock import RWLock
-
-
-class MatrixDataCacheItem(object):
- """This class provides access and caching for a dataset. The first time a dataset is accessed, it is
- opened and cached. Later accesses use the cached version. It may also be deleted by the
- MatrixDataCacheManager to make room for another dataset. While a dataset is actively being used
- (during the lifetime of a api request), a reader lock is locked. During that time, the dataset cannot
- be removed."""
-
- def __init__(self, loader):
- self.loader = loader
- self.data_adaptor = None
- self.data_lock = RWLock()
-
- def acquire_existing(self):
- """If the data_adaptor exists, take a read lock and return it, else return None"""
- self.data_lock.r_acquire()
- if self.data_adaptor:
- return self.data_adaptor
-
- self.data_lock.r_release()
- return None
-
- def acquire_and_open(self, app_config, dataset_config=None):
- """returns the data_adaptor if cached. opens the data_adaptor if not.
- In either case, the a reader lock is taken. Must call release when
- the data_adaptor is no longer needed"""
- self.data_lock.r_acquire()
- if self.data_adaptor:
- return self.data_adaptor
- self.data_lock.r_release()
-
- self.data_lock.w_acquire()
- # the data may have been loaded while waiting on the lock
- if not self.data_adaptor:
- try:
- self.loader.pre_load_validation()
- self.data_adaptor = self.loader.open(app_config, dataset_config)
- except Exception as e:
- # necessary to hold the reader lock after an exception, since
- # the release will occur when the context exits.
- self.data_lock.w_demote()
- raise DatasetAccessError(str(e))
-
- # demote the write lock to a read lock.
- self.data_lock.w_demote()
- return self.data_adaptor
-
- def release(self):
- """Release the reader lock"""
- self.data_lock.r_release()
-
- def delete(self):
- """Clear resources used by this dataset"""
- with self.data_lock.w_locked():
- if self.data_adaptor:
- self.data_adaptor.cleanup()
- self.data_adaptor = None
-
- def attempt_delete(self):
- """Delete, but only if the write lock can be immediately locked. Return True if the delete happened"""
- if self.data_lock.w_acquire_non_blocking():
- if self.data_adaptor:
- try:
- self.data_adaptor.cleanup()
- self.data_adaptor = None
- except Exception:
- # catch all exceptions to ensure the lock is released
- pass
-
- self.data_lock.w_release()
- return True
- else:
- return False
-
-
-class MatrixDataCacheInfo(object):
- def __init__(self, cache_item, timestamp):
- # The MatrixDataCacheItem in the cache
- self.cache_item = cache_item
- # The last time the cache_item was accessed
- self.last_access = timestamp
- # The number of times the cache_item was accessed (used for testing)
- self.num_access = 1
-
-
-class MatrixDataCacheManager(object):
- """A class to manage the cached datasets. This is intended to be used as a context manager
- for handling api requests. When the context is created, the data_adator is either loaded or
- retrieved from a cache. In either case, the reader lock is taken during this time, and release
- when the context ends. This class currently implements a simple least recently used cache,
- which can delete a dataset from the cache to make room for a new one.
-
- This is the intended usage pattern:
-
- m = MatrixDataCacheManager(max_cached=..., timelimmit_s = ...)
- with m.data_adaptor(location, app_config) as data_adaptor:
- # use the data_adaptor for some operation
- """
-
- # FIXME: If the number of active datasets exceeds the max_cached, then each request could
- # lead to a dataset being deleted and a new only being opened: the cache will get thrashed.
- # In this case, we may need to send back a 503 (Server Unavailable), or some other error message.
-
- # NOTE: If the actual dataset is changed. E.g. a new set of datafiles replaces an existing set,
- # then the cache will not react to this, however once the cache time limit is reached, the dataset
- # will automatically be refreshed.
-
- def __init__(self, max_cached, timelimit_s=None):
- # key is tuple(url_dataroot, location), value is a MatrixDataCacheInfo
- self.datasets = {}
-
- # lock to protect the datasets
- self.lock = threading.Lock()
-
- # The number of datasets to cache. When max_cached is reached, the least recently used
- # cache is replaced with the newly requested one.
- # TODO: This is very simple. This can be improved by taking into account how much space is actually
- # taken by each dataset, instead of arbitrarily picking a max datasets to cache.
- self.max_cached = max_cached
-
- # items are automatically removed from the cache once this time limit is reached
- self.timelimit_s = timelimit_s
-
- @contextmanager
- def data_adaptor(self, url_dataroot, location, app_config):
- # create a loader for to this location if it does not already exist
-
- delete_adaptor = None
- data_adaptor = None
- cache_item = None
-
- key = (url_dataroot, location)
- with self.lock:
- self.evict_old_datasets()
- info = self.datasets.get(key)
- if info is not None:
- info.last_access = time.time()
- info.num_access += 1
- self.datasets[key] = info
- data_adaptor = info.cache_item.acquire_existing()
- cache_item = info.cache_item
-
- if data_adaptor is None:
- while True:
- if len(self.datasets) < self.max_cached:
- break
-
- items = list(self.datasets.items())
- items = sorted(items, key=lambda x: x[1].last_access)
- # close the least recently used loader
- oldest = items[0]
- oldest_cache = oldest[1].cache_item
- oldest_key = oldest[0]
- del self.datasets[oldest_key]
- delete_adaptor = oldest_cache
-
- loader = MatrixDataLoader(location, app_config=app_config)
- cache_item = MatrixDataCacheItem(loader)
- item = MatrixDataCacheInfo(cache_item, time.time())
- self.datasets[key] = item
-
- try:
- assert cache_item
- if delete_adaptor:
- delete_adaptor.delete()
- if data_adaptor is None:
- dataset_config = app_config.get_dataset_config(url_dataroot)
- data_adaptor = cache_item.acquire_and_open(app_config, dataset_config)
- yield data_adaptor
- except DatasetAccessError:
- cache_item.release()
- with self.lock:
- del self.datasets[key]
- cache_item.delete()
- cache_item = None
- raise
-
- finally:
- if cache_item:
- cache_item.release()
-
- def evict_old_datasets(self):
- # must be called with the lock held
- if self.timelimit_s is None:
- return
-
- now = time.time()
- to_del = []
- for key, info in self.datasets.items():
- if (now - info.last_access) > self.timelimit_s:
- # remove the data_cache when if it has been in the cache too long
- to_del.append((key, info))
-
- for key, info in to_del:
- # try and get the write_lock for the dataset.
- # if this returns false, it means the dataset is being used, and should
- # not be removed.
- if info.cache_item.attempt_delete():
- del self.datasets[key]
-
-
-class MatrixDataType(Enum):
- H5AD = "h5ad"
- CXG = "cxg"
- UNKNOWN = "unknown"
-
-
-class MatrixDataLoader(object):
- def __init__(self, location, matrix_data_type=None, app_config=None):
- """ location can be a string or DataLocator """
- region_name = None if app_config is None else app_config.server_config.data_locator__s3__region_name
- self.location = DataLocator(location, region_name=region_name)
- if not self.location.exists():
- raise DatasetAccessError("Dataset does not exist.", HTTPStatus.NOT_FOUND)
-
- # matrix_data_type is an enum value of type MatrixDataType
- self.matrix_data_type = matrix_data_type
- # matrix_type is a DataAdaptor type, which corresponds to the matrix_data_type
- self.matrix_type = None
-
- if matrix_data_type is None:
- self.matrix_data_type = self.__matrix_data_type()
-
- if not self.__matrix_data_type_allowed(app_config):
- raise DatasetAccessError("Dataset does not have an allowed type.")
-
- if self.matrix_data_type == MatrixDataType.H5AD:
- from backend.czi_hosted.data_anndata.anndata_adaptor import AnndataAdaptor
-
- self.matrix_type = AnndataAdaptor
- elif self.matrix_data_type == MatrixDataType.CXG:
- from backend.czi_hosted.data_cxg.cxg_adaptor import CxgAdaptor
-
- self.matrix_type = CxgAdaptor
-
- def __matrix_data_type(self):
- if self.location.path.endswith(".h5ad"):
- return MatrixDataType.H5AD
- elif ".cxg" in self.location.path:
- return MatrixDataType.CXG
- else:
- return MatrixDataType.UNKNOWN
-
- def __matrix_data_type_allowed(self, app_config):
- if self.matrix_data_type == MatrixDataType.UNKNOWN:
- return False
-
- if not app_config:
- return True
- if not app_config.is_multi_dataset():
- return True
- if len(app_config.server_config.multi_dataset__allowed_matrix_types) == 0:
- return True
-
- for val in app_config.server_config.multi_dataset__allowed_matrix_types:
- try:
- if self.matrix_data_type == MatrixDataType(val):
- return True
- except ValueError:
- # Check case where multi_dataset_allowed_matrix_type does not have a
- # valid MatrixDataType value. TODO: Add a feature to check
- # the AppConfig for errors on startup
- return False
-
- return False
-
- def pre_load_validation(self):
- if self.matrix_data_type == MatrixDataType.UNKNOWN:
- raise DatasetAccessError("Dataset does not have a recognized type: .h5ad or .cxg")
- self.matrix_type.pre_load_validation(self.location)
-
- def file_size(self):
- return self.matrix_type.file_size(self.location)
-
- def open(self, app_config, dataset_config=None):
- # create and return a DataAdaptor object
- return self.matrix_type.open(self.location, app_config, dataset_config)
diff --git a/backend/czi_hosted/data_common/rwlock.py b/backend/czi_hosted/data_common/rwlock.py
deleted file mode 100644
index f701afa1..00000000
--- a/backend/czi_hosted/data_common/rwlock.py
+++ /dev/null
@@ -1,135 +0,0 @@
-# -*- coding: utf-8 -*-
-""" rwlock.py
-
- A class to implement read-write locks on top of the standard threading
- library.
-
- This is implemented with two mutexes (threading.Lock instances) as per this
- wikipedia pseudocode:
-
- https://en.wikipedia.org/wiki/Readers%E2%80%93writer_lock#Using_two_mutexes
-
- Code written by Tyler Neylon at Unbox Research.
-
- This file is public domain.
-
- Modified to add a w_demote function to convert a writer lock to a reader lock
-"""
-
-
-# _______________________________________________________________________
-# Imports
-
-from contextlib import contextmanager
-from threading import Lock
-
-
-# _______________________________________________________________________
-# Class
-
-
-class RWLock(object):
- """ RWLock class; this is meant to allow an object to be read from by
- multiple threads, but only written to by a single thread at a time. See:
- https://en.wikipedia.org/wiki/Readers%E2%80%93writer_lock
-
- Usage:
-
- from rwlock import RWLock
-
- my_obj_rwlock = RWLock()
-
- # When reading from my_obj:
- with my_obj_rwlock.r_locked():
- do_read_only_things_with(my_obj)
-
- # When writing to my_obj:
- with my_obj_rwlock.w_locked():
- mutate(my_obj)
- """
-
- def __init__(self):
-
- self.w_lock = Lock()
- self.num_r_lock = Lock()
- self.num_r = 0
-
- # The d_lock is needed to handle the demotion case,
- # so that the writer can become a reader without releasing the w_lock.
- # the d_lock is held by the writer, and prevents any other thread from taking the
- # num_r_lock during that time, which means the writer thread is able to take the
- # num_r_lock to update the num_r.
- self.d_lock = Lock()
-
- # ___________________________________________________________________
- # Reading methods.
-
- def r_acquire(self):
- self.d_lock.acquire()
- self.num_r_lock.acquire()
- self.num_r += 1
-
- if self.num_r == 1:
- self.w_lock.acquire()
-
- self.num_r_lock.release()
- self.d_lock.release()
-
- def r_release(self):
- assert self.num_r > 0
- self.num_r_lock.acquire()
- self.num_r -= 1
- if self.num_r == 0:
- self.w_lock.release()
-
- self.num_r_lock.release()
-
- @contextmanager
- def r_locked(self):
- """ This method is designed to be used via the `with` statement. """
- try:
- self.r_acquire()
- yield
- finally:
- self.r_release()
-
- # ___________________________________________________________________
- # Writing methods.
-
- def w_acquire(self):
- self.d_lock.acquire()
- self.w_lock.acquire()
-
- def w_acquire_non_blocking(self):
- # if d_lock and w_lock can be acquired without blocking, acquire and return True,
- # else immediately return False.
- if self.d_lock.acquire(blocking=False):
- if self.w_lock.acquire(blocking=False):
- return True
- else:
- self.d_lock.release()
- return False
-
- def w_release(self):
- self.w_lock.release()
- self.d_lock.release()
-
- def w_demote(self):
- """demote a writer lock to a reader lock"""
-
- # the d_lock is already held from w_acquire.
- # releasing the d_lock at the end of this function allows multiple readers.
- # incrementing num_r makes this thread one of those readers.
- self.num_r_lock.acquire()
- self.num_r += 1
- self.num_r_lock.release()
- self.d_lock.release()
-
- @contextmanager
- def w_locked(self):
- """ This method is designed to be used via the `with` statement. """
- try:
- self.w_acquire()
- yield
- finally:
- self.w_release()
diff --git a/backend/czi_hosted/data_cxg/__init__.py b/backend/czi_hosted/data_cxg/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/backend/czi_hosted/data_cxg/cxg_adaptor.py b/backend/czi_hosted/data_cxg/cxg_adaptor.py
deleted file mode 100644
index a0643935..00000000
--- a/backend/czi_hosted/data_cxg/cxg_adaptor.py
+++ /dev/null
@@ -1,470 +0,0 @@
-import json
-import logging
-import os
-import threading
-
-import numpy as np
-import pandas as pd
-import tiledb
-from server_timing import Timing as ServerTiming
-
-from backend.common.constants import Axis, XApproximateDistribution
-from backend.common.errors import DatasetAccessError, ConfigurationError
-from backend.czi_hosted.common.immutable_kvcache import ImmutableKVCache
-from backend.common.utils.type_conversion_utils import get_schema_type_hint_from_dtype
-from backend.common.utils.utils import path_join
-from backend.czi_hosted.compute import diffexp_cxg
-from backend.czi_hosted.data_common.data_adaptor import DataAdaptor
-from backend.common.fbs.matrix import encode_matrix_fbs
-from backend.czi_hosted.data_cxg.cxg_util import pack_selector_from_mask
-
-
-class CxgAdaptor(DataAdaptor):
- # TODO: The tiledb context parameters should be a configuration option
- tiledb_ctx = tiledb.Ctx(
- {"sm.tile_cache_size": 8 * 1024 * 1024 * 1024, "sm.num_reader_threads": 32, "vfs.s3.region": "us-east-1"}
- )
-
- def __init__(self, data_locator, app_config=None, dataset_config=None):
- super().__init__(data_locator, app_config, dataset_config)
- self.lock = threading.Lock()
-
- self.url = data_locator.uri_or_path
- if self.url[-1] != "/":
- self.url += "/"
-
- # caching immutable state
- self.lsuri_results = ImmutableKVCache(lambda key: self._lsuri(uri=key, tiledb_ctx=self.tiledb_ctx))
- self.arrays = ImmutableKVCache(lambda key: self._open_array(uri=key, tiledb_ctx=self.tiledb_ctx))
- self.schema = None
- self.X_approximate_distribution = None
-
- self._validate_and_initialize()
-
- def cleanup(self):
- """close all the open tiledb arrays"""
- for array in self.arrays.values():
- array.close()
- self.arrays.clear()
-
- @staticmethod
- def set_tiledb_context(context_params):
- """Set the tiledb context. This should be set before any instances of CxgAdaptor are created"""
- try:
- CxgAdaptor.tiledb_ctx = tiledb.Ctx(context_params)
- tiledb.default_ctx(context_params)
-
- except tiledb.libtiledb.TileDBError as e:
- if e.message == "Global context already initialized!":
- if tiledb.default_ctx().config().dict() != CxgAdaptor.tiledb_ctx.config().dict():
- raise ConfigurationError("Cannot change tiledb configuration once it is set")
- else:
- raise ConfigurationError(f"Invalid tiledb context: {str(e)}")
-
- @staticmethod
- def pre_load_validation(data_locator):
- location = data_locator.uri_or_path
- if not CxgAdaptor.isvalid(location):
- logging.error(f"cxg matrix is not valid: {location}")
- raise DatasetAccessError("cxg matrix is not valid")
-
- @staticmethod
- def file_size(data_locator):
- return 0
-
- @staticmethod
- def open(data_locator, app_config, dataset_config=None):
- return CxgAdaptor(data_locator, app_config, dataset_config)
-
- def get_about(self):
- return self.about if self.about else super().get_about()
-
- def get_title(self):
- return self.title if self.title else super().get_title()
-
- def get_corpora_props(self):
- return self.corpora_props if self.corpora_props else super().get_corpora_props()
-
- def get_name(self):
- return "cellxgene cxg adaptor version"
-
- def get_library_versions(self):
- return dict(tiledb=tiledb.__version__)
-
- def get_path(self, *urls):
- return path_join(self.url, *urls)
-
- @staticmethod
- def _lsuri(uri, tiledb_ctx):
- def _cleanpath(p):
- if p[-1] == "/":
- return p[:-1]
- else:
- return p
-
- result = []
- tiledb.ls(uri, lambda path, type: result.append((_cleanpath(path), type)), ctx=tiledb_ctx)
- return result
-
- def lsuri(self, uri):
- """
- given a URI, do a tiledb.ls but normalizing for all path weirdness:
- * S3 URIs require trailing slash. file: doesn't care.
- * results on S3 *have* a trailing slash, Posix does not.
-
- returns list of (absolute paths, type) *without* trailing slash
- in the path.
- """
- if uri[-1] != "/":
- uri += "/"
- return self.lsuri_results[uri]
-
- @staticmethod
- def isvalid(url):
- """
- Return True if this looks like a valid CXG, False if not. Just a quick/cheap
- test, not to be fully trusted.
- """
- if not tiledb.object_type(url, ctx=CxgAdaptor.tiledb_ctx) == "group":
- return False
- if not tiledb.object_type(path_join(url, "obs"), ctx=CxgAdaptor.tiledb_ctx) == "array":
- return False
- if not tiledb.object_type(path_join(url, "var"), ctx=CxgAdaptor.tiledb_ctx) == "array":
- return False
- if not tiledb.object_type(path_join(url, "X"), ctx=CxgAdaptor.tiledb_ctx) == "array":
- return False
- if not tiledb.object_type(path_join(url, "emb"), ctx=CxgAdaptor.tiledb_ctx) == "group":
- return False
- return True
-
- def has_array(self, name):
- a_type = tiledb.object_type(path_join(self.url, name), ctx=self.tiledb_ctx)
- return a_type == "array"
-
- def _validate_and_initialize(self):
- """
- remember, preload_validation() has already been called, so
- no need to repeat anything it has done.
-
- Load the CXG "group" metadata and cache instance values.
- Be very aware of multiple versions of the CXG object.
-
- CXG versions in the wild:
- * version 0, aka "no version" -- can be detected by the lack
- of a cxg_group_metadata array.
- * version 0.1 -- metadata attache to cxg_group_metadata array.
- Same as 0, except it adds group metadata.
- """
- title = None
- about = None
- corpora_props = None
- if self.has_array("cxg_group_metadata"):
- # version >0
- gmd = self.open_array("cxg_group_metadata")
- cxg_version = gmd.meta["cxg_version"]
- # version 0.1 used a malformed/shorthand semver string.
- if cxg_version == "0.1" or cxg_version == "0.2.0":
- cxg_properties = json.loads(gmd.meta["cxg_properties"])
- title = cxg_properties.get("title", None)
- about = cxg_properties.get("about", None)
- if cxg_version == "0.2.0":
- corpora_props = json.loads(gmd.meta["corpora"]) if "corpora" in gmd.meta else None
- else:
- # version 0
- cxg_version = "0.0"
-
- if cxg_version not in ["0.0", "0.1", "0.2.0"]:
- raise DatasetAccessError(f"cxg matrix is not valid: {self.url}")
-
- if self.dataset_config.X_approximate_distribution == "auto":
- raise ConfigurationError("X-approximate-distribution 'auto' mode unsupported.")
- self.X_approximate_distribution = self.dataset_config.X_approximate_distribution
-
- self.title = title
- self.about = about
- self.cxg_version = cxg_version
- self.corpora_props = corpora_props
-
- @staticmethod
- def _open_array(uri, tiledb_ctx):
- with tiledb.Array(uri, mode="r", ctx=tiledb_ctx) as array:
- if array.schema.sparse:
- return tiledb.SparseArray(uri, mode="r", ctx=tiledb_ctx)
- else:
- return tiledb.DenseArray(uri, mode="r", ctx=tiledb_ctx)
-
- def open_array(self, name):
- try:
- p = self.get_path(name)
- return self.arrays[p]
- except tiledb.libtiledb.TileDBError:
- raise DatasetAccessError(name)
-
- def get_embedding_array(self, ename, dims=2):
- array = self.open_array(f"emb/{ename}")
- return array[:, 0:dims]
-
- def compute_diffexp_ttest(self, maskA, maskB, top_n=None, lfc_cutoff=None):
- if top_n is None:
- top_n = self.dataset_config.diffexp__top_n
- if lfc_cutoff is None:
- lfc_cutoff = self.dataset_config.diffexp__lfc_cutoff
- return diffexp_cxg.diffexp_ttest(
- adaptor=self, maskA=maskA, maskB=maskB, top_n=top_n, diffexp_lfc_cutoff=lfc_cutoff
- )
-
- def get_colors(self):
- if self.cxg_version == "0.0":
- return dict()
- meta = self.open_array("cxg_group_metadata").meta
- return json.loads(meta["cxg_category_colors"]) if "cxg_category_colors" in meta else dict()
-
- def __remap_indices(self, coord_range, coord_mask, coord_data):
- """
- This function maps the indices in coord_data, which could be in the range [0,coord_range), to
- a range that only includes the number of indices encoded in coord_mask.
- coord_range is the maxinum size of the range (e.g. get_shape()[0] or get_shape()[1])
- coord_mask is a mask passed into the get_X_array, of size coord_range
- coord_data are indices representing locations of non-zero values, in the range [0,coord_range).
-
- For example, say
- coord_mask = [1,0,1,0,0,1]
- coord_data = [2,0,2,2,5]
-
- The function computes the following:
- indices = [0,2,5]
- ncoord = 3
- maprange = [0,1,2]
- mapindex = [0,0,1,0,0,2]
- coordindices = [1,0,1,1,2]
- """
- if coord_mask is None:
- return coord_range, coord_data
-
- indices = np.where(coord_mask)[0]
- ncoord = indices.shape[0]
- maprange = np.arange(ncoord)
- mapindex = np.zeros(indices[-1] + 1, dtype=int)
- mapindex[indices] = maprange
- coordindices = mapindex[coord_data]
- return ncoord, coordindices
-
- def get_X_array(self, obs_mask=None, var_mask=None):
- obs_items = pack_selector_from_mask(obs_mask)
- var_items = pack_selector_from_mask(var_mask)
- if obs_items is None or var_items is None:
- # If either zero rows or zero columns were selected, return an empty 2d array.
- shape = self.get_shape()
- obs_size = 0 if obs_items is None else shape[0] if obs_mask is None else np.count_nonzero(obs_mask)
- var_size = 0 if var_items is None else shape[1] if var_mask is None else np.count_nonzero(var_mask)
- return np.ndarray((obs_size, var_size))
-
- X = self.open_array("X")
-
- if X.schema.sparse:
- if obs_items == slice(None) and var_items == slice(None):
- data = X[:, :]
- else:
- data = X.multi_index[obs_items, var_items]
-
- nrows, obsindices = self.__remap_indices(X.shape[0], obs_mask, data.get("coords", data)["obs"])
- ncols, varindices = self.__remap_indices(X.shape[1], var_mask, data.get("coords", data)["var"])
- densedata = np.zeros((nrows, ncols), dtype=self.get_X_array_dtype())
- densedata[obsindices, varindices] = data[""]
- if self.has_array("X_col_shift"):
- X_col_shift = self.open_array("X_col_shift")
- if var_items == slice(None):
- densedata += X_col_shift[:]
- else:
- densedata += X_col_shift.multi_index[var_items][""]
-
- return densedata
-
- else:
- if obs_items == slice(None) and var_items == slice(None):
- data = X[:, :]
- else:
- data = X.multi_index[obs_items, var_items][""]
- return data
-
- def get_X_approximate_distribution(self) -> XApproximateDistribution:
- return self.X_approximate_distribution
-
- def get_shape(self):
- X = self.open_array("X")
- return X.shape
-
- def get_X_array_dtype(self):
- X = self.open_array("X")
- return X.dtype
-
- def query_var_array(self, term_name):
- var = self.open_array("var")
- data = var.query(attrs=[term_name])[:][term_name]
- return data
-
- def query_obs_array(self, term_name):
- var = self.open_array("obs")
- try:
- data = var.query(attrs=[term_name])[:][term_name]
- except tiledb.libtiledb.TileDBError:
- raise DatasetAccessError("query_obs")
- return data
-
- def get_obs_names(self):
- # get the index from the meta data
- obs = self.open_array("obs")
- meta = json.loads(obs.meta["cxg_schema"])
- index_name = meta["index"]
- return index_name
-
- def get_obs_index(self):
- obs = self.open_array("obs")
- meta = json.loads(obs.meta["cxg_schema"])
- index_name = meta["index"]
- data = obs.query(attrs=[index_name])[:][index_name]
- return data
-
- def get_obs_columns(self):
- obs = self.open_array("obs")
- schema = obs.schema
- col_names = [attr.name for attr in schema]
- return pd.Index(col_names)
-
- def get_obs_keys(self):
- obs = self.open_array("obs")
- schema = obs.schema
- return [attr.name for attr in schema]
-
- def get_var_keys(self):
- var = self.open_array("var")
- schema = var.schema
- return [attr.name for attr in schema]
-
- # function to get the embedding
- # this function to iterate through embeddings.
- def get_embedding_names(self):
- with ServerTiming.time("layout.lsuri"):
- pemb = self.get_path("emb")
- embeddings = [os.path.basename(p) for (p, t) in self.lsuri(pemb) if t == "array"]
- if len(embeddings) == 0:
- raise DatasetAccessError("cxg matrix missing embeddings")
- return embeddings
-
- def _get_schema(self):
- if self.schema:
- return self.schema
-
- shape = self.get_shape()
- dtype = self.get_X_array_dtype()
-
- dataframe = {"nObs": shape[0], "nVar": shape[1], **get_schema_type_hint_from_dtype(dtype)}
-
- annotations = {}
- for ax in ("obs", "var"):
- A = self.open_array(ax)
- schema_hints = json.loads(A.meta["cxg_schema"]) if "cxg_schema" in A.meta else {}
- if type(schema_hints) is not dict:
- raise TypeError("Array schema was malformed.")
-
- cols = []
- for attr in A.schema:
- schema = dict(name=attr.name, writable=False)
- type_hint = schema_hints.get(attr.name, {})
- # type hints take precedence
- if "type" in type_hint:
- schema["type"] = type_hint["type"]
- if schema["type"] == "categorical" and "categories" in type_hint:
- schema["categories"] = type_hint["categories"]
- else:
- schema.update(get_schema_type_hint_from_dtype(attr.dtype))
- cols.append(schema)
-
- annotations[ax] = dict(columns=cols)
-
- if "index" in schema_hints:
- annotations[ax].update({"index": schema_hints["index"]})
-
- obs_layout = []
- embeddings = self.get_embedding_names()
- for ename in embeddings:
- A = self.open_array(f"emb/{ename}")
- obs_layout.append({"name": ename, "type": "float32", "dims": [f"{ename}_{d}" for d in range(0, A.ndim)]})
-
- schema = {"dataframe": dataframe, "annotations": annotations, "layout": {"obs": obs_layout}}
- return schema
-
- def get_schema(self):
- if self.schema is None:
- with self.lock:
- self.schema = self._get_schema()
- return self.schema
-
- def _annotations_field_split(self, axis, fields, A, labels):
- """
- fields: requested fields, may be None (all)
- labels: writable user annotations dataframe, if any
-
- Remove redundant fields, raise KeyError on non-existant fields,
- and split into three lists:
- fields_to_fetch_from_cxg
- fields_to_fetch_from_labels
- fields_to_return
-
- if we have to return from labels, the fetch fields will contain the index
- to join on, which may not be in fields_to_return
- """
- need_labels = axis == Axis.OBS and labels is not None and not labels.empty
- index_key = self.get_obs_names() if need_labels else None
-
- if not fields:
- return (None, None, None, index_key)
-
- cxg_keys = frozenset([a.name for a in A.schema])
- user_anno_keys = frozenset(labels.columns.tolist()) if need_labels else frozenset()
- return_keys = frozenset(fields)
-
- label_join_index = frozenset([index_key]) if need_labels and (return_keys & user_anno_keys) else frozenset()
-
- unknown_fields = return_keys - (cxg_keys | user_anno_keys)
- if unknown_fields:
- raise KeyError("_".join(unknown_fields))
-
- return (
- list((return_keys & cxg_keys) | label_join_index),
- list(return_keys & user_anno_keys),
- list(return_keys),
- index_key,
- )
-
- def annotation_to_fbs_matrix(self, axis, fields=None, labels=None):
- with ServerTiming.time(f"annotations.{axis}.query"):
- A = self.open_array(str(axis))
-
- # may raise if fields contains unknown key
- cxg_fields, anno_fields, return_fields, index_field = self._annotations_field_split(axis, fields, A, labels)
-
- if cxg_fields is None:
- data = A[:]
- elif cxg_fields:
- data = A.query(attrs=cxg_fields)[:]
- else:
- data = {}
-
- df = pd.DataFrame.from_dict(data)
-
- if axis == Axis.OBS and labels is not None and not labels.empty:
- if anno_fields is None:
- assert index_field
- df = df.join(labels, index_field)
- elif anno_fields:
- assert index_field
- df = df.join(labels[anno_fields], index_field)
-
- if return_fields:
- df = df[return_fields]
-
- with ServerTiming.time(f"annotations.{axis}.encode"):
- fbs = encode_matrix_fbs(df, col_idx=df.columns)
-
- return fbs
diff --git a/backend/czi_hosted/data_cxg/cxg_util.py b/backend/czi_hosted/data_cxg/cxg_util.py
deleted file mode 100644
index f5759e8d..00000000
--- a/backend/czi_hosted/data_cxg/cxg_util.py
+++ /dev/null
@@ -1,37 +0,0 @@
-import numpy as np
-
-
-def pack_selector_from_mask(boolarray):
- """
- pack all contiguous selectors into slices. Remember that
- tiledb multi_index requires INCLUSIVE indices.
- """
-
- if boolarray is None:
- return slice(None)
-
- assert type(boolarray) == np.ndarray
- assert boolarray.dtype == bool
-
- selector = np.nonzero(boolarray)[0]
- return pack_selector_from_indices(selector)
-
-
-def pack_selector_from_indices(selector):
-
- if len(selector) == 0:
- return None
-
- result = []
- current = slice(selector[0], selector[0])
- for sel in selector[1:]:
- if sel == current.stop + 1:
- current = slice(current.start, sel)
- else:
- result.append(current if current.start != current.stop else current.start)
- current = slice(sel, sel)
-
- if len(result) == 0 or result[-1] != current:
- result.append(current if current.start != current.stop else current.start)
-
- return result
diff --git a/backend/czi_hosted/db/__init__.py b/backend/czi_hosted/db/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/backend/czi_hosted/db/cellxgene_orm.py b/backend/czi_hosted/db/cellxgene_orm.py
deleted file mode 100644
index 2860f6b1..00000000
--- a/backend/czi_hosted/db/cellxgene_orm.py
+++ /dev/null
@@ -1,58 +0,0 @@
-import uuid
-
-from sqlalchemy import Column, DateTime, ForeignKey, String, func, JSON
-from sqlalchemy.dialects.postgresql import UUID
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import relationship
-
-Base = declarative_base()
-
-
-class CellxGeneUser(Base):
- """
- A registered CellxGene user.
- Links a user to their annotations
- """
-
- __tablename__ = "cxguser"
-
- id = Column(String, primary_key=True)
- created_at = Column(DateTime, nullable=False, server_default=func.now())
- updated_at = Column(DateTime, nullable=False, server_default=func.now(), onupdate=func.now())
-
- # Relationships
- annotations = relationship("Annotation", back_populates="cxguser")
-
-
-class Annotation(Base):
- """
- An annotation is a link between a user, a dataset and tiledb dataframe. A user can have multiple annotations for a
- dataset, the most recent annotation (based on created_at) will be the default returned when queried
- """
-
- __tablename__ = "annotation"
-
- id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4, unique=True, nullable=False)
- tiledb_uri = Column(String)
- user_id = Column(String, ForeignKey("cxguser.id"), nullable=False)
- dataset_id = Column(UUID, ForeignKey("cxgdataset.id"), nullable=False)
-
- created_at = Column(DateTime, nullable=False, server_default=func.now())
- schema_hints = Column(JSON)
- # Relationships
- cxguser = relationship("CellxGeneUser", back_populates="annotations")
- dataset = relationship("CellxGeneDataset", back_populates="annotations")
-
-
-class CellxGeneDataset(Base):
- """
- Datasets refer to datasets stored by cellxgene
- """
-
- __tablename__ = "cxgdataset"
-
- id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4, unique=True, nullable=False)
- name = Column(String, unique=True, index=True)
-
- created_at = Column(DateTime, nullable=False, server_default=func.now())
- annotations = relationship("Annotation", back_populates="dataset")
diff --git a/backend/czi_hosted/db/create_db.py b/backend/czi_hosted/db/create_db.py
deleted file mode 100644
index 58e2c6de..00000000
--- a/backend/czi_hosted/db/create_db.py
+++ /dev/null
@@ -1,18 +0,0 @@
-"""
-Drops and recreates all tables for local testing according to cellxgene_orm.py
-"""
-from sqlalchemy import create_engine
-
-from backend.czi_hosted.db.cellxgene_orm import Base
-
-
-def create_db(database_uri: str = "postgresql://postgres:test_pw@localhost:5432"):
- engine = create_engine(database_uri)
- print("Dropping tables")
- Base.metadata.drop_all(engine)
- print("Recreating tables")
- Base.metadata.create_all(engine)
-
-
-if __name__ == "__main__":
- create_db()
diff --git a/backend/czi_hosted/db/db_utils.py b/backend/czi_hosted/db/db_utils.py
deleted file mode 100644
index 85230f63..00000000
--- a/backend/czi_hosted/db/db_utils.py
+++ /dev/null
@@ -1,71 +0,0 @@
-import typing
-import uuid
-
-from sqlalchemy import create_engine
-from sqlalchemy.orm import sessionmaker
-
-from backend.czi_hosted.db.cellxgene_orm import Base, CellxGeneDataset, CellxGeneUser
-
-
-class DbUtils:
- def __init__(self, database_uri: str = "postgresql://postgres:test_pw@localhost:5432"):
- self.session = DBSessionMaker(database_uri).session()
- self.engine = self.session.get_bind()
-
- def get(self, table: Base, entity_id: typing.Union[str, typing.Tuple[str]]) -> typing.Union[Base, None]:
- """
- Query a table row by its primary key
- :param table: SQLAlchemy Table to query
- :param entity_id: Primary key of desired row
- :return: SQLAlchemy Table object, None if not found
- """
- return self.session.query(table).get(entity_id)
-
- def query(self, table_args: typing.List[Base], filter_args: typing.List[bool] = None) -> typing.List[Base]:
- """
- Query the database using the current DB session
- :param table_args: List of SQLAlchemy Tables to query/join
- :param filter_args: List of SQLAlchemy filter conditions
- :return: List of SQLAlchemy query response objects
- """
- return (
- self.session.query(*table_args).filter(*filter_args).all()
- if filter_args
- else self.session.query(*table_args).all()
- )
-
- def query_for_most_recent(self, table: Base, filter_args: typing.List[bool] = None) -> Base:
- try:
- return self.session.query(table).filter(*filter_args).order_by(table.created_at.desc()).limit(1).all()[0]
- except IndexError:
- return None
-
- def get_or_create_dataset(self, dataset_name):
- try:
- dataset_id = self.query(table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == dataset_name])[
- 0
- ].id
- except IndexError:
- dataset_id = uuid.uuid4()
- dataset = CellxGeneDataset(id=dataset_id, name=dataset_name)
- self.session.add(dataset)
- self.session.commit()
- return str(dataset_id)
-
- def get_or_create_user(self, user_id):
- try:
- user_id = self.query(table_args=[CellxGeneUser], filter_args=[CellxGeneUser.id == user_id])[0].id
- except IndexError:
- user = CellxGeneUser(id=user_id)
- self.session.add(user)
- self.session.commit()
- return str(user_id)
-
-
-class DBSessionMaker:
- def __init__(self, database_uri):
- self.engine = create_engine(database_uri, connect_args={"connect_timeout": 5})
- self.session_maker = sessionmaker(bind=self.engine)
-
- def session(self, **kwargs):
- return self.session_maker(**kwargs)
diff --git a/backend/czi_hosted/default_config.py b/backend/czi_hosted/default_config.py
deleted file mode 100644
index 214eeb31..00000000
--- a/backend/czi_hosted/default_config.py
+++ /dev/null
@@ -1,266 +0,0 @@
-import yaml
-
-default_config = """
-server:
- app:
- verbose: false
- debug: false
- host: localhost
- port : null
- open_browser: false
- force_https: false
- flask_secret_key: null
- generate_cache_control_headers: false
- server_timing_headers: false
- csp_directives: null
-
- # By default, cellxgene will serve api requests from the same base url as the webpage.
- # In general api_base_url and web_base_url will not need to be set.
- # There are two reasons to set these parameters:
- # 1. Oauth authentication is used; the oauth server will redirect back to the api_base_url after login,
- # which then redirects back to the web_base_url. If the web_base_url is not set, it will default to
- # the api_base_url. If oauth authentication is used, the api_base_url must be set.
- # For a local test (where the server runs on "http://localhost:"), then the api_base_url may be
- # set to the string "local".
- # 2. The cellxgene deploymnent is in an environment where the webpage and api have
- # different base urls. In this case both api_base_url and web_base_url must be set.
- # It is up to the server admin to ensure that the networking is setup correctly for this environment.
- api_base_url: null
- web_base_url: null
-
- authentication:
- # The authentication types may be "none", "session", "oauth" or "test"
- # none: No authentication support, features like user_annotations must not be enabled.
- # session: A session based userid is automatically generated. (no params needed)
- # oauth: oauth2 is used for authentication; parameters are defined in params_oauth.
- # test: Simple module for testing the authentication logic without connecting to an external service
- type: test
- insecure_test_environment: true
-
- params_oauth:
- # url to the oauth server
- oauth_api_base_url: null
- # client_id of this app
- client_id: null
- # the client_secret known to the auth server and this app
- client_secret: null
- # jwt_decode_options, to specify non default decode options define
- # jwt_decode_options to be a dictionary with key/values described by
- # the options parameter of the jose.jwt.decode function:
- # (https://python-jose.readthedocs.io/en/latest/jwt/api.html)
- jwt_decode_options: null
-
- # if true, the jwt containing the id_token is stored in a session cookie
- session_cookie: true
-
- # if session_cookie is false, then a regular cookie will be used. In that case
- # the cookie will be defined by a dictionary of parameters.
- # The keys of the dictionary match the parameters of the flask set_cookie api
- # (https://flask.palletsprojects.com/en/1.1.x/api/), and with the same meaning.
- # legal keys: key, max_age, expires, path, domain, secure, httponly, and samesite.
- cookie: null
-
- multi_dataset:
- # If dataroot is set, then cellxgene may serve multiple datasets. This parameter is not
- # compatible with single_dataset/datapath.
- # dataroot may be a string, representing the path to a directory or S3 prefix. In this
- # case the datasets in that location are accessed from /d/.
- # example:
- # dataroot: /path/to/datasets/
- # or
- # dataroot: s3://bucket/prefix/
- #
- # As an alternative, dataroot can be a dictionary, where a dataset key is associated with a base_url
- # and a dataroot.
- # example:
- # dataroot:
- # d1:
- # base_url: set1
- # dataroot: /path/to/set1_datasets/
- # d2:
- # base_url: set2/subdir
- # dataroot: /path/to/set2_datasets/
- #
- # In this case, datasets can be accessed from /set1/ or
- # /set2/subdir/. It is possible to have different dataset configurations
- # for datasets accessed through different dataroots. For example, in one dataroot, the
- # user annotations could be enabled, and in another dataroot they could be disabled.
- # To specify dataroot configurations, add a new top level dictionary to the config named
- # per_dataset_config. Within per_dataset_config create a dictionary for each dataroot to specialize
- # ("d1" or "d2" from the example). Each of these dictionaries has the exact same form as the "dataset"
- # dictionary (see below).
- # When this approach is used, the values for each configuration option are checked in
- # this order: per_dataset_config/, dataset, then the default values.
- #
- # example:
- #
- # per_dataset_config:
- # d1:
- # user_annotations:
- # enable: false
- # d2:
- # user_annotations:
- # enable: true
-
- dataroot: null
-
- # The index page when in multi-dataset mode:
- # false or null: this returns a 404 code
- # true: loads a test index page, which links to the datasets that are available in the dataroot
- # string/URL: redirect to this URL: flask.redirect(config.multi_dataset__index)
- index: false
-
- # A list of allowed matrix types. If an empty list, then all matrix types are allowed
- allowed_matrix_types: []
-
- matrix_cache:
- # The maximum number of datasets that may be opened at one time. The least recently used dataset
- # is evicted from the cache first.
- max_datasets: 5
-
- # A matrix is automatically removed from the cache after timelimit_s number of seconds.
- # If timelimit_s is set to None, then there is no time limit.
- timelimit_s: 30
-
- single_dataset:
- # If datapath is set, then cellxgene with serve a single dataset located at datapath. This parameter is not
- # compatible with multi_dataset/dataroot.
- datapath: null
- obs_names: null
- var_names: null
- about: null
- title: null
-
- diffexp:
- alg_cxg:
- # The number of threads to use is computed from: min(max_workers, cpu_multipler * cpu_count).
- # Where cpu_count is determined at runtime.
- max_workers: 64
- cpu_multiplier: 4
-
- # The target number of matrix elements that are evaluated
- # together in one thread.
- target_workunit: 16_000_000
-
- data_locator:
- s3:
- # s3 region name.
- # if true, then the s3 location is automatically determined from the datapath or dataroot.
- # if false/null, then do not set.
- # if a string, then use that value (e.g. us-east-1).
- region_name: true
-
- adaptor:
- cxg_adaptor:
- # The key/values under tiledb_ctx will be used to initialize the tiledb Context.
- # If 'vfs.s3.region' is not set, then it will automatically use the setting from
- # data_locator / s3 / region_name.
- tiledb_ctx:
- sm.tile_cache_size: 8589934592
- sm.num_reader_threads: 32
-
- anndata_adaptor:
- backed: false
-
- limits:
- column_request_max: 32
- diffexp_cellcount_max: null
-
-
-dataset:
- app:
- # Scripts can be a list of either file names (string) or dicts containing keys src, integrity and crossorigin.
- # these will be injected into the index template as script tags with these attributes set.
- scripts: []
- # Inline scripts are a list of file names, where the contents of the file will be injected into the index.
- inline_scripts: []
-
- about_legal_tos: null
- about_legal_privacy: null
-
- # allow authentication support
- authentication_enable: true
-
- presentation:
- max_categories: 1000
- custom_colors: true
-
- user_annotations:
- enable: true
- type: local_file_csv
- hosted_tiledb_array:
- db_uri: null
- hosted_file_directory: null
- local_file_csv:
- directory: null
- file: null
-
- embeddings:
- names : []
-
- diffexp:
- enable: true
- lfc_cutoff: 0.01
- top_n: 10
-
- X_approximate_distribution: normal # currently fixed config
-
-external:
- # You can retrieve configuration parameters from this config file, the environment,
- # the AWS secrets manager, or from the "cellxgene launch" command line arguments.
- # They are applied in that order, meaning that if a parameter is defined in more
- # than one location, the last one applied takes effect.
-
- # environment variables:
- # This section describes how to map environment variables to configuration parameters.
- # The format is a list defining an environment variable.
- # Each entry in the list is a dictionary with three entries:
- # name: the name of the environment variable
- # path: the path within the cellxgene configuration to update.
- # required: (default=False) a boolean. If true, then it is an error if the environment variable is not set.
-
- environment:
- - name: CXG_SECRET_KEY
- path: [server, app, flask_secret_key]
- required: false
- - name: CXG_OAUTH_CLIENT_SECRET
- path: [server, authentication, params_oauth, client_secret]
- required: false
-
- # AWS Secrets Manager
- # This section describes how to map aws secrets to configuration parameters.
- # The format is the region for the secrets manager, then a list of secrets.
- # each secret has a name, and a list of values.
- # Each entry in the list of values is a dictionary with three entries:
- # key: the key of the aws secret.
- # path: the path within the cellxgene configuration to update.
- # required: (default=False) a boolean. If true, then it is an error if the key does not exist in the secret.
- #
- # example:
- # aws_secrets_manager:
- # region: us-west-2
- # - name: my_first_secret
- # values:
- # - key: flask_secret_key
- # path: [server, app, flask_secret_key]
- # required: true
- # - key: db_uri
- # path: [dataset, user_annotations, hosted_tiledb_array, db_uri]
- # required: true
- # - name: my_auth_secret
- # values:
- # - key: client_secret
- # path: [server, authentication, params_oauth, client_secret]
- # required: true
- # - key: client_id
- # path: [server, authentication, params_oauth, client_id]
- # required: true
-
- aws_secrets_manager:
- region: null
- secrets: []
-"""
-
-
-def get_default_config():
- return yaml.load(default_config, Loader=yaml.Loader)
diff --git a/backend/czi_hosted/eb/.ebextensions/database.config b/backend/czi_hosted/eb/.ebextensions/database.config
deleted file mode 100644
index e69de29b..00000000
diff --git a/backend/czi_hosted/eb/.ebextensions/enable_mod_deflate.config b/backend/czi_hosted/eb/.ebextensions/enable_mod_deflate.config
deleted file mode 100644
index 3456869b..00000000
--- a/backend/czi_hosted/eb/.ebextensions/enable_mod_deflate.config
+++ /dev/null
@@ -1,30 +0,0 @@
-files:
- "/etc/httpd/conf.d/enable_mod_deflate.conf":
- mode: "000644"
- owner: root
- group: root
- content: |
-
-
- AddOutputFilterByType DEFLATE text/plain
- AddOutputFilterByType DEFLATE text/html
- AddOutputFilterByType DEFLATE application/xhtml+xml
- AddOutputFilterByType DEFLATE text/xml
- AddOutputFilterByType DEFLATE application/xml
- AddOutputFilterByType DEFLATE application/xml+rss
- AddOutputFilterByType DEFLATE application/x-javascript
- AddOutputFilterByType DEFLATE text/javascript
- AddOutputFilterByType DEFLATE text/css
- AddOutputFilterByType DEFLATE application/octet-stream
-
- DeflateCompressionLevel 9
-
- BrowserMatch ^Mozilla/4 gzip-only-text/html
- BrowserMatch ^Mozilla/4\.0[678] no-gzip
- BrowserMatch \bMSI[E] !no-gzip !gzip-only-text/html
-
-
- Header append Vary User-Agent env=!dont-vary
-
-
-
diff --git a/backend/czi_hosted/eb/.ebextensions/wsgi_app_group_global.config b/backend/czi_hosted/eb/.ebextensions/wsgi_app_group_global.config
deleted file mode 100644
index cf2ecc61..00000000
--- a/backend/czi_hosted/eb/.ebextensions/wsgi_app_group_global.config
+++ /dev/null
@@ -1,10 +0,0 @@
-# Configure WSGI so that it will work with numpy, scanpy, etc, which all use the
-# Python SWIG, and therefore will deadlock on start. For more information, see
-# https://modwsgi.readthedocs.io/en/develop/user-guides/application-issues.html#python-simplified-gil-state-api
-files:
- "/etc/httpd/conf.d/wsgi_custom.conf":
- mode: "000644"
- owner: root
- group: root
- content: |
- WSGIApplicationGroup %{GLOBAL}
diff --git a/backend/czi_hosted/eb/.gitignore b/backend/czi_hosted/eb/.gitignore
deleted file mode 100644
index bca646a7..00000000
--- a/backend/czi_hosted/eb/.gitignore
+++ /dev/null
@@ -1,5 +0,0 @@
-
-# Elastic Beanstalk Files
-.elasticbeanstalk/*
-!.elasticbeanstalk/*.cfg.yml
-!.elasticbeanstalk/*.global.yml
diff --git a/backend/czi_hosted/eb/Makefile b/backend/czi_hosted/eb/Makefile
deleted file mode 100644
index f00f6a92..00000000
--- a/backend/czi_hosted/eb/Makefile
+++ /dev/null
@@ -1,59 +0,0 @@
-include ../../../common.mk
-
-.PHONY: clean
-clean:
- rm -f artifact.zip
- rm -rf artifact.dir
-
-
-# Build the ElasticBeanstalk configuration and deployment bundle,
-# such that deployment can be done with a simple `eb deploy`.
-# Presumes that a top-level `make build-client` has been done to
-# create the client static assets.
-
-cwd := $(shell pwd)
-
-.PHONY: build
-build: clean
- mkdir artifact.dir; \
- (cd ../../.. ; \
- git ls-files backend/czi_hosted/ | cpio -pdm $(cwd)/artifact.dir ; ); \
- $(call copy_client_assets,../../../client/build,artifact.dir/backend/czi_hosted) ; \
- set -e ; \
- cp app.py artifact.dir/application.py; \
- cp -r ../../../backend/common artifact.dir/backend/common; \
- cp ../requirements.txt artifact.dir; \
- cp -r .ebextensions artifact.dir; \
- if [ -d customize ] ; then \
- if [ -f customize/config.yaml ] ; then \
- cp customize/config.yaml artifact.dir; \
- fi ; \
- if [ -f customize/Dockerfile ] ; then \
- cp customize/Dockerfile artifact.dir; \
- fi ; \
- if [ -f customize/requirements.txt ] ; then \
- pip install requirements-parser ; \
- pip install packaging ; \
- python3 check_requirements.py ../requirements.txt customize/requirements.txt; \
- cp customize/requirements.txt artifact.dir; \
- fi ; \
- if [ -d customize/deploy ] ; then \
- mkdir -p artifact.dir/backend/czi_hosted/common/web/static/cellxgene; \
- cp -r customize/deploy artifact.dir/backend/czi_hosted/common/web/static/cellxgene; \
- fi; \
- if [ -d customize/inline_scripts ] ; then \
- cp -r customize/inline_scripts/* artifact.dir/backend/czi_hosted/common/web/templates; \
- fi; \
- if [ -d customize/ebextensions ] ; then \
- cp -r customize/ebextensions/* artifact.dir/.ebextensions; \
- fi; \
- fi; \
- if [ -d customize/plugins ] ; then \
- cp -r customize/plugins artifact.dir/backend/czi_hosted/; \
- fi; \
- (cd artifact.dir; \
- cp -r backend/czi_hosted/common/web/static static; \
- zip -r ../artifact.zip . --exclude backend/czi_hosted/test/\* backend/czi_hosted/eb/\* ; ); \
- if ! grep artifact.zip .elasticbeanstalk/config.yml ; then \
- mkdir -p .elasticbeanstalk ; cat config_deploy.yaml >> .elasticbeanstalk/config.yml ; fi
-
diff --git a/backend/czi_hosted/eb/README.md b/backend/czi_hosted/eb/README.md
deleted file mode 100644
index 306eda00..00000000
--- a/backend/czi_hosted/eb/README.md
+++ /dev/null
@@ -1,300 +0,0 @@
-# AWS Elastic Beanstalk
-
-This directory contains scripts to aid in creating and deploying cellxgene on
-AWS Elastic Beanstalk.
-
-This will result in a variant of cellxgene, running on AWS EC2 instances, serving data from S3.
-All datasets must be in the CXG (tiledb) format (see `cellxene convert --help`),
-and located under a single S3 prefix, which is accessible to the instance.
-In the current incarnation, no access control is available
-(outside of anything you configure yourself), so this is most appropriate for public datasets.
-
-This is early development work, and will change significantly in the near future.
-We would love feedback on it, but please assume it will change.
-
-## Prerequisites
-
-1. Some familiarity with AWS EB, S3, and IAM are needed.
-
-2. Install the awsebcli.
- Instruction are here:
- https://docs.aws.amazon.com/elasticbeanstalk/latest/dg/eb-cli3-install.html
-
-3. In the top level directory, run `make build-client` to create the client static assets.
-
-## Steps
-
-These steps are meant to serve as an example.
-There are many more options to these commands that may be important or necessary for your environment.
-
-### 1. Make your matrix files available to the EB servers.
-
-The following choices are known to work.
-
-- S3 Bucket.
-- POSIX filesystem (such as Lustre)
-- Lustre filesystem backed by S3
-
-S3 is convenient and the relatively inexpensive option.
-Lustre is higher performance, but more expensive, and slightly more complex to setup and manage.
-AWS supports a feature to back the Lustre filesystem with S3, which gives an easy to manage, high
-performance option.
-
-Once the storage is in place, the next step is to copy your data files to that location.
-Currently cellxgene supports a flat file organization. Each matrix file is located under
-the same s3 prefix or filesystem directory. This location is specified in the configuration
-as the dataroot.
-
-### 2. Create an elastic beanstalk application. For example:
-
-```
-EB_APP=cellxgene-app
-eb init -p python-3.6 $EB_APP
-```
-
-### 3. Configuring cellxgene
-
-All the cellxgene configuration options can be set from a configuration file.
-A yaml config file containing all of the default configuration options can be generated like this:
-
-`cellxgene launch --dump-default-config > myconfig.yaml`
-
-The config file may then be customized before the app is deployed.
-
-There are two ways to set the config file location, evaluated in this order:
-
-First, if your config file is named "config.yaml" and exists in `customize/config.yaml`,
-then it will be bundled with the application zip file and installed along
-side the app on the EB servers.
-
-Second, a potentially more flexible approach is to place your config file in a location accessible
-to the EB servers, such as in S3. For example: s3://my-bucket/my-datasets/config.yaml.
-Set the CXG_CONFIG_FILE environment variable to specify this location.
-
-Another option is to set the CXG_DATAROOT environment variable. The dataroot
-is the location where the matrix files are located.
-This environment variable will override the dataroot in the config file (if specified).
-
-### 4. Customization
-
-The deployment can be customized in several ways, by adding files to a directory called
-`customize` which is placed in this directory.
-
-#### config file
-
-This was described in the previous section.
-
-#### static files
-
-The cellxgene server can serve additional static webpages that will be associated with the app.
-These include the about_legal_tos (terms of service), and about_legal_privacy, for example.
-To use this feature, do the following:
-
-- In this directory, create a sub directory called "customize/deploy/".
-- Copy the files you want to serve into this directory
-- modify your configuration file to set the location to these file: /static/cellxgene/deploy/
-
-Example: you want to include an "about_legal_tos" and "about_legal_privacy" page to cellxgene.
-Assume files called "tos.html" and "privacy.html" exist.
-
-```
-$ mkdir -p customize/deploy
-$ cp /tos.html customize/deploy/tos.html
-$ cp /privacy.html customize/deploy/privacy.html
-
-# edit config.yaml
-$ grep "/static/cellxgene/deploy" config.yaml
-about_legal_tos: /static/cellxgene/deploy/tos.html
-about_legal_privacy: /static/cellxgene/deploy/privacy.html
-```
-
-#### Inline javascript scripts
-
-Additional scripts can be added using the server/inline_scripts config parameters.
-To include these scripts in the deployment, use the following steps:
-
-- In this directory, create a sub directory called "customize/inline_scripts".
-- Copy the script files into this directory
-- Modify your configuration file to set the location to these file (leaving off customize/inline_scripts)
-
-For example, to add an inline script called "myscript.js":
-
-```
-$ mkdir -p customize/inline_scripts
-$ cp /myscript.js customize/inline_scripts/myscript.js
-# edit the config.yaml
-$ grep inline_scripts config.yaml
- inline_scripts : [ myscript.js ]
-```
-
-#### Plugins
-
-Optionally, you can add plugins to the server python code. To include a plugin in the deployment use the following steps:
-
-```
-$ mkdir -p customize/plugins
-$ cp /.py customize/plugins/.py
-```
-
-#### ebextensions
-
-Any additional config files intended for the `.ebextensions` directory of the artifact can be added
-to the `customize/ebextensions` directory. Any file found here will be copied over.
-
-#### requirements.txt
-
-A custom requirements.txt can be supplied in customize/requirements.txt.
-This file must fully specify the versions of all the python modules used by the server in the deployment.
-This is useful to ensure that the dependencies do not change from one deployment to the next.
-Therefore the custom/requirements.txt must all have exact versions specified (e.g. anndata==0.7.1).
-
-This file can be generated the first time using a process like this:
-
-```
-# assume you are running in this directory
-$ virtualenv temp
-$ source temp/bin/activate
-$ pip install -r ../requirements.txt
-$ mkdir -p customize
-$ pip freeze > customize/requirements.txt
-$ deactivate
-$ rm -rf temp/
-```
-
-Keep the customize/requirememts.txt file, and reuse it for each deployment.
-If a future cellxgene version updates its requirements by modifying a module version
-or adding a new dependency, then the `make build` process will detect any
-incompatibilities and raise an error.
-
-#### File structure for customizations
-
-The following diagram shows the file structure for the customization directory.
-
-```
-customization
-+-- config.yaml
-+-- deploy/
-+-- inline_scripts/
-+-- plugins/
-+-- ebextensions/
-+-- requirements.txt
-```
-
-### 5. Create the artifact.zip file for the application
-
-```
-$ make build
-```
-
-### 6. Flask secret key
-
-The application requires a secret key to be provided to flask, the web framework used by cellxgene.
-There are three ways to provide the secret key:
-
-- In the configuration file, update the server/flask_secret_key attribute.
-- In the configuration file, update the external/aws_secrets_manager section to set the
- secret name and key that defines the flask secret key.
-- An environment variable: `CXG_SECRET_KEY`
-
-### 7. Create an environment
-
-```
-# name of the environment
-$ EB_ENV=cellxgene-env
-
-# type of ec2 instance to run the cellxgene server (for example)
-$ EB_INSTANCE=m5.large
-
-# One or both of the following environment variables needs to be set
-$ CXG_DATAROOT=
-$ CXG_CONFIG_FILE=
-
-# Potentially also set an environment variable for the flask secret key,
-# and other environemet variable described in the configuration file.
-
-$ eb create $EB_ENV --instance-type $EB_INSTANCE \
- --envvars CXG_DATAROOT=$CXG_DATAROOT,CXG_CONFIG_FILE=$CXG_CONFIG_FILE
-```
-
-### 8. Give the elastic beanstalk environment access to the dataroot.
-
-If using S3, this link may provide some useful information:
-https://aws.amazon.com/premiumsupport/knowledge-center/elastic-beanstalk-s3-bucket-instance/
-If using Lustre, then this link may provide a place to start:
-https://aws.amazon.com/fsx/lustre/
-
-### 9. Deploy the application
-
-```
-$ eb deploy $EB_ENV
-```
-
-### 10. Open the application in a browser
-
-```
-$ eb open $EB_ENV
-```
-
-## Advanced Features
-
-### Authentication
-
-Authentication can be configured in the configuration file. Authentication is required
-for User Annotations (see below). User Annotations is a feature where annotations can be
-created by the user
-, and
-then associated with the user's id.
-When the user revisits the site, their annotations will be available.
-
-There are three main authentication modes: null, session, or oauth.
-In the configuration file specify the authentication mode by setting
-`server / authentication / type`.
-
-#### null
-
-Authentication is disabled: user annotations cannot be enabled.
-
-#### session
-
-The user is associated with their client browser session. This approach is
-simple to setup, but not recommended for hosted cellxgene, since the user will not have access to
-their annotations when running from a different browser, or if their cookies get cleared.
-
-#### oauth
-
-A user logs into cellxgene using an identity provider (like Google), or logs in using
-an email/password. This is the best option, but requires making use of an oauth service and
-additional configuration of the cellxgene server.
-
-To see what this looks like, please look at https://cellxgene.cziscience.com/,
-and view one of the cellxgene datasets.
-For this server, Auth0 (auth0.com) is used for authentication, but there are other options.
-There are good sources of documentation online that describe how to use one of these
-services.
-
-The `params_oauth` section in the configuration file describes characteristics of the
-authentication service, like "client_id" and "client_secret".
-For security, the client_secret needs to be protected. One option is to
-store it in the AWS Secrets Manager.
-
-### User Annotations
-
-User annotations can be configured in the configuration file both generally and for a specific data route. The annotations feature is only available when Authorization is enabled.
-To enable Annotations, it is necessary to create a relational database and add the database uri (typically `postgresql://[user[:password]@][netloc][:port][/dbname]`) to the secrets manager under `DB_URI`.
-The hosted version of cellxgene runs on AWS's [Aurora PostgreSQL](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/Aurora.AuroraPostgreSQL.html) but any sqlalchemy compatible relational database should work.
-Once the database is set up apply the cellxgene schema to your database by running the following inside the cellxgene repo
-`PROJECT_ROOT=$(git rev-parse --show-toplevel)`
-`python3`
-Inside the python console
-`from sqlalchemy import create_engine`
-`from server.db.cellxgene_orm import Base`
-`uri = "[DB_URI]”`
-`engine = create_engine(uri)`
-
- Base.metadata.create_all(engine)`
-
-To check the schema was properly applied (or just to check what is in the database at any point)
-ssh into your database. For a postgres database this entails running:
-`psql [DB_URI]`
-
-You'll also need to update your IAM policies to allow the instance to write to the s3 bucket.
diff --git a/backend/czi_hosted/eb/__init__.py b/backend/czi_hosted/eb/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/backend/czi_hosted/eb/app.py b/backend/czi_hosted/eb/app.py
deleted file mode 100644
index 3a54f18b..00000000
--- a/backend/czi_hosted/eb/app.py
+++ /dev/null
@@ -1,190 +0,0 @@
-"""cellxgene AWS elastic beanstalk application"""
-
-import sys
-import os
-import hashlib
-import base64
-from urllib.parse import urlparse
-from flask import json
-import logging
-from flask_talisman import Talisman
-from flask_cors import CORS
-
-
-if os.path.isdir("/opt/python/log"):
- # This is the standard location where Amazon EC2 instances store the application logs.
- logging.basicConfig(
- filename="/opt/python/log/app.log",
- level=logging.INFO,
- format="%(asctime)s.%(msecs)03d %(levelname)s %(module)s - %(funcName)s: %(message)s",
- datefmt="%Y-%m-%d %H:%M:%S",
- )
-
-SERVERDIR = os.path.dirname(os.path.realpath(__file__))
-sys.path.append(SERVERDIR)
-
-try:
- from backend.czi_hosted.common.config.app_config import AppConfig
- from backend.czi_hosted.app.app import Server
- from backend.common.utils.data_locator import DataLocator, discover_s3_region_name
-except Exception:
- logging.critical("Exception importing server modules", exc_info=True)
- sys.exit(1)
-
-
-class WSGIServer(Server):
- def __init__(self, app_config):
- super().__init__(app_config)
-
- @staticmethod
- def _before_adding_routes(app, app_config):
- script_hashes = WSGIServer.get_csp_hashes(app, app_config)
- server_config = app_config.server_config
-
- # add the api_base_url to the connect_src csp header.
- extra_connect_src = []
- api_base_url = server_config.get_api_base_url()
- if api_base_url:
- parse_api_base_url = urlparse(api_base_url)
- extra_connect_src = [f"{parse_api_base_url.scheme}://{parse_api_base_url.netloc}"]
-
- # This hash should be in sync with the script within
- # `client/configuration/webpack/obsoleteHTMLTemplate.html`
-
- # It is _very_ difficult to generate the correct hash manually,
- # consider forcing CSP to fail on the local server by intercepting the response via Requestly
- # this should print the failing script's hash to console.
- # See more here: https://github.com/chanzuckerberg/cellxgene/pull/1745
- obsolete_browser_script_hash = ["'sha256-/rmgOi/skq9MpiZxPv6lPb1PNSN+Uf4NaUHO/IjyfwM='"]
- csp = {
- "default-src": ["'self'"],
- "connect-src": ["'self'"] + extra_connect_src,
- "script-src": ["'self'", "'unsafe-eval'"] + obsolete_browser_script_hash + script_hashes,
- "style-src": ["'self'", "'unsafe-inline'"],
- "img-src": ["'self'", "https://cellxgene.cziscience.com", "data:"],
- "object-src": ["'none'"],
- "base-uri": ["'none'"],
- "frame-ancestors": ["'none'"],
- }
-
- if not app.debug:
- csp["upgrade-insecure-requests"] = ""
-
- if server_config.app__csp_directives:
- for k, v in server_config.app__csp_directives.items():
- if not isinstance(v, list):
- v = [v]
- csp[k] = csp.get(k, []) + v
-
- # Add the web_base_url to the CORS header
- web_base_url = server_config.get_web_base_url()
- if web_base_url:
- web_base_url_parse = urlparse(web_base_url)
- allowed_origin = f"{web_base_url_parse.scheme}://{web_base_url_parse.netloc}"
- CORS(app, supports_credentials=True, origins=allowed_origin)
-
- Talisman(
- app, force_https=server_config.app__force_https, frame_options="DENY", content_security_policy=csp,
- )
-
- @staticmethod
- def load_static_csp_hashes(app):
- csp_hashes = None
- try:
- with app.open_resource("../common/web/csp-hashes.json") as f:
- csp_hashes = json.load(f)
- except FileNotFoundError:
- pass
- if not isinstance(csp_hashes, dict):
- csp_hashes = {}
- script_hashes = [f"'{hash}'" for hash in csp_hashes.get("script-hashes", [])]
- if len(script_hashes) == 0:
- logging.error("Content security policy hashes are missing, falling back to unsafe-inline policy")
-
- return script_hashes
-
- @staticmethod
- def compute_inline_csp_hashes(app, app_config):
- dataset_configs = [app_config.default_dataset_config] + list(app_config.dataroot_config.values())
- hashes = []
- for dataset_config in dataset_configs:
- inline_scripts = dataset_config.app__inline_scripts
- for script in inline_scripts:
- with app.open_resource(f"../common/web/templates/{script}") as f:
- content = f.read()
- # we use jinja2 template include, which trims final newline if present.
- if content[-1] == 0x0A:
- content = content[0:-1]
- hash = base64.b64encode(hashlib.sha256(content).digest())
- hashes.append(f"'sha256-{hash.decode('utf-8')}'")
- return hashes
-
- @staticmethod
- def get_csp_hashes(app, app_config):
- script_hashes = WSGIServer.load_static_csp_hashes(app)
- script_hashes += WSGIServer.compute_inline_csp_hashes(app, app_config)
- return script_hashes
-
-
-try:
- app_config = AppConfig()
-
- has_config = False
- # config file: look first for "config.yaml" in the current working directory
- config_file = "config.yaml"
- config_location = DataLocator(config_file)
- if config_location.exists():
- with config_location.local_handle() as lh:
- logging.info(f"Configuration from {config_file}")
- app_config.update_from_config_file(lh)
- has_config = True
-
- else:
- # config file: second, use the CXG_CONFIG_FILE
- config_file = os.getenv("CXG_CONFIG_FILE")
- if config_file:
- region_name = discover_s3_region_name(config_file)
- config_location = DataLocator(config_file, region_name)
- if config_location.exists():
- with config_location.local_handle() as lh:
- logging.info(f"Configuration from {config_file}")
- app_config.update_from_config_file(lh)
- has_config = True
- else:
- logging.critical(f"Configuration file not found {config_file}")
- sys.exit(1)
-
- if not has_config:
- logging.critical("No config file found")
- sys.exit(1)
-
- dataroot = os.getenv("CXG_DATAROOT")
- if dataroot:
- logging.info("Configuration from CXG_DATAROOT")
- app_config.update_server_config(multi_dataset__dataroot=dataroot)
-
- # overwrite configuration for the eb app
- app_config.update_server_config(multi_dataset__allowed_matrix_types=["cxg"],)
-
- # complete config
- app_config.complete_config(logging.info)
-
- server = WSGIServer(app_config)
- debug = False
- application = server.app
-
-except Exception:
- logging.critical("Caught exception during initialization", exc_info=True)
- sys.exit(1)
-
-if app_config.is_multi_dataset():
- logging.info(f"starting server with multi_dataset__dataroot={app_config.server_config.multi_dataset__dataroot}")
-else:
- logging.info(f"starting server with single_dataset__datapath={app_config.server_config.single_dataset__datapath}")
-
-if __name__ == "__main__":
- try:
- application.run(host=app_config.server_config.app__host, debug=debug, threaded=not debug, use_debugger=False)
- except Exception:
- logging.critical("Caught exception during initialization", exc_info=True)
- sys.exit(1)
diff --git a/backend/czi_hosted/eb/check_config.py b/backend/czi_hosted/eb/check_config.py
deleted file mode 100644
index 223a21b3..00000000
--- a/backend/czi_hosted/eb/check_config.py
+++ /dev/null
@@ -1,38 +0,0 @@
-import sys
-import argparse
-import yaml
-
-from backend.czi_hosted.common.config.app_config import AppConfig
-
-def main():
- parser = argparse.ArgumentParser("A script to check hosted configuration files")
- parser.add_argument("config_file", help="the configuration file")
- parser.add_argument(
- "-s",
- "--show",
- default=False,
- action="store_true",
- help="print the configuration. NOTE: this may print secret values to stdout",
- )
-
- args = parser.parse_args()
-
- app_config = AppConfig()
- try:
- app_config.update_from_config_file(args.config_file)
- app_config.complete_config()
- except Exception as e:
- print(f"Error: {str(e)}")
- print("FAIL:", args.config_file)
- sys.exit(1)
-
- if args.show:
- yaml_config = app_config.config_to_dict()
- yaml.dump(yaml_config, sys.stdout)
-
- print("PASS:", args.config_file)
- sys.exit(0)
-
-
-if __name__ == "__main__":
- main()
diff --git a/backend/czi_hosted/eb/check_requirements.py b/backend/czi_hosted/eb/check_requirements.py
deleted file mode 100644
index 008afede..00000000
--- a/backend/czi_hosted/eb/check_requirements.py
+++ /dev/null
@@ -1,98 +0,0 @@
-"""This is a simple script to ensure the custom requirements.txt do not violate
-the server requirements.txt. A hosted cellxgene deployment may specify the exact
-version requirements on all the modules, and may add additional modules.
-This script is meant to aid in making that list of custom requirements easier to maintain.
-If cellxgene adds a new dependency, or changes the version requirements of an existing
-dependency, then this script can check if the custom requirements are still valid"""
-
-import sys
-import requirements
-from packaging.version import Version
-import pkg_resources
-
-
-def check(expected, custom):
- """checks that the custom requirements meet all the requirements of the expected requirements.
- The custom set of requirements may contain additional entries than expected.
- The requirements in custom must all be exact (==).
- An expected requirement must be present in custom, and must match all the specs
- for that requirement.
-
- expected : name of the expected requirement.txt file
- custom : name of the custom requirements.txt file
- """
- edict = parse_requirements(expected)
- cdict = parse_requirements(custom)
-
- okay = True
-
- # cdict must only have exact requirements (==)
- for cname, cspecs in cdict.items():
- if len(cspecs) != 1 or cspecs[0][0] != "==":
- print(f"Error, spec must be an exact requirement {custom}: {cname} {str(cspecs)}")
- okay = False
-
- for ename, especs in edict.items():
- if ename not in cdict:
- print(f"Error, missing requirement from {custom}: {ename} {str(especs)}")
- okay = False
- continue
-
- cver = Version(cdict[ename][0][1])
- for espec in especs:
- rokay = check_version(cver, espec[0], Version(espec[1]))
- if not rokay:
- print(f"Error, failed requirement from {custom}: {ename} {espec}, {cver}")
- okay = False
-
- if okay:
- print("requirements check successful")
- sys.exit(0)
- else:
- sys.exit(1)
-
-
-def parse_requirements(fname):
- """Read a requirements file and return a dict of modules name / specification"""
- try:
- with open(fname, "r") as fd:
- try:
- # pylint: disable=no-member
- rdict = {req.name: req.specs for req in requirements.parse(fd)}
- except pkg_resources.RequirementParseError:
- print(f"Unable to parse the requirements file: {fname}")
- sys.exit(1)
- except Exception as e:
- print(f"Unable to open file {fname}: {str(e)}")
- sys.exit(1)
-
- return rdict
-
-
-# pylint: disable=too-many-return-statements
-def check_version(cver, optype, ever):
- """
- Simple version check.
- Note: There is more complexity to comparing version (PEP440).
- However the use cases in cellxgene are limited, and do not require a general solution.
- """
-
- if optype == "==":
- return cver == ever
- if optype == "!=":
- return cver != ever
- if optype == ">=":
- return cver >= ever
- if optype == ">":
- return cver > ever
- if optype == "<=":
- return cver <= ever
- if optype == "<":
- return cver < ever
-
- print(f"Error, optype not handled: {optype}")
- return False
-
-
-if __name__ == "__main__":
- check(sys.argv[1], sys.argv[2])
diff --git a/backend/czi_hosted/eb/config_deploy.yaml b/backend/czi_hosted/eb/config_deploy.yaml
deleted file mode 100644
index 8c6e4cb0..00000000
--- a/backend/czi_hosted/eb/config_deploy.yaml
+++ /dev/null
@@ -1,2 +0,0 @@
-deploy:
- artifact: artifact.zip
diff --git a/backend/czi_hosted/requirements-dev.txt b/backend/czi_hosted/requirements-dev.txt
deleted file mode 100644
index 12f50599..00000000
--- a/backend/czi_hosted/requirements-dev.txt
+++ /dev/null
@@ -1,12 +0,0 @@
-Authlib>=0.14.3
-black
-bumpversion>=0.5
-codecov>=2.0.15
-parameterized>=0.7.0
-psycopg2-binary>=2.8.5
-pytest>=3.6.3
-python-jose>=3.2.0
-twine>=1.12.1
--r requirements.txt
--r requirements-prepare.txt
-rsa>=4.7 # not directly required, pinned by Snyk to avoid a vulnerability
diff --git a/backend/czi_hosted/requirements-prepare.txt b/backend/czi_hosted/requirements-prepare.txt
deleted file mode 100644
index ae6d4ca6..00000000
--- a/backend/czi_hosted/requirements-prepare.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-python-igraph
-louvain>=0.6
-scanpy
-umap-learn<0.5.0 # The pinned version scanpy is not compatible with latest umap-learn
diff --git a/backend/czi_hosted/requirements.txt b/backend/czi_hosted/requirements.txt
deleted file mode 100644
index 504ad432..00000000
--- a/backend/czi_hosted/requirements.txt
+++ /dev/null
@@ -1,24 +0,0 @@
-anndata>=0.7.6 # we need to_memory(), added in 0.7.6
-boto3>=1.12.18
-click>=7.1.2
-Flask>=1.0.2,<2.0.0 # Flask 2.0 is not compatible with the latest version of Flask-RESTful (0.3.8)
-Flask-Compress>=1.4.0
-Flask-Cors>=3.0.9 # CVE-2020-25032
-Flask-RESTful>=0.3.6
-flask-server-timing>=0.1.2
-flask-talisman>=0.7.0
-flatbuffers>=1.11.0,<2.0.0 # cellxgene is not compatible with 2.0.0. Requires migration
-flatten-dict>=0.2.0
-fsspec>=0.4.4,<0.8.0
-gunicorn>=20.0.4
-h5py>=3.0.0
-numba>=0.51.2
-numpy>=1.17.5
-packaging>=20.0
-pandas>=1.0,!=1.1 # pandas 1.1 breaks tests, https://github.com/pandas-dev/pandas/issues/35446
-PyYAML>=5.4 # CVE-2020-14343
-scipy>=1.4
-requests>=2.22.0
-tiledb>=0.5.9,>=0.6.2,!=0.7.2, !=0.8.6
-s3fs==0.4.2
-sqlalchemy>=1.3.18
diff --git a/backend/server/Makefile b/backend/server/Makefile
index 74c0dd77..bd34ec30 100644
--- a/backend/server/Makefile
+++ b/backend/server/Makefile
@@ -20,7 +20,3 @@ unit-test:
.PHONY: test-annotations-performance
test-annotations-performance:
python ../test/test_server/performance/performance_test_annotations_backend.py
-
-.PHONY: test-annotations-scale
-test-annotations-scale:
- locust -f ../test/test_server/performance/scale_test_annotations.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt
diff --git a/backend/test/fixtures/czi_hosted_dataset_config_outline.py b/backend/test/fixtures/czi_hosted_dataset_config_outline.py
deleted file mode 100644
index 81ca0df7..00000000
--- a/backend/test/fixtures/czi_hosted_dataset_config_outline.py
+++ /dev/null
@@ -1,35 +0,0 @@
-f"""
-dataset:
- app:
- scripts: {scripts} #list of strs (filenames) or dicts containing keys
- inline_scripts: {inline_scripts} #list of strs (filenames)
-
- about_legal_tos: {about_legal_tos}
- about_legal_privacy: {about_legal_privacy}
-
- authentication_enable: {authentication_enable}
-
- presentation:
- max_categories: {max_categories}
- custom_colors: {custom_colors}
-
- user_annotations:
- enable: {enable_users_annotations}
- type: {annotation_type}
- hosted_tiledb_array:
- db_uri: {db_uri}
- hosted_file_directory: {hosted_file_directory}
- local_file_csv:
- directory: {local_file_csv_directory}
- file: {local_file_csv_file}
-
- embeddings:
- names: {embedding_names}
-
- diffexp:
- enable: {enable_difexp}
- lfc_cutoff: {lfc_cutoff}
- top_n: {top_n}
-
- X_approximate_distribution: {X_approximate_distribution}
-"""
diff --git a/backend/test/fixtures/czi_hosted_server_config_outline.py b/backend/test/fixtures/czi_hosted_server_config_outline.py
deleted file mode 100644
index a23f7d24..00000000
--- a/backend/test/fixtures/czi_hosted_server_config_outline.py
+++ /dev/null
@@ -1,63 +0,0 @@
-f"""server:
- app:
- verbose: {verbose}
- debug: {debug}
- host: {host}
- port: {port}
- open_browser: {open_browser}
- force_https: {force_https}
- flask_secret_key: {flask_secret_key}
- generate_cache_control_headers: {generate_cache_control_headers}
- server_timing_headers: {server_timing_headers}
- csp_directives: {csp_directives}
- api_base_url: {api_base_url}
- web_base_url: {web_base_url}
- authentication:
- type: {auth_type}
- insecure_test_environment: {insecure_test_environment}
- params_oauth:
- oauth_api_base_url: {oauth_api_base_url}
- client_id: {client_id}
- client_secret: {client_secret}
- jwt_decode_options: {jwt_decode_options}
- session_cookie: {session_cookie}
- cookie: {cookie}
-
- multi_dataset:
- dataroot: {dataroot}
- index: {index}
- allowed_matrix_types: {allowed_matrix_types}
- matrix_cache:
- max_datasets: {max_cached_datasets}
- timelimit_s: {timelimit_s}
-
- single_dataset:
- datapath: {dataset_datapath}
- obs_names: {obs_names}
- var_names: {var_names}
- about: {about}
- title: {title}
-
- diffexp:
- alg_cxg: # number of threads to use is computed from: min(max_workers, cpu_multipler * cpu_count)
- max_workers: {diffexp_max_workers}
- cpu_multiplier: {cpu_multiplier}
- target_workunit: {target_workunit} # The target number of matrix elements that are evaluated in one thread.
-
- data_locator:
- s3:
- region_name: {data_locater_region_name}
-
- adaptor:
- cxg_adaptor:
- tiledb_ctx:
- sm.tile_cache_size: {cxg_tile_cache_size}
- sm.num_reader_threads: {cxg_num_reader_threads}
-
- anndata_adaptor:
- backed: {anndata_backed}
-
- limits:
- column_request_max: {column_request_max}
- diffexp_cellcount_max: {diffexp_cellcount_max}
-"""
diff --git a/backend/test/fixtures/database/__init__.py b/backend/test/fixtures/database/__init__.py
deleted file mode 100644
index bf97d6a3..00000000
--- a/backend/test/fixtures/database/__init__.py
+++ /dev/null
@@ -1,79 +0,0 @@
-import string
-import random
-
-
-from sqlalchemy import func
-
-from backend.czi_hosted.db.cellxgene_orm import CellxGeneUser, CellxGeneDataset, Annotation, Base
-from backend.czi_hosted.db.create_db import create_db
-from backend.czi_hosted.db.db_utils import DbUtils
-
-
-class TestDatabase:
- def __init__(self):
- local_db_uri = "postgresql://postgres:test_pw@localhost:5432"
- create_db(local_db_uri)
- self.db = DbUtils(local_db_uri)
- self._populate_test_data()
- self._populate_test_data_many()
-
- def _populate_test_data(self):
- self._create_test_user()
- self._create_test_dataset()
- self._create_test_annotation()
-
- def _populate_test_data_many(self):
- self._create_test_users()
- self._create_test_datasets()
- self._create_test_annotations()
-
- def _create_test_user(self):
- user = CellxGeneUser(id="test_user_id")
- user2 = CellxGeneUser(id="1234")
- self.db.session.add(user)
- self.db.session.add(user2)
- self.db.session.commit()
-
- def _create_test_dataset(self):
- dataset = CellxGeneDataset(name="test_dataset",)
- self.db.session.add(dataset)
- self.db.session.commit()
-
- def _create_test_annotation(self):
- dataset = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == "test_dataset"],)[0]
- annotation = Annotation(tiledb_uri="tiledb_uri", user_id="test_user_id", dataset_id=str(dataset.id))
- self.db.session.add(annotation)
- self.db.session.commit()
-
- @staticmethod
- def get_random_string():
- letters = string.ascii_lowercase
- return "".join(random.choice(letters) for i in range(12))
-
- def _create_test_users(self, user_count: int = 10):
- users = []
- for i in range(user_count):
- users.append(CellxGeneUser(id=self.get_random_string()))
- self.db.session.add_all(users)
- self.db.session.commit()
-
- def _create_test_datasets(self, dataset_count: int = 10):
- datasets = []
- for i in range(dataset_count):
- datasets.append(CellxGeneDataset(name=self.get_random_string()))
- self.db.session.add_all(datasets)
- self.db.session.commit()
-
- def order_by_random(self, table: Base):
- return self.db.session.query(table).order_by(func.random()).first()
-
- def _create_test_annotations(self, annotation_count: int = 10):
- annotations = []
- for i in range(annotation_count):
- dataset = self.order_by_random(CellxGeneDataset)
- user = self.order_by_random(CellxGeneUser)
- annotations.append(
- Annotation(tiledb_uri=self.get_random_string(), user_id=user.id, dataset_id=str(dataset.id))
- )
- self.db.session.add_all(annotations)
- self.db.session.commit()
diff --git a/backend/test/test_czi_hosted/__init__.py b/backend/test/test_czi_hosted/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/backend/test/test_czi_hosted/locust/README.md b/backend/test/test_czi_hosted/locust/README.md
deleted file mode 100644
index b677af6a..00000000
--- a/backend/test/test_czi_hosted/locust/README.md
+++ /dev/null
@@ -1,39 +0,0 @@
-# Locust Load Test
-
-This directory contains scripts to load test cellxgene's backend. It
-primary simulates initial data loading and expression data fetch, which
-are the most common data routes. It currently does not include tests
-for differential expression or re-clustering routes.
-
-## Prerequisites
-
-You need:
-
-- Python 3.6+, and pip
-- cellxgene installed
-- install the locust dependencies in `requirements-locust.txt`
-
-## To test
-
-1. Choose to run cellxgene in either single dataset or data root mode.
-2. Edit config.py to indicate which datasets to load:
- - in single dataset mode, just set `DataSets=[""]`
- - in dataroot (multi-dataset) mode, add the route names, eg, `DataSets=['foo.cxg', 'bar.cxg']`
-3. Launch cellxgene in the appropriate mode
-4. launch locust, specifying the correct --host argument
-5. point your web browser to the locust http server, usually `http://localhost:8089/`
-
-### Single dataset mode
-
-- Edit config.py and set `DataSets=[""]`
-- in a shell, run `cellxgene launch somefile.h5ad`
-- launch locust in another shell, `locust --host http://localhost:5005/` (or wherever you are running cellxgene)
-- point a browser to the locust port, usually http://localhost:8089/
-- run test
-
-### Multi-dataset mode
-
-- Edit config.py and set `DataSets=["datapath1", ...]`
-- in a shell, run `cellxgene launch --dataroot path`
-
-The remainder of the steps are same as single dataset.
diff --git a/backend/test/test_czi_hosted/locust/__init__.py b/backend/test/test_czi_hosted/locust/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/backend/test/test_czi_hosted/locust/config.py b/backend/test/test_czi_hosted/locust/config.py
deleted file mode 100644
index 7b256ee2..00000000
--- a/backend/test/test_czi_hosted/locust/config.py
+++ /dev/null
@@ -1,15 +0,0 @@
-"""
-Locust test config
-"""
-
-
-""" Data routes that will be tested """
-
-# single dataset, for non-dataroot tests
-# DataSets = [""]
-
-# multi-dataset, for dataroot tests. these are varied in size/shape
-DataSets = [
- "GSE60361.cxg",
- "WongAdultRetina.cxg",
-]
diff --git a/backend/test/test_czi_hosted/locust/locustfile.py b/backend/test/test_czi_hosted/locust/locustfile.py
deleted file mode 100644
index 9e38cd7c..00000000
--- a/backend/test/test_czi_hosted/locust/locustfile.py
+++ /dev/null
@@ -1,165 +0,0 @@
-import json
-import random
-
-import requests
-from config import DataSets
-from locust import HttpUser, SequentialTaskSet, task, between, TaskSet
-from locust.clients import HttpSession
-from requests.packages.urllib3.exceptions import InsecureRequestWarning
-
-import backend.test.decode_fbs as decode_fbs
-
-requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
-
-"""
-Simple locust stress test definition for cellxgene
-"""
-
-API_SUFFIX = "api/v0.2"
-
-
-class CellXGeneTasks(TaskSet):
- """
- Simulate use against a single dataset
- """
-
- def on_start(self):
-
- self.client.verify = False
- self.dataset = random.choice(DataSets)
-
- with self.client.get(
- f"{self.dataset}/{API_SUFFIX}/schema", stream=True, catch_response=True
- ) as schema_response:
- if schema_response.status_code == 200:
- self.schema = schema_response.json()["schema"]
- else:
- self.schema = None
-
- with self.client.get(
- f"{self.dataset}/{API_SUFFIX}/config", stream=True, catch_response=True
- ) as config_response:
- if config_response.status_code == 200:
- self.config = config_response.json()["config"]
- else:
- self.config = None
-
- with self.client.get(
- f"{self.dataset}/{API_SUFFIX}/annotations/var?annotation-name={self.var_index_name()}",
- headers={"Accept": "application/octet-stream"},
- catch_response=True,
- ) as var_index_response:
- if var_index_response.status_code == 200:
- df = decode_fbs.decode_matrix_FBS(var_index_response.content)
- gene_names_idx = df["col_idx"].index(self.var_index_name())
- self.gene_names = df["columns"][gene_names_idx]
- else:
- self.gene_names = []
-
- def var_index_name(self):
- if self.schema is not None:
- return self.schema["annotations"]["var"]["index"]
- return None
-
- def obs_annotation_names(self):
- if self.schema is not None:
- return [col["name"] for col in self.schema["annotations"]["obs"]["columns"]]
- return []
-
- def layout_names(self):
- if self.schema is not None:
- return [layout["name"] for layout in self.schema["layout"]["obs"]]
- else:
- return []
-
- @task(2)
- class InitializeClient(SequentialTaskSet):
- """
- Initial loading of cellxgene - when the user hits the main route.
-
- Currently this sequence skips some of the static assets, which are quite small and should be served by the
- HTTP server directly.
-
- 1. Load index.html, etc.
- 2. Concurrently load /config, /schema
- 3. Concurrently load /layout/obs, /annotations/var?annotation-name=
- -- Does initial render --
- 4. Concurrently load all /annotations/obs and all /layouts/obs
- -- Fully initialized --
- """
-
- # Users hit all of the init routes as fast as they can, subject to the ordering constraints and network latency.
- wait_time = between(0.01, 0.1)
-
- def on_start(self):
- self.dataset = self.parent.dataset
- self.client.verify = False
- self.api_less_client = HttpSession(
- base_url=self.client.base_url.replace("api.", "").replace("cellxgene/", ""),
- request_success=self.client.request_success,
- request_failure=self.client.request_failure,
- )
-
- @task
- def index(self):
- self.api_less_client.get(f"{self.dataset}", stream=True)
-
- @task
- def loadConfigAndSchema(self):
- self.client.get(f"{self.dataset}/{API_SUFFIX}/schema", stream=True, catch_response=True)
- self.client.get(f"{self.dataset}/{API_SUFFIX}/config", stream=True, catch_response=True)
-
- @task
- def loadBootstrapData(self):
- self.client.get(
- f"{self.dataset}/{API_SUFFIX}/layout/obs", headers={"Accept": "application/octet-stream"}, stream=True
- )
- self.client.get(
- f"{self.dataset}/{API_SUFFIX}/annotations/var?annotation-name={self.parent.var_index_name()}",
- headers={"Accept": "application/octet-stream"},
- catch_response=True,
- )
-
- @task
- def loadObsAnnotationsAndLayouts(self):
- obs_names = self.parent.obs_annotation_names()
- for name in obs_names:
- self.client.get(
- f"{self.dataset}/{API_SUFFIX}/annotations/obs?annotation-name={name}",
- headers={"Accept": "application/octet-stream"},
- stream=True,
- )
-
- layouts = self.parent.layout_names()
- for name in layouts:
- self.client.get(
- f"{self.dataset}/{API_SUFFIX}/annotations/obs?layout-name={name}",
- headers={"Accept": "application/octet-stream"},
- stream=True,
- )
-
- @task
- def done(self):
- self.interrupt()
-
- @task(1)
- def load_expression(self):
- """
- Simulate user occasionally loading some expression data for a gene
- """
-
- gene_name = random.choice(self.gene_names)
- filter = {"filter": {"var": {"annotation_value": [{"name": self.var_index_name(), "values": [gene_name]}]}}}
- self.client.put(
- f"{self.dataset}/{API_SUFFIX}/data/var",
- data=json.dumps(filter),
- headers={"Content-Type": "application/json", "Accept": "application/octet-stream"},
- stream=True,
- ).close()
-
-
-class CellxgeneUser(HttpUser):
- tasks = [CellXGeneTasks]
-
- # Most ops do not require back-end interaction, so slow cadence for users
- wait_time = between(10, 60)
diff --git a/backend/test/test_czi_hosted/locust/requirements-locust.txt b/backend/test/test_czi_hosted/locust/requirements-locust.txt
deleted file mode 100644
index c52c2541..00000000
--- a/backend/test/test_czi_hosted/locust/requirements-locust.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-locust
--r ../../../czi_hosted/requirements.txt
diff --git a/backend/test/test_czi_hosted/performance/__init__.py b/backend/test/test_czi_hosted/performance/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/backend/test/test_czi_hosted/performance/create_test_matrix.py b/backend/test/test_czi_hosted/performance/create_test_matrix.py
deleted file mode 100644
index 476b6387..00000000
--- a/backend/test/test_czi_hosted/performance/create_test_matrix.py
+++ /dev/null
@@ -1,44 +0,0 @@
-import anndata
-import argparse
-import random
-import scipy
-import numpy as np
-
-
-def main():
- parser = argparse.ArgumentParser("A command to generate test h5ad files")
- parser.add_argument("output", help="Name of the output file")
- parser.add_argument("nobs", type=int, help="Number of observations (rows)")
- parser.add_argument("nvar", type=int, help="Number of variables (columns)")
- parser.add_argument("-n", "--nnz-percent", type=float, default=100, help="percent of non-zeros")
- parser.add_argument("-c", "--col-shift", action="store_true", help="add a random value to each column")
- parser.add_argument("--seed", type=int, default=None, help="add a random value to each column")
-
- args = parser.parse_args()
- create_test_h5ad(args.output, args.nobs, args.nvar, args.nnz_percent, args.col_shift, args.seed)
-
-
-def create_test_h5ad(outfile, nobs, nvar, nnz_percent=100, apply_col_shift=False, seed=None):
- random.seed(seed)
- np.random.seed(seed)
- x = create_X_array(nobs, nvar, nnz_percent, apply_col_shift)
- obsm = {"X_random": np.random.rand(nobs, 2).astype(np.float32)}
- adata = anndata.AnnData(x, obsm=obsm)
- adata.write(outfile)
-
-
-def create_X_array(nobs, nvar, nnz_percent, apply_col_shift):
- if nnz_percent < 100:
- array = scipy.sparse.random(nobs, nvar, nnz_percent * 0.01, dtype=np.float32, format="csc")
- else:
- array = np.random.rand(nobs, nvar).astype(np.float32)
-
- if apply_col_shift:
- col_shift = np.random.rand((nvar))
- array += col_shift
-
- return array
-
-
-if __name__ == "__main__":
- main()
diff --git a/backend/test/test_czi_hosted/performance/performance_test_annotations_backend.py b/backend/test/test_czi_hosted/performance/performance_test_annotations_backend.py
deleted file mode 100644
index 3272d669..00000000
--- a/backend/test/test_czi_hosted/performance/performance_test_annotations_backend.py
+++ /dev/null
@@ -1,216 +0,0 @@
-import json
-import string
-from contextlib import contextmanager
-from timeit import default_timer
-import concurrent.futures
-import numpy as np
-import requests
-import sys
-import pandas as pd
-import random
-
-from backend.common.fbs.matrix import encode_matrix_fbs
-
-"""
-Before running, sign into the dataportal, copy the cookie and paste it below. To test in staging or prod update the
-url base below. It is also possible to configure the number of categories created and the number of unique labels per
-category.
-"""
-
-cookie = ""
-
-test_datasets = {
- "smallest": {
- "dataset_url": "kampmann_lab_human_AD_snRNAseq_EC_inhibitoryNeurons-53-remixed.cxg",
- "name": "smallest",
- "num_cells": 5270,
- },
- "10k": {
- "dataset_url": "krasnow_lab_human_lung_cell_atlas_smartseq2-2-remixed.cxg",
- "name": "10k",
- "num_cells": 9409,
- },
- "80k": {
- "dataset_url": "Single_cell_gene_expression_profiling_of_SARS_CoV_2_infected_human_cell_lines_H1299-27-remixed.cxg", # noqa E501
- "name": "80k",
- "num_cells": 81736,
- },
- "140k": {"dataset_url": "Single_cell_drug_screening_a549-42-remixed.cxg", "name": "140k", "num_cells": 143015},
- "largest": {"dataset_url": "human_cell_landscape.cxg", "name": "largest", "num_cells": 599926},
- "1million": {"dataset_url": None, "name": "1million", "num_cells": 1000000},
- "4million": {"dataset_url": None, "name": "4million", "num_cells": 4000000},
-}
-
-url_base = "https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/"
-annotations_category_count = [1, 10, 50]
-max_labels = [5, 50, 100]
-
-
-class PerformanceTestingAnnotations:
- def __init__(
- self,
- datasets=test_datasets,
- annotations_category_count=annotations_category_count,
- max_labels=max_labels,
- url_base=url_base,
- ):
- self.test_datasets = datasets
- self.annotations_category_count = annotations_category_count
- self.max_labels = max_labels
- self.url_base = url_base
- self.test_notes = self.create_info_dict()
-
- def set_cell_count(self, dataset_name):
- dataset_url = self.test_datasets[dataset_name]["dataset_url"]
- headers = {"Content-Type": "application/octet-stream", "Cookie": cookie}
- response = self.client.get(f"{self.url_base}{dataset_url}/api/v0.2/schema", headers=headers)
- cell_count = json.loads(response._content)["schema"]["dataframe"]["nObs"]
- self.test_datasets[dataset_name]["cell_count"] = cell_count
-
- def create_info_dict(self):
- request_info = {}
- for dataset in self.test_datasets.keys():
- request_info[dataset] = {}
- for cat_count in self.annotations_category_count:
- request_info[dataset][f"num_categories_{cat_count}"] = {}
- for unique_labels in self.max_labels:
- request_info[dataset][f"num_categories_{cat_count}"][f"max_label_{unique_labels}"] = {}
- return request_info
-
- def create_annotations_dict_multi_process(self, dataset_name, category_count, label_max):
- annotation_dict = {}
- futures = []
- categories = [f"Category{i}" for i in range(category_count)]
- if not self.test_datasets[dataset_name]["num_cells"]:
- self.set_cell_count(dataset_name)
- with concurrent.futures.ProcessPoolExecutor(max_workers=5) as executor:
- for category in categories:
- futures.append(
- executor.submit(
- self.build_array_for_category,
- category,
- self.test_datasets[dataset_name]["num_cells"],
- label_max,
- )
- )
- for future in concurrent.futures.as_completed(futures):
- try:
- result = future.result()
- category_name, cells = result
- annotation_dict[category_name] = pd.Series(cells, dtype="category")
- except Exception as e:
- print(f"Issue creating the annotations dict: {e}")
- return annotation_dict
-
- def build_array_for_category(self, category_name, cell_count, label_max):
- unique_label_count = label_max
- labels = self.generate_labels(unique_label_count)
- cells_per_label = int(cell_count / len(labels))
- extra = cell_count % len(labels)
- cells = []
- for label in labels:
- cells.extend([label] * cells_per_label)
- cells.extend(["extra"] * extra)
- rng = np.random.default_rng()
- rng.shuffle(cells)
- return category_name, cells
-
- @staticmethod
- def convert_to_fbs(annotation_dict):
- df = pd.DataFrame(annotation_dict)
- return encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
-
- @staticmethod
- def generate_labels(unique_label_count):
- labels = ["undefined"]
- for i in range(unique_label_count):
- length = random.randrange(10, 20)
- labels.append(f"{i}__" + "".join(random.choice(string.ascii_letters) for z in range(length)))
- return labels
-
- @contextmanager
- def elapsed_timer(self):
- start = default_timer()
- elapser = lambda: default_timer() - start # noqa E731
- yield lambda: elapser()
- end = default_timer()
- elapser = lambda: end - start # noqa E731
-
- def create_matrix(self, dataset_name, num_cat, max_labels):
- with self.elapsed_timer() as elapsed:
- annon_dict = self.create_annotations_dict_multi_process(dataset_name, num_cat, max_labels)
- dict_size = sum(sys.getsizeof(value) for value in annon_dict.values()) / 1024 ** 2
- self.test_notes[dataset_name][f"num_categories_{num_cat}"][f"max_label_{max_labels}"]["annotation_dict"] = {
- "creation_time": str(elapsed()),
- "size": f"{dict_size} mb",
- }
- df = pd.DataFrame(annon_dict)
- df_size = sys.getsizeof(df) / 1024 ** 2
- self.test_notes[dataset_name][f"num_categories_{num_cat}"][f"max_label_{max_labels}"]["data_frame"] = {
- "creation_time": str(elapsed()),
- "size": f"{df_size} mb",
- }
- try:
- matrix = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
- matrix_size = sys.getsizeof(matrix) / 1024 ** 2
- self.test_notes[dataset_name][f"num_categories_{num_cat}"][f"max_label_{max_labels}"]["fbs_matrix"] = {
- "creation_time": str(elapsed()),
- "size": f"{matrix_size} mb",
- }
- return matrix
- except Exception as e:
- print(f"Issue creating fbs matrix: {e}, for {dataset_name}")
- return []
-
- def send_put_request(self, dataset_url, data):
- url = self.url_base + f"{dataset_url}/api/v0.2/annotations/obs"
- with self.elapsed_timer() as elapsed:
- try:
- headers = {"Content-Type": "application/octet-stream", "Cookie": cookie}
- response = requests.put(url=url, data=data, headers=headers)
- except Exception as e:
- print(f"Issue with put request: {e}")
- return None, elapsed()
- return response, elapsed()
-
- def test_categories_max_label_matrix(self, dataset_name):
- for unique_labels in self.max_labels:
- for category_count in self.annotations_category_count:
- print(f"Starting dataset: {dataset_name}, categories: {category_count}, labels: {unique_labels}")
- fbs_matrix = self.create_matrix(dataset_name, category_count, unique_labels)
- if self.test_datasets[dataset_name]["dataset_url"] and fbs_matrix:
- response, response_time = self.send_put_request(
- self.test_datasets[dataset_name]["dataset_url"], fbs_matrix
- )
- if response is None:
- self.test_notes[dataset_name][f"num_categories_{category_count}"][f"max_label_{unique_labels}"][
- "put_request"
- ] = {"response_status": "failed", "request_time": str(response_time)}
- else:
- self.test_notes[dataset_name][f"num_categories_{category_count}"][f"max_label_{unique_labels}"][
- "put_request"
- ] = {"response_status": response.status_code, "request_time": str(response_time)}
-
-
-def test_all_datasets():
- """
- Run time is dependent on number of datasets, dataset size, number of categories/number being tested and number of
- unique label counts being tested. However it generally takes a long time. I recommend running this in tmux
- """
- perf_test = PerformanceTestingAnnotations()
- for dataset_name in perf_test.test_datasets.keys():
- print(f"Testing annotation creation for: {dataset_name}")
- try:
- perf_test.test_categories_max_label_matrix(dataset_name)
- except Exception as e:
- print(f"something went wrong with {dataset_name}: {e}")
- return perf_test.test_notes
-
-
-def main():
- notes = test_all_datasets()
- print(notes)
-
-
-if __name__ == "__main__":
- main()
diff --git a/backend/test/test_czi_hosted/performance/run_diffexp.py b/backend/test/test_czi_hosted/performance/run_diffexp.py
deleted file mode 100644
index cb6fa847..00000000
--- a/backend/test/test_czi_hosted/performance/run_diffexp.py
+++ /dev/null
@@ -1,116 +0,0 @@
-import sys
-import argparse
-import random
-import time
-import numpy as np
-
-from backend.czi_hosted.common.config.app_config import AppConfig
-from backend.czi_hosted.compute import diffexp_cxg
-from backend.common.compute import diffexp_generic
-from backend.czi_hosted.data_common.matrix_loader import MatrixDataLoader
-from backend.czi_hosted.data_cxg.cxg_adaptor import CxgAdaptor
-
-
-def main():
- parser = argparse.ArgumentParser("A command to test diffexp")
- parser.add_argument("dataset", help="name of a dataset to load")
- parser.add_argument("-na", "--numA", type=int, help="number of rows in group A")
- parser.add_argument("-nb", "--numB", type=int, help="number of rows in group B")
- parser.add_argument("-va", "--varA", help="obs variable:value to use for group A")
- parser.add_argument("-vb", "--varB", help="obs variable:value to use for group B")
- parser.add_argument("-t", "--trials", default=1, type=int, help="number of trials")
- parser.add_argument(
- "-a", "--alg", choices=("default", "generic", "cxg"), default="default", help="algorithm to use"
- )
- parser.add_argument("-s", "--show", default=False, action="store_true", help="show the results")
- parser.add_argument(
- "-n", "--new-selection", default=False, action="store_true", help="change the selection between each trial"
- )
- parser.add_argument("--seed", default=1, type=int, help="set the random seed")
-
- args = parser.parse_args()
-
- app_config = AppConfig()
- app_config.update_server_config(single_dataset__datapath=args.dataset)
- app_config.update_server_config(app__verbose=True)
- app_config.complete_config()
-
- loader = MatrixDataLoader(args.dataset)
- adaptor = loader.open(app_config)
-
- if args.show:
- if isinstance(adaptor, CxgAdaptor):
- adaptor.open_array("X").schema.dump()
-
- random.seed(args.seed)
- np.random.seed(args.seed)
- rows = adaptor.get_shape()[0]
-
- if args.numA:
- filterA = random.sample(range(rows), args.numA)
- elif args.varA:
- vname, vval = args.varA.split(":")
- filterA = get_filter_from_obs(adaptor, vname, vval)
- else:
- print("must supply numA or varA")
- sys.exit(1)
-
- if args.numB:
- filterB = random.sample(range(rows), args.numB)
- elif args.varB:
- vname, vval = args.varB.split(":")
- filterB = get_filter_from_obs(adaptor, vname, vval)
- else:
- print("must supply numB or varB")
- sys.exit(1)
-
- for i in range(args.trials):
- if args.new_selection:
- if args.numA:
- filterA = random.sample(range(rows), args.numA)
- if args.numB:
- filterB = random.sample(range(rows), args.numB)
-
- maskA = np.zeros(rows, dtype=bool)
- maskA[filterA] = True
- maskB = np.zeros(rows, dtype=bool)
- maskB[filterB] = True
-
- t1 = time.time()
- if args.alg == "default":
- results = adaptor.compute_diffexp_ttest(maskA, maskB)
- elif args.alg == "generic":
- results = diffexp_generic.diffexp_ttest(adaptor, maskA, maskB)
- elif args.alg == "cxg":
- if not isinstance(adaptor, CxgAdaptor):
- print("cxg only works with CxgAdaptor")
- sys.exit(1)
- results = diffexp_cxg.diffexp_ttest(adaptor, maskA, maskB)
-
- t2 = time.time()
- print("TIME=", t2 - t1)
-
- if args.show:
- for res in results:
- print(res)
-
-
-def get_filter_from_obs(adaptor, obsname, obsval):
- attrs = adaptor.get_obs_columns()
- if obsname not in attrs:
- print(f"Unknown obs attr {obsname}: expected on of {attrs}")
- sys.exit(1)
- obsvals = adaptor.query_obs_array(obsname)[:]
- obsval = type(obsvals[0])(obsval)
-
- vfilter = np.where(obsvals == obsval)[0]
- if len(vfilter) == 0:
- u = np.unique(obsvals)
- print(f"Unknown value in variable {obsname}:{obsval}: expected one of {list(u)}")
- sys.exit(1)
-
- return vfilter
-
-
-if __name__ == "__main__":
- main()
diff --git a/backend/test/test_czi_hosted/performance/scale_test_annotations.py b/backend/test/test_czi_hosted/performance/scale_test_annotations.py
deleted file mode 100644
index f671c3f0..00000000
--- a/backend/test/test_czi_hosted/performance/scale_test_annotations.py
+++ /dev/null
@@ -1,45 +0,0 @@
-import time
-import random
-
-from locust import HttpUser, between, task
-
-random.seed(time.time())
-"""
-To run this script sign into cellxgene in the desired environment and grab the returned cookie, update the cookie
-variable below with your cookie and run the following command to see results in the terminal:
-locust -f backend/test/test_czi_hosted/performance/scale_test_annotations.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt
-
-Or if you want to use the locust gui run:
-locust -f backend/test/test_czi_hosted/performance/scale_test_annotations.py -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/
-
-If you want to test staging you'll need to substitute staging for dev in the host url
-To test prod you'll need to replace dev.single-cell.czi.technology with cziscience.com
-If you'd like to test additional datasets you'll need to add them to the dataset_urls array
-
-Todo @mdunitz update script to retrieve different annotation categories -- may need to create them to ensure the
-categories are shared across datasets for a given user.
-"""
-cookie = ""
-
-
-class WebsiteUser(HttpUser):
- wait_time = between(1, 2)
- dataset_urls = [
- "human_cell_landscape.cxg",
- "Single_cell_drug_screening_a549-42-remixed.cxg",
- "krasnow_lab_human_lung_cell_atlas_smartseq2-2-remixed.cxg",
- "Single_cell_gene_expression_profiling_of_SARS_CoV_2_infected_human_cell_lines_H1299-27-remixed.cxg",
- ]
-
- @task
- def get_annotations(self):
- dataset_url = random.choice(self.dataset_urls)
- url = f"{dataset_url}/api/v0.2/annotations/obs?annotation-name=cell_type"
- headers = {"Content-Type": "application/octet-stream", "Cookie": cookie}
- self.client.get(url, headers=headers)
-
- @task
- def get_schema(self):
- dataset_url = random.choice(self.dataset_urls)
- headers = {"Content-Type": "application/octet-stream", "Cookie": cookie}
- self.client.get(f"{dataset_url}/api/v0.2/schema", headers=headers)
diff --git a/backend/test/test_czi_hosted/test_database/__init__.py b/backend/test/test_czi_hosted/test_database/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/backend/test/test_czi_hosted/test_database/test_database.py b/backend/test/test_czi_hosted/test_database/test_database.py
deleted file mode 100644
index 18ffe1c6..00000000
--- a/backend/test/test_czi_hosted/test_database/test_database.py
+++ /dev/null
@@ -1,63 +0,0 @@
-import unittest
-from backend.czi_hosted.db.cellxgene_orm import CellxGeneUser, CellxGeneDataset, Annotation
-from backend.czi_hosted.db.db_utils import DbUtils
-from backend.test.fixtures.database import TestDatabase
-
-
-class DatabaseTest(unittest.TestCase):
- db = DbUtils("postgresql://postgres:test_pw@localhost:5432")
-
- @classmethod
- def setUpClass(cls) -> None:
- TestDatabase()
-
- @classmethod
- def tearDownClass(cls) -> None:
- del cls.db
-
- def test_user_creation(self):
- one_user = self.db.get(table=CellxGeneUser, entity_id="test_user_id")
- self.assertEqual(one_user.id, "test_user_id")
- user_count = self.db.session.query(CellxGeneUser).count()
- self.assertGreater(user_count, 10)
-
- def test_dataset_creation(self):
- one_dataset = self.db.query(
- table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == "test_dataset"]
- )
- self.assertEqual(one_dataset[0].name, "test_dataset")
- dataset_count = self.db.session.query(CellxGeneDataset).count()
- self.assertGreater(dataset_count, 10)
-
- def test_annotation_creation(self):
- one_annotation = self.db.query(table_args=[Annotation], filter_args=[Annotation.tiledb_uri == "tiledb_uri"])[0]
- self.assertEqual(one_annotation.tiledb_uri, "tiledb_uri")
- annotation_count = self.db.session.query(Annotation).count()
- self.assertGreater(annotation_count, 10)
-
- def test_get_most_recent_annotation_for_user_dataset(self):
- dataset_id = str(
- self.db.query(table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == "test_dataset"])[0].id
- )
-
- # have to commit separately because created_at time written on the db server
- self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_0"))
- self.db.session.commit()
-
- self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_1"))
- self.db.session.commit()
-
- self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_2"))
- self.db.session.commit()
-
- self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_3"))
- self.db.session.commit()
-
- self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_4"))
- self.db.session.commit()
-
- most_recent_annotation = self.db.query_for_most_recent(
- Annotation, [Annotation.dataset_id == dataset_id, Annotation.user_id == "test_user_id"]
- )
-
- self.assertEqual(most_recent_annotation.tiledb_uri, "tiledb_uri_4")
diff --git a/backend/test/test_czi_hosted/unit/__init__.py b/backend/test/test_czi_hosted/unit/__init__.py
deleted file mode 100644
index 31af2bd3..00000000
--- a/backend/test/test_czi_hosted/unit/__init__.py
+++ /dev/null
@@ -1,180 +0,0 @@
-import logging
-import shutil
-import tempfile
-import unittest
-
-from os import path
-
-import pandas as pd
-from flask_compress import Compress
-from flask_cors import CORS
-
-from backend.czi_hosted.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
-from backend.czi_hosted.common.annotations.local_file_csv import AnnotationsLocalFile
-from backend.czi_hosted.common.config.app_config import AppConfig
-from backend.common.utils.data_locator import DataLocator
-from backend.common.fbs.matrix import encode_matrix_fbs
-from backend.czi_hosted.data_common.matrix_loader import MatrixDataType, MatrixDataLoader
-from backend.czi_hosted.db.db_utils import DbUtils
-from backend.czi_hosted.app.app import Server
-from backend.test import PROJECT_ROOT, FIXTURES_ROOT
-
-
-def data_with_tmp_tiledb_annotations(ext: MatrixDataType):
- tmp_dir = tempfile.mkdtemp()
- fname = {
- MatrixDataType.H5AD: f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
- MatrixDataType.CXG: "test/fixtures/pbmc3k.cxg",
- }[ext]
- data_locator = DataLocator(fname)
- config = AppConfig()
- config.update_server_config(
- app__flask_secret_key="secret",
- multi_dataset__dataroot=data_locator.path,
- authentication__type="test",
- authentication__insecure_test_environment=True,
- )
- config.update_default_dataset_config(
- embeddings__names=["umap"],
- presentation__max_categories=100,
- diffexp__lfc_cutoff=0.01,
- user_annotations__type="hosted_tiledb_array",
- user_annotations__hosted_tiledb_array__db_uri="postgresql://postgres:test_pw@localhost:5432",
- user_annotations__hosted_tiledb_array__hosted_file_directory=tmp_dir,
- )
-
- config.complete_config()
-
- data = MatrixDataLoader(data_locator.abspath()).open(config)
- annotations = AnnotationsHostedTileDB(
- {
- "user-annotations": True,
- "genesets-save": False,
- },
- tmp_dir,
- DbUtils("postgresql://postgres:test_pw@localhost:5432"),
- )
- return data, tmp_dir, annotations
-
-
-def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False):
- tmp_dir = tempfile.mkdtemp()
- annotations_file = path.join(tmp_dir, "test_annotations.csv")
- if annotations_fixture:
- shutil.copyfile(f"{FIXTURES_ROOT}/pbmc3k-annotations.csv", annotations_file)
- fname = {
- MatrixDataType.H5AD: f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
- MatrixDataType.CXG: f"{FIXTURES_ROOT}/pbmc3k.cxg",
- }[ext]
- data_locator = DataLocator(fname)
- config = AppConfig()
- config.update_server_config(
- app__flask_secret_key="secret",
- single_dataset__obs_names=None,
- single_dataset__var_names=None,
- single_dataset__datapath=data_locator.path,
- )
- config.update_default_dataset_config(
- embeddings__names=["umap"],
- presentation__max_categories=100,
- diffexp__lfc_cutoff=0.01,
- )
-
- config.complete_config()
- data = MatrixDataLoader(data_locator.abspath()).open(config)
- annotations = AnnotationsLocalFile(
- {
- "user-annotations": True,
- "genesets-save": False,
- },
- None,
- annotations_file,
- )
- return data, tmp_dir, annotations, config
-
-
-def make_fbs(data):
- df = pd.DataFrame(data)
- return encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
-
-
-def skip_if(condition, reason: str):
- def decorator(f):
- def wraps(self, *args, **kwargs):
- if condition(self):
- self.skipTest(reason)
- else:
- f(self, *args, **kwargs)
-
- return wraps
-
- return decorator
-
-
-def app_config(data_locator, backed=False, extra_server_config={}, extra_dataset_config={}):
- config = AppConfig()
- config.update_server_config(
- app__flask_secret_key="secret",
- single_dataset__obs_names=None,
- single_dataset__var_names=None,
- adaptor__anndata_adaptor__backed=backed,
- single_dataset__datapath=data_locator,
- limits__diffexp_cellcount_max=None,
- limits__column_request_max=None,
- )
- config.update_default_dataset_config(
- embeddings__names=["umap", "tsne", "pca"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01
- )
- config.update_server_config(**extra_server_config)
- config.update_default_dataset_config(**extra_dataset_config)
- config.complete_config()
- return config
-
-
-class TestServer(Server):
- def __init__(self, app_config):
- super().__init__(app_config)
-
- @staticmethod
- def _before_adding_routes(app, app_config):
- app.config["COMPRESS_MIMETYPES"] = [
- "text/html",
- "text/css",
- "text/xml",
- "application/json",
- "application/javascript",
- "application/octet-stream",
- ]
- Compress(app)
- if app_config.server_config.app__debug:
- CORS(app, supports_credentials=True)
-
-
-class BaseTest(unittest.TestCase):
- @classmethod
- def setUpClass(cls, app_config=None):
- cls.TEST_URL_BASE = "/d/pbmc3k.cxg/api/v0.2/"
- cls.maxDiff = None
- cls.app = cls.create_app(app_config)
-
- @classmethod
- def create_app(cls, app_config=None):
- if not app_config:
- app_config = AppConfig()
- app_config.update_server_config(
- authentication__type="test",
- authentication__insecure_test_environment=True,
- app__flask_secret_key="testing",
- app__debug=True,
- multi_dataset__dataroot=f"{FIXTURES_ROOT}",
- multi_dataset__index=True,
- multi_dataset__allowed_matrix_types=["cxg"]
- )
- app_config.complete_config(logging.info)
-
- app = TestServer(app_config).app
-
- app.testing = True
- app.debug = True
-
- return app
diff --git a/backend/test/test_czi_hosted/unit/auth/__init__.py b/backend/test/test_czi_hosted/unit/auth/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/backend/test/test_czi_hosted/unit/auth/test_auth.py b/backend/test/test_czi_hosted/unit/auth/test_auth.py
deleted file mode 100644
index 67a7fdcb..00000000
--- a/backend/test/test_czi_hosted/unit/auth/test_auth.py
+++ /dev/null
@@ -1,168 +0,0 @@
-import json
-import unittest
-
-from backend.czi_hosted.common.config.app_config import AppConfig
-from backend.test import FIXTURES_ROOT
-from backend.test.test_czi_hosted.unit import BaseTest
-
-
-class AuthTest(BaseTest):
- def setUp(self):
- self.dataset_dataroot = FIXTURES_ROOT
-
- def test_auth_none(self):
- app_config = AppConfig()
- app_config.update_server_config(app__flask_secret_key="secret")
- app_config.update_server_config(authentication__type=None, multi_dataset__dataroot=self.dataset_dataroot)
- app_config.update_default_dataset_config(user_annotations__enable=False)
-
- app_config.complete_config()
- server= self.create_app(app_config)
- server.testing = True
- session = server.test_client()
- config = json.loads(session.get(f"{self.TEST_URL_BASE}config").data)
- userinfo = json.loads(session.get(f"{self.TEST_URL_BASE}userinfo").data)
- self.assertNotIn("authentication", config["config"])
- self.assertIsNone(userinfo)
-
- def test_auth_session(self):
- app_config = AppConfig()
- app_config.update_server_config(app__flask_secret_key="secret")
- app_config.update_server_config(authentication__type="session", multi_dataset__dataroot=self.dataset_dataroot)
- app_config.update_default_dataset_config(user_annotations__enable=True)
- app_config.complete_config()
-
- server = self.create_app(app_config)
- server.auth.is_user_authenticated = lambda: True
- server.testing = True
- session = server.test_client()
- config = json.loads(session.get(f"{self.TEST_URL_BASE}config").data)
- userinfo = json.loads(session.get(f"{self.TEST_URL_BASE}userinfo").data)
-
- self.assertFalse(config["config"]["authentication"]["requires_client_login"])
- self.assertTrue(userinfo["userinfo"]["is_authenticated"])
- self.assertEqual(userinfo["userinfo"]["username"], "anonymous")
-
- def test_auth_test(self):
- app_config = AppConfig()
- app_config.update_server_config(app__flask_secret_key="secret")
- app_config.update_server_config(authentication__type="test")
- app_config.update_server_config(authentication__insecure_test_environment=True)
- app_config.update_server_config(
- multi_dataset__dataroot=dict(
- a1=dict(dataroot=self.dataset_dataroot, base_url="auth"),
- a2=dict(dataroot=self.dataset_dataroot, base_url="no-auth"),
- )
- )
-
- # specialize the configs
- app_config.add_dataroot_config("a1", app__authentication_enable=True, user_annotations__enable=True)
- app_config.add_dataroot_config("a2", app__authentication_enable=False, user_annotations__enable=False)
-
- app_config.complete_config()
-
- server=self.create_app(app_config)
- server.testing = True
- session = server.test_client()
-
- # auth datasets
- config = json.loads(session.get("/auth/pbmc3k.cxg/api/v0.2/config").data)
- userinfo = json.loads(session.get(f"/auth/pbmc3k.cxg/api/v0.2/userinfo").data)
-
- self.assertFalse(userinfo["userinfo"]["is_authenticated"])
- self.assertIsNone(userinfo["userinfo"]["username"])
- self.assertTrue(config["config"]["authentication"]["requires_client_login"])
- self.assertTrue(config["config"]["parameters"]["annotations"])
-
- login_uri = config["config"]["authentication"]["login"]
- logout_uri = config["config"]["authentication"]["logout"]
-
- self.assertEqual(login_uri, "/login?dataset=auth/pbmc3k.cxg")
- self.assertEqual(logout_uri, "/logout?dataset=auth/pbmc3k.cxg")
-
- response = session.get(login_uri)
- # check that the login redirect worked
-
- self.assertEqual(response.status_code, 302)
- self.assertEqual(response.headers['Location'], 'http://localhost/auth/pbmc3k.cxg')
- config = json.loads(session.get("/auth/pbmc3k.cxg/api/v0.2/config").data)
- userinfo = json.loads(session.get("/auth/pbmc3k.cxg/api/v0.2/userinfo").data)
-
- self.assertTrue(userinfo["userinfo"]["is_authenticated"])
- self.assertEqual(userinfo["userinfo"]["username"], "test_account")
- self.assertEqual(userinfo["userinfo"]["picture"], None)
- self.assertTrue(config["config"]["parameters"]["annotations"])
-
- response = session.get(logout_uri)
- # check that the logout redirect worked
-
- self.assertEqual(response.status_code, 302)
- config = json.loads(session.get("/auth/pbmc3k.cxg/api/v0.2/config").data)
- userinfo = json.loads(session.get("/auth/pbmc3k.cxg/api/v0.2/userinfo").data)
- self.assertFalse(userinfo["userinfo"]["is_authenticated"])
- self.assertIsNone(userinfo["userinfo"]["username"])
- self.assertTrue(config["config"]["parameters"]["annotations"])
-
- # no-auth datasets
- config = json.loads(session.get("/no-auth/pbmc3k.cxg/api/v0.2/config").data)
- userinfo = json.loads(session.get("/no-auth/pbmc3k.cxg/api/v0.2/userinfo").data)
- self.assertIsNone(userinfo)
- self.assertFalse(config["config"]["parameters"]["annotations"])
-
- # login with a picture
- session.get(f"{login_uri}&picture=myimage.png")
- userinfo = json.loads(session.get("/auth/pbmc3k.cxg/api/v0.2/userinfo").data)
- self.assertTrue(userinfo["userinfo"]["is_authenticated"])
- self.assertEqual(userinfo["userinfo"]["picture"], "myimage.png")
-
- def test_auth_test_single(self):
- app_config = AppConfig()
- app_config.update_server_config(app__flask_secret_key="secret")
- app_config.update_server_config(
- authentication__type="test", single_dataset__datapath=f"{self.dataset_dataroot}/pbmc3k.cxg"
- )
- app_config.update_server_config(authentication__insecure_test_environment=True)
-
- app_config.complete_config()
-
- server = self.create_app(app_config)
- server.testing = True
- session = server.test_client()
-
- config = json.loads(session.get("/api/v0.2/config").data)
- userinfo = json.loads(session.get("/api/v0.2/userinfo").data)
- self.assertFalse(userinfo["userinfo"]["is_authenticated"])
- self.assertIsNone(userinfo["userinfo"]["username"])
- self.assertTrue(config["config"]["authentication"]["requires_client_login"])
- self.assertTrue(config["config"]["parameters"]["annotations"])
-
- login_uri = config["config"]["authentication"]["login"]
- logout_uri = config["config"]["authentication"]["logout"]
-
- self.assertEqual(login_uri, "/login")
- self.assertEqual(logout_uri, "/logout")
-
-
- # check that the login redirect worked
- with server.test_client() as session:
- response = session.get(login_uri)
- self.assertEqual(response.status_code, 302)
- self.assertEqual(response.headers['Location'], "http://localhost/")
-
- config = json.loads(session.get("api/v0.2/config").data)
- userinfo = json.loads(session.get("/api/v0.2/userinfo").data)
- self.assertTrue(userinfo["userinfo"]["is_authenticated"])
- self.assertEqual(userinfo["userinfo"]["username"], "test_account")
- self.assertTrue(config["config"]["parameters"]["annotations"])
-
- response = session.get(logout_uri)
- # check that the logout redirect worked
-
- self.assertEqual(response.status_code, 302)
- self.assertEqual(response.headers['Location'], "http://localhost/")
- config = json.loads(session.get("/api/v0.2/config").data)
-
- userinfo = json.loads(session.get("/api/v0.2/userinfo").data)
- self.assertFalse(userinfo["userinfo"]["is_authenticated"])
- self.assertIsNone(userinfo["userinfo"]["username"])
- self.assertTrue(config["config"]["parameters"]["annotations"])
diff --git a/backend/test/test_czi_hosted/unit/auth/test_oauth.py b/backend/test/test_czi_hosted/unit/auth/test_oauth.py
deleted file mode 100644
index f88d24f1..00000000
--- a/backend/test/test_czi_hosted/unit/auth/test_oauth.py
+++ /dev/null
@@ -1,229 +0,0 @@
-import unittest
-import random
-import time
-import base64
-import json
-import requests
-
-from flask import Flask, jsonify, make_response, request, redirect
-from multiprocessing import Process
-
-import jose
-from backend.czi_hosted.common.config.app_config import AppConfig
-from backend.test import FIXTURES_ROOT
-
-# This tests the oauth authentication type.
-# This test starts a cellxgene server and a mock oauth server.
-# API requests to login and logout and get the userinfo are made
-# to the cellxgene server, which then sends requests to the mock
-# oauth server.
-
-# number of seconds that the oauth token is valid
-from backend.test.test_czi_hosted.unit import BaseTest
-
-TOKEN_EXPIRES = 2
-
-# Create a mocked out oauth token, which servers all the endpoints needed by the oauth type.
-mock_oauth_app = Flask("mock_oauth_app")
-
-
-@mock_oauth_app.route("/authorize")
-def authorize():
- callback = request.args.get("redirect_uri")
- state = request.args.get("state")
- return redirect(callback + f"?code=fakecode&state={state}")
-
-
-@mock_oauth_app.route("/oauth/token", methods=["POST"])
-def token():
- now = time.time()
- expires_at = now + TOKEN_EXPIRES
- headers = dict(alg="RS256", kid="fake_kid")
- payload = dict(name="fake_user", sub="fake_id", email="fake_user@email.com", email_verified=True, exp=expires_at)
- jwt = jose.jwt.encode(claims=payload, key="mysecret", algorithm="HS256", headers=headers)
- r = {
- "access_token": f"access-{now}",
- "id_token": jwt,
- "refresh_token": f"random-{now}",
- "scope": "openid profile email",
- "expires_in": TOKEN_EXPIRES,
- "token_type": "Bearer",
- "expires_at": expires_at,
- }
- return make_response(jsonify(r))
-
-
-@mock_oauth_app.route("/v2/logout")
-def logout():
- return_to = request.args.get("returnTo")
- return redirect(return_to)
-
-
-@mock_oauth_app.route("/.well-known/jwks.json")
-def jwks():
- data = dict(alg="RS256", kty="RSA", use="sig", kid="fake_kid",)
- return make_response(jsonify(dict(keys=[data])))
-
-
-# function to launch the mock oauth server
-def launch_mock_oauth(mock_port):
- mock_oauth_app.run(port=mock_port)
-
-
-class AuthTest(BaseTest):
- @classmethod
- def setUpClass(cls):
- # The port that the mock oauth server will listen on
- cls.mock_port = random.randint(10000, 12000)
- cls.dataset_dataroot = FIXTURES_ROOT
- cls.mock_oauth_process = Process(target=launch_mock_oauth, args=(cls.mock_port,))
- cls.mock_oauth_process.start()
-
- # Verify that the mock oauth server is ready (accepting requests) before starting the tests.
-
- # The following lines are polling until the mock server is ready.
- # The issue is we are starting a mock oauth server, then we are starting a cellxgene server,
- # which will start making requests to the mock oauth server.
- # So there is a race condition because the mock oauth server needs to be ready before it gets requests.
- # We check to see if it is ready, and if not we wait 1 second, then try again.
- # If it gets to 5 seconds, which is shouldn't, we assume something has gone wrong and fail the test.
- server_okay = False
- for _ in range(5):
- try:
- response = requests.get(f"http://localhost:{cls.mock_port}/.well-known/jwks.json")
- if response.status_code == 200:
- server_okay = True
- break
- except: # noqa: E722
- pass
-
- # wait one second and try again
- time.sleep(1)
-
- assert(server_okay)
-
- @classmethod
- def tearDownClass(cls):
- cls.mock_oauth_process.terminate()
-
- def auth_flow(self, app_config, cookie_key=None):
-
- app_config.update_server_config(
- app__api_base_url="local",
- authentication__type="oauth",
- authentication__params_oauth__oauth_api_base_url=f"http://localhost:{self.mock_port}",
- authentication__params_oauth__client_id="mock_client_id",
- authentication__params_oauth__client_secret="mock_client_secret",
- authentication__params_oauth__jwt_decode_options={"verify_signature": False, "verify_iss": False},
- )
-
- app_config.update_server_config(multi_dataset__dataroot=self.dataset_dataroot)
- app_config.complete_config()
-
- server= self.create_app(app_config)
- server.testing = True
- session = server.test_client()
-
- # auth datasets
- config = json.loads(session.get("/d/pbmc3k.cxg/api/v0.2/config").data)
- userinfo = json.loads(session.get("/d/pbmc3k.cxg/api/v0.2/userinfo").data)
-
- self.assertFalse(userinfo["userinfo"]["is_authenticated"])
- self.assertIsNone(userinfo["userinfo"]["username"])
- self.assertTrue(config["config"]["authentication"]["requires_client_login"])
- self.assertTrue(config["config"]["parameters"]["annotations"])
-
- login_uri = config["config"]["authentication"]["login"]
- logout_uri = config["config"]["authentication"]["logout"]
-
- self.assertEqual(login_uri, "http://localhost:5005/login?dataset=d/pbmc3k.cxg/")
- self.assertEqual(logout_uri, "http://localhost:5005/logout?dataset=d/pbmc3k.cxg/")
-
- response = session.get(login_uri)
- # check that the login redirect worked
-
- self.assertEqual(response.status_code, 302)
-
- config = json.loads(session.get("/d/pbmc3k.cxg/api/v0.2/config").data)
- userinfo = json.loads(session.get("/d/pbmc3k.cxg/api/v0.2/userinfo").data)
-
- self.assertTrue(userinfo["userinfo"]["is_authenticated"])
- self.assertEqual(userinfo["userinfo"]["username"], "fake_user")
- self.assertEqual(userinfo["userinfo"]["email"], "fake_user@email.com")
- self.assertTrue(config["config"]["parameters"]["annotations"])
-
- if cookie_key:
- cookie = session.cookies.get(cookie_key)
- token = json.loads(base64.b64decode(cookie))
- access_token_before = token.get("access_token")
- id_token_before = token.get("id_token")
-
- # let the token expire
- time.sleep(TOKEN_EXPIRES + 1)
-
- # check that refresh works
- session.get(login_uri)
- userinfo = json.loads(session.get(f"/d/pbmc3k.cxg/api/v0.2/userinfo").data)
- self.assertTrue(userinfo["userinfo"]["is_authenticated"])
- self.assertEqual(userinfo["userinfo"]["username"], "fake_user")
-
- cookie = session.cookies.get(cookie_key)
- token = json.loads(base64.b64decode(cookie))
- access_token_after = token.get("access_token")
- id_token_after = token.get("id_token")
-
- self.assertNotEqual(access_token_before, access_token_after)
- self.assertNotEqual(id_token_before, id_token_after)
-
- # invalid cookie is rejected
- session.cookies.set(cookie_key, "TEST_" + cookie)
- self.assertTrue(cookie_key in session.cookies)
- response = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo")
- # this is not an error, the invalid cookie is just ignored.
- self.assertEqual(response.status_code, 200)
- userinfo = json.loads(response.data)
- self.assertFalse(userinfo["userinfo"]["is_authenticated"])
- self.assertIsNone(userinfo["userinfo"]["username"])
-
- # invalid id_token is rejected
- test_token = token
- test_token["id_token"] = "TEST_" + id_token_after
- encoded_cookie = base64.b64encode(json.dumps(test_token).encode()).decode()
- session.cookies.set(cookie_key, encoded_cookie)
- response = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo")
- # this is not an error, the invalid id_token is just ignored.
- self.assertEqual(response.status_code, 200)
- userinfo = json.loads(response.data)
- self.assertFalse(userinfo["userinfo"]["is_authenticated"])
- self.assertIsNone(userinfo["userinfo"]["username"])
-
- r = session.get(logout_uri)
- # check that the logout redirect worked
-
- self.assertEqual(r.history[0].status_code, 302)
- self.assertEqual(r.url, f"{server}/d/pbmc3k.cxg/")
- config = json.loads(session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config").data)
- userinfo = json.loads(session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").data)
- self.assertFalse(userinfo["userinfo"]["is_authenticated"])
- self.assertIsNone(userinfo["userinfo"]["username"])
- self.assertTrue(config["config"]["parameters"]["annotations"])
-
- @unittest.skip("turn on when we utilizing auth in the explorer")
- def test_auth_oauth_session(self):
- # test with session cookies
- app_config = AppConfig()
- app_config.update_server_config(app__flask_secret_key="secret")
- app_config.update_server_config(authentication__params_oauth__session_cookie=True,)
- self.auth_flow(app_config)
-
- @unittest.skip("turn on when we utilizing auth in the explorer")
- def test_auth_oauth_cookie(self):
- # test with specified cookie
- app_config = AppConfig()
- app_config.update_server_config(app__flask_secret_key="secret")
- app_config.update_server_config(
- authentication__params_oauth__session_cookie=False,
- authentication__params_oauth__cookie=dict(key="test_cxguser", httponly=True, max_age=60),
- )
-
- self.auth_flow(app_config, "test_cxguser")
diff --git a/backend/test/test_czi_hosted/unit/cli/__init__.py b/backend/test/test_czi_hosted/unit/cli/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/backend/test/test_czi_hosted/unit/cli/test_launch.py b/backend/test/test_czi_hosted/unit/cli/test_launch.py
deleted file mode 100644
index 5550e775..00000000
--- a/backend/test/test_czi_hosted/unit/cli/test_launch.py
+++ /dev/null
@@ -1,27 +0,0 @@
-import filecmp
-import os
-import shutil
-import unittest
-
-import yaml
-
-from backend.czi_hosted.default_config import default_config
-from backend.test import FIXTURES_ROOT
-
-
-class CLIPLaunchTests(unittest.TestCase):
- tmp_dir = os.path.join(FIXTURES_ROOT, "dump_configs")
-
- @classmethod
- def setUpClass(cls) -> None:
- os.mkdir(cls.tmp_dir)
-
- @classmethod
- def tearDownClass(cls) -> None:
- shutil.rmtree(cls.tmp_dir)
-
- def test_dump_default_config(self):
- os.system(f"cellxgene launch --dump-default-config > {self.tmp_dir}/test_config_dump.txt")
- with open(f"{self.tmp_dir}/expected_config_dump.txt", "w") as expected_config:
- expected_config.write(yaml.dump(default_config))
- filecmp.cmp(f"{self.tmp_dir}/expected_config_dump.txt", f"{self.tmp_dir}/test_config_dump.txt")
diff --git a/backend/test/test_czi_hosted/unit/cli/test_prepare.py b/backend/test/test_czi_hosted/unit/cli/test_prepare.py
deleted file mode 100644
index d4c78fdf..00000000
--- a/backend/test/test_czi_hosted/unit/cli/test_prepare.py
+++ /dev/null
@@ -1,15 +0,0 @@
-import unittest
-
-import pandas as pd
-
-from backend.czi_hosted.cli.prepare import make_index_unique
-
-
-class CLIPrepareTests(unittest.TestCase):
- """ Test cases for CLI prepare logic """
-
- def test_make_index_unique(self):
- index = pd.Index(["SNORD113", "SNORD113", "SNORD113-1"])
- result = make_index_unique(index)
- expected = pd.Index(["SNORD113", "SNORD113-2", "SNORD113-1"])
- self.assertTrue(all(left == right for left, right in zip(result.values, expected.values)))
diff --git a/backend/test/test_czi_hosted/unit/cli/test_upgrade.py b/backend/test/test_czi_hosted/unit/cli/test_upgrade.py
deleted file mode 100644
index a9ae85d9..00000000
--- a/backend/test/test_czi_hosted/unit/cli/test_upgrade.py
+++ /dev/null
@@ -1,28 +0,0 @@
-import unittest
-
-from backend.czi_hosted.cli.upgrade import validate_version_str, split_version, version_gt
-
-
-class CLIUpgradeTests(unittest.TestCase):
- """ Test cases for CLI logic """
-
- def test_validate_version_str(self):
- self.assertTrue(validate_version_str("0.1.2"))
- self.assertTrue(validate_version_str("0.1.2-RC", release_only=False))
- self.assertFalse(validate_version_str("0.1"))
- self.assertFalse(validate_version_str("0.1.2.3"))
- self.assertFalse(validate_version_str("0.1.2-RC"))
-
- def test_split_version_str(self):
- self.assertEqual(split_version("0.1.2"), [0, 1, 2])
- with self.assertRaises(AttributeError):
- split_version("0.1")
-
- def test_assert_verstion_gt(self):
- self.assertTrue(version_gt("1.0.0", "0.1.1"))
- self.assertTrue(version_gt("0.1.0", "0.0.1"))
- self.assertTrue(version_gt("0.0.1", "0.0.0"))
- self.assertFalse(version_gt("0.0.0", "0.0.0"))
- self.assertFalse(version_gt("0.0.0", "0.0.1"))
- self.assertFalse(version_gt("0.0.1", "0.1.0"))
- self.assertFalse(version_gt("0.1.1", "1.0.0"))
diff --git a/backend/test/test_czi_hosted/unit/common/__init__.py b/backend/test/test_czi_hosted/unit/common/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/backend/test/test_czi_hosted/unit/common/config/__init__.py b/backend/test/test_czi_hosted/unit/common/config/__init__.py
deleted file mode 100644
index 1cfebb17..00000000
--- a/backend/test/test_czi_hosted/unit/common/config/__init__.py
+++ /dev/null
@@ -1,274 +0,0 @@
-import os
-import shutil
-import random
-import yaml
-
-from backend.test import FIXTURES_ROOT
-from backend.test.test_czi_hosted.unit import BaseTest
-
-
-
-class ConfigTests(BaseTest):
- tmp_fixtures_directory = os.path.join(FIXTURES_ROOT, "tmp_dir")
-
- @classmethod
- def tearDownClass(cls) -> None:
- shutil.rmtree(cls.tmp_fixtures_directory)
-
- @classmethod
- def setUpClass(cls) -> None:
- os.makedirs(cls.tmp_fixtures_directory)
-
- def custom_server_config(
- self,
- verbose="false",
- debug="false",
- host="localhost",
- port="null",
- open_browser="false",
- force_https="false",
- flask_secret_key="secret",
- generate_cache_control_headers="false",
- server_timing_headers="false",
- csp_directives="null",
- api_base_url="null",
- web_base_url="null",
- auth_type="session",
- insecure_test_environment="false",
- oauth_api_base_url="null",
- client_id="null",
- client_secret="null",
- jwt_decode_options="null",
- session_cookie="true",
- cookie="null",
- dataroot="null",
- index="false",
- allowed_matrix_types=[],
- max_cached_datasets=5,
- timelimit_s=5,
- dataset_datapath="null",
- obs_names="null",
- var_names="null",
- about="null",
- title="null",
- diffexp_max_workers=64,
- cpu_multiplier=4,
- target_workunit="16_000_000",
- data_locater_region_name="us-east-1",
- cxg_tile_cache_size=8589934592,
- cxg_num_reader_threads=32,
- anndata_backed="false",
- column_request_max=32,
- diffexp_cellcount_max="null",
- config_file_name="server_config.yaml",
- ):
- configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
- server_config_outline_path = os.path.join(FIXTURES_ROOT, "czi_hosted_server_config_outline.py")
- with open(server_config_outline_path, "r") as config_skeleton:
- config = config_skeleton.read()
- server_config = eval(config)
- with open(configfile, "w") as server_config_file:
- server_config_file.write(server_config)
- return configfile
-
- def custom_app_config(
- self,
- verbose="false",
- debug="false",
- host="localhost",
- port="null",
- open_browser="false",
- force_https="false",
- flask_secret_key="secret",
- generate_cache_control_headers="false",
- server_timing_headers="false",
- csp_directives="null",
- api_base_url="null",
- web_base_url="null",
- auth_type="session",
- oauth_api_base_url="null",
- client_id="null",
- client_secret="null",
- jwt_decode_options="null",
- session_cookie="true",
- cookie="null",
- dataroot="null",
- index="false",
- allowed_matrix_types=[],
- max_cached_datasets=5,
- timelimit_s=5,
- dataset_datapath="null",
- obs_names="null",
- var_names="null",
- about="null",
- title="null",
- diffexp_max_workers=64,
- cpu_multiplier=4,
- target_workunit="16_000_000",
- data_locater_region_name="us-east-1",
- cxg_tile_cache_size=8589934592,
- cxg_num_reader_threads=32,
- anndata_backed="false",
- column_request_max=32,
- diffexp_cellcount_max="null",
- scripts=[],
- inline_scripts=[],
- about_legal_tos="null",
- about_legal_privacy="null",
- authentication_enable="true",
- max_categories=1000,
- custom_colors="true",
- enable_users_annotations="true",
- annotation_type="local_file_csv",
- db_uri="null",
- hosted_file_directory="null",
- local_file_csv_directory="null",
- local_file_csv_file="null",
- embedding_names=[],
- enable_difexp="true",
- lfc_cutoff=0.01,
- top_n=10,
- environment=None,
- aws_secrets_manager_region=None,
- aws_secrets_manager_secrets=[],
- X_approximate_distribution="normal",
- config_file_name="app_config.yml",
- ):
- random_num = random.randrange(999999)
- configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
- server_config = self.custom_server_config(
- verbose=verbose,
- debug=debug,
- host=host,
- port=port,
- open_browser=open_browser,
- force_https=force_https,
- flask_secret_key=flask_secret_key,
- generate_cache_control_headers=generate_cache_control_headers,
- server_timing_headers=server_timing_headers,
- csp_directives=csp_directives,
- api_base_url=api_base_url,
- web_base_url=web_base_url,
- auth_type=auth_type,
- oauth_api_base_url=oauth_api_base_url,
- client_id=client_id,
- client_secret=client_secret,
- jwt_decode_options=jwt_decode_options,
- session_cookie=session_cookie,
- cookie=cookie,
- dataroot=dataroot,
- index=index,
- allowed_matrix_types=allowed_matrix_types,
- max_cached_datasets=max_cached_datasets,
- timelimit_s=timelimit_s,
- dataset_datapath=dataset_datapath,
- obs_names=obs_names,
- var_names=var_names,
- about=about,
- title=title,
- diffexp_max_workers=diffexp_max_workers,
- cpu_multiplier=cpu_multiplier,
- target_workunit=target_workunit,
- data_locater_region_name=data_locater_region_name,
- cxg_tile_cache_size=cxg_tile_cache_size,
- cxg_num_reader_threads=cxg_num_reader_threads,
- anndata_backed=anndata_backed,
- column_request_max=column_request_max,
- diffexp_cellcount_max=diffexp_cellcount_max,
- config_file_name=f"temp_server_config_{random_num}.yml",
- )
- dataset_config = self.custom_dataset_config(
- scripts=scripts,
- inline_scripts=inline_scripts,
- about_legal_tos=about_legal_tos,
- about_legal_privacy=about_legal_privacy,
- authentication_enable=authentication_enable,
- max_categories=max_categories,
- custom_colors=custom_colors,
- enable_users_annotations=enable_users_annotations,
- annotation_type=annotation_type,
- db_uri=db_uri,
- hosted_file_directory=hosted_file_directory,
- local_file_csv_directory=local_file_csv_directory,
- local_file_csv_file=local_file_csv_file,
- embedding_names=embedding_names,
- enable_difexp=enable_difexp,
- lfc_cutoff=lfc_cutoff,
- top_n=top_n,
- X_approximate_distribution=X_approximate_distribution,
- config_file_name=f"temp_dataset_config_{random_num}.yml",
- )
- external_config = self.custom_external_config(
- environment=environment,
- aws_secrets_manager_region=aws_secrets_manager_region,
- aws_secrets_manager_secrets=aws_secrets_manager_secrets,
- config_file_name=f"temp_external_config_{random_num}.yml",
- )
-
- with open(configfile, "w") as app_config_file:
- app_config_file.write(open(server_config).read())
- app_config_file.write(open(dataset_config).read())
- app_config_file.write(open(external_config).read())
-
- return configfile
-
- def custom_dataset_config(
- self,
- scripts=[],
- inline_scripts=[],
- about_legal_tos="null",
- about_legal_privacy="null",
- authentication_enable="true",
- max_categories=1000,
- custom_colors="true",
- enable_users_annotations="true",
- annotation_type="local_file_csv",
- db_uri="null",
- hosted_file_directory="null",
- local_file_csv_directory="null",
- local_file_csv_file="null",
- embedding_names=[],
- enable_difexp="true",
- lfc_cutoff=0.01,
- top_n=10,
- X_approximate_distribution="normal",
- config_file_name="dataset_config.yml",
- ):
- configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
- dataset_config_outline_path = os.path.join(FIXTURES_ROOT, "czi_hosted_dataset_config_outline.py")
- with open(dataset_config_outline_path, "r") as config_skeleton:
- config = config_skeleton.read()
- dataset_config = eval(config)
- with open(configfile, "w") as dataset_config_file:
- dataset_config_file.write(dataset_config)
-
- return configfile
-
- def custom_external_config(
- self,
- environment=None,
- aws_secrets_manager_region=None,
- aws_secrets_manager_secrets=[],
- config_file_name="external_config.yaml",
- ):
- # set to the default if environment is None
- if environment is None:
- environment = [
- dict(name="CXG_SECRET_KEY", path=["server", "app", "flask_secret_key"], required=False),
- dict(
- name="CXG_OAUTH_CLIENT_SECRET",
- path=["server", "authentication", "params_oauth", "client_secret"],
- required=False,
- ),
- ]
- external_config = {
- "external": {
- "environment": environment,
- "aws_secrets_manager": {"region": aws_secrets_manager_region, "secrets": aws_secrets_manager_secrets},
- }
- }
-
- configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
- with open(configfile, "w") as external_config_file:
- yaml.dump(external_config, external_config_file)
- return configfile
diff --git a/backend/test/test_czi_hosted/unit/common/config/test_app_config.py b/backend/test/test_czi_hosted/unit/common/config/test_app_config.py
deleted file mode 100644
index f6e7bc31..00000000
--- a/backend/test/test_czi_hosted/unit/common/config/test_app_config.py
+++ /dev/null
@@ -1,220 +0,0 @@
-import os
-import tempfile
-import unittest
-
-import yaml
-
-from backend.czi_hosted.common.config.app_config import AppConfig
-from backend.common.errors import ConfigurationError
-from backend.czi_hosted.default_config import default_config
-from backend.test import FIXTURES_ROOT
-from backend.test.test_czi_hosted.unit.common.config import ConfigTests
-
-
-class AppConfigTest(ConfigTests):
- def setUp(self):
- self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
- self.config = AppConfig()
- self.config.update_server_config(app__flask_secret_key="secret")
- self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
- self.server_config = self.config.server_config
- self.config.complete_config()
-
- message_list = []
-
- def noop(message):
- message_list.append(message)
-
- messagefn = noop
- self.context = dict(messagefn=messagefn, messages=message_list)
-
- def get_config(self, **kwargs):
- file_name = self.custom_app_config(
- dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
- )
- config = AppConfig()
- config.update_from_config_file(file_name)
- return config
-
- def test_get_default_config_correctly_reads_default_config_file(self):
- app_default_config = AppConfig().default_config
-
- expected_config = yaml.load(default_config, Loader=yaml.Loader)
-
- server_config = app_default_config["server"]
- dataset_config = app_default_config["dataset"]
-
- expected_server_config = expected_config["server"]
- expected_dataset_config = expected_config["dataset"]
-
- self.assertDictEqual(app_default_config, expected_config)
- self.assertDictEqual(server_config, expected_server_config)
- self.assertDictEqual(dataset_config, expected_dataset_config)
-
- def test_get_dataset_config_returns_default_dataset_config_for_single_datasets(self):
- datapath = f"{FIXTURES_ROOT}/1e4dfec4-c0b2-46ad-a04e-ff3ffb3c0a8f.h5ad"
- file_name = self.custom_app_config(dataset_datapath=datapath, config_file_name=self.config_file_name)
- config = AppConfig()
- config.update_from_config_file(file_name)
-
- self.assertEqual(config.get_dataset_config(""), config.default_dataset_config)
-
- def test_update_server_config_updates_server_config_and_config_status(self):
- config = self.get_config()
- config.complete_config()
- config.check_config()
- config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
- with self.assertRaises(ConfigurationError):
- config.server_config.check_config()
-
- def test_write_config_outputs_yaml_with_all_config_vars(self):
- config = self.get_config()
- config.write_config(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml")
- with open(f"{FIXTURES_ROOT}/tmp_dir/{self.config_file_name}", "r") as default_config:
- default_config_yml = yaml.safe_load(default_config)
-
- with open(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml", "r") as output_config:
- output_config_yml = yaml.safe_load(output_config)
- self.maxDiff = None
- self.assertEqual(default_config_yml, output_config_yml)
-
- def test_update_app_config(self):
- config = AppConfig()
- config.update_server_config(app__verbose=True, multi_dataset__dataroot="datadir")
- vars = config.server_config.changes_from_default()
- self.assertCountEqual(vars, [("app__verbose", True, False), ("multi_dataset__dataroot", "datadir", None)])
-
- config = AppConfig()
- config.update_default_dataset_config(app__scripts=(), app__inline_scripts=())
- vars = config.server_config.changes_from_default()
- self.assertCountEqual(vars, [])
-
- config = AppConfig()
- config.update_default_dataset_config(app__scripts=[], app__inline_scripts=[])
- vars = config.default_dataset_config.changes_from_default()
- self.assertCountEqual(vars, [])
-
- config = AppConfig()
- config.update_default_dataset_config(app__scripts=("a", "b"), app__inline_scripts=["c", "d"])
- vars = config.default_dataset_config.changes_from_default()
- self.assertCountEqual(vars, [("app__scripts", ["a", "b"], []), ("app__inline_scripts", ["c", "d"], [])])
-
- def test_configfile_no_dataset_section(self):
- # test a config file without a dataset section
-
- with tempfile.TemporaryDirectory() as tempdir:
- configfile = os.path.join(tempdir, "config.yaml")
- with open(configfile, "w") as fconfig:
- config = """
- server:
- app:
- flask_secret_key: secret
- multi_dataset:
- dataroot: test_dataroot
-
- """
- fconfig.write(config)
-
- app_config = AppConfig()
- app_config.update_from_config_file(configfile)
- server_changes = app_config.server_config.changes_from_default()
- dataset_changes = app_config.default_dataset_config.changes_from_default()
- self.assertEqual(
- server_changes,
- [("app__flask_secret_key", "secret", None), ("multi_dataset__dataroot", "test_dataroot", None)],
- )
- self.assertEqual(dataset_changes, [])
-
- def test_configfile_no_server_section(self):
- # test a config file without a dataset section
-
- with tempfile.TemporaryDirectory() as tempdir:
- configfile = os.path.join(tempdir, "config.yaml")
- with open(configfile, "w") as fconfig:
- config = """
- dataset:
- user_annotations:
- enable: false
- """
- fconfig.write(config)
-
- app_config = AppConfig()
- app_config.update_from_config_file(configfile)
- server_changes = app_config.server_config.changes_from_default()
- dataset_changes = app_config.default_dataset_config.changes_from_default()
- self.assertEqual(server_changes, [])
- self.assertEqual(dataset_changes, [("user_annotations__enable", False, True)])
-
- def test_simple_update_single_config_from_path_and_value(self):
- """Update a simple config parameter"""
-
- config = AppConfig()
- config.server_config.multi_dataset__dataroot = dict(
- s1=dict(dataroot="my_dataroot_s1", base_url="my_baseurl_s1"),
- s2=dict(dataroot="my_dataroot_s2", base_url="my_baseurl_s2"),
- )
- config.add_dataroot_config("s1")
- config.add_dataroot_config("s2")
-
- # test simple value in server
- config.update_single_config_from_path_and_value(["server", "app", "flask_secret_key"], "mysecret")
- self.assertEqual(config.server_config.app__flask_secret_key, "mysecret")
-
- # test simple value in default dataset
- config.update_single_config_from_path_and_value(
- ["dataset", "user_annotations", "hosted_tiledb_array", "db_uri"], "mydburi",
- )
- self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mydburi")
- self.assertEqual(config.dataroot_config["s1"].user_annotations__hosted_tiledb_array__db_uri, "mydburi")
- self.assertEqual(config.dataroot_config["s2"].user_annotations__hosted_tiledb_array__db_uri, "mydburi")
-
- # test simple value in specific dataset
- config.update_single_config_from_path_and_value(
- ["per_dataset_config", "s1", "user_annotations", "hosted_tiledb_array", "db_uri"], "s1dburi"
- )
- self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mydburi")
- self.assertEqual(config.dataroot_config["s1"].user_annotations__hosted_tiledb_array__db_uri, "s1dburi")
- self.assertEqual(config.dataroot_config["s2"].user_annotations__hosted_tiledb_array__db_uri, "mydburi")
-
- # error checking
- bad_paths = [
- (
- ["dataset", "does", "not", "exist"],
- "unknown config parameter at path: '['dataset', 'does', 'not', 'exist']'",
- ),
- (["does", "not", "exist"], "path must start with 'server', 'dataset', or 'per_dataset_config'"),
- ([], "path must start with 'server', 'dataset', or 'per_dataset_config'"),
- (["per_dataset_config"], "missing dataroot when using per_dataset_config: got '['per_dataset_config']'"),
- (
- ["per_dataset_config", "unknown"],
- "unknown dataroot when using per_dataset_config: got '['per_dataset_config', 'unknown']',"
- " dataroots specified in config are ['s1', 's2']",
- ),
- ([1, 2, 3], "path must be a list of strings, got '[1, 2, 3]'"),
- ("string", "path must be a list of strings, got 'string'"),
- ]
- for bad_path, error_message in bad_paths:
- with self.assertRaises(ConfigurationError) as config_error:
- config.update_single_config_from_path_and_value(bad_path, "value")
-
- self.assertEqual(config_error.exception.message, error_message)
-
- def test_dict_update_single_config_from_path_and_value(self):
- """Update a config parameter that has a value of dict"""
-
- # the path leads to a dict config param, set the config parameter to the new value
- config = AppConfig()
- config.update_single_config_from_path_and_value(
- ["server", "authentication", "params_oauth", "cookie"], dict(key="mykey1", max_age=100)
- )
- self.assertEqual(config.server_config.authentication__params_oauth__cookie, dict(key="mykey1", max_age=100))
-
- # the path leads to an entry within a dict config param, the value is simple
- config = AppConfig()
- config.server_config.authentication__params_oauth__cookie = dict(key="mykey1", max_age=100)
- config.update_single_config_from_path_and_value(
- ["server", "authentication", "params_oauth", "cookie", "httponly"], True,
- )
- self.assertEqual(
- config.server_config.authentication__params_oauth__cookie, dict(key="mykey1", max_age=100, httponly=True)
- )
diff --git a/backend/test/test_czi_hosted/unit/common/config/test_base_config.py b/backend/test/test_czi_hosted/unit/common/config/test_base_config.py
deleted file mode 100644
index 959fbfd4..00000000
--- a/backend/test/test_czi_hosted/unit/common/config/test_base_config.py
+++ /dev/null
@@ -1,66 +0,0 @@
-import unittest
-
-from backend.czi_hosted.common.config.app_config import AppConfig
-from backend.test import FIXTURES_ROOT
-from backend.common.errors import ConfigurationError
-from backend.test.test_czi_hosted.unit.common.config import ConfigTests
-
-
-class BaseConfigTest(ConfigTests):
- def setUp(self):
- self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
- self.config = AppConfig()
- self.config.update_server_config(app__flask_secret_key="secret")
- self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
- self.server_config = self.config.server_config
- self.config.complete_config()
-
- message_list = []
-
- def noop(message):
- message_list.append(message)
-
- messagefn = noop
- self.context = dict(messagefn=messagefn, messages=message_list)
-
- def get_config(self, **kwargs):
- file_name = self.custom_app_config(
- dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
- )
- config = AppConfig()
- config.update_from_config_file(file_name)
- return config
-
- def test_mapping_creation_returns_map_of_server_and_dataset_config(self):
- config = AppConfig()
- mapping = config.default_dataset_config.create_mapping(config.default_config)
- self.assertIsNotNone(mapping["server__app__verbose"])
- self.assertIsNotNone(mapping["dataset__presentation__max_categories"])
- self.assertIsNotNone(mapping["server__multi_dataset__allowed_matrix_types"])
-
- def test_changes_from_default_returns_list_of_nondefault_config_values(self):
- config = self.get_config(verbose="true", lfc_cutoff=0.05)
- server_changes = config.server_config.changes_from_default()
- dataset_changes = config.default_dataset_config.changes_from_default()
-
- self.assertEqual(
- server_changes,
- [
- ("app__verbose", True, False),
- ("app__flask_secret_key", "secret", None),
- ('authentication__type', 'session', 'test'),
- ('authentication__insecure_test_environment', False, True),
- ("multi_dataset__dataroot", FIXTURES_ROOT, None),
- ("multi_dataset__matrix_cache__timelimit_s", 5, 30),
- ("data_locator__s3__region_name", "us-east-1", True),
- ],
- )
- self.assertEqual(dataset_changes, [("diffexp__lfc_cutoff", 0.05, 0.01)])
-
- def test_check_config_throws_error_if_attr_has_not_been_checked(self):
- config = self.get_config(verbose="true")
- config.complete_config()
- config.check_config()
- config.update_server_config(app__verbose=False)
- with self.assertRaises(ConfigurationError):
- config.check_config()
diff --git a/backend/test/test_czi_hosted/unit/common/config/test_dataset_config.py b/backend/test/test_czi_hosted/unit/common/config/test_dataset_config.py
deleted file mode 100644
index 149e5373..00000000
--- a/backend/test/test_czi_hosted/unit/common/config/test_dataset_config.py
+++ /dev/null
@@ -1,253 +0,0 @@
-import json
-import os
-import tempfile
-
-import unittest
-from unittest.mock import patch
-
-from backend.czi_hosted.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
-from backend.czi_hosted.common.annotations.local_file_csv import AnnotationsLocalFile
-from backend.czi_hosted.common.config.app_config import AppConfig
-from backend.czi_hosted.common.config.base_config import BaseConfig
-from backend.test import PROJECT_ROOT, FIXTURES_ROOT
-
-from backend.common.errors import ConfigurationError
-from backend.test.test_czi_hosted.unit.common.config import ConfigTests
-
-
-class TestDatasetConfig(ConfigTests):
- def setUp(self):
- self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
- self.config = AppConfig()
- self.config.update_server_config(app__flask_secret_key="secret")
- self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
- self.dataset_config = self.config.default_dataset_config
- self.config.complete_config()
- message_list = []
-
- def noop(message):
- message_list.append(message)
-
- messagefn = noop
- self.context = dict(messagefn=messagefn, messages=message_list)
-
- def get_config(self, **kwargs):
- file_name = self.custom_app_config(
- dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
- )
- config = AppConfig()
- config.update_from_config_file(file_name)
- return config
-
- def test_init_datatset_config_sets_vars_from_default_config(self):
- config = AppConfig()
- self.assertEqual(config.default_dataset_config.presentation__max_categories, 1000)
- self.assertEqual(config.default_dataset_config.user_annotations__type, "local_file_csv")
- self.assertEqual(config.default_dataset_config.diffexp__lfc_cutoff, 0.01)
-
- @patch("backend.czi_hosted.common.config.dataset_config.BaseConfig.validate_correct_type_of_configuration_attribute")
- def test_complete_config_checks_all_attr(self, mock_check_attrs):
- mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
- self.dataset_config.complete_config(self.context)
- self.assertEqual(mock_check_attrs.call_count, 19)
-
- def test_app_sets_script_vars(self):
- config = self.get_config(scripts=["path/to/script"])
- config.default_dataset_config.handle_app()
-
- self.assertEqual(config.default_dataset_config.app__scripts, [{"src": "path/to/script"}])
-
- config = self.get_config(scripts=[{"src": "path/to/script", "more": "different/script/path"}])
- config.default_dataset_config.handle_app()
- self.assertEqual(
- config.default_dataset_config.app__scripts, [{"src": "path/to/script", "more": "different/script/path"}]
- )
-
- config = self.get_config(scripts=["path/to/script", "different/script/path"])
- config.default_dataset_config.handle_app()
- # TODO @madison -- is this the desired functionality?
- self.assertEqual(
- config.default_dataset_config.app__scripts, [{"src": "path/to/script"}, {"src": "different/script/path"}]
- )
-
- config = self.get_config(scripts=[{"more": "different/script/path"}])
- with self.assertRaises(ConfigurationError):
- config.default_dataset_config.handle_app()
-
- def test_handle_user_annotations_ensures_auth_is_enabled_with_valid_auth_type(self):
- config = self.get_config(enable_users_annotations="true", authentication_enable="false")
- config.server_config.complete_config(self.context)
- with self.assertRaises(ConfigurationError):
- config.default_dataset_config.handle_user_annotations(self.context)
-
- config = self.get_config(enable_users_annotations="true", authentication_enable="true", auth_type="pretend")
- with self.assertRaises(ConfigurationError):
- config.server_config.complete_config(self.context)
-
- def test_handle_user_annotations__adds_warning_message_if_annotation_vars_set_when_annotations_disabled(self):
- config = self.get_config(
- enable_users_annotations="false", authentication_enable="false", db_uri="shouldnt/be/set"
- )
- config.default_dataset_config.handle_user_annotations(self.context)
-
- self.assertEqual(self.context["messages"], ["Warning: db_uri ignored as annotations are disabled."])
-
- @patch("backend.czi_hosted.common.config.dataset_config.DbUtils")
- def test_handle_user_annotations__instantiates_user_annotations_class_correctly(self, mock_db_utils):
- mock_db_utils.return_value = "123"
- config = self.get_config(
- enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv"
- )
- config.server_config.complete_config(self.context)
- config.default_dataset_config.handle_user_annotations(self.context)
- self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsLocalFile)
-
- config = self.get_config(
- enable_users_annotations="true",
- authentication_enable="true",
- annotation_type="hosted_tiledb_array",
- db_uri="gotta/set/this",
- hosted_file_directory="and/this",
- )
- config.server_config.complete_config(self.context)
- config.default_dataset_config.handle_user_annotations(self.context)
- self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsHostedTileDB)
-
- config = self.get_config(
- enable_users_annotations="true", authentication_enable="true", annotation_type="NOT_REAL"
- )
- config.server_config.complete_config(self.context)
- with self.assertRaises(ConfigurationError):
- config.default_dataset_config.handle_user_annotations(self.context)
-
- def test_handle_local_file_csv_annotations__sets_dir_if_not_passed_in(self):
- config = self.get_config(
- enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv"
- )
- config.server_config.complete_config(self.context)
- config.default_dataset_config.handle_local_file_csv_annotations()
- self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsLocalFile)
- cwd = os.getcwd()
- self.assertEqual(config.default_dataset_config.user_annotations._get_output_dir(), cwd)
-
- def test_handle_diffexp__raises_warning_for_large_datasets(self):
- config = self.get_config(lfc_cutoff=0.02, enable_difexp="true", top_n=15)
- config.server_config.complete_config(self.context)
- config.default_dataset_config.handle_diffexp(self.context)
- self.assertEqual(len(self.context["messages"]), 0)
-
- def test_multi_dataset(self):
- config = AppConfig()
- # test for illegal url_dataroots
- for illegal in ("../b", "!$*", "\\n", "", "(bad)"):
- config.update_server_config(
- app__flask_secret_key="secret",
- multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": f"{PROJECT_ROOT}/example-dataset"}},
- )
- with self.assertRaises(ConfigurationError):
- config.complete_config()
-
- # test for legal url_dataroots
- for legal in ("d", "this.is-okay_", "a/b"):
- config.update_server_config(
- app__flask_secret_key="secret",
- multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": f"{PROJECT_ROOT}/example-dataset"}},
- )
- config.complete_config()
-
- # test that multi dataroots work end to end
- config.update_server_config(
- app__flask_secret_key="secret",
- multi_dataset__dataroot=dict(
- s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"),
- s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"),
- s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"),
- ),
- )
-
- # Change this default to test if the dataroot overrides below work.
- config.update_default_dataset_config(app__about_legal_tos="tos_default.html")
-
- # specialize the configs for set1
- config.add_dataroot_config(
- "s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html"
- )
-
- # specialize the configs for set2
- config.add_dataroot_config(
- "s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html"
- )
-
- # no specializations for set3 (they get the default dataset config)
- config.complete_config()
-
- server = self.create_app(config)
-
- server.testing = True
- session = server.test_client()
-
- response = session.get("/set1/1/2/pbmc3k.h5ad/api/v0.2/config")
- data_config = json.loads(response.data)
-
- assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
- assert data_config["config"]["parameters"]["annotations"] is False
- assert data_config["config"]["parameters"]["disable-diffexp"] is False
- assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html"
-
- response = session.get("/set2/pbmc3k.cxg/api/v0.2/config")
- data_config = json.loads(response.data)
- assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
- assert data_config["config"]["parameters"]["annotations"] is True
- assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html"
-
- response = session.get("/set3/pbmc3k.cxg/api/v0.2/config")
- data_config = json.loads(response.data)
- assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
- assert data_config["config"]["parameters"]["annotations"] is True
- assert data_config["config"]["parameters"]["disable-diffexp"] is False
- assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html"
-
- response = session.get("/health")
-
- assert json.loads(response.data)["status"] == "pass"
-
- def test_configfile_with_specialization(self):
- # test that per_dataset_config config load the default config, then the specialized config
-
- with tempfile.TemporaryDirectory() as tempdir:
- configfile = os.path.join(tempdir, "config.yaml")
- with open(configfile, "w") as fconfig:
- config = """
- server:
- multi_dataset:
- dataroot:
- test:
- base_url: test
- dataroot: fake_dataroot
-
- dataset:
- user_annotations:
- enable: false
- type: hosted_tiledb_array
- hosted_tiledb_array:
- db_uri: fake_db_uri
- hosted_file_directory: fake_dir
-
- per_dataset_config:
- test:
- user_annotations:
- enable: true
- """
- fconfig.write(config)
-
- app_config = AppConfig()
- app_config.update_from_config_file(configfile)
-
- test_config = app_config.dataroot_config["test"]
-
- # test config from default
- self.assertEqual(test_config.user_annotations__type, "hosted_tiledb_array")
- self.assertEqual(test_config.user_annotations__hosted_tiledb_array__db_uri, "fake_db_uri")
-
- # test config from specialization
- self.assertTrue(test_config.user_annotations__enable)
diff --git a/backend/test/test_czi_hosted/unit/common/config/test_external_config.py b/backend/test/test_czi_hosted/unit/common/config/test_external_config.py
deleted file mode 100644
index 6db40023..00000000
--- a/backend/test/test_czi_hosted/unit/common/config/test_external_config.py
+++ /dev/null
@@ -1,243 +0,0 @@
-import json
-import os
-from unittest.mock import patch
-
-import yaml
-
-from backend.common.errors import ConfigurationError
-from backend.czi_hosted.common.config.app_config import AppConfig
-from backend.common.utils.type_conversion_utils import convert_string_to_value
-from backend.test import FIXTURES_ROOT
-from backend.test.test_czi_hosted.unit.common.config import ConfigTests
-
-
-class TestExternalConfig(ConfigTests):
- def test_type_convert(self):
- # The values from environment variables and aws secrets are returned as strings.
- # These values need to be converted to the proper types.
-
- self.assertEqual(convert_string_to_value("1"), int(1))
- self.assertEqual(convert_string_to_value("1.1"), float(1.1))
- self.assertEqual(convert_string_to_value("string"), "string")
- self.assertEqual(convert_string_to_value("true"), True)
- self.assertEqual(convert_string_to_value("True"), True)
- self.assertEqual(convert_string_to_value("false"), False)
- self.assertEqual(convert_string_to_value("False"), False)
- self.assertEqual(convert_string_to_value("null"), None)
- self.assertEqual(convert_string_to_value("None"), None)
- self.assertEqual(convert_string_to_value("{'a':10, 'b':'string'}"), dict(a=int(10), b="string"))
-
- def test_environment_variable(self):
- configfile = self.custom_external_config(
- environment=[
- dict(name="DATAPATH", path=["server", "single_dataset", "datapath"], required=True),
- dict(name="DIFFEXP", path=["dataset", "diffexp", "enable"], required=True),
- ],
- config_file_name="environment_external_config.yaml",
- )
-
- env = os.environ
- env["DATAPATH"] = f"{FIXTURES_ROOT}/pbmc3k.cxg"
- env["DIFFEXP"] = "False"
- config = AppConfig()
- config.update_from_config_file(configfile)
- config.update_server_config(app__flask_secret_key="123 magic")
-
- server = self.create_app(config)
-
- server.testing = True
- session = server.test_client()
-
- response = session.get("/api/v0.2/config")
- data_config = json.loads(response.data)
- self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k")
- self.assertTrue(data_config["config"]["parameters"]["disable-diffexp"])
-
- os.environ["DATAPATH"] = f"{FIXTURES_ROOT}/a95c59b4-7f5d-4b80-ad53-a694834ca18b.h5ad"
- os.environ["DIFFEXP"] = "True"
-
- server= self.create_app(config)
-
- server.testing = True
- session = server.test_client()
-
- # session = requests.Session()
- response = session.get("/api/v0.2/config")
- data_config = json.loads(response.data)
- self.assertEqual(data_config["config"]["displayNames"]["dataset"], "a95c59b4-7f5d-4b80-ad53-a694834ca18b")
- self.assertFalse(data_config["config"]["parameters"]["disable-diffexp"])
-
- def test_environment_variable_errors(self):
- # no name
- app_config = AppConfig()
- app_config.external_config.environment = [dict(required=True, path=["this", "is", "a", "path"])]
- with self.assertRaises(ConfigurationError) as config_error:
- app_config.complete_config()
- self.assertEqual(config_error.exception.message, "environment: 'name' is missing")
-
- # required has wrong type
- app_config = AppConfig()
- app_config.external_config.environment = [
- dict(name="myenvar", required="optional", path=["this", "is", "a", "path"])
- ]
- with self.assertRaises(ConfigurationError) as config_error:
- app_config.complete_config()
- self.assertEqual(config_error.exception.message, "environment: 'required' must be a bool")
-
- # no path
- app_config = AppConfig()
- app_config.external_config.environment = [dict(name="myenvar", required=True)]
- with self.assertRaises(ConfigurationError) as config_error:
- app_config.complete_config()
- self.assertEqual(config_error.exception.message, "environment: 'path' is missing")
-
- # required environment variable is not set
- app_config = AppConfig()
- app_config.external_config.environment = [
- dict(name="THIS_ENV_IS_NOT_SET", required=True, path=["this", "is", "a", "path"])
- ]
- with self.assertRaises(ConfigurationError) as config_error:
- app_config.complete_config()
- self.assertEqual(config_error.exception.message, "required environment variable 'THIS_ENV_IS_NOT_SET' not set")
-
- @patch("backend.czi_hosted.common.config.external_config.get_secret_key")
- def test_aws_secrets_manager(self, mock_get_secret_key):
- mock_get_secret_key.return_value = {
- "oauth_client_secret": "mock_oauth_secret",
- "db_uri": "mock_db_uri",
- }
- configfile = self.custom_external_config(
- aws_secrets_manager_region="us-west-2",
- aws_secrets_manager_secrets=[
- dict(
- name="my_secret",
- values=[
- dict(key="flask_secret_key", path=["server", "app", "flask_secret_key"], required=False),
- dict(
- key="db_uri",
- path=["dataset", "user_annotations", "hosted_tiledb_array", "db_uri"],
- required=True,
- ),
- dict(
- key="oauth_client_secret",
- path=["server", "authentication", "params_oauth", "client_secret"],
- required=True,
- ),
- ],
- )
- ],
- config_file_name="secret_external_config.yaml",
- )
-
- app_config = AppConfig()
- app_config.update_from_config_file(configfile)
- app_config.server_config.single_dataset__datapath = f"{FIXTURES_ROOT}/pbmc3k.cxg"
- app_config.server_config.app__flask_secret_key = "original"
- app_config.server_config.single_dataset__datapath = f"{FIXTURES_ROOT}/pbmc3k.cxg"
-
- app_config.complete_config()
-
- self.assertEqual(app_config.server_config.app__flask_secret_key, "original")
- self.assertEqual(app_config.server_config.authentication__params_oauth__client_secret, "mock_oauth_secret")
- self.assertEqual(app_config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mock_db_uri")
-
- @patch("backend.czi_hosted.common.config.external_config.get_secret_key")
- def test_aws_secrets_manager_error(self, mock_get_secret_key):
- mock_get_secret_key.return_value = {
- "oauth_client_secret": "mock_oauth_secret",
- "db_uri": "mock_db_uri",
- }
-
- # no region
- app_config = AppConfig()
- app_config.external_config.aws_secrets_manager__region = None
- app_config.external_config.aws_secrets_manager__secrets = [
- dict(name="secret1", values=[dict(key="key1", required=True, path=["this", "is", "my", "path"])])
- ]
- with self.assertRaises(ConfigurationError) as config_error:
- app_config.complete_config()
- self.assertEqual(
- config_error.exception.message,
- "Invalid type for attribute: aws_secrets_manager__region, expected type str, got NoneType",
- )
-
- # missing secret name
- app_config = AppConfig()
- app_config.external_config.aws_secrets_manager__region = "us-west-2"
- app_config.external_config.aws_secrets_manager__secrets = [
- dict(values=[dict(key="db_uri", required=True, path=["this", "is", "my", "path"])])
- ]
- with self.assertRaises(ConfigurationError) as config_error:
- app_config.complete_config()
- self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'name' is missing")
-
- # secret name wrong type
- app_config = AppConfig()
- app_config.external_config.aws_secrets_manager__region = "us-west-2"
- app_config.external_config.aws_secrets_manager__secrets = [
- dict(name=1, values=[dict(key="db_uri", required=True, path=["this", "is", "my", "path"])])
- ]
- with self.assertRaises(ConfigurationError) as config_error:
- app_config.complete_config()
- self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'name' must be a string")
-
- # missing values name
- app_config = AppConfig()
- app_config.external_config.aws_secrets_manager__region = "us-west-2"
- app_config.external_config.aws_secrets_manager__secrets = [dict(name="mysecret")]
- with self.assertRaises(ConfigurationError) as config_error:
- app_config.complete_config()
- self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'values' is missing")
-
- # values wrong type
- app_config = AppConfig()
- app_config.external_config.aws_secrets_manager__region = "us-west-2"
- app_config.external_config.aws_secrets_manager__secrets = [
- dict(name="mysecret", values=dict(key="db_uri", required=True, path=["this", "is", "my", "path"]))
- ]
- with self.assertRaises(ConfigurationError) as config_error:
- app_config.complete_config()
- self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'values' must be a list")
-
- # entry missing key
- app_config = AppConfig()
- app_config.external_config.aws_secrets_manager__region = "us-west-2"
- app_config.external_config.aws_secrets_manager__secrets = [
- dict(name="mysecret", values=[dict(required=True, path=["this", "is", "my", "path"])])
- ]
- with self.assertRaises(ConfigurationError) as config_error:
- app_config.complete_config()
- self.assertEqual(config_error.exception.message, "missing 'key' in secret values: mysecret")
-
- # entry required is wrong type
- app_config = AppConfig()
- app_config.external_config.aws_secrets_manager__region = "us-west-2"
- app_config.external_config.aws_secrets_manager__secrets = [
- dict(name="mysecret", values=[dict(key="db_uri", required="optional", path=["this", "is", "my", "path"])])
- ]
- with self.assertRaises(ConfigurationError) as config_error:
- app_config.complete_config()
- self.assertEqual(config_error.exception.message, "wrong type for 'required' in secret values: mysecret")
-
- # entry missing path
- app_config = AppConfig()
- app_config.external_config.aws_secrets_manager__region = "us-west-2"
- app_config.external_config.aws_secrets_manager__secrets = [
- dict(name="mysecret", values=[dict(key="db_uri", required=True)])
- ]
- with self.assertRaises(ConfigurationError) as config_error:
- app_config.complete_config()
- self.assertEqual(config_error.exception.message, "missing 'path' in secret values: mysecret")
-
- # secret missing required key
- app_config = AppConfig()
- app_config.external_config.aws_secrets_manager__region = "us-west-2"
- app_config.external_config.aws_secrets_manager__secrets = [
- dict(
- name="mysecret",
- values=[dict(key="KEY_DOES_NOT_EXIST", required=True, path=["this", "is", "a", "path"])],
- )
- ]
- with self.assertRaises(ConfigurationError) as config_error:
- app_config.complete_config()
- self.assertEqual(config_error.exception.message, "required secret 'mysecret:KEY_DOES_NOT_EXIST' not set")
diff --git a/backend/test/test_czi_hosted/unit/common/config/test_server_config.py b/backend/test/test_czi_hosted/unit/common/config/test_server_config.py
deleted file mode 100644
index 674656d7..00000000
--- a/backend/test/test_czi_hosted/unit/common/config/test_server_config.py
+++ /dev/null
@@ -1,324 +0,0 @@
-import json
-import os
-import unittest
-from unittest.mock import patch
-
-
-from backend.czi_hosted.common.config.base_config import BaseConfig
-from backend.common.utils.utils import find_available_port
-from backend.test import PROJECT_ROOT, FIXTURES_ROOT
-
-from backend.czi_hosted.common.config.app_config import AppConfig
-from backend.common.errors import ConfigurationError
-from backend.test.test_czi_hosted.unit.common.config import ConfigTests
-
-
-class TestServerConfig(ConfigTests):
- def setUp(self):
- self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
- self.config = AppConfig()
- self.config.update_server_config(app__flask_secret_key="secret")
- self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
- self.server_config = self.config.server_config
- self.config.complete_config()
-
- message_list = []
-
- def noop(message):
- message_list.append(message)
-
- messagefn = noop
- self.context = dict(messagefn=messagefn, messages=message_list)
-
- def get_config(self, **kwargs):
- file_name = self.custom_app_config(
- dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
- )
- config = AppConfig()
- config.update_from_config_file(file_name)
- return config
-
- def test_init_raises_error_if_default_config_is_invalid(self):
- invalid_config = self.get_config(port="not_valid")
- with self.assertRaises(ConfigurationError):
- invalid_config.complete_config()
-
- @patch("backend.czi_hosted.common.config.server_config.BaseConfig.validate_correct_type_of_configuration_attribute")
- def test_complete_config_checks_all_attr(self, mock_check_attrs):
- mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
- self.server_config.complete_config(self.context)
- self.assertEqual(mock_check_attrs.call_count, 41)
-
- def test_handle_app__throws_error_if_port_doesnt_exist(self):
- config = self.get_config(port=99999999)
- with self.assertRaises(ConfigurationError):
- config.server_config.handle_app(self.context)
-
- @patch("backend.czi_hosted.common.config.server_config.discover_s3_region_name")
- def test_handle_data_locator_works_for_default_types(self, mock_discover_region_name):
- mock_discover_region_name.return_value = None
- # Default config
- self.assertEqual(self.config.server_config.data_locator__s3__region_name, None)
- # hard coded
- config = self.get_config()
- self.assertEqual(config.server_config.data_locator__s3__region_name, "us-east-1")
- # incorrectly formatted
- dataroot = {
- "d1": {"base_url": "set1", "dataroot": "/path/to/set1_datasets/"},
- "d2": {"base_url": "set2/subdir", "dataroot": "s3://shouldnt/work"},
- }
- file_name = self.custom_app_config(
- dataroot=dataroot, config_file_name=self.config_file_name, data_locater_region_name="true"
- )
- config = AppConfig()
- config.update_from_config_file(file_name)
- with self.assertRaises(ConfigurationError):
- config.server_config.handle_data_locator()
-
- @patch("backend.czi_hosted.common.config.server_config.discover_s3_region_name")
- def test_handle_data_locator_can_read_from_dataroot(self, mock_discover_region_name):
- mock_discover_region_name.return_value = "us-west-2"
- dataroot = {
- "d1": {"base_url": "set1", "dataroot": "/path/to/set1_datasets/"},
- "d2": {"base_url": "set2/subdir", "dataroot": "s3://hosted-cellxgene-dev"},
- }
- file_name = self.custom_app_config(
- dataroot=dataroot, config_file_name=self.config_file_name, data_locater_region_name="true"
- )
- config = AppConfig()
- config.update_from_config_file(file_name)
- config.server_config.handle_data_locator()
- self.assertEqual(config.server_config.data_locator__s3__region_name, "us-west-2")
- mock_discover_region_name.assert_called_once_with("s3://hosted-cellxgene-dev")
-
- def test_handle_app___can_use_envar_port(self):
- config = self.get_config(port=24)
- self.assertEqual(config.server_config.app__port, 24)
-
- # Note if the port is set in the config file it will NOT be overwritten by a different envvar
- os.environ["CXG_SERVER_PORT"] = "4008"
- self.config = AppConfig()
- self.config.update_server_config(app__flask_secret_key="secret")
- self.config.server_config.handle_app(self.context)
- self.assertEqual(self.config.server_config.app__port, 4008)
- del os.environ["CXG_SERVER_PORT"]
-
- def test_handle_app__can_get_secret_key_from_envvar_or_config_file_with_envvar_given_preference(self):
- config = self.get_config(flask_secret_key="KEY_FROM_FILE")
- self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_FILE")
-
- os.environ["CXG_SECRET_KEY"] = "KEY_FROM_ENV"
- config.external_config.handle_environment(self.context)
- self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_ENV")
-
- def test_handle_app__sets_web_base_url(self):
- config = self.get_config(web_base_url="anything.com")
- self.assertEqual(config.server_config.app__web_base_url, "anything.com")
-
- def test_handle_auth__gets_client_secret_from_envvars_or_config_with_envvars_given_preference(self):
- config = self.get_config(client_secret="KEY_FROM_FILE")
- config.server_config.handle_authentication()
- self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "KEY_FROM_FILE")
-
- os.environ["CXG_OAUTH_CLIENT_SECRET"] = "KEY_FROM_ENV"
- config.external_config.handle_environment(self.context)
-
- self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "KEY_FROM_ENV")
-
- def test_handle_data_source__errors_when_passed_zero_or_two_dataroots(self):
- file_name = self.custom_app_config(
- dataroot=f"{FIXTURES_ROOT}",
- config_file_name="two_data_roots.yml",
- dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad",
- )
- config = AppConfig()
- config.update_from_config_file(file_name)
- with self.assertRaises(ConfigurationError):
- config.server_config.handle_data_source()
-
- file_name = self.custom_app_config(config_file_name="zero_roots.yml")
- config = AppConfig()
- config.update_from_config_file(file_name)
- with self.assertRaises(ConfigurationError):
- config.server_config.handle_data_source()
-
- @unittest.skip("skip when running in github action")
- def test_get_api_base_url_works(self):
- # test the api_base_url feature, and that it can contain a path
- config = AppConfig()
- backend_port = find_available_port("localhost", 10000)
- config.update_server_config(
- app__flask_secret_key="secret",
- app__api_base_url=f"http://localhost:{backend_port}/additional/path",
- multi_dataset__dataroot=f"{PROJECT_ROOT}/example-dataset",
- multi_dataset__allowed_matrix_types=["cxg"],
- )
-
- config.complete_config()
- server = self.create_app(config)
- server.testing = True
- session = server.test_client()
- response = session.get(f"/additional/path/d/pbmc3k.h5ad/api/v0.2/config")
-
- self.assertEqual(response.status_code, 200)
- data_config = json.loads(response.data)
- self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k")
-
- # test the health check at the correct url
- response = session.get(f"/additional/path/health")
- assert json.loads(response.data)["status"] == "pass"
-
- def test_get_web_base_url_works(self):
- config = self.get_config(web_base_url="www.thisisawebsite.com")
- web_base_url = config.server_config.get_web_base_url()
- self.assertEqual(web_base_url, "www.thisisawebsite.com")
-
- config = self.get_config(web_base_url="local", port=12)
- web_base_url = config.server_config.get_web_base_url()
- self.assertEqual(web_base_url, "http://localhost:12")
-
- config = self.get_config(web_base_url="www.thisisawebsite.com/")
- web_base_url = config.server_config.get_web_base_url()
- self.assertEqual(web_base_url, "www.thisisawebsite.com")
-
- config = self.get_config(api_base_url="www.api_base.com/")
- web_base_url = config.server_config.get_web_base_url()
- self.assertEqual(web_base_url, "www.api_base.com")
-
- def test_config_for_single_dataset(self):
- file_name = self.custom_app_config(
- config_file_name="single_dataset.yml", dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k.cxg"
- )
- config = AppConfig()
- config.update_from_config_file(file_name)
- config.server_config.handle_single_dataset(self.context)
- self.assertIsNotNone(config.server_config.matrix_data_cache_manager)
-
- file_name = self.custom_app_config(
- config_file_name="single_dataset_with_about.yml",
- about="www.cziscience.com",
- dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k.cxg",
- )
- config = AppConfig()
- config.update_from_config_file(file_name)
- with self.assertRaises(ConfigurationError):
- config.server_config.handle_single_dataset(self.context)
-
- def test_multi_dataset_raises_error_for_illegal_routes(self):
- # test for illegal url_dataroots
- for illegal in ("../b", "!$*", "\\n", "", "(bad)"):
- self.config.update_server_config(
- multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": f"{PROJECT_ROOT}/example-dataset"}}
- )
- with self.assertRaises(ConfigurationError):
- self.config.complete_config()
-
- def test_multidataset_works_for_legal_routes(self):
- # test for legal url_dataroots
- for legal in ("d", "this.is-okay_", "a/b"):
- self.config.update_server_config(
- multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": f"{PROJECT_ROOT}/example-dataset"}}
- )
- self.config.complete_config()
-
- @patch("backend.czi_hosted.app.app.render_template")
- def test_mulitdatasets_work_e2e(self, mock_render_template):
- mock_render_template.return_value = "something"
- # test that multi dataroots work end to end
- self.config.update_server_config(
- multi_dataset__dataroot=dict(
- s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"),
- s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"),
- s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"),
- )
- )
-
- # Change this default to test if the dataroot overrides below work.
- self.config.update_default_dataset_config(app__about_legal_tos="tos_default.html")
-
- # specialize the configs for set1
- self.config.add_dataroot_config(
- "s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html"
- )
-
- # specialize the configs for set2
- self.config.add_dataroot_config(
- "s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html"
- )
-
- # no specializations for set3 (they get the default dataset config)
- self.config.complete_config()
-
- server = self.create_app(self.config)
- server.auth.requires_client_login = lambda: False
- server.testing = True
- session = server.test_client()
-
- response = session.get(f"/set1/1/2/pbmc3k.h5ad/api/v0.2/config")
-
- data_config = json.loads(response.data)
- assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
- assert data_config["config"]["parameters"]["annotations"] is False
- assert data_config["config"]["parameters"]["disable-diffexp"] is False
- assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html"
-
- response = session.get("/set2/pbmc3k.cxg/api/v0.2/config")
-
- data_config = json.loads(response.data)
- assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
- assert data_config["config"]["parameters"]["annotations"] is True
- assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html"
-
- response = session.get("/set3/pbmc3k.cxg/api/v0.2/config")
- data_config = json.loads(response.data)
- assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
- assert data_config["config"]["parameters"]["annotations"] is True
- assert data_config["config"]["parameters"]["disable-diffexp"] is False
- assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html"
-
- response = session.get("/health")
- assert json.loads(response.data)["status"] == "pass"
-
- # access a dataset (no slash)
- response = session.get("/set2/pbmc3k.cxg")
- self.assertEqual(response.status_code, 200)
-
- # access a dataset (with slash)
- response = session.get("/set2/pbmc3k.cxg/")
- self.assertEqual(response.status_code, 200)
-
- @patch("backend.czi_hosted.common.config.server_config.diffexp_tiledb.set_config")
- def test_handle_diffexp(self, mock_tiledb_config):
- custom_config_file = self.custom_app_config(
- dataroot=f"{FIXTURES_ROOT}",
- cpu_multiplier=3,
- diffexp_max_workers=1,
- target_workunit=4,
- config_file_name=self.config_file_name,
- )
- config = AppConfig()
- config.update_from_config_file(custom_config_file)
- config.server_config.handle_diffexp()
- # called with the min of diffexp_max_workers and cpus*cpu_multiplier
- mock_tiledb_config.assert_called_once_with(1, 4)
-
- @patch("backend.czi_hosted.data_cxg.cxg_adaptor.CxgAdaptor.set_tiledb_context")
- def test_handle_adaptor(self, mock_tiledb_context):
- custom_config = self.custom_app_config(
- dataroot=f"{FIXTURES_ROOT}", cxg_tile_cache_size=10, cxg_num_reader_threads=2
- )
- config = AppConfig()
- config.update_from_config_file(custom_config)
- config.server_config.handle_adaptor()
- mock_tiledb_context.assert_called_once_with(
- {"sm.tile_cache_size": 10, "sm.num_reader_threads": 2, "vfs.s3.region": "us-east-1"}
- )
-
- def test_test_auth_only_in_insecure(self):
-
- config = self.get_config(auth_type="test")
- with self.assertRaises(ConfigurationError):
- config.complete_config()
-
- config.update_server_config(authentication__insecure_test_environment=True)
- config.complete_config()
diff --git a/backend/test/test_czi_hosted/unit/common/test_api.py b/backend/test/test_czi_hosted/unit/common/test_api.py
deleted file mode 100644
index 6072ee53..00000000
--- a/backend/test/test_czi_hosted/unit/common/test_api.py
+++ /dev/null
@@ -1,505 +0,0 @@
-import json
-import os
-import time
-from http import HTTPStatus
-import hashlib
-
-import requests
-
-
-from backend.czi_hosted.common.config.app_config import AppConfig
-from backend.test import decode_fbs
-from backend.test.fixtures.fixtures import pbmc3k_colors
-from backend.test.test_czi_hosted.unit import BaseTest, skip_if
-
-BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
-
-class EndPoints(BaseTest):
- @classmethod
- def setUpClass(cls, app_config=None):
- super().setUpClass(app_config)
- cls.app.testing = True
- cls.client = cls.app.test_client()
- os.environ["SKIP_STATIC"] = "True"
- for i in range(90):
- try:
- result = cls.client.get(f"{cls.TEST_URL_BASE}schema")
- cls.schema = json.loads(result.data)
- except requests.exceptions.ConnectionError:
- time.sleep(1)
-
- def test_initialize(self):
- endpoint = "schema"
- url = f"{self.TEST_URL_BASE}{endpoint}"
- result = self.client.get(url)
- self.assertEqual(result.status_code, HTTPStatus.OK)
- self.assertEqual(result.headers["Content-Type"], "application/json")
- result_data = json.loads(result.data)
- self.assertEqual(result_data["schema"]["dataframe"]["nObs"], 2638)
- self.assertEqual(len(result_data["schema"]["annotations"]["obs"]), 2)
- self.assertEqual(
- len(result_data["schema"]["annotations"]["obs"]["columns"]), 5
- )
-
- def test_config(self):
- endpoint = "config"
- url = f"{self.TEST_URL_BASE}{endpoint}"
- result = self.client.get(url)
- self.assertEqual(result.status_code, HTTPStatus.OK)
- self.assertEqual(result.headers["Content-Type"], "application/json")
- result_data = json.loads(result.data)
- self.assertIn("library_versions", result_data["config"])
- self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k")
-
- def test_get_layout_fbs(self):
- endpoint = "layout/obs"
- url = f"{self.TEST_URL_BASE}{endpoint}"
- header = {"Accept": "application/octet-stream"}
- result = self.client.get(url, headers=header)
- self.assertEqual(result.status_code, HTTPStatus.OK)
- self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
- df = decode_fbs.decode_matrix_FBS(result.data)
- self.assertEqual(df["n_rows"], 2638)
- self.assertEqual(df["n_cols"], 8)
- self.assertIsNotNone(df["columns"])
- self.assertSetEqual(
- set(df["col_idx"]),
- {"pca_0", "pca_1", "tsne_0", "tsne_1", "umap_0", "umap_1", "draw_graph_fr_0", "draw_graph_fr_1"},
- )
- self.assertIsNone(df["row_idx"])
- self.assertEqual(len(df["columns"]), df["n_cols"])
-
- def test_bad_filter(self):
- endpoint = "data/var"
- url = f"{self.TEST_URL_BASE}{endpoint}"
- header = {"Accept": "application/octet-stream"}
- result = self.client.put(url, headers=header, json=BAD_FILTER)
- self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
-
- def test_get_annotations_obs_fbs(self):
- endpoint = "annotations/obs"
- url = f"{self.TEST_URL_BASE}{endpoint}"
- header = {"Accept": "application/octet-stream"}
- result = self.client.get(url, headers=header)
- self.assertEqual(result.status_code, HTTPStatus.OK)
- self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
- df = decode_fbs.decode_matrix_FBS(result.data)
- self.assertEqual(df["n_rows"], 2638)
- self.assertEqual(df["n_cols"], 5)
- self.assertIsNotNone(df["columns"])
- self.assertIsNone(df["row_idx"])
- self.assertEqual(len(df["columns"]), df["n_cols"])
- obs_index_col_name = self.schema["schema"]["annotations"]["obs"]["index"]
- self.assertCountEqual(
- df["col_idx"],
- [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"],
- )
-
- def test_get_annotations_obs_keys_fbs(self):
- endpoint = "annotations/obs"
- query = "annotation-name=n_genes&annotation-name=percent_mito"
- url = f"{self.TEST_URL_BASE}{endpoint}?{query}"
- header = {"Accept": "application/octet-stream"}
- result = self.client.get(url, headers=header)
- self.assertEqual(result.status_code, HTTPStatus.OK)
- self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
- df = decode_fbs.decode_matrix_FBS(result.data)
- self.assertEqual(df["n_rows"], 2638)
- self.assertEqual(df["n_cols"], 2)
- self.assertIsNotNone(df["columns"])
- self.assertIsNone(df["row_idx"])
- self.assertEqual(len(df["columns"]), df["n_cols"])
- self.assertCountEqual(df["col_idx"], ["n_genes", "percent_mito"])
-
- def test_get_annotations_obs_error(self):
- endpoint = "annotations/obs"
- query = "annotation-name=notakey"
- url = f"{self.TEST_URL_BASE}{endpoint}?{query}"
- header = {"Accept": "application/octet-stream"}
- result = self.client.get(url, headers=header)
- self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
-
-# TEMP: Testing count 15 to match hardcoded values for diffexp
-# TODO(#1281): Switch back to dynamic values
- def test_diff_exp(self):
- endpoint = "diffexp/obs"
- url = f"{self.TEST_URL_BASE}{endpoint}"
- params = {
- "mode": "topN",
- "set1": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["NK cells"]}]}}},
- "set2": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["CD8 T cells"]}]}}},
- "count": 15,
- }
- result = self.client.post(url, json=params)
- self.assertEqual(result.status_code, HTTPStatus.OK)
- self.assertEqual(result.headers["Content-Type"], "application/json")
- result_data = json.loads(result.data)
- self.assertEqual(len(result_data['positive']), 15)
- self.assertEqual(len(result_data['negative']), 15)
-
- def test_diff_exp_indices(self):
- endpoint = "diffexp/obs"
- url = f"{self.TEST_URL_BASE}{endpoint}"
- params = {
- "mode": "topN",
- "count": 15,
- "set1": {"filter": {"obs": {"index": [[0, 500]]}}},
- "set2": {"filter": {"obs": {"index": [[500, 1000]]}}},
- }
- result = self.client.post(url, json=params)
- self.assertEqual(result.status_code, HTTPStatus.OK)
- self.assertEqual(result.headers["Content-Type"], "application/json")
- result_data = json.loads(result.data)
- self.assertEqual(len(result_data['positive']), 15)
- self.assertEqual(len(result_data['negative']), 15)
-
- def test_get_annotations_var_fbs(self):
- endpoint = "annotations/var"
- url = f"{self.TEST_URL_BASE}{endpoint}"
- header = {"Accept": "application/octet-stream"}
- result = self.client.get(url, headers=header)
- self.assertEqual(result.status_code, HTTPStatus.OK)
- self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
- df = decode_fbs.decode_matrix_FBS(result.data)
- self.assertEqual(df["n_rows"], 1838)
- self.assertEqual(df["n_cols"], 2)
- self.assertIsNotNone(df["columns"])
- self.assertIsNone(df["row_idx"])
- self.assertEqual(len(df["columns"]), df["n_cols"])
- var_index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
- self.assertCountEqual(df["col_idx"], [var_index_col_name, "n_cells"])
-
- def test_get_annotations_var_keys_fbs(self):
- endpoint = "annotations/var"
- query = "annotation-name=n_cells"
- url = f"{self.TEST_URL_BASE}{endpoint}?{query}"
- header = {"Accept": "application/octet-stream"}
- result = self.client.get(url, headers=header)
- self.assertEqual(result.status_code, HTTPStatus.OK)
- self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
- df = decode_fbs.decode_matrix_FBS(result.data)
- self.assertEqual(df["n_rows"], 1838)
- self.assertEqual(df["n_cols"], 1)
- self.assertIsNotNone(df["columns"])
- self.assertIsNone(df["row_idx"])
- self.assertEqual(len(df["columns"]), df["n_cols"])
- self.assertCountEqual(df["col_idx"], ["n_cells"])
-
- def test_get_annotations_var_error(self):
- endpoint = "annotations/var"
- query = "annotation-name=notakey"
- url = f"{self.TEST_URL_BASE}{endpoint}?{query}"
- header = {"Accept": "application/octet-stream"}
- result = self.client.get(url, headers=header)
- self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
-
- def test_data_mimetype_error(self):
- endpoint = "data/var"
- header = {"Accept": "xxx"}
- url = f"{self.TEST_URL_BASE}{endpoint}"
- result = self.client.put(url, headers=header)
- self.assertEqual(result.status_code, HTTPStatus.NOT_ACCEPTABLE)
-
- def test_fbs_default(self):
- endpoint = "data/var"
- url = f"{self.TEST_URL_BASE}{endpoint}"
- headers = {"Accept": "application/octet-stream"}
- result = self.client.put(url, headers=headers)
- self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
-
- filter = {"filter": {"var": {"index": [0, 1, 4]}}}
- result = self.client.put(url, headers=headers, json=filter)
- self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
-
- def test_data_put_fbs(self):
- endpoint = "data/var"
- url = f"{self.TEST_URL_BASE}{endpoint}"
- header = {"Accept": "application/octet-stream"}
- result = self.client.put(url, headers=header)
- self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
-
- def test_data_get_fbs(self):
- endpoint = "data/var"
- url = f"{self.TEST_URL_BASE}{endpoint}"
- header = {"Accept": "application/octet-stream"}
- result = self.client.get(url, headers=header)
- self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
-
- def test_data_put_filter_fbs(self):
- endpoint = "data/var"
- url = f"{self.TEST_URL_BASE}{endpoint}"
- header = {"Accept": "application/octet-stream"}
- filter = {"filter": {"var": {"index": [0, 1, 4]}}}
- result = self.client.put(url, headers=header, json=filter)
- self.assertEqual(result.status_code, HTTPStatus.OK)
- self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
- df = decode_fbs.decode_matrix_FBS(result.data)
- self.assertEqual(df["n_rows"], 2638)
- self.assertEqual(df["n_cols"], 3)
- self.assertIsNotNone(df["columns"])
- self.assertIsNone(df["row_idx"])
- self.assertEqual(len(df["columns"]), df["n_cols"])
- self.assertListEqual(df["col_idx"].tolist(), [0, 1, 4])
-
- def test_data_get_filter_fbs(self):
- index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
- endpoint = "data/var"
- query = f"var:{index_col_name}=SIK1"
- url = f"{self.TEST_URL_BASE}{endpoint}?{query}"
- header = {"Accept": "application/octet-stream"}
- result = self.client.get(url, headers=header)
- self.assertEqual(result.status_code, HTTPStatus.OK)
- self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
- df = decode_fbs.decode_matrix_FBS(result.data)
- self.assertEqual(df["n_rows"], 2638)
- self.assertEqual(df["n_cols"], 1)
-
- def test_data_get_unknown_filter_fbs(self):
- index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
- endpoint = "data/var"
- query = f"var:{index_col_name}=UNKNOWN"
- url = f"{self.TEST_URL_BASE}{endpoint}?{query}"
- header = {"Accept": "application/octet-stream"}
- result = self.client.get(url, headers=header)
- self.assertEqual(result.status_code, HTTPStatus.OK)
- self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
- df = decode_fbs.decode_matrix_FBS(result.data)
- self.assertEqual(df["n_rows"], 2638)
- self.assertEqual(df["n_cols"], 0)
-
- def test_data_put_single_var(self):
- endpoint = "data/var"
- url = f"{self.TEST_URL_BASE}{endpoint}"
- header = {"Accept": "application/octet-stream"}
- index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
- var_filter = {"filter": {"var": {"annotation_value": [{"name": index_col_name, "values": ["RER1"]}]}}}
- result = self.client.put(url, headers=header, json=var_filter)
- self.assertEqual(result.status_code, HTTPStatus.OK)
- self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
- df = decode_fbs.decode_matrix_FBS(result.data)
- self.assertEqual(df["n_rows"], 2638)
- self.assertEqual(df["n_cols"], 1)
-
- def test_colors(self):
- endpoint = "colors"
- url = f"{self.TEST_URL_BASE}{endpoint}"
- result = self.client.get(url)
- self.assertEqual(result.status_code, HTTPStatus.OK)
- self.assertEqual(result.headers["Content-Type"], "application/json")
- result_data = json.loads(result.data)
- self.assertEqual(result_data, pbmc3k_colors)
-
- @skip_if(lambda x: os.getenv("SKIP_STATIC"), "Skip static test when running locally")
- def test_static(self):
- endpoint = "static"
- file = "assets/favicon.ico"
- url = f"{endpoint}/{file}"
- result = self.client.get(url)
- self.assertEqual(result.status_code, HTTPStatus.OK)
-
- def test_genesets_config(self):
- result = self.client.get(f"{self.TEST_URL_BASE}config")
- config_data = json.loads(result.data)
- params = config_data["config"]["parameters"]
- annotations_genesets = params["annotations_genesets"]
- annotations_genesets_readonly = params["annotations_genesets_readonly"]
- annotations_genesets_summary_methods = params["annotations_genesets_summary_methods"]
- self.assertTrue(annotations_genesets)
- self.assertTrue(annotations_genesets_readonly)
- self.assertEqual(annotations_genesets_summary_methods, ["mean"])
-
- def test_get_genesets(self):
- endpoint = "genesets"
- url = f"{self.TEST_URL_BASE}{endpoint}"
- result = self.client.get(url, headers={"Accept": "application/json"})
- self.assertEqual(result.status_code, HTTPStatus.OK)
- self.assertEqual(result.headers["Content-Type"], "application/json")
- result_data = json.loads(result.data)
- self.assertIsNotNone(result_data["genesets"])
-
- def test_get_summaryvar(self):
- index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
- endpoint = "summarize/var"
-
- # single column
- filter = f"var:{index_col_name}=F5"
- query = f"method=mean&{filter}"
- query_hash = hashlib.sha1(query.encode()).hexdigest()
- url = f"{self.TEST_URL_BASE}{endpoint}?{query}"
- header = {"Accept": "application/octet-stream"}
- result = self.client.get(url, headers=header)
- self.assertEqual(result.status_code, HTTPStatus.OK)
- self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
- df = decode_fbs.decode_matrix_FBS(result.data)
- self.assertEqual(df["n_rows"], 2638)
- self.assertEqual(df["n_cols"], 1)
- self.assertEqual(df["col_idx"], [query_hash])
- self.assertAlmostEqual(df["columns"][0][0], -0.110451095)
-
- # multi-column
- col_names = ["F5", "BEB3", "SIK1"]
- filter = "&".join([f"var:{index_col_name}={name}" for name in col_names])
- query = f"method=mean&{filter}"
- query_hash = hashlib.sha1(query.encode()).hexdigest()
- url = f"{self.TEST_URL_BASE}{endpoint}?{query}"
- header = {"Accept": "application/octet-stream"}
- result = self.client.get(url, headers=header)
- self.assertEqual(result.status_code, HTTPStatus.OK)
- self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
- df = decode_fbs.decode_matrix_FBS(result.data)
- self.assertEqual(df["n_rows"], 2638)
- self.assertEqual(df["n_cols"], 1)
- self.assertEqual(df["col_idx"], [query_hash])
- self.assertAlmostEqual(df["columns"][0][0], -0.16628358)
-
- def test_post_summaryvar(self):
- index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
- endpoint = "summarize/var"
- headers = {"Content-Type": "application/x-www-form-urlencoded", "Accept": "application/octet-stream"}
-
- # single column
- filter = f"var:{index_col_name}=F5"
- query = f"method=mean&{filter}"
- query_hash = hashlib.sha1(query.encode()).hexdigest()
- url = f"{self.TEST_URL_BASE}{endpoint}?key={query_hash}"
- result = self.client.post(url, headers=headers, data=query)
-
- self.assertEqual(result.status_code, HTTPStatus.OK)
- self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
- df = decode_fbs.decode_matrix_FBS(result.data)
- self.assertEqual(df["n_rows"], 2638)
- self.assertEqual(df["n_cols"], 1)
- self.assertEqual(df["col_idx"], [query_hash])
- self.assertAlmostEqual(df["columns"][0][0], -0.110451095)
-
- # multi-column
- col_names = ["F5", "BEB3", "SIK1"]
- filter = "&".join([f"var:{index_col_name}={name}" for name in col_names])
- query = f"method=mean&{filter}"
- query_hash = hashlib.sha1(query.encode()).hexdigest()
- url = f"{self.TEST_URL_BASE}{endpoint}?key={query_hash}"
- result = self.client.post(url, headers=headers, data=query)
- self.assertEqual(result.status_code, HTTPStatus.OK)
- self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
- df = decode_fbs.decode_matrix_FBS(result.data)
- self.assertEqual(df["n_rows"], 2638)
- self.assertEqual(df["n_cols"], 1)
- self.assertEqual(df["col_idx"], [query_hash])
- self.assertAlmostEqual(df["columns"][0][0], -0.16628358)
-
-
-
-class EndPointsCxg(EndPoints):
- """Test Case for endpoints"""
- @classmethod
- def setUpClass(cls):
- app_config = AppConfig()
- app_config.update_default_dataset_config(user_annotations__enable=False)
-
- def test_get_genesets_json(self):
- self.app.auth.is_user_authenticated = lambda: True
- endpoint = "genesets"
- url = f"{self.TEST_URL_BASE}{endpoint}"
- result = self.client.get(url, headers={"Accept": "application/json"})
- self.assertEqual(result.status_code, HTTPStatus.OK)
- self.assertEqual(result.headers["Content-Type"], "application/json")
- result_data = json.loads(result.data)
- self.assertIsNotNone(result_data["genesets"])
- self.assertIsNotNone(result_data["tid"])
-
- self.assertEqual(
- result_data,
- {
- "genesets": [
- {
- "genes": [
- {"gene_description": " a gene_description", "gene_symbol": "F5"},
- {"gene_description": "", "gene_symbol": "SUMO3"},
- {"gene_description": "", "gene_symbol": "SRM"},
- ],
- "geneset_description": "a description",
- "geneset_name": "first gene set name",
- },
- {
- "genes": [
- {"gene_description": "", "gene_symbol": "RER1"},
- {"gene_description": "", "gene_symbol": "SIK1"},
- ],
- "geneset_description": "",
- "geneset_name": "second_gene_set",
- },
- {"genes": [], "geneset_description": "", "geneset_name": "third gene set"},
- {"genes": [], "geneset_description": "fourth description", "geneset_name": "fourth_gene_set"},
- {"genes": [], "geneset_description": "", "geneset_name": "fifth_dataset"},
- {
- "genes": [
- {"gene_description": "", "gene_symbol": "ACD"},
- {"gene_description": "", "gene_symbol": "AATF"},
- {"gene_description": "", "gene_symbol": "F5"},
- {"gene_description": "", "gene_symbol": "PIGU"},
- ],
- "geneset_description": "",
- "geneset_name": "summary test",
- },
- {'genes': [], 'geneset_description': '', 'geneset_name': 'geneset_to_delete'},
- {'genes': [], 'geneset_description': '', 'geneset_name': 'geneset_to_edit'},
- {
- 'genes': [],
- 'geneset_description': '',
- 'geneset_name': 'fill_this_geneset'
- },
- {
- 'genes': [{'gene_description': '', 'gene_symbol': 'SIK1'}],
- 'geneset_description': '',
- 'geneset_name': 'empty_this_geneset'
- },
- {
- 'genes': [{'gene_description': '', 'gene_symbol': 'SIK1'}],
- 'geneset_description': '',
- 'geneset_name': 'brush_this_gene'
- }
- ],
- "tid": 0,
- },
- )
-
- def test_get_genesets_csv(self):
- endpoint = "genesets"
- url = f"{self.TEST_URL_BASE}{endpoint}"
- self.app.auth.is_user_authenticated = lambda: True
- result = self.client.get(url, headers={"Accept": "text/csv"})
- self.assertEqual(result.status_code, HTTPStatus.OK)
- self.assertEqual(result.headers["Content-Type"], "text/csv")
- expected_data = """gene_set_name,gene_set_description,gene_symbol,gene_description\r
-first gene set name,a description,F5, a gene_description\r
-first gene set name,a description,SUMO3,\r
-first gene set name,a description,SRM,\r
-second_gene_set,,RER1,\r
-second_gene_set,,SIK1,\r
-third gene set,,,\r
-fourth_gene_set,fourth description,,\r
-fifth_dataset,,,\r
-summary test,,ACD,\r
-summary test,,AATF,\r
-summary test,,F5,\r
-summary test,,PIGU,\r
-geneset_to_delete,,,\r
-geneset_to_edit,,,\r
-fill_this_geneset,,,\r
-empty_this_geneset,,SIK1,\r
-brush_this_gene,,SIK1,\r
-"""
- self.assertEqual(result.data.decode("utf-8"), expected_data)
-
- def test_put_genesets(self):
- endpoint = "genesets"
- url = f"{self.TEST_URL_BASE}{endpoint}"
-
- result = self.client.get(url, headers={"Accept": "application/json"})
- self.assertEqual(result.status_code, HTTPStatus.OK)
-
- test1 = {"tid": 3, "genesets": []}
- result = self.client.put(url, json=test1)
-
- self.assertEqual(result.status_code, HTTPStatus.METHOD_NOT_ALLOWED)
-
diff --git a/backend/test/test_czi_hosted/unit/common/test_corpora.py b/backend/test/test_czi_hosted/unit/common/test_corpora.py
deleted file mode 100644
index edec33bb..00000000
--- a/backend/test/test_czi_hosted/unit/common/test_corpora.py
+++ /dev/null
@@ -1,168 +0,0 @@
-import json
-import shutil
-import tempfile
-import unittest
-from http import HTTPStatus
-
-import anndata
-
-from backend.czi_hosted.common.config.app_config import AppConfig
-from backend.czi_hosted.common.corpora import (
- corpora_get_versions_from_anndata,
- corpora_is_version_supported,
- corpora_get_props_from_anndata,
-)
-from backend.test.test_czi_hosted.unit import BaseTest
-from backend.test import PROJECT_ROOT
-
-VERSION = "v0.2"
-
-
-class CorporaAPITest(unittest.TestCase):
- def test_corpora_get_versions_from_anndata(self):
- adata = self._get_h5ad()
-
- if "version" in adata.uns:
- del adata.uns["version"]
- self.assertIsNone(corpora_get_versions_from_anndata(adata))
-
- # something bogus
- adata.uns["version"] = 99
- self.assertIsNone(corpora_get_versions_from_anndata(adata))
-
- # something legit
- adata.uns["version"] = {"corpora_schema_version": "0.0.0", "corpora_encoding_version": "9.9.9"}
- self.assertEqual(corpora_get_versions_from_anndata(adata), ["0.0.0", "9.9.9"])
-
- def test_corpora_is_version_supported(self):
- self.assertTrue(corpora_is_version_supported("1.0.0", "0.1.0"))
- self.assertFalse(corpora_is_version_supported("0.0.0", "0.1.0"))
- self.assertFalse(corpora_is_version_supported("1.0.0", "0.0.0"))
-
- def test_corpora_get_props_from_anndata(self):
- adata = self._get_h5ad()
-
- if "version" in adata.uns:
- del adata.uns["version"]
- self.assertIsNone(corpora_get_props_from_anndata(adata))
-
- # something bogus
- adata.uns["version"] = 99
- self.assertIsNone(corpora_get_props_from_anndata(adata))
-
- # unsupported version, but missing required values
- adata.uns["version"] = {"corpora_schema_version": "99.0.0", "corpora_encoding_version": "32.1.0"}
- with self.assertRaises(ValueError):
- corpora_get_props_from_anndata(adata)
-
- # legit version, but missing required values
- adata.uns["version"] = {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"}
- with self.assertRaises(KeyError):
- corpora_get_props_from_anndata(adata)
-
- some_fields = {
- "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"},
- "title": "title",
- "layer_descriptions": "layer_descriptions",
- "organism": "organism",
- "organism_ontology_term_id": "organism_ontology_term_id",
- "project_name": "project_name",
- "project_description": "project_description",
- "contributors": json.dumps([{"contributors": "contributors"}]),
- "project_links": json.dumps([{"link_name": "link_name", "link_url": "link_url", "link_type": "SUMMARY"}]),
- }
- for k in some_fields:
- adata.uns[k] = some_fields[k]
- some_fields["contributors"] = json.loads(some_fields["contributors"])
- some_fields["project_links"] = json.loads(some_fields["project_links"])
- self.assertEqual(corpora_get_props_from_anndata(adata), some_fields)
-
- def test_corpora_get_props_from_anndata_v110(self):
- adata = self._get_h5ad()
-
- if "version" in adata.uns:
- del adata.uns["version"]
- self.assertIsNone(corpora_get_props_from_anndata(adata))
-
- # legit version, but missing required values
- adata.uns["version"] = {"corpora_schema_version": "1.1.0", "corpora_encoding_version": "0.1.0"}
- with self.assertRaises(KeyError):
- corpora_get_props_from_anndata(adata)
-
- # Metadata following schema 1.1.0, which removes some fields relative to 1.1.0
- some_110_fields = {
- "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"},
- "title": "title",
- "layer_descriptions": "layer_descriptions",
- "organism": "organism",
- "organism_ontology_term_id": "organism_ontology_term_id",
- }
- for k in some_110_fields:
- adata.uns[k] = some_110_fields[k]
- self.assertEqual(corpora_get_props_from_anndata(adata), some_110_fields)
-
- def _get_h5ad(self):
- return anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
-
-
-class CorporaRESTAPITest(BaseTest):
- """ Confirm endpoints reflect Corpora-specific features """
-
- @classmethod
- def setCorporaFields(cls, path):
- adata = anndata.read_h5ad(path)
- corpora_props = {
- "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"},
- "title": "PBMC3K",
- "contributors": json.dumps([{"name": "name"}]),
- "layer_descriptions": {"X": "raw counts"},
- "organism": "human",
- "organism_ontology_term_id": "unknown",
- "project_name": "test project",
- "project_description": "test description",
- "project_links": json.dumps(
- [{"link_name": "test link", "link_type": "SUMMARY", "link_url": "https://a.u.r.l/"}]
- ),
- "default_embedding": "X_tsne",
- }
- adata.uns.update(corpora_props)
- adata.write(path)
-
- @classmethod
- def setUpClass(cls, app_config=None):
- if not app_config:
- app_config = AppConfig()
- cls.tmp_dir = tempfile.TemporaryDirectory()
- src = f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad"
- dst = f"{cls.tmp_dir.name}/pbmc3k.h5ad"
- shutil.copyfile(src, dst)
- cls.setCorporaFields(dst)
- app_config.update_server_config(single_dataset__datapath=dst)
-
- super().setUpClass(app_config)
- cls.app.testing = True
- cls.client = cls.app.test_client()
-
- def setUp(self):
- self.session = self.client
- self.url_base = "/api/v0.2/"
-
- def test_config(self):
- endpoint = "config"
- url = f"{self.url_base}{endpoint}"
- header = {"Content-Type": "application/json"}
- result = self.session.get(url, headers=header)
- self.assertEqual(result.status_code, HTTPStatus.OK)
- self.assertEqual(result.headers["Content-Type"], "application/json")
-
- result_data = json.loads(result.data)
- self.assertIsInstance(result_data["config"]["corpora_props"], dict)
- self.assertIsInstance(result_data["config"]["parameters"], dict)
-
- corpora_props = result_data["config"]["corpora_props"]
- parameters = result_data["config"]["parameters"]
-
- self.assertEqual(corpora_props["version"]["corpora_schema_version"], "1.0.0")
-
- self.assertEqual(corpora_props["organism"], "human")
- self.assertEqual(parameters["default_embedding"], "tsne")
diff --git a/backend/test/test_czi_hosted/unit/common/test_nan_rest.py b/backend/test/test_czi_hosted/unit/common/test_nan_rest.py
deleted file mode 100644
index a5ba0059..00000000
--- a/backend/test/test_czi_hosted/unit/common/test_nan_rest.py
+++ /dev/null
@@ -1,65 +0,0 @@
-from http import HTTPStatus
-import math
-
-import backend.test.decode_fbs as decode_fbs
-
-from backend.czi_hosted.common.config.app_config import AppConfig
-from backend.test import FIXTURES_ROOT
-from backend.test.test_czi_hosted.unit import BaseTest
-
-VERSION = "v0.2"
-BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
-
-
-class WithNaNs(BaseTest):
- """Test Case for endpoints"""
-
- @classmethod
- def setUpClass(cls):
- app_config = AppConfig()
- app_config.update_server_config(single_dataset__datapath=f"{FIXTURES_ROOT}/nan.h5ad")
- app_config.update_default_dataset_config(user_annotations__enable=True)
- super().setUpClass(app_config)
- cls.app.testing = True
- cls.client = cls.app.test_client()
-
- def setUp(self):
- self.session = self.client
- self.url_base = "api/v0.2/"
-
- def test_initialize(self):
- endpoint = "schema"
- url = f"{self.url_base}{endpoint}"
- result = self.session.get(url)
- self.assertEqual(result.status_code, HTTPStatus.OK)
-
- def test_data(self):
- endpoint = "data/var"
- url = f"{self.url_base}{endpoint}"
- filter = {"filter": {"var": {"index": [[0, 20]]}}}
- header = {"Accept": "application/octet-stream"}
- result = self.session.put(url, headers=header, json=filter)
- self.assertEqual(result.status_code, HTTPStatus.OK)
- self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
- df = decode_fbs.decode_matrix_FBS(result.data)
- self.assertTrue(math.isnan(df["columns"][3][3]))
-
- def test_annotation_obs(self):
- endpoint = "annotations/obs"
- url = f"{self.url_base}{endpoint}"
- header = {"Accept": "application/octet-stream"}
- result = self.session.get(url, headers=header)
- self.assertEqual(result.status_code, HTTPStatus.OK)
- self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
- df = decode_fbs.decode_matrix_FBS(result.data)
- self.assertTrue(math.isnan(df["columns"][2][0]))
-
- def test_annotation_var(self):
- endpoint = "annotations/var"
- url = f"{self.url_base}{endpoint}"
- header = {"Accept": "application/octet-stream"}
- result = self.session.get(url, headers=header)
- self.assertEqual(result.status_code, HTTPStatus.OK)
- self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
- df = decode_fbs.decode_matrix_FBS(result.data)
- self.assertTrue(math.isnan(df["columns"][2][0]))
diff --git a/backend/test/test_czi_hosted/unit/common/test_rest.py b/backend/test/test_czi_hosted/unit/common/test_rest.py
deleted file mode 100644
index 8b8151b3..00000000
--- a/backend/test/test_czi_hosted/unit/common/test_rest.py
+++ /dev/null
@@ -1,79 +0,0 @@
-import unittest
-from urllib.parse import parse_qs
-from werkzeug.datastructures import MultiDict
-from backend.common.errors import FilterError
-from backend.czi_hosted.common.rest import _query_parameter_to_filter
-
-
-def _qsparse(qs):
- """ emulate what Flask/Werkzeug do to our QS """
- return MultiDict(parse_qs(qs))
-
-
-class FilterParseTests(unittest.TestCase):
- """ Test cases for various filter parsing """
-
- def test_queryparam_to_filter_parse(self):
- # categories
- self.assertEqual(
- _query_parameter_to_filter(_qsparse("obs:foo=bar&var:baz=133&var:baz=A&obs:baz=foo")),
- {
- "obs": {"annotation_value": [{"name": "foo", "values": ["bar"]}, {"name": "baz", "values": ["foo"]}]},
- "var": {"annotation_value": [{"name": "baz", "values": ["133", "A"]}]},
- },
- )
-
- # ranges
- self.assertEqual(
- _query_parameter_to_filter(_qsparse("obs:A=1,99&obs:B=*,100&obs:C=0,*")),
- {
- "obs": {
- "annotation_value": [
- {"name": "A", "min": 1, "max": 99.0},
- {"name": "B", "max": 100.0},
- {"name": "C", "min": 0.0},
- ]
- },
- },
- )
-
- # combo
- self.assertEqual(
- _query_parameter_to_filter(_qsparse("var:B=YES&var:A=1,99&var:B=NO")),
- {
- "var": {
- "annotation_value": [
- {"name": "B", "values": ["YES", "NO"]},
- {"name": "A", "min": 1.0, "max": 99.0},
- ]
- },
- },
- )
-
- def test_queryparam_to_filter_escaping(self):
- self.assertEqual(
- _query_parameter_to_filter(_qsparse("obs:var=%2521%252C%253AOK%253D&obs:A%2521=YO")),
- {"obs": {"annotation_value": [{"name": "var", "values": ["!,:OK="]}, {"name": "A!", "values": ["YO"]}]}},
- )
-
- def test_queryparam_to_filter_errors(self):
-
- # should raise FilterError
- filter_errors = [
- "foo=bar", # no axis
- "X=&Y=3", # no value
- "X&Y=3", # no value
- "moo:foo=bar", # bad axis
- "obs:x=1,A", # non-numeric range
- "var:X=1,2&var:X=3,4", # duplicate ranges
- "var:Y=,",
- "var:Y=2,",
- "var:Y=,5",
- "var:Y=*,",
- "var:Y=,*",
- "var:Y=*,*",
- ]
-
- for qs in filter_errors:
- with self.assertRaises(FilterError):
- _query_parameter_to_filter(_qsparse(qs))
diff --git a/backend/test/test_czi_hosted/unit/common/test_writable_annotation.py b/backend/test/test_czi_hosted/unit/common/test_writable_annotation.py
deleted file mode 100644
index 5b2faf8b..00000000
--- a/backend/test/test_czi_hosted/unit/common/test_writable_annotation.py
+++ /dev/null
@@ -1,317 +0,0 @@
-import json
-import shutil
-import unittest
-from os import path, listdir
-from unittest.mock import MagicMock
-
-import numpy as np
-import pandas as pd
-import tiledb
-from flask import Flask
-
-from backend.common.errors import AnnotationCategoryNameError
-from backend.czi_hosted.common.rest import annotations_put_fbs_helper, schema_get_helper
-from backend.czi_hosted.data_common.matrix_loader import MatrixDataType
-from backend.czi_hosted.db.cellxgene_orm import CellxGeneDataset, Annotation
-from backend.test.fixtures.database import TestDatabase
-from backend.test.test_czi_hosted.unit import make_fbs, data_with_tmp_tiledb_annotations, data_with_tmp_annotations
-from backend.test import decode_fbs
-
-TestDatabase()
-
-class auth(object):
- def get_user_id():
- return "1234"
-
- def get_user_name():
- return "person name"
-
-
-class WritableTileDBStoredAnnotationTest(unittest.TestCase):
- def setUp(self):
- self.user_id = "1234"
- self.data, self.tmp_dir, self.annotations = data_with_tmp_tiledb_annotations(MatrixDataType.H5AD)
- self.data.dataset_config.user_annotations = self.annotations
- self.db = self.annotations.db
- self.n_rows = self.data.get_shape()[0]
- self.test_dict = {
- "cat_A": pd.Series(["label_A"] * self.n_rows, dtype="category"),
- "cat_B": pd.Series(["label_B"] * self.n_rows, dtype="category"),
- }
- self.fbs = make_fbs(self.test_dict)
- self.df = pd.DataFrame(self.test_dict)
- self.app = Flask("fake_app")
- self.app.__setattr__("auth", auth)
-
- def tearDown(self):
- shutil.rmtree(self.tmp_dir)
-
- def annotation_put_fbs(self, fbs):
- annotations_put_fbs_helper(self.data, fbs)
- res = json.dumps({"status": "OK"})
- return res
-
- def test_category_name_throws_errors_for_categories_that_cant_be_converted_to_filenames(self):
- with self.app.test_request_context():
- bad_category_names = make_fbs(
- {
- "cat_A": pd.Series(["label_A"] * self.n_rows, dtype="category"),
- "cat/B": pd.Series(["label_B"] * self.n_rows, dtype="category"),
- }
- )
- with self.assertRaises(AnnotationCategoryNameError):
- self.annotation_put_fbs(bad_category_names)
-
- def test_convert_to_pandas__converts_tiledb_to_pandas_df(self):
- with self.app.test_request_context():
- self.annotations.write_labels(self.df, self.data)
- dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id
- annotation = self.db.query_for_most_recent(
- Annotation, [Annotation.user_id == self.user_id, Annotation.dataset_id == str(dataset_id)]
- )
- # retrieve tiledb array
- df = tiledb.open(annotation.tiledb_uri)
- self.assertEqual(type(df), tiledb.array.SparseArray)
-
- # convert to pandas df
- pandas_df = self.annotations.convert_to_pandas_df(df, annotation.schema_hints)
- self.assertEqual(type(pandas_df), pd.DataFrame)
-
- def test_write_labels_creates_a_dataset_if_it_doesnt_exist(self):
- with self.app.test_request_context():
- new_name = "new_dataset/location"
- self.data.get_location = MagicMock(return_value=new_name)
- num_datasets = len(self.db.query([CellxGeneDataset]))
- self.annotation_put_fbs(self.fbs)
- more_datasets = len(self.db.query([CellxGeneDataset]))
- self.assertGreater(more_datasets, num_datasets)
-
- self.assertGreater(len(self.db.query([CellxGeneDataset], [CellxGeneDataset.name == new_name])), 0)
-
- def test_write_labels_links_to_existing_dataset(self):
- with self.app.test_request_context():
- # add dataset to to db
- self.annotation_put_fbs(self.fbs)
-
- num_datasets = len(self.db.query([CellxGeneDataset]))
-
- # create another annotation with the same dataset
- self.annotation_put_fbs(self.fbs)
-
- same_num_datasets = len(self.db.query([CellxGeneDataset]))
-
- self.assertEqual(num_datasets, same_num_datasets)
-
- def test_read_labels_returns_pandas_df(self):
- with self.app.test_request_context():
- self.annotation_put_fbs(self.fbs)
- pandas_df = self.annotations.read_labels(self.data)
- self.assertEqual(type(pandas_df), pd.DataFrame)
-
- def test_read_labels_returns_df_matching_original(self):
- with self.app.test_request_context():
- self.annotation_put_fbs(self.fbs)
- pandas_df = self.annotations.read_labels(self.data)
-
- self.assertEqual(pandas_df.shape, (self.n_rows, 2))
- self.assertEqual(set(pandas_df.columns), {"cat_A", "cat_B"})
-
- self.assertTrue(self.data.original_obs_index.equals(pandas_df.index))
-
- self.assertTrue(np.all(pandas_df["cat_A"] == ["label_A"] * self.n_rows))
- self.assertTrue(np.all(pandas_df["cat_B"] == ["label_B"] * self.n_rows))
-
- def test_error_checks(self):
- # verify that the expected errors are generated
- with self.app.test_request_context():
- n_rows = self.data.get_shape()[0]
- fbs_bad = make_fbs({"louvain": pd.Series(["undefined"] * n_rows, dtype="category")})
-
- # ensure we catch attempt to overwrite non-writable data
- with self.assertRaises(KeyError):
- self.annotation_put_fbs(fbs_bad)
-
- def test_write_labels_stores_df_as_tiledb_array(self):
- with self.app.test_request_context():
- self.annotations.write_labels(self.df, self.data)
- # get uri
- dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id
- annotation = self.db.query_for_most_recent(
- Annotation, [Annotation.user_id == "1234", Annotation.dataset_id == str(dataset_id)]
- )
-
- df = tiledb.open(annotation.tiledb_uri)
- self.assertEqual(type(df), tiledb.array.SparseArray)
-
- def test_remove_categories(self):
- with self.app.test_request_context():
- # update empty category data, which is how annotations are removed
- empty = make_fbs({})
- self.annotation_put_fbs(empty)
-
- # verify that the tiledb uri is an empty string.
- dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id
- annotation = self.db.query_for_most_recent(
- Annotation, [Annotation.user_id == self.user_id, Annotation.dataset_id == str(dataset_id)]
- )
- self.assertEqual(annotation.tiledb_uri, "")
-
- # verify that read_labels returns None
- df = self.annotations.read_labels(self.data)
- self.assertIsNone(df)
-
-
-class WritableAnnotationTest(unittest.TestCase):
- def setUp(self):
- self.data, self.tmp_dir, self.annotations, self.config= data_with_tmp_annotations(MatrixDataType.H5AD)
- self.data.dataset_config.user_annotations = self.annotations
-
- def tearDown(self):
- shutil.rmtree(self.tmp_dir)
-
- def annotation_put_fbs(self, fbs):
- annotations_put_fbs_helper(self.data, fbs)
- res = json.dumps({"status": "OK"})
- return res
-
- def test_error_checks(self):
- # verify that the expected errors are generated
- n_rows = self.data.get_shape()[0]
- fbs_bad = make_fbs({"louvain": pd.Series(["undefined"] * n_rows, dtype="category")})
-
- # ensure we catch attempt to overwrite non-writable data
- with self.assertRaises(KeyError):
- self.annotation_put_fbs(fbs_bad)
-
- def test_write_to_file(self):
- # verify the file is written as expected
- n_rows = self.data.get_shape()[0]
- fbs = make_fbs(
- {
- "cat_A": pd.Series(["label_A"] * n_rows, dtype="category"),
- "cat_B": pd.Series(["label_B"] * n_rows, dtype="category"),
- }
- )
- res = self.annotation_put_fbs(fbs)
- self.assertEqual(res, json.dumps({"status": "OK"}))
- self.assertTrue(path.exists(self.annotations.output_file))
- df = pd.read_csv(self.annotations.output_file, index_col=0, header=0, comment="#")
- self.assertEqual(df.shape, (n_rows, 2))
- self.assertEqual(set(df.columns), {"cat_A", "cat_B"})
- self.assertTrue(self.data.original_obs_index.equals(df.index))
- self.assertTrue(np.all(df["cat_A"] == ["label_A"] * n_rows))
- self.assertTrue(np.all(df["cat_B"] == ["label_B"] * n_rows))
-
- # verify complete overwrite on second attempt, AND rotation occurs
- fbs = make_fbs(
- {
- "cat_A": pd.Series(["label_A1"] * n_rows, dtype="category"),
- "cat_C": pd.Series(["label_C"] * n_rows, dtype="category"),
- }
- )
- res = self.annotation_put_fbs(fbs)
- self.assertEqual(res, json.dumps({"status": "OK"}))
- self.assertTrue(path.exists(self.annotations.output_file))
- df = pd.read_csv(self.annotations.output_file, index_col=0, header=0, comment="#")
- self.assertEqual(set(df.columns), {"cat_A", "cat_C"})
- self.assertTrue(np.all(df["cat_A"] == ["label_A1"] * n_rows))
- self.assertTrue(np.all(df["cat_C"] == ["label_C"] * n_rows))
-
- # rotation
- name, ext = path.splitext(self.annotations.output_file)
- backup_dir = f"{name}-backups"
- self.assertTrue(path.isdir(backup_dir))
- found_files = listdir(backup_dir)
- self.assertEqual(len(found_files), 1)
-
- def test_file_rotation_to_max_9(self):
- # verify we stop rotation at 9
- n_rows = self.data.get_shape()[0]
- fbs = make_fbs(
- {
- "cat_A": pd.Series(["label_A"] * n_rows, dtype="category"),
- "cat_B": pd.Series(["label_B"] * n_rows, dtype="category"),
- }
- )
- for i in range(0, 11):
- res = self.annotation_put_fbs(fbs)
- self.assertEqual(res, json.dumps({"status": "OK"}))
-
- name, ext = path.splitext(self.annotations.output_file)
- backup_dir = f"{name}-backups"
- self.assertTrue(path.isdir(backup_dir))
- found_files = listdir(backup_dir)
- self.assertTrue(len(found_files) <= 9)
-
- def test_put_get_roundtrip(self):
- # verify that OBS PUTs (annotation_put_fbs) are accessible via
- # GET (annotation_to_fbs_matrix)
-
- n_rows = self.data.get_shape()[0]
- fbs = make_fbs(
- {
- "cat_A": pd.Series(["label_A"] * n_rows, dtype="category"),
- "cat_B": pd.Series(["label_B"] * n_rows, dtype="category"),
- }
- )
-
- # put
- res = self.annotation_put_fbs(fbs)
- self.assertEqual(res, json.dumps({"status": "OK"}))
-
- # get
- labels = self.annotations.read_labels(None)
- fbsAll = self.data.annotation_to_fbs_matrix("obs", None, labels)
- schema = schema_get_helper(self.data)
- annotations = decode_fbs.decode_matrix_FBS(fbsAll)
- obs_index_col_name = schema["annotations"]["obs"]["index"]
- self.assertEqual(annotations["n_rows"], n_rows)
- self.assertEqual(annotations["n_cols"], 7)
- self.assertIsNone(annotations["row_idx"])
- self.assertEqual(
- annotations["col_idx"],
- [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain", "cat_A", "cat_B"],
- )
- col_idx = annotations["col_idx"]
- self.assertEqual(annotations["columns"][col_idx.index("cat_A")], ["label_A"] * n_rows)
- self.assertEqual(annotations["columns"][col_idx.index("cat_B")], ["label_B"] * n_rows)
-
- # verify the schema was updated
- all_col_schema = {c["name"]: c for c in schema["annotations"]["obs"]["columns"]}
- self.assertEqual(
- all_col_schema["cat_A"],
- {"name": "cat_A", "type": "categorical", "categories": ["label_A"], "writable": True},
- )
- self.assertEqual(
- all_col_schema["cat_B"],
- {"name": "cat_B", "type": "categorical", "categories": ["label_B"], "writable": True},
- )
-
- def test_put_float_data(self):
- # verify that OBS PUTs (annotation_put_fbs) are accessible via
- # GET (annotation_to_fbs_matrix)
-
- n_rows = self.data.get_shape()[0]
-
- # verifies that floating point with decimals fail.
- fbs = make_fbs({"cat_F_FAIL": pd.Series([1.1] * n_rows, dtype=np.dtype("float"))})
- with self.assertRaises(ValueError) as exception_context:
- res = self.annotation_put_fbs(fbs)
- self.assertEqual(str(exception_context.exception), "Columns may not have floating point types")
-
- # verifies that floating point that can be converted to int passes
- fbs = make_fbs({"cat_F_PASS": pd.Series([1.0] * n_rows, dtype="float")})
- res = self.annotation_put_fbs(fbs)
- self.assertEqual(res, json.dumps({"status": "OK"}))
-
- # check read_labels
- labels = self.annotations.read_labels(None)
- fbsAll = self.data.annotation_to_fbs_matrix("obs", None, labels)
- schema = schema_get_helper(self.data)
- annotations = decode_fbs.decode_matrix_FBS(fbsAll)
- self.assertEqual(annotations["n_rows"], n_rows)
- all_col_schema = {c["name"]: c for c in schema["annotations"]["obs"]["columns"]}
- self.assertEqual(
- all_col_schema["cat_F_PASS"],
- {"name": "cat_F_PASS", "type": "int32", "writable": True},
- )
diff --git a/backend/test/test_czi_hosted/unit/common/utils/__init__.py b/backend/test/test_czi_hosted/unit/common/utils/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/backend/test/test_czi_hosted/unit/common/utils/test_cxg_generation_utils.py b/backend/test/test_czi_hosted/unit/common/utils/test_cxg_generation_utils.py
deleted file mode 100644
index 5437b23b..00000000
--- a/backend/test/test_czi_hosted/unit/common/utils/test_cxg_generation_utils.py
+++ /dev/null
@@ -1,158 +0,0 @@
-import json
-import unittest
-from os import path, mkdir
-from shutil import rmtree
-from uuid import uuid4
-
-import numpy as np
-import tiledb
-from pandas import Series, DataFrame
-
-from backend.czi_hosted.common.utils.cxg_generation_utils import (
- convert_dictionary_to_cxg_group,
- convert_dataframe_to_cxg_array,
- convert_ndarray_to_cxg_dense_array,
- convert_matrix_to_cxg_array,
-)
-from backend.test import FIXTURES_ROOT
-
-
-class TestCxgGenerationUtils(unittest.TestCase):
- def setUp(self):
- self.testing_cxg_temp_directory = f"{FIXTURES_ROOT}/{uuid4()}"
- mkdir(self.testing_cxg_temp_directory)
-
- def tearDown(self):
- if path.isdir(self.testing_cxg_temp_directory):
- rmtree(self.testing_cxg_temp_directory)
-
- def test__convert_dictionary_to_cxg_group__writes_successfully(self):
- random_dictionary = {"cookies": "chocolate_chip", "brownies": "chocolate", "cake": "double chocolate"}
- dictionary_name = "favorite_desserts"
- expected_array_directory = f"{self.testing_cxg_temp_directory}/{dictionary_name}"
-
- convert_dictionary_to_cxg_group(
- self.testing_cxg_temp_directory, random_dictionary, group_metadata_name=dictionary_name
- )
-
- array = tiledb.open(expected_array_directory)
- actual_stored_metadata = dict(array.meta.items())
-
- self.assertTrue(path.isdir(expected_array_directory))
- self.assertTrue(isinstance(array, tiledb.DenseArray))
- self.assertEqual(random_dictionary, actual_stored_metadata)
-
- def test__convert_dataframe_to_cxg_array__writes_successfully(self):
- random_int_category = Series(data=[3, 1, 2, 4], dtype=np.int64)
- random_bool_category = Series(data=[True, True, False, True], dtype=np.bool_)
- random_dataframe_name = f"random_dataframe_{uuid4()}"
- random_dataframe = DataFrame(data={"int_category": random_int_category, "bool_category": random_bool_category})
-
- convert_dataframe_to_cxg_array(
- self.testing_cxg_temp_directory, random_dataframe_name, random_dataframe, "int_category", tiledb.Ctx()
- )
-
- expected_array_directory = f"{self.testing_cxg_temp_directory}/{random_dataframe_name}"
- expected_array_metadata = {
- "cxg_schema": json.dumps(
- {"int_category": {"type": "int32"}, "bool_category": {"type": "boolean"}, "index": "int_category"}
- )
- }
-
- actual_stored_dataframe_array = tiledb.open(expected_array_directory)
- actual_stored_dataframe_metadata = dict(actual_stored_dataframe_array.meta.items())
-
- self.assertTrue(path.isdir(expected_array_directory))
- self.assertTrue(isinstance(actual_stored_dataframe_array, tiledb.DenseArray))
- self.assertDictEqual(expected_array_metadata, actual_stored_dataframe_metadata)
- self.assertTrue((actual_stored_dataframe_array[0:4]["int_category"] == random_int_category.to_numpy()).all())
- self.assertTrue((actual_stored_dataframe_array[0:4]["bool_category"] == random_bool_category.to_numpy()).all())
-
- def test__convert_ndarray_to_cxg_dense_array__writes_successfully(self):
- ndarray = np.random.rand(3, 2)
- ndarray_name = f"{self.testing_cxg_temp_directory}/awesome_ndarray_{uuid4()}"
-
- convert_ndarray_to_cxg_dense_array(ndarray_name, ndarray, tiledb.Ctx())
-
- actual_stored_array = tiledb.open(ndarray_name)
-
- self.assertTrue(path.isdir(ndarray_name))
- self.assertTrue(isinstance(actual_stored_array, tiledb.DenseArray))
- self.assertTrue((actual_stored_array[:, :] == ndarray).all())
-
- def test__convert_matrix_to_cxg_array__dense_array_writes_successfully(self):
- matrix = np.float32(np.random.rand(3, 2))
- matrix_name = f"{self.testing_cxg_temp_directory}/awesome_matrix_{uuid4()}"
-
- convert_matrix_to_cxg_array(matrix_name, matrix, False, tiledb.Ctx())
-
- actual_stored_array = tiledb.open(matrix_name)
-
- self.assertTrue(path.isdir(matrix_name))
- self.assertTrue(isinstance(actual_stored_array, tiledb.DenseArray))
- self.assertTrue((actual_stored_array[:, :] == matrix).all())
-
- def test__convert_matrix_to_cxg_array__sparse_array_only_store_nonzeros_empty_array(self):
- matrix = np.zeros([3, 2])
- matrix_name = f"{self.testing_cxg_temp_directory}/awesome_zero_matrix_{uuid4()}"
-
- convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx())
-
- actual_stored_array = tiledb.open(matrix_name)
-
- self.assertTrue(path.isdir(matrix_name))
- self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
- self.assertTrue(actual_stored_array[:, :][""].size == 0)
-
- def test__convert_matrix_to_cxg_array__sparse_array_only_store_nonzeros(self):
- matrix = np.zeros([3, 3])
- matrix[0, 0] = 1
- matrix[1, 1] = 1
- matrix[2, 2] = 2
- matrix_name = f"{self.testing_cxg_temp_directory}/awesome_sparse_matrix_{uuid4()}"
-
- convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx())
-
- actual_stored_array = tiledb.open(matrix_name)
-
- self.assertTrue(path.isdir(matrix_name))
- self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
- self.assertTrue(actual_stored_array[0, 0][""] == 1)
- self.assertTrue(actual_stored_array[1, 1][""] == 1)
- self.assertTrue(actual_stored_array[2, 2][""] == 2)
- self.assertTrue(actual_stored_array[:, :][""].size == 3)
-
- def test__convert_matrix_to_cxg_array__sparse_array_with_column_encoding_empty_array(self):
- matrix_name = f"{self.testing_cxg_temp_directory}/awesome_column_shift_matrix_{uuid4()}"
- matrix = np.ones((3, 2))
- # The column shift will be equal to the matrix since subtracting the column shift from the matrix will create
- # a matrix of zeros which is sparse.
- column_shift = np.ones((3, 2))
-
- convert_matrix_to_cxg_array(
- matrix_name, matrix, True, tiledb.Ctx(), column_shift_for_sparse_encoding=column_shift
- )
-
- actual_stored_array = tiledb.open(matrix_name)
-
- self.assertTrue(path.isdir(matrix_name))
- self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
- self.assertTrue(actual_stored_array[:, :][""].size == 0)
-
- def test__convert_matrix_to_cxg_array__sparse_array_with_column_encoding_partial_array(self):
- matrix_name = f"{self.testing_cxg_temp_directory}/awesome_column_shift_matrix_{uuid4()}"
- matrix = np.ones((2, 2))
- # Only column shift the first column of ones.
- column_shift = np.array([[1, 0], [1, 0]])
-
- convert_matrix_to_cxg_array(
- matrix_name, matrix, True, tiledb.Ctx(), column_shift_for_sparse_encoding=column_shift
- )
-
- actual_stored_array = tiledb.open(matrix_name)
-
- self.assertTrue(path.isdir(matrix_name))
- self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
- self.assertTrue(actual_stored_array[0, 1][""] == 1)
- self.assertTrue(actual_stored_array[1, 1][""] == 1)
- self.assertTrue(actual_stored_array[:, :][""].size == 2)
diff --git a/backend/test/test_czi_hosted/unit/common/utils/test_matrix_utils.py b/backend/test/test_czi_hosted/unit/common/utils/test_matrix_utils.py
deleted file mode 100644
index c7b3cc19..00000000
--- a/backend/test/test_czi_hosted/unit/common/utils/test_matrix_utils.py
+++ /dev/null
@@ -1,66 +0,0 @@
-import unittest
-
-import numpy as np
-
-from backend.czi_hosted.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix
-
-
-class TestMatrixUtils(unittest.TestCase):
- def test__is_matrix_sparse__zero_and_one_hundred_percent_threshold(self):
- matrix = np.array([1, 2, 3])
-
- self.assertFalse(is_matrix_sparse(matrix, 0))
- self.assertTrue(is_matrix_sparse(matrix, 100))
-
- def test__is_matrix_sparse__partially_populated_sparse_matrix_returns_true(self):
- matrix = np.zeros([3, 4])
- matrix[2][3] = 1.0
- matrix[1][1] = 2.2
-
- self.assertTrue(is_matrix_sparse(matrix, 50))
-
- def test__is_matrix_sparse__partially_populated_dense_matrix_returns_false(self):
- matrix = np.zeros([2, 2])
- matrix[0][0] = 1.0
- matrix[0][1] = 2.2
- matrix[1][1] = 3.7
-
- self.assertFalse(is_matrix_sparse(matrix, 50))
-
- def test__is_matrix_sparse__giant_matrix_returns_false_early(self):
- matrix = np.ones([20000, 20])
-
- with self.assertLogs(level="INFO") as logger:
- self.assertFalse(is_matrix_sparse(matrix, 1))
-
- # Because the function returns early a log will output the _estimate_ instead of the _exact_ percentage of
- # non-zero elements in the matrix.
- self.assertIn("Percentage of non-zero elements (estimate)", logger.output[0])
-
- def test__is_matrix_sparse_with_column_shift_encoding__regular_sparse_returns_true(self):
- matrix = np.zeros([2, 2])
- matrix[0][0] = 1.0
-
- self.assertIsNotNone(get_column_shift_encode_for_matrix(matrix, 50))
-
- def test__is_matrix_sparse_with_column_shift_encoding__column_shift_returns_same_value(self):
- matrix = np.ones([2, 2])
- expected_column_shift = [1, 1]
-
- actual_column_shift = get_column_shift_encode_for_matrix(matrix, 50)
- self.assertTrue((expected_column_shift == actual_column_shift).all())
-
- def test__is_matrix_sparse_with_column_shift_encoding__impossible_column_shift_returns_none(self):
- matrix = np.array([[1, 2], [3, 4]])
-
- self.assertIsNone(get_column_shift_encode_for_matrix(matrix, 50))
-
- def test__is_matrix_sparse_with_column_shift_encoding__giant_matrix_returns_false_early(self):
- matrix = np.random.rand(20000, 20)
-
- with self.assertLogs(level="INFO") as logger:
- self.assertFalse(is_matrix_sparse(matrix, 1))
-
- # Because the function returns early a log will output the _estimate_ instead of the _exact_ percentage of
- # non-zero elements in the matrix.
- self.assertIn("Percentage of non-zero elements (estimate)", logger.output[0])
diff --git a/backend/test/test_czi_hosted/unit/common/utils/test_sanitization_utils.py b/backend/test/test_czi_hosted/unit/common/utils/test_sanitization_utils.py
deleted file mode 100644
index 9ae3cede..00000000
--- a/backend/test/test_czi_hosted/unit/common/utils/test_sanitization_utils.py
+++ /dev/null
@@ -1,55 +0,0 @@
-import unittest
-
-from backend.czi_hosted.common.utils.sanitization_utils import sanitize_values_in_list, sanitize_keys_in_dictionary
-
-
-class TestSanitizationUtils(unittest.TestCase):
- def test__sanitize_values_in_list__not_strings_raises_exception(self):
- keys_to_sanitize = [1, 2, 3]
-
- with self.assertRaises(Exception) as exception_context:
- sanitize_values_in_list(keys_to_sanitize)
-
- self.assertIn("must contain all strings", str(exception_context.exception))
-
- def test__sanitize_values_in_list__not_all_strings_raises_exception(self):
- keys_to_sanitize = ["1", "2", 3]
-
- with self.assertRaises(Exception) as exception_context:
- sanitize_values_in_list(keys_to_sanitize)
-
- self.assertIn("must contain all strings", str(exception_context.exception))
-
- def test__sanitize_values_in_list__replace_non_ascii_character_with_underscore(self):
- keys_to_sanitize = ["abc.", "~abc", "a~b/c"]
- expected_sanitized_keys_dict = dict(zip(keys_to_sanitize, ["abc_", "_abc", "a_b_c"]))
-
- actual_sanitized_keys_dict = sanitize_values_in_list(keys_to_sanitize)
-
- self.assertEqual(expected_sanitized_keys_dict, actual_sanitized_keys_dict)
-
- def test__sanitize_keys_in_dictionary__replace_non_ascii_character_with_underscore(self):
- dictionary_to_sanitize = {"abc.": 3, "~abc": 4, "a~b/c": 5}
- expected_sanitized_dict = {"abc_": 3, "_abc": 4, "a_b_c": 5}
-
- actual_sanitized_dict = dictionary_to_sanitize
- sanitize_keys_in_dictionary(actual_sanitized_dict)
-
- self.assertEqual(expected_sanitized_dict, actual_sanitized_dict)
-
- def test__sanitize_keys_in_dictionary__non_string_key_raises_exception(self):
- dictionary_to_sanitize = {4: 3, "~abc": 4, "a~b/c": 5}
-
- with self.assertRaises(Exception) as exception_context:
- sanitize_keys_in_dictionary(dictionary_to_sanitize)
-
- self.assertIn("must contain all strings", str(exception_context.exception))
-
- def test__sanitize_keys_in_dictionary__replace_only_some_keys(self):
- dictionary_to_sanitize = {"abc": 3, "~abc": 4, "a~b/c": 5}
- expected_sanitized_dict = {"abc": 3, "_abc": 4, "a_b_c": 5}
-
- actual_sanitized_dict = dictionary_to_sanitize
- sanitize_keys_in_dictionary(actual_sanitized_dict)
-
- self.assertEqual(expected_sanitized_dict, actual_sanitized_dict)
diff --git a/backend/test/test_czi_hosted/unit/common/utils/test_utils.py b/backend/test/test_czi_hosted/unit/common/utils/test_utils.py
deleted file mode 100644
index fd1f9442..00000000
--- a/backend/test/test_czi_hosted/unit/common/utils/test_utils.py
+++ /dev/null
@@ -1,34 +0,0 @@
-import os
-import shutil
-import unittest
-
-from backend.common.utils.utils import import_plugins
-from backend.test import PROJECT_ROOT, random_string
-
-
-class TestPlugins(unittest.TestCase):
- """ Test plugin import functionality """
-
- plugins_dir = f"{PROJECT_ROOT}/backend/test/test_czi_hosted/unit/plugins"
- test_plugin_path = f"{plugins_dir}/foo.py"
- secret = random_string(8)
-
- @classmethod
- def setUpClass(cls) -> None:
- if not os.path.isdir(cls.plugins_dir):
- os.mkdir(cls.plugins_dir)
- with open(cls.test_plugin_path, "w") as fh:
- fh.write(f'SECRET = "{cls.secret}"\n')
-
- @classmethod
- def tearDownClass(cls) -> None:
- if os.path.isdir(cls.plugins_dir):
- shutil.rmtree(cls.plugins_dir)
-
- def test_import_plugins(self):
- self.assertTrue(os.path.isfile(self.test_plugin_path))
- loaded_modules = import_plugins("backend.test.test_czi_hosted.unit.plugins")
- # test that import plugins found the file
- self.assertEqual(["backend.test.test_czi_hosted.unit.plugins.foo"], [ele.__name__ for ele in loaded_modules])
- # test that the module was properly executed
- self.assertEqual(self.secret, loaded_modules[0].SECRET)
diff --git a/backend/test/test_czi_hosted/unit/compute/__init__.py b/backend/test/test_czi_hosted/unit/compute/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/backend/test/test_czi_hosted/unit/compute/test_diffexp_cxg.py b/backend/test/test_czi_hosted/unit/compute/test_diffexp_cxg.py
deleted file mode 100644
index 14ddf400..00000000
--- a/backend/test/test_czi_hosted/unit/compute/test_diffexp_cxg.py
+++ /dev/null
@@ -1,183 +0,0 @@
-import os
-import tempfile
-import unittest
-
-import numpy as np
-
-from backend.czi_hosted.compute import diffexp_cxg
-from backend.common.compute import diffexp_generic
-from backend.czi_hosted.compute.diffexp_cxg import diffexp_ttest
-from backend.czi_hosted.converters.h5ad_data_file import H5ADDataFile
-from backend.common.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs
-from backend.czi_hosted.data_common.matrix_loader import MatrixDataLoader
-from backend.test.test_czi_hosted.performance.create_test_matrix import create_test_h5ad
-from backend.test.test_czi_hosted.unit import app_config
-from backend.test import PROJECT_ROOT, FIXTURES_ROOT
-
-
-class DiffExpTest(unittest.TestCase):
- """Tests the diffexp returns the expected results for one test case, using different
- adaptor types and different algorithms."""
-
- def load_dataset(self, path, extra_server_config={}, extra_dataset_config={}):
- extra_dataset_config["X_approximate_distribution"] = "normal" # hardwired for now
- config = app_config(path, extra_server_config=extra_server_config, extra_dataset_config=extra_dataset_config)
- loader = MatrixDataLoader(path)
- adaptor = loader.open(config)
- return adaptor
-
- def get_mask(self, adaptor, start, stride):
- """Simple function to return a mask or rows"""
- rows = adaptor.get_shape()[0]
- sel = list(range(start, rows, stride))
- mask = np.zeros(rows, dtype=bool)
- mask[sel] = True
- return mask
-
- def compare_diffexp_results(self, results, expects):
- self.assertEqual(len(results), len(expects))
- for result, expect in zip(results, expects):
- self.assertEqual(result[0], expect[0])
- self.assertTrue(np.isclose(result[1], expect[1], 1e-6, 1e-4))
- self.assertTrue(np.isclose(result[2], expect[2], 1e-6, 1e-4))
- self.assertTrue(np.isclose(result[3], expect[3], 1e-6, 1e-4))
-
-
- def check_1_10_2_10(self, results):
- """Checks the results for a specific set of rows selections"""
-
- positive_expects = [
- [1712, 0.24104056, 0.0051788902660723345, 1.0],
- [1575, 0.2615018, 0.007830310753043345, 1.0],
- [693, 0.23106655, 0.008715846769131548, 1.0],
- [916, 0.2395215, 0.009080596532247588, 1.0],
- [77, 0.22927025, 0.010070392939027756, 1.0],
- [782, 0.20581803, 0.010161745218916036, 1.0],
- [913, 0.23841085, 0.010782030711612685, 1.0],
- [910, 0.21493295, 0.014596411069229197, 1.0],
- [1727, 0.21911663, 0.015168372104237176, 1.0],
- [1443, 0.19814226, 0.015337080567465522, 1.0],
- ]
- negative_expects = [
- [956, -0.29662406, 0.0008649321884808977, 1.0],
- [1124, -0.2607333, 0.0011717216548271284, 1.0],
- [1809, -0.24854594, 0.0019304405196777848, 1.0],
- [1754, -0.24683577, 0.005691734062127954, 1.0],
- [948, -0.18708363, 0.006622111055981219, 1.0],
- [1810, -0.2172082, 0.007055917428377063, 1.0],
- [779, -0.21150622, 0.007202934422407284, 1.0],
- [576, -0.19008157, 0.008272092578813124, 1.0],
- [538, -0.21803819, 0.01062259019889307, 1.0],
- [436, -0.2100364, 0.01127515110543434, 1.0],
- ]
-
- self.compare_diffexp_results(results['positive'], positive_expects)
- self.compare_diffexp_results(results['negative'], negative_expects)
-
- def get_X_col(self, adaptor, cols):
- varmask = np.zeros(adaptor.get_shape()[1], dtype=bool)
- varmask[cols] = True
- return adaptor.get_X_array(None, varmask)
-
- def test_anndata_default(self):
- """Test an anndata adaptor with its default diffexp algorithm (diffexp_generic)"""
- adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
- maskA = self.get_mask(adaptor, 1, 10)
- maskB = self.get_mask(adaptor, 2, 10)
- results = adaptor.compute_diffexp_ttest(maskA, maskB, 10)
- self.check_1_10_2_10(results)
-
- def test_cxg_default(self):
- """Test a cxg adaptor with its default diffexp algorithm (diffexp_cxg)"""
- adaptor = self.load_dataset(f"{FIXTURES_ROOT}/pbmc3k.cxg")
- maskA = self.get_mask(adaptor, 1, 10)
- maskB = self.get_mask(adaptor, 2, 10)
-
- # run it through the adaptor
- results = adaptor.compute_diffexp_ttest(maskA, maskB, 10)
- self.check_1_10_2_10(results)
-
- # run it directly
-
- results = diffexp_ttest(adaptor, maskA, maskB, 10)
- self.check_1_10_2_10(results)
-
- def test_cxg_generic(self):
- """Test a cxg adaptor with the generic adaptor"""
- adaptor = self.load_dataset(f"{FIXTURES_ROOT}/pbmc3k.cxg")
- maskA = self.get_mask(adaptor, 1, 10)
- maskB = self.get_mask(adaptor, 2, 10)
- # run it directly
- results = diffexp_generic.diffexp_ttest(adaptor, maskA, maskB, 10)
- self.check_1_10_2_10(results)
-
- def test_cxg_sparse(self):
- self.sparse_diffexp(False)
-
- def test_cxg_sparse_col_shift(self):
- self.sparse_diffexp(True)
-
- def sparse_diffexp(self, apply_col_shift):
- with tempfile.TemporaryDirectory() as dirname:
- # create a sparse matrix
- h5adfile_path = os.path.join(dirname, "sparse.h5ad")
- create_test_h5ad(h5adfile_path, 2000, 2000, 10, apply_col_shift)
-
- h5ad_file_to_convert = H5ADDataFile(h5adfile_path, use_corpora_schema=False)
-
- sparsename = os.path.join(dirname, "sparse.cxg")
- h5ad_file_to_convert.to_cxg(sparsename, 11, True)
-
- adaptor_anndata = self.load_dataset(h5adfile_path, extra_dataset_config=dict(embeddings__names=[]))
-
- adaptor_sparse = self.load_dataset(sparsename)
- assert adaptor_sparse.open_array("X").schema.sparse
- assert adaptor_sparse.has_array("X_col_shift") == apply_col_shift
-
- densename = os.path.join(dirname, "dense.cxg")
- h5ad_file_to_convert.to_cxg(densename, True, 0)
- adaptor_dense = self.load_dataset(densename)
- assert not adaptor_dense.open_array("X").schema.sparse
- assert not adaptor_dense.has_array("X_col_shift")
-
- maskA = self.get_mask(adaptor_anndata, 1, 10)
- maskB = self.get_mask(adaptor_anndata, 2, 10)
-
- diffexp_results_anndata = diffexp_generic.diffexp_ttest(adaptor_anndata, maskA, maskB, 10)
- diffexp_results_sparse = diffexp_cxg.diffexp_ttest(adaptor_sparse, maskA, maskB, 10)
- diffexp_results_dense = diffexp_cxg.diffexp_ttest(adaptor_dense, maskA, maskB, 10)
-
- self.compare_diffexp_results(diffexp_results_anndata['positive'], diffexp_results_sparse['positive'])
- self.compare_diffexp_results(diffexp_results_anndata['negative'], diffexp_results_sparse['negative'])
-
- self.compare_diffexp_results(diffexp_results_anndata['positive'], diffexp_results_dense['positive'])
- self.compare_diffexp_results(diffexp_results_anndata['negative'], diffexp_results_dense['negative'])
-
- topcols_pos = np.array([x[0] for x in diffexp_results_anndata['positive']])
- topcols_neg = np.array([x[0] for x in diffexp_results_anndata['negative']])
- topcols = np.concatenate((topcols_pos, topcols_neg))
-
- cols_anndata = self.get_X_col(adaptor_anndata, topcols)
- cols_sparse = self.get_X_col(adaptor_sparse, topcols)
- cols_dense = self.get_X_col(adaptor_dense, topcols)
-
- assert cols_anndata.shape[0] == adaptor_sparse.get_shape()[0]
- assert cols_anndata.shape[1] == len(diffexp_results_anndata['positive']) + len(diffexp_results_anndata['negative'])
-
- def convert(mat, cols):
- return decode_matrix_fbs(encode_matrix_fbs(mat, col_idx=cols)).to_numpy()
-
- cols_anndata = convert(cols_anndata, topcols)
- cols_sparse = convert(cols_sparse, topcols)
- cols_dense = convert(cols_dense, topcols)
-
- x = adaptor_sparse.get_X_array()
- assert x.shape == adaptor_sparse.get_shape()
-
- for row in range(cols_anndata.shape[0]):
- for col in range(cols_anndata.shape[1]):
- vanndata = cols_anndata[row][col]
- vsparse = cols_sparse[row][col]
- vdense = cols_dense[row][col]
- self.assertTrue(np.isclose(vanndata, vsparse, 1e-6, 1e-6))
- self.assertTrue(np.isclose(vanndata, vdense, 1e-6, 1e-6))
diff --git a/backend/test/test_czi_hosted/unit/converters/__init__.py b/backend/test/test_czi_hosted/unit/converters/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/backend/test/test_czi_hosted/unit/converters/schema/__init__.py b/backend/test/test_czi_hosted/unit/converters/schema/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/backend/test/test_czi_hosted/unit/converters/schema/test_gene_symbol.py b/backend/test/test_czi_hosted/unit/converters/schema/test_gene_symbol.py
deleted file mode 100644
index a0a825be..00000000
--- a/backend/test/test_czi_hosted/unit/converters/schema/test_gene_symbol.py
+++ /dev/null
@@ -1,61 +0,0 @@
-import os
-import unittest
-
-import pandas as pd
-
-from backend.czi_hosted.converters.schema import gene_symbol
-from backend.test import FIXTURES_ROOT
-
-
-class TestHGNCSymbolChecker(unittest.TestCase):
-
- def setUp(self):
- self.test_hgnc_path = os.path.join(FIXTURES_ROOT, "hgnc_example.txt.gz")
- self.hgnc_checker = gene_symbol.HGNCSymbolChecker.from_hgnc_records(self.test_hgnc_path)
-
- def test_symbol_upgrade(self):
- self.assertEqual(self.hgnc_checker.upgrade_symbol("SEPT1"), "SEPTIN1")
- self.assertEqual(self.hgnc_checker.upgrade_symbol("ADRB2R"), "ADRB2")
- self.assertEqual(self.hgnc_checker.upgrade_symbol("BAR"), "ADRB2")
- self.assertEqual(self.hgnc_checker.upgrade_symbol("sept1"), "SEPTIN1")
- self.assertEqual(self.hgnc_checker.upgrade_symbol("AdRb2R"), "ADRB2")
- self.assertEqual(self.hgnc_checker.upgrade_symbol("bar"), "ADRB2")
-
- # Strip off seurat endings when appropriate
- self.assertEqual(self.hgnc_checker.upgrade_symbol("SEPT1.1"), "SEPTIN1")
- self.assertEqual(self.hgnc_checker.upgrade_symbol("ADRB2-1"), "ADRB2")
-
- # DIFF6 is ambiguous so don't upgrade it
- self.assertEqual(self.hgnc_checker.upgrade_symbol("DIFF6"), "DIFF6")
- self.assertEqual(self.hgnc_checker.upgrade_symbol("diff6"), "diff6")
-
- # ARG1 is approved
- self.assertEqual(self.hgnc_checker.upgrade_symbol("ARG1"), "ARG1")
- self.assertEqual(self.hgnc_checker.upgrade_symbol("arg1"), "ARG1")
-
- # HAP1 is both approved and withdrawn
- self.assertEqual(self.hgnc_checker.upgrade_symbol("HAP1"), "HAP1")
- self.assertEqual(self.hgnc_checker.upgrade_symbol("hap1"), "HAP1")
-
- # Leave unknown symbols alone
- self.assertEqual(self.hgnc_checker.upgrade_symbol("NOTASYMBOL"), "NOTASYMBOL")
- self.assertEqual(self.hgnc_checker.upgrade_symbol("notasymbol"), "notasymbol")
-
- # Upgrade HGNC ids unless you can't find it
- self.assertEqual(self.hgnc_checker.upgrade_symbol("HGNC:286"), "ADRB2")
- self.assertEqual(self.hgnc_checker.upgrade_symbol("HGNC:4812"), "HAP1")
- self.assertEqual(self.hgnc_checker.upgrade_symbol("HGNC:123456"), "HGNC:123456")
-
- def test_check_symbol(self):
- self.assertEqual(self.hgnc_checker.check_symbol("SEPT1"), gene_symbol.SymbolStatus.UPGRADABLE)
- self.assertEqual(self.hgnc_checker.check_symbol("DIFF6"), gene_symbol.SymbolStatus.AMBIGUOUS)
- self.assertEqual(self.hgnc_checker.check_symbol("NOTASYMBOL"), gene_symbol.SymbolStatus.UNKNOWN)
-
- # HAP1 is one of the approved and withdrawn symbols
- self.assertEqual(self.hgnc_checker.check_symbol("HAP1"), gene_symbol.SymbolStatus.APPROVED)
-
- def test_upgrade_index(self):
- index = pd.Index(["SEPT1", "DIFF6", "NOTASYMBOL", "bar", "SEPTIN1"])
- var_df = pd.DataFrame([[0] * len(index)], index=index)
- upgraded_index = gene_symbol.get_upgraded_var_index(var_df, hgnc_path=self.test_hgnc_path)
- self.assertEqual(upgraded_index.tolist(), ["SEPTIN1", "DIFF6", "NOTASYMBOL", "ADRB2", "SEPTIN1"])
diff --git a/backend/test/test_czi_hosted/unit/converters/schema/test_ontology.py b/backend/test/test_czi_hosted/unit/converters/schema/test_ontology.py
deleted file mode 100644
index 6cda5b12..00000000
--- a/backend/test/test_czi_hosted/unit/converters/schema/test_ontology.py
+++ /dev/null
@@ -1,128 +0,0 @@
-import json
-
-import unittest.mock
-
-from backend.czi_hosted.converters.schema import ontology
-
-
-class TestOntologyParsing(unittest.TestCase):
- def setUp(self):
-
- self.curies = ["UBERON:0002048", "HsapDv:0000174", "NCBITaxon:9606", "EFO:0008995"]
-
- self.names = ["UBERON", "HsapDv", "NCBITaxon", "EFO"]
-
- self.values = ["0002048", "0000174", "9606", "0008995"]
-
- self.iris = [
- "http://purl.obolibrary.org/obo/UBERON_0002048",
- "http://purl.obolibrary.org/obo/HsapDv_0000174",
- "http://purl.obolibrary.org/obo/NCBITaxon_9606",
- "http://www.ebi.ac.uk/efo/EFO_0008995",
- ]
-
- URL_ROOT = "http://www.ebi.ac.uk/ols/api/ontologies/"
- self.urls = [
- URL_ROOT + "UBERON/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FUBERON_0002048",
- URL_ROOT + "HsapDv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FHsapDv_0000174",
- URL_ROOT + "NCBITaxon/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_9606",
- URL_ROOT + "EFO/terms/http%253A%252F%252Fwww.ebi.ac.uk%252Fefo%252FEFO_0008995",
- ]
-
- self.responses = {
- "UBERON:0002048": {
- "iri": "http://purl.obolibrary.org/obo/UBERON_0002048",
- "description": ["Respiration organ that develops as an outpocketing of the esophagus."],
- "label": "lung",
- },
- "HsapDv:0000174": {
- "iri": "http://purl.obolibrary.org/obo/HsapDv_0000174",
- "description": ["Infant stage that refers to an infant who is over 1 and under 2 months old."],
- "label": "1-month-old human stage",
- },
- "NCBITaxon:9606": {
- "iri": "http://purl.obolibrary.org/obo/NCBITaxon_9606",
- "description": None,
- "label": "Homo sapiens",
- },
- "EFO:0008995": {
- "iri": "http://www.ebi.ac.uk/efo/EFO_0008995",
- "description": [
- (
- '10X is a "synthetic long-read" technology and works by capturing a barcoded oligo-coated '
- "gel-bead and 0.3x genome copies into a single emulsion droplet, processing the equivalent "
- "of 1 million pipetting steps. Successive versions of the 10x chemistry use different "
- "barcode locations to improve the sequencing yield and quality of 10x experiments."
- )
- ],
- "label": "10X sequencing",
- },
- }
-
- def test_ontololgy_name(self):
- for curie, expected_name in zip(self.curies, self.names):
- self.assertEqual(ontology._ontology_name(curie), expected_name)
-
- def test_ontololgy_value(self):
- for curie, expected_value in zip(self.curies, self.values):
- self.assertEqual(ontology._ontology_value(curie), expected_value)
-
- def test_iri(self):
- for curie, expected_iri in zip(self.curies, self.iris):
- self.assertEqual(ontology._iri(curie), expected_iri)
-
- def test_ontology_info_url(self):
- for curie, expected_url in zip(self.curies, self.urls):
- self.assertEqual(ontology._ontology_info_url(curie), expected_url)
-
- def test_empty_ontology_info_url(self):
- self.assertEqual(ontology._ontology_info_url(""), "")
-
-
-class TestOntologyLookup(unittest.TestCase):
- def setUp(self):
- self.responses = {
- "UBERON:0002048": {
- "iri": "http://purl.obolibrary.org/obo/UBERON_0002048",
- "description": ["Respiration organ that develops as an outpocketing of the esophagus."],
- "label": "lung",
- },
- "HsapDv:0000174": {
- "iri": "http://purl.obolibrary.org/obo/HsapDv_0000174",
- "description": ["Infant stage that refers to an infant who is over 1 and under 2 months old."],
- "label": "1-month-old human stage",
- },
- "NCBITaxon:9606": {
- "iri": "http://purl.obolibrary.org/obo/NCBITaxon_9606",
- "description": None,
- "label": "Homo sapiens",
- },
- "EFO:0008995": {
- "iri": "http://www.ebi.ac.uk/efo/EFO_0008995",
- "description": [
- ('10X is a "synthetic long-read" technology and works by capturing a barcoded oligo-coated '
- 'gel-bead and 0.3x genome copies into a single emulsion droplet, processing the equivalent '
- 'of 1 million pipetting steps. Successive versions of the 10x chemistry use different barcode '
- 'locations to improve the sequencing yield and quality of 10x experiments.')
- ],
- "label": "10X sequencing",
- },
- }
-
- self.labels = {
- "UBERON:0002048": "lung",
- "HsapDv:0000174": "1-month-old human stage",
- "NCBITaxon:9606": "Homo sapiens",
- "EFO:0008995": "10X sequencing",
- }
-
- @unittest.mock.patch("requests.get")
- def test_lookup_label(self, mock_get):
-
- for curie, response in self.responses.items():
- mock_get.return_value.content = json.dumps(response)
- mock_get.return_value.json.return_value = response
- mock_get.return_value.status_code = 200
-
- label = ontology.get_ontology_label(curie)
- self.assertEqual(label, self.labels[curie])
diff --git a/backend/test/test_czi_hosted/unit/converters/schema/test_remix.py b/backend/test/test_czi_hosted/unit/converters/schema/test_remix.py
deleted file mode 100644
index 4b4e37a4..00000000
--- a/backend/test/test_czi_hosted/unit/converters/schema/test_remix.py
+++ /dev/null
@@ -1,257 +0,0 @@
-import json
-import os
-import unittest
-import unittest.mock
-
-import anndata
-import numpy
-import pandas as pd
-import scanpy as sc
-
-from backend.czi_hosted.converters.schema import remix
-
-from backend.test import PROJECT_ROOT, FIXTURES_ROOT
-
-
-class TestApplySchema(unittest.TestCase):
-
- def setUp(self):
- self.source_h5ad_path = f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad"
- self.output_h5ad_path = f"{FIXTURES_ROOT}/test_remix.h5ad"
- self.config_path = f"{FIXTURES_ROOT}/test_config.yaml"
- self.bad_config_path = f"{FIXTURES_ROOT}/test_bad_config.yaml"
-
- def tearDown(self):
- try:
- os.remove(self.output_h5ad_path)
- except OSError:
- pass
-
- @unittest.mock.patch("backend.czi_hosted.converters.schema.ontology.get_ontology_label")
- def test_apply_schema(self, mock_get_ontology_label):
- mock_get_ontology_label.return_value = "test label"
- remix.apply_schema(self.source_h5ad_path, self.config_path, self.output_h5ad_path)
- new_adata = sc.read_h5ad(self.output_h5ad_path)
-
- self.assertIn("cell_type", new_adata.obs.columns)
- self.assertListEqual(["test label"], new_adata.obs["cell_type"].unique().tolist())
- self.assertListEqual(
- ["CL:00001", "CL:00002", "CL:00003", "CL:00004", "CL:00005", "CL:00006", "CL:00007", "CL:00008"],
- sorted(new_adata.obs["cell_type_ontology_term_id"].unique().tolist())
- )
-
- self.assertIn("version", new_adata.uns_keys())
-
- @unittest.mock.patch("backend.czi_hosted.converters.schema.ontology.get_ontology_label")
- def test_apply_bad_schema(self, mock_get_ontology_label):
- mock_get_ontology_label.return_value = "test label"
- remix.apply_schema(self.source_h5ad_path, self.bad_config_path, self.output_h5ad_path)
- new_adata = sc.read_h5ad(self.output_h5ad_path)
-
- # Should refuse to write the version
- self.assertNotIn("version", new_adata.uns_keys())
-
-class TestFieldParsing(unittest.TestCase):
-
- def test_is_curie(self):
- self.assertTrue(remix.is_curie("EFO:00001"))
- self.assertTrue(remix.is_curie("UBERON:123456"))
- self.assertTrue(remix.is_curie("HsapDv:0001"))
- self.assertFalse(remix.is_curie("UBERON"))
- self.assertFalse(remix.is_curie("UBERON:"))
- self.assertFalse(remix.is_curie("123456"))
-
- def test_is_ontology_field(self):
- self.assertTrue(remix.is_ontology_field("tissue_ontology_term_id"))
- self.assertTrue(remix.is_ontology_field("cell_type_ontology_term_id"))
- self.assertFalse(remix.is_ontology_field("cell_ontology"))
- self.assertFalse(remix.is_ontology_field("method"))
-
- def test_get_label_field_name(self):
- self.assertEqual("tissue", remix.get_label_field_name("tissue_ontology_term_id"))
- self.assertEqual("cell_type", remix.get_label_field_name("cell_type_ontology_term_id"))
-
- def test_split_suffix(self):
- self.assertEqual(("UBERON:1234", " (organoid)"), remix.split_suffix("UBERON:1234 (organoid)"))
- self.assertEqual(("UBERON:1234", " (cell culture)"), remix.split_suffix("UBERON:1234 (cell culture)"))
- self.assertEqual(("UBERON:1234", ""), remix.split_suffix("UBERON:1234"))
- self.assertEqual(("UBERON:1234 (something)", ""), remix.split_suffix("UBERON:1234 (something)"))
-
- @unittest.mock.patch("backend.czi_hosted.converters.schema.ontology.get_ontology_label")
- def test_get_curie_and_label(self, mock_get_ontology_label):
- mock_get_ontology_label.return_value = "test label"
- self.assertEqual(
- remix.get_curie_and_label("UBERON:1234"),
- ("UBERON:1234", "test label")
- )
- self.assertEqual(
- remix.get_curie_and_label("UBERON:1234 (cell culture)"),
- ("UBERON:1234 (cell culture)", "test label (cell culture)")
- )
- self.assertEqual(
- remix.get_curie_and_label("whatever"),
- ("", "whatever")
- )
-
-
-class TestManipulateAnndata(unittest.TestCase):
-
- def setUp(self):
-
- self.cell_count = 20
- self.gene_count = 200
- X = numpy.random.randint(0, 1000, (self.cell_count, self.gene_count))
- uns = {"organism": "monkey", "experiment": "monkey experiment"}
- obs = pd.DataFrame(
- index=[f"Cell{d}" for d in range(self.cell_count)],
- columns=["tissue", "CellType"],
- data=[["lung", "epithelial"]] * (self.cell_count // 2) + [["lung", "endothelial"]] * (self.cell_count // 2)
- )
- var = pd.DataFrame(index=[f"SEPT{d}" for d in range(self.gene_count)])
-
- self.adata = anndata.AnnData(X=X, obs=obs, var=var, uns=uns)
-
- def test_safe_add_field(self):
-
- remix.safe_add_field(self.adata.obs, "tissue", ["monkey lung"] * self.cell_count)
- self.assertEqual(self.adata.obs["tissue_original"].tolist(), ["lung"] * self.cell_count)
- self.assertEqual(self.adata.obs["tissue"].tolist(), ["monkey lung"] * self.cell_count)
-
- remix.safe_add_field(self.adata.uns, "contributors", [{"name": "contributor1"}, {"name": "contributor2"}])
- self.assertEqual(
- self.adata.uns["contributors"],
- json.dumps([{"name": "contributor1"}, {"name": "contributor2"}])
- )
-
- @unittest.mock.patch("backend.czi_hosted.converters.schema.ontology.get_ontology_label")
- def test_remix_uns(self, mock_get_ontology_label):
- mock_get_ontology_label.return_value = "Pan troglodytes"
- uns_config = {
- "version": {
- "corpora_schema_version": "1.0.0",
- "corpora_encoding_version": "0.1.0"
- },
- "organism_ontology_term_id": "NCBITaxon:9598",
- "contributors": [
- {
- "name": "scientist",
- "email": "scientist@science.com"
- }
- ]
- }
-
- remix.remix_uns(self.adata, uns_config)
-
- self.assertEqual(
- sorted(self.adata.uns_keys()),
- sorted(["organism_original", "organism", "organism_ontology_term_id",
- "contributors", "version", "experiment"])
- )
-
- self.assertEqual(self.adata.uns['organism'], "Pan troglodytes")
- self.assertEqual(self.adata.uns['organism_original'], "monkey")
- self.assertEqual(self.adata.uns['organism_ontology_term_id'], "NCBITaxon:9598")
- self.assertEqual(self.adata.uns['contributors'],
- json.dumps([{"name": "scientist", "email": "scientist@science.com"}]))
-
- @unittest.mock.patch("backend.czi_hosted.converters.schema.ontology.get_ontology_label")
- def test_remix_obs(self, mock_get_ontology_label):
- mock_get_ontology_label.return_value = "lung (in a monkey)"
- obs_config = {
- "tissue_ontology_term_id": {
- "tissue": {
- "lung": "UBERON:00000"
- }
- },
- "cell_color": {
- "CellType": {
- "epithelial": "fuschia",
- "endothelial": "khaki"
- }
- },
- "sex": "male"
- }
-
- remix.remix_obs(self.adata, obs_config)
- self.assertEqual(
- sorted(self.adata.obs_keys()),
- sorted(["tissue", "tissue_ontology_term_id", "tissue_original", "CellType", "cell_color", "sex"])
- )
-
- self.assertTrue(all(v == "lung" for v in self.adata.obs.tissue_original))
- self.assertTrue(all(v == "UBERON:00000" for v in self.adata.obs.tissue_ontology_term_id))
- self.assertTrue(all(v == "lung (in a monkey)" for v in self.adata.obs.tissue))
- self.assertTrue(all(v == "male" for v in self.adata.obs.sex))
- self.assertTrue(all(v in (("epithelial", "fuschia"), ("endothelial", "khaki"))
- for v in zip(self.adata.obs.CellType, self.adata.obs.cell_color)))
-
-
-class TestFixupGeneSymbols(unittest.TestCase):
-
- def setUp(self):
- self.seurat_path = f"{PROJECT_ROOT}/czi_hosted/test/fixtures/schema_test_data/seurat_tutorial.h5ad"
- self.seurat_merged_path = f"{PROJECT_ROOT}/czi_hosted/test/fixtures/schema_test_data/seurat_tutorial_merged.h5ad"
- self.sctransform_path = f"{PROJECT_ROOT}/czi_hosted/test/fixtures/schema_test_data/sctransform.h5ad"
- self.sctransform_merged_path = f"{PROJECT_ROOT}/czi_hosted/test/fixtures/schema_test_data/sctransform_merged.h5ad"
-
- # There's lots of MALAT1, but it doesn't collide with any other names,
- # so it shouldn't change during merging.
- self.stable_gene = "MALAT1"
-
- def test_fixup_gene_symbols_seurat(self):
-
- if not os.path.isfile(self.seurat_path):
- return unittest.skip(
- "Skipping gene symbol conversion tests because test h5ads are not present. To create them, "
- "run czi_hosted/test/fixtures/schema_test_data/generate_test_data.sh"
- )
-
- original_adata = sc.read_h5ad(self.seurat_path)
- merged_adata = sc.read_h5ad(self.seurat_merged_path)
-
- fixup_config = {"X": "log1p", "counts": "raw", "scale.data": "log1p"}
-
- fixed_adata = remix.fixup_gene_symbols(original_adata, fixup_config)
-
- self.assertEqual(
- merged_adata.layers["counts"][:, merged_adata.var.index == self.stable_gene].sum(),
- fixed_adata.raw.X[:, fixed_adata.var.index == self.stable_gene].sum()
- )
- self.assertAlmostEqual(
- merged_adata.X[:, merged_adata.var.index == self.stable_gene].sum(),
- fixed_adata.X[:, fixed_adata.var.index == self.stable_gene].sum()
- )
-
- self.assertAlmostEqual(
- merged_adata.layers["scale.data"][:, merged_adata.var.index == self.stable_gene].sum(),
- fixed_adata.layers["scale.data"][:, fixed_adata.var.index == self.stable_gene].sum()
- )
-
- def test_fixup_gene_symbols_sctransform(self):
-
- if not os.path.isfile(self.sctransform_path):
- return unittest.skip(
- "Skipping gene symbol conversion tests because test h5ads are not present. To create them, "
- "run czi_hosted/test/fixtures/schema_test_data/generate_test_data.sh"
- )
-
- original_adata = sc.read_h5ad(self.sctransform_path)
- merged_adata = sc.read_h5ad(self.sctransform_merged_path)
-
- fixup_config = {"X": "log1p", "counts": "raw"}
-
- fixed_adata = remix.fixup_gene_symbols(original_adata, fixup_config)
-
- # sctransform does a bunch of stuff, including slightly modifying the
- # raw counts. So we can't assert for exact equality the way we do with
- # the vanilla seurat tutorial. But, the results should still be very
- # close.
- merged_raw_stable = merged_adata.layers["counts"][:, merged_adata.var.index == self.stable_gene].sum()
- fixed_raw_stable = fixed_adata.raw.X[:, fixed_adata.var.index == self.stable_gene].sum()
- self.assertLess(abs(merged_raw_stable - fixed_raw_stable), .001 * merged_raw_stable)
-
- self.assertAlmostEqual(
- merged_adata.X[:, merged_adata.var.index == self.stable_gene].sum(),
- fixed_adata.X[:, fixed_adata.var.index == self.stable_gene].sum(),
- 0
- )
diff --git a/backend/test/test_czi_hosted/unit/converters/schema/test_validate.py b/backend/test/test_czi_hosted/unit/converters/schema/test_validate.py
deleted file mode 100644
index 01978222..00000000
--- a/backend/test/test_czi_hosted/unit/converters/schema/test_validate.py
+++ /dev/null
@@ -1,434 +0,0 @@
-import json
-import unittest
-
-import pandas as pd
-import scanpy as sc
-
-from backend.czi_hosted.converters.schema import validate
-
-from backend.test import PROJECT_ROOT
-
-
-class TestFieldValidation(unittest.TestCase):
-
- def test_validate_stringified_list_of_dicts(self):
-
- good = json.dumps([{"a": 1}, {2: "x", "z": "y"}])
- not_stringified = [{"a": 1}, {2: "x", "z": "y"}]
- not_a_list = json.dumps({"bad": "dict"})
- not_json = "oh hey!"
-
- self.assertTrue(validate._validate_stringified_list_of_dicts(good))
-
- self.assertFalse(validate._validate_stringified_list_of_dicts(not_stringified))
- self.assertFalse(validate._validate_stringified_list_of_dicts(not_a_list))
- self.assertFalse(validate._validate_stringified_list_of_dicts(not_json))
-
- def test_validate_human_readable_string(self):
-
- good = "oh hey!"
- curie = "EFO:0001"
- ensg = "ENSG000001234"
- enst = "ENST000005678"
-
- self.assertTrue(validate._validate_human_readable_string(good))
-
- self.assertFalse(validate._validate_human_readable_string(curie))
- self.assertFalse(validate._validate_human_readable_string(ensg))
- self.assertFalse(validate._validate_human_readable_string(enst))
-
- def test_validate_curie(self):
-
- self.assertTrue(validate._validate_curie("UBERON:00001", ["UBERON", "EFO"]))
- self.assertTrue(validate._validate_curie("HsapDv:00002", ["HsapDv"]))
-
- self.assertFalse(validate._validate_curie("HsapDv:00002", ["UBERON", "EFO"]))
- self.assertFalse(validate._validate_curie("EFO:00002 (organoid)", ["UBERON", "EFO"]))
- self.assertFalse(validate._validate_curie("EFO:00002 extra", ["UBERON", "EFO"]))
- self.assertFalse(validate._validate_curie("UBERON:ABCD", ["UBERON", "EFO"]))
- self.assertFalse(validate._validate_curie("Uberon:00002", ["UBERON", "EFO"]))
- self.assertFalse(validate._validate_curie("UBERON:", ["UBERON", "EFO"]))
- self.assertFalse(validate._validate_curie("UBERON", ["UBERON", "EFO"]))
-
- def test_validate_suffixed_curie(self):
-
- self.assertTrue(validate._validate_suffixed_curie("EFO:00001", ["UBERON", "EFO"]))
- self.assertTrue(validate._validate_suffixed_curie("UBERON:00001 (cell culture)", ["UBERON", "EFO"]))
-
- self.assertFalse(validate._validate_suffixed_curie("HsapDv:00002 (organoid)", ["UBERON", "EFO"]))
- self.assertFalse(validate._validate_suffixed_curie("HsapDv:00002(organoid)", ["UBERON", "EFO"]))
- self.assertFalse(validate._validate_suffixed_curie("HsapDv:00002", ["UBERON", "EFO"]))
- self.assertFalse(validate._validate_suffixed_curie("EFO:00002 extra", ["UBERON", "EFO"]))
- self.assertFalse(validate._validate_suffixed_curie("UBERON:ABCD", ["UBERON", "EFO"]))
- self.assertFalse(validate._validate_suffixed_curie("Uberon:00002", ["UBERON", "EFO"]))
- self.assertFalse(validate._validate_suffixed_curie("UBERON:", ["UBERON", "EFO"]))
- self.assertFalse(validate._validate_suffixed_curie("UBERON", ["UBERON", "EFO"]))
-
-
-class TestColumnValidation(unittest.TestCase):
-
- def test_validate_unique(self):
- unique = pd.DataFrame([["abc", "def"], ["ghi", "jkl"], ["mnop", "qrs"]],
- index=["X", "Y", "Z"], columns=["col1", "col2"])
- duped = pd.DataFrame([["abc", "def"], ["ghi", "qrs"], ["abc", "qrs"]],
- index=["X", "Y", "X"], columns=["col1", "col2"])
-
- schema_def = {"unique": True}
-
- errors = validate._validate_column(unique.index, "index", "unique_df", schema_def)
- self.assertFalse(errors)
-
- errors = validate._validate_column(duped.index, "index", "duped_df", schema_def)
- self.assertEqual(len(errors), 1)
- self.assertIn("is not unique", errors[0])
-
- errors = validate._validate_column(unique["col1"], "col1", "unique_df", schema_def)
- self.assertFalse(errors)
-
- errors = validate._validate_column(duped["col1"], "col1", "duped_df", schema_def)
- self.assertEqual(len(errors), 1)
- self.assertIn("is not unique", errors[0])
-
- schema_def = {"unique": False}
- errors = validate._validate_column(duped["col1"], "col1", "duped_df", schema_def)
- self.assertFalse(errors)
-
- def test_validate_nullable(self):
- non_null = pd.DataFrame([["abc", "def"], ["ghi", "jkl"], ["mnop", "qrs"]],
- index=["X", "Y", "Z"], columns=["col1", "col2"])
- has_null = pd.DataFrame([["abc", "", None], ["ghi", "jkl", 1], ["mnop", "qrs", 2]],
- index=["X", "Y", "Z"], columns=["col1", "col2", "col3"])
-
- schema_def = {"nullable": False}
- errors = validate._validate_column(non_null["col1"], "col1", "nonnull_df", schema_def)
- self.assertFalse(errors)
- errors = validate._validate_column(has_null["col1"], "col1", "hasnull_df", schema_def)
- self.assertFalse(errors)
-
- errors = validate._validate_column(has_null["col2"], "col2", "hasnull_df", schema_def)
- self.assertEqual(len(errors), 1)
- self.assertIn("contains empty values", errors[0])
- errors = validate._validate_column(has_null["col3"], "col3", "hasnull_df", schema_def)
- self.assertEqual(len(errors), 1)
- self.assertIn("contains empty values", errors[0])
-
- schema_def = {"nullable": True}
- errors = validate._validate_column(has_null["col2"], "col2", "hasnull_df", schema_def)
- self.assertFalse(errors)
-
- def test_human_readable(self):
- hr_df = pd.DataFrame(
- [["for you, a human", "UBERON:12345", "UBERON:1234 (thundercat)"],
- ["hope you're well", "bit of lungs", "brain"]],
- index=["ENSG00001", "ENSG00002"],
- columns=["good", "curie", "suffixed_curie"])
-
- schema_def = {"type": "human-readable string"}
- errors = validate._validate_column(hr_df["good"], "good", "hr", schema_def)
- self.assertFalse(errors)
-
- errors = validate._validate_column(hr_df["curie"], "curie", "hr", schema_def)
- self.assertEqual(len(errors), 1)
- self.assertIn("non-human-readable", errors[0])
-
- errors = validate._validate_column(hr_df["suffixed_curie"], "suffixed_curie", "hr", schema_def)
- self.assertEqual(len(errors), 1)
- self.assertIn("non-human-readable", errors[0])
-
- errors = validate._validate_column(hr_df.index, "ensg", "hr", schema_def)
- self.assertEqual(len(errors), 1)
- self.assertIn("non-human-readable", errors[0])
-
- def test_curie(self):
-
- curie_df = pd.DataFrame(
- [["EFO:00001", "HsapDv:00001 (cell culture)", "EFO:", "MONDO:0001 cell culture"],
- ["UBERON:00002", "HsapDv:00002 (organoid)", "EFO:12345", "MONDO:0002 (baba yaga)"],
- ["EFO:0000000005", "HsapDv:000004 (humanzee)", "EFO:000002", "MONDO:0004 (TMNT)"]],
- index=["X", "Y", "Z"],
- columns=["good", "good_suffix", "bad", "bad_suffix"])
-
- # Good
- schema_def = {"type": "curie", "prefixes": ["EFO", "UBERON"]}
- errors = validate._validate_column(curie_df["good"], "good", "curie_df", schema_def)
- self.assertFalse(errors)
-
- # Good suffix
- schema_def = {"type": "suffixed curie", "prefixes": ["HsapDv", "WHATEVER"]}
- errors = validate._validate_column(curie_df["good_suffix"], "good_suffix", "curie_df", schema_def)
- self.assertFalse(errors)
-
- # Bad prefix
- schema_def = {"type": "curie", "prefixes": ["EFO"]}
- errors = validate._validate_column(curie_df["good"], "good", "curie_df", schema_def)
- self.assertEqual(len(errors), 1)
- self.assertIn("invalid ontology", errors[0])
- self.assertIn("must be curies from one of these", errors[0])
-
- # Bad curies
- schema_def = {"type": "curie", "prefixes": ["EFO"]}
- errors = validate._validate_column(curie_df["bad"], "bad", "curie_df", schema_def)
- self.assertEqual(len(errors), 1)
- self.assertIn("invalid ontology", errors[0])
-
- # Bad suffixes
- schema_def = {"type": "suffixed curie", "prefixes": ["EFO"]}
- errors = validate._validate_column(curie_df["bad_suffix"], "bad_suffix", "curie_df", schema_def)
- self.assertEqual(len(errors), 1)
- self.assertIn("invalid ontology", errors[0])
-
- def test_enum(self):
- enum_df = pd.DataFrame(
- [["abc", "ghi"],
- ["def", "jkl"]],
- index=["X", "Y"],
- columns=["col1", "col2"])
-
- # All match
- schema_def = {"type": "string", "enum": ["abc", "def", "xyz"]}
- errors = validate._validate_column(enum_df["col1"], "col1", "enum_df", schema_def)
- self.assertFalse(errors)
-
- # Missing value
- schema_def = {"type": "string", "enum": ["abc", "xyz"]}
- errors = validate._validate_column(enum_df["col1"], "col1", "enum_df", schema_def)
- self.assertEqual(len(errors), 1)
- self.assertIn("unpermitted values", errors[0])
-
-
-class TestDictValidations(unittest.TestCase):
-
-
- def test_key_presence(self):
-
- schema_def = {"keys": {"abc": None, "def": None}}
-
- dict_ = {"abc": "123", "def": "456"}
- errors = validate._validate_dict(dict_, "d", schema_def)
- self.assertFalse(errors)
-
- # Missing keys are bad
- dict_ = {"abc": "123"}
- errors = validate._validate_dict(dict_, "d", schema_def)
- self.assertEqual(len(errors), 1)
- self.assertIn("missing key", errors[0])
-
- # Extra keys are okay
- dict_ = {"abc": "123", "def": "456", "xyz": "789"}
- errors = validate._validate_dict(dict_, "d", schema_def)
- self.assertFalse(errors)
-
- # Better not be empty come on
- dict_ = {}
- errors = validate._validate_dict(dict_, "d", schema_def)
- self.assertEqual(len(errors), 2)
-
- def test_nullable(self):
-
- schema_def = {"keys": {"abc": {"type": "string", "nullable": False},
- "def": {"type": "string", "nullable": True}}}
-
- dict_ = {"abc": "xyz", "def": ""}
- errors = validate._validate_dict(dict_, "d", schema_def)
- self.assertFalse(errors)
-
- dict_ = {"abc": "", "def": ""}
- errors = validate._validate_dict(dict_, "d", schema_def)
- self.assertEqual(len(errors), 1)
- self.assertIn("empty value", errors[0])
-
- def test_recurse(self):
-
- schema_def = {
- "keys": {
- "subdict": {
- "type": "dict",
- "keys": {
- "subdict_key1": None,
- "subdict_key2": None
- }
- },
- "ontology": {
- "type": "curie",
- "prefixes": ["ONTOLOGY"]
- },
- "blob": {
- "type": "stringified list of dicts"
- }
- }
- }
-
- dict_ = {
- "subdict": {"subdict_key1": "any", "subdict_key2": "any"},
- "ontology": "ONTOLOGY:123456",
- "blob": json.dumps([{"abc": 123}, {"def": 456}])
- }
- errors = validate._validate_dict(dict_, "d", schema_def)
- self.assertFalse(errors)
-
- dict_ = {
- "subdict": {"subdict_key1": "any"},
- "ontology": "ONTOLOGY:123456",
- "blob": json.dumps([{"abc": 123}, {"def": 456}])
- }
- errors = validate._validate_dict(dict_, "d", schema_def)
- self.assertEqual(len(errors), 1)
- self.assertIn("missing key", errors[0])
-
- dict_ = {
- "subdict": {"subdict_key1": "any", "subdict_key2": "any"},
- "ontology": "oh no not an ontology term",
- "blob": json.dumps([{"abc": 123}, {"def": 456}])
- }
- errors = validate._validate_dict(dict_, "d", schema_def)
- self.assertEqual(len(errors), 1)
- self.assertIn("invalid ontology", errors[0])
-
- dict_ = {
- "subdict": {"subdict_key1": "any", "subdict_key2": "any"},
- "ontology": "ONTOLOGY:123456",
- "blob": [{"abc": 123}, {"def": 456}]
- }
- errors = validate._validate_dict(dict_, "d", schema_def)
- self.assertEqual(len(errors), 1)
- self.assertIn("JSON-encoded list of dicts", errors[0])
-
- # Multiple errors
- dict_ = {
- "subdict": {"subdict_key1": "any"},
- "ontology": "oh no not an ontology term",
- "blob": json.dumps([{"abc": 123}, {"def": 456}])
- }
- errors = validate._validate_dict(dict_, "d", schema_def)
- self.assertEqual(len(errors), 2)
-
-
-class TestDataframeValidation(unittest.TestCase):
-
- def test_column_presence(self):
- df = pd.DataFrame(
- [["abc", "EFO:123"],
- ["def", "UBERON:456"]],
- columns=["hr_string", "ontology"],
- index=["X", "Y"]
- )
-
- schema_def = {
- "columns": {
- "hr_string": {"type": "human-readable string"},
- "ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]}
- }
- }
- errors = validate._validate_dataframe(df, "df", schema_def)
- self.assertFalse(errors)
-
- schema_def = {
- "columns": {
- "hr_string": {"type": "human-readable string"},
- "another_hr_string": {"type": "human-readable string"},
- "ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]}
- }
- }
- errors = validate._validate_dataframe(df, "df", schema_def)
- self.assertEqual(len(errors), 1)
- self.assertIn("missing column", errors[0])
-
- # Extra is okay
- df = pd.DataFrame(
- [["abc", "EFO:123", "extra"],
- ["def", "UBERON:456", "extra"]],
- columns=["hr_string", "ontology", "extra"],
- index=["X", "Y"]
- )
- schema_def = {
- "columns": {
- "hr_string": {"type": "human-readable string"},
- "ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]}
- }
- }
- errors = validate._validate_dataframe(df, "df", schema_def)
- self.assertFalse(errors)
-
-
- def test_index(self):
- df = pd.DataFrame(
- [["abc", "123"],
- ["def", "456"]],
- columns=["col1", "col2"],
- index=["ENSG0001", "ENSG0002"]
- )
-
- schema_def = {"index": {"unique": True}}
- errors = validate._validate_dataframe(df, "df", schema_def)
- self.assertFalse(errors)
-
- schema_def = {"index": {"type": "human-readable string"}}
- errors = validate._validate_dataframe(df, "df", schema_def)
- self.assertEqual(len(errors), 1)
- self.assertIn("non-human-readable", errors[0])
-
- df = pd.DataFrame(
- [["abc", "123"],
- ["def", "456"]],
- columns=["col1", "col2"],
- index=["ENSG0001", "ENSG0001"]
- )
- schema_def = {"index": {"unique": True}}
- errors = validate._validate_dataframe(df, "df", schema_def)
- self.assertEqual(len(errors), 1)
- self.assertIn("is not unique", errors[0])
-
- def test_recurse(self):
-
- df = pd.DataFrame(
- [["abc", "HsapDv:0001"],
- ["EFO:123", "UBERON:456"]],
- columns=["hr_string", "ontology"],
- index=["X", "Y"]
- )
- schema_def = {
- "columns": {
- "hr_string": {"type": "human-readable string"},
- "ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]}
- }
- }
- errors = validate._validate_dataframe(df, "df", schema_def)
- self.assertEqual(len(errors), 2)
- self.assertEqual(len([e for e in errors if "non-human-readable" in e]), 1)
- self.assertEqual(len([e for e in errors if "invalid ontology" in e]), 1)
-
-
-class TestGetSchema(unittest.TestCase):
-
- def test_get_schema(self):
- self.assertIsInstance(validate.get_schema_definition("1.0.0"), dict)
-
- with self.assertRaises(ValueError):
- validate.get_schema_definition("10.1.5")
-
-
-class TestValidate(unittest.TestCase):
-
- def setUp(self):
- self.source_h5ad_path = f"{PROJECT_ROOT}/backend/test/fixtures/pbmc3k-CSC-gz.h5ad"
-
- def test_shallow(self):
-
- adata = sc.read_h5ad(self.source_h5ad_path)
- self.assertFalse(validate.validate_adata(adata, True))
-
- adata.uns["version"] = {
- "corpora_schema_version": "1.0.0",
- "corpora_encoding_version": "0.1.0"
- }
- self.assertTrue(validate.validate_adata(adata, True))
-
- def test_deep(self):
- adata = sc.read_h5ad(self.source_h5ad_path)
- self.assertFalse(validate.validate_adata(adata, False))
-
- adata.uns["version"] = {
- "corpora_schema_version": "1.0.0",
- "corpora_encoding_version": "0.1.0"
- }
- self.assertFalse(validate.validate_adata(adata, False))
diff --git a/backend/test/test_czi_hosted/unit/converters/test_h5ad_data_file.py b/backend/test/test_czi_hosted/unit/converters/test_h5ad_data_file.py
deleted file mode 100644
index c750ca1e..00000000
--- a/backend/test/test_czi_hosted/unit/converters/test_h5ad_data_file.py
+++ /dev/null
@@ -1,284 +0,0 @@
-import json
-import unittest
-from glob import glob
-from os import remove, path
-from shutil import rmtree
-from uuid import uuid4
-
-import anndata
-import numpy as np
-from pandas import Series, DataFrame
-import tiledb
-
-from backend.czi_hosted.common.corpora import CorporaConstants
-from backend.czi_hosted.converters.h5ad_data_file import H5ADDataFile
-
-from backend.test import PROJECT_ROOT
-
-
-class TestH5ADDataFile(unittest.TestCase):
- def setUp(self):
- self.sample_anndata = self._create_sample_anndata_dataset()
- self.sample_h5ad_filename = self._write_anndata_to_file(self.sample_anndata)
-
- self.sample_output_directory = path.splitext(self.sample_h5ad_filename)[0] + ".cxg"
-
- def tearDown(self):
- if self.sample_h5ad_filename:
- remove(self.sample_h5ad_filename)
-
- if path.isdir(self.sample_output_directory):
- rmtree(self.sample_output_directory)
-
- def test__create_h5ad_data_file__non_h5ad_raises_exception(self):
- non_h5ad_filename = "my_fancy_dataset.csv"
-
- with self.assertRaises(Exception) as exception_context:
- H5ADDataFile(non_h5ad_filename)
-
- self.assertIn("File must be an H5AD", str(exception_context.exception))
-
- def test__create_h5ad_data_file__assert_warning_outputted_if_dataset_title_or_about_given(self):
- with self.assertLogs(level="WARN") as logger:
- H5ADDataFile(
- self.sample_h5ad_filename,
- dataset_title="My Awesome Dataset",
- dataset_about="http://www.awesomedataset.com",
- use_corpora_schema=False,
- )
-
- self.assertIn("will override any metadata that is extracted", logger.output[0])
-
- def test__create_h5ad_data_file__reads_anndata_successfully(self):
- h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False)
-
- self.assertTrue((h5ad_file.anndata.X == self.sample_anndata.X).all())
- self.assertEqual(
- h5ad_file.anndata.obs.sort_index(inplace=True), self.sample_anndata.obs.sort_index(inplace=True)
- )
- self.assertEqual(
- h5ad_file.anndata.var.sort_index(inplace=True), self.sample_anndata.var.sort_index(inplace=True)
- )
-
- for key in h5ad_file.anndata.obsm.keys():
- self.assertIn(key, self.sample_anndata.obsm.keys())
- self.assertTrue((h5ad_file.anndata.obsm[key] == self.sample_anndata.obsm[key]).all())
-
- for key in self.sample_anndata.obsm.keys():
- self.assertIn(key, h5ad_file.anndata.obsm.keys())
- self.assertTrue((h5ad_file.anndata.obsm[key] == self.sample_anndata.obsm[key]).all())
-
- def test__create_h5ad_data_file__copies_index_of_obs_and_var_to_column(self):
- h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False)
-
- # The automatic name chosen for the index should be "name_0"
- self.assertNotIn("name_0", self.sample_anndata.obs.columns)
- self.assertIn("name_0", h5ad_file.obs.columns)
-
- self.assertNotIn("name_0", self.sample_anndata.var.columns)
- self.assertIn("name_0", h5ad_file.var.columns)
-
- def test__create_h5ad_data_file__no_copy_if_obs_and_var_index_names_specified(self):
- h5ad_file = H5ADDataFile(
- self.sample_h5ad_filename,
- use_corpora_schema=False,
- obs_index_column_name="float_category",
- vars_index_column_name="int_category",
- )
-
- self.assertNotIn("name_0", h5ad_file.obs.columns)
- self.assertNotIn("name_0", h5ad_file.var.columns)
-
- def test__create_h5ad_data_file__obs_and_var_index_names_specified_not_unique_raises_exception(self):
-
- with self.assertRaises(Exception) as exception_context:
- H5ADDataFile(
- self.sample_h5ad_filename,
- use_corpora_schema=False,
- obs_index_column_name="float_category",
- vars_index_column_name="bool_category",
- )
-
- self.assertIn("Please prepare data to contain unique values", str(exception_context.exception))
-
- def test__create_h5ad_data_file__obs_and_var_index_names_specified_doesnt_exist_raises_exception(self):
- with self.assertRaises(Exception) as exception_context:
- H5ADDataFile(
- self.sample_h5ad_filename,
- use_corpora_schema=False,
- obs_index_column_name="unknown_category",
- vars_index_column_name="i_dont_exist",
- )
-
- self.assertIn("does not exist", str(exception_context.exception))
-
- def test__create_h5ad_data_file__extract_about_and_title_from_dataset(self):
- h5ad_file = H5ADDataFile(self.sample_h5ad_filename)
-
- self.assertEqual(h5ad_file.dataset_title, "random_link_name")
- self.assertEqual(h5ad_file.dataset_about, "www.link.com")
-
- def test__create_h5ad_data_file__inputted_dataset_title_and_about_overrides_extracted(self):
- h5ad_file = H5ADDataFile(
- self.sample_h5ad_filename, dataset_about="override_about", dataset_title="override_title"
- )
-
- self.assertEqual(h5ad_file.dataset_title, "override_title")
- self.assertEqual(h5ad_file.dataset_about, "override_about")
-
- def test__to_cxg__simple_anndata_no_corpora_and_sparse(self):
- h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False)
- h5ad_file.to_cxg(self.sample_output_directory, 100)
-
- self._validate_cxg_and_h5ad_content_match(self.sample_h5ad_filename, self.sample_output_directory, True)
-
- def test__to_cxg__simple_anndata_with_corpora_and_sparse(self):
- h5ad_file = H5ADDataFile(self.sample_h5ad_filename)
- h5ad_file.to_cxg(self.sample_output_directory, 100)
-
- self._validate_cxg_and_h5ad_content_match(self.sample_h5ad_filename, self.sample_output_directory, True)
-
- def test__to_cxg__simple_anndata_no_corpora_and_dense(self):
- h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False)
- h5ad_file.to_cxg(self.sample_output_directory, 0)
-
- self._validate_cxg_and_h5ad_content_match(self.sample_h5ad_filename, self.sample_output_directory, False)
-
- def test__to_cxg__simple_anndata_with_corpora_and_dense(self):
- h5ad_file = H5ADDataFile(self.sample_h5ad_filename)
- h5ad_file.to_cxg(self.sample_output_directory, 0)
-
- self._validate_cxg_and_h5ad_content_match(self.sample_h5ad_filename, self.sample_output_directory, False)
-
- def test__to_cxg__with_sparse_column_encoding(self):
- anndata = self._create_sample_anndata_dataset()
- anndata.X = np.ones((3, 4))
- sparse_with_column_shift_filename = self._write_anndata_to_file(anndata)
-
- h5ad_file = H5ADDataFile(sparse_with_column_shift_filename)
- h5ad_file.to_cxg(self.sample_output_directory, 50)
-
- self._validate_cxg_and_h5ad_content_match(
- sparse_with_column_shift_filename, self.sample_output_directory, False, has_column_encoding=True
- )
-
- # Clean up
- remove(sparse_with_column_shift_filename)
-
- def _validate_cxg_and_h5ad_content_match(self, h5ad_filename, cxg_directory, is_sparse, has_column_encoding=False):
- anndata_object = anndata.read_h5ad(h5ad_filename)
-
- # Array locations
- metadata_array_location = f"{cxg_directory}/cxg_group_metadata"
- main_x_array_location = f"{cxg_directory}/X"
- embedding_array_location = f"{cxg_directory}/emb"
- specific_embedding_array_location = f"{self.sample_output_directory}/emb/awesome_embedding"
- obs_array_location = f"{cxg_directory}/obs"
- var_array_location = f"{cxg_directory}/var"
- x_col_shift_array_location = f"{cxg_directory}/X_col_shift"
-
- # Assert CXG structure
- self.assertEqual(tiledb.object_type(cxg_directory), "group")
- self.assertEqual(tiledb.object_type(obs_array_location), "array")
- self.assertEqual(tiledb.object_type(var_array_location), "array")
- self.assertEqual(tiledb.object_type(main_x_array_location), "array")
- self.assertEqual(tiledb.object_type(embedding_array_location), "group")
- self.assertEqual(tiledb.object_type(specific_embedding_array_location), "array")
-
- if has_column_encoding:
- self.assertEqual(tiledb.object_type(x_col_shift_array_location), "array")
-
- # Validate metadata
- metadata_array = tiledb.DenseArray(metadata_array_location, mode="r")
- self.assertIn("cxg_version", metadata_array.meta)
-
- # Validate obs index
- obs_array = tiledb.DenseArray(obs_array_location, mode="r")
- expected_index_data = anndata_object.obs.index.to_numpy()
- index_name = json.loads(obs_array.meta["cxg_schema"])["index"]
- actual_index_data = obs_array.query(attrs=[index_name])[:][index_name]
- self.assertTrue(np.array_equal(expected_index_data, actual_index_data))
-
- # Validate obs columns
- expected_columns = list(anndata_object.obs.columns.values)
- for column_name in expected_columns:
- expected_data = anndata_object.obs[column_name].to_numpy()
- actual_data = obs_array.query(attrs=[column_name])[:][column_name]
- self.assertTrue(np.array_equal(expected_data, actual_data))
-
- # Validate var index
- var_array = tiledb.DenseArray(var_array_location, mode="r")
- expected_index_data = anndata_object.var.index.to_numpy()
- index_name = json.loads(var_array.meta["cxg_schema"])["index"]
- actual_index_data = var_array.query(attrs=[index_name])[:][index_name]
- self.assertTrue(np.array_equal(expected_index_data, actual_index_data))
-
- # Validate var columns
- expected_columns = anndata_object.var.columns.values
- for column_name in expected_columns:
- expected_data = anndata_object.var[column_name].to_numpy()
- actual_data = var_array.query(attrs=[column_name])[:][column_name]
- self.assertTrue(np.array_equal(expected_data, actual_data))
-
- # Validate embedding
- expected_embedding_data = anndata_object.obsm.get("X_awesome_embedding")
- embedding_array = tiledb.DenseArray(specific_embedding_array_location, mode="r")
- actual_embedding_data = embedding_array[:, 0:2]
- self.assertTrue(np.array_equal(expected_embedding_data, actual_embedding_data))
-
- # Validate X matrix if not column shifted
- if not has_column_encoding:
- expected_x_data = anndata_object.X
- if is_sparse:
- x_array = tiledb.SparseArray(main_x_array_location, mode="r")
- actual_x_data = np.reshape(x_array[:, :][""], expected_x_data.shape)
- else:
- x_array = tiledb.DenseArray(main_x_array_location, mode="r")
- actual_x_data = x_array[:, :]
- self.assertTrue(np.array_equal(expected_x_data, actual_x_data))
-
- def _write_anndata_to_file(self, anndata):
- temporary_filename = f"{PROJECT_ROOT}/backend/test/fixtures/{uuid4()}.h5ad"
- anndata.write(temporary_filename)
-
- return temporary_filename
-
- def _create_sample_anndata_dataset(self):
- # Create X
- X = np.random.rand(3, 4)
-
- # Create obs
- random_string_category = Series(data=["a", "b", "b"], dtype="category")
- random_float_category = Series(data=[3.2, 1.1, 2.2], dtype=np.float32)
- obs_dataframe = DataFrame(
- data={"string_category": random_string_category, "float_category": random_float_category}
- )
- obs = obs_dataframe
-
- # Create vars
- random_int_category = Series(data=[3, 1, 2, 4], dtype=np.int32)
- random_bool_category = Series(data=[True, True, False, True], dtype=np.bool_)
- var_dataframe = DataFrame(data={"int_category": random_int_category, "bool_category": random_bool_category})
- var = var_dataframe
-
- # Create embeddings
- random_embedding = np.random.rand(3, 2)
- obsm = {"X_awesome_embedding": random_embedding}
-
- # Create uns corpora metadata
- uns = {}
- for metadata_field in CorporaConstants.REQUIRED_SIMPLE_METADATA_FIELDS:
- uns[metadata_field] = "random"
-
- for metadata_field in CorporaConstants.OPTIONAL_JSON_ENCODED_METADATA_FIELD:
- uns[metadata_field] = json.dumps({"random_key": "random_value"})
-
- # Need to carefully set the corpora schema versions in order for tests to pass.
- uns["version"] = {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"}
-
- # Set project links to be a dictionary
- uns["project_links"] = json.dumps(
- [{"link_name": "random_link_name", "link_url": "www.link.com", "link_type": "SUMMARY"}]
- )
-
- return anndata.AnnData(X=X, obs=obs, var=var, obsm=obsm, uns=uns)
diff --git a/backend/test/test_czi_hosted/unit/data_anndata/__init__.py b/backend/test/test_czi_hosted/unit/data_anndata/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/backend/test/test_czi_hosted/unit/data_anndata/test_anndata_adaptor.py b/backend/test/test_czi_hosted/unit/data_anndata/test_anndata_adaptor.py
deleted file mode 100644
index 7788257e..00000000
--- a/backend/test/test_czi_hosted/unit/data_anndata/test_anndata_adaptor.py
+++ /dev/null
@@ -1,198 +0,0 @@
-import json
-import sys
-import time
-import unittest
-
-import numpy as np
-import pandas as pd
-import pytest
-from parameterized import parameterized_class
-
-from backend.common.utils.data_locator import DataLocator
-from backend.common.errors import FilterError
-from backend.czi_hosted.data_anndata.anndata_adaptor import AnndataAdaptor
-from backend.test.test_czi_hosted.unit import app_config
-from backend.test import PROJECT_ROOT, FIXTURES_ROOT, decode_fbs
-from backend.test.fixtures.fixtures import pbmc3k_colors
-
-"""
-Test the anndata adaptor using the pbmc3k data set.
-"""
-
-
-@parameterized_class(
- ("data_locator", "backed"),
- [
- (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", False),
- (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", False),
- (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", False),
- (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", True),
- (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", True),
- (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", True),
- ],
-)
-class AdaptorTest(unittest.TestCase):
- def setUp(self):
- config = app_config(self.data_locator, self.backed)
- self.data = AnndataAdaptor(DataLocator(self.data_locator), config)
-
- def test_init(self):
- self.assertEqual(self.data.cell_count, 2638)
- self.assertEqual(self.data.gene_count, 1838)
- epsilon = 0.000_005
- self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
-
- def test_mandatory_annotations(self):
- obs_index_col_name = self.data.get_schema()["annotations"]["obs"]["index"]
- self.assertIn(obs_index_col_name, self.data.data.obs)
- self.assertEqual(list(self.data.data.obs.index), list(range(2638)))
- var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
- self.assertIn(var_index_col_name, self.data.data.var)
- self.assertEqual(list(self.data.data.var.index), list(range(1838)))
-
- @pytest.mark.filterwarnings("ignore:Anndata data matrix")
- def test_data_type(self):
- # don't run the test on the more exotic data types, as they don't
- # support the astype() interface (used by this test, but not underlying app)
- if isinstance(self.data.data.X, np.ndarray):
- self.data.data.X = self.data.data.X.astype("float64")
- with self.assertWarns(UserWarning):
- self.data._validate_data_types()
-
- def test_filter_idx(self):
- filter_ = {"filter": {"var": {"index": [1, 99, [200, 300]]}}}
- fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
- data = decode_fbs.decode_matrix_FBS(fbs)
- self.assertEqual(data["n_rows"], 2638)
- self.assertEqual(data["n_cols"], 102)
-
- def test_filter_complex(self):
- filter_ = {
- "filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 10}], "index": [1, 99, [200, 300]]}}
- }
- fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
- data = decode_fbs.decode_matrix_FBS(fbs)
- self.assertEqual(data["n_rows"], 2638)
- self.assertEqual(data["n_cols"], 91)
-
- def test_obs_and_var_names(self):
- self.assertEqual(np.sum(self.data.data.var[self.data.get_schema()["annotations"]["var"]["index"]].isna()), 0)
- self.assertEqual(np.sum(self.data.data.obs[self.data.get_schema()["annotations"]["obs"]["index"]].isna()), 0)
-
- def test_get_colors(self):
- self.assertEqual(self.data.get_colors(), pbmc3k_colors)
-
- def test_get_schema(self):
- with open(f"{FIXTURES_ROOT}/schema.json") as fh:
- schema = json.load(fh)
- self.assertDictEqual(self.data.get_schema(), schema)
-
- def test_schema_produces_error(self):
- self.data.data.obs["time"] = pd.Series(
- list([time.time() for i in range(self.data.cell_count)]), dtype="datetime64[ns]",
- )
- with pytest.raises(TypeError):
- self.data._create_schema()
-
- def test_layout(self):
- fbs = self.data.layout_to_fbs_matrix(fields=None)
- layout = decode_fbs.decode_matrix_FBS(fbs)
- self.assertEqual(layout["n_cols"], 6)
- self.assertEqual(layout["n_rows"], 2638)
-
- X = layout["columns"][0]
- self.assertTrue((X >= 0).all() and (X <= 1).all())
- Y = layout["columns"][1]
- self.assertTrue((Y >= 0).all() and (Y <= 1).all())
-
- def test_layout_fields(self):
- """ X_pca, X_tsne, X_umap are available """
- fbs = self.data.layout_to_fbs_matrix(["pca"])
- layout = decode_fbs.decode_matrix_FBS(fbs)
- self.assertEqual(layout["n_cols"], 2)
- self.assertEqual(layout["n_rows"], 2638)
- self.assertCountEqual(layout["col_idx"], ["pca_0", "pca_1"])
-
- fbs = self.data.layout_to_fbs_matrix(["tsne", "pca"])
- layout = decode_fbs.decode_matrix_FBS(fbs)
- self.assertEqual(layout["n_cols"], 4)
- self.assertEqual(layout["n_rows"], 2638)
- self.assertCountEqual(layout["col_idx"], ["tsne_0", "tsne_1", "pca_0", "pca_1"])
-
- def test_annotations(self):
- fbs = self.data.annotation_to_fbs_matrix("obs")
- annotations = decode_fbs.decode_matrix_FBS(fbs)
- self.assertEqual(annotations["n_rows"], 2638)
- self.assertEqual(annotations["n_cols"], 5)
- obs_index_col_name = self.data.get_schema()["annotations"]["obs"]["index"]
- self.assertEqual(
- annotations["col_idx"], [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"],
- )
-
- fbs = self.data.annotation_to_fbs_matrix("var")
- annotations = decode_fbs.decode_matrix_FBS(fbs)
- self.assertEqual(annotations["n_rows"], 1838)
- self.assertEqual(annotations["n_cols"], 2)
- var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
- self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells"])
-
- def test_annotation_fields(self):
- fbs = self.data.annotation_to_fbs_matrix("obs", ["n_genes", "n_counts"])
- annotations = decode_fbs.decode_matrix_FBS(fbs)
- self.assertEqual(annotations["n_rows"], 2638)
- self.assertEqual(annotations["n_cols"], 2)
-
- var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
- fbs = self.data.annotation_to_fbs_matrix("var", [var_index_col_name])
- annotations = decode_fbs.decode_matrix_FBS(fbs)
- self.assertEqual(annotations["n_rows"], 1838)
- self.assertEqual(annotations["n_cols"], 1)
-
- def test_diffexp_topN(self):
- f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
- f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
- result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
- self.assertEqual(len(result['positive']), 10)
- self.assertEqual(len(result['negative']), 10)
- result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
- self.assertEqual(len(result['positive']), 20)
- self.assertEqual(len(result['negative']), 20)
-
- def test_data_frame(self):
- f1 = {"var": {"index": [[0, 10]]}}
- fbs = self.data.data_frame_to_fbs_matrix(f1, "var")
- data = decode_fbs.decode_matrix_FBS(fbs)
- self.assertEqual(data["n_rows"], 2638)
- self.assertEqual(data["n_cols"], 10)
-
- with self.assertRaises(ValueError):
- self.data.data_frame_to_fbs_matrix(None, "obs")
-
- def test_filtered_data_frame(self):
- filter_ = {"filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 100}]}}}
- fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
- data = decode_fbs.decode_matrix_FBS(fbs)
- self.assertEqual(data["n_rows"], 2638)
- self.assertEqual(data["n_cols"], 1040)
-
- filter_ = {"filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}}
- with self.assertRaises(FilterError):
- self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
-
- def test_data_named_gene(self):
- var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
- filter_ = {"filter": {"var": {"annotation_value": [{"name": var_index_col_name, "values": ["RER1"]}]}}}
- fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
- data = decode_fbs.decode_matrix_FBS(fbs)
- self.assertEqual(data["n_rows"], 2638)
- self.assertEqual(data["n_cols"], 1)
- self.assertEqual(data["col_idx"], [4])
-
- filter_ = {
- "filter": {"var": {"annotation_value": [{"name": var_index_col_name, "values": ["SPEN", "TYMP", "PRMT2"]}]}}
- }
- fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
- data = decode_fbs.decode_matrix_FBS(fbs)
- self.assertEqual(data["n_rows"], 2638)
- self.assertEqual(data["n_cols"], 3)
- self.assertTrue((data["col_idx"] == [15, 1818, 1837]).all())
diff --git a/backend/test/test_czi_hosted/unit/data_anndata/test_anndata_adaptor_data_load.py b/backend/test/test_czi_hosted/unit/data_anndata/test_anndata_adaptor_data_load.py
deleted file mode 100644
index 9a258198..00000000
--- a/backend/test/test_czi_hosted/unit/data_anndata/test_anndata_adaptor_data_load.py
+++ /dev/null
@@ -1,86 +0,0 @@
-import unittest
-import json
-
-from backend.common.utils.data_locator import DataLocator
-from backend.czi_hosted.data_anndata.anndata_adaptor import AnndataAdaptor
-from backend.czi_hosted.common.config.app_config import AppConfig
-from backend.test import PROJECT_ROOT
-
-
-class DataLoadAdaptorTest(unittest.TestCase):
- """
- Test file loading, including deferred loading/update.
- """
-
- def setUp(self):
- self.data_file = DataLocator(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
- config = AppConfig()
- config.update_server_config(single_dataset__datapath=self.data_file.path)
- config.update_server_config(app__flask_secret_key="secret")
- config.complete_config()
- self.data = AnndataAdaptor(self.data_file, config)
-
- def test_delayed_load_data(self):
- self.data._create_schema()
- self.assertEqual(self.data.cell_count, 2638)
- self.assertEqual(self.data.gene_count, 1838)
- epsilon = 0.000_005
- self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
-
- def test_diffexp_topN(self):
- f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
- f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
-
- result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
-
- self.assertEqual(len(result['positive']), 10)
- self.assertEqual(len(result['negative']), 10)
-
- result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
- self.assertEqual(len(result['positive']), 20)
- self.assertEqual(len(result['negative']), 20)
-
-
-class DataLocatorAdaptorTest(unittest.TestCase):
- """
- Test various types of data locators we expect to consume
- """
-
- def get_basic_config(self):
- config = AppConfig()
- config.update_server_config(
- single_dataset__obs_names=None, single_dataset__var_names=None,
- )
- config.update_server_config(app__flask_secret_key="secret")
- config.update_default_dataset_config(
- embeddings__names=["umap"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01,
- )
- return config
-
- def stdAsserts(self, data):
- """ run these each time we load the data """
- self.assertIsNotNone(data)
- self.assertEqual(data.cell_count, 2638)
- self.assertEqual(data.gene_count, 1838)
-
- def test_posix_file(self):
- locator = DataLocator("../../example-dataset/pbmc3k.h5ad")
- config = self.get_basic_config()
- config.update_server_config(single_dataset__datapath=locator.path)
- config.complete_config()
- data = AnndataAdaptor(locator, config)
- self.stdAsserts(data)
-
- def test_url_https(self):
- url = "https://raw.githubusercontent.com/chanzuckerberg/cellxgene/main/example-dataset/pbmc3k.h5ad"
- locator = DataLocator(url)
- config = self.get_basic_config()
- data = AnndataAdaptor(locator, config)
- self.stdAsserts(data)
-
- def test_url_http(self):
- url = "http://raw.githubusercontent.com/chanzuckerberg/cellxgene/main/example-dataset/pbmc3k.h5ad"
- locator = DataLocator(url)
- config = self.get_basic_config()
- data = AnndataAdaptor(locator, config)
- self.stdAsserts(data)
diff --git a/backend/test/test_czi_hosted/unit/data_anndata/test_nan_anndata_adaptor.py b/backend/test/test_czi_hosted/unit/data_anndata/test_nan_anndata_adaptor.py
deleted file mode 100644
index 16a20fe7..00000000
--- a/backend/test/test_czi_hosted/unit/data_anndata/test_nan_anndata_adaptor.py
+++ /dev/null
@@ -1,64 +0,0 @@
-import math
-import unittest
-import warnings
-
-import pytest
-
-from backend.common.utils.data_locator import DataLocator
-from backend.common.errors import FilterError
-from backend.czi_hosted.data_anndata.anndata_adaptor import AnndataAdaptor
-from backend.test.test_czi_hosted.unit import app_config
-from backend.test import FIXTURES_ROOT, decode_fbs
-
-
-class NaNTest(unittest.TestCase):
- def setUp(self):
- self.data_locator = DataLocator(f"{FIXTURES_ROOT}/nan.h5ad")
- self.config = app_config(self.data_locator.path)
-
- with warnings.catch_warnings():
- warnings.simplefilter("ignore", category=UserWarning)
- self.data = AnndataAdaptor(self.data_locator, self.config)
- self.data._create_schema()
-
- def test_load(self):
- with self.assertLogs(level="WARN") as logger:
- self.data = AnndataAdaptor(self.data_locator, self.config)
- self.assertTrue(logger.output)
-
- def test_init(self):
- self.assertEqual(self.data.cell_count, 100)
- self.assertEqual(self.data.gene_count, 100)
- epsilon = 0.000_005
- self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
-
- def test_dataframe(self):
- data_frame_var = decode_fbs.decode_matrix_FBS(self.data.data_frame_to_fbs_matrix(None, "var"))
- self.assertIsNotNone(data_frame_var)
- self.assertEqual(data_frame_var["n_rows"], 100)
- self.assertEqual(data_frame_var["n_cols"], 100)
- self.assertTrue(math.isnan(data_frame_var["columns"][3][3]))
-
- with pytest.raises(FilterError):
- self.data.data_frame_to_fbs_matrix("an erroneous filter", "var")
- with pytest.raises(FilterError):
- filter_ = {"filter": {"obs": {"index": [1, 99, [200, 300]]}}}
- self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
-
- def test_dataframe_obs_not_implemented(self):
- with self.assertRaises(ValueError) as cm:
- decode_fbs.decode_matrix_FBS(self.data.data_frame_to_fbs_matrix(None, "obs"))
- self.assertIsNotNone(cm.exception)
-
- def test_annotation(self):
- annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("obs"))
- obs_index_col_name = self.data.schema["annotations"]["obs"]["index"]
- self.assertEqual(annotations["col_idx"], [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"])
- self.assertEqual(annotations["n_rows"], 100)
- self.assertTrue(math.isnan(annotations["columns"][2][0]))
-
- annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("var"))
- var_index_col_name = self.data.schema["annotations"]["var"]["index"]
- self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells", "var_with_nans"])
- self.assertEqual(annotations["n_rows"], 100)
- self.assertTrue(math.isnan(annotations["columns"][2][0]))
diff --git a/backend/test/test_czi_hosted/unit/data_common/__init__.py b/backend/test/test_czi_hosted/unit/data_common/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/backend/test/test_czi_hosted/unit/data_common/fbs/__init__.py b/backend/test/test_czi_hosted/unit/data_common/fbs/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/backend/test/test_czi_hosted/unit/data_common/fbs/test_matrix.py b/backend/test/test_czi_hosted/unit/data_common/fbs/test_matrix.py
deleted file mode 100644
index 21389fe0..00000000
--- a/backend/test/test_czi_hosted/unit/data_common/fbs/test_matrix.py
+++ /dev/null
@@ -1,82 +0,0 @@
-import unittest
-import pandas as pd
-import numpy as np
-from scipy import sparse
-
-from backend.test import decode_fbs
-from backend.common.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs
-
-
-class FbsTests(unittest.TestCase):
- """Test Case for Matrix FBS data encode/decode """
-
- def test_encode_boundary(self):
- """ test various boundary checks """
-
- # row indexing is unsupported
- with self.assertRaises(ValueError):
- encode_matrix_fbs(matrix=pd.DataFrame(), row_idx=[])
-
- # matrix must be 2D
- with self.assertRaises(ValueError):
- encode_matrix_fbs(matrix=np.zeros((3, 2, 1)))
- with self.assertRaises(ValueError):
- encode_matrix_fbs(matrix=np.ones((10,)))
-
- def fbs_checks(self, fbs, dims, expected_types, expected_column_idx):
- d = decode_fbs.decode_matrix_FBS(fbs)
- self.assertEqual(d["n_rows"], dims[0])
- self.assertEqual(d["n_cols"], dims[1])
- self.assertIsNone(d["row_idx"])
- self.assertEqual(len(d["columns"]), dims[1])
- for i in range(0, len(d["columns"])):
- self.assertEqual(len(d["columns"][i]), dims[0])
- self.assertIsInstance(d["columns"][i], expected_types[i][0])
- if expected_types[i][1] is not None:
- self.assertEqual(d["columns"][i].dtype, expected_types[i][1])
- if expected_column_idx is not None:
- self.assertSetEqual(set(expected_column_idx), set(d["col_idx"]))
-
- def test_encode_DataFrame(self):
- df = pd.DataFrame(
- data={
- "a": np.zeros((10,), dtype=np.float32),
- "b": np.ones((10,), dtype=np.int64),
- "c": np.array([i for i in range(0, 10)], dtype=np.uint16),
- "d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"),
- }
- )
- expected_types = ((np.ndarray, np.float32), (np.ndarray, np.int32), (np.ndarray, np.int32), (list, None))
- fbs = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
- self.fbs_checks(fbs, (10, 4), expected_types, ["a", "b", "c", "d"])
-
- def test_encode_ndarray(self):
- arr = np.zeros((3, 2), dtype=np.float32)
- expected_types = ((np.ndarray, np.float32), (np.ndarray, np.float32), (np.ndarray, np.float32))
- fbs = encode_matrix_fbs(matrix=arr, row_idx=None, col_idx=None)
- self.fbs_checks(fbs, (3, 2), expected_types, None)
-
- def test_encode_sparse(self):
- csc = sparse.csc_matrix(np.array([[0, 1, 2], [3, 0, 4]]))
- expected_types = ((np.ndarray, np.int32), (np.ndarray, np.int32), (np.ndarray, np.int32))
- fbs = encode_matrix_fbs(matrix=csc, row_idx=None, col_idx=None)
- self.fbs_checks(fbs, (2, 3), expected_types, None)
-
- def test_roundtrip(self):
- dfSrc = pd.DataFrame(
- data={
- "a": np.zeros((10,), dtype=np.float32),
- "b": np.ones((10,), dtype=np.int64),
- "c": np.array([i for i in range(0, 10)], dtype=np.uint16),
- "d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"),
- }
- )
- dfDst = decode_matrix_fbs(encode_matrix_fbs(matrix=dfSrc, col_idx=dfSrc.columns))
- self.assertEqual(dfSrc.shape, dfDst.shape)
- self.assertEqual(set(dfSrc.columns), set(dfDst.columns))
- for c in dfSrc.columns:
- self.assertTrue(c in dfDst.columns)
- if isinstance(dfSrc[c], pd.Series):
- self.assertTrue(np.all(dfSrc[c] == dfDst[c]))
- else:
- self.assertEqual(dfSrc[c], dfDst[c])
diff --git a/backend/test/test_czi_hosted/unit/data_common/test_matrix_loader.py b/backend/test/test_czi_hosted/unit/data_common/test_matrix_loader.py
deleted file mode 100644
index 3b39e7c3..00000000
--- a/backend/test/test_czi_hosted/unit/data_common/test_matrix_loader.py
+++ /dev/null
@@ -1,126 +0,0 @@
-import os
-import shutil
-import tempfile
-import time
-import unittest
-
-from backend.czi_hosted.common.config.app_config import AppConfig
-from backend.common.errors import DatasetAccessError
-from backend.czi_hosted.data_common.matrix_loader import MatrixDataCacheManager
-from backend.test import FIXTURES_ROOT
-
-
-class MatrixCacheTest(unittest.TestCase):
- def setup(self):
- pass
-
- def make_temporay_datasets(self, dirname, num):
- source = f"{FIXTURES_ROOT}/pbmc3k.cxg"
- for i in range(num):
- target = os.path.join(dirname, str(i) + ".cxg")
- shutil.copytree(source, target)
-
- def use_dataset(self, matrix_cache, dirname, app_config, dataset_index):
- with matrix_cache.data_adaptor(None, os.path.join(dirname, str(dataset_index) + ".cxg"), app_config) as adaptor:
- pass
- return adaptor
-
- def use_dataset_with_error(self, matrix_cache, dirname, app_config, dataset_index):
- try:
- with matrix_cache.data_adaptor(None, os.path.join(dirname, str(dataset_index) + ".cxg"), app_config):
- raise DatasetAccessError("something bad happened")
- except DatasetAccessError:
- # the MatrixDataCacheManager rethrows the exception, so catch and ignore
- pass
-
- def get_datasets(self, matrix_cache, dirname):
- datasets = matrix_cache.datasets
- result = {}
- for k, v in datasets.items():
- # filter out the dirname and the .cxg from the name
- newk = int(k[1][len(dirname) + 1 : -4])
- result[newk] = v
-
- return result
-
- def check_datasets(self, matrix_cache, dirname, expected):
- res = self.get_datasets(matrix_cache, dirname)
- actual = res.keys()
- self.assertSetEqual(set(actual), set(expected))
-
- def test_basic(self):
- with tempfile.TemporaryDirectory() as dirname:
- self.make_temporay_datasets(dirname, 5)
- app_config = AppConfig()
- m = MatrixDataCacheManager(max_cached=3, timelimit_s=None)
-
- # should have only dataset 0
- self.use_dataset(m, dirname, app_config, 0)
- self.check_datasets(m, dirname, [0])
-
- # should have datasets 0, 1
- self.use_dataset(m, dirname, app_config, 1)
- self.check_datasets(m, dirname, [0, 1])
-
- # should have datasets 0, 1, 2
- self.use_dataset(m, dirname, app_config, 2)
- self.check_datasets(m, dirname, [0, 1, 2])
-
- # should have datasets 1, 2, 3
- self.use_dataset(m, dirname, app_config, 3)
- self.check_datasets(m, dirname, [1, 2, 3])
-
- # use dataset 1, making is more recent than dataset 2
- self.use_dataset(m, dirname, app_config, 1)
- self.check_datasets(m, dirname, [1, 2, 3])
-
- # use dataset 4, should have 1,3,4
- self.use_dataset(m, dirname, app_config, 4)
- self.check_datasets(m, dirname, [1, 3, 4])
-
- # use dataset 4 a few more times, get the count to 3
- self.use_dataset(m, dirname, app_config, 4)
- self.use_dataset(m, dirname, app_config, 4)
-
- datasets = self.get_datasets(m, dirname)
- self.assertEqual(datasets[1].num_access, 2)
- self.assertEqual(datasets[3].num_access, 1)
- self.assertEqual(datasets[4].num_access, 3)
-
- def test_timelimit(self):
- with tempfile.TemporaryDirectory() as dirname:
- self.make_temporay_datasets(dirname, 2)
-
- app_config = AppConfig()
- m = MatrixDataCacheManager(max_cached=3, timelimit_s=1)
-
- adaptor = self.use_dataset(m, dirname, app_config, 0)
- adaptor1 = self.use_dataset(m, dirname, app_config, 0)
- self.assertTrue(adaptor is adaptor1)
-
- # wait until the timelimit expires and check that there is a new adaptor
- time.sleep(1.1)
- adaptor2 = self.use_dataset(m, dirname, app_config, 0)
- self.assertTrue(adaptor is not adaptor2)
- self.check_datasets(m, dirname, [0])
-
- # now load a different dataset and see if dataset 0 gets evicted
- time.sleep(1.1)
- self.use_dataset(m, dirname, app_config, 1)
- self.check_datasets(m, dirname, [1])
-
- def test_access_error(self):
- with tempfile.TemporaryDirectory() as dirname:
- self.make_temporay_datasets(dirname, 1)
-
- app_config = AppConfig()
- m = MatrixDataCacheManager(max_cached=3, timelimit_s=1)
-
- # use the 0 datasets
- self.use_dataset(m, dirname, app_config, 0)
- self.check_datasets(m, dirname, [0])
-
- # use the 0 datasets, but this time a DatasetAccessError is raised.
- # verify that dataset is removed from the cache.
- self.use_dataset_with_error(m, dirname, app_config, 0)
- self.check_datasets(m, dirname, [])
diff --git a/backend/test/test_czi_hosted/unit/data_cxg/__init__.py b/backend/test/test_czi_hosted/unit/data_cxg/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/backend/test/test_czi_hosted/unit/data_cxg/test_cxg_adaptor.py b/backend/test/test_czi_hosted/unit/data_cxg/test_cxg_adaptor.py
deleted file mode 100644
index 8775a1f3..00000000
--- a/backend/test/test_czi_hosted/unit/data_cxg/test_cxg_adaptor.py
+++ /dev/null
@@ -1,20 +0,0 @@
-import unittest
-
-from backend.common.utils.data_locator import DataLocator
-from backend.czi_hosted.data_cxg.cxg_adaptor import CxgAdaptor
-from backend.test.test_czi_hosted.unit import app_config
-from backend.test import FIXTURES_ROOT
-from backend.test.fixtures.fixtures import pbmc3k_colors
-
-
-class TestCxgAdaptor(unittest.TestCase):
- def test_get_colors(self):
- data = self.get_data("pbmc3k.cxg")
- self.assertDictEqual(data.get_colors(), pbmc3k_colors)
- data = self.get_data("pbmc3k_v0.cxg")
- self.assertDictEqual(data.get_colors(), dict())
-
- def get_data(self, fixture):
- data_locator = f"{FIXTURES_ROOT}/{fixture}"
- config = app_config(data_locator)
- return CxgAdaptor(DataLocator(data_locator), config)
diff --git a/backend/test/test_czi_hosted/unit/eb/__init__.py b/backend/test/test_czi_hosted/unit/eb/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/backend/test/test_czi_hosted/unit/eb/test_eb.py b/backend/test/test_czi_hosted/unit/eb/test_eb.py
deleted file mode 100644
index 9dba1bc9..00000000
--- a/backend/test/test_czi_hosted/unit/eb/test_eb.py
+++ /dev/null
@@ -1,82 +0,0 @@
-import os
-from unittest.mock import patch
-
-import requests
-import subprocess
-import tempfile
-import time
-import unittest
-
-from contextlib import contextmanager
-from backend.czi_hosted.common.config.app_config import AppConfig
-from backend.test import PROJECT_ROOT, FIXTURES_ROOT
-
-
-@contextmanager
-def run_eb_app(tempdirname):
- ps = subprocess.Popen(["python", "artifact.dir/application.py"], cwd=tempdirname)
- server = "http://localhost:5000"
-
- for _ in range(10):
- try:
- requests.get(f"{server}/health")
- break
- except requests.exceptions.ConnectionError:
- time.sleep(1)
-
- try:
- yield server
- finally:
- try:
- ps.terminate()
- except ProcessLookupError:
- pass
-
-
-class Elastic_Beanstalk_Test(unittest.TestCase):
- def test_run(self):
- tempdir = tempfile.TemporaryDirectory(dir=f"{PROJECT_ROOT}/backend/czi_hosted")
- tempdirname = tempdir.name
- config = AppConfig()
- # test that eb works
- config.update_server_config(multi_dataset__dataroot=f"{FIXTURES_ROOT}", app__flask_secret_key="open sesame")
- config.complete_config()
- config.write_config(f"{tempdirname}/config.yaml")
- subprocess.check_call(f"git ls-files . | cpio -pdm {tempdirname}", cwd=f"{PROJECT_ROOT}/backend/czi_hosted/eb",
- shell=True)
- subprocess.check_call(["make", "build"], cwd=tempdirname)
- with run_eb_app(tempdirname) as server:
- session = requests.Session()
- response = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config")
- data_config = response.json()
- assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
-
- def test_config(self):
- check_config_script = os.path.join(PROJECT_ROOT, "backend", "czi_hosted", "eb", "check_config.py")
- with tempfile.TemporaryDirectory() as tempdir:
- configfile = os.path.join(tempdir, "config.yaml")
- app_config = AppConfig()
- app_config.update_server_config(multi_dataset__dataroot=f"{FIXTURES_ROOT}")
- app_config.write_config(configfile)
-
- command = ["python", check_config_script, configfile]
-
- # test failure mode (flask_secret_key not set)
- env = os.environ.copy()
- env.pop("CXG_SECRET_KEY", None)
- with self.assertRaises(subprocess.CalledProcessError) as exception_context:
- subprocess.check_output(command, env=env)
- output = str(exception_context.exception.stdout, "utf-8")
- self.assertTrue(
- output.startswith(
- "Error: Invalid type for attribute: app__flask_secret_key, expected type str, got NoneType"
- )
- )
- self.assertEqual(exception_context.exception.returncode, 1)
-
- # test passing case
- env = os.environ.copy()
- env["CXG_SECRET_KEY"] = "secret"
- output = subprocess.check_output(command, env=env)
- output = str(output, "utf-8")
- self.assertTrue(output.startswith("PASS"))
diff --git a/client/__tests__/e2e/e2e.test.js b/client/__tests__/e2e/e2e.test.js
index bd3e852a..d8a4b5e9 100644
--- a/client/__tests__/e2e/e2e.test.js
+++ b/client/__tests__/e2e/e2e.test.js
@@ -16,7 +16,6 @@ import {
getTestId,
goToPage,
waitByID,
- clickOnUntil,
} from "./puppeteerUtils";
import {
@@ -26,8 +25,6 @@ import {
getAllCategoriesAndCounts,
getCellSetCount,
selectCategory,
- login,
- logout,
} from "./cellxgeneActions";
const data = datasets[DATASET];
@@ -351,34 +348,4 @@ test("lasso moves after pan", async () => {
expect(panCount).toBe(initialCount);
});
-
-const describeIfCalledByMakeFileTarget =
- process.env.CXG_AUTH_TYPE?.toLowerCase() === "test"
- ? describe
- : describe.skip;
-
-describeIfCalledByMakeFileTarget("auth buttons", () => {
- test("login then logout", async () => {
- await goToPage(appUrlBase);
- await clickOnUntil("log-in", async () => {
- await page.waitForNavigation({ waitUntil: "networkidle0" });
- await waitByID("user-info");
- });
- await logout();
- });
-});
-
-const conditionalDescribe =
- process.env.TEST_AUTH_INTEGRATION === "true" ? describe : describe.skip;
-
-conditionalDescribe("AuthN Integration", () => {
- it("logs in", async () => {
- await login();
- });
-
- it("logs out", async () => {
- await login();
- await logout();
- });
-});
/* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */
diff --git a/setup.cfg b/setup.cfg
index 5841d387..7913312c 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,4 +1,4 @@
[flake8]
max-line-length = 120
ignore = E203, W503
-exclude = backend/common/fbs/NetEncoding/,.git,__pycache__,venv,backend/czi_hosted/venv,old,build,dist,backend/czi_hosted/eb/artifact.dir/
+exclude = backend/common/fbs/NetEncoding/,.git,__pycache__,venv,old,build,dist
diff --git a/setup_hosted.py b/setup_hosted.py
deleted file mode 100644
index d09b5eb5..00000000
--- a/setup_hosted.py
+++ /dev/null
@@ -1,43 +0,0 @@
-from setuptools import setup, find_packages
-
-with open("README.md", "rb") as fh:
- long_description = fh.read().decode()
-
-with open("backend/czi_hosted/requirements.txt") as fh:
- requirements = fh.read().splitlines()
-
-with open("backend/czi_hosted/requirements-prepare.txt") as fh:
- requirements_prepare = fh.read().splitlines()
-
-setup(
- name="cellxgene",
- version="0.16.0",
- packages=find_packages(),
- url="https://github.com/chanzuckerberg/cellxgene",
- license="MIT",
- author="Chan Zuckerberg Initiative",
- author_email="cellxgene@chanzuckerberg.com",
- description="Web application for exploration of large scale scRNA-seq datasets",
- long_description=long_description,
- long_description_content_type="text/markdown",
- install_requires=requirements,
- include_package_data=True,
- zip_safe=False,
- classifiers=[
- "Framework :: Flask",
- "Intended Audience :: Science/Research",
- "License :: OSI Approved :: MIT License",
- "Natural Language :: English",
- "Operating System :: POSIX",
- "Operating System :: Unix",
- "Operating System :: MacOS :: MacOS X",
- "Programming Language :: JavaScript",
- "Programming Language :: Python :: 3",
- "Programming Language :: Python :: 3.6",
- "Programming Language :: Python :: 3.7",
- "Programming Language :: Python :: 3 :: Only",
- "Topic :: Scientific/Engineering :: Bio-Informatics",
- ],
- entry_points={"console_scripts": ["cellxgene = backend.czi_hosted.cli.cli:cli"]},
- extras_require=dict(prepare=requirements_prepare),
-)