diff --git a/.github/workflows/push_tests.yml b/.github/workflows/push_tests.yml index 70fcf947..0e21c56b 100644 --- a/.github/workflows/push_tests.yml +++ b/.github/workflows/push_tests.yml @@ -36,7 +36,7 @@ jobs: npm install - name: Format with black and lint with flake8 run: | - make lint-server + make lint-servers - name: Lint src with eslint working-directory: ./client run: | @@ -72,6 +72,36 @@ jobs: bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k server -cF backend,python,unitTest cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest + unit-test-local: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.7 + uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: Python cache + uses: actions/cache@v1 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Node cache + uses: actions/cache@v1 + with: + path: ~/.npm + key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node- + - name: Install dependencies + run: make pydist-local install-dist dev-env-local-server + - name: Unit tests + run: | + make unit-test-local + bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k local_server -cF backend,python,unitTest + cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest + smoke-tests: runs-on: macos-latest timeout-minutes: 20 @@ -126,7 +156,7 @@ jobs: restore-keys: | ${{ runner.os }}-node- - name: Install dependencies - run: make pydist install-dist + run: make pydist-local install-dist - name: Smoke tests (with annotations feature) run: | cd client && make smoke-test-annotations diff --git a/MANIFEST.in b/MANIFEST.in index 3a3b5b2b..4dbe4057 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,7 @@ -recursive-include server/common/web/templates * -recursive-include server/common/web/static * +recursive-include local_server/common/web/templates * +recursive-include local_server/common/web/static * -include server/requirements.txt -include server/requirements-prepare.txt -include server/converters/schema/hgnc_complete_set.txt.gz -include server/converters/schema/schema_definitions/* +include local_server/requirements.txt +include local_server/requirements-prepare.txt +include local_server/converters/schema/hgnc_complete_set.txt.gz +include local_server/converters/schema/schema_definitions/* diff --git a/MANIFEST_hosted.in b/MANIFEST_hosted.in new file mode 100644 index 00000000..3a3b5b2b --- /dev/null +++ b/MANIFEST_hosted.in @@ -0,0 +1,7 @@ +recursive-include server/common/web/templates * +recursive-include server/common/web/static * + +include server/requirements.txt +include server/requirements-prepare.txt +include server/converters/schema/hgnc_complete_set.txt.gz +include server/converters/schema/schema_definitions/* diff --git a/Makefile b/Makefile index 8f9a4083..e987cee9 100644 --- a/Makefile +++ b/Makefile @@ -3,13 +3,14 @@ include common.mk BUILDDIR := build CLIENTBUILD := $(BUILDDIR)/client SERVERBUILD := $(BUILDDIR)/server +LOCALSERVERBUILD := $(BUILDDIR)/local_server CLEANFILES := $(BUILDDIR)/ client/build build dist cellxgene.egg-info PART ?= patch # CLEANING .PHONY: clean -clean: clean-lite clean-server clean-client +clean: clean-lite clean-local-server clean-server clean-client # cleaning the client's node_modules is the longest one, so we avoid that if possible .PHONY: clean-lite @@ -17,7 +18,7 @@ clean-lite: rm -rf $(CLEANFILES) clean-%: - cd $(*) && $(MAKE) clean + cd $(subst -,_,$*) && $(MAKE) clean # BUILDING PACKAGE @@ -26,18 +27,35 @@ clean-%: build-client: cd client && $(MAKE) ci build +.PHONY: build-local +build-local: clean build-client + git ls-files local_server/ | grep -v 'local_server/test/' | cpio -pdm $(BUILDDIR) + cp -r client/build/ $(CLIENTBUILD) + $(call copy_client_assets,$(CLIENTBUILD),$(LOCALSERVERBUILD)) + cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR) + .PHONY: build build: clean build-client git ls-files server/ | grep -v 'server/test/' | cpio -pdm $(BUILDDIR) cp -r client/build/ $(CLIENTBUILD) $(call copy_client_assets,$(CLIENTBUILD),$(SERVERBUILD)) - cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR) + cp MANIFEST_hosted.in README.md setup.cfg setup_hosted.py $(BUILDDIR) + mv $(BUILDDIR)/setup_hosted.py $(BUILDDIR)/setup.py + mv $(BUILDDIR)/MANIFEST_hosted.in $(BUILDDIR)/MANIFEST.in # If you are actively developing in the server folder use this, dirties the source tree +.PHONY: build-for-server-dev-local +build-for-server-dev-local: clean-local-server build-client + $(call copy_client_assets,client/build,local_server) + .PHONY: build-for-server-dev build-for-server-dev: clean-server build-client $(call copy_client_assets,client/build,server) +.PHONY: copy-client-assets-local +copy-client-assets-local: + $(call copy_client_assets,client/build,local_server) + .PHONY: copy-client-assets copy-client-assets: $(call copy_client_assets,client/build,server) @@ -46,11 +64,17 @@ copy-client-assets: .PHONY: test test: unit-test smoke-test +.PHONY: test-local +test-local: unit-test-local smoke-test + +.PHONY: unit-test-local +unit-test-local: unit-test-local-server unit-test-client + .PHONY: unit-test unit-test: unit-test-server unit-test-client unit-test-%: - cd $(*) && $(MAKE) unit-test + cd $(subst -,_,$*) && $(MAKE) unit-test .PHONY: smoke-test smoke-test: @@ -60,14 +84,17 @@ smoke-test: smoke-test-annotations: cd client && $(MAKE) smoke-test-annotations +.PHONY: test-db-local +test-db-local: + cd local_server && $(MAKE) test-db + .PHONY: test-db test-db: cd server && $(MAKE) test-db - # FORMATTING CODE -.PHOHY: fmt +.PHONY: fmt fmt: fmt-client fmt-py .PHONY: fmt-client @@ -79,19 +106,30 @@ fmt-py: black . .PHONY: lint -lint: lint-server lint-client +lint: lint-servers lint-client + +.PHONY: lint-servers +lint-servers: lint-local-server lint-server + +.PHONY: lint-local-server +lint-local-server: fmt-py + flake8 local_server --per-file-ignores='local_server/test/fixtures/dataset_config_outline.py:F821 local_server/test/fixtures/server_config_outline.py:F821 local_server/test/performance/scale_test_annotations.py:E501' .PHONY: lint-server lint-server: fmt-py flake8 server --per-file-ignores='server/test/fixtures/dataset_config_outline.py:F821 server/test/fixtures/server_config_outline.py:F821 server/test/performance/scale_test_annotations.py:E501' - .PHONY: lint-client lint-client: cd client && $(MAKE) lint # CREATING DISTRIBUTION RELEASE +.PHONY: pydist-local +pydist-local: build-local + cd $(BUILDDIR); python setup.py sdist -d ../dist + @echo "done" + .PHONY: pydist pydist: build cd $(BUILDDIR); python setup.py sdist -d ../dist @@ -137,16 +175,19 @@ release-directly-to-prod: dev-env pydist twine-prod @echo " make install-release" .PHONY: dev-env -dev-env: dev-env-client dev-env-server +dev-env: dev-env-client dev-env-local-server .PHONY: dev-env-client dev-env-client: cd client && $(MAKE) ci +.PHONY: dev-env-local-server +dev-env-local-server: + pip install -r local_server/requirements-dev.txt + .PHONY: dev-env-server dev-env-server: pip install -r server/requirements-dev.txt - # Set PART=[major, minor, patch] as param to make bump. # This will create a release candidate. (i.e. 0.16.1 -> 0.16.2-rc.0 for a patch bump) .PHONY: bump-version diff --git a/client/Makefile b/client/Makefile index 1a8d3a81..ad1ed798 100644 --- a/client/Makefile +++ b/client/Makefile @@ -1,6 +1,6 @@ include ../common.mk -ANNOTATIONS := $(if $(ANNOTATIONS),$(ANNOTATIONS),../server/test/fixtures/pbmc3k-annotations.csv) +ANNOTATIONS := $(if $(ANNOTATIONS),$(ANNOTATIONS),../local_server/test/fixtures/pbmc3k-annotations.csv) ANNOTATIONS_FILENAME := $(shell basename $(ANNOTATIONS)) CXG_CONFIG := $(if $(CXG_CONFIG), $(CXG_CONFIG), ./__tests__/e2e/test_config.yaml) diff --git a/client/__tests__/e2e/test_config.yaml b/client/__tests__/e2e/test_config.yaml index 11576878..0c37dafb 100644 --- a/client/__tests__/e2e/test_config.yaml +++ b/client/__tests__/e2e/test_config.yaml @@ -1,20 +1,7 @@ server: app: force_https: true - - # By default, cellxgene will serve api requests from the same base url as the webpage. - # In general api_base_url and web_base_url will not need to be set. - # There are two reasons to set these parameters: - # 1. Oauth authentication is used; the oauth server will redirect back to the api_base_url after login, - # which then redirects back to the web_base_url. If the web_base_url is not set, it will default to - # the api_base_url. If oauth authentication is used, the api_base_url must be set. - # For a local test (where the server runs on "http://localhost:"), then the api_base_url may be - # set to the string "local". - # 2. The cellxgene deploymnent is in an environment where the webpage and api have - # different base urls. In this case both api_base_url and web_base_url must be set. - # It is up to the server admin to ensure that the networking is setup correctly for this environment. - api_base_url: http://localhost:5005 - web_base_url: http://localhost:3000 + port: 5005 authentication: # The authentication types may be "none", "session", "oauth" @@ -22,12 +9,9 @@ server: # session: A session based userid is automatically generated. (no params needed) # oauth: oauth2 is used for authentication; parameters are defined in params_oauth. type: test + insecure_test_environment: true dataset: - app: - about_legal_tos: null - about_legal_privacy: null - presentation: max_categories: 1000 custom_colors: true diff --git a/local_server/Makefile b/local_server/Makefile new file mode 100644 index 00000000..872b9ae5 --- /dev/null +++ b/local_server/Makefile @@ -0,0 +1,49 @@ +include ../common.mk + +.PHONY: clean +clean: + rm -f common/web/templates/index.html + rm -rf common/web/static + rm -f common/web/csp-hashes.json + +.PHONY: unit-test +unit-test: create-test-db + PYTHONWARNINGS=ignore:ResourceWarning coverage run \ + --source=app,cli,common,compute,converters,data_anndata,data_common \ + --omit=.coverage,data_common/fbs/NetEncoding,venv \ + -m unittest discover \ + --start-directory test/ \ + --top-level-directory ../ \ + --verbose; test_result=$$?; \ + $(MAKE) clean-test-db; \ + exit $$test_result \ + + +.PHONY: test-db +test-db: create-test-db + PYTHONWARNINGS=ignore:ResourceWarning coverage run \ + --source=app,cli,common,compute,converters,data_anndata,data_common \ + --omit=.coverage,data_common/fbs/NetEncoding,venv \ + -m unittest discover \ + --start-directory test/test_database \ + --top-level-directory ../ \ + --verbose; test_result=$$?; \ + $(MAKE) clean-test-db; \ + exit $$test_result + +.PHONY: create-test-db +create-test-db: + -docker run -d -p 5432:5432 --name test_db -e POSTGRES_PASSWORD=test_pw postgres + +.PHONY: clean-test-db +clean-test-db: + -docker stop test_db + -docker rm test_db + +.PHONY: test-annotations-performance +test-annotations-performance: + python test/performance/performance_test_annotations_backend.py + +.PHONY: test-annotations-scale +test-annotations-scale: + locust -f test/performance/scale_test_annotations.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt diff --git a/local_server/__init__.py b/local_server/__init__.py new file mode 100644 index 00000000..7c3e1b4e --- /dev/null +++ b/local_server/__init__.py @@ -0,0 +1,13 @@ +import logging +import sys +from local_server.common.utils.utils import import_plugins + +__version__ = "0.16.0" +display_version = "cellxgene v" + __version__ + +try: + import_plugins("server.plugins") +except Exception as e: + # Make sure to exit in this case, as the server may not be configured as expected. + logging.critical(f"Error in import_plugins: {str(e)}") + sys.exit(1) diff --git a/local_server/__main__.py b/local_server/__main__.py new file mode 100644 index 00000000..b582e892 --- /dev/null +++ b/local_server/__main__.py @@ -0,0 +1,15 @@ +# Work around bug https://github.com/pallets/werkzeug/issues/461 +if __package__ is None: + import sys + from pathlib import Path + + PKG_PATH = Path(__file__).parent + sys.path.insert(0, str(PKG_PATH.parent)) + import server # noqa F401 + + __package__ = PKG_PATH.name + +# Main thing +from .cli.cli import cli # noqa F402 + +cli() diff --git a/local_server/app/__init__.py b/local_server/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/app/app.py b/local_server/app/app.py new file mode 100644 index 00000000..84d37565 --- /dev/null +++ b/local_server/app/app.py @@ -0,0 +1,225 @@ +import datetime +import logging +from functools import wraps +from http import HTTPStatus + +from flask import ( + Flask, + current_app, + make_response, + render_template, + Blueprint, + request, + send_from_directory, +) +from flask_restful import Api, Resource + +import local_server.common.rest as common_rest +from local_server.common.errors import DatasetAccessError, RequestException +from local_server.common.health import health_check +from local_server.common.utils.utils import Float32JSONEncoder + +webbp = Blueprint("webapp", "local_server.common.web", template_folder="templates") + + +@webbp.route("/", methods=["GET"]) +def dataset_index(): + app_config = current_app.app_config + + dataset_config = app_config.get_dataset_config() + scripts = dataset_config.app__scripts + inline_scripts = dataset_config.app__inline_scripts + + try: + args = {"SCRIPTS": scripts, "INLINE_SCRIPTS": inline_scripts} + return render_template("index.html", **args) + + except DatasetAccessError as e: + return common_rest.abort_and_log( + e.status_code, f"Invalid dataset: {e.message}", loglevel=logging.INFO, include_exc_info=True + ) + + +@webbp.errorhandler(RequestException) +def handle_request_exception(error): + return common_rest.abort_and_log(error.status_code, error.message, loglevel=logging.INFO, include_exc_info=True) + + +def requires_authentication(func): + @wraps(func) + def wrapped_function(self, *args, **kwargs): + auth = current_app.auth + if auth.is_user_authenticated(): + return func(self, *args, **kwargs) + else: + return make_response("not authenticated", HTTPStatus.UNAUTHORIZED) + + return wrapped_function + + +def rest_get_data_adaptor(func): + @wraps(func) + def wrapped_function(self): + try: + return func(self, current_app.data_adaptor) + except DatasetAccessError as e: + return common_rest.abort_and_log( + e.status_code, f"Invalid dataset: {e.message}", loglevel=logging.INFO, include_exc_info=True + ) + + return wrapped_function + + +class HealthAPI(Resource): + def get(self): + config = current_app.app_config + return health_check(config) + + +class SchemaAPI(Resource): + # TODO @mdunitz separate dataset schema and user schema + @rest_get_data_adaptor + def get(self, data_adaptor): + return common_rest.schema_get(data_adaptor) + + +class ConfigAPI(Resource): + @rest_get_data_adaptor + def get(self, data_adaptor): + return common_rest.config_get(current_app.app_config, data_adaptor) + + +class UserInfoAPI(Resource): + @rest_get_data_adaptor + def get(self, data_adaptor): + return common_rest.userinfo_get(current_app.app_config, data_adaptor) + + +class AnnotationsObsAPI(Resource): + @rest_get_data_adaptor + def get(self, data_adaptor): + return common_rest.annotations_obs_get(request, data_adaptor) + + @requires_authentication + @rest_get_data_adaptor + def put(self, data_adaptor): + return common_rest.annotations_obs_put(request, data_adaptor) + + +class AnnotationsVarAPI(Resource): + @rest_get_data_adaptor + def get(self, data_adaptor): + return common_rest.annotations_var_get(request, data_adaptor) + + +class DataVarAPI(Resource): + @rest_get_data_adaptor + def put(self, data_adaptor): + return common_rest.data_var_put(request, data_adaptor) + + @rest_get_data_adaptor + def get(self, data_adaptor): + return common_rest.data_var_get(request, data_adaptor) + + +class ColorsAPI(Resource): + @rest_get_data_adaptor + def get(self, data_adaptor): + return common_rest.colors_get(data_adaptor) + + +class DiffExpObsAPI(Resource): + @rest_get_data_adaptor + def post(self, data_adaptor): + return common_rest.diffexp_obs_post(request, data_adaptor) + + +class LayoutObsAPI(Resource): + @rest_get_data_adaptor + def get(self, data_adaptor): + return common_rest.layout_obs_get(request, data_adaptor) + + @rest_get_data_adaptor + def put(self, data_adaptor): + return common_rest.layout_obs_put(request, data_adaptor) + + +def get_api_base_resources(bp_base): + """Add resources that are accessed from the api url""" + api = Api(bp_base) + + # Diagnostics routes + api.add_resource(HealthAPI, "/health") + return api + + +def get_api_dataroot_resources(bp_dataroot): + """Add resources that refer to a dataset""" + api = Api(bp_dataroot) + + def add_resource(resource, url): + """convenience function to make the outer function less verbose""" + api.add_resource(resource, url) + + # Initialization routes + add_resource(SchemaAPI, "/schema") + add_resource(ConfigAPI, "/config") + add_resource(UserInfoAPI, "/userinfo") + # Data routes + add_resource(AnnotationsObsAPI, "/annotations/obs") + add_resource(AnnotationsVarAPI, "/annotations/var") + add_resource(DataVarAPI, "/data/var") + # Display routes + add_resource(ColorsAPI, "/colors") + # Computation routes + add_resource(DiffExpObsAPI, "/diffexp/obs") + add_resource(LayoutObsAPI, "/layout/obs") + return api + + +class Server: + @staticmethod + def _before_adding_routes(app, app_config): + """ will be called before routes are added, during __init__. Subclass protocol """ + pass + + def __init__(self, app_config): + self.app = Flask(__name__, static_folder=None) + self._before_adding_routes(self.app, app_config) + self.app.json_encoder = Float32JSONEncoder + server_config = app_config.server_config + + # enable session data + self.app.permanent_session_lifetime = datetime.timedelta(days=50 * 365) + + # Config + secret_key = server_config.app__flask_secret_key + self.app.config.update(SECRET_KEY=secret_key) + + self.app.register_blueprint(webbp) + + api_version = "/api/v0.2" + api_path = "/" + + bp_base = Blueprint("bp_base", __name__, url_prefix=api_path) + base_resources = get_api_base_resources(bp_base) + self.app.register_blueprint(base_resources.blueprint) + + bp_api = Blueprint("api", __name__, url_prefix=f"{api_path}{api_version}") + resources = get_api_dataroot_resources(bp_api) + self.app.register_blueprint(resources.blueprint) + self.app.add_url_rule( + "/static/", + "static_assets", + view_func=lambda filename: send_from_directory("../common/web/static", filename), + methods=["GET"], + ) + + self.app.data_adaptor = server_config.data_adaptor + self.app.app_config = app_config + + auth = server_config.auth + self.app.auth = auth + if auth.requires_client_login(): + auth.add_url_rules(self.app) + auth.complete_setup(self.app) diff --git a/local_server/auth/__init__.py b/local_server/auth/__init__.py new file mode 100644 index 00000000..c0c238b4 --- /dev/null +++ b/local_server/auth/__init__.py @@ -0,0 +1,5 @@ +# import the built in auth types so they can be registered + +import local_server.auth.auth_none # noqa: F401 +import local_server.auth.auth_test # noqa: F401 +import local_server.auth.auth_session # noqa: F401 diff --git a/local_server/auth/auth.py b/local_server/auth/auth.py new file mode 100644 index 00000000..03184e8d --- /dev/null +++ b/local_server/auth/auth.py @@ -0,0 +1,91 @@ +from abc import ABC, abstractmethod + + +class AuthTypeBase(ABC): + """Base type for all authentication types.""" + + def __init__(self): + super().__init__() + + @abstractmethod + def is_valid_authentication_type(self): + """Return True if the auth type is valid, e.g. it can return userinfo and username. + (AuthTypeNone is the only one type that returns False)""" + pass + + def requires_client_login(self): + """Return True if the user needs to login from the client (e.g. Login button is shown)""" + return False + + @abstractmethod + def complete_setup(self, app): + """complete any setup that may be needed by this auth type. The Flask app is passed in. + This is the last auth function called before the server starts to run.""" + pass + + @abstractmethod + def is_user_authenticated(self): + """Return True if the user is authenticated""" + pass + + @abstractmethod + def get_user_id(self): + """Return the id for this user (string)""" + pass + + @abstractmethod + def get_user_name(self): + """Return the name of the user (string)""" + pass + + @abstractmethod + def get_user_email(self): + """Return the name of the user (string)""" + pass + + def get_user_picture(self): + """Return the location to the user's picture""" + return None + + +class AuthTypeClientBase(AuthTypeBase): + """Base type for all authentication types that require the client to login""" + + def __init__(self): + super().__init__() + + def requires_client_login(self): + return True + + @abstractmethod + def add_url_rules(self, selfapp): + """Add url rules to the app (like /login, /logout, etc)""" + pass + + @abstractmethod + def get_login_url(self, data_adaptor): + """Return the url for the login route""" + pass + + @abstractmethod + def get_logout_url(self, data_adaptor): + """Return the url for the logout route""" + pass + + +class AuthTypeFactory: + """Factory class to create an authentication type""" + + auth_types = {} + + @staticmethod + def register(name, auth_type): + assert issubclass(auth_type, AuthTypeBase) + AuthTypeFactory.auth_types[name] = auth_type + + @staticmethod + def create(name, app_config): + auth_type = AuthTypeFactory.auth_types.get(name) + if auth_type is None: + return None + return auth_type(app_config) diff --git a/local_server/auth/auth_none.py b/local_server/auth/auth_none.py new file mode 100644 index 00000000..2d6f1f75 --- /dev/null +++ b/local_server/auth/auth_none.py @@ -0,0 +1,27 @@ +from local_server.auth.auth import AuthTypeBase, AuthTypeFactory + + +class AuthTypeNone(AuthTypeBase): + def __init__(self, app_config): + super().__init__() + + def is_valid_authentication_type(self): + return False + + def complete_setup(self, app): + pass + + def is_user_authenticated(self): + return True + + def get_user_id(self): + return None + + def get_user_name(self): + return None + + def get_user_email(self): + return None + + +AuthTypeFactory.register(None, AuthTypeNone) diff --git a/local_server/auth/auth_session.py b/local_server/auth/auth_session.py new file mode 100644 index 00000000..de460fb0 --- /dev/null +++ b/local_server/auth/auth_session.py @@ -0,0 +1,39 @@ +from local_server.auth.auth import AuthTypeBase, AuthTypeFactory +from flask import session +from uuid import uuid4 + + +class AuthTypeSession(AuthTypeBase): + """Session based authentication. The user is always logged. The user id is a random number + associated with the session. This is a good choice for desktop servers.""" + + # key in the session token for userid + CXGUID = "cxguid" + + def __init__(self, app_config): + super().__init__() + + def is_valid_authentication_type(self): + return True + + def complete_setup(self, app): + pass + + def is_user_authenticated(self): + # always authenticated + return True + + def get_user_id(self): + if self.CXGUID not in session: + session[self.CXGUID] = uuid4().hex + session.permanent = True + return session[self.CXGUID] + + def get_user_name(self): + return "anonymous" + + def get_user_email(self): + return None + + +AuthTypeFactory.register("session", AuthTypeSession) diff --git a/local_server/auth/auth_test.py b/local_server/auth/auth_test.py new file mode 100644 index 00000000..f5eaf1e3 --- /dev/null +++ b/local_server/auth/auth_test.py @@ -0,0 +1,73 @@ +from local_server.auth.auth import AuthTypeClientBase, AuthTypeFactory +from flask import session, request, redirect + + +class AuthTypeTest(AuthTypeClientBase): + """An authentication type for testing client based logins. When the login route is accessed + the user is automatically logged in with a default or configured username""" + + # key in session token with userid and username + CXGUID = "cxguid_test" + CXGUNAME = "cxguname_test" + CXGUEMAIL = "cxguemail_test" + CXGUPICTURE = "cxgupicture_test" + + def __init__(self, app_config): + super().__init__() + self.user_name = "test_account" + self.user_id = "id0001" + self.user_email = "test_account@test.com" + self.user_picture = None + + def is_valid_authentication_type(self): + return True + + def requires_client_login(self): + return True + + def add_url_rules(self, app): + app.add_url_rule("/login", "login", self.login, methods=["GET"]) + app.add_url_rule("/logout", "logout", self.logout, methods=["GET"]) + + def complete_setup(self, app): + pass + + def is_user_authenticated(self): + return self.CXGUID in session + + def get_user_id(self): + return session.get(self.CXGUID) + + def get_user_name(self): + return session.get(self.CXGUNAME) + + def get_user_email(self): + return session.get(self.CXGUEMAIL) + + def get_user_picture(self): + return session.get(self.CXGUPICTURE) + + def login(self): + args = request.args + return_to = args.get("dataset", "/") + session[self.CXGUID] = args.get("userid", self.user_id) + session[self.CXGUNAME] = args.get("username", self.user_name) + session[self.CXGUEMAIL] = args.get("email", self.user_email) + session[self.CXGUPICTURE] = args.get("picture", self.user_picture) + return redirect(return_to) + + def logout(self): + session.clear() + return_to = request.args.get("dataset", "/") + return redirect(return_to) + + def get_login_url(self, data_adaptor): + """Return the url for the login route""" + return "/login" + + def get_logout_url(self, data_adaptor): + """Return the url for the logout route""" + return "/logout" + + +AuthTypeFactory.register("test", AuthTypeTest) diff --git a/local_server/cli/__init__.py b/local_server/cli/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/cli/cli.py b/local_server/cli/cli.py new file mode 100644 index 00000000..a093df13 --- /dev/null +++ b/local_server/cli/cli.py @@ -0,0 +1,33 @@ +import click + +from .launch import launch +from .prepare import prepare +from .upgrade import log_upgrade_check +from .schema import schema_cli +from .. import __version__ + + +@click.group( + name="cellxgene", + subcommand_metavar="COMMAND ", + options_metavar="", + context_settings=dict(max_content_width=85, help_option_names=["-h", "--help"]), +) +@click.help_option("--help", "-h", help="Show this message and exit.") +@click.version_option( + version=__version__, + prog_name="cellxgene", + message="[%(prog)s] Version %(version)s", + help="Show the software version and exit.", +) +@click.option( + "--upgrade-check/--no-upgrade-check", default=True, show_default=True, help="Check for release upgrades on start.", +) +def cli(upgrade_check): + if upgrade_check: + log_upgrade_check() + + +cli.add_command(launch) +cli.add_command(prepare) +cli.add_command(schema_cli) diff --git a/local_server/cli/launch.py b/local_server/cli/launch.py new file mode 100644 index 00000000..23963fde --- /dev/null +++ b/local_server/cli/launch.py @@ -0,0 +1,445 @@ +import errno +import functools +import logging +import sys +import webbrowser +import os +import click +from flask_compress import Compress +from flask_cors import CORS + +from local_server.default_config import default_config +from local_server.app.app import Server +from local_server.common.config.app_config import AppConfig +from local_server.common.errors import DatasetAccessError, ConfigurationError +from local_server.common.utils.utils import sort_options + +DEFAULT_CONFIG = AppConfig() + + +def annotation_args(func): + @click.option( + "--disable-annotations", + is_flag=True, + default=not DEFAULT_CONFIG.dataset_config.user_annotations__enable, + show_default=True, + help="Disable user annotation of data.", + ) + @click.option( + "--annotations-file", + default=DEFAULT_CONFIG.dataset_config.user_annotations__local_file_csv__file, + show_default=True, + multiple=False, + metavar="", + help="CSV file to initialize editing of existing annotations; will be altered in-place. " + "Incompatible with --annotations-dir.", + ) + @click.option( + "--annotations-dir", + default=DEFAULT_CONFIG.dataset_config.user_annotations__local_file_csv__directory, + show_default=False, + multiple=False, + metavar="", + help="Directory of where to save output annotations; filename will be specified in the application. " + "Incompatible with --annotations-file.", + ) + @click.option( + "--experimental-annotations-ontology", + is_flag=True, + default=DEFAULT_CONFIG.dataset_config.user_annotations__ontology__enable, + show_default=True, + help="When creating annotations, optionally autocomplete names from ontology terms.", + ) + @click.option( + "--experimental-annotations-ontology-obo", + default=DEFAULT_CONFIG.dataset_config.user_annotations__ontology__obo_location, + show_default=True, + metavar="", + help="Location of OBO file defining cell annotation autosuggest terms.", + ) + @functools.wraps(func) + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + + return wrapper + + +def config_args(func): + @click.option( + "--max-category-items", + default=DEFAULT_CONFIG.dataset_config.presentation__max_categories, + metavar="", + show_default=True, + help="Will not display categories with more distinct values than specified.", + ) + @click.option( + "--disable-custom-colors", + is_flag=True, + default=False, + show_default=False, + help="Disable user-defined category-label colors drawn from source data file.", + ) + @click.option( + "--diffexp-lfc-cutoff", + "-de", + default=DEFAULT_CONFIG.dataset_config.diffexp__lfc_cutoff, + show_default=True, + metavar="", + help="Minimum log fold change threshold for differential expression.", + ) + @click.option( + "--disable-diffexp", + is_flag=True, + default=not DEFAULT_CONFIG.dataset_config.diffexp__enable, + show_default=False, + help="Disable on-demand differential expression.", + ) + @click.option( + "--embedding", + "-e", + default=DEFAULT_CONFIG.dataset_config.embeddings__names, + multiple=True, + show_default=False, + metavar="", + help="Embedding name, eg, 'umap'. Repeat option for multiple embeddings. Defaults to all.", + ) + @click.option( + "--experimental-enable-reembedding", + is_flag=True, + default=DEFAULT_CONFIG.dataset_config.embeddings__enable_reembedding, + show_default=False, + hidden=True, + help="Enable experimental on-demand re-embedding using UMAP. WARNING: may be very slow.", + ) + @functools.wraps(func) + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + + return wrapper + + +def dataset_args(func): + @click.option( + "--obs-names", + "-obs", + default=DEFAULT_CONFIG.server_config.single_dataset__obs_names, + metavar="", + help="Name of annotation field to use for observations. If not specified cellxgene will use the the obs index.", + ) + @click.option( + "--var-names", + "-var", + default=DEFAULT_CONFIG.server_config.single_dataset__var_names, + metavar="", + help="Name of annotation to use for variables. If not specified cellxgene will use the the var index.", + ) + @click.option( + "--backed", + "-b", + is_flag=True, + default=DEFAULT_CONFIG.server_config.adaptor__anndata_adaptor__backed, + show_default=False, + help="Load anndata in file-backed mode. " "This may save memory, but may result in slower overall performance.", + ) + @click.option( + "--title", + "-t", + default=DEFAULT_CONFIG.server_config.single_dataset__title, + metavar="", + help="Title to display. If omitted will use file name.", + ) + @click.option( + "--about", + default=DEFAULT_CONFIG.server_config.single_dataset__about, + metavar="", + help="URL providing more information about the dataset (hint: must be a fully specified absolute URL).", + ) + @functools.wraps(func) + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + + return wrapper + + +def server_args(func): + @click.option( + "--debug", + "-d", + is_flag=True, + default=DEFAULT_CONFIG.server_config.app__debug, + show_default=True, + help="Run in debug mode. This is helpful for cellxgene developers, " + "or when you want more information about an error condition.", + ) + @click.option( + "--verbose", + "-v", + is_flag=True, + default=DEFAULT_CONFIG.server_config.app__verbose, + show_default=True, + help="Provide verbose output, including warnings and all server requests.", + ) + @click.option( + "--port", + "-p", + metavar="", + default=DEFAULT_CONFIG.server_config.app__port, + type=int, + show_default=True, + help="Port to run server on. If not specified cellxgene will find an available port.", + ) + @click.option( + "--host", + metavar="", + default=DEFAULT_CONFIG.server_config.app__host, + show_default=False, + help="Host IP address. By default cellxgene will use localhost (e.g. 127.0.0.1).", + ) + @click.option( + "--scripts", + "-s", + default=DEFAULT_CONFIG.dataset_config.app__scripts, + multiple=True, + metavar="", + help="Additional script files to include in HTML page. If not specified, " + "no additional script files will be included.", + show_default=False, + ) + @functools.wraps(func) + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + + return wrapper + + +def launch_args(func): + @annotation_args + @config_args + @dataset_args + @server_args + @click.argument("datapath", required=False, metavar="") + @click.option( + "--open", + "-o", + "open_browser", + is_flag=True, + default=DEFAULT_CONFIG.server_config.app__open_browser, + show_default=True, + help="Open web browser after launch.", + ) + @click.option( + "--config-file", + "-c", + "config_file", + default=None, + show_default=True, + help="Location to yaml file with configuration settings", + ) + @click.option( + "--dump-default-config", + "dump_default_config", + is_flag=True, + default=False, + show_default=True, + help="Print default configuration settings and exit", + ) + @click.help_option("--help", "-h", help="Show this message and exit.") + @functools.wraps(func) + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + + return wrapper + + +def handle_scripts(scripts): + if scripts: + click.echo( + r""" + / / /\ \ \__ _ _ __ _ __ (_)_ __ __ _ + \ \/ \/ / _` | '__| '_ \| | '_ \ / _` | + \ /\ / (_| | | | | | | | | | | (_| | + \/ \/ \__,_|_| |_| |_|_|_| |_|\__, | + |___/ + The --scripts flag is intended for developers to include google analytics etc. You could be opening yourself to a + security risk by including the --scripts flag. Make sure you trust the scripts that you are including. + """ + ) + scripts_pretty = ", ".join(scripts) + click.confirm(f"Are you sure you want to inject these scripts: {scripts_pretty}?", abort=True) + + +class CliLaunchServer(Server): + """ + the CLI runs a local web server, and needs to enable a few more features. + """ + + def __init__(self, app_config): + super().__init__(app_config) + + @staticmethod + def _before_adding_routes(app, app_config): + app.config["COMPRESS_MIMETYPES"] = [ + "text/html", + "text/css", + "text/xml", + "application/json", + "application/javascript", + "application/octet-stream", + ] + Compress(app) + if app_config.server_config.app__debug: + CORS(app, supports_credentials=True) + + +@sort_options +@click.command( + short_help="Launch the cellxgene data viewer. " "Run `cellxgene launch --help` for more information.", + options_metavar="", +) +@launch_args +def launch( + datapath, + verbose, + debug, + open_browser, + port, + host, + embedding, + obs_names, + var_names, + max_category_items, + disable_custom_colors, + diffexp_lfc_cutoff, + title, + scripts, + about, + disable_annotations, + annotations_file, + annotations_dir, + backed, + disable_diffexp, + experimental_annotations_ontology, + experimental_annotations_ontology_obo, + experimental_enable_reembedding, + config_file, + dump_default_config, +): + """Launch the cellxgene data viewer. + This web app lets you explore single-cell expression data. + Data must be in a format that cellxgene expects. + Read the "getting started" guide to learn more: + https://chanzuckerberg.github.io/cellxgene/getting-started.html + + Examples: + + > cellxgene launch example-dataset/pbmc3k.h5ad --title pbmc3k + + > cellxgene launch --title + + > cellxgene launch """ + + if dump_default_config: + print(default_config) + sys.exit(0) + + # Startup message + click.echo("[cellxgene] Starting the CLI...") + + # app config + app_config = AppConfig() + server_config = app_config.server_config + + try: + if config_file: + app_config.update_from_config_file(config_file) + + # Determine which config options were give on the command line. + # Those will override the ones provided in the config file (if provided). + cli_config = AppConfig() + cli_config.update_server_config( + app__verbose=verbose, + app__debug=debug, + app__host=host, + app__port=port, + app__open_browser=open_browser, + single_dataset__datapath=datapath, + single_dataset__title=title, + single_dataset__about=about, + single_dataset__obs_names=obs_names, + single_dataset__var_names=var_names, + adaptor__anndata_adaptor__backed=backed, + ) + cli_config.update_dataset_config( + app__scripts=scripts, + user_annotations__enable=not disable_annotations, + user_annotations__local_file_csv__file=annotations_file, + user_annotations__local_file_csv__directory=annotations_dir, + user_annotations__ontology__enable=experimental_annotations_ontology, + user_annotations__ontology__obo_location=experimental_annotations_ontology_obo, + presentation__max_categories=max_category_items, + presentation__custom_colors=not disable_custom_colors, + embeddings__names=embedding, + embeddings__enable_reembedding=experimental_enable_reembedding, + diffexp__enable=not disable_diffexp, + diffexp__lfc_cutoff=diffexp_lfc_cutoff, + ) + + diff = cli_config.server_config.changes_from_default() + changes = {key: val for key, val, _ in diff} + app_config.update_server_config(**changes) + + diff = cli_config.dataset_config.changes_from_default() + changes = {key: val for key, val, _ in diff} + app_config.update_dataset_config(**changes) + + # process the configuration + # any errors will be thrown as an exception. + # any info messages will be passed to the messagefn function. + + def messagefn(message): + click.echo("[cellxgene] " + message) + + # Use a default secret if one is not provided + if not server_config.app__flask_secret_key: + app_config.update_server_config(app__flask_secret_key="SparkleAndShine") + + app_config.complete_config(messagefn) + + except (ConfigurationError, DatasetAccessError) as e: + raise click.ClickException(e) + + handle_scripts(scripts) + + # create the server + server = CliLaunchServer(app_config) + + if not server_config.app__verbose: + log = logging.getLogger("werkzeug") + log.setLevel(logging.ERROR) + + cellxgene_url = f"http://{app_config.server_config.app__host}:{app_config.server_config.app__port}" + if server_config.app__open_browser: + click.echo(f"[cellxgene] Launching! Opening your browser to {cellxgene_url} now.") + webbrowser.open(cellxgene_url) + else: + click.echo(f"[cellxgene] Launching! Please go to {cellxgene_url} in your browser.") + + click.echo("[cellxgene] Type CTRL-C at any time to exit.") + + if not server_config.app__verbose: + f = open(os.devnull, "w") + sys.stdout = f + + try: + server.app.run( + host=server_config.app__host, + debug=server_config.app__debug, + port=server_config.app__port, + threaded=not server_config.app__debug, + use_debugger=False, + use_reloader=False, + ) + except OSError as e: + if e.errno == errno.EADDRINUSE: + raise click.ClickException("Port is in use, please specify an open port using the --port flag.") from e + raise diff --git a/local_server/cli/prepare.py b/local_server/cli/prepare.py new file mode 100644 index 00000000..df5c8dd7 --- /dev/null +++ b/local_server/cli/prepare.py @@ -0,0 +1,274 @@ +from os.path import expanduser, isdir, isfile, sep, splitext + +import click +import pandas as pd +from numpy import ndarray, unique +from scipy.sparse.csc import csc_matrix + +from local_server.common.utils.utils import sort_options + + +@sort_options +@click.command( + short_help="Preprocess data for use with cellxgene. " "Run `cellxgene prepare --help` for more information.", + options_metavar="", +) +@click.argument("data", nargs=1, metavar="", required=True) +@click.option( + "--embedding", + "-e", + default=["umap", "tsne"], + multiple=True, + type=click.Choice(["umap", "tsne"]), + help="Embedding algorithm(s). Repeat option for multiple embeddings.", + show_default=True, +) +@click.option( + "--recipe", "-r", default="none", type=click.Choice(["none", "seurat", "zheng17"]), show_default=True, +) +@click.option("--output", "-o", default="", help="Save a new file to filename.", metavar="") +@click.option("--plotting", "-p", default=False, is_flag=True, help="Generate plots.", show_default=True) +@click.option("--sparse", default=False, is_flag=True, help="Force sparsity.", show_default=True) +@click.option("--overwrite", default=False, is_flag=True, help="Allow file overwriting.", show_default=True) +@click.option("--set-obs-names", default="", help="Named field to set as index for obs.", metavar="") +@click.option("--set-var-names", default="", help="Named field to set as index for var.", metavar="") +@click.option( + "--skip-qc", + default=False, + is_flag=True, + help="Do not run quality control metrics. By default cellxgene runs them " + "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).", +) +@click.option( + "--make-obs-names-unique/--no-make-obs-names-unique", + default=True, + help="Ensure obs index is unique.", + show_default=True, +) +@click.option( + "--make-var-names-unique/--no-make-var-names-unique", + default=True, + help="Ensure var index is unique.", + show_default=True, +) +@click.help_option("--help", "-h", help="Show this message and exit.") +def prepare( + data, + embedding, + recipe, + output, + plotting, + sparse, + overwrite, + set_obs_names, + set_var_names, + skip_qc, + make_obs_names_unique, + make_var_names_unique, +): + """ + Preprocess data for use with cellxgene. + This tool runs a series of scanpy routines for preparing a dataset for use + with cellxgene. It loads data from different formats + (h5ad, loom, or a 10x directory), runs dimensionality reduction, + computes nearest neighbors, computes an embedding, performs clustering, + and saves the results. Includes additional options for naming annotations, + ensuring sparsity, and plotting results. + """ + + # collect slow imports here to make CLI startup more responsive + click.echo("[cellxgene] Starting CLI...") + try: + import matplotlib + + matplotlib.use("Agg") + import scanpy as sc + except ImportError: + raise click.ClickException( + "[cellxgene] cellxgene prepare has not been installed. Please run `pip install 'cellxgene[prepare]'` " + "to install the necessary requirements." + ) + + # scanpy settings + sc.settings.verbosity = 0 + sc.settings.autosave = True + + # check args + if sparse and not recipe == "none": + raise click.UsageError("Cannot use a recipe when forcing sparsity") + + output = expanduser(output) + + if not output: + click.echo( + "Warning: No file will be saved, to save the results of cellxgene prepare include " + "--output to save output to a new file" + ) + if isfile(output) and not overwrite: + raise click.UsageError(f"Cannot overwrite existing file {output}, try using the flag --overwrite") + + def load_data(data): + if isfile(data): + name, extension = splitext(data) + if extension == ".h5ad": + adata = sc.read_h5ad(data) + elif extension == ".loom": + adata = sc.read_loom(data) + else: + raise click.FileError(data, hint="does not have a valid extension [.h5ad | .loom]") + elif isdir(data): + if not data.endswith(sep): + data += sep + adata = sc.read_10x_mtx(data) + else: + raise click.FileError(data, hint="not a valid file or path") + + if not set_obs_names == "": + if set_obs_names not in adata.obs_keys(): + raise click.UsageError(f"obs {set_obs_names} not found, options are: {adata.obs_keys()}") + adata.obs_names = adata.obs[set_obs_names] + if not set_var_names == "": + if set_var_names not in adata.var_keys(): + raise click.UsageError(f"var {set_var_names} not found, options are: {adata.var_keys()}") + adata.var_names = adata.var[set_var_names] + if make_obs_names_unique: + adata.obs.index = make_index_unique(adata.obs.index) + if make_var_names_unique: + adata.var.index = make_index_unique(adata.var.index) + if not adata._obs.index.is_unique: + click.echo("Warning: obs index is not unique") + if not adata._var.index.is_unique: + click.echo("Warning: var index is not unique") + return adata + + def calculate_qc_metrics(adata): + if not skip_qc: + sc.pp.calculate_qc_metrics(adata, inplace=True) + return adata + + def make_sparse(adata): + if (type(adata.X) is ndarray) and sparse: + adata.X = csc_matrix(adata.X) + + def run_recipe(adata): + if recipe == "seurat": + sc.pp.recipe_seurat(adata) + elif recipe == "zheng17": + sc.pp.recipe_zheng17(adata) + else: + sc.pp.filter_cells(adata, min_genes=5) + sc.pp.filter_genes(adata, min_cells=25) + if sparse: + sc.pp.scale(adata, zero_center=False) + else: + sc.pp.scale(adata) + + def run_pca(adata): + if sparse: + sc.pp.pca(adata, svd_solver="arpack", zero_center=False) + else: + sc.pp.pca(adata, svd_solver="arpack") + + def run_neighbors(adata): + sc.pp.neighbors(adata) + + def run_louvain(adata): + sc.tl.louvain(adata) + + def run_embedding(adata): + if len(unique(adata.obs["louvain"].values)) < 10: + palette = "tab10" + else: + palette = "tab20" + + if "umap" in embedding: + sc.tl.umap(adata) + if plotting: + sc.pl.umap(adata, color="louvain", palette=palette, save="_louvain") + + if "tsne" in embedding: + sc.tl.tsne(adata) + if plotting: + sc.pl.tsne(adata, color="louvain", palette=palette, save="_louvain") + + def show_step(item): + if not skip_qc: + qc_name = "Calculating QC metrics" + else: + qc_name = "Skipping QC" + names = { + "calculate_qc_metrics": qc_name, + "make_sparse": "Ensuring sparsity", + "run_recipe": f'Running preprocessing recipe "{recipe}"', + "run_pca": "Running PCA", + "run_neighbors": "Calculating neighbors", + "run_louvain": "Calculating clusters", + "run_embedding": "Computing embedding", + } + if item is not None: + return names[item.__name__] + + steps = [calculate_qc_metrics, make_sparse, run_recipe, run_pca, run_neighbors, run_louvain, run_embedding] + + click.echo(f"[cellxgene] Loading data from {data}, please wait...") + adata = load_data(data) + + click.echo("[cellxgene] Beginning preprocessing...") + with click.progressbar(steps, label="[cellxgene] Progress", show_eta=False, item_show_func=show_step) as bar: + for step in bar: + step(adata) + + # saving + if not output == "": + click.echo(f"[cellxgene] Saving results to {output}...") + adata.write(output) + + click.echo("[cellxgene] Success!") + + +# TODO (mweiden): remove this once this issue is resolved https://github.com/theislab/anndata/issues/344 +# Note: tentative solution here https://github.com/theislab/anndata/pull/345 +def make_index_unique(index: pd.Index, join: str = "-"): + """ + Makes the index unique by appending a number string to each duplicate index element: '1', '2', etc. + + If a tentative name created by the algorithm already exists in the index, it tries the next integer in the sequence. + + The first occurrence of a non-unique value is ignored. + Parameters + ---------- + join + The connecting string between name and integer. + Examples + -------- + >>> from anndata import AnnData + >>> adata1 = AnnData(np.ones((3, 2)), dict(obs_names=['a', 'b', 'c'])) + >>> adata2 = AnnData(np.zeros((3, 2)), dict(obs_names=['d', 'b', 'b'])) + >>> adata = adata1.concatenate(adata2) + >>> adata.obs_names + Index(['a', 'b', 'c', 'd', 'b', 'b'], dtype='object') + >>> adata.obs_names_make_unique() + >>> adata.obs_names + Index(['a', 'b', 'c', 'd', 'b-1', 'b-2'], dtype='object') + """ + if index.is_unique: + return index + from collections import defaultdict + + values = index.values + values_set = set(values) + indices_dup = index.duplicated(keep="first") + values_dup = values[indices_dup] + counter = defaultdict(lambda: 0) + for i, v in enumerate(values_dup): + while True: + counter[v] += 1 + tentative_new_name = v + join + str(counter[v]) + if tentative_new_name not in values_set: + values_set.add(tentative_new_name) + values_dup[i] = tentative_new_name + break + + values[indices_dup] = values_dup + index = pd.Index(values) + return index diff --git a/local_server/cli/schema.py b/local_server/cli/schema.py new file mode 100644 index 00000000..83c9c07f --- /dev/null +++ b/local_server/cli/schema.py @@ -0,0 +1,72 @@ +import click + +from local_server.converters.schema import remix, validate + + +@click.group( + name="schema", + subcommand_metavar="COMMAND ", + short_help="Apply and validate the cellxgene data integration schema to an h5ad file.", + context_settings=dict(max_content_width=85, help_option_names=["-h", "--help"]), +) +def schema_cli(): + try: + import scanpy # noqa: F401 + except ImportError: + raise click.ClickException( + "[cellxgene] cellxgene schema requires scanpy" + ) + + +@click.command( + name="apply", + short_help="(experimental) Apply the cellxgene data integration schema to an h5ad.", + help="(experimental) Using a yaml file that describes schema values to insert or convert and in input " + "h5ad file, apply the schema changes and create a new, conforming h5ad.", +) +@click.option( + "--source-h5ad", + help="Input h5ad file.", + nargs=1, + required=True, + type=click.Path(exists=True, dir_okay=False), +) +@click.option( + "--remix-config", + help="Config yaml with information on how to apply the schema.", + nargs=1, + required=True, + type=click.Path(exists=True, dir_okay=False), +) +@click.option( + "--output-filename", + help="Filename for the new, schema-conforming h5ad file.", + required=True, + nargs=1 +) +def schema_apply(source_h5ad, remix_config, output_filename): + remix.apply_schema(source_h5ad, remix_config, output_filename) + + +@click.command( + name="validate", + short_help="(experimental) Check that an h5ad follows the cellxgene data integration schema.", +) +@click.argument( + "h5ad", + nargs=1, + type=click.Path(exists=True, dir_okay=False), +) +@click.option( + "--shallow", + help="When true, just check that the correct version information is present.", + default=False, + show_default=True, + is_flag=True, +) +def schema_validate(h5ad, shallow): + validate.validate(h5ad, shallow) + + +schema_cli.add_command(schema_apply) +schema_cli.add_command(schema_validate) diff --git a/local_server/cli/upgrade.py b/local_server/cli/upgrade.py new file mode 100644 index 00000000..222d7e81 --- /dev/null +++ b/local_server/cli/upgrade.py @@ -0,0 +1,85 @@ +import re + +import click +import requests +from requests.exceptions import ConnectionError + +from .. import __version__ + +# Official SemVer regex: https://semver.org/ +SEMVER_FORMAT = re.compile( + r"^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[" + r"a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(" + r"?:\.[0-9a-zA-Z-]+)*))?$" +) + + +def log_upgrade_check(): + # Sanity-check that the CLI version is a properly-formatted SemVer string + assert validate_version_str(__version__, release_only=False) + + # Get the current latest release + try: + release_tag_generator = (r["tag_name"] for r in _request_cellxgene_releases()) + latest_release = next(release_tag_generator, lambda tag_name: validate_version_str(tag_name)) + if version_gt(latest_release, __version__): + click.echo(f"There's a new version of cellxgene available ({latest_release})!", err=True) + click.echo("To upgrade, run the following: pip install --upgrade cellxgene\n", err=True) + except (ConnectionError, RateLimitException): + click.echo("Upgrade check failed.\n") + + +class RateLimitException(Exception): + """ + Github API Rate Limit Exception + """ + + +def _request_cellxgene_releases(): + def raise_on_rate_limit(response): + if response.status_code == 403 and res.headers.get("X-RateLimit-Remaining") == "0": + raise RateLimitException + + url = "https://api.github.com/repos/chanzuckerberg/cellxgene/releases" + res = requests.get(url) + raise_on_rate_limit(res) + for release in res.json(): + yield release + while "next" in res.links.keys(): + res = requests.get(res.links["next"]["url"]) + raise_on_rate_limit(res) + for release in res.json(): + yield release + + +def validate_version_str(version_str, release_only=True): + """ + Test if a string conforms to SemVer format (https://semver.org/) + :param version_str: a string to be validated + :param release_only: only declare releases (not prereleases) valid + :return: True if the version string is of a valid SemVer format else False + """ + match = SEMVER_FORMAT.match(version_str) + has_match = match is not None + if has_match and release_only: + return not match.group("prerelease") + return has_match + + +def split_version(version_string): + """ + Split a SemVer-formatted string into its component integers + :param version_string: a SemVer string to be split + :return: an array of three integers + """ + match = SEMVER_FORMAT.match(version_string) + return [int(match.group(group)) for group in ["major", "minor", "patch"]] + + +def version_gt(left_version, right_version): + for left, right in zip(split_version(left_version), split_version(right_version)): + if left > right: + return True + elif right > left: + return False + return False diff --git a/local_server/common/__init__.py b/local_server/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/common/annotations/__init__.py b/local_server/common/annotations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/common/annotations/annotations.py b/local_server/common/annotations/annotations.py new file mode 100644 index 00000000..232a354f --- /dev/null +++ b/local_server/common/annotations/annotations.py @@ -0,0 +1,70 @@ +from abc import ABCMeta, abstractmethod + +import fastobo +import fsspec + +from local_server.common.errors import OntologyLoadFailure +from local_server.common.utils.type_conversion_utils import get_schema_type_hint_of_array + + +class Annotations(metaclass=ABCMeta): + """ baseclass for annotations, including ontologies""" + + """ our default ontology is the PURL for the Cell Ontology. + See http://www.obofoundry.org/ontology/cl.html """ + DefaultOnotology = "http://purl.obolibrary.org/obo/cl.obo" + + def __init__(self): + self.ontology_data = None + + def load_ontology(self, path): + """Load and parse ontologies - currently support OBO files only.""" + if path is None: + path = self.DefaultOnotology + + try: + with fsspec.open(path) as f: + obo = fastobo.iter(f) + terms = filter(lambda stanza: type(stanza) is fastobo.term.TermFrame, obo) + names = [tag.name for term in terms for tag in term if type(tag) is fastobo.term.NameClause] + self.ontology_data = names + + except FileNotFoundError as e: + raise OntologyLoadFailure("Unable to find OBO ontology path") from e + + except SyntaxError as e: + raise OntologyLoadFailure("Syntax error loading OBO ontology") from e + + except Exception as e: + raise OntologyLoadFailure("Error loading OBO file") from e + + def get_schema(self, data_adaptor): + schema = [] + labels = self.read_labels(data_adaptor) + if labels is not None and not labels.empty: + for col in labels.columns: + col_schema = dict(name=col, writable=True) + col_schema.update(get_schema_type_hint_of_array(labels[col])) + schema.append(col_schema) + + return schema + + @abstractmethod + def set_collection(self, name): + """set or create a new annotation collection""" + pass + + @abstractmethod + def read_labels(self, data_adaptor): + """Return the labels as a pandas.DataFrame""" + pass + + @abstractmethod + def write_labels(self, df, data_adaptor): + """Write the labels (df) to a persistent storage such that it can later be read""" + pass + + @abstractmethod + def update_parameters(self, parameters, data_adaptor): + """Update configuration parameters that describe information about the annotations feature""" + pass diff --git a/local_server/common/annotations/local_file_csv.py b/local_server/common/annotations/local_file_csv.py new file mode 100644 index 00000000..639c1e0b --- /dev/null +++ b/local_server/common/annotations/local_file_csv.py @@ -0,0 +1,196 @@ +import base64 +import os +import re +import threading +from datetime import datetime +from hashlib import blake2b + +import pandas as pd +from flask import session, has_request_context, current_app + +from local_server import __version__ as cellxgene_version +from local_server.common.annotations.annotations import Annotations +from local_server.common.errors import AnnotationsError + + +class AnnotationsLocalFile(Annotations): + CXG_ANNO_COLLECTION = "cxg_anno_collection" + + def __init__(self, output_dir, output_file): + super().__init__() + self.output_dir = output_dir + self.output_file = output_file + # lock used to protect label file write ops + self.label_lock = threading.RLock() + + # cache the most recent annotations + self.last_fname = None + self.last_labels = None + + def is_safe_collection_name(self, name): + """ + return true if this is a safe collection name + this is ultra conservative. If we want to allow full legal file name syntax, + we could look at modules like `pathvalidate` + """ + if name is None: + return False + return re.match(r"^[\w\-]+$", name) is not None + + def set_collection(self, name): + session[self.CXG_ANNO_COLLECTION] = name + session.permanent = True + + def get_collection(self): + if session is None: + return None + return session.get(self.CXG_ANNO_COLLECTION) + + def read_labels(self, data_adaptor): + if has_request_context(): + if not current_app.auth.is_user_authenticated(): + return pd.DataFrame() + + fname = self._get_filename(data_adaptor) + with self.label_lock: + if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0: + # returned the cached labels if possible, otherwise read them from the file + if fname == self.last_fname: + return self.last_labels + else: + labels = pd.read_csv( + fname, dtype="category", index_col=0, header=0, comment="#", keep_default_na=False + ) + # update the cache + self.last_fname = fname + self.last_labels = labels + return labels + else: + return pd.DataFrame() + + def write_labels(self, df, data_adaptor): + # update our internal state and save it. Multi-threading often enabled, + # so treat this as a critical section. + with self.label_lock: + lastmod = data_adaptor.get_last_mod_time() + lastmodstr = "'unknown'" if lastmod is None else lastmod.isoformat(timespec="seconds") + header = ( + f"# Annotations generated on {datetime.now().isoformat(timespec='seconds')} " + f"using cellxgene version {cellxgene_version}\n" + f"# Input data file was {data_adaptor.get_location()}, " + f"which was last modified on {lastmodstr}\n" + ) + + fname = self._get_filename(data_adaptor) + self._backup(fname) + if not df.empty: + with open(fname, "w", newline="") as f: + if header is not None: + f.write(header) + df.to_csv(f) + else: + open(fname, "w").close() + + # update the cache + self.last_fname = fname + self.last_labels = df + + def _get_userdata_idhash(self, data_adaptor): + """ + Return a short hash that weakly identifies the user and dataset. + Used to create safe annotations output file names. + """ + uid = current_app.auth.get_user_id() + id = (uid + data_adaptor.get_location()).encode() + idhash = base64.b32encode(blake2b(id, digest_size=5).digest()).decode("utf-8") + return idhash + + def _get_output_dir(self): + if self.output_dir: + return self.output_dir + + if self.output_file: + return os.path.dirname(self.path.abspath(self.output_dir)) + + return os.getcwd() + + def _get_filename(self, data_adaptor): + """ return the current annotation file name """ + if self.output_file: + return self.output_file + + # we need to generate a file name, which we can only do if we have a UID and collection name + if session is None: + raise AnnotationsError("unable to determine file name for annotations") + + collection = self.get_collection() + if collection is None: + return None + + if data_adaptor is None: + raise AnnotationsError("unable to determine file name for annotations") + + idhash = self._get_userdata_idhash(data_adaptor) + return os.path.join(self._get_output_dir(), f"{collection}-{idhash}.csv") + + def _backup(self, fname, max_backups=9): + """ + save N backups of file to backup_dir. + 1. fname -> backup_dir/fname-TIME + 2. delete excess files in backup_dir + """ + root, ext = os.path.splitext(fname) + backup_dir = f"{root}-backups" + + # Make sure there is work to do + if not os.path.exists(fname): + return + + # Ensure backup_dir exists + if not os.path.exists(backup_dir): + os.mkdir(backup_dir) + + # Save current file to backup_dir + fname_base = os.path.basename(fname) + fname_base_root, fname_base_ext = os.path.splitext(fname_base) + # don't use ISO standard time format, as it contains characters illegal on some filesytems. + nowish = datetime.now().strftime("%Y-%m-%dT%H-%M-%S") + backup_fname = os.path.join(backup_dir, f"{fname_base_root}-{nowish}{fname_base_ext}") + if os.path.exists(backup_fname): + os.remove(backup_fname) + os.rename(fname, backup_fname) + + # prune the backup_dir to max number of backup files, keeping the most recent backups + backups = list(filter(lambda s: s.startswith(fname_base_root), os.listdir(backup_dir))) + excess_count = len(backups) - max_backups + if excess_count > 0: + backups.sort() + for bu in backups[0:excess_count]: + os.remove(os.path.join(backup_dir, bu)) + + def update_parameters(self, parameters, data_adaptor): + params = {} + params["annotations"] = True + params["user_annotation_collection_name_enabled"] = True + + if self.ontology_data: + params["annotations_cell_ontology_enabled"] = True + params["annotations_cell_ontology_terms"] = self.ontology_data + else: + params["annotations_cell_ontology_enabled"] = False + + if self.output_file is not None: + # user has hard-wired the name of the annotation data collection + fname = os.path.basename(self.output_file) + collection_fname = os.path.splitext(fname)[0] + params["annotations-data-collection-is-read-only"] = True + params["annotations-data-collection-name"] = collection_fname + + elif session is not None: + collection = self.get_collection() + if current_app.auth.is_user_authenticated(): + params["annotations-user-data-idhash"] = self._get_userdata_idhash(data_adaptor) + params["annotations-data-collection-is-read-only"] = False + params["annotations-data-collection-name"] = collection + + parameters.update(params) diff --git a/local_server/common/aws_secret_utils.py b/local_server/common/aws_secret_utils.py new file mode 100644 index 00000000..4cfb1159 --- /dev/null +++ b/local_server/common/aws_secret_utils.py @@ -0,0 +1,23 @@ +import logging + +import boto3 +from flask import json + +from local_server.common.errors import SecretKeyRetrievalError + + +def get_secret_key(region_name, secret_name): + session = boto3.session.Session() + client = session.client(service_name="secretsmanager", region_name=region_name) + + try: + get_secret_value_response = client.get_secret_value(SecretId=secret_name) + if "SecretString" in get_secret_value_response: + var = get_secret_value_response["SecretString"] + secret = json.loads(var) + return secret + except Exception as e: + logging.critical(f"Caught exception during get_secret_key, {e}", exc_info=True) + raise SecretKeyRetrievalError(str(e)) + + return None diff --git a/local_server/common/colors.py b/local_server/common/colors.py new file mode 100644 index 00000000..b1e8d0bb --- /dev/null +++ b/local_server/common/colors.py @@ -0,0 +1,233 @@ +import re + +from local_server.common.errors import ColorFormatException + +HEX_COLOR_FORMAT = re.compile("^#[a-fA-F0-9]{6,6}$") + +# https://www.w3.org/TR/css-color-4/#named-colors +CSS4_NAMED_COLORS = dict( + aliceblue="#f0f8ff", + antiquewhite="#faebd7", + aqua="#00ffff", + aquamarine="#7fffd4", + azure="#f0ffff", + beige="#f5f5dc", + bisque="#ffe4c4", + black="#000000", + blanchedalmond="#ffebcd", + blue="#0000ff", + blueviolet="#8a2be2", + brown="#a52a2a", + burlywood="#deb887", + cadetblue="#5f9ea0", + chartreuse="#7fff00", + chocolate="#d2691e", + coral="#ff7f50", + cornflowerblue="#6495ed", + cornsilk="#fff8dc", + crimson="#dc143c", + cyan="#00ffff", + darkblue="#00008b", + darkcyan="#008b8b", + darkgoldenrod="#b8860b", + darkgray="#a9a9a9", + darkgreen="#006400", + darkgrey="#a9a9a9", + darkkhaki="#bdb76b", + darkmagenta="#8b008b", + darkolivegreen="#556b2f", + darkorange="#ff8c00", + darkorchid="#9932cc", + darkred="#8b0000", + darksalmon="#e9967a", + darkseagreen="#8fbc8f", + darkslateblue="#483d8b", + darkslategray="#2f4f4f", + darkslategrey="#2f4f4f", + darkturquoise="#00ced1", + darkviolet="#9400d3", + deeppink="#ff1493", + deepskyblue="#00bfff", + dimgray="#696969", + dimgrey="#696969", + dodgerblue="#1e90ff", + firebrick="#b22222", + floralwhite="#fffaf0", + forestgreen="#228b22", + fuchsia="#ff00ff", + gainsboro="#dcdcdc", + ghostwhite="#f8f8ff", + gold="#ffd700", + goldenrod="#daa520", + gray="#808080", + green="#008000", + greenyellow="#adff2f", + grey="#808080", + honeydew="#f0fff0", + hotpink="#ff69b4", + indianred="#cd5c5c", + indigo="#4b0082", + ivory="#fffff0", + khaki="#f0e68c", + lavender="#e6e6fa", + lavenderblush="#fff0f5", + lawngreen="#7cfc00", + lemonchiffon="#fffacd", + lightblue="#add8e6", + lightcoral="#f08080", + lightcyan="#e0ffff", + lightgoldenrodyellow="#fafad2", + lightgray="#d3d3d3", + lightgreen="#90ee90", + lightgrey="#d3d3d3", + lightpink="#ffb6c1", + lightsalmon="#ffa07a", + lightseagreen="#20b2aa", + lightskyblue="#87cefa", + lightslategray="#778899", + lightslategrey="#778899", + lightsteelblue="#b0c4de", + lightyellow="#ffffe0", + lime="#00ff00", + limegreen="#32cd32", + linen="#faf0e6", + magenta="#ff00ff", + maroon="#800000", + mediumaquamarine="#66cdaa", + mediumblue="#0000cd", + mediumorchid="#ba55d3", + mediumpurple="#9370db", + mediumseagreen="#3cb371", + mediumslateblue="#7b68ee", + mediumspringgreen="#00fa9a", + mediumturquoise="#48d1cc", + mediumvioletred="#c71585", + midnightblue="#191970", + mintcream="#f5fffa", + mistyrose="#ffe4e1", + moccasin="#ffe4b5", + navajowhite="#ffdead", + navy="#000080", + oldlace="#fdf5e6", + olive="#808000", + olivedrab="#6b8e23", + orange="#ffa500", + orangered="#ff4500", + orchid="#da70d6", + palegoldenrod="#eee8aa", + palegreen="#98fb98", + paleturquoise="#afeeee", + palevioletred="#db7093", + papayawhip="#ffefd5", + peachpuff="#ffdab9", + peru="#cd853f", + pink="#ffc0cb", + plum="#dda0dd", + powderblue="#b0e0e6", + purple="#800080", + rebeccapurple="#663399", + red="#ff0000", + rosybrown="#bc8f8f", + royalblue="#4169e1", + saddlebrown="#8b4513", + salmon="#fa8072", + sandybrown="#f4a460", + seagreen="#2e8b57", + seashell="#fff5ee", + sienna="#a0522d", + silver="#c0c0c0", + skyblue="#87ceeb", + slateblue="#6a5acd", + slategray="#708090", + slategrey="#708090", + snow="#fffafa", + springgreen="#00ff7f", + steelblue="#4682b4", + tan="#d2b48c", + teal="#008080", + thistle="#d8bfd8", + tomato="#ff6347", + turquoise="#40e0d0", + violet="#ee82ee", + wheat="#f5deb3", + white="#ffffff", + whitesmoke="#f5f5f5", + yellow="#ffff00", + yellowgreen="#9acd32", +) + + +def convert_color_to_hex_format(unknown): + """ + Try to convert color info to a hex triplet string https://en.wikipedia.org/wiki/Web_colors#Hex_triplet. + + The function accepts for the following formats: + - A CSS4 color name, as supported by matplotlib https://matplotlib.org/3.1.0/gallery/color/named_colors.html + - RGB tuple/list with values ranging from 0.0 to 1.0, as in [0.5, 0.75, 1.0] + - RFB tuple/list with values ranging from 0 to 255, as in [128, 192, 255] + - Hex triplet string, as in "#08c0ff" + + :param unknown: color info of unknown format + :return: a hex triplet representing that color + """ + try: + if type(unknown) in (list, tuple) and len(unknown) == 3: + if all(0.0 <= ele <= 1.0 for ele in unknown): + tup = tuple(int(ele * 255) for ele in unknown) + elif all(0 <= ele <= 255 and isinstance(ele, int) for ele in unknown): + tup = tuple(unknown) + else: + raise ColorFormatException("Unknown color iterable format!") + return "#%02x%02x%02x" % tup + elif isinstance(unknown, str) and unknown.lower() in CSS4_NAMED_COLORS: + return CSS4_NAMED_COLORS[unknown.lower()] + elif isinstance(unknown, str) and HEX_COLOR_FORMAT.match(unknown): + return unknown.lower() + else: + raise ColorFormatException("Unknown color format type!") + except Exception as e: + raise ColorFormatException(e) + + +def convert_anndata_category_colors_to_cxg_category_colors(data): + """ + Convert color information from anndata files to the cellxgene color data format as described below: + { + "": { + "": "", + ... + }, + ... + } + + For more on the cxg color data structure, see https://github.com/chanzuckerberg/cellxgene/issues/1307. + + For more on the anndata color data structure, see + https://github.com/chanzuckerberg/cellxgene/issues/1152#issuecomment-587276178. + + Handling of malformed data: + - For any color info in a adata.uns[f"{category}_colors"] color array that convert_color_to_hex_format cannot + convert to a hex triplet string, a ColorFormatException is raised + - No category_name key group is returned for adata.uns[f"{category}_colors"] keys for which there is no + adata.obs[f"{category}"] key + + :param data: the anndata file + :return: cellxgene color data structure as described above + """ + cxg_colors = dict() + color_key_suffix = "_colors" + for uns_key in data.uns.keys(): + # find uns array that describes colors for a category + if not uns_key.endswith(color_key_suffix): + continue + + # check to see if we actually have observations for that category + category_name = uns_key[: -len(color_key_suffix)] + if category_name not in data.obs.keys(): + continue + + # create the cellxgene color entry for this category + cxg_colors[category_name] = dict( + zip(data.obs[category_name].cat.categories, [convert_color_to_hex_format(c) for c in data.uns[uns_key]]) + ) + return cxg_colors diff --git a/local_server/common/config/__init__.py b/local_server/common/config/__init__.py new file mode 100644 index 00000000..fa1a0dcc --- /dev/null +++ b/local_server/common/config/__init__.py @@ -0,0 +1,4 @@ +from local_server.common.aws_secret_utils import get_secret_key # noqa F504 + +DEFAULT_SERVER_PORT = 5005 +BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20 # 100MB diff --git a/local_server/common/config/app_config.py b/local_server/common/config/app_config.py new file mode 100644 index 00000000..8ecd67f7 --- /dev/null +++ b/local_server/common/config/app_config.py @@ -0,0 +1,171 @@ +import yaml +from flatten_dict import unflatten + +from local_server.default_config import get_default_config +from local_server.common.config.dataset_config import DatasetConfig +from local_server.common.config.server_config import ServerConfig +from local_server.common.config.external_config import ExternalConfig +from local_server.common.errors import ConfigurationError + + +class AppConfig(object): + """ + AppConfig stores all the configuration for cellxgene. + AppConfig contains one or more DatasetConfig(s) and one ServerConfig. + The server_config contains attributes that refer to the server process as a whole. + The dataset_config refers to attributes that are associated with the features and + presentations of a dataset. + AppConfig has methods to initialize, modify, and access the configuration. + """ + + def __init__(self): + + # the default configuration (see default_config.py) + # TODO @madison -- if we always read from the default config (hard coded path) can we set those values as + # defaults within the config class? + self.default_config = get_default_config() + # the server configuration + self.server_config = ServerConfig(self, self.default_config["server"]) + # the dataset config + self.dataset_config = DatasetConfig(None, self, self.default_config["dataset"]) + # external config + self.external_config = ExternalConfig(self, self.default_config["external"]) + + # Set to true when config_completed is called + self.is_completed = False + + def get_dataset_config(self): + return self.dataset_config + + def check_config(self): + """Verify all the attributes in the config have been type checked""" + if not self.is_completed: + raise ConfigurationError("The configuration has not been completed") + self.server_config.check_config() + self.dataset_config.check_config() + self.external_config.check_config() + + def update_server_config(self, **kw): + self.server_config.update(**kw) + self.is_complete = False + + def update_dataset_config(self, **kw): + self.dataset_config.update(**kw) + self.is_complete = False + + def update_single_config_from_path_and_value(self, path, value): + """Update a single config parameter with the value. + Path is a list of string, that gives a path to the config parameter to be updated. + For example, path may be ["server","app","port"]. + """ + self.is_complete = False + if not isinstance(path, list): + raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'") + for part in path: + if not isinstance(part, str): + raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'") + + if len(path) < 1 or path[0] not in ("server", "dataset"): + raise ConfigurationError("path must start with 'server', or 'dataset'") + + if path[0] == "server": + attr = "__".join(path[1:]) + try: + self.update_server_config(**{attr: value}) + except ConfigurationError: + raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'") + elif path[0] == "dataset": + attr = "__".join(path[1:]) + try: + self.update_dataset_config(**{attr: value}) + except ConfigurationError: + raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'") + + def update_from_config_file(self, config_file): + try: + with open(config_file) as yml_file: + config = yaml.safe_load(yml_file) + except yaml.YAMLError as e: + raise ConfigurationError(f"The specified config file contained an error: {e}") + except OSError as e: + raise ConfigurationError(f"Issue retrieving the specified config file: {e}") + + if config.get("server"): + self.server_config.update_from_config(config["server"], "server") + if config.get("dataset"): + self.dataset_config.update_from_config(config["dataset"], "dataset") + + if config.get("external"): + self.external_config.update_from_config(config["external"], "external") + + self.is_complete = False + + def config_to_dict(self): + """return the configuration as an unflattened dict""" + server = self.server_config.create_mapping(self.server_config.default_config) + dataset = self.dataset_config.create_mapping(self.dataset_config.default_config) + external = self.external_config.create_mapping(self.external_config.default_config) + config = dict(server={}, dataset={}) + for attrname in server.keys(): + config["server__" + attrname] = getattr(self.server_config, attrname) + for attrname in dataset.keys(): + config["dataset__" + attrname] = getattr(self.dataset_config, attrname) + for attrname in external.keys(): + config["external__" + attrname] = getattr(self.external_config, attrname) + + config = unflatten(config, splitter=lambda key: key.split("__")) + return config + + def write_config(self, config_file): + """output the config to a yaml file""" + config = self.config_to_dict() + yaml.dump(config, open(config_file, "w")) + + def changes_from_default(self): + """Return all the attribute that are different from the default""" + diff_server = self.server_config.changes_from_default() + diff_dataset = self.dataset_config.changes_from_default() + diff_external = self.external.changes_from_default() + diff = dict(server=diff_server, dataset=diff_dataset, external=diff_external) + return diff + + def complete_config(self, messagefn=None): + """The configure options are checked, and any additional setup based on the config + parameters is done""" + + if messagefn is None: + + def noop(message): + pass + + messagefn = noop + + # TODO: to give better error messages we can add a mapping between where each config + # attribute originated (e.g. command line argument or config file), then in the error + # messages we can give correct context for attributes with bad value. + context = dict(messagefn=messagefn) + + # complete config for external_config first, since this may update values in the other sections + self.external_config.complete_config(context) + self.server_config.complete_config(context) + self.dataset_config.complete_config(context) + + self.is_completed = True + self.check_config() + + def get_matrix_data_cache_manager(self): + return self.server_config.matrix_data_cache_manager + + def get_title(self, data_adaptor): + return ( + self.server_config.single_dataset__title + if self.server_config.single_dataset__title + else data_adaptor.get_title() + ) + + def get_about(self, data_adaptor): + return ( + self.server_config.single_dataset__about + if self.server_config.single_dataset__about + else data_adaptor.get_about() + ) diff --git a/local_server/common/config/base_config.py b/local_server/common/config/base_config.py new file mode 100644 index 00000000..8238f2ed --- /dev/null +++ b/local_server/common/config/base_config.py @@ -0,0 +1,99 @@ +import copy + +from flatten_dict import flatten +from local_server.common.errors import ConfigurationError + + +class BaseConfig(object): + """ + This class handles the mechanics of updating and checking attributes. + Derived classes are expected to store the actual attributes + Currently DatasetConfig and ServerConfig both inherit from BaseConfig. + """ + + def __init__(self, app_config, default_config): + # reference back to the app_config + self.app_config = app_config + # the complete set of attributes and their default values (unflattened) + self.default_config = default_config + # used to make sure every attribute value is checked + self.attr_checked = {key_name: False for key_name in self.create_mapping(default_config).keys()} + + def create_mapping(self, config): + """ + Create a dictionary where the keys are the name of attributes (using double underscore convention) + For example: authentication__type + + The values are a tuple, + - the first item of the tuple is a tuple of path elements (location in config 'tree') + - the second item is the value of the config parameter + + For example: (('authentication', 'type'), 'session')) + """ + config_copy = copy.deepcopy(config) + mapping = {} + + flat_config = flatten(config_copy) + for key, value in flat_config.items(): + # name of the attribute + attr = "__".join(key) + mapping[attr] = (key, value) + + return mapping + + def validate_correct_type_of_configuration_attribute(self, attrname, vtype): + val = getattr(self, attrname) + if type(vtype) in (list, tuple): + if type(val) not in vtype: + tnames = ",".join([x.__name__ for x in vtype]) + raise ConfigurationError( + f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}" + ) + else: + if type(val) != vtype: + raise ConfigurationError( + f"Invalid type for attribute: {attrname}, " + f"expected type {vtype.__name__}, got {type(val).__name__}" + ) + + self.attr_checked[attrname] = True + + def check_config(self): + mapping = self.create_mapping(self.default_config) + for key in mapping.keys(): + if not self.attr_checked[key]: + raise ConfigurationError(f"The attr '{key}' has not been checked") + + def update(self, **kw): + """Update the attributes defined in kw with their new values.""" + for key, value in kw.items(): + if not hasattr(self, key): + raise ConfigurationError(f"unknown config parameter {key}.") + try: + if type(value) == tuple: + # convert tuple values to list values + value = list(value) + setattr(self, key, value) + except KeyError: + raise ConfigurationError(f"Unable to set config parameter {key}.") + + self.attr_checked[key] = False + + def update_from_config(self, config, prefix): + mapping = self.create_mapping(config) + for attr, (key, value) in mapping.items(): + if not hasattr(self, attr): + raise ConfigurationError(f"Unknown key from config file: {prefix}__{attr}") + setattr(self, attr, value) + + self.attr_checked[attr] = False + + def changes_from_default(self): + """Return all the attribute that are different from the default""" + mapping = self.create_mapping(self.default_config) + diff = [] + for attrname, (key, defval) in mapping.items(): + curval = getattr(self, attrname) + if curval != defval: + diff.append((attrname, curval, defval)) + return diff diff --git a/local_server/common/config/client_config.py b/local_server/common/config/client_config.py new file mode 100644 index 00000000..c6cde91c --- /dev/null +++ b/local_server/common/config/client_config.py @@ -0,0 +1,120 @@ +from local_server import display_version as cellxgene_display_version + + +def get_client_config(app_config, data_adaptor): + """ + Return the configuration as required by the /config REST route + """ + + server_config = app_config.server_config + dataset_config = data_adaptor.dataset_config + annotation = dataset_config.user_annotations + auth = server_config.auth + + # FIXME The current set of config is not consistently presented: + # we have camalCase, hyphen-text, and underscore_text + + # make sure the configuration has been checked. + app_config.check_config() + + # display_names + title = app_config.get_title(data_adaptor) + about = app_config.get_about(data_adaptor) + + display_names = dict(engine=data_adaptor.get_name(), dataset=title) + + # library_versions + library_versions = {} + library_versions.update(data_adaptor.get_library_versions()) + library_versions["cellxgene"] = cellxgene_display_version + + # links + links = {"about-dataset": about} + + # parameters + parameters = { + "layout": dataset_config.embeddings__names, + "max-category-items": dataset_config.presentation__max_categories, + "obs_names": server_config.single_dataset__obs_names, + "var_names": server_config.single_dataset__var_names, + "diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff, + "backed": server_config.adaptor__anndata_adaptor__backed, + "disable-diffexp": not dataset_config.diffexp__enable, + "enable-reembedding": dataset_config.embeddings__enable_reembedding, + "annotations": False, + "annotations_file": None, + "annotations_dir": None, + "annotations_cell_ontology_enabled": False, + "annotations_cell_ontology_obopath": None, + "annotations_cell_ontology_terms": None, + "custom_colors": dataset_config.presentation__custom_colors, + "diffexp-may-be-slow": False, + } + + # corpora dataset_props + # TODO/Note: putting info from the dataset into the /config is not ideal. + # However, it is definitely not part of /schema, and we do not have a top-level + # route for data properties. Consider creating one at some point. + corpora_props = data_adaptor.get_corpora_props() + if corpora_props and "default_embedding" in corpora_props: + default_embedding = corpora_props["default_embedding"] + if isinstance(default_embedding, str) and default_embedding.startswith("X_"): + default_embedding = default_embedding[2:] # drop X_ prefix + if default_embedding in data_adaptor.get_embedding_names(): + parameters["default_embedding"] = default_embedding + + data_adaptor.update_parameters(parameters) + if annotation: + annotation.update_parameters(parameters, data_adaptor) + + # gather it all together + client_config = {} + config = client_config["config"] = {} + config["displayNames"] = display_names + config["library_versions"] = library_versions + config["links"] = links + config["parameters"] = parameters + config["corpora_props"] = corpora_props + config["limits"] = { + "column_request_max": server_config.limits__column_request_max, + "diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max, + } + + if dataset_config.app__authentication_enable and auth.is_valid_authentication_type(): + config["authentication"] = { + "requires_client_login": auth.requires_client_login(), + } + if auth.requires_client_login(): + config["authentication"].update( + { + # Todo why are these stored on the data_adaptor? + "login": auth.get_login_url(data_adaptor), + "logout": auth.get_logout_url(data_adaptor), + } + ) + + return client_config + + +def get_client_userinfo(app_config, data_adaptor): + """ + Return the userinfo as required by the /userinfo REST route + """ + + server_config = app_config.server_config + dataset_config = data_adaptor.dataset_config + auth = server_config.auth + + # make sure the configuration has been checked. + app_config.check_config() + + if dataset_config.app__authentication_enable and auth.is_valid_authentication_type(): + userinfo = {} + userinfo["userinfo"] = { + "is_authenticated": auth.is_user_authenticated(), + "username": auth.get_user_name(), + "user_id": auth.get_user_id(), + "email": auth.get_user_email(), + "picture": auth.get_user_picture(), + } + return userinfo diff --git a/local_server/common/config/dataset_config.py b/local_server/common/config/dataset_config.py new file mode 100644 index 00000000..91993ae2 --- /dev/null +++ b/local_server/common/config/dataset_config.py @@ -0,0 +1,191 @@ +import os +from os.path import splitext, isdir + +from local_server.common.annotations.local_file_csv import AnnotationsLocalFile +from local_server.common.config.base_config import BaseConfig +from local_server.common.errors import ConfigurationError, OntologyLoadFailure +from local_server.compute.scanpy import get_scanpy_module +from local_server.data_common.matrix_loader import MatrixDataLoader + + +class DatasetConfig(BaseConfig): + """Manages the config attribute associated with a dataset.""" + + def __init__(self, tag, app_config, default_config): + super().__init__(app_config, default_config) + self.tag = tag + try: + self.app__scripts = default_config["app"]["scripts"] + self.app__inline_scripts = default_config["app"]["inline_scripts"] + self.app__authentication_enable = default_config["app"]["authentication_enable"] + + self.presentation__max_categories = default_config["presentation"]["max_categories"] + self.presentation__custom_colors = default_config["presentation"]["custom_colors"] + + self.user_annotations__enable = default_config["user_annotations"]["enable"] + self.user_annotations__type = default_config["user_annotations"]["type"] + self.user_annotations__local_file_csv__directory = default_config["user_annotations"]["local_file_csv"][ + "directory" + ] + self.user_annotations__local_file_csv__file = default_config["user_annotations"]["local_file_csv"]["file"] + self.user_annotations__ontology__enable = default_config["user_annotations"]["ontology"]["enable"] + self.user_annotations__ontology__obo_location = default_config["user_annotations"]["ontology"][ + "obo_location" + ] + + self.embeddings__names = default_config["embeddings"]["names"] + self.embeddings__enable_reembedding = default_config["embeddings"]["enable_reembedding"] + + self.diffexp__enable = default_config["diffexp"]["enable"] + self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"] + self.diffexp__top_n = default_config["diffexp"]["top_n"] + + except KeyError as e: + raise ConfigurationError(f"Unexpected config: {str(e)}") + + # The annotation object is created during complete_config and stored here. + self.user_annotations = None + + def complete_config(self, context): + self.handle_app() + self.handle_presentation() + self.handle_user_annotations(context) + self.handle_embeddings() + self.handle_diffexp(context) + + def get_data_adaptor(self): + server_config = self.app_config.server_config + if not server_config.data_adaptor: + matrix_data_loader = MatrixDataLoader( + server_config.single_dataset__datapath, app_config=self.app_config + ) + server_config.data_adaptor = matrix_data_loader.open(self.app_config) + + return server_config.data_adaptor + + def handle_app(self): + self.validate_correct_type_of_configuration_attribute("app__scripts", list) + self.validate_correct_type_of_configuration_attribute("app__inline_scripts", list) + self.validate_correct_type_of_configuration_attribute("app__authentication_enable", bool) + + # scripts can be string (filename) or dict (attributes). Convert string to dict. + scripts = [] + for script in self.app__scripts: + try: + if isinstance(script, str): + scripts.append({"src": script}) + elif isinstance(script, dict) and isinstance(script["src"], str): + scripts.append(script) + else: + raise Exception + except Exception as e: + raise ConfigurationError(f"Scripts must be string or a dict containing an src key: {e}") + + self.app__scripts = scripts + + def handle_presentation(self): + self.validate_correct_type_of_configuration_attribute("presentation__max_categories", int) + self.validate_correct_type_of_configuration_attribute("presentation__custom_colors", bool) + + def handle_user_annotations(self, context): + self.validate_correct_type_of_configuration_attribute("user_annotations__enable", bool) + self.validate_correct_type_of_configuration_attribute("user_annotations__type", str) + self.validate_correct_type_of_configuration_attribute( + "user_annotations__local_file_csv__directory", (type(None), str) + ) + self.validate_correct_type_of_configuration_attribute( + "user_annotations__local_file_csv__file", (type(None), str) + ) + self.validate_correct_type_of_configuration_attribute("user_annotations__ontology__enable", bool) + self.validate_correct_type_of_configuration_attribute( + "user_annotations__ontology__obo_location", (type(None), str) + ) + if self.user_annotations__enable: + server_config = self.app_config.server_config + if not self.app__authentication_enable: + raise ConfigurationError("user annotations requires authentication to be enabled") + if not server_config.auth.is_valid_authentication_type(): + auth_type = server_config.authentication__type + raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations") + + if self.user_annotations__type == "local_file_csv": + self.handle_local_file_csv_annotations() + else: + raise ConfigurationError('The only annotation type support is "local_file_csv"') + if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location: + try: + self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location) + except OntologyLoadFailure as e: + raise ConfigurationError("Unable to load ontology terms\n" + str(e)) + else: + self.check_annotation_config_vars_not_set(context) + + def handle_local_file_csv_annotations(self): + dirname = self.user_annotations__local_file_csv__directory + filename = self.user_annotations__local_file_csv__file + if filename is not None and dirname is not None: + raise ConfigurationError("'annotations-file' and 'annotations-dir' may not be used together.") + + if filename is not None: + lf_name, lf_ext = splitext(filename) + if lf_ext and lf_ext != ".csv": + raise ConfigurationError(f"annotation file type must be .csv: {filename}") + + if dirname is not None and not isdir(dirname): + try: + os.mkdir(dirname) + except OSError: + raise ConfigurationError("Unable to create directory specified by --annotations-dir") + + self.user_annotations = AnnotationsLocalFile(dirname, filename) + + # if the user has specified a fixed label file, go ahead and validate it + # so that we can remove errors early in the process. + server_config = self.app_config.server_config + if server_config.single_dataset__datapath and self.user_annotations__local_file_csv__file: + data_adaptor = self.get_data_adaptor() + data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor)) + + def check_annotation_config_vars_not_set(self, context): + if self.user_annotations__type is not None: + dirname = self.user_annotations__local_file_csv__directory + filename = self.user_annotations__local_file_csv__file + if filename is not None: + context["messagefn"]("Warning: --annotations-file ignored as annotations are disabled.") + if dirname is not None: + context["messagefn"]("Warning: --annotations-dir ignored as annotations are disabled.") + + if self.user_annotations__ontology__enable: + context["messagefn"]("Warning: --experimental-annotations-ontology ignored as annotations are disabled.") + if self.user_annotations__ontology__obo_location is not None: + context["messagefn"]( + "Warning: --experimental-annotations-ontology-obo ignored as annotations are disabled." + ) + + def handle_embeddings(self): + self.validate_correct_type_of_configuration_attribute("embeddings__names", list) + self.validate_correct_type_of_configuration_attribute("embeddings__enable_reembedding", bool) + + server_config = self.app_config.server_config + if self.embeddings__enable_reembedding: + if server_config.single_dataset__datapath: + if server_config.adaptor__anndata_adaptor__backed: + raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.") + + try: + get_scanpy_module() + except NotImplementedError: + # Todo add scanpy to requirements.txt and remove this check once re-embeddings is fully supported + raise ConfigurationError("Please install scanpy to enable UMAP re-embedding") + + def handle_diffexp(self, context): + self.validate_correct_type_of_configuration_attribute("diffexp__enable", bool) + self.validate_correct_type_of_configuration_attribute("diffexp__lfc_cutoff", float) + self.validate_correct_type_of_configuration_attribute("diffexp__top_n", int) + + data_adaptor = self.get_data_adaptor() + if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False): + context["messagefn"]( + "CAUTION: due to the size of your dataset, " + "running differential expression may take longer or fail." + ) diff --git a/local_server/common/config/external_config.py b/local_server/common/config/external_config.py new file mode 100644 index 00000000..19d29a8e --- /dev/null +++ b/local_server/common/config/external_config.py @@ -0,0 +1,96 @@ +import os + +from local_server.common.config.base_config import BaseConfig +from local_server.common.errors import ConfigurationError +from local_server.common.config import get_secret_key +from local_server.common.errors import SecretKeyRetrievalError +from local_server.common.utils.type_conversion_utils import convert_string_to_value + + +class ExternalConfig(BaseConfig): + """Manages the config attribute associated with external configuration sources, such as + environment variables or the AWS Secrets Manager.""" + + def __init__(self, app_config, default_config): + super().__init__(app_config, default_config) + try: + self.environment = default_config["environment"] + self.aws_secrets_manager__region = default_config["aws_secrets_manager"]["region"] + self.aws_secrets_manager__secrets = default_config["aws_secrets_manager"]["secrets"] + + except KeyError as e: + raise ConfigurationError(f"Unexpected config: {str(e)}") + + def complete_config(self, context): + self.handle_environment(context) + self.handle_aws_secrets_manager(context) + + def handle_environment(self, context): + """For each environment variable defined, get the value (if it is set), + and set the specified config parameter""" + self.validate_correct_type_of_configuration_attribute("environment", list) + for envdict in self.environment: + name = envdict.get("name") + if name is None: + raise ConfigurationError("environment: 'name' is missing") + required = envdict.get("required", False) + if type(required) != bool: + raise ConfigurationError("environment: 'required' must be a bool") + path = envdict.get("path") + if path is None: + raise ConfigurationError("environment: 'path' is missing") + + value = os.environ.get(name) + if value is None: + if required: + raise ConfigurationError(f"required environment variable '{name}' not set") + else: + value = convert_string_to_value(value) + self.app_config.update_single_config_from_path_and_value(path, value) + + def handle_aws_secrets_manager(self, context): + """For each aws secret defined, get the key/values, and set the specified config parameter""" + self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", (type(None), str)) + self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__secrets", list) + + if not self.aws_secrets_manager__secrets: + return + + self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", str) + + for secret in self.aws_secrets_manager__secrets: + secret_name = secret.get("name") + if secret_name is None: + raise ConfigurationError("aws_secrets_manager: 'name' is missing") + if not isinstance(secret_name, str): + raise ConfigurationError("aws_secrets_manager: 'name' must be a string") + + try: + secret_dict = get_secret_key(self.aws_secrets_manager__region, secret_name) + except SecretKeyRetrievalError as e: + raise ConfigurationError(f"Unable to retrieve secret {secret_name}: {str(e)}") + + values = secret.get("values") + if values is None: + raise ConfigurationError("aws_secrets_manager: 'values' is missing") + if not isinstance(values, list): + raise ConfigurationError("aws_secrets_manager: 'values' must be a list") + + for value in values: + key = value.get("key") + if key is None: + raise ConfigurationError(f"missing 'key' in secret values: {secret_name}") + path = value.get("path") + if path is None: + raise ConfigurationError(f"missing 'path' in secret values: {secret_name}") + required = value.get("required", False) + if type(required) != bool: + raise ConfigurationError(f"wrong type for 'required' in secret values: {secret_name}") + + secret_value = secret_dict.get(key) + if secret_value is None: + if required: + raise ConfigurationError(f"required secret '{secret_name}:{key}' not set") + else: + secret_value = convert_string_to_value(secret_value) + self.app_config.update_single_config_from_path_and_value(path, secret_value) diff --git a/local_server/common/config/server_config.py b/local_server/common/config/server_config.py new file mode 100644 index 00000000..210c9667 --- /dev/null +++ b/local_server/common/config/server_config.py @@ -0,0 +1,183 @@ +import os +import sys +import warnings +from os.path import basename +from urllib.parse import urlparse + +from local_server.auth.auth import AuthTypeFactory +from local_server.common.config.base_config import BaseConfig +from local_server.common.config import DEFAULT_SERVER_PORT, BIG_FILE_SIZE_THRESHOLD +from local_server.common.errors import ConfigurationError, DatasetAccessError +from local_server.common.data_locator import discover_s3_region_name +from local_server.common.utils.utils import is_port_available, find_available_port, custom_format_warning +from local_server.data_common.matrix_loader import MatrixDataLoader + + +class ServerConfig(BaseConfig): + """Manages the config attribute associated with the server.""" + + def __init__(self, app_config, default_config): + super().__init__(app_config, default_config) + + try: + self.app__verbose = default_config["app"]["verbose"] + self.app__debug = default_config["app"]["debug"] + self.app__host = default_config["app"]["host"] + self.app__port = default_config["app"]["port"] + self.app__open_browser = default_config["app"]["open_browser"] + self.app__force_https = default_config["app"]["force_https"] + self.app__flask_secret_key = default_config["app"]["flask_secret_key"] + + self.authentication__type = default_config["authentication"]["type"] + self.authentication__insecure_test_environment = default_config["authentication"][ + "insecure_test_environment" + ] + + self.single_dataset__datapath = default_config["single_dataset"]["datapath"] + self.single_dataset__obs_names = default_config["single_dataset"]["obs_names"] + self.single_dataset__var_names = default_config["single_dataset"]["var_names"] + self.single_dataset__about = default_config["single_dataset"]["about"] + self.single_dataset__title = default_config["single_dataset"]["title"] + + self.data_locator__s3__region_name = default_config["data_locator"]["s3"]["region_name"] + + self.adaptor__anndata_adaptor__backed = default_config["adaptor"]["anndata_adaptor"]["backed"] + + self.limits__diffexp_cellcount_max = default_config["limits"]["diffexp_cellcount_max"] + self.limits__column_request_max = default_config["limits"]["column_request_max"] + + except KeyError as e: + raise ConfigurationError(f"Unexpected config: {str(e)}") + + self.data_adaptor = None + + # The authentication object + self.auth = None + + def complete_config(self, context): + self.handle_app(context) + self.handle_data_source() + self.handle_authentication() + self.handle_data_locator() + self.handle_adaptor() # may depend on data_locator + self.handle_single_dataset(context) # may depend on adaptor + self.handle_limits() + + self.check_config() + + def handle_app(self, context): + self.validate_correct_type_of_configuration_attribute("app__verbose", bool) + self.validate_correct_type_of_configuration_attribute("app__debug", bool) + self.validate_correct_type_of_configuration_attribute("app__host", str) + self.validate_correct_type_of_configuration_attribute("app__port", (type(None), int)) + self.validate_correct_type_of_configuration_attribute("app__open_browser", bool) + self.validate_correct_type_of_configuration_attribute("app__force_https", bool) + self.validate_correct_type_of_configuration_attribute("app__flask_secret_key", str) + + if self.app__port: + try: + if not is_port_available(self.app__host, self.app__port): + raise ConfigurationError( + f"The port selected {self.app__port} is in use, please configure an open port." + ) + except OverflowError: + raise ConfigurationError(f"Invalid port: {self.app__port}") + else: + try: + default_server_port = int(os.environ.get("CXG_SERVER_PORT", DEFAULT_SERVER_PORT)) + except ValueError: + raise ConfigurationError( + "Invalid port from environment variable CXG_SERVER_PORT: " + os.environ.get("CXG_SERVER_PORT") + ) + try: + self.app__port = find_available_port(self.app__host, default_server_port) + except OverflowError: + raise ConfigurationError(f"Invalid port: {default_server_port}") + + if self.app__debug: + context["messagefn"]("in debug mode, setting verbose=True and open_browser=False") + self.app__verbose = True + self.app__open_browser = False + else: + warnings.formatwarning = custom_format_warning + + if not self.app__verbose: + sys.tracebacklimit = 0 + + def handle_authentication(self): + self.validate_correct_type_of_configuration_attribute("authentication__type", (type(None), str)) + self.validate_correct_type_of_configuration_attribute("authentication__insecure_test_environment", bool) + + if self.authentication__type == "test" and not self.authentication__insecure_test_environment: + raise ConfigurationError("Test auth can only be used in an insecure test environment") + + self.auth = AuthTypeFactory.create(self.authentication__type, self) + if self.auth is None: + raise ConfigurationError(f"Unknown authentication type: {self.authentication__type}") + + def handle_data_locator(self): + self.validate_correct_type_of_configuration_attribute("data_locator__s3__region_name", (type(None), bool, str)) + if self.data_locator__s3__region_name is True: + path = self.single_dataset__datapath + + if path.startswith("s3://"): + region_name = discover_s3_region_name(path) + if region_name is None: + raise ConfigurationError(f"Unable to discover s3 region name from {path}") + else: + region_name = None + self.data_locator__s3__region_name = region_name + + def handle_data_source(self): + self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", str) + + def handle_single_dataset(self, context): + self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None))) + self.validate_correct_type_of_configuration_attribute("single_dataset__title", (str, type(None))) + self.validate_correct_type_of_configuration_attribute("single_dataset__about", (str, type(None))) + self.validate_correct_type_of_configuration_attribute("single_dataset__obs_names", (str, type(None))) + self.validate_correct_type_of_configuration_attribute("single_dataset__var_names", (str, type(None))) + + # preload this data set + matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath, app_config=self.app_config) + try: + matrix_data_loader.pre_load_validation() + except DatasetAccessError as e: + raise ConfigurationError(str(e)) + + file_size = matrix_data_loader.file_size() + file_basename = basename(self.single_dataset__datapath) + if file_size > BIG_FILE_SIZE_THRESHOLD: + context["messagefn"](f"Loading data from {file_basename}, this may take a while...") + else: + context["messagefn"](f"Loading data from {file_basename}.") + + if self.single_dataset__about: + + def url_check(url): + try: + result = urlparse(url) + if all([result.scheme, result.netloc]): + return True + else: + return False + except ValueError: + return False + + if not url_check(self.single_dataset__about): + raise ConfigurationError( + "Must provide an absolute URL for --about. (Example format: http://example.com)" + ) + + def handle_adaptor(self): + self.validate_correct_type_of_configuration_attribute("adaptor__anndata_adaptor__backed", bool) + + def handle_limits(self): + self.validate_correct_type_of_configuration_attribute("limits__diffexp_cellcount_max", (type(None), int)) + self.validate_correct_type_of_configuration_attribute("limits__column_request_max", (type(None), int)) + + def exceeds_limit(self, limit_name, value): + limit_value = getattr(self, "limits__" + limit_name, None) + if limit_value is None: # disabled + return False + return value > limit_value diff --git a/local_server/common/constants.py b/local_server/common/constants.py new file mode 100644 index 00000000..a883e23a --- /dev/null +++ b/local_server/common/constants.py @@ -0,0 +1,30 @@ +from enum import Enum + + +class AugmentedEnum(Enum): + def __hash__(self): + return self.value.__hash__() + + def __eq__(self, other): + if isinstance(other, type(self)) or isinstance(other, str): + return self.value == other + return False + + def __str__(self) -> str: + return self.value + + +class Axis(AugmentedEnum): + OBS = "obs" + VAR = "var" + + +class DiffExpMode(AugmentedEnum): + TOP_N = "topN" + VAR_FILTER = "varFilter" + + +JSON_NaN_to_num_warning_msg = "JSON encoding failure - please verify all data are finite values (no NaN or Infinities)" +REACTIVE_LIMIT = 1_000_000 + +MAX_LAYOUTS = 30 diff --git a/local_server/common/corpora.py b/local_server/common/corpora.py new file mode 100644 index 00000000..e9ed5240 --- /dev/null +++ b/local_server/common/corpora.py @@ -0,0 +1,78 @@ +""" +Corpora schema conventions support. Helper functions for reading. + +https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md + +https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema_h5ad_implementation.md +""" +import collections +import json + +from local_server.cli.upgrade import validate_version_str +from local_server.common.utils.corpora_constants import CorporaConstants + + +def corpora_get_versions_from_anndata(adata): + """ + Given an AnnData object, return: + * None - if not a Corpora object + * [ corpora_schema_version, corpora_encoding_version ] - if a Corpora object + + Implements the identification protocol defined in the specification. + """ + + # per Corpora AnnData spec, this is a corpora file if the following is true + if "version" not in adata.uns_keys(): + return None + version = adata.uns["version"] + if not isinstance(version, collections.abc.Mapping) or "corpora_schema_version" not in version: + return None + + corpora_schema_version = version.get("corpora_schema_version") + corpora_encoding_version = version.get("corpora_encoding_version") + + # TODO: spec says these must be SEMVER values, so check. + if validate_version_str(corpora_schema_version) and validate_version_str(corpora_encoding_version): + return [corpora_schema_version, corpora_encoding_version] + + +def corpora_is_version_supported(corpora_schema_version, corpora_encoding_version): + return ( + corpora_schema_version + and corpora_encoding_version + and corpora_schema_version.startswith("1.") + and corpora_encoding_version.startswith("0.1.") + ) + + +def corpora_get_props_from_anndata(adata): + """ + Get Corpora dataset properties from an AnnData + """ + versions = corpora_get_versions_from_anndata(adata) + if versions is None: + return None + [corpora_schema_version, corpora_encoding_version] = versions + version_is_supported = corpora_is_version_supported(corpora_schema_version, corpora_encoding_version) + if not version_is_supported: + raise ValueError("Unsupported Corpora schema version") + + corpora_props = {} + for key in CorporaConstants.REQUIRED_SIMPLE_METADATA_FIELDS: + if key not in adata.uns: + raise KeyError(f"missing Corpora schema field {key}") + corpora_props[key] = adata.uns[key] + + for key in CorporaConstants.OPTIONAL_JSON_ENCODED_METADATA_FIELD: + if key not in adata.uns: + continue + try: + corpora_props[key] = json.loads(adata.uns[key]) + except json.JSONDecodeError: + raise json.JSONDecodeError(f"Corpora schema field {key} is expected to be a valid JSON string") + + for key in CorporaConstants.OPTIONAL_SIMPLE_METADATA_FIELDS: + if key in adata.uns: + corpora_props[key] = adata.uns[key] + + return corpora_props diff --git a/local_server/common/data_locator.py b/local_server/common/data_locator.py new file mode 100644 index 00000000..bf47d724 --- /dev/null +++ b/local_server/common/data_locator.py @@ -0,0 +1,154 @@ +import os +import tempfile +import fsspec +from datetime import datetime +import boto3 +import botocore +from urllib.parse import urlparse + + +class DataLocator: + """ + DataLocator is a simple wrapper around fsspec functionality, and provides a + set of functions to encapsulate a data location (URI or path), interogate + metadata about the object at that location (size, existance, etc) and + access the underlying data. + + https://filesystem-spec.readthedocs.io/en/latest/index.html + + Example: + dl = DataLocator("/tmp/foo.h5ad") + if dl.exists(): + print(dl.size()) + with dl.open() as f: + thecontents = f.read() + + DataLocator will accept a URI or native path. Error handling is as defined + in fsspec. + + """ + + def __init__(self, uri_or_path, region_name=None): + if isinstance(uri_or_path, DataLocator): + locator = uri_or_path + self.uri_or_path = locator.uri_or_path + self.protocol = locator.protocol + self.path = locator.path + self.cname = locator.cname + else: + self.uri_or_path = uri_or_path + self.protocol, self.path = DataLocator._get_protocol_and_path(uri_or_path) + # work-around for LocalFileSystem not treating file: and None as the same scheme/protocol + self.cname = self.path if self.protocol == "file" else self.uri_or_path + + # fsspec.filesystem will throw RuntimeError if the protocol is unsupported + if self.protocol == "s3": + if region_name: + config_kwargs = dict(region_name=region_name) + self.fs = fsspec.filesystem(self.protocol, listings_expiry_time=30, config_kwargs=config_kwargs) + else: + self.fs = fsspec.filesystem(self.protocol, listings_expiry_time=30) + else: + self.fs = fsspec.filesystem(self.protocol) + + def __repr__(self): + return f"DataLocator(protocol={self.protocol}, cname={self.cname}, " + f"path={self.path}, uri_or_path={self.uri_or_path})" + + @staticmethod + def _get_protocol_and_path(uri_or_path): + if "://" in uri_or_path: + protocol, path = uri_or_path.split("://", 1) + # windows!!! Ignore single letter drive identifiers, + # eg, G:\foo.txt + if len(protocol) > 1: + return protocol, path + return None, uri_or_path + + def exists(self): + return self.fs.exists(self.cname) + + def size(self): + return self.fs.size(self.cname) + + def lastmodtime(self): + """ return datetime object representing last modification time, or None if unavailable """ + info = self.fs.info(self.cname) + if self.islocal() and info is not None: + return datetime.fromtimestamp(info["mtime"]) + else: + return getattr(info, "LastModified", None) + + def abspath(self): + """ + return the absolute path for the locator - only really does something + for file: protocol, as all others are already absolute + """ + if self.islocal(): + return os.path.abspath(self.path) + else: + return self.uri_or_path + + def isfile(self): + return self.fs.isfile(self.cname) + + def open(self, *args): + return self.fs.open(self.uri_or_path, *args) + + def islocal(self): + return self.protocol is None or self.protocol == "file" + + def local_handle(self): + if self.islocal(): + return LocalFilePath(self.path) + + # if not local, create a tmp file system object to contain the data, + # and clean it up when done. If the path has a suffix/extension, + # do our best to create a file with the same. + ext = os.path.splitext(self.path) + suffix = None if ext[1] == "" else ext[1] + with self.open() as src, tempfile.NamedTemporaryFile(prefix="cellxgene_", suffix=suffix, delete=False) as tmp: + tmp.write(src.read()) + tmp.close() + src.close() + tmp_path = tmp.name + return LocalFilePath(tmp_path, delete=True) + + def ls(self): + paths = self.fs.ls(self.uri_or_path) + return [os.path.basename(p) for p in paths] + + +class LocalFilePath: + def __init__(self, tmp_path, delete=False): + self.tmp_path = tmp_path + self.delete = delete + + def __enter__(self): + return self.tmp_path + + def __exit__(self, *args): + if self.delete: + os.unlink(self.tmp_path) + + +def discover_s3_region_name(uri): + """If this is an s3 protocol, discover and return the (aws) region name. + If a return name could not be discovered, or if the uri is not an s3 protocol, return None.""" + + protocol, _ = DataLocator._get_protocol_and_path(uri) + if protocol == "s3": + bucket = urlparse(uri).netloc + client = boto3.client("s3") + try: + res = client.head_bucket(Bucket=bucket) + except botocore.exceptions.ClientError: + return None + + region = res.get("ResponseMetadata", {}).get("HTTPHeaders", {}).get("x-amz-bucket-region") + if region: + return region + else: + return None + + return None diff --git a/local_server/common/errors.py b/local_server/common/errors.py new file mode 100644 index 00000000..ca339bee --- /dev/null +++ b/local_server/common/errors.py @@ -0,0 +1,57 @@ +from http import HTTPStatus + + +class CellxgeneException(Exception): + """Base class for cellxgene exceptions""" + + def __init__(self, message): + self.message = message + super().__init__(message) + + +class RequestException(CellxgeneException): + """Baseclass for exceptions that can be raised from a request.""" + + # The default status code is 400 (Bad Request) + default_status_code = HTTPStatus.BAD_REQUEST + + def __init__(self, message, status_code=None): + super().__init__(message) + self.status_code = status_code or self.default_status_code + + +def define_exception(name, doc): + globals()[name] = type(name, (CellxgeneException,), dict(__doc__=doc)) + + +def define_request_exception(name, doc, default_status_code=HTTPStatus.BAD_REQUEST): + globals()[name] = type(name, (RequestException,), dict(__doc__=doc, default_status_code=default_status_code)) + + +define_request_exception("FilterError", "Raised when filter is malformed") +define_request_exception("JSONEncodingValueError", "Raised when data cannot be encoded into json") +define_request_exception("MimeTypeError", "Raised when incompatible MIME type selected") +define_request_exception("DatasetAccessError", "Raised when file loaded into a DataAdaptor is misformatted") +define_request_exception("DisabledFeatureError", "Raised when an attempt to use a disabled feature occurs") +define_request_exception("AnnotationsError", "Raised when an attempt to use the annotations feature fails") +define_request_exception( + "ComputeError", + "Raised when an error occurs during a compute algorithm (such as diffexp)", + HTTPStatus.INTERNAL_SERVER_ERROR, +) +define_request_exception("ExceedsLimitError", "Raised when an HTTP request exceeds a limit/quota") +define_request_exception("ColorFormatException", "Raised when color helper functions encounter an unknown color format") +define_request_exception( + "AuthenticationError", "Raised when there is an authentication error", default_status_code=HTTPStatus.UNAUTHORIZED +) + +define_request_exception( + "AnnotationCategoryNameError", + "Raised when an annotation category name cant be saved", + default_status_code=HTTPStatus.UNPROCESSABLE_ENTITY, +) + +define_exception("OntologyLoadFailure", "Raised when reading the ontology file fails") +define_exception("ConfigurationError", "Raised when checking configuration errors") +define_exception("PrepareError", "Raised when data is misprepared") +define_exception("SecretKeyRetrievalError", "Raised when get_secret_key from AWS fails") diff --git a/local_server/common/health.py b/local_server/common/health.py new file mode 100644 index 00000000..8916d95b --- /dev/null +++ b/local_server/common/health.py @@ -0,0 +1,33 @@ +from http import HTTPStatus +from flask import make_response, jsonify + +from local_server import __version__ as cellxgene_version +from local_server.common.data_locator import DataLocator + + +def _is_accessible(path, config): + if path is None: + return True + + try: + dl = DataLocator(path, region_name=config.data_locator__s3__region_name) + return dl.exists() + except RuntimeError: + return False + + +def health_check(config): + """ + simple health check - return HTTP response. + See https://tools.ietf.org/id/draft-inadarei-api-health-check-01.html + """ + health = {"status": None, "version": "1", "releaseID": cellxgene_version} + + server_config = config.server_config + check = _is_accessible(server_config.single_dataset__datapath, server_config) + + health["status"] = "pass" if check else "fail" + code = HTTPStatus.OK if health["status"] == "pass" else HTTPStatus.BAD_REQUEST + response = make_response(jsonify(health), code) + response.headers["Content-Type"] = "application/health+json" + return response diff --git a/local_server/common/rest.py b/local_server/common/rest.py new file mode 100644 index 00000000..8e12bd2a --- /dev/null +++ b/local_server/common/rest.py @@ -0,0 +1,330 @@ +import copy +import logging +import sys +from http import HTTPStatus +import zlib + +from flask import make_response, jsonify, current_app, abort +from werkzeug.urls import url_unquote + +from local_server.common.config.client_config import get_client_config, get_client_userinfo +from local_server.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg +from local_server.common.errors import ( + FilterError, + JSONEncodingValueError, + PrepareError, + DisabledFeatureError, + ExceedsLimitError, + DatasetAccessError, + ColorFormatException, +) + +import json +from local_server.data_common.fbs.matrix import decode_matrix_fbs + + +def abort_and_log(code, logmsg, loglevel=logging.DEBUG, include_exc_info=False): + """ + Log the message, then abort with HTTP code. If include_exc_info is true, + also include current exception via sys.exc_info(). + """ + if include_exc_info: + exc_info = sys.exc_info() + else: + exc_info = False + current_app.logger.log(loglevel, logmsg, exc_info=exc_info) + # Do NOT send log message to HTTP response. + return abort(code) + + +def _query_parameter_to_filter(args): + """ + Convert an annotation value filter, if present in the query args, + into the standard dict filter format used by internal code. + + Query param filters look like: :name=value, where value + may be one of: + - a range, min,max, where either may be an open range by using an asterisc, eg, 10,* + - a value + Eg, + ...?tissue=lung&obs:tissue=heart&obs:num_reads=1000,* + """ + filters = { + "obs": {}, + "var": {}, + } + + # args has already been url-unquoted once. We assume double escaping + # on name and value. + try: + for key, value in args.items(multi=True): + axis, name = key.split(":") + if axis not in ("obs", "var"): + raise FilterError("unknown filter axis") + name = url_unquote(name) + current = filters[axis].setdefault(name, {"name": name}) + + val_split = value.split(",") + if len(val_split) == 1: + if "min" in current or "max" in current: + raise FilterError("do not mix range and value filters") + value = url_unquote(value) + values = current.setdefault("values", []) + values.append(value) + + elif len(val_split) == 2: + if len(current) > 1: + raise FilterError("duplicate range specification") + min = url_unquote(val_split[0]) + max = url_unquote(val_split[1]) + if min != "*": + current["min"] = float(min) + if max != "*": + current["max"] = float(max) + if len(current) < 2: + raise FilterError("must specify at least min or max in range filter") + + else: + raise FilterError("badly formated filter value") + + except ValueError as e: + raise FilterError(str(e)) + + result = {} + for axis in ("obs", "var"): + axis_filter = filters[axis] + if len(axis_filter) > 0: + result[axis] = {"annotation_value": [val for val in axis_filter.values()]} + + return result + + +def schema_get_helper(data_adaptor): + """helper function to gather the schema from the data source and annotations""" + schema = data_adaptor.get_schema() + schema = copy.deepcopy(schema) + + # add label obs annotations as needed + annotations = data_adaptor.dataset_config.user_annotations + if annotations is not None: + label_schema = annotations.get_schema(data_adaptor) + schema["annotations"]["obs"]["columns"].extend(label_schema) + + return schema + + +def schema_get(data_adaptor): + schema = schema_get_helper(data_adaptor) + return make_response(jsonify({"schema": schema}), HTTPStatus.OK) + + +def config_get(app_config, data_adaptor): + config = get_client_config(app_config, data_adaptor) + return make_response(jsonify(config), HTTPStatus.OK) + + +def userinfo_get(app_config, data_adaptor): + config = get_client_userinfo(app_config, data_adaptor) + return make_response(jsonify(config), HTTPStatus.OK) + + +def annotations_obs_get(request, data_adaptor): + fields = request.args.getlist("annotation-name", None) + num_columns_requested = len(data_adaptor.get_obs_keys()) if len(fields) == 0 else len(fields) + if data_adaptor.server_config.exceeds_limit("column_request_max", num_columns_requested): + return abort(HTTPStatus.BAD_REQUEST) + preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"]) + if preferred_mimetype != "application/octet-stream": + return abort(HTTPStatus.NOT_ACCEPTABLE) + + try: + labels = None + annotations = data_adaptor.dataset_config.user_annotations + if annotations: + labels = annotations.read_labels(data_adaptor) + fbs = data_adaptor.annotation_to_fbs_matrix(Axis.OBS, fields, labels) + return make_response(fbs, HTTPStatus.OK, {"Content-Type": "application/octet-stream"}) + except KeyError as e: + return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True) + + +def annotations_put_fbs_helper(data_adaptor, fbs): + """helper function to write annotations from fbs""" + annotations = data_adaptor.dataset_config.user_annotations + if annotations is None: + raise DisabledFeatureError("Writable annotations are not enabled") + + new_label_df = decode_matrix_fbs(fbs) + if not new_label_df.empty: + new_label_df = data_adaptor.check_new_labels(new_label_df) + annotations.write_labels(new_label_df, data_adaptor) + + +def inflate(data): + return zlib.decompress(data) + + +def annotations_obs_put(request, data_adaptor): + annotations = data_adaptor.dataset_config.user_annotations + if annotations is None: + return abort(HTTPStatus.NOT_IMPLEMENTED) + + anno_collection = request.args.get("annotation-collection-name", default=None) + fbs = inflate(request.get_data()) + + if anno_collection is not None: + if not annotations.is_safe_collection_name(anno_collection): + return abort(HTTPStatus.BAD_REQUEST, "Bad annotation collection name") + annotations.set_collection(anno_collection) + + try: + annotations_put_fbs_helper(data_adaptor, fbs) + res = json.dumps({"status": "OK"}) + return make_response(res, HTTPStatus.OK, {"Content-Type": "application/json"}) + except (ValueError, DisabledFeatureError, KeyError) as e: + return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True) + + +def annotations_var_get(request, data_adaptor): + fields = request.args.getlist("annotation-name", None) + num_columns_requested = len(data_adaptor.get_var_keys()) if len(fields) == 0 else len(fields) + if data_adaptor.server_config.exceeds_limit("column_request_max", num_columns_requested): + return abort(HTTPStatus.BAD_REQUEST) + preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"]) + if preferred_mimetype != "application/octet-stream": + return abort(HTTPStatus.NOT_ACCEPTABLE) + + try: + labels = None + annotations = data_adaptor.dataset_config.user_annotations + if annotations is not None: + labels = annotations.read_labels(data_adaptor) + return make_response( + data_adaptor.annotation_to_fbs_matrix(Axis.VAR, fields, labels), + HTTPStatus.OK, + {"Content-Type": "application/octet-stream"}, + ) + except KeyError as e: + return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True) + + +def data_var_put(request, data_adaptor): + preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"]) + if preferred_mimetype != "application/octet-stream": + return abort(HTTPStatus.NOT_ACCEPTABLE) + + filter_json = request.get_json() + filter = filter_json["filter"] if filter_json else None + try: + return make_response( + data_adaptor.data_frame_to_fbs_matrix(filter, axis=Axis.VAR), + HTTPStatus.OK, + {"Content-Type": "application/octet-stream"}, + ) + except (FilterError, ValueError, ExceedsLimitError) as e: + return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True) + + +def data_var_get(request, data_adaptor): + preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"]) + if preferred_mimetype != "application/octet-stream": + return abort(HTTPStatus.NOT_ACCEPTABLE) + + try: + filter = _query_parameter_to_filter(request.args) + return make_response( + data_adaptor.data_frame_to_fbs_matrix(filter, axis=Axis.VAR), + HTTPStatus.OK, + {"Content-Type": "application/octet-stream"}, + ) + except (FilterError, ValueError, ExceedsLimitError) as e: + return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True) + + +def colors_get(data_adaptor): + if not data_adaptor.dataset_config.presentation__custom_colors: + return make_response(jsonify({}), HTTPStatus.OK) + try: + return make_response(jsonify(data_adaptor.get_colors()), HTTPStatus.OK) + except ColorFormatException as e: + return abort_and_log(HTTPStatus.NOT_FOUND, str(e), include_exc_info=True) + + +def diffexp_obs_post(request, data_adaptor): + if not data_adaptor.dataset_config.diffexp__enable: + return abort(HTTPStatus.NOT_IMPLEMENTED) + + args = request.get_json() + try: + # TODO: implement varfilter mode + mode = DiffExpMode(args["mode"]) + + if mode == DiffExpMode.VAR_FILTER or "varFilter" in args: + return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, "varFilter not enabled") + + set1_filter = args.get("set1", {"filter": {}})["filter"] + set2_filter = args.get("set2", {"filter": {}})["filter"] + count = args.get("count", None) + + if set1_filter is None or set2_filter is None or count is None: + return abort_and_log(HTTPStatus.BAD_REQUEST, "missing required parameter") + if Axis.VAR in set1_filter or Axis.VAR in set2_filter: + return abort_and_log(HTTPStatus.BAD_REQUEST, "var axis filter not enabled") + + except (KeyError, TypeError) as e: + return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True) + + try: + diffexp = data_adaptor.diffexp_topN(set1_filter, set2_filter, count) + return make_response(diffexp, HTTPStatus.OK, {"Content-Type": "application/json"}) + except (ValueError, DisabledFeatureError, FilterError, ExceedsLimitError) as e: + return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True) + except JSONEncodingValueError: + # JSON encoding failure, usually due to bad data. Just let it ripple up + # to default exception handler. + current_app.logger.warning(JSON_NaN_to_num_warning_msg) + raise + + +def layout_obs_get(request, data_adaptor): + fields = request.args.getlist("layout-name", None) + num_columns_requested = len(data_adaptor.get_embedding_names()) if len(fields) == 0 else len(fields) + if data_adaptor.server_config.exceeds_limit("column_request_max", num_columns_requested): + return abort(HTTPStatus.BAD_REQUEST) + + preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"]) + if preferred_mimetype != "application/octet-stream": + return abort(HTTPStatus.NOT_ACCEPTABLE) + + try: + return make_response( + data_adaptor.layout_to_fbs_matrix(fields), HTTPStatus.OK, {"Content-Type": "application/octet-stream"} + ) + except (KeyError, DatasetAccessError) as e: + return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True) + except PrepareError: + return abort_and_log( + HTTPStatus.NOT_IMPLEMENTED, + f"No embedding available {request.path}", + loglevel=logging.ERROR, + include_exc_info=True, + ) + + +def layout_obs_put(request, data_adaptor): + if not data_adaptor.dataset_config.embeddings__enable_reembedding: + return abort(HTTPStatus.NOT_IMPLEMENTED) + + args = request.get_json() + filter = args["filter"] if args else None + if not filter: + return abort_and_log(HTTPStatus.BAD_REQUEST, "obs filter is required") + method = args["method"] if args else "umap" + + try: + schema = data_adaptor.compute_embedding(method, filter) + return make_response(jsonify(schema), HTTPStatus.OK, {"Content-Type": "application/json"}) + except NotImplementedError as e: + return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, str(e)) + except (ValueError, DisabledFeatureError, FilterError) as e: + return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True) diff --git a/local_server/common/utils/__init__.py b/local_server/common/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/common/utils/corpora_constants.py b/local_server/common/utils/corpora_constants.py new file mode 100644 index 00000000..fa1641d5 --- /dev/null +++ b/local_server/common/utils/corpora_constants.py @@ -0,0 +1,22 @@ +class CorporaConstants(object): + REQUIRED_SIMPLE_METADATA_FIELDS = [ + "version", + "title", + "layer_descriptions", + "organism", + "organism_ontology_term_id", + ] + + # The Corpora specification requires some values encoded as JSON due to the inability of AnnData to store complex + # types. + OPTIONAL_JSON_ENCODED_METADATA_FIELD = ["contributors", "project_links"] + + OPTIONAL_SIMPLE_METADATA_FIELDS = [ + "preprint_doi", + "publication_doi", + "default_embedding", + "default_field", + "tags", + "project_name", + "project_description", + ] diff --git a/local_server/common/utils/type_conversion_utils.py b/local_server/common/utils/type_conversion_utils.py new file mode 100644 index 00000000..971bf641 --- /dev/null +++ b/local_server/common/utils/type_conversion_utils.py @@ -0,0 +1,158 @@ +import logging + +import numpy as np +import pandas as pd + + +def get_dtypes_and_schemas_of_dataframe(dataframe: pd.DataFrame): + dtypes_by_column_name = {} + schema_type_hints_by_column_name = {} + + for column_name, column_values in dataframe.items(): + ( + dtypes_by_column_name[column_name], + schema_type_hints_by_column_name[column_name], + ) = get_dtype_and_schema_of_array(column_values) + + return dtypes_by_column_name, schema_type_hints_by_column_name + + +def get_dtype_of_array(array: pd.Series): + return get_dtype_and_schema_of_array(array)[0] + + +def get_schema_type_hint_of_array(array: pd.Series): + return get_dtype_and_schema_of_array(array)[1] + + +def get_dtype_and_schema_of_array(array: pd.Series): + return ( + get_dtype_from_dtype(array.dtype, array_values=array), + get_schema_type_hint_from_dtype(array.dtype, array_values=array), + ) + + +def get_dtype_from_dtype(dtype, array_values=None): + """ + Given a data type, finds the equivalent data type that the array should be encoded as. Notably, this is relevant + for 64 bit values which will get downcast to 32 bit. + """ + + dtype_name = dtype.name + dtype_kind = dtype.kind + + if dtype_name == "bool": + return np.uint8 + if dtype_name == "object" and dtype_kind == "O": + return str + if dtype_name == "category": + return get_dtype_from_dtype(dtype.categories.dtype, array_values) + + if can_cast_to_int32(dtype, array_values): + return np.int32 + if can_cast_to_float32(dtype, array_values): + return np.float32 + if not can_cast_to_float32(dtype, array_values): + return np.float64 + + raise TypeError(f"Annotations of type {dtype} are unsupported.") + + +def get_schema_type_hint_from_dtype(dtype, array_values=None): + """ + Returns a dictionary that contains type hints about the data type given, especially if the data type is 64 bit + and will be downcast to 32 bit. + """ + + dtype_name = dtype.name + dtype_kind = dtype.kind + + if dtype == np.float32 or dtype == np.int32: + return {"type": dtype_name} + if dtype_name == "bool": + return {"type": "boolean"} + if dtype_name == "object" and dtype_kind == "O": + return {"type": "string"} + if dtype_name == "category": + return {"type": "categorical", "categories": dtype.categories.tolist()} + + if can_cast_to_int32(dtype, array_values): + return {"type": "int32"} + if can_cast_to_float32(dtype, array_values): + return {"type": "float32"} + if dtype_kind == "f" and not can_cast_to_float32(dtype, array_values): + return {"type": "float64"} + + raise TypeError(f"Annotations of type {dtype} are unsupported.") + + +def can_cast_to_float32(dtype, array_values): + """ + Optimistically returns True signifying that a type downcast to float32 is possible whenever the incoming type is + a float. + + We also handle a special case here where the array is a Series object with integer categorical values AND NaNs. + Since NaNs are floating points in numpy, we upcast the integer array to float32 and return True. + """ + + if dtype.kind == "f": + if not np.can_cast(dtype, np.float32): + logging.warning(f"Type {dtype.name} will be converted to 32 bit float and may lose precision.") + + return True + + if dtype.kind == "O" and array_values.hasnans: + return True + + return False + + +def can_cast_to_int32(dtype, array_values=None): + """ + A type can be cast to 32 bit, overriding the numpy `cast_cast` function if the values in the array that are of + the higher precision type has values that are entirely within the range of the downcast type. + """ + + # Since a NaN is technically a float, any array that contains NaNs cannot be cast to an integer so immediately + # return False. + if array_values.hasnans: + return False + + # If the array is categorical, then we need to order the array values so that functions min and max that occur + # later, can function. They do not function on unordered categories. + ordered_array_values = array_values + if array_values.dtype.name == "category" and not array_values.cat.ordered: + ordered_array_values = array_values.cat.as_ordered() + + if dtype.kind in ["i", "u"]: + if np.can_cast(dtype, np.int32): + return True + ii32 = np.iinfo(np.int32) + if ( + not ordered_array_values.empty + and (ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max) + or ordered_array_values.empty + ): + return True + return False + + +def convert_pandas_series_to_numpy(series_to_convert: pd.Series, dtype): + if series_to_convert.hasnans and dtype == np.int32: + logging.error("Cannot convert a pandas Series object to an integer dtype if it contains NaNs.") + + return series_to_convert.to_numpy(dtype) + + +def convert_string_to_value(value: str): + """convert a string to value with the most appropriate type""" + if value.lower() == "true": + return True + if value.lower() == "false": + return False + if value == "null": + return None + try: + return eval(value) + except: # noqa E722 + return value diff --git a/local_server/common/utils/utils.py b/local_server/common/utils/utils.py new file mode 100644 index 00000000..909999c8 --- /dev/null +++ b/local_server/common/utils/utils.py @@ -0,0 +1,118 @@ +import contextlib +import errno +import importlib.util +import logging +import os +import pkgutil +import socket +from urllib.parse import urlsplit, urljoin + +import numpy as np +from flask import json + +from local_server.common.errors import ConfigurationError + + +def find_available_port(host, port=5005): + """ + Helper method to find open port on host. Tries 5000 ports incremented from the specified port + """ + # Takes approx 2 seconds to do a scan of 5000 ports on my laptop + num_ports_to_try = 5000 + for port_to_try in range(port, port + num_ports_to_try): + if is_port_available(host, port_to_try): + return port_to_try + raise socket.error(errno.EADDRINUSE, f"No port in range {port} - {port + num_ports_to_try - 1} available.") + + +def is_port_available(host, port): + is_available = False + with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: + try: + s.bind((host, port)) + is_available = True + except socket.error: + pass + return is_available + + +def sort_options(command): + """ + Helper for the click options - will sort options in a command, and can + be used as a decorator. + """ + command.params.sort(key=lambda p: p.name) + return command + + +def path_join(base, *urls): + """ + this is like urllib.parse.urljoin, except it works around the scheme-specific + cleverness in the aforementioned code, ignores anything in the url except the path, + and accepts more than one url. + """ + if not base.endswith("/"): + base += "/" + btpl = urlsplit(base) + path = btpl.path + for url in urls: + utpl = urlsplit(url) + if btpl.scheme == "": + path = os.path.join(path, utpl.path) + path = os.path.normpath(path) + else: + path = urljoin(path, utpl.path) + return btpl._replace(path=path).geturl() + + +class Float32JSONEncoder(json.JSONEncoder): + def __init__(self, *args, **kwargs): + """ + NaN/Infinities are illegal in standard JSON. Python extends JSON with + non-standard symbols that most JavaScript JSON parsers do not understand. + The `allow_nan` parameter will force Python simplejson to throw an ValueError + if it runs into non-finite floating point values which are unsupported by + standard JSON. + """ + kwargs["allow_nan"] = False + super().__init__(*args, **kwargs) + + def default(self, obj): + if isinstance(obj, np.float32): + return float(obj) + elif isinstance(obj, np.integer): + return int(obj) + return json.JSONEncoder.default(self, obj) + + +def custom_format_warning(msg, *args, **kwargs): + return f"[cellxgene] Warning: {msg} \n" + + +def jsonify_numpy(data): + return json.dumps(data, cls=Float32JSONEncoder, allow_nan=False) + + +def import_plugins(plugin_module): + """ + Load optional plugin modules from local_server.common.plugins + + If you would like to customize cellxgene, you can add submodules to server.common.plugins before running the app. + This code will import each, loading the code in each. If no plugins are defined, initializing the app continues as + normal. + """ + loaded_modules = [] + try: + pkg = importlib.import_module(plugin_module) + for loader, name, is_pkg in pkgutil.walk_packages(pkg.__path__): + full_name = f"{plugin_module}.{name}" + try: + module = importlib.import_module(full_name) + except Exception as e: + raise ConfigurationError(f"Unexpected error while importing plugin: {plugin_module}.{name}: {str(e)}") + loaded_modules.append(module) + except ModuleNotFoundError as e: + # This exception occurs when the plugin_module does not exist (not an error). + logging.debug(f"No plugins found in module: {plugin_module}: {str(e)}") + + return loaded_modules diff --git a/local_server/common/web/__init__.py b/local_server/common/web/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/compute/__init__.py b/local_server/compute/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/compute/diffexp_generic.py b/local_server/compute/diffexp_generic.py new file mode 100644 index 00000000..71fe9a59 --- /dev/null +++ b/local_server/compute/diffexp_generic.py @@ -0,0 +1,134 @@ +import numpy as np +from scipy import sparse, stats + + +def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01): + """ + Return differential expression statistics for top N variables. + + Algorithm: + - compute log fold change (log2(meanA/meanB)) + - compute Welch's t-test statistic and pvalue (w/ Bonferroni correction) + - return top N abs(logfoldchange) where lfc > diffexp_lfc_cutoff + + If there are not N which meet criteria, augment by removing the logfoldchange + threshold requirement. + + Notes on alogrithm: + - Welch's ttest provides basic statistics test. + https://en.wikipedia.org/wiki/Welch%27s_t-test + - p-values adjusted with Bonferroni correction. + https://en.wikipedia.org/wiki/Bonferroni_correction + + :param adaptor: DataAdaptor instance + :param maskA: observation selection mask for set 1 + :param maskB: observation selection mask for set 2 + :param top_n: number of variables to return stats for + :param diffexp_lfc_cutoff: minimum + :return: for top N genes, [ varindex, logfoldchange, pval, pval_adj ] + """ + + dataA = adaptor.get_X_array(maskA, None) + dataB = adaptor.get_X_array(maskB, None) + + # mean, variance, N - calculate for both selections + meanA, vA, nA = mean_var_n(dataA) + meanB, vB, nB = mean_var_n(dataB) + res = diffexp_ttest_from_mean_var(meanA, vA, nA, meanB, vB, nB, top_n, diffexp_lfc_cutoff) + + return res + + +def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp_lfc_cutoff): + n_var = meanA.shape[0] + top_n = min(top_n, n_var) + + # variance / N + vnA = varA / min(nA, nB) # overestimate variance, would normally be nA + vnB = varB / min(nA, nB) # overestimate variance, would normally be nB + sum_vn = vnA + vnB + + # degrees of freedom for Welch's t-test + with np.errstate(divide="ignore", invalid="ignore"): + dof = sum_vn ** 2 / (vnA ** 2 / (nA - 1) + vnB ** 2 / (nB - 1)) + dof[np.isnan(dof)] = 1 + + # Welch's t-test score calculation + with np.errstate(divide="ignore", invalid="ignore"): + tscores = (meanA - meanB) / np.sqrt(sum_vn) + tscores[np.isnan(tscores)] = 0 + + # p-value + pvals = stats.t.sf(np.abs(tscores), dof) * 2 + pvals_adj = pvals * n_var + pvals_adj[pvals_adj > 1] = 1 # cap adjusted p-value at 1 + + # logfoldchanges: log2(meanA / meanB) + logfoldchanges = np.log2(np.abs((meanA + 1e-9) / (meanB + 1e-9))) + + # find all with lfc > cutoff + lfc_above_cutoff_idx = np.nonzero(np.abs(logfoldchanges) > diffexp_lfc_cutoff)[0] + stats_to_sort = np.abs(tscores) + + # derive sort order + if lfc_above_cutoff_idx.shape[0] > top_n: + # partition top N + rel_t_partition = np.argpartition(stats_to_sort[lfc_above_cutoff_idx], -top_n)[-top_n:] + t_partition = lfc_above_cutoff_idx[rel_t_partition] + # sort the top N partition + rel_sort_order = np.argsort(stats_to_sort[t_partition])[::-1] + sort_order = t_partition[rel_sort_order] + else: + # partition and sort top N, ignoring lfc cutoff + partition = np.argpartition(stats_to_sort, -top_n)[-top_n:] + rel_sort_order = np.argsort(stats_to_sort[partition])[::-1] + indices = np.indices(stats_to_sort.shape)[0] + sort_order = indices[partition][rel_sort_order] + + # top n slice based upon sort order + logfoldchanges_top_n = logfoldchanges[sort_order] + pvals_top_n = pvals[sort_order] + pvals_adj_top_n = pvals_adj[sort_order] + + # varIndex, logfoldchange, pval, pval_adj + result = [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in range(top_n)] + return result + + +# Convenience function which handles sparse data +def mean_var_n(X): + """ + Two-pass variance calculation. Numerically (more) stable + than naive methods (and same method used by numpy.var()) + https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Two-pass + """ + # fp_err_occurred is a flag indicating that a floating point error + # occured somewhere in our compute. Used to trigger non-finite + # number handling. + fp_err_occurred = False + + def fp_err_set(err, flag): + nonlocal fp_err_occurred + fp_err_occurred = True + + with np.errstate(divide="call", invalid="call", call=fp_err_set): + n = X.shape[0] + if sparse.issparse(X): + mean = X.mean(axis=0).A1 + dfm = X - mean + sumsq = np.sum(np.multiply(dfm, dfm), axis=0).A1 + v = sumsq / (n - 1) + else: + mean = X.mean(axis=0) + dfm = X - mean + sumsq = np.sum(np.multiply(dfm, dfm), axis=0) + v = sumsq / (n - 1) + + if fp_err_occurred: + mean[np.isfinite(mean) == False] = 0 # noqa: E712 + v[np.isfinite(v) == False] = 0 # noqa: E712 + else: + mean[np.isnan(mean)] = 0 + v[np.isnan(v)] = 0 + + return mean, v, n diff --git a/local_server/compute/scanpy.py b/local_server/compute/scanpy.py new file mode 100644 index 00000000..309eb270 --- /dev/null +++ b/local_server/compute/scanpy.py @@ -0,0 +1,53 @@ +import importlib +import numpy as np + +""" +Wrapper for various scanpy modules. Will raise NotImplementedError if the scanpy +module is not installed/available +""" + + +def get_scanpy_module(): + try: + sc = importlib.import_module("scanpy") + # Future: we could enforce versions here, eg, lookat sc.__version__ + return sc + except ModuleNotFoundError as e: + raise NotImplementedError("Please install scanpy to enable UMAP re-embedding") from e + except Exception as e: + # will capture other ImportError corner cases + raise NotImplementedError() from e + + +def scanpy_umap(adata, obs_mask=None, pca_options={}, neighbors_options={}, umap_options={}): + """ + Given adata and an obs mask, return a new embedding for adata[obs_mask, :] + as an ndarray of shape (len(obs_mask), N), where N>=2. + + Do NOT mutate adata. + """ + + # backed mode is incompatible with the current implementation + if adata.isbacked: + raise NotImplementedError("Backed mode is incompatible with re-embedding") + + # safely get scanpy module, which may not be present. + sc = get_scanpy_module() + + # https://github.com/theislab/anndata/issues/311 + obs_mask = slice(None) if obs_mask is None else obs_mask + adata = adata[obs_mask, :].copy() + + for k in list(adata.obsm.keys()): + del adata.obsm[k] + for k in list(adata.uns.keys()): + del adata.uns[k] + + sc.pp.pca(adata, zero_center=None, n_comps=min(adata.n_vars - 1, 50), **pca_options) + sc.pp.neighbors(adata, **neighbors_options) + sc.tl.umap(adata, **umap_options) + + umap = adata.obsm["X_umap"] + result = np.full((obs_mask.shape[0], umap.shape[1]), np.NaN) + result[obs_mask] = umap + return result diff --git a/local_server/converters/__init__.py b/local_server/converters/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/converters/schema/__init__.py b/local_server/converters/schema/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/converters/schema/gene_symbol.py b/local_server/converters/schema/gene_symbol.py new file mode 100644 index 00000000..2d0de5a7 --- /dev/null +++ b/local_server/converters/schema/gene_symbol.py @@ -0,0 +1,211 @@ +"""Helpers for converting and checking HGNC gene symbols.""" + +import argparse +import enum +import logging +import os +import re +import numpy as np +import pandas as pd + + +def get_upgraded_var_index(var, hgnc_path=None): + """Given an anndata var dataframe, return a new index for the dataframe + where human gene symbols have been upgraded to the current HGNC set. + """ + + if not hgnc_path: + hgnc_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "hgnc_complete_set.txt.gz") + + hgnc_symbol_checker = HGNCSymbolChecker.from_hgnc_records(hgnc_path) + + return pd.Index([hgnc_symbol_checker.upgrade_symbol(s) for s in var.index]) + + +class SymbolStatus(enum.Enum): + """The status of a symbol in the HGNC database. + + APPROVED: Currently a valid symbol + WITHDRAWN: A previously approved HGNC symbol for a gene that has since been shown + not to exist _unless_ that symbol is also approved + AMBIGUOUS: A symbol that is not approved but is an alias or previous symbol for + multiple approved symbols + UPGRADABLE: A symbol that is not approved but unambiguously maps to an approved + symbol + UNKNOWN: A symbol that does not appear in HGNC + """ + + APPROVED = 1 + WITHDRAWN = 2 + AMBIGUOUS = 3 + UPGRADABLE = 4 + UNKNOWN = 5 + + +class HGNCSymbolChecker: + """Handle checking and correcting HGNC symbols.""" + + def __init__(self, approved_symbols, withdrawn_symbols, ambiguous_symbols, symbol_map): + self.approved_symbols = approved_symbols + self.withdrawn_symbols = withdrawn_symbols + self.ambiguous_symbols = ambiguous_symbols + self.symbol_map = symbol_map + + def print_symbol_map(self): + """Print out a map from old symbol to new symbol.""" + + for symbol_pair in self.symbol_map.items(): + print("\t".join(symbol_pair)) + + def check_symbol(self, symbol): + """See if a symbol if approved or something else.""" + if symbol in self.approved_symbols: + return SymbolStatus.APPROVED + + if symbol in self.withdrawn_symbols: + return SymbolStatus.WITHDRAWN + + if symbol in self.ambiguous_symbols: + return SymbolStatus.AMBIGUOUS + + if symbol in self.symbol_map: + return SymbolStatus.UPGRADABLE + + return SymbolStatus.UNKNOWN + + def upgrade_symbol(self, symbol): + """Return the approved symbol for the given symbol. + + If the symbol cannot be upgraded, just return the original symbol. + """ + + fixed_symbol, stripped_symbol = format_symbol(symbol) + + if fixed_symbol in self.approved_symbols: + return fixed_symbol + elif fixed_symbol in self.symbol_map: + return self.symbol_map[fixed_symbol] + elif stripped_symbol in self.approved_symbols: + return stripped_symbol + elif stripped_symbol in self.symbol_map: + return self.symbol_map[stripped_symbol] + + return symbol + + @classmethod + def from_hgnc_records(cls, hgnc_dataset_path): + """Parse a hgnc database download into a HGNCSymbolChecker object.""" + + def all_symbols(record): + """Get all the symbols associated with an HGNC record including previous, alias, + and approved.""" + yield format_symbol(record["symbol"])[0] + for symbol in alias_and_previous_symbols(record): + yield symbol + + def alias_and_previous_symbols(record): + """Get alias and previous symbols from an HGNC record.""" + for field in ("alias_symbol", "prev_symbol"): + if record[field] is not np.nan: + for symbol in record[field].split("|"): + yield format_symbol(symbol)[0] + # Sometimes something like HGNC:1234 appears in datasets, which we + # want to fix as well. + yield record["hgnc_id"] + + hgnc_records = pd.read_csv(hgnc_dataset_path, sep="\t", header=0, low_memory=False).to_dict("records") + + # Get all symbols that are currently approved. + approved_symbols = set() + for record in hgnc_records: + if record["status"] == "Approved": + approved_symbols.add(format_symbol(record["symbol"])[0]) + + # Get all symbols that have been withdrawn + withdrawn_symbols = set() + for record in hgnc_records: + if record["status"] == "Entry Withdrawn": + for symbol in all_symbols(record): + withdrawn_symbols.add(symbol) + + # If a symbol is both approved and withdrawn, be optimistic and call it approved + logging.warning( + f"Some symbols are simulaneously withdrawn and approved\n" + f"We will treat them at approved:\n" + f"{withdrawn_symbols.intersection(approved_symbols)}" + ) + withdrawn_symbols = withdrawn_symbols.difference(approved_symbols) + + # Now try to map from symbols that are not approved but are an alias or previous symbol for an approved symbol + alias_previous_to_approved = {} + ambiguous_symbols = set() + + for record in hgnc_records: + if record["status"] == "Approved": + + # The approved symbol is what we'll map to + approved_symbol = format_symbol(record["symbol"])[0] + + for symbol in alias_and_previous_symbols(record): + + # If the alias or previous symbol is also an approved symbol, + # we'll just leave it alone + if symbol in approved_symbols: + continue + + # If the alias or previous symbol maps to a different approved symbol, mark it as ambiguous + if symbol in alias_previous_to_approved and alias_previous_to_approved[symbol] != approved_symbol: + ambiguous_symbols.add(symbol) + else: + alias_previous_to_approved[symbol] = approved_symbol + + # Remove all the ambiguous symbols from the map + for ambiguous_symbol in ambiguous_symbols: + alias_previous_to_approved.pop(ambiguous_symbol) + + return HGNCSymbolChecker(approved_symbols, withdrawn_symbols, ambiguous_symbols, alias_previous_to_approved) + + +def format_symbol(symbol): + """HGNC rules say symbols should all be upper case except for C#orf#. However, case is + variable in both alias and previous symbols as well as in the symbols we get in + submissions. So, upper case everything except for the one situation where mixed-case + is allowed, which are the genes like C2orf157. + + Also, seurat and scanpy append ".1" or "-1" to duplicated gene names, and these altered + names persist throughout the life of the object. They won't match against the HGNC database + and we want to merge them, so we need to strip off the suffix and try matching again. + + This function takes a symbol and returns the symbol with the fixed case and also with the + seurat/scanpy suffix stripped off. + """ + + match = re.match(r"^(C)(\d+)(orf)(\d+)$", symbol, re.IGNORECASE) + + if match: + fixed_case = f"C{match.group(2)}orf{match.group(4)}" + else: + fixed_case = symbol.upper() + + suffix_stripped = re.sub(r"[\.\-]\d+$", "", fixed_case) + + return fixed_case, suffix_stripped + + +def main(): + """When called as main, parse a given hgnc download and print out a map from old to new + symbol. + """ + parser = argparse.ArgumentParser() + parser.add_argument( + "hgnc_dataset", help="HGNC dataset tsv, available from www.genenames.org/download/statistics-and-files/" + ) + args = parser.parse_args() + + hgnc_symbol_checker = HGNCSymbolChecker.from_hgnc_records(args.hgnc_dataset) + + hgnc_symbol_checker.print_symbol_map() + + +if __name__ == "__main__": + main() diff --git a/local_server/converters/schema/hgnc_complete_set.txt.gz b/local_server/converters/schema/hgnc_complete_set.txt.gz new file mode 100644 index 00000000..29c3c7a9 Binary files /dev/null and b/local_server/converters/schema/hgnc_complete_set.txt.gz differ diff --git a/local_server/converters/schema/ontology.py b/local_server/converters/schema/ontology.py new file mode 100644 index 00000000..8a524402 --- /dev/null +++ b/local_server/converters/schema/ontology.py @@ -0,0 +1,86 @@ +"""Methods for working with ontologies and the OLS.""" +from urllib.parse import quote_plus + +import requests + +OLS_API_ROOT = "http://www.ebi.ac.uk/ols/api" + +# Curie means something like CL:0000001 + + +def _ontology_name(curie): + """Get the name of the ontology from the curie, CL or UBERON for example.""" + return curie.split(":")[0] + + +def _ontology_value(curie): + """Get the id component of the curie, 0000001 from CL:0000001 for example.""" + return curie.split(":")[1] + + +def _double_encode(url): + """Double url encode a url. This is required by the OLS API.""" + return quote_plus(quote_plus(url)) + + +def _iri(curie): + """Get the iri from a curie. This is a bit hopeful that they all map to purl.obolibrary.org""" + if _ontology_name(curie) == "EFO": + return f"http://www.ebi.ac.uk/efo/EFO_{_ontology_value(curie)}" + return f"http://purl.obolibrary.org/obo/{_ontology_name(curie)}_{_ontology_value(curie)}" + + +class OntologyLookupError(Exception): + """Exception for some problem with looking up ontology information.""" + + +def _ontology_info_url(curie): + """Get the to make a GET to to get information about an ontology term.""" + + # If the curie is empty, just return an empty string. This happens when there is no + # valid ontology value. + if not curie: + return "" + else: + return f"{OLS_API_ROOT}/ontologies/{_ontology_name(curie)}/terms/{_double_encode(_iri(curie))}" + + +def get_ontology_label(curie): + """For a given curie like 'CL:1000413', get the label like 'endothelial cell of artery'""" + + url = _ontology_info_url(curie) + + if not url: + return "" + + response = requests.get(url) + + if not response.ok: + raise OntologyLookupError( + f"Curie {curie} lookup failed, got status code {response.status_code}: {response.text}" + ) + return response.json()["label"] + + +def lookup_candidate_term(label, ontology="cl", method="select"): + """Lookup candidate terms for a label. This is useful when there is an existing label in a + submitted dataset, and you want to find an appropriate ontology term. + + Args: + label: the label to find ontology terms for + ontology: the ontology to search in, cl or uberon or efo for example + method: select or search. search provides much broader results + + Returns: + list of (curie, label) tuples returned by OLS + """ + # using OLS REST API [https://www.ebi.ac.uk/ols/docs/api] + url = f"{OLS_API_ROOT}/{method}?q={quote_plus(label)}&ontology={ontology.lower()}" + response = requests.get(url) + + if not response.ok: + raise OntologyLookupError( + f"Label {label} lookup failed, got status code {response.status_code}: {response.text}" + ) + + return [(r["obo_id"], r["label"]) for r in response.json()["response"]["docs"]] diff --git a/local_server/converters/schema/remix.py b/local_server/converters/schema/remix.py new file mode 100644 index 00000000..5cc60746 --- /dev/null +++ b/local_server/converters/schema/remix.py @@ -0,0 +1,264 @@ +import argparse +import collections +import json +import logging +import math +import string + +import anndata +import numpy as np +import pandas as pd +import yaml + +from . import gene_symbol +from . import ontology +from . import validate + +REPLACE_SUFFIX = "_original" +ONTOLOGY_SUFFIX = "_ontology_term_id" + + +def is_curie(value): + """Return True iff the value is an OBO-id CURIE like EFO:000001""" + return (value.count(":") + and all(len(part) > 0 for part in value.split(":")) + and all(c in string.digits for c in value.split(":")[1])) + + +def is_ontology_field(field_name): + """Return True iff the field_name is an ontology field like tissue_ontology_term_id""" + return field_name.endswith(ONTOLOGY_SUFFIX) + + +def get_label_field_name(field_name): + """Get the associated label field from an ontology field, assay_ontology_term_id --> assay""" + return field_name[: -len(ONTOLOGY_SUFFIX)] + + +def split_suffix(maybe_curie): + """Split off the (cell culture) or (organoid) suffix.""" + + suffixes = [" (cell culture)", " (organoid)"] + for suffix in suffixes: + if maybe_curie.endswith(suffix): + return maybe_curie[:-len(suffix)], suffix + return maybe_curie, "" + + +def get_curie_and_label(maybe_curie): + """Given a string that might be a curie, return a (curie, label) pair""" + + maybe_curie, suffix = split_suffix(maybe_curie) + if not is_curie(maybe_curie): + return ("", maybe_curie + suffix) + return (maybe_curie + suffix, ontology.get_ontology_label(maybe_curie) + suffix) + + +def safe_add_field(adata_attr, field_name, field_value): + """Add a field and value to an AnnData, but don't clobber an exising value.""" + + if ( + isinstance(field_value, list) + and field_value + and isinstance(field_value[0], dict) + ): + field_value = json.dumps(field_value) + if field_name in adata_attr: + adata_attr[field_name + REPLACE_SUFFIX] = adata_attr[field_name] + adata_attr[field_name] = field_value + + +def remix_uns(adata, uns_config): + """Add fields from the config to adata.uns""" + for field_name, field_value in uns_config.items(): + + if is_ontology_field(field_name): + # If it's an ontology field, look it up + label_field_name = get_label_field_name(field_name) + ontology_term, ontology_label = get_curie_and_label(field_value) + safe_add_field(adata.uns, field_name, ontology_term) + safe_add_field(adata.uns, label_field_name, ontology_label) + else: + safe_add_field(adata.uns, field_name, field_value) + + +def remix_obs(adata, obs_config): + """Add fields from the config to adata.obs""" + + for field_name, field_value in obs_config.items(): + + if isinstance(field_value, dict): + # If the value is a dict, that means we are supposed to map from an + # existing column to the new one + source_column, column_map = next(iter(field_value.items())) + nan_value = None + for key in column_map: + if isinstance(key, float) and math.isnan(key): + nan_value = column_map[key] + if nan_value is not None: + column_map["nan"] = nan_value + + for key in column_map: + if key not in adata.obs[source_column].unique(): + logging.warning(f'Key {key} not in adata.obs["{source_column}"]') + + for value in adata.obs[source_column].unique(): + if value not in column_map: + logging.warning(f'Value {value} in adata.obs["{source_column}"] not in translation dict') + + if is_ontology_field(field_name): + ontology_term_map, ontology_label_map = {}, {} + logging.info(f"Looking up labels for {field_name}") + for original_value, maybe_curie in column_map.items(): + curie, label = get_curie_and_label(maybe_curie) + ontology_term_map[original_value] = curie + ontology_label_map[original_value] = label + logging.info(f"Mapping {original_value} -> {curie} -> {label}") + + ontology_column = adata.obs[source_column].replace( + ontology_term_map, inplace=False + ) + label_column = adata.obs[source_column].replace( + ontology_label_map, inplace=False + ) + + safe_add_field(adata.obs, field_name, ontology_column) + safe_add_field( + adata.obs, get_label_field_name(field_name), label_column + ) + else: + label_column = adata.obs[source_column].replace( + column_map, inplace=False + ) + safe_add_field(adata.obs, field_name, label_column) + + else: + if is_ontology_field(field_name): + # If it's an ontology field, look it up + label_field_name = get_label_field_name(field_name) + ontology_term, ontology_label = get_curie_and_label(field_value) + safe_add_field(adata.obs, field_name, ontology_term) + safe_add_field(adata.obs, label_field_name, ontology_label) + else: + safe_add_field(adata.obs, field_name, field_value) + + +def merge_df(df, domain, index, columns): + """ + Given a dataframe with duplicate column labels, merge and return a dataframe where + the duplicates have been merged together, resulting in a dataframe with unique column + labels. + + "merge" depends on the value of domain. If the domain is "raw", then duplicate columns + can just be summed. If it's "log1p" or "sqrt", it needs to be exp1m'd or squared, then + summed, and then logged or sqrt'd again. + """ + + if not isinstance(df, np.ndarray): + to_merge = df.toarray() + else: + to_merge = df + if domain == "raw": + merged_df = pd.DataFrame(to_merge, index=index, columns=columns).sum( + axis=1, level=0, skipna=False + ) + elif domain == "log1p": + merged_df = ( + pd.DataFrame(np.expm1(to_merge, dtype=np.float128), index=index, columns=columns) + .sum(axis=1, level=0, skipna=False) + ) + merged_df = pd.DataFrame(np.log1p(merged_df.to_numpy()), index=merged_df.index, columns=merged_df.columns) + elif domain == "sqrt": + merged_df = ( + pd.DataFrame(np.square(to_merge), index=index, columns=columns) + .sum(axis=1, level=0, skipna=False) + ) + merged_df = pd.DataFrame(np.sqrt(merged_df.to_numpy()), index=merged_df.index, columns=merged_df.columns) + + return merged_df + + +def fixup_gene_symbols(adata, fixup_config): + """Update the var index to hold a consistent set of HGNC gene symbols.""" + + upgraded_var_index = gene_symbol.get_upgraded_var_index(adata.var) + + merged_X = merge_df(adata.X, fixup_config["X"], adata.obs.index, upgraded_var_index) + fixup_adata = anndata.AnnData( + X=merged_X, + obs=adata.obs, + var=merged_X.columns.to_frame(name="hgnc_gene_symbol"), + uns=adata.uns, + obsm=adata.obsm, + ) + + for layer, domain in fixup_config.items(): + if layer == "X": + continue + if layer == "raw.X": + df = adata.raw.X + else: + df = adata.layers[layer] + + merged_df = merge_df(df, domain, adata.obs.index, upgraded_var_index) + assert merged_df.index.equals(merged_X.index) + assert merged_df.columns.equals(merged_X.columns) + + if domain == "raw": + fixup_raw = anndata.AnnData( + X=merged_df, + obs=adata.obs, + var=merged_X.columns.to_frame(name="hgnc_gene_symbol"), + ) + fixup_adata.raw = fixup_raw + else: + fixup_adata.layers[layer] = merged_df + + return fixup_adata + +def _strip_version(adata): + """Remove version information from the AnnData object.""" + + if "version" in adata.uns_keys(): + del adata.uns["version"] + +def apply_schema(source_h5ad, remix_config, output_filename): + + try: + import scanpy + except ImportError: + raise ImportError("scanpy must be installed for cellxgene schema") + adata = scanpy.read_h5ad(source_h5ad) + config = yaml.load(open(remix_config), Loader=yaml.FullLoader) + remix_uns(adata, config["uns"]) + remix_obs(adata, config["obs"]) + + if config.get("fixup_gene_symbols"): + adata = fixup_gene_symbols(adata, config["fixup_gene_symbols"]) + + if ("version" in adata.uns_keys() + and isinstance(adata.uns["version"], collections.Mapping) + and "corpora_schema_version" in adata.uns["version"]): + schema_version = adata.uns["version"]["corpora_schema_version"] + try: + validate.get_schema_definition(schema_version) + except ValueError: + logging.warning(f"Stripping version information out of AnnData because schema " + f"version {schema_version} is unknown.") + _strip_version(adata) + + if not validate.validate_adata(adata, shallow=False): + logging.warning(f"Stripping version information out of AnnData because it does not " + f"follow schema version {schema_version} .") + _strip_version(adata) + + adata.write_h5ad(output_filename, compression="gzip") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--source-h5ad", required=True) + parser.add_argument("--remix-config", required=True) + parser.add_argument("--output-filename", required=True) + args = parser.parse_args() + apply_schema(args.source_h5ad, args.remix_config, args.output_filename) diff --git a/local_server/converters/schema/schema_definitions/1_0_0.yaml b/local_server/converters/schema/schema_definitions/1_0_0.yaml new file mode 100644 index 00000000..acff2238 --- /dev/null +++ b/local_server/converters/schema/schema_definitions/1_0_0.yaml @@ -0,0 +1,95 @@ +title: Corpora schema version 1.0.0 +type: anndata +components: + uns: + type: dict + keys: + version: + type: dict + keys: + corpora_schema_version: null + corpora_encoding_version: null + title: + type: string + contributors: + type: stringified list of dicts + layer_descriptions: + type: dict + keys: + X: null + organism: + type: string + nullable: false + organism_ontology_term_id: + type: curie + prefixes: + - NCBITaxon + var: + type: dataframe + index: + type: human-readable string + unique: true + obs: + type: dataframe + index: + unique: true + columns: + tissue: + type: human-readable string + nullable: false + tissue_ontology_term_id: + type: suffixed curie + nullable: true + prefixes: + - UBERON + assay: + type: human-readable string + nullable: false + assay_ontology_term_id: + type: curie + nullable: true + prefixes: + - EFO + disease: + type: human-readable string + nullable: false + disease_ontology_term_id: + type: curie + nullable: true + prefixes: + - MONDO + - PATO + cell_type: + type: human-readable string + nullable: false + cell_type_ontology_term_id: + type: curie + nullable: true + prefixes: + - CL + - UBERON + sex: + type: string + enum: + - male + - female + - mixed + - unknown + - other + ethnicity: + type: human-readable string + nullable: false + ethnicity_ontology_term_id: + type: curie + nullable: true + prefixes: + - HANCESTRO + development_stage: + type: human-readable string + nullable: false + development_stage_ontology_term_id: + type: curie + nullable: true + prefixes: + - HsapDv + - EFO diff --git a/local_server/converters/schema/schema_definitions/1_1_0.yaml b/local_server/converters/schema/schema_definitions/1_1_0.yaml new file mode 100644 index 00000000..7531c72e --- /dev/null +++ b/local_server/converters/schema/schema_definitions/1_1_0.yaml @@ -0,0 +1,93 @@ +title: Corpora schema version 1.1.0 +type: anndata +components: + uns: + type: dict + keys: + version: + type: dict + keys: + corpora_schema_version: null + corpora_encoding_version: null + title: + type: string + layer_descriptions: + type: dict + keys: + X: null + organism: + type: string + nullable: false + organism_ontology_term_id: + type: curie + prefixes: + - NCBITaxon + var: + type: dataframe + index: + type: human-readable string + unique: true + obs: + type: dataframe + index: + unique: true + columns: + tissue: + type: human-readable string + nullable: false + tissue_ontology_term_id: + type: suffixed curie + nullable: true + prefixes: + - UBERON + assay: + type: human-readable string + nullable: false + assay_ontology_term_id: + type: curie + nullable: true + prefixes: + - EFO + disease: + type: human-readable string + nullable: false + disease_ontology_term_id: + type: curie + nullable: true + prefixes: + - MONDO + - PATO + cell_type: + type: human-readable string + nullable: false + cell_type_ontology_term_id: + type: curie + nullable: true + prefixes: + - CL + - UBERON + sex: + type: string + enum: + - male + - female + - mixed + - unknown + - other + ethnicity: + type: human-readable string + nullable: false + ethnicity_ontology_term_id: + type: curie + nullable: true + prefixes: + - HANCESTRO + development_stage: + type: human-readable string + nullable: false + development_stage_ontology_term_id: + type: curie + nullable: true + prefixes: + - HsapDv + - EFO diff --git a/local_server/converters/schema/validate.py b/local_server/converters/schema/validate.py new file mode 100644 index 00000000..ec463dd5 --- /dev/null +++ b/local_server/converters/schema/validate.py @@ -0,0 +1,236 @@ +import json +import re +import os +import sys + +import pandas as pd +import yaml + + +def _is_null(v): + """Return True if v is null, for one of the multiple ways a "null" value shows up in an h5ad.""" + return pd.isnull(v) or (hasattr(v, "__len__") and len(v) == 0) + + +def _validate_stringified_list_of_dicts(s): + """Verify that a string can be parsed into a list. + + We have some types that are lists of dicts. Those cannot be stored directly in an h5ad, so we have to + json.dumps them. This verifies that we can load them back. + """ + + try: + list_ = json.loads(s) + if not isinstance(list_, list): + return False + for el in list_: + if not isinstance(el, dict): + return False + return True + except (json.JSONDecodeError, TypeError): + pass + return False + + +def _validate_human_readable_string(s): + """Verify that a string is human-readable. + + There are parts of the schema where a "human-readable" string is required. "Human-readable" is kind + of vague and subjective. I feel like I can read many strings. So here we just check for the main ways + that fails: someone puts in an ontology term id or and ensembl gene/transcript id. + + Returns False if s is not a string or is one of those bad string types. + """ + + return isinstance(s, str) and (not re.match(r"[A-Z]\w+:\d+", s)) and (not re.match(r"ENS[GT]\d+$", s)) + + +def _validate_curie(c, prefixes): + """Verify that a string is a valid compact URI, like EFO:000001. If prefixes is not empty, make sure the + prefix of the curies is in prefixes. + """ + + if not c: + return True + + match = re.match(r"([A-Z]\w+):\d+$", c) + + if prefixes: + return match and match.group(1) in prefixes + else: + return match + + +def _validate_suffixed_curie(c, prefixes): + """Verify that a string is a compact URI with an optional suffix like 'EFO:00001 (cell culture)'""" + + # Pull off the suffix + suffix = re.findall(r"\ \(.*\)$", c) + if suffix: + c = c[: -len(suffix[0])] + return _validate_curie(c, prefixes) + + +def _validate_column(column, column_name, df_name, schema_def): + """Given a schema definition and the column of a dataframe, verify that the column satifies + the schema. + """ + + errors = [] + + if schema_def.get("unique"): + if column.nunique() != len(column): + errors.append(f"Column {column_name} in dataframe {df_name} is not unique.") + + if "nullable" in schema_def and not schema_def["nullable"]: + if any(_is_null(v) for v in column): + errors.append(f"Column {column_name} in dataframe {df_name} contains empty values.") + + if schema_def.get("type") == "human-readable string": + non_readables = [v for v in column if not _validate_human_readable_string(v)] + if non_readables: + errors.append( + f"Column {column_name} in dataframe {df_name} contains non-human-readable " + f"values like {non_readables[0]}" + ) + + if schema_def.get("type") in ("curie", "suffixed curie"): + validation_func = _validate_curie if schema_def.get("type") == "curie" else _validate_suffixed_curie + non_valid_curies = [v for v in column if not validation_func(v, schema_def.get("prefixes"))] + if non_valid_curies: + errors.append( + f"Column {column_name} in dataframe {df_name} contains invalid ontology values like " + f"{non_valid_curies[0]}." + ) + if "prefixes" in schema_def: + errors[-1] += f" Values must be curies from one of these ontologies {schema_def['prefixes']}." + + if "enum" in schema_def: + bad_enums = [v for v in column if v not in schema_def["enum"]] + if bad_enums: + errors.append( + f"Column {column_name} in dataframe {df_name} contains unpermitted values like " + f"{bad_enums[0]}. Values must be one of {schema_def['enum']}." + ) + + return errors + + +def _validate_dict(dict_, dict_name, schema_def): + """Given a schema definition and dict, verify that the dict satifies the schema.""" + + errors = [] + + for key in schema_def.get("keys", []): + if key not in dict_: + errors.append(f"{dict_name} is missing key {key}.") + elif schema_def["keys"][key]: + if schema_def["keys"][key]["type"] == "stringified list of dicts": + if not _validate_stringified_list_of_dicts(dict_[key]): + errors.append( + f"Key {key} in {dict_name} should be a JSON-encoded list of dicts, but it is {dict_[key]}" + ) + elif schema_def["keys"][key]["type"] == "dict": + errors.extend(_validate_dict(dict_[key], key, schema_def["keys"][key])) + elif schema_def["keys"][key]["type"] == "curie": + if not _validate_curie(dict_[key], schema_def["keys"][key]["prefixes"]): + errors.append(f"Key {key} in {dict_name} contains invalid ontology value.") + if "nullable" in schema_def["keys"][key] and not schema_def["keys"][key]["nullable"]: + if _is_null(dict_[key]): + errors.append(f"Key {key} in dict {dict_name} is an empty value.") + + return errors + + +def _validate_dataframe(df, df_name, schema_def): + """Given a dataframe and schema definition, verify that the dataframe follows the schema.""" + + errors = [] + + if "index" in schema_def: + errors.extend(_validate_column(df.index, "index", df_name, schema_def["index"])) + + for column in schema_def.get("columns", []): + if column not in df.columns: + errors.append(f"Dataframe {df_name} is missing column {column}.") + else: + errors.extend(_validate_column(df[column], column, df_name, schema_def["columns"][column])) + + return errors + + +def get_schema_definition(version): + """Look up and read a schema definition based on a version number like "1.0.0".""" + + path = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "schema_definitions", version.replace(".", "_") + ".yaml" + ) + + if not os.path.isfile(path): + raise ValueError(f"No definition for version {version} found.") + + return yaml.load(open(path), Loader=yaml.FullLoader) + + +def deep_check(adata, schema_def): + """Perform a "deep" check of the AnnData object using the schema definition. + + This checks all the columns and unstructured metadata rather than just the version. + + Returns a list of error messages. If that list is empty, the object passed validation. + """ + + errors = [] + + for component, component_def in schema_def["components"].items(): + if component_def["type"] == "dataframe": + errors.extend(_validate_dataframe(getattr(adata, component), component, component_def)) + elif component_def["type"] == "dict": + errors.extend(_validate_dict(getattr(adata, component), component, component_def)) + else: + raise ValueError(f"Unexpected component type {component['type']}") + + return errors + + +def validate_adata(adata, shallow): + """Validate an AnnData object. If shallow, just check that the required version information is + present. + """ + + # Does it have the version information written into uns? + if "version" not in adata.uns_keys() or "corpora_schema_version" not in adata.uns["version"]: + print("AnnData file is missing corpora version information") + return False + + # We can stop here if it's a "shallow" check, that is, if we're just + # checking that version is present. + if shallow: + return True + + schema_def = get_schema_definition(adata.uns["version"]["corpora_schema_version"]) + + errors = deep_check(adata, schema_def) + + for error in errors: + print(error) + + return not errors + + +def validate(h5ad_path, shallow=False): + """Entry point for validation.""" + + try: + import scanpy + except ImportError: + raise ImportError("scanpy must be installed for cellxgene schema") + + try: + adata = scanpy.read_h5ad(h5ad_path, backed="r") + except (OSError, TypeError): + print(f"Unable to open {h5ad_path} with scanpy.") + sys.exit(1) + + if not validate_adata(adata, shallow): + sys.exit(1) diff --git a/local_server/data_anndata/__init__.py b/local_server/data_anndata/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/data_anndata/anndata_adaptor.py b/local_server/data_anndata/anndata_adaptor.py new file mode 100644 index 00000000..12f468c7 --- /dev/null +++ b/local_server/data_anndata/anndata_adaptor.py @@ -0,0 +1,369 @@ +import warnings +from datetime import datetime + +import anndata +import numpy as np +from packaging import version +from pandas.core.dtypes.dtypes import CategoricalDtype +from scipy import sparse +from server_timing import Timing as ServerTiming + +import local_server.compute.diffexp_generic as diffexp_generic +from local_server.common.colors import convert_anndata_category_colors_to_cxg_category_colors +from local_server.common.constants import Axis, MAX_LAYOUTS +from local_server.common.corpora import corpora_get_props_from_anndata +from local_server.common.errors import PrepareError, DatasetAccessError, FilterError +from local_server.common.utils.type_conversion_utils import get_schema_type_hint_of_array +from local_server.compute.scanpy import scanpy_umap +from local_server.data_common.data_adaptor import DataAdaptor +from local_server.data_common.fbs.matrix import encode_matrix_fbs + +anndata_version = version.parse(str(anndata.__version__)).release + + +def anndata_version_is_pre_070(): + major = anndata_version[0] + minor = anndata_version[1] if len(anndata_version) > 1 else 0 + return major == 0 and minor < 7 + + +class AnndataAdaptor(DataAdaptor): + def __init__(self, data_locator, app_config=None, dataset_config=None): + super().__init__(data_locator, app_config, dataset_config) + self.data = None + self._load_data(data_locator) + self._validate_and_initialize() + + def cleanup(self): + pass + + @staticmethod + def pre_load_validation(data_locator): + if data_locator.islocal(): + # if data locator is local, apply file system conventions and other "cheap" + # validation checks. If a URI, defer until we actually fetch the data and + # try to read it. Many of these tests don't make sense for URIs (eg, extension- + # based typing). + if not data_locator.exists(): + raise DatasetAccessError("does not exist") + if not data_locator.isfile(): + raise DatasetAccessError("is not a file") + + @staticmethod + def file_size(data_locator): + return data_locator.size() if data_locator.islocal() else 0 + + @staticmethod + def open(data_locator, app_config, dataset_config=None): + return AnndataAdaptor(data_locator, app_config, dataset_config) + + def get_corpora_props(self): + return corpora_get_props_from_anndata(self.data) + + def get_name(self): + return "cellxgene anndata adaptor version" + + def get_library_versions(self): + return dict(anndata=str(anndata.__version__)) + + @staticmethod + def _create_unique_column_name(df, col_name_prefix): + """ given the columns of a dataframe, and a name prefix, return a column name which + does not exist in the dataframe, AND which is prefixed by `prefix` + + The approach is to append a numeric suffix, starting at zero and increasing by + one, until an unused name is found (eg, prefix_0, prefix_1, ...). + """ + suffix = 0 + while f"{col_name_prefix}{suffix}" in df: + suffix += 1 + return f"{col_name_prefix}{suffix}" + + def _alias_annotation_names(self): + """ + The front-end relies on the existance of a unique, human-readable + index for obs & var (eg, var is typically gene name, obs the cell name). + The user can specify these via the --obs-names and --var-names config. + If they are not specified, use the existing index to create them, giving + the resulting column a unique name (eg, "name"). + + In both cases, enforce that the result is unique, and communicate the + index column name to the front-end via the obs_names and var_names config + (which is incorporated into the schema). + """ + self.original_obs_index = self.data.obs.index + + for (ax_name, var_name) in ((Axis.OBS, "obs"), (Axis.VAR, "var")): + config_name = f"single_dataset__{var_name}_names" + parameter_name = f"{var_name}_names" + name = getattr(self.server_config, config_name) + df_axis = getattr(self.data, str(ax_name)) + if name is None: + # Default: create unique names from index + if not df_axis.index.is_unique: + raise KeyError( + f"Values in {ax_name}.index must be unique. " + "Please prepare data to contain unique index values, or specify an " + "alternative with --{ax_name}-name." + ) + name = self._create_unique_column_name(df_axis.columns, "name_") + self.parameters[parameter_name] = name + # reset index to simple range; alias name to point at the + # previously specified index. + df_axis.rename_axis(name, inplace=True) + df_axis.reset_index(inplace=True) + elif name in df_axis.columns: + # User has specified alternative column for unique names, and it exists + if not df_axis[name].is_unique: + raise KeyError( + f"Values in {ax_name}.{name} must be unique. " "Please prepare data to contain unique values." + ) + df_axis.reset_index(drop=True, inplace=True) + self.parameters[parameter_name] = name + else: + # user specified a non-existent column name + raise KeyError(f"Annotation name {name}, specified in --{ax_name}-name does not exist.") + + def _create_schema(self): + self.schema = { + "dataframe": {"nObs": self.cell_count, "nVar": self.gene_count, "type": str(self.data.X.dtype)}, + "annotations": { + "obs": {"index": self.parameters.get("obs_names"), "columns": []}, + "var": {"index": self.parameters.get("var_names"), "columns": []}, + }, + "layout": {"obs": []}, + } + for ax in Axis: + curr_axis = getattr(self.data, str(ax)) + for ann in curr_axis: + ann_schema = {"name": ann, "writable": False} + ann_schema.update(get_schema_type_hint_of_array(curr_axis[ann])) + self.schema["annotations"][ax]["columns"].append(ann_schema) + + for layout in self.get_embedding_names(): + layout_schema = {"name": layout, "type": "float32", "dims": [f"{layout}_0", f"{layout}_1"]} + self.schema["layout"]["obs"].append(layout_schema) + + def get_schema(self): + return self.schema + + def _load_data(self, data_locator): + # as of AnnData 0.6.19, backed mode performs initial load fast, but at the + # cost of significantly slower access to X data. + try: + # there is no guarantee data_locator indicates a local file. The AnnData + # API will only consume local file objects. If we get a non-local object, + # make a copy in tmp, and delete it after we load into memory. + with data_locator.local_handle() as lh: + # as of AnnData 0.6.19, backed mode performs initial load fast, but at the + # cost of significantly slower access to X data. + backed = "r" if self.server_config.adaptor__anndata_adaptor__backed else None + self.data = anndata.read_h5ad(lh, backed=backed) + + except ValueError: + raise DatasetAccessError( + "File must be in the .h5ad format. Please read " + "https://github.com/theislab/scanpy_usage/blob/master/170505_seurat/info_h5ad.md to " + "learn more about this format. You may be able to convert your file into this format " + "using `cellxgene prepare`, please run `cellxgene prepare --help` for more " + "information." + ) + except MemoryError: + raise DatasetAccessError("Out of memory - file is too large for available memory.") + except Exception: + raise DatasetAccessError( + "File not found or is inaccessible. File must be an .h5ad object. " + "Please check your input and try again." + ) + + def _validate_and_initialize(self): + if anndata_version_is_pre_070(): + warnings.warn( + "Use of anndata versions older than 0.7 will have serious issues. Please update to at " + "least anndata 0.7 or later." + ) + + # var and obs column names must be unique + if not self.data.obs.columns.is_unique or not self.data.var.columns.is_unique: + raise KeyError("All annotation column names must be unique.") + + self._alias_annotation_names() + self._validate_data_types() + self.cell_count = self.data.shape[0] + self.gene_count = self.data.shape[1] + self._create_schema() + + # heuristic + n_values = self.data.shape[0] * self.data.shape[1] + if (n_values > 1e8 and self.server_config.adaptor__anndata_adaptor__backed is True) or (n_values > 5e8): + self.parameters.update({"diffexp_may_be_slow": True}) + + def _is_valid_layout(self, arr): + """ return True if this layout data is a valid array for front-end presentation: + * ndarray, dtype float/int/uint + * with shape (n_obs, >= 2) + * with all values finite or NaN (no +Inf or -Inf) + """ + is_valid = type(arr) == np.ndarray and arr.dtype.kind in "fiu" + is_valid = is_valid and arr.shape[0] == self.data.n_obs and arr.shape[1] >= 2 + is_valid = is_valid and not np.any(np.isinf(arr)) and not np.all(np.isnan(arr)) + return is_valid + + def _validate_data_types(self): + # The backed API does not support interrogation of the underlying sparsity or sparse matrix type + # Fake it by asking for a small subarray and testing it. NOTE: if the user has ignored our + # anndata <= 0.7 warning, opted for the --backed option, and specified a large, sparse dataset, + # this "small" indexing request will load the entire X array. This is due to a bug in anndata<=0.7 + # which will load the entire X matrix to fullfill any slicing request if X is sparse. See + # user warning in _load_data(). + X0 = self.data.X[0, 0:1] + if sparse.isspmatrix(X0) and not sparse.isspmatrix_csc(X0): + warnings.warn( + "Anndata data matrix is sparse, but not a CSC (columnar) matrix. " + "Performance may be improved by using CSC." + ) + if self.data.X.dtype != "float32": + warnings.warn( + f"Anndata data matrix is in {self.data.X.dtype} format not float32. " f"Precision may be truncated." + ) + for ax in Axis: + curr_axis = getattr(self.data, str(ax)) + for ann in curr_axis: + datatype = curr_axis[ann].dtype + downcast_map = { + "int64": "int32", + "uint32": "int32", + "uint64": "int32", + "float64": "float32", + } + if datatype in downcast_map: + warnings.warn( + f"Anndata annotation {ax}:{ann} is in unsupported format: {datatype}. " + f"Data will be downcast to {downcast_map[datatype]}." + ) + if isinstance(datatype, CategoricalDtype): + category_num = len(curr_axis[ann].dtype.categories) + if category_num > 500 and category_num > self.dataset_config.presentation__max_categories: + warnings.warn( + f"{str(ax).title()} annotation '{ann}' has {category_num} categories, this may be " + f"cumbersome or slow to display. We recommend setting the " + f"--max-category-items option to 500, this will hide categorical " + f"annotations with more than 500 categories in the UI" + ) + + def annotation_to_fbs_matrix(self, axis, fields=None, labels=None): + if axis == Axis.OBS: + if labels is not None and not labels.empty: + df = self.data.obs.join(labels, self.parameters.get("obs_names")) + else: + df = self.data.obs + else: + df = self.data.var + + if fields is not None and len(fields) > 0: + df = df[fields] + return encode_matrix_fbs(df, col_idx=df.columns) + + def get_embedding_names(self): + """ + Return pre-computed embeddings. + + function: + a) generate list of default layouts + b) validate layouts are legal. remove/warn on any that are not + c) cap total list of layouts at global const MAX_LAYOUTS + """ + # load default layouts from the data. + layouts = self.dataset_config.embeddings__names + + if layouts is None or len(layouts) == 0: + layouts = [key[2:] for key in self.data.obsm_keys() if type(key) == str and key.startswith("X_")] + + # remove invalid layouts + valid_layouts = [] + obsm_keys = self.data.obsm_keys() + for layout in layouts: + layout_name = f"X_{layout}" + if layout_name not in obsm_keys: + warnings.warn(f"Ignoring unknown layout name: {layout}.") + elif not self._is_valid_layout(self.data.obsm[layout_name]): + warnings.warn(f"Ignoring layout due to malformed shape or data type: {layout}") + else: + valid_layouts.append(layout) + + if len(valid_layouts) == 0: + raise PrepareError("No valid layout data.") + + # cap layouts to MAX_LAYOUTS + return valid_layouts[0:MAX_LAYOUTS] + + def get_embedding_array(self, ename, dims=2): + full_embedding = self.data.obsm[f"X_{ename}"] + return full_embedding[:, 0:dims] + + def compute_embedding(self, method, obsFilter): + if Axis.VAR in obsFilter: + raise FilterError("Observation filters may not contain variable conditions") + if method != "umap": + raise NotImplementedError(f"re-embedding method {method} is not available.") + try: + shape = self.get_shape() + obs_mask = self._axis_filter_to_mask(Axis.OBS, obsFilter["obs"], shape[0]) + except (KeyError, IndexError): + raise FilterError("Error parsing filter") + with ServerTiming.time("layout.compute"): + X_umap = scanpy_umap(self.data, obs_mask) + + # Server picks reemedding name, which must not collide with any other + # embedding name generated by this backend. + name = f"reembed:{method}_{datetime.now().isoformat(timespec='milliseconds')}" + dims = [f"{name}_0", f"{name}_1"] + layout_schema = {"name": name, "type": "float32", "dims": dims} + self.schema["layout"]["obs"].append(layout_schema) + self.data.obsm[f"X_{name}"] = X_umap + return layout_schema + + def compute_diffexp_ttest(self, maskA, maskB, top_n=None, lfc_cutoff=None): + if top_n is None: + top_n = self.dataset_config.diffexp__top_n + if lfc_cutoff is None: + lfc_cutoff = self.dataset_config.diffexp__lfc_cutoff + return diffexp_generic.diffexp_ttest(self, maskA, maskB, top_n, lfc_cutoff) + + def get_colors(self): + return convert_anndata_category_colors_to_cxg_category_colors(self.data) + + def get_X_array(self, obs_mask=None, var_mask=None): + if obs_mask is None: + obs_mask = slice(None) + if var_mask is None: + var_mask = slice(None) + X = self.data.X[obs_mask, var_mask] + return X + + def get_shape(self): + return self.data.shape + + def query_var_array(self, term_name): + return getattr(self.data.var, term_name) + + def query_obs_array(self, term_name): + return getattr(self.data.obs, term_name) + + def get_obs_index(self): + name = self.server_config.single_dataset__obs_names + if name is None: + return self.original_obs_index + else: + return self.data.obs[name] + + def get_obs_columns(self): + return self.data.obs.columns + + def get_obs_keys(self): + # return list of keys + return self.data.obs.keys().to_list() + + def get_var_keys(self): + # return list of keys + return self.data.var.keys().to_list() diff --git a/local_server/data_common/__init__.py b/local_server/data_common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/data_common/data_adaptor.py b/local_server/data_common/data_adaptor.py new file mode 100644 index 00000000..c9f6d079 --- /dev/null +++ b/local_server/data_common/data_adaptor.py @@ -0,0 +1,391 @@ +from abc import ABCMeta, abstractmethod +from os.path import basename, splitext + +import numpy as np +import pandas as pd +from server_timing import Timing as ServerTiming + +from local_server.common.config.app_config import AppConfig +from local_server.common.constants import Axis +from local_server.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError +from local_server.common.utils.utils import jsonify_numpy +from local_server.data_common.fbs.matrix import encode_matrix_fbs + + +class DataAdaptor(metaclass=ABCMeta): + """Base class for loading and accessing matrix data""" + + def __init__(self, data_locator, app_config, dataset_config=None): + if type(app_config) != AppConfig: + raise TypeError("config expected to be of type AppConfig") + + # location to the dataset + self.data_locator = data_locator + + # config is the application configuration + self.app_config = app_config + self.server_config = self.app_config.server_config + self.dataset_config = dataset_config or app_config.dataset_config + + # parameters set by this data adaptor based on the data. + self.parameters = {} + + @staticmethod + @abstractmethod + def pre_load_validation(data_locator): + pass + + @staticmethod + @abstractmethod + def open(data_locator, app_config, dataset_config): + pass + + @staticmethod + @abstractmethod + def file_size(data_locator): + pass + + @abstractmethod + def get_name(self): + """return a string name for this data adaptor""" + pass + + @abstractmethod + def get_library_versions(self): + """return a dictionary of library name to library versions""" + pass + + @abstractmethod + def get_embedding_names(self): + """return a list of pre-computed embedding names""" + pass + + @abstractmethod + def get_embedding_array(self, ename, dims=2): + """return an numpy array for the given pre-computed embedding name.""" + pass + + @abstractmethod + def compute_embedding(self, method, filter): + """compute a new embedding on the specified obs subset, and return the embedding schema. """ + pass + + @abstractmethod + def get_X_array(self, obs_mask=None, var_mask=None): + """return the X array, possibly filtered by obs_mask or var_mask. + the return type is either ndarray or scipy.sparse.spmatrix.""" + pass + + @abstractmethod + def get_shape(self): + pass + + @abstractmethod + def query_var_array(self, term_var): + pass + + @abstractmethod + def query_obs_array(self, term_var): + pass + + @abstractmethod + def get_colors(self): + pass + + @abstractmethod + def get_obs_index(self): + pass + + @abstractmethod + def get_obs_columns(self): + pass + + @abstractmethod + def get_obs_keys(self): + # return list of keys + pass + + @abstractmethod + def get_var_keys(self): + # return list of keys + pass + + @abstractmethod + def cleanup(self): + pass + + def get_data_locator(self): + return self.data_locator + + def get_location(self): + return self.data_locator.uri_or_path + + def get_about(self): + return None + + def get_title(self): + # default to file name + location = self.get_location() + if location.endswith("/"): + location = location[:-1] + return splitext(basename(location))[0] + + def get_corpora_props(self): + return None + + @abstractmethod + def get_schema(self): + """ + Return current schema + """ + pass + + @abstractmethod + def annotation_to_fbs_matrix(self, axis, field=None, uid=None): + """ + Gets annotation value for each observation + :param axis: string obs or var + :param fields: list of keys for annotation to return, returns all annotation values if not set. + :return: flatbuffer: in fbs/matrix.fbs encoding + """ + pass + + def update_parameters(self, parameters): + parameters.update(self.parameters) + + def _index_filter_to_mask(self, filter, count): + mask = np.zeros((count,), dtype=np.bool) + for i in filter: + if type(i) == list: + mask[i[0] : i[1]] = True + else: + mask[i] = True + return mask + + def _axis_filter_to_mask(self, axis, filter, count): + mask = np.ones((count,), dtype=np.bool) + if "index" in filter: + mask = np.logical_and(mask, self._index_filter_to_mask(filter["index"], count)) + if "annotation_value" in filter: + mask = np.logical_and(mask, self._annotation_filter_to_mask(axis, filter["annotation_value"], count)) + + return mask + + def _annotation_filter_to_mask(self, axis, filter, count): + mask = np.ones((count,), dtype=np.bool) + for v in filter: + name = v["name"] + if axis == Axis.VAR: + anno_data = self.query_var_array(name) + elif axis == Axis.OBS: + anno_data = self.query_obs_array(name) + + if anno_data.dtype.name in ["boolean", "category", "object"]: + values = v.get("values", []) + key_idx = np.in1d(anno_data, values) + mask = np.logical_and(mask, key_idx) + + else: + min_ = v.get("min", None) + max_ = v.get("max", None) + if min_ is not None: + key_idx = (anno_data >= min_).ravel() + mask = np.logical_and(mask, key_idx) + if max_ is not None: + key_idx = (anno_data <= max_).ravel() + mask = np.logical_and(mask, key_idx) + + return mask + + def _filter_to_mask(self, filter): + """ + Return the filter as a row and column selection list. + No filter on a dimension means 'all' + """ + shape = self.get_shape() + var_selector = None + obs_selector = None + if filter is not None: + if Axis.OBS in filter: + obs_selector = self._axis_filter_to_mask(Axis.OBS, filter["obs"], shape[0]) + + if Axis.VAR in filter: + var_selector = self._axis_filter_to_mask(Axis.VAR, filter["var"], shape[1]) + + return (obs_selector, var_selector) + + def check_new_labels(self, labels_df): + """Check the new annotations labels, then set the labels_df index""" + if labels_df is None or labels_df.empty: + return + + labels_df.index = self.get_obs_index() + if labels_df.index.name is None: + labels_df.index.name = "index" + + # all labels must have a name, which must be unique and not used in obs column names + if not labels_df.columns.is_unique: + raise KeyError("All column names specified in user annotations must be unique.") + + # the label index must be unique, and must have same values the anndata obs index + if not labels_df.index.is_unique: + raise KeyError("All row index values specified in user annotations must be unique.") + + obs_columns = self.get_obs_columns() + + duplicate_columns = list(set(labels_df.columns) & set(obs_columns)) + if len(duplicate_columns) > 0: + raise KeyError( + "Labels file may not contain column names which overlap " f"with h5ad obs columns {duplicate_columns}" + ) + + # labels must have same count as obs annotations + shape = self.get_shape() + if labels_df.shape[0] != shape[0]: + raise ValueError("Labels file must have same number of rows as data file.") + + # This will convert a float column that contains integer data into an integer type. + # This case can occur when a user makes a copy of a category that originally contained integer data. + # The client always copies array data to floats, therefore the copy will contain floats instead of integers. + # float data is not allowed as a categorical type. + if any([np.issubdtype(coltype.type, np.floating) for coltype in labels_df.dtypes]): + labels_df = labels_df.convert_dtypes() + for col, dtype in zip(labels_df, labels_df.dtypes): + if isinstance(dtype, pd.Int32Dtype): + labels_df[col] = labels_df[col].astype("int32") + if isinstance(dtype, pd.Int64Dtype): + labels_df[col] = labels_df[col].astype("int64") + + if any([np.issubdtype(coltype.type, np.floating) for coltype in labels_df.dtypes]): + raise ValueError("Columns may not have floating point types") + + return labels_df + + def data_frame_to_fbs_matrix(self, filter, axis): + """ + Retrieves data 'X' and returns in a flatbuffer Matrix. + :param filter: filter: dictionary with filter params + :param axis: string obs or var + :return: flatbuffer Matrix + + Caveats: + * currently only supports access on VAR axis + * currently only supports filtering on VAR axis + """ + if axis != Axis.VAR: + raise ValueError("Only VAR dimension access is supported") + + try: + obs_selector, var_selector = self._filter_to_mask(filter) + except (KeyError, IndexError, TypeError, AttributeError): + raise FilterError("Error parsing filter") + + if obs_selector is not None: + raise FilterError("filtering on obs unsupported") + + num_columns = self.get_shape()[1] if var_selector is None else np.count_nonzero(var_selector) + if self.server_config.exceeds_limit("column_request_max", num_columns): + raise ExceedsLimitError("Requested dataframe columns exceed column request limit") + + X = self.get_X_array(obs_selector, var_selector) + col_idx = np.nonzero([] if var_selector is None else var_selector)[0] + return encode_matrix_fbs(X, col_idx=col_idx, row_idx=None) + + def diffexp_topN(self, obsFilterA, obsFilterB, top_n=None): + """ + Computes the top N differentially expressed variables between two observation sets. If mode + is "TOP_N", then stats for the top N + dataframes + contain a subset of variables, then statistics for all variables will be returned, otherwise + only the top N vars will be returned. + :param obsFilterA: filter: dictionary with filter params for first set of observations + :param obsFilterB: filter: dictionary with filter params for second set of observations + :param top_n: Limit results to top N (Top var mode only) + :return: top N genes and corresponding stats + """ + if Axis.VAR in obsFilterA or Axis.VAR in obsFilterB: + raise FilterError("Observation filters may not contain variable conditions") + try: + shape = self.get_shape() + obs_mask_A = self._axis_filter_to_mask(Axis.OBS, obsFilterA["obs"], shape[0]) + obs_mask_B = self._axis_filter_to_mask(Axis.OBS, obsFilterB["obs"], shape[0]) + except (KeyError, IndexError): + raise FilterError("Error parsing filter") + if top_n is None: + top_n = self.dataset_config.diffexp__top_n + + if self.server_config.exceeds_limit( + "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B) + ): + raise ExceedsLimitError("Diffexp request exceeds max cell count limit") + + result = self.compute_diffexp_ttest(obs_mask_A, obs_mask_B, top_n, self.dataset_config.diffexp__lfc_cutoff) + + try: + return jsonify_numpy(result) + except ValueError: + raise JSONEncodingValueError("Error encoding differential expression to JSON") + + @abstractmethod + def compute_diffexp_ttest(self, maskA, maskB, top_n, lfc_cutoff): + pass + + @staticmethod + def normalize_embedding(embedding): + """Normalize embedding layout to meet client assumptions. + Embedding is an ndarray, shape (n_obs, n)., where n is normally 2 + """ + + # scale isotropically + try: + min = np.nanmin(embedding, axis=0) + max = np.nanmax(embedding, axis=0) + except RuntimeError: + # indicates entire array was NaN, which should propagate + min = np.NaN + max = np.NaN + + scale = np.amax(max - min) + normalized_layout = (embedding - min) / scale + + # translate to center on both axis + translate = 0.5 - ((max - min) / scale / 2) + normalized_layout = normalized_layout + translate + + normalized_layout = normalized_layout.astype(dtype=np.float32) + return normalized_layout + + def layout_to_fbs_matrix(self, fields): + """ + return specified embeddings as a flatbuffer, using the cellxgene matrix fbs encoding. + + * returns only first two dimensions, with name {ename}_0 and {ename}_1, + where {ename} is the embedding name. + * client assumes each will be individually centered & scaled (isotropically) + to a [0, 1] range. + * does not support filtering + + """ + embeddings = self.get_embedding_names() if fields is None or len(fields) == 0 else fields + layout_data = [] + with ServerTiming.time("layout.query"): + for ename in embeddings: + embedding = self.get_embedding_array(ename, 2) + normalized_layout = DataAdaptor.normalize_embedding(embedding) + layout_data.append(pd.DataFrame(normalized_layout, columns=[f"{ename}_0", f"{ename}_1"])) + + with ServerTiming.time("layout.encode"): + if layout_data: + df = pd.concat(layout_data, axis=1, copy=False) + else: + df = pd.DataFrame() + fbs = encode_matrix_fbs(df, col_idx=df.columns, row_idx=None) + + return fbs + + def get_last_mod_time(self): + try: + lastmod = self.get_data_locator().lastmodtime() + except RuntimeError: + lastmod = None + return lastmod diff --git a/local_server/data_common/fbs/NetEncoding/Column.py b/local_server/data_common/fbs/NetEncoding/Column.py new file mode 100644 index 00000000..89b786d6 --- /dev/null +++ b/local_server/data_common/fbs/NetEncoding/Column.py @@ -0,0 +1,41 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# namespace: NetEncoding + +import flatbuffers + +class Column(object): + __slots__ = ['_tab'] + + @classmethod + def GetRootAsColumn(cls, buf, offset): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) + x = Column() + x.Init(buf, n + offset) + return x + + # Column + def Init(self, buf, pos): + self._tab = flatbuffers.table.Table(buf, pos) + + # Column + def UType(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos) + return 0 + + # Column + def U(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) + if o != 0: + from flatbuffers.table import Table + obj = Table(bytearray(), 0) + self._tab.Union(obj, o) + return obj + return None + +def ColumnStart(builder): builder.StartObject(2) +def ColumnAddUType(builder, uType): builder.PrependUint8Slot(0, uType, 0) +def ColumnAddU(builder, u): builder.PrependUOffsetTRelativeSlot(1, flatbuffers.number_types.UOffsetTFlags.py_type(u), 0) +def ColumnEnd(builder): return builder.EndObject() diff --git a/local_server/data_common/fbs/NetEncoding/Float32Array.py b/local_server/data_common/fbs/NetEncoding/Float32Array.py new file mode 100644 index 00000000..1acc426c --- /dev/null +++ b/local_server/data_common/fbs/NetEncoding/Float32Array.py @@ -0,0 +1,46 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# namespace: NetEncoding + +import flatbuffers + +class Float32Array(object): + __slots__ = ['_tab'] + + @classmethod + def GetRootAsFloat32Array(cls, buf, offset): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) + x = Float32Array() + x.Init(buf, n + offset) + return x + + # Float32Array + def Init(self, buf, pos): + self._tab = flatbuffers.table.Table(buf, pos) + + # Float32Array + def Data(self, j): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + a = self._tab.Vector(o) + return self._tab.Get(flatbuffers.number_types.Float32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4)) + return 0 + + # Float32Array + def DataAsNumpy(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Float32Flags, o) + return 0 + + # Float32Array + def DataLength(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + return self._tab.VectorLen(o) + return 0 + +def Float32ArrayStart(builder): builder.StartObject(1) +def Float32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0) +def Float32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4) +def Float32ArrayEnd(builder): return builder.EndObject() diff --git a/local_server/data_common/fbs/NetEncoding/Float64Array.py b/local_server/data_common/fbs/NetEncoding/Float64Array.py new file mode 100644 index 00000000..2ec343a2 --- /dev/null +++ b/local_server/data_common/fbs/NetEncoding/Float64Array.py @@ -0,0 +1,46 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# namespace: NetEncoding + +import flatbuffers + +class Float64Array(object): + __slots__ = ['_tab'] + + @classmethod + def GetRootAsFloat64Array(cls, buf, offset): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) + x = Float64Array() + x.Init(buf, n + offset) + return x + + # Float64Array + def Init(self, buf, pos): + self._tab = flatbuffers.table.Table(buf, pos) + + # Float64Array + def Data(self, j): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + a = self._tab.Vector(o) + return self._tab.Get(flatbuffers.number_types.Float64Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 8)) + return 0 + + # Float64Array + def DataAsNumpy(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Float64Flags, o) + return 0 + + # Float64Array + def DataLength(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + return self._tab.VectorLen(o) + return 0 + +def Float64ArrayStart(builder): builder.StartObject(1) +def Float64ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0) +def Float64ArrayStartDataVector(builder, numElems): return builder.StartVector(8, numElems, 8) +def Float64ArrayEnd(builder): return builder.EndObject() diff --git a/local_server/data_common/fbs/NetEncoding/Int32Array.py b/local_server/data_common/fbs/NetEncoding/Int32Array.py new file mode 100644 index 00000000..f3f8156f --- /dev/null +++ b/local_server/data_common/fbs/NetEncoding/Int32Array.py @@ -0,0 +1,46 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# namespace: NetEncoding + +import flatbuffers + +class Int32Array(object): + __slots__ = ['_tab'] + + @classmethod + def GetRootAsInt32Array(cls, buf, offset): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) + x = Int32Array() + x.Init(buf, n + offset) + return x + + # Int32Array + def Init(self, buf, pos): + self._tab = flatbuffers.table.Table(buf, pos) + + # Int32Array + def Data(self, j): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + a = self._tab.Vector(o) + return self._tab.Get(flatbuffers.number_types.Int32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4)) + return 0 + + # Int32Array + def DataAsNumpy(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Int32Flags, o) + return 0 + + # Int32Array + def DataLength(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + return self._tab.VectorLen(o) + return 0 + +def Int32ArrayStart(builder): builder.StartObject(1) +def Int32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0) +def Int32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4) +def Int32ArrayEnd(builder): return builder.EndObject() diff --git a/local_server/data_common/fbs/NetEncoding/JSONEncodedArray.py b/local_server/data_common/fbs/NetEncoding/JSONEncodedArray.py new file mode 100644 index 00000000..366ebadd --- /dev/null +++ b/local_server/data_common/fbs/NetEncoding/JSONEncodedArray.py @@ -0,0 +1,46 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# namespace: NetEncoding + +import flatbuffers + +class JSONEncodedArray(object): + __slots__ = ['_tab'] + + @classmethod + def GetRootAsJSONEncodedArray(cls, buf, offset): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) + x = JSONEncodedArray() + x.Init(buf, n + offset) + return x + + # JSONEncodedArray + def Init(self, buf, pos): + self._tab = flatbuffers.table.Table(buf, pos) + + # JSONEncodedArray + def Data(self, j): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + a = self._tab.Vector(o) + return self._tab.Get(flatbuffers.number_types.Uint8Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 1)) + return 0 + + # JSONEncodedArray + def DataAsNumpy(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Uint8Flags, o) + return 0 + + # JSONEncodedArray + def DataLength(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + return self._tab.VectorLen(o) + return 0 + +def JSONEncodedArrayStart(builder): builder.StartObject(1) +def JSONEncodedArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0) +def JSONEncodedArrayStartDataVector(builder, numElems): return builder.StartVector(1, numElems, 1) +def JSONEncodedArrayEnd(builder): return builder.EndObject() diff --git a/local_server/data_common/fbs/NetEncoding/Matrix.py b/local_server/data_common/fbs/NetEncoding/Matrix.py new file mode 100644 index 00000000..8c9d0eb8 --- /dev/null +++ b/local_server/data_common/fbs/NetEncoding/Matrix.py @@ -0,0 +1,98 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# namespace: NetEncoding + +import flatbuffers + +class Matrix(object): + __slots__ = ['_tab'] + + @classmethod + def GetRootAsMatrix(cls, buf, offset): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) + x = Matrix() + x.Init(buf, n + offset) + return x + + # Matrix + def Init(self, buf, pos): + self._tab = flatbuffers.table.Table(buf, pos) + + # Matrix + def NRows(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + return self._tab.Get(flatbuffers.number_types.Uint32Flags, o + self._tab.Pos) + return 0 + + # Matrix + def NCols(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) + if o != 0: + return self._tab.Get(flatbuffers.number_types.Uint32Flags, o + self._tab.Pos) + return 0 + + # Matrix + def Columns(self, j): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8)) + if o != 0: + x = self._tab.Vector(o) + x += flatbuffers.number_types.UOffsetTFlags.py_type(j) * 4 + x = self._tab.Indirect(x) + from .Column import Column + obj = Column() + obj.Init(self._tab.Bytes, x) + return obj + return None + + # Matrix + def ColumnsLength(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8)) + if o != 0: + return self._tab.VectorLen(o) + return 0 + + # Matrix + def ColIndexType(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10)) + if o != 0: + return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos) + return 0 + + # Matrix + def ColIndex(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12)) + if o != 0: + from flatbuffers.table import Table + obj = Table(bytearray(), 0) + self._tab.Union(obj, o) + return obj + return None + + # Matrix + def RowIndexType(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14)) + if o != 0: + return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos) + return 0 + + # Matrix + def RowIndex(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(16)) + if o != 0: + from flatbuffers.table import Table + obj = Table(bytearray(), 0) + self._tab.Union(obj, o) + return obj + return None + +def MatrixStart(builder): builder.StartObject(7) +def MatrixAddNRows(builder, nRows): builder.PrependUint32Slot(0, nRows, 0) +def MatrixAddNCols(builder, nCols): builder.PrependUint32Slot(1, nCols, 0) +def MatrixAddColumns(builder, columns): builder.PrependUOffsetTRelativeSlot(2, flatbuffers.number_types.UOffsetTFlags.py_type(columns), 0) +def MatrixStartColumnsVector(builder, numElems): return builder.StartVector(4, numElems, 4) +def MatrixAddColIndexType(builder, colIndexType): builder.PrependUint8Slot(3, colIndexType, 0) +def MatrixAddColIndex(builder, colIndex): builder.PrependUOffsetTRelativeSlot(4, flatbuffers.number_types.UOffsetTFlags.py_type(colIndex), 0) +def MatrixAddRowIndexType(builder, rowIndexType): builder.PrependUint8Slot(5, rowIndexType, 0) +def MatrixAddRowIndex(builder, rowIndex): builder.PrependUOffsetTRelativeSlot(6, flatbuffers.number_types.UOffsetTFlags.py_type(rowIndex), 0) +def MatrixEnd(builder): return builder.EndObject() diff --git a/local_server/data_common/fbs/NetEncoding/TypedArray.py b/local_server/data_common/fbs/NetEncoding/TypedArray.py new file mode 100644 index 00000000..e36c4f1b --- /dev/null +++ b/local_server/data_common/fbs/NetEncoding/TypedArray.py @@ -0,0 +1,12 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# namespace: NetEncoding + +class TypedArray(object): + NONE = 0 + Float32Array = 1 + Int32Array = 2 + Uint32Array = 3 + Float64Array = 4 + JSONEncodedArray = 5 + diff --git a/local_server/data_common/fbs/NetEncoding/Uint32Array.py b/local_server/data_common/fbs/NetEncoding/Uint32Array.py new file mode 100644 index 00000000..01c7dfdd --- /dev/null +++ b/local_server/data_common/fbs/NetEncoding/Uint32Array.py @@ -0,0 +1,46 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# namespace: NetEncoding + +import flatbuffers + +class Uint32Array(object): + __slots__ = ['_tab'] + + @classmethod + def GetRootAsUint32Array(cls, buf, offset): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) + x = Uint32Array() + x.Init(buf, n + offset) + return x + + # Uint32Array + def Init(self, buf, pos): + self._tab = flatbuffers.table.Table(buf, pos) + + # Uint32Array + def Data(self, j): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + a = self._tab.Vector(o) + return self._tab.Get(flatbuffers.number_types.Uint32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4)) + return 0 + + # Uint32Array + def DataAsNumpy(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Uint32Flags, o) + return 0 + + # Uint32Array + def DataLength(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + return self._tab.VectorLen(o) + return 0 + +def Uint32ArrayStart(builder): builder.StartObject(1) +def Uint32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0) +def Uint32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4) +def Uint32ArrayEnd(builder): return builder.EndObject() diff --git a/local_server/data_common/fbs/NetEncoding/__init__.py b/local_server/data_common/fbs/NetEncoding/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/data_common/fbs/__init__.py b/local_server/data_common/fbs/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/data_common/fbs/matrix.py b/local_server/data_common/fbs/matrix.py new file mode 100644 index 00000000..14fa71ba --- /dev/null +++ b/local_server/data_common/fbs/matrix.py @@ -0,0 +1,248 @@ +import json + +import numpy as np +import pandas as pd +from flatbuffers import Builder +from scipy import sparse + +import local_server.data_common.fbs.NetEncoding.Column as Column +import local_server.data_common.fbs.NetEncoding.Float32Array as Float32Array +import local_server.data_common.fbs.NetEncoding.Float64Array as Float64Array +import local_server.data_common.fbs.NetEncoding.Int32Array as Int32Array +import local_server.data_common.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray +import local_server.data_common.fbs.NetEncoding.Matrix as Matrix +import local_server.data_common.fbs.NetEncoding.TypedArray as TypedArray +import local_server.data_common.fbs.NetEncoding.Uint32Array as Uint32Array + + +# Serialization helper +def serialize_column(builder, typed_arr): + """ Serialize NetEncoding.Column """ + + (u_type, u_value) = typed_arr + Column.ColumnStart(builder) + Column.ColumnAddUType(builder, u_type) + Column.ColumnAddU(builder, u_value) + return Column.ColumnEnd(builder) + + +# Serialization helper +def serialize_matrix(builder, n_rows, n_cols, columns, col_idx): + """ Serialize NetEncoding.Matrix """ + + Matrix.MatrixStart(builder) + Matrix.MatrixAddNRows(builder, n_rows) + Matrix.MatrixAddNCols(builder, n_cols) + Matrix.MatrixAddColumns(builder, columns) + if col_idx is not None: + (u_type, u_val) = col_idx + Matrix.MatrixAddColIndexType(builder, u_type) + Matrix.MatrixAddColIndex(builder, u_val) + return Matrix.MatrixEnd(builder) + + +# Serialization helper +def serialize_typed_array(builder, source_array, encoding_info): + """ + Serialize any of the various typed arrays, eg, Float32Array. Specific means of serialization and type conversion + are provided by type_info. + """ + + arr = source_array + (array_type, as_type) = encoding_info(source_array) + + if isinstance(arr, pd.Index): + arr = arr.to_series() + + # convert to a simple ndarray + if as_type == "json": + as_json = arr.to_json(orient="records") + arr = np.array(bytearray(as_json, "utf-8")) + else: + if sparse.issparse(arr): + arr = arr.toarray() + elif isinstance(arr, pd.Series): + arr = arr.to_numpy() + if arr.dtype != as_type: + arr = arr.astype(as_type) + + # serialize the ndarray into a vector + if arr.ndim == 2: + if arr.shape[0] == 1: + arr = arr[0] + elif arr.shape[1] == 1: + arr = arr.T[0] + + vec = builder.CreateNumpyVector(arr) + + # serialize the typed array table + builder.StartObject(1) + builder.PrependUOffsetTRelativeSlot(0, vec, 0) + array_value = builder.EndObject() + return (array_type, array_value) + + +def column_encoding(arr): + column_encoding_type_map = { + # array protocol string: ( array_type, as_type ) + np.dtype(np.float64).str: (TypedArray.TypedArray.Float64Array, np.float64), + np.dtype(np.float32).str: (TypedArray.TypedArray.Float32Array, np.float32), + np.dtype(np.float16).str: (TypedArray.TypedArray.Float32Array, np.float32), + np.dtype(np.int8).str: (TypedArray.TypedArray.Int32Array, np.int32), + np.dtype(np.int16).str: (TypedArray.TypedArray.Int32Array, np.int32), + np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32), + np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32), + np.dtype(np.uint8).str: (TypedArray.TypedArray.Uint32Array, np.uint32), + np.dtype(np.uint16).str: (TypedArray.TypedArray.Uint32Array, np.uint32), + np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32), + np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32), + } + column_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, "json") + + return column_encoding_type_map.get(arr.dtype.str, column_encoding_default) + + +def index_encoding(arr): + index_encoding_type_map = { + # array protocol string: ( array_type, as_type ) + np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32), + np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32), + np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32), + np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32), + } + index_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, "json") + + return index_encoding_type_map.get(arr.dtype.str, index_encoding_default) + + +def guess_at_mem_needed(matrix): + (n_rows, n_cols) = matrix.shape + if isinstance(matrix, np.ndarray) or sparse.issparse(matrix): + guess = (n_rows * n_cols * matrix.dtype.itemsize) + 1024 + elif isinstance(matrix, pd.DataFrame): + # XXX TODO - DataFrame type estimate + guess = 1 + else: + guess = 1 + + # round up to nearest 1024 bytes + guess = (guess + 0x400) & (~0x3FF) + return guess + + +def encode_matrix_fbs(matrix, row_idx=None, col_idx=None): + """ + Given a 2D DataFrame, ndarray or sparse equivalent, create and return a Matrix flatbuffer. + + :param matrix: 2D DataFrame, ndarray or sparse equivalent + :param row_idx: index for row dimension, Index or ndarray + :param col_idx: index for col dimension, Index or ndarray + + NOTE: row indices are (currently) unsupported and must be None + """ + + if row_idx is not None: + raise ValueError("row indexing not supported for FBS Matrix") + if matrix.ndim != 2: + raise ValueError("FBS Matrix must be 2D") + + (n_rows, n_cols) = matrix.shape + + # estimate size needed, so we don't unnecessarily realloc. + builder = Builder(guess_at_mem_needed(matrix)) + + columns = [] + for cidx in range(n_cols - 1, -1, -1): + # serialize the typed array + col = matrix.iloc[:, cidx] if isinstance(matrix, pd.DataFrame) else matrix[:, cidx] + typed_arr = serialize_typed_array(builder, col, column_encoding) + + # serialize the Column union + columns.append(serialize_column(builder, typed_arr)) + + # Serialize Matrix.columns[] + Matrix.MatrixStartColumnsVector(builder, n_cols) + for c in columns: + builder.PrependUOffsetTRelative(c) + matrix_column_vec = builder.EndVector(n_cols) + + # serialize the colIndex if provided + cidx = None + if col_idx is not None: + cidx = serialize_typed_array(builder, col_idx, index_encoding) + + # Serialize Matrix + matrix = serialize_matrix(builder, n_rows, n_cols, matrix_column_vec, cidx) + + builder.Finish(matrix) + return builder.Output() + + +def deserialize_typed_array(tarr): + type_map = { + TypedArray.TypedArray.NONE: None, + TypedArray.TypedArray.Uint32Array: Uint32Array.Uint32Array, + TypedArray.TypedArray.Int32Array: Int32Array.Int32Array, + TypedArray.TypedArray.Float32Array: Float32Array.Float32Array, + TypedArray.TypedArray.Float64Array: Float64Array.Float64Array, + TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray, + } + (u_type, u) = tarr + if u_type is TypedArray.TypedArray.NONE: + return None + + TarType = type_map.get(u_type, None) + if TarType is None: + raise TypeError(f"FBS contains unknown data type: {u_type}") + + arr = TarType() + arr.Init(u.Bytes, u.Pos) + narr = arr.DataAsNumpy() + if u_type == TypedArray.TypedArray.JSONEncodedArray: + narr = json.loads(narr.tostring().decode("utf-8")) + return narr + + +def decode_matrix_fbs(fbs): + """ + Given an FBS-encoded Matrix, return a Pandas DataFrame the contains the data and indices. + """ + + matrix = Matrix.Matrix.GetRootAsMatrix(fbs, 0) + n_rows = matrix.NRows() + n_cols = matrix.NCols() + if n_rows == 0 or n_cols == 0: + return pd.DataFrame() + + if matrix.RowIndexType() is not TypedArray.TypedArray.NONE: + raise ValueError("row indexing not supported for FBS Matrix") + + columns_length = matrix.ColumnsLength() + + columns_index = deserialize_typed_array((matrix.ColIndexType(), matrix.ColIndex())) + if columns_index is None: + columns_index = range(0, n_cols) + + # sanity checks + if len(columns_index) != n_cols or columns_length != n_cols: + raise ValueError("FBS column count does not match number of columns in underlying matrix") + + columns_data = {} + columns_type = {} + for col_idx in range(0, columns_length): + col = matrix.Columns(col_idx) + tarr = (col.UType(), col.U()) + data = deserialize_typed_array(tarr) + columns_data[columns_index[col_idx]] = data + if len(data) != n_rows: + raise ValueError("FBS column length does not match number of rows") + if col.UType() is TypedArray.TypedArray.JSONEncodedArray: + columns_type[columns_index[col_idx]] = "category" + + df = pd.DataFrame.from_dict(data=columns_data).astype(columns_type, copy=False) + + # more sanity checks + if not df.columns.is_unique or len(df.columns) != n_cols: + raise KeyError("FBS column indices are not unique") + + return df diff --git a/local_server/data_common/matrix_loader.py b/local_server/data_common/matrix_loader.py new file mode 100644 index 00000000..065e2d91 --- /dev/null +++ b/local_server/data_common/matrix_loader.py @@ -0,0 +1,55 @@ +from enum import Enum +from local_server.common.errors import DatasetAccessError +from local_server.common.data_locator import DataLocator +from http import HTTPStatus + + +class MatrixDataType(Enum): + H5AD = "h5ad" + UNKNOWN = "unknown" + + +class MatrixDataLoader(object): + def __init__(self, location, matrix_data_type=None, app_config=None): + """ location can be a string or DataLocator """ + region_name = None if app_config is None else app_config.server_config.data_locator__s3__region_name + self.location = DataLocator(location, region_name=region_name) + if not self.location.exists(): + raise DatasetAccessError("Dataset does not exist.", HTTPStatus.NOT_FOUND) + + # matrix_data_type is an enum value of type MatrixDataType + self.matrix_data_type = matrix_data_type + # matrix_type is a DataAdaptor type, which corresonds to the matrix_data_type + self.matrix_type = None + + if matrix_data_type is None: + self.matrix_data_type = self.__matrix_data_type() + + if not self.__matrix_data_type_allowed(app_config): + raise DatasetAccessError("Dataset does not have an allowed type.") + + if self.matrix_data_type == MatrixDataType.H5AD: + from local_server.data_anndata.anndata_adaptor import AnndataAdaptor + + self.matrix_type = AnndataAdaptor + + def __matrix_data_type(self): + if self.location.path.endswith(".h5ad"): + return MatrixDataType.H5AD + else: + return MatrixDataType.UNKNOWN + + def __matrix_data_type_allowed(self, app_config): + return self.matrix_data_type != MatrixDataType.UNKNOWN + + def pre_load_validation(self): + if self.matrix_data_type == MatrixDataType.UNKNOWN: + raise DatasetAccessError("Dataset does not have a recognized type: .h5ad") + self.matrix_type.pre_load_validation(self.location) + + def file_size(self): + return self.matrix_type.file_size(self.location) + + def open(self, app_config, dataset_config=None): + # create and return a DataAdaptor object + return self.matrix_type.open(self.location, app_config, dataset_config) diff --git a/local_server/default_config.py b/local_server/default_config.py new file mode 100644 index 00000000..e722a8d5 --- /dev/null +++ b/local_server/default_config.py @@ -0,0 +1,135 @@ +import yaml + +default_config = """ +server: + app: + verbose: false + debug: false + host: localhost + port : null + open_browser: false + force_https: false + flask_secret_key: null + + authentication: + # The authentication types may be "none" or "session" + # none: No authentication support, features like user_annotations must not be enabled. + # session: A session based userid is automatically generated. (no params needed) + type: session + insecure_test_environment: false + + single_dataset: + # If datapath is set, then cellxgene with serve a single dataset located at datapath. + datapath: null + obs_names: null + var_names: null + about: null + title: null + + data_locator: + s3: + # s3 region name. + # if true, then the s3 location is automatically determined from the datapath or dataroot. + # if false/null, then do not set. + # if a string, then use that value (e.g. us-east-1). + region_name: true + + adaptor: + anndata_adaptor: + backed: false + + limits: + column_request_max: 32 + diffexp_cellcount_max: null + + +dataset: + app: + # Scripts can be a list of either file names (string) or dicts containing keys src, integrity and crossorigin. + # these will be injected into the index template as script tags with these attributes set. + scripts: [] + # Inline scripts are a list of file names, where the contents of the file will be injected into the index. + inline_scripts: [] + + # allow authentication support + authentication_enable: true + + presentation: + max_categories: 1000 + custom_colors: true + + user_annotations: + enable: true + type: local_file_csv + local_file_csv: + directory: null + file: null + ontology: + enable: false + obo_location: null + + embeddings: + names : [] + enable_reembedding: false + + diffexp: + enable: true + lfc_cutoff: 0.01 + top_n: 10 + +external: + # You can retrieve configuration parameters from this config file, the environment, + # the AWS secrets manager, or from the "cellxgene launch" command line arguments. + # They are applied in that order, meaning that if a parameter is defined in more + # than one location, the last one applied takes effect. + + # environment variables: + # This section describes how to map environment variables to configuration parameters. + # The format is a list defining an environment variable. + # Each entry in the list is a dictionary with three entries: + # name: the name of the environment variable + # path: the path within the cellxgene configuration to update. + # required: (default=False) a boolean. If true, then it is an error if the environment variable is not set. + + environment: + - name: CXG_SECRET_KEY + path: [server, app, flask_secret_key] + required: false + + # AWS Secrets Manager + # This section describes how to map aws secrets to configuration parameters. + # The format is the region for the secrets manager, then a list of secrets. + # each secret has a name, and a list of values. + # Each entry in the list of values is a dictionary with three entries: + # key: the key of the aws secret. + # path: the path within the cellxgene configuration to update. + # required: (default=False) a boolean. If true, then it is an error if the key does not exist in the secret. + # + # example: + # aws_secrets_manager: + # region: us-west-2 + # - name: my_first_secret + # values: + # - key: flask_secret_key + # path: [server, app, flask_secret_key] + # required: true + # - key: db_uri + # path: [dataset, user_annotations, db_uri] + # required: true + # - name: my_auth_secret + # values: + # - key: client_secret + # path: [server, authentication, client_secret] + # required: true + # - key: client_id + # path: [server, authentication, client_id] + # required: true + + aws_secrets_manager: + region: null + secrets: [] +""" + + +def get_default_config(): + return yaml.load(default_config, Loader=yaml.Loader) diff --git a/local_server/requirements-dev.txt b/local_server/requirements-dev.txt new file mode 100644 index 00000000..9f4fd724 --- /dev/null +++ b/local_server/requirements-dev.txt @@ -0,0 +1,10 @@ +Authlib>=0.14.3 +black +bumpversion>=0.5 +codecov>=2.0.15 +parameterized>=0.7.0 +psycopg2-binary>=2.8.5 +pytest>=3.6.3 +python-jose>=3.2.0 +twine>=1.12.1 +-r requirements.txt diff --git a/local_server/requirements-prepare.txt b/local_server/requirements-prepare.txt new file mode 100644 index 00000000..254f827b --- /dev/null +++ b/local_server/requirements-prepare.txt @@ -0,0 +1,2 @@ +python-igraph +louvain>=0.6 diff --git a/local_server/requirements.txt b/local_server/requirements.txt new file mode 100644 index 00000000..f60c5b13 --- /dev/null +++ b/local_server/requirements.txt @@ -0,0 +1,24 @@ +anndata>=0.7.0 +boto3>=1.12.18 +click>=7.1.2 +fastobo>=0.6.1 +Flask>=1.0.2 +Flask-Compress>=1.4.0 +Flask-Cors>=3.0.6 +Flask-RESTful>=0.3.6 +flask-server-timing>=0.1.2 +flask-talisman>=0.7.0 +flatbuffers>=1.11.0 +flatten-dict>=0.2.0 +fsspec>=0.4.4,<0.8.0 +gunicorn>=20.0.4 +h5py<3.0.0 # h5py>=3.0.0 had a breaking change; there is a fix in anndata>=0.7.5 +numpy>=1.15.0 +packaging>=20.0 +pandas>=1.0,!=1.1 # pandas 1.1 breaks tests, https://github.com/pandas-dev/pandas/issues/35446 +PyYAML>=5.3 +scipy>=1.0 +requests>=2.22.0 +s3fs==0.4.2 +scanpy==1.4.6 # Until we move to anndata 0.7.4 scanpy needs to be pinned here +umap-learn<0.5.0 # The pinned version scanpy is not compatible with latest umap-learn diff --git a/local_server/test/__init__.py b/local_server/test/__init__.py new file mode 100644 index 00000000..c18cafd0 --- /dev/null +++ b/local_server/test/__init__.py @@ -0,0 +1,166 @@ +import os +import random +import shutil +import string +import tempfile +import time +from contextlib import contextmanager +from os import path, popen +from subprocess import Popen + +import pandas as pd +import requests + +from local_server.common.annotations.local_file_csv import AnnotationsLocalFile +from local_server.common.config.app_config import AppConfig +from local_server.common.config import DEFAULT_SERVER_PORT +from local_server.common.data_locator import DataLocator +from local_server.common.utils.utils import find_available_port +from local_server.data_common.fbs.matrix import encode_matrix_fbs +from local_server.data_common.matrix_loader import MatrixDataLoader, MatrixDataType + +PROJECT_ROOT = popen("git rev-parse --show-toplevel").read().strip() +FIXTURES_ROOT = PROJECT_ROOT + "/local_server/test/fixtures" +H5AD_FIXTURE = FIXTURES_ROOT + "/pbmc3k-CSC-gz.h5ad" + + +def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False): + tmp_dir = tempfile.mkdtemp() + annotations_file = path.join(tmp_dir, "test_annotations.csv") + if annotations_fixture: + shutil.copyfile(f"{PROJECT_ROOT}/local_server/test/fixtures/pbmc3k-annotations.csv", annotations_file) + fname = { + MatrixDataType.H5AD: f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", + }[ext] + data_locator = DataLocator(fname) + config = AppConfig() + config.update_server_config( + app__flask_secret_key="secret", + single_dataset__obs_names=None, + single_dataset__var_names=None, + single_dataset__datapath=data_locator.path, + ) + config.update_dataset_config( + embeddings__names=["umap"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01, + ) + + config.complete_config() + data = MatrixDataLoader(data_locator.abspath()).open(config) + annotations = AnnotationsLocalFile(None, annotations_file) + return data, tmp_dir, annotations + + +def make_fbs(data): + df = pd.DataFrame(data) + return encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns) + + +def skip_if(condition, reason: str): + def decorator(f): + def wraps(self, *args, **kwargs): + if condition(self): + self.skipTest(reason) + else: + f(self, *args, **kwargs) + + return wraps + + return decorator + + +def app_config(data_locator, backed=False, extra_server_config={}, extra_dataset_config={}): + config = AppConfig() + config.update_server_config( + app__flask_secret_key="secret", + single_dataset__obs_names=None, + single_dataset__var_names=None, + adaptor__anndata_adaptor__backed=backed, + single_dataset__datapath=data_locator, + limits__diffexp_cellcount_max=None, + limits__column_request_max=None, + ) + config.update_dataset_config( + embeddings__names=["umap", "tsne", "pca"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01 + ) + config.update_server_config(**extra_server_config) + config.update_dataset_config(**extra_dataset_config) + config.complete_config() + return config + + +def random_string(n): + return "".join(random.choice(string.ascii_letters) for _ in range(n)) + + +def start_test_server(command_line_args=[], app_config=None, env=None): + """ + Command line arguments can be passed in, as well as an app_config. + This function is meant to be used like this, for example: + + with test_server(...) as server: + r = requests.get(f"{server}/...") + // check r + + where the server can be accessed within the context, and is terminated when + the context is exited. + The port is automatically set using find_available_port, unless passed in as a command line arg. + The verbose flag is automatically set to True. + If an app_config is provided, then this function writes a temporary + yaml config file, which this server will read and parse. + """ + + command = ["cellxgene", "--no-upgrade-check", "launch", "--verbose"] + if "-p" in command_line_args: + port = int(command_line_args[command_line_args.index("-p") + 1]) + elif "--port" in command_line_args: + port = int(command_line_args[command_line_args.index("--port") + 1]) + else: + start = random.randint(DEFAULT_SERVER_PORT, 2 ** 16 - 1) + port = int(os.environ.get("CXG_SERVER_PORT", start)) + port = find_available_port("localhost", port) + command += ["--port=%d" % port] + + command += command_line_args + + tempdir = None + if app_config: + tempdir = tempfile.TemporaryDirectory() + config_file = os.path.join(tempdir.name, "config.yaml") + app_config.write_config(config_file) + command.extend(["-c", config_file]) + + server = f"http://localhost:{port}" + ps = Popen(command, env=env) + + for _ in range(10): + try: + requests.get(f"{server}/health") + break + except requests.exceptions.ConnectionError: + time.sleep(1) + + if tempdir: + tempdir.cleanup() + + return ps, server + + +def stop_test_server(ps): + try: + ps.terminate() + except ProcessLookupError: + pass + + +@contextmanager +def test_server(command_line_args=[], app_config=None, env=None): + """A context to run the cellxgene server.""" + + ps, server = start_test_server(command_line_args, app_config, env) + try: + yield server + finally: + try: + stop_test_server(ps) + except ProcessLookupError: + pass diff --git a/local_server/test/fixtures/1e4dfec4-c0b2-46ad-a04e-ff3ffb3c0a8f.h5ad b/local_server/test/fixtures/1e4dfec4-c0b2-46ad-a04e-ff3ffb3c0a8f.h5ad new file mode 100644 index 00000000..eee29036 Binary files /dev/null and b/local_server/test/fixtures/1e4dfec4-c0b2-46ad-a04e-ff3ffb3c0a8f.h5ad differ diff --git a/local_server/test/fixtures/__init__.py b/local_server/test/fixtures/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/test/fixtures/a95c59b4-7f5d-4b80-ad53-a694834ca18b.h5ad b/local_server/test/fixtures/a95c59b4-7f5d-4b80-ad53-a694834ca18b.h5ad new file mode 100644 index 00000000..9d7ffcc6 Binary files /dev/null and b/local_server/test/fixtures/a95c59b4-7f5d-4b80-ad53-a694834ca18b.h5ad differ diff --git a/local_server/test/fixtures/dataset_config_outline.py b/local_server/test/fixtures/dataset_config_outline.py new file mode 100644 index 00000000..c41a0980 --- /dev/null +++ b/local_server/test/fixtures/dataset_config_outline.py @@ -0,0 +1,31 @@ +f""" +dataset: + app: + scripts: {scripts} #list of strs (filenames) or dicts containing keys + inline_scripts: {inline_scripts} #list of strs (filenames) + + authentication_enable: {authentication_enable} + + presentation: + max_categories: {max_categories} + custom_colors: {custom_colors} + + user_annotations: + enable: {enable_users_annotations} + type: {annotation_type} + local_file_csv: + directory: {local_file_csv_directory} + file: {local_file_csv_file} + ontology: + enable: {ontology_enabled} + obo_location: {obo_location} + + embeddings: + names: {embedding_names} + enable_reembedding: {enable_reembedding} + + diffexp: + enable: {enable_difexp} + lfc_cutoff: {lfc_cutoff} + top_n: {top_n} +""" diff --git a/local_server/test/fixtures/fixtures.py b/local_server/test/fixtures/fixtures.py new file mode 100644 index 00000000..e1a1368d --- /dev/null +++ b/local_server/test/fixtures/fixtures.py @@ -0,0 +1,12 @@ +pbmc3k_colors = { + "louvain": { + "B cells": "#2ca02c", + "CD14+ Monocytes": "#ff7f0e", + "CD4 T cells": "#1f77b4", + "CD8 T cells": "#d62728", + "Dendritic cells": "#e377c2", + "FCGR3A+ Monocytes": "#8c564b", + "Megakaryocytes": "#bcbd22", + "NK cells": "#9467bd", + } +} diff --git a/local_server/test/fixtures/hgnc_example.txt.gz b/local_server/test/fixtures/hgnc_example.txt.gz new file mode 100644 index 00000000..4c7704c4 Binary files /dev/null and b/local_server/test/fixtures/hgnc_example.txt.gz differ diff --git a/local_server/test/fixtures/nan.h5ad b/local_server/test/fixtures/nan.h5ad new file mode 100644 index 00000000..d97078b0 Binary files /dev/null and b/local_server/test/fixtures/nan.h5ad differ diff --git a/local_server/test/fixtures/pbmc3k-CSC-gz.h5ad b/local_server/test/fixtures/pbmc3k-CSC-gz.h5ad new file mode 100644 index 00000000..2c2909c5 Binary files /dev/null and b/local_server/test/fixtures/pbmc3k-CSC-gz.h5ad differ diff --git a/local_server/test/fixtures/pbmc3k-CSR-gz.h5ad b/local_server/test/fixtures/pbmc3k-CSR-gz.h5ad new file mode 100644 index 00000000..3053f0a9 Binary files /dev/null and b/local_server/test/fixtures/pbmc3k-CSR-gz.h5ad differ diff --git a/local_server/test/fixtures/pbmc3k-annotations.csv b/local_server/test/fixtures/pbmc3k-annotations.csv new file mode 100644 index 00000000..8031f7cd --- /dev/null +++ b/local_server/test/fixtures/pbmc3k-annotations.csv @@ -0,0 +1,2641 @@ +# Annotations generated on 2020-02-24T23:19:31 using cellxgene version 0.14.0 +# Input data file was example-dataset/pbmc3k.h5ad, which was last modified on 2019-12-17T17:13:17 +index,cluster-test +AAACATACAACCAC-1,four +AAACATTGAGCTAC-1,three +AAACATTGATCAGC-1,four +AAACCGTGCTTCCG-1,two +AAACCGTGTATGCG-1,five +AAACGCACTGGTAC-1,four +AAACGCTGACCAGT-1,unassigned +AAACGCTGGTTCTT-1,four +AAACGCTGTAGCCA-1,five +AAACGCTGTTTCTG-1,one +AAACTTGAAAAACG-1,three +AAACTTGATCCAGA-1,four +AAAGAGACGAGATA-1,four +AAAGAGACGCGAGA-1,two +AAAGAGACGGACTT-1,five +AAAGAGACGGCATT-1,four +AAAGCAGATATCGG-1,two +AAAGCCTGTATGCG-1,four +AAAGGCCTGTCTAG-1,three +AAAGTTTGATCACG-1,three +AAAGTTTGGGGTGA-1,three +AAAGTTTGTAGAGA-1,four +AAAGTTTGTAGCGT-1,two +AAATCAACAATGCC-1,three +AAATCAACACCAGT-1,four +AAATCAACCAGGAG-1,four +AAATCAACCCTATT-1,seven +AAATCAACGGAAGC-1,four +AAATCAACTCGCAA-1,four +AAATCATGACCACA-1,one +AAATCCCTCCACAA-1,four +AAATCCCTGCTATG-1,three +AAATGTTGAACGAA-1,two +AAATGTTGCCACAA-1,four +AAATGTTGTGGCAT-1,two +AAATTCGAAGGTTC-1,four +AAATTCGAATCACG-1,two +AAATTCGAGCTGAT-1,one +AAATTCGAGGAGTG-1,four +AAATTCGATTCTCA-1,five +AAATTGACACGACT-1,five +AAATTGACTCGCTC-1,unassigned +AACAAACTCATTTC-1,four +AACAAACTTTCGTT-1,four +AACAATACGACGAG-1,four +AACACGTGCAGAGG-1,four +AACACGTGGAAAGT-1,four +AACACGTGGAACCT-1,two +AACACGTGGCTACA-1,four +AACACGTGTACGAC-1,one +AACAGCACAAGAGT-1,two +AACATTGATGGGAG-1,one +AACCAGTGATACCG-1,one +AACCCAGATCGCTC-1,four +AACCGATGCTCCCA-1,four +AACCGATGGTCATG-1,three +AACCGATGTTCTAC-1,two +AACCGCCTAGCGTT-1,four +AACCGCCTCTACGA-1,two +AACCTACTGTGAGG-1,two +AACCTACTGTGTTG-1,four +AACCTTACGAGACG-1,four +AACCTTACGCGAGA-1,five +AACCTTACTAACGC-1,one +AACCTTTGGACGGA-1,four +AACCTTTGTACGCA-1,one +AACGCAACAAGTAG-1,four +AACGCATGACCCAA-1,four +AACGCATGCCTTCG-1,four +AACGCATGTACTTC-1,four +AACGCCCTCGGGAA-1,four +AACGCCCTCGTACA-1,five +AACGCCCTGGCATT-1,four +AACGTCGAGTATCG-1,five +AACGTGTGAAAGCA-1,four +AACGTGTGGCGGAA-1,three +AACGTGTGTCCAAG-1,four +AACGTGTGTGCTTT-1,four +AACTACCTTAGAGA-1,four +AACTCACTCAAGCT-1,two +AACTCACTTGGAGG-1,two +AACTCGGAAAGTGA-1,five +AACTCGGAAGGTCT-1,two +AACTCTTGCAGGAG-1,four +AACTGTCTCCCTTG-1,two +AACTTGCTACGCTA-1,three +AACTTGCTGGGACA-1,four +AAGAACGAGTGTTG-1,four +AAGAAGACGTAGGG-1,three +AAGACAGAAGTCTG-1,three +AAGACAGAGGATCT-1,four +AAGACAGATTACCT-1,two +AAGAGATGGGTAGG-1,three +AAGATGGAAAACAG-1,three +AAGATGGAGAACTC-1,two +AAGATGGAGATAAG-1,two +AAGATTACAACCTG-1,four +AAGATTACAGATCC-1,two +AAGATTACCCGTTC-1,two +AAGATTACCGCCTT-1,unassigned +AAGATTACCTCAAG-1,five +AAGATTACTCCTCG-1,two +AAGCAAGAGCGAGA-1,four +AAGCAAGAGCTTAG-1,five +AAGCAAGAGGTGTT-1,five +AAGCACTGAGCAAA-1,three +AAGCACTGCATACG-1,five +AAGCACTGGTTCTT-1,three +AAGCCAACGTGTTG-1,four +AAGCCATGAACTGC-1,unassigned +AAGCCATGACACGT-1,two +AAGCCATGCGTGAT-1,four +AAGCCATGTCTCGC-1,four +AAGCCTGACATGCA-1,four +AAGCCTGACCGAAT-1,two +AAGCGACTCCTCAC-1,one +AAGCGACTGTGTCA-1,four +AAGCGACTTACAGC-1,three +AAGCGACTTTGACG-1,one +AAGCGTACGTCTTT-1,four +AAGGCTTGCGAACT-1,five +AAGGTCACGGTTAC-1,two +AAGGTCACTGTTTC-1,three +AAGGTCACTTCCCG-1,five +AAGGTCTGACAGTC-1,four +AAGGTCTGCAGATC-1,four +AAGTAACTCTGAAC-1,two +AAGTAACTGAGATA-1,three +AAGTAGGATACAGC-1,five +AAGTATACCGAACT-1,four +AAGTCCGACTTGTT-1,four +AAGTCCGATAGAAG-1,four +AAGTCTCTAGTCGT-1,four +AAGTCTCTCGGAGA-1,five +AAGTGGCTTGGAGG-1,three +AAGTTCCTCATTCT-1,unassigned +AAGTTCCTTCTTAC-1,four +AATAAGCTCGAATC-1,four +AATAAGCTCGTTGA-1,four +AATACCCTGGACGA-1,three +AATACCCTGGCATT-1,four +AATACTGAAAGGGC-1,five +AATACTGAATTGGC-1,five +AATAGGGAACCCTC-1,three +AATAGGGAGAATGA-1,five +AATCAAACTATCGG-1,five +AATCCGGAATGCTG-1,three +AATCCTACCGGTAT-1,four +AATCCTTGACGGGA-1,two +AATCCTTGGTGAGG-1,five +AATCGGTGGAACTC-1,five +AATCGGTGTGCTTT-1,four +AATCTAGAAAAGTG-1,four +AATCTAGAATCGGT-1,four +AATCTCACAGCCTA-1,two +AATCTCACTCTAGG-1,four +AATCTCTGAACAGA-1,four +AATCTCTGCTTTAC-1,five +AATGATACACCAAC-1,five +AATGATACGGTCAT-1,two +AATGCGTGACACCA-1,three +AATGCGTGGACGGA-1,unassigned +AATGCGTGGCTATG-1,three +AATGGAGAATCGTG-1,two +AATGGAGATCCTTA-1,five +AATGGCTGACACCA-1,three +AATGGCTGCGTGAT-1,four +AATGGCTGTAAAGG-1,five +AATGGCTGTACTCT-1,five +AATGGCTGTGAAGA-1,four +AATGTAACGGTGGA-1,four +AATGTCCTCTTCTA-1,four +AATGTTGACAGTCA-1,four +AATGTTGAGTTGAC-1,two +AATGTTGATCTACT-1,four +AATTACGAATTCCT-1,unassigned +AATTACGACTTCTA-1,four +AATTACGAGTGAGG-1,unassigned +AATTACGATTGGCA-1,three +AATTCCTGCTCAGA-1,three +AATTGATGTCGCAA-1,one +AATTGTGACTTGGA-1,three +ACAAAGGAGGGTGA-1,four +ACAAATTGATTCTC-1,one +ACAAATTGCTCAGA-1,two +ACAAATTGTTGCGA-1,five +ACAACCGAGGGATG-1,five +ACAACCGAGTTACG-1,four +ACAAGAGAAGTCGT-1,two +ACAAGAGACTTATC-1,four +ACAAGAGAGTTGAC-1,three +ACAATCCTAACCGT-1,four +ACAATCCTTAGCGT-1,three +ACAATTGACTGACA-1,five +ACAATTGATGACTG-1,five +ACACAGACCATACG-1,five +ACACCAGAGGGCAA-1,five +ACACCCTGGTGTTG-1,five +ACACGAACAGTTCG-1,four +ACACGATGACGCAT-1,five +ACACGATGATGTGC-1,unassigned +ACACGATGTCGTAG-1,five +ACACGATGTGGTCA-1,five +ACAGACACGGCATT-1,four +ACAGACACGTTGTG-1,four +ACAGCAACACCTAG-1,two +ACAGCAACCTCAAG-1,one +ACAGGTACCCCACT-1,four +ACAGGTACGCTGTA-1,five +ACAGGTACTGGTGT-1,five +ACAGTCGACCCAAA-1,four +ACAGTCGACCGATA-1,five +ACAGTGACTCACCC-1,five +ACAGTGACTCTATC-1,five +ACAGTGTGGTCACA-1,four +ACAGTGTGTTGCGA-1,two +ACATCACTCTACTT-1,two +ACATGGTGAAGCCT-1,four +ACATGGTGCAACCA-1,three +ACATGGTGCGAGTT-1,four +ACATTCTGGCATAC-1,four +ACATTCTGGGAACG-1,two +ACCAACGACATGCA-1,four +ACCACAGAAAGTAG-1,four +ACCACAGAGTTGGT-1,four +ACCACCTGTGTGCA-1,four +ACCACGCTACAGCT-1,four +ACCACGCTACCCAA-1,five +ACCACGCTGCGAGA-1,four +ACCACGCTGCTGTA-1,four +ACCAGCCTGACAGG-1,four +ACCAGTGAACGGTT-1,three +ACCAGTGAATACCG-1,one +ACCAGTGAGGGATG-1,four +ACCAGTGATGACTG-1,two +ACCATTACCTTCTA-1,four +ACCATTACGAGATA-1,four +ACCATTTGTCATTC-1,four +ACCCAAGAACTGTG-1,five +ACCCAAGAATTCCT-1,one +ACCCAAGAGGACAG-1,four +ACCCAAGATTCACT-1,three +ACCCACTGCGCCTT-1,two +ACCCACTGGACAGG-1,four +ACCCACTGGTTCAG-1,six +ACCCACTGTCGTAG-1,two +ACCCAGCTCAGAAA-1,four +ACCCAGCTGTTAGC-1,two +ACCCAGCTTGCTTT-1,four +ACCCGTTGATGACC-1,three +ACCCGTTGCTGCAA-1,four +ACCCGTTGCTTCTA-1,unassigned +ACCCTCGACCTATT-1,four +ACCCTCGACGGTAT-1,three +ACCCTCGATAAGGA-1,two +ACCCTCGATCAAGC-1,unassigned +ACCGTGCTACCAGT-1,four +ACCGTGCTGGAACG-1,three +ACCTATTGCTGAGT-1,three +ACCTATTGTGCCCT-1,one +ACCTCCGAGTCCTC-1,four +ACCTCCGATATGCG-1,four +ACCTCCGATGCTGA-1,four +ACCTCGTGAACCAC-1,unassigned +ACCTGAGATATCGG-1,one +ACCTGGCTAAGTAG-1,five +ACCTTTGACTCCCA-1,two +ACCTTTGAGGAACG-1,two +ACCTTTGAGGAAGC-1,one +ACGAACACCTTGTT-1,five +ACGAACTGGCTATG-1,six +ACGAAGCTCTCCAC-1,four +ACGAAGCTCTGAGT-1,three +ACGACCCTATCTCT-1,one +ACGACCCTGATGAA-1,four +ACGACCCTTGACAC-1,three +ACGACCCTTGACCA-1,four +ACGAGGGACAGGAG-1,unassigned +ACGAGGGACGAACT-1,four +ACGAGGGATGTAGC-1,four +ACGAGTACCCTAAG-1,three +ACGAGTACGAATCC-1,three +ACGATCGAGGACTT-1,five +ACGATCGAGTCACA-1,two +ACGATGACAATGCC-1,four +ACGATGACTGGTCA-1,four +ACGATTCTACGGGA-1,four +ACGCAATGGTTCAG-1,five +ACGCACCTGTTAGC-1,three +ACGCCACTGAACTC-1,two +ACGCCGGAAACCAC-1,two +ACGCCGGAAAGCCT-1,four +ACGCCGGAAATGCC-1,four +ACGCCTTGCTCCCA-1,one +ACGCGGTGGCGAGA-1,four +ACGCGGTGTGTGGT-1,three +ACGCGGTGTTTGCT-1,five +ACGCTCACAGTACC-1,four +ACGCTCACCCTTGC-1,four +ACGCTGCTGTTCTT-1,five +ACGGAACTCAGATC-1,five +ACGGAACTGTCGTA-1,one +ACGGAGGACTCTTA-1,two +ACGGATTGGGAGGT-1,three +ACGGATTGGTTAGC-1,four +ACGGCTCTGAGCAG-1,four +ACGGCTCTTGCACA-1,four +ACGGTAACCGCTAA-1,four +ACGGTAACCTTCGC-1,five +ACGGTAACGGTGGA-1,four +ACGGTAACTCGCAA-1,five +ACGGTATGAGTCGT-1,two +ACGGTATGGGTATC-1,three +ACGGTATGGTTGTG-1,three +ACGGTCCTAACGGG-1,four +ACGGTCCTCGGGAA-1,four +ACGTAGACAACCAC-1,four +ACGTAGACTACAGC-1,three +ACGTCAGAAACGAA-1,four +ACGTCAGAGAGCTT-1,two +ACGTCAGAGGGATG-1,four +ACGTCCTGATAAGG-1,two +ACGTCCTGTGAACC-1,unassigned +ACGTCGCTCCTGAA-1,five +ACGTCGCTCTATTC-1,unassigned +ACGTCGCTTCTCAT-1,three +ACGTGATGCCATGA-1,unassigned +ACGTGATGGGTCTA-1,four +ACGTGATGTAACCG-1,one +ACGTGATGTGACAC-1,five +ACGTGCCTCCGTAA-1,four +ACGTGCCTTCTATC-1,five +ACGTTACTTTCCAT-1,four +ACGTTGGAAAAGCA-1,four +ACGTTGGAAACCTG-1,five +ACGTTGGACCGTAA-1,two +ACGTTGGAGCCAAT-1,five +ACGTTGGATATGGC-1,three +ACGTTGGATCAGGT-1,two +ACGTTTACATCAGC-1,four +ACTAAAACCCACAA-1,five +ACTAAAACTCGACA-1,two +ACTACGGAATTTCC-1,four +ACTACGGACCTATT-1,four +ACTACGGATCGCTC-1,four +ACTACTACTAAGGA-1,four +ACTAGGTGGAACCT-1,five +ACTAGGTGGAACTC-1,four +ACTATCACCTTGGA-1,five +ACTATCACTGCCAA-1,four +ACTCAGGACTGAAC-1,four +ACTCAGGATCTATC-1,three +ACTCAGGATTCGTT-1,two +ACTCCTCTCAACTG-1,four +ACTCGCACGAAAGT-1,two +ACTCGCACTACGAC-1,two +ACTCTCCTGACACT-1,four +ACTCTCCTGCATAC-1,four +ACTCTCCTGTTTGG-1,four +ACTGAGACAACCAC-1,five +ACTGAGACCCATAG-1,one +ACTGAGACGTTGGT-1,three +ACTGCCACACACGT-1,five +ACTGCCACTCCGTC-1,five +ACTGGCCTTCAGTG-1,five +ACTGTGGACGTGTA-1,five +ACTGTGGATCTAGG-1,three +ACTGTTACCCACAA-1,two +ACTGTTACTGCAGT-1,four +ACTTAAGAACCACA-1,four +ACTTAAGATTACTC-1,unassigned +ACTTAGCTGCGTAT-1,one +ACTTAGCTGGGAGT-1,four +ACTTCAACAAGCAA-1,unassigned +ACTTCAACGTAGGG-1,five +ACTTCCCTTTCCGC-1,four +ACTTCTGACATGCA-1,three +ACTTGACTCCACAA-1,three +ACTTGGGAGAAAGT-1,two +ACTTGGGAGGTTTG-1,four +ACTTGGGATGTGAC-1,one +ACTTGGGATTGACG-1,two +ACTTGTACCTGTCC-1,four +ACTTTGTGGAAAGT-1,four +ACTTTGTGGATAGA-1,two +AGAAACGAAAGTAG-1,three +AGAAAGTGCGCAAT-1,four +AGAAAGTGGGGATG-1,two +AGAACAGAAATGCC-1,five +AGAACAGACGACTA-1,three +AGAACAGAGACAAA-1,four +AGAACGCTTTGCTT-1,four +AGAAGATGTGACTG-1,four +AGAATGGAAGAAGT-1,five +AGAATTTGTAACCG-1,four +AGAATTTGTAGAGA-1,four +AGACACACTGTAGC-1,four +AGACACTGTCAAGC-1,two +AGACCTGAAGTAGA-1,two +AGACCTGACCAACA-1,four +AGACCTGAGGAAGC-1,five +AGACGTACAGAGGC-1,four +AGACGTACCCCTAC-1,three +AGACGTACCTCTTA-1,unassigned +AGACGTACTCGTGA-1,one +AGACTGACCATCAG-1,five +AGACTGACCCTTTA-1,four +AGACTTCTCATGCA-1,three +AGAGATGACAGTCA-1,three +AGAGATGACTGAAC-1,four +AGAGATGAGGTTTG-1,two +AGAGATGATCTCGC-1,three +AGAGATGATTGTGG-1,four +AGAGCGGAGGCAAG-1,two +AGAGTCTGGTCGTA-1,one +AGAGTGCTCAGCTA-1,four +AGAGTGCTCGAATC-1,four +AGAGTGCTGTCATG-1,four +AGAGTGCTGTCCTC-1,five +AGAGTGCTGTGTTG-1,three +AGATATACCCGTAA-1,five +AGATATACGATGAA-1,four +AGATATACTGTTCT-1,three +AGATATTGCCTACC-1,five +AGATATTGGCCAAT-1,two +AGATCGTGTCTGGA-1,two +AGATCGTGTTTGTC-1,three +AGATCTCTATCACG-1,three +AGATTAACGTTCTT-1,one +AGATTCCTATCGTG-1,four +AGATTCCTCACTTT-1,two +AGATTCCTGACGAG-1,two +AGATTCCTGTTCAG-1,five +AGCAAAGATATGCG-1,four +AGCACAACAGTCTG-1,four +AGCACTGAGGGAGT-1,four +AGCACTGATATGCG-1,four +AGCACTGATGCTTT-1,unassigned +AGCACTGATTGCGA-1,four +AGCATCGAAGATCC-1,four +AGCATCGAAGGGTG-1,three +AGCATCGAGCTTCC-1,two +AGCATCGAGTGAGG-1,four +AGCATCGATAACCG-1,two +AGCATGACGATGAA-1,five +AGCCAATGGGGAGT-1,one +AGCCAATGTATCTC-1,four +AGCCACCTGGATCT-1,five +AGCCGGTGCCAATG-1,four +AGCCGGTGTGTTTC-1,four +AGCCGTCTCAATCG-1,four +AGCCGTCTGAGAGC-1,four +AGCCTCACGTTCGA-1,four +AGCCTCACTGTCAG-1,one +AGCCTCTGCAGTTG-1,three +AGCCTCTGCCAATG-1,five +AGCGAACTGGATCT-1,four +AGCGAACTTACTGG-1,four +AGCGATACGGAGCA-1,four +AGCGATTGAGATCC-1,five +AGCGCCGAATCTCT-1,four +AGCGCCGACAGAGG-1,four +AGCGCTCTACCTTT-1,four +AGCGGCACCGGGAA-1,four +AGCGGCTGATGTGC-1,four +AGCGGGCTTGCCAA-1,four +AGCGTAACATGCTG-1,four +AGCGTAACTGAGAA-1,two +AGCTCGCTACTGGT-1,four +AGCTCGCTCTGCTC-1,four +AGCTGAACCATACG-1,one +AGCTGAACCTCTCG-1,four +AGCTGCCTTGGGAG-1,two +AGCTGCCTTTCATC-1,four +AGCTGCCTTTCTGT-1,four +AGCTGTGATCCAAG-1,four +AGCTTTACAAGTAG-1,four +AGCTTTACACCAAC-1,five +AGCTTTACTCTCAT-1,three +AGGAAATGAGGAGC-1,four +AGGAACCTCTTAGG-1,one +AGGAACCTTGCCTC-1,five +AGGAATGATAACGC-1,four +AGGAATGATTTGTC-1,four +AGGAGTCTGGTTTG-1,four +AGGAGTCTTGTCAG-1,four +AGGATAGACATTTC-1,four +AGGATAGAGGATTC-1,five +AGGATGCTACTAGC-1,three +AGGATGCTTTAGGC-1,five +AGGCAACTGAAGGC-1,four +AGGCAGGAGTACCA-1,five +AGGCCTCTAGTCGT-1,four +AGGCCTCTCGGAGA-1,four +AGGCCTCTCGTAAC-1,four +AGGGACGACGTTGA-1,four +AGGGACGAGTCAAC-1,five +AGGGACGAGTTGTG-1,four +AGGGACGATAGAGA-1,five +AGGGACGATGCATG-1,five +AGGGAGTGAGCCTA-1,four +AGGGCCACCATACG-1,four +AGGGCGCTAACCAC-1,unassigned +AGGGCGCTATGGTC-1,four +AGGGTGGACAGTCA-1,three +AGGGTGGACTCAAG-1,two +AGGGTGGAGTTGCA-1,five +AGGGTTTGTTCATC-1,two +AGGTCATGAGTGTC-1,three +AGGTCATGCTTATC-1,five +AGGTCTGATTCTCA-1,one +AGGTGGGAAGAATG-1,unassigned +AGGTGGGAAGTTCG-1,four +AGGTGTTGGTTACG-1,three +AGGTTCGAACCTCC-1,two +AGGTTCGAACGTAC-1,two +AGGTTCGAGGGTGA-1,four +AGTAAGGAGTTTGG-1,four +AGTAAGGATTCTTG-1,four +AGTAATACATCACG-1,five +AGTAATACCGAACT-1,three +AGTAATTGTCCCAC-1,four +AGTACGTGAGGGTG-1,four +AGTACGTGCTGCAA-1,four +AGTACGTGCTTGGA-1,four +AGTACTCTACGTGT-1,three +AGTACTCTCAACCA-1,five +AGTACTCTCGGTAT-1,five +AGTAGGCTTGCCTC-1,one +AGTATAACTTGTCT-1,one +AGTATCCTAGAACA-1,four +AGTCACGATGAGCT-1,five +AGTCAGACGAATAG-1,four +AGTCAGACGCTTAG-1,four +AGTCAGACTAGAGA-1,five +AGTCAGACTGCACA-1,four +AGTCCAGATATCTC-1,four +AGTCCAGATTTCAC-1,four +AGTCGAACCAACCA-1,four +AGTCGCCTCCGTAA-1,four +AGTCTACTAGGGTG-1,two +AGTCTACTTGCATG-1,one +AGTCTTACACCACA-1,four +AGTCTTACTTCGCC-1,five +AGTCTTACTTCGGA-1,six +AGTGACTGCAACTG-1,one +AGTGTTCTAACCTG-1,four +AGTGTTCTATAAGG-1,four +AGTGTTCTCACTTT-1,two +AGTTAAACCACTTT-1,four +AGTTATGAACAGTC-1,two +AGTTATGACTGAGT-1,four +AGTTATGAGTTCAG-1,four +AGTTCTACCAGCTA-1,two +AGTTCTTGAAGCCT-1,five +AGTTGTCTACTACG-1,four +AGTTTAGATGGTCA-1,four +AGTTTCACGGTCTA-1,five +AGTTTGCTACAGTC-1,four +AGTTTGCTACTGGT-1,five +AGTTTGCTCCAAGT-1,four +ATAAACACAGTGCT-1,three +ATAAACACCACCAA-1,four +ATAACAACATGCTG-1,two +ATAACAACGTCTAG-1,four +ATAACAACTTTGTC-1,four +ATAACATGTACTCT-1,four +ATAACCCTGTTGGT-1,four +ATAACCCTTGGTAC-1,four +ATAAGTACGAATGA-1,three +ATAAGTTGGTACGT-1,one +ATAAGTTGTCTAGG-1,four +ATAATCGAGCTGAT-1,three +ATAATCGATGGTTG-1,four +ATAATGACCTACTT-1,four +ATAATGACTCGTGA-1,two +ATACAATGTTAGGC-1,four +ATACCACTCGTACA-1,two +ATACCACTCTAAGC-1,unassigned +ATACCACTGCCAAT-1,five +ATACCGGAATGCTG-1,unassigned +ATACCGGACATTTC-1,two +ATACCGGACTTCGC-1,four +ATACCGGAGGTGTT-1,two +ATACCGGATCTCGC-1,four +ATACCTACGCATCA-1,unassigned +ATACCTTGGGGCAA-1,four +ATACGGACAGACTC-1,two +ATACGGACCTACTT-1,two +ATACGGACGAGGTG-1,three +ATACGGACTATGCG-1,three +ATACGGACTCTGGA-1,two +ATACGTCTTAACGC-1,two +ATACTCTGCTTCGC-1,two +ATACTCTGGTATGC-1,five +ATAGATACCATGGT-1,four +ATAGATACGACGAG-1,four +ATAGATTGGTGTAC-1,three +ATAGCCGAACGGAG-1,four +ATAGCGTGCAGATC-1,four +ATAGCGTGCCCTTG-1,two +ATAGCGTGGTATCG-1,two +ATAGCGTGTCTCTA-1,four +ATAGCTCTCTGATG-1,four +ATAGCTCTGAGGTG-1,four +ATAGGAGAAACAGA-1,two +ATAGGCTGTCAGAC-1,four +ATAGTCCTAGTGTC-1,two +ATAGTCCTTGCATG-1,five +ATAGTCCTTGTCGA-1,four +ATAGTTGACAACTG-1,four +ATAGTTGACCCTCA-1,one +ATAGTTGAGACGTT-1,four +ATAGTTGATAAGCC-1,four +ATATACGAAGCCAT-1,four +ATATACGAATTGGC-1,four +ATATAGTGGAATGA-1,two +ATATGCCTAGATCC-1,two +ATATGCCTGGACAG-1,four +ATATGCCTTCTCTA-1,four +ATATGCCTTGGTAC-1,four +ATCAAATGAGCCTA-1,two +ATCAAATGGGTAAA-1,one +ATCAACCTAAACGA-1,four +ATCAACCTGAGGAC-1,four +ATCAACCTTCTCTA-1,five +ATCAACCTTTGTCT-1,five +ATCACACTTTGTCT-1,four +ATCACGGATTTCGT-1,three +ATCATCTGACACCA-1,six +ATCATGCTAGAGTA-1,five +ATCATGCTGAACCT-1,three +ATCCAGGACGCTAA-1,one +ATCCAGGATGGAAA-1,four +ATCCATACTCCTTA-1,two +ATCCATACTTCATC-1,one +ATCCCGTGCAGTCA-1,five +ATCCCGTGCATGCA-1,three +ATCCCGTGGCTGAT-1,three +ATCCGCACGCATCA-1,three +ATCCTAACGACGGA-1,three +ATCCTAACGCTACA-1,five +ATCGACGAAACTGC-1,two +ATCGACGAATGACC-1,four +ATCGAGTGGACGTT-1,five +ATCGCAGAATCTCT-1,one +ATCGCAGAGTGTCA-1,three +ATCGCCACTGAGGG-1,four +ATCGCCTGGGTCAT-1,four +ATCGCCTGTGGCAT-1,three +ATCGCGCTCAGAGG-1,two +ATCGCGCTGGGATG-1,four +ATCGCGCTTTTCGT-1,three +ATCGGAACCAGTCA-1,four +ATCGGTGAGTCAAC-1,four +ATCGGTGATTGCAG-1,four +ATCGTTTGCCTACC-1,two +ATCGTTTGGGTACT-1,two +ATCGTTTGTGCCAA-1,two +ATCTACACCCGCTT-1,two +ATCTACACCGGGAA-1,four +ATCTCAACAGGAGC-1,three +ATCTCAACCTCGAA-1,four +ATCTCAACCTTGTT-1,two +ATCTGGGAAACCAC-1,two +ATCTGGGAAGTGTC-1,unassigned +ATCTGGGATTCCGC-1,five +ATCTGTTGAACGGG-1,four +ATCTGTTGACCTCC-1,four +ATCTGTTGCCTTCG-1,one +ATCTGTTGGTTGCA-1,four +ATCTTGACACCAAC-1,four +ATCTTGACCTCCCA-1,two +ATCTTTCTGCATCA-1,unassigned +ATCTTTCTGTTTCT-1,five +ATCTTTCTTGTCCC-1,five +ATGAAACTCTGTGA-1,five +ATGAAACTGAGGCA-1,four +ATGAAGGAACAGCT-1,five +ATGAAGGACCTGTC-1,three +ATGAAGGACCTTAT-1,three +ATGAAGGACTAGTG-1,three +ATGAAGGACTTGCC-1,five +ATGACGTGACGACT-1,three +ATGACGTGATCGGT-1,one +ATGAGAGAAAGTGA-1,four +ATGAGAGAACGCAT-1,four +ATGAGAGAAGTAGA-1,three +ATGAGCACACAGCT-1,four +ATGAGCACATCTTC-1,four +ATGATAACTTCACT-1,five +ATGATATGAAACAG-1,four +ATGATATGACTGGT-1,four +ATGATATGAGCACT-1,four +ATGATATGGTCATG-1,four +ATGATATGGTGCTA-1,five +ATGATATGTTGTCT-1,two +ATGCACGAATGTCG-1,three +ATGCACGACTGTAG-1,four +ATGCACGAGAACCT-1,three +ATGCACGAGTTCGA-1,four +ATGCACGATTGGTG-1,two +ATGCAGTGTTACCT-1,three +ATGCAGTGTTCTAC-1,four +ATGCCAGAACGACT-1,four +ATGCCAGACAGTCA-1,two +ATGCCGCTTGAACC-1,two +ATGCGATGCTATGG-1,two +ATGCGATGCTGAGT-1,four +ATGCGATGGTTACG-1,three +ATGCGCCTTCATTC-1,four +ATGCTTTGCGAATC-1,four +ATGCTTTGGGCGAA-1,five +ATGCTTTGTAGTCG-1,three +ATGGACACATCGGT-1,four +ATGGACACGCATCA-1,five +ATGGGTACAACCTG-1,three +ATGGGTACATCGGT-1,five +ATGGGTACTATTCC-1,four +ATGGGTACTGGGAG-1,four +ATGTAAACACCTCC-1,four +ATGTAAACCCGCTT-1,four +ATGTAAACGGGATG-1,unassigned +ATGTAAACTCTCCG-1,unassigned +ATGTAAACTTCACT-1,two +ATGTACCTCAGTCA-1,four +ATGTACCTTAGTCG-1,four +ATGTACCTTTATCC-1,two +ATGTACCTTTCACT-1,five +ATGTCACTAATGCC-1,two +ATGTCACTCTGCTC-1,four +ATGTCGGAGGTGAG-1,four +ATGTTCACAGTCTG-1,two +ATGTTCACCGTAGT-1,four +ATGTTGCTTTCAGG-1,four +ATTAACGATGAGAA-1,five +ATTAACGATGCAAC-1,five +ATTAAGACTGCAGT-1,four +ATTACCTGCCTTAT-1,two +ATTACCTGGAGGAC-1,one +ATTAGATGTTTCAC-1,four +ATTATGGAATCTCT-1,four +ATTCAAGAACGGGA-1,four +ATTCAAGACCTTTA-1,four +ATTCAGCTCATTGG-1,six +ATTCCAACCATTGG-1,two +ATTCCAACTTAGGC-1,five +ATTCGACTCACTAG-1,four +ATTCGACTGAATAG-1,two +ATTCGACTTTTGTC-1,one +ATTCGGGAAAGGCG-1,two +ATTCGGGATTAGGC-1,four +ATTCTTCTGATACC-1,unassigned +ATTGAATGGACGGA-1,four +ATTGATGAAGGTTC-1,four +ATTGATGACTGAGT-1,four +ATTGATGAGCGAAG-1,four +ATTGATGATCTATC-1,four +ATTGCACTGACGGA-1,four +ATTGCACTGAGAGC-1,one +ATTGCACTGGAGCA-1,two +ATTGCACTTAGCCA-1,four +ATTGCACTTGCTTT-1,one +ATTGCTTGTTACTC-1,three +ATTGGTCTGACTAC-1,three +ATTGGTCTTGTCTT-1,three +ATTGTAGATTCCCG-1,unassigned +ATTGTAGATTGCAG-1,four +ATTGTCTGCGTACA-1,four +ATTTAGGAACCATG-1,one +ATTTAGGACAGAGG-1,four +ATTTCCGAGATGAA-1,four +ATTTCCGAGTGCTA-1,four +ATTTCGTGTATGGC-1,five +ATTTCTCTACTTTC-1,one +ATTTCTCTAGCAAA-1,four +ATTTCTCTCACTTT-1,unassigned +ATTTCTCTTCCCAC-1,four +ATTTGCACAAGATG-1,two +CAAAGCACAGCTCA-1,four +CAAAGCACCGTAAC-1,three +CAAAGCACGGTAAA-1,five +CAAAGCTGAAAGTG-1,four +CAAAGCTGTTGCTT-1,two +CAAATATGTGACAC-1,four +CAAATTGAGGGCAA-1,four +CAAATTGATGGAGG-1,two +CAACCAGAAAAGTG-1,four +CAACCAGAAGTGCT-1,four +CAACCAGAGTTCAG-1,two +CAACCAGATAGAAG-1,two +CAACCGCTGTTCAG-1,four +CAACCGCTTTGAGC-1,four +CAACGATGCGCAAT-1,one +CAACGTGACTCCAC-1,two +CAACGTGAGCCATA-1,five +CAACGTGATCAAGC-1,two +CAAGAAGACCACAA-1,four +CAAGAAGACGTCTC-1,two +CAAGAAGATTCTAC-1,four +CAAGACTGACCTGA-1,three +CAAGACTGAGTAGA-1,three +CAAGCTGACCATAG-1,one +CAAGCTGATCTATC-1,five +CAAGGACTGTTCAG-1,four +CAAGGACTTCTTTG-1,five +CAAGGTTGCTCCAC-1,four +CAAGGTTGTCATTC-1,three +CAAGGTTGTCTGGA-1,five +CAAGTCGAAACAGA-1,three +CAAGTCGATAGCGT-1,two +CAATAAACGCCATA-1,two +CAATAATGAACTGC-1,two +CAATATGACATGGT-1,four +CAATATGACCTTCG-1,five +CAATATGACGTTAG-1,four +CAATATGAGGAGCA-1,four +CAATCGGAGAAACA-1,four +CAATCTACTGACTG-1,five +CAATTCACCCAACA-1,five +CAATTCACGATAGA-1,four +CAATTCACTTGTGG-1,four +CAATTCTGCTTGTT-1,two +CAATTCTGGCGTAT-1,two +CACAACGATACGAC-1,four +CACAATCTTGTTCT-1,four +CACAATCTTTCCAT-1,five +CACACCTGCTTGAG-1,four +CACACCTGTATGGC-1,four +CACAGAACCCTTGC-1,four +CACAGAACCTGATG-1,four +CACAGATGGGATTC-1,four +CACAGATGGTTTCT-1,two +CACAGCCTGATACC-1,unassigned +CACAGCCTTGCCAA-1,one +CACAGCCTTGTAGC-1,four +CACAGTGATGAAGA-1,two +CACATACTACAGCT-1,five +CACATGGAACACGT-1,four +CACATGGAAGTCGT-1,four +CACCACTGCCAACA-1,four +CACCACTGGCGAAG-1,five +CACCCATGTTCTGT-1,four +CACCGGGAATCGAC-1,unassigned +CACCGGGACGAGAG-1,five +CACCGGGACGTGTA-1,four +CACCGGGACTTCTA-1,unassigned +CACCGGGACTTGCC-1,five +CACCGGGATTCGGA-1,five +CACCGTACTAAGGA-1,three +CACCGTACTAGCGT-1,four +CACCTGACACCCAA-1,five +CACCTGACCAGAAA-1,four +CACCTGACCTCAAG-1,five +CACCTGACGAAAGT-1,three +CACCTGACTCGTAG-1,four +CACGAAACTTCCGC-1,three +CACGACCTCGATAC-1,three +CACGCTACAGAAGT-1,four +CACGCTACTGTTCT-1,three +CACGGGACAGAGTA-1,four +CACGGGACATAAGG-1,four +CACGGGACGTAGGG-1,four +CACGGGTGCTTCGC-1,five +CACGGGTGGAGGAC-1,five +CACGGGTGTGTTTC-1,four +CACTAACTCCTAAG-1,three +CACTAACTGAAAGT-1,four +CACTAGGATGATGC-1,three +CACTATACCCCGTT-1,three +CACTATACGTTTGG-1,five +CACTGAGACAGTCA-1,two +CACTGCACTTCATC-1,four +CACTGCTGAGACTC-1,unassigned +CACTGCTGGAAAGT-1,four +CACTTAACCGAATC-1,four +CACTTAACCGTACA-1,three +CACTTTGACTCTAT-1,five +CACTTTGAGCTGTA-1,five +CAGAAGCTCTCAAG-1,four +CAGACATGAACGGG-1,four +CAGACATGTCGACA-1,four +CAGACCCTAAGGTA-1,one +CAGACCCTAATGCC-1,four +CAGACCCTAGGAGC-1,four +CAGACTGAGTATGC-1,four +CAGATCGAATGTCG-1,three +CAGATCGACCTGAA-1,four +CAGATCGATATGGC-1,four +CAGATGACATTCTC-1,five +CAGCAATGCCTTCG-1,four +CAGCAATGGAGGGT-1,five +CAGCAATGGTGCTA-1,four +CAGCAATGTCTACT-1,two +CAGCAATGTGACCA-1,four +CAGCAATGTGAGGG-1,two +CAGCACCTAAGCCT-1,four +CAGCACCTAGGCGA-1,four +CAGCACCTGTAGGG-1,two +CAGCATGACAACCA-1,two +CAGCATGAGACGTT-1,unassigned +CAGCCTACCCAACA-1,four +CAGCCTTGCTACCC-1,two +CAGCCTTGGGGACA-1,four +CAGCGGACACCCTC-1,four +CAGCGGACCTTTAC-1,five +CAGCGTCTAAAGCA-1,four +CAGCGTCTTATCGG-1,four +CAGCTAGATGTGAC-1,two +CAGCTCTGAGGCGA-1,four +CAGCTCTGCAAGCT-1,five +CAGCTCTGTCGTAG-1,two +CAGCTCTGTGTGGT-1,five +CAGGAACTAACTGC-1,five +CAGGAACTCTCAGA-1,two +CAGGCCGAACACCA-1,four +CAGGCCGAACACGT-1,four +CAGGCCGAACGACT-1,four +CAGGCCGAATCTCT-1,unassigned +CAGGCCGACTAGCA-1,three +CAGGGCACCATACG-1,three +CAGGGCACCCAACA-1,four +CAGGGCACTCCCGT-1,four +CAGGTAACAGACTC-1,two +CAGGTATGAGTCGT-1,four +CAGGTATGTGCTTT-1,four +CAGGTTGAGGATCT-1,three +CAGTGATGGACGGA-1,three +CAGTGATGGCTAAC-1,five +CAGTGATGGGACAG-1,four +CAGTGATGTAAGGA-1,three +CAGTGATGTACGCA-1,four +CAGTGTGATGTCAG-1,four +CAGTTACTAAGGTA-1,three +CAGTTACTGATAGA-1,four +CAGTTGGAAAGAGT-1,two +CAGTTGGACATACG-1,four +CAGTTTACACACGT-1,five +CAGTTTACCCCAAA-1,four +CATAAAACGGAGCA-1,two +CATAAATGAACTGC-1,four +CATAACCTTCTCCG-1,four +CATACTACCTCGAA-1,four +CATACTACCTGAAC-1,four +CATACTACGTACCA-1,two +CATACTTGGGTTAC-1,seven +CATAGTCTAATCGC-1,four +CATAGTCTCACTTT-1,four +CATATAGACTAAGC-1,unassigned +CATATAGATCAGGT-1,three +CATCAACTAGAAGT-1,four +CATCAACTCCCTCA-1,four +CATCAGGACTTCCG-1,five +CATCAGGATAGCCA-1,five +CATCAGGATCCTAT-1,one +CATCAGGATGCACA-1,two +CATCAGGATTTCGT-1,two +CATCATACCGCATA-1,two +CATCATACGGAGCA-1,three +CATCATACTCAAGC-1,four +CATCGCTGGGATCT-1,four +CATCGCTGTGGCAT-1,three +CATCGGCTATGCTG-1,four +CATCGGCTTTGGCA-1,four +CATCTCCTATGTGC-1,three +CATCTCCTCGAACT-1,unassigned +CATGAGACACGGGA-1,three +CATGAGACGTTGAC-1,five +CATGAGACTCGCCT-1,four +CATGCCACGGGTGA-1,four +CATGCCACTGCCAA-1,four +CATGCGCTAGTCAC-1,one +CATGCGCTCAGATC-1,two +CATGCGCTTTGCAG-1,one +CATGGCCTAGGGTG-1,two +CATGGCCTGTGCAT-1,four +CATGTACTATCGTG-1,five +CATGTTACAGTCGT-1,five +CATGTTACCTGAGT-1,four +CATGTTTGGGGATG-1,two +CATTACACACGGAG-1,three +CATTACACCAACTG-1,five +CATTACACGGAGTG-1,four +CATTACACTACTCT-1,four +CATTAGCTCCACAA-1,two +CATTGACTAGCGGA-1,two +CATTGGGACTCGAA-1,four +CATTGTACAGCGTT-1,four +CATTGTACTCGATG-1,three +CATTGTACTTATCC-1,four +CATTGTACTTTGCT-1,one +CATTGTTGCTAGTG-1,four +CATTTCGACTCTAT-1,five +CATTTCGAGATACC-1,five +CATTTGACCACACA-1,five +CATTTGACCCTGAA-1,five +CATTTGTGACGACT-1,three +CATTTGTGCATTGG-1,four +CATTTGTGCGGAGA-1,four +CATTTGTGGGATCT-1,three +CCAAAGTGCTACGA-1,two +CCAAAGTGTGAGAA-1,two +CCAACCTGAAGTAG-1,five +CCAACCTGACGTAC-1,two +CCAACCTGTTCGCC-1,unassigned +CCAAGAACCCAATG-1,four +CCAAGAACGTAGCT-1,four +CCAAGAACGTGTCA-1,four +CCAAGAACTACTGG-1,five +CCAAGAACTCCTAT-1,three +CCAAGATGTCATTC-1,four +CCAAGATGTTTCAC-1,four +CCAAGTGAGGAACG-1,four +CCAAGTGATCAAGC-1,four +CCAATTTGAACGTC-1,two +CCACCATGAACGTC-1,four +CCACCATGATCGGT-1,four +CCACCATGGACGAG-1,one +CCACCATGGGGAGT-1,four +CCACCATGTCCTGC-1,five +CCACTGACCCGCTT-1,two +CCACTGTGGGAAGC-1,two +CCACTGTGTGTAGC-1,four +CCACTTCTCGGGAA-1,two +CCAGAAACCCTGTC-1,three +CCAGAAACGAACTC-1,four +CCAGAAACGGTCTA-1,four +CCAGACCTCTGAGT-1,unassigned +CCAGACCTTGTGGT-1,four +CCAGCACTGCGATT-1,one +CCAGCGGAAAGGCG-1,two +CCAGCGGACGACTA-1,two +CCAGCGGATGGGAG-1,four +CCAGCTACACAGTC-1,four +CCAGCTACCAGCTA-1,three +CCAGGTCTACACCA-1,three +CCAGGTCTAGCATC-1,two +CCAGGTCTATGGTC-1,four +CCAGTCACACTGGT-1,unassigned +CCAGTCACACTGTG-1,three +CCAGTCACGTTGTG-1,four +CCAGTGCTAACCAC-1,four +CCAGTGCTCGTAGT-1,four +CCATCCGAAAGCAA-1,four +CCATCCGAACGACT-1,unassigned +CCATCCGAAGGTTC-1,three +CCATCCGATTCGCC-1,two +CCATCGTGAACGGG-1,four +CCATCGTGCTAGAC-1,unassigned +CCCAACACCTCGCT-1,three +CCCAACACGCATCA-1,four +CCCAACACTTTGTC-1,two +CCCAACTGCAATCG-1,two +CCCAGACTGCCTTC-1,two +CCCAGACTGGTTTG-1,three +CCCAGACTTTCGCC-1,one +CCCAGTTGCAGTTG-1,two +CCCAGTTGGGTACT-1,four +CCCAGTTGTCTATC-1,three +CCCGATTGTGTTTC-1,four +CCCGGAGAAGGGTG-1,two +CCCTACGAATTGGC-1,two +CCCTAGTGCAAAGA-1,four +CCCTCAGACACTTT-1,three +CCCTCAGACGAGAG-1,four +CCCTCAGAGGTCAT-1,four +CCCTGAACTAAAGG-1,three +CCCTGATGCAACCA-1,four +CCCTGATGCAAGCT-1,three +CCCTTACTAACCAC-1,two +CCCTTACTGCAGTT-1,four +CCGAAAACCTTGTT-1,unassigned +CCGACACTGGTTTG-1,four +CCGACTACCCAGTA-1,two +CCGACTACCGTGTA-1,four +CCGACTACTGAGGG-1,four +CCGATAGACCTAAG-1,four +CCGATAGAGTTGGT-1,two +CCGCGAGACACACA-1,two +CCGCGAGAGGTTCA-1,five +CCGCTATGGGACGA-1,three +CCGCTATGTGCAAC-1,two +CCGCTATGTGCACA-1,two +CCGGTACTGTCCTC-1,four +CCGTACACAAGCAA-1,four +CCGTACACAGCGTT-1,three +CCGTACACGTCATG-1,five +CCGTACACGTTGGT-1,two +CCGTACACTAACGC-1,five +CCTAAACTTTCGTT-1,four +CCTAAGGACCCAAA-1,four +CCTAAGGACTAGCA-1,two +CCTAAGGAGGGCAA-1,five +CCTAAGGATGATGC-1,four +CCTAAGGATGTCAG-1,four +CCTACCGACTCTTA-1,four +CCTACCGAGGGATG-1,three +CCTAGAGAGGTGAG-1,three +CCTATAACCAAAGA-1,four +CCTATAACGAGACG-1,one +CCTATAACTCAGAC-1,four +CCTATAACTGCATG-1,one +CCTCGAACACTTTC-1,five +CCTCGAACCCGTAA-1,unassigned +CCTCGAACGTATCG-1,four +CCTCGAACTTACTC-1,four +CCTCTACTCTTCGC-1,unassigned +CCTCTACTGGCATT-1,four +CCTGACTGAAGTAG-1,five +CCTGACTGGGGAGT-1,four +CCTGACTGTGTCTT-1,three +CCTGCAACACGTTG-1,two +CCTGGACTCGTGAT-1,four +CCTTAATGCCCAAA-1,three +CCTTAATGTTCTAC-1,one +CCTTCACTACGACT-1,four +CCTTCACTCAGTCA-1,three +CCTTCACTGGAGTG-1,two +CCTTTAGATTCATC-1,two +CGAACATGCCCTAC-1,two +CGAACATGTCAGAC-1,four +CGAAGACTGGAACG-1,five +CGAAGACTGTTACG-1,four +CGAAGGGAAACCTG-1,five +CGAAGGGATCCGAA-1,three +CGAAGTACCAACTG-1,three +CGAATCGAGGAGCA-1,two +CGAATCGAGGAGGT-1,four +CGACAAACCCATAG-1,four +CGACAAACCGACAT-1,three +CGACCACTAAAGTG-1,five +CGACCACTGCCAAT-1,five +CGACCGGAAGGTCT-1,four +CGACCGGATGGAAA-1,four +CGACCTTGCTAGTG-1,four +CGACGTCTATCGTG-1,four +CGACGTCTCGTGTA-1,four +CGACGTCTGAGGCA-1,four +CGACTCACGTCGTA-1,four +CGACTCACGTTGCA-1,four +CGACTCTGTGTGAC-1,unassigned +CGACTGCTTCCTCG-1,two +CGAGAACTAAGGCG-1,five +CGAGAACTACGTTG-1,four +CGAGAACTTGTTCT-1,two +CGAGATTGGACACT-1,five +CGAGATTGGCCATA-1,five +CGAGCCGAACACCA-1,four +CGAGCCGAGGCGAA-1,four +CGAGCGTGCTCCAC-1,four +CGAGCGTGGATACC-1,four +CGAGCGTGTATGCG-1,two +CGAGGAGACCTCCA-1,three +CGAGGAGATGTCGA-1,four +CGAGGCACCTATGG-1,five +CGAGGCACTATGCG-1,two +CGAGGCACTCTTCA-1,four +CGAGGCTGACGCTA-1,five +CGAGGCTGGCAGTT-1,two +CGAGGGCTACGACT-1,three +CGAGGGCTCGAATC-1,four +CGAGTATGTCACCC-1,one +CGATACGAACAGTC-1,two +CGATACGACAGGAG-1,unassigned +CGATACGATTCACT-1,three +CGATAGACCCGTAA-1,unassigned +CGATAGACCGTACA-1,four +CGATAGACGTAGGG-1,two +CGATAGACTGTTCT-1,four +CGATCAGAAGAACA-1,five +CGATCAGAGAGGGT-1,four +CGATCAGAGGTACT-1,four +CGATCAGATGTGAC-1,unassigned +CGATCCACCGGGAA-1,five +CGATCCACTTCCAT-1,three +CGCAAATGCTCGAA-1,four +CGCAACCTCCTTGC-1,four +CGCAACCTGGACGA-1,four +CGCACGGAGGACGA-1,three +CGCACGGATCTTTG-1,four +CGCACTACAGAATG-1,four +CGCACTACAGCCAT-1,four +CGCACTACATTGGC-1,four +CGCACTACTCGCCT-1,three +CGCACTACTCGTGA-1,four +CGCACTTGTCACGA-1,two +CGCAGGACAGATCC-1,five +CGCAGGACCTACTT-1,four +CGCAGGACTTGTCT-1,one +CGCAGGTGCACTGA-1,four +CGCAGGTGCCATAG-1,one +CGCAGGTGGGAACG-1,four +CGCATAGATCACGA-1,two +CGCCATACTGCAAC-1,four +CGCCATTGAGAGGC-1,four +CGCCATTGCTATGG-1,four +CGCCATTGGAGACG-1,four +CGCCATTGGAGCAG-1,three +CGCCATTGTACTGG-1,three +CGCCGAGAGCTTAG-1,five +CGCCTAACGAATGA-1,unassigned +CGCCTAACTGCTCC-1,four +CGCGAGACACAGCT-1,four +CGCGAGACAGGTCT-1,three +CGCGAGACGCTACA-1,unassigned +CGCGATCTCAGTCA-1,two +CGCGATCTGTTGAC-1,two +CGCGATCTTTCTTG-1,three +CGCGGATGGCCAAT-1,four +CGCTAAGAATGTCG-1,four +CGCTAAGACAACTG-1,four +CGCTAAGACCCTTG-1,four +CGCTACTGAACAGA-1,two +CGCTACTGAGAACA-1,four +CGCTACTGTGAGCT-1,four +CGCTACTGTTCCCG-1,five +CGCTCATGCATTTC-1,two +CGGAATTGGTTTGG-1,four +CGGAATTGTGGAGG-1,four +CGGACCGATGCGTA-1,five +CGGACCGATGGGAG-1,four +CGGACTCTAAACAG-1,three +CGGACTCTCCAATG-1,four +CGGACTCTCCTCGT-1,five +CGGAGGCTATTCCT-1,four +CGGAGGCTTGGATC-1,four +CGGATAACAACGAA-1,two +CGGATAACAGCTCA-1,four +CGGATAACTCAGTG-1,two +CGGCACGAACTCAG-1,five +CGGCACGAAGGGTG-1,four +CGGCACGACTACGA-1,one +CGGCATCTTAGAAG-1,five +CGGCATCTTCGTAG-1,two +CGGCCAGAAAGGTA-1,four +CGGCCAGAGAGGCA-1,five +CGGCGAACCAGTCA-1,two +CGGCGAACGACAAA-1,five +CGGCGAACGGTCTA-1,four +CGGCGAACTACTTC-1,five +CGGGACTGCGTGTA-1,four +CGGGACTGGAATAG-1,five +CGGGCATGACCCAA-1,five +CGGGCATGTCTCTA-1,five +CGGGCATGTTGTGG-1,four +CGGTAAACTCGCAA-1,unassigned +CGGTCACTGTTTGG-1,four +CGGTCACTTACTTC-1,four +CGTAACGATCGCCT-1,four +CGTACCACACACAC-1,four +CGTACCACACGTTG-1,four +CGTACCACCTCATT-1,four +CGTACCACGGAGCA-1,four +CGTACCTGGCATCA-1,five +CGTAGCCTCTCTCG-1,two +CGTAGCCTGCGAAG-1,four +CGTAGCCTGTATGC-1,one +CGTCAAGAAAGGTA-1,two +CGTCAAGAACGTGT-1,four +CGTCAAGACAGAGG-1,four +CGTCAAGACAGGAG-1,five +CGTCCATGCTCTTA-1,four +CGTCGACTTTCCGC-1,five +CGTGATGACGCTAA-1,four +CGTGATGAGGTTCA-1,four +CGTGCACTTATGGC-1,two +CGTGTAGAAAAACG-1,four +CGTGTAGACGATAC-1,five +CGTGTAGAGTTACG-1,five +CGTGTAGAGTTCAG-1,two +CGTGTAGATTCGGA-1,five +CGTTAGGAAACCAC-1,four +CGTTAGGATCATTC-1,two +CGTTATACCCTGAA-1,five +CGTTTAACTGGTCA-1,four +CTAAACCTCTGACA-1,five +CTAAACCTGTGCAT-1,three +CTAACACTAACGTC-1,four +CTAACACTAGTGCT-1,one +CTAACGGAACCGAT-1,five +CTAACGGATTTCTG-1,four +CTAACTACGGCAAG-1,four +CTAAGGACACCATG-1,four +CTAAGGACCGTTAG-1,four +CTAAGGACGCCATA-1,two +CTAAGGTGCCTAAG-1,three +CTAAGGTGTTGCAG-1,four +CTAAGGTGTTTCTG-1,four +CTAATAGAGCTATG-1,four +CTAATGCTTGTGGT-1,two +CTACAACTCCCGTT-1,five +CTACCTCTCAACCA-1,unassigned +CTACGCACACCTAG-1,one +CTACGCACTCTCCG-1,four +CTACGCACTGGTCA-1,five +CTACGGCTTTCTTG-1,two +CTACTATGAACCAC-1,two +CTACTATGATGTGC-1,two +CTACTATGCTAAGC-1,unassigned +CTACTATGTAAAGG-1,three +CTACTCCTATGTCG-1,five +CTACTCCTGCCATA-1,two +CTAGAGACACTTTC-1,four +CTAGAGACAGCATC-1,five +CTAGAGACTTTGGG-1,two +CTAGATCTCTCTAT-1,four +CTAGATCTTCGACA-1,four +CTAGGATGAGCCTA-1,three +CTAGGATGATCGTG-1,three +CTAGGCCTCTCAGA-1,four +CTAGGTGATGGTTG-1,one +CTAGTTACCAGAGG-1,four +CTAGTTACCGCATA-1,four +CTAGTTACGAAACA-1,five +CTATAAGATCGTTT-1,three +CTATACTGAGGTTC-1,two +CTATACTGCCAGTA-1,four +CTATACTGCGCTAA-1,two +CTATACTGCTACGA-1,five +CTATACTGTCTCAT-1,five +CTATACTGTTCGTT-1,four +CTATAGCTGTCACA-1,four +CTATAGCTTCGCTC-1,two +CTATAGCTTGCCTC-1,two +CTATCAACGAACTC-1,four +CTATCAACGCAGAG-1,four +CTATCAACTTTGGG-1,four +CTATCCCTCCACCT-1,two +CTATGTACGAGAGC-1,four +CTATGTACGCTTAG-1,four +CTATGTACTGTTTC-1,four +CTATGTTGAAAGCA-1,four +CTATGTTGTCCTCG-1,four +CTATGTTGTCTCGC-1,four +CTATTGACAAACGA-1,two +CTATTGACACTGGT-1,two +CTATTGACGGTGAG-1,one +CTATTGTGGCAAGG-1,four +CTCAATTGGTTCAG-1,four +CTCAATTGGTTGCA-1,two +CTCAGAGATAGAAG-1,four +CTCAGCACTCTAGG-1,four +CTCAGCACTGAACC-1,two +CTCAGCACTTGCAG-1,four +CTCAGCTGAACCTG-1,two +CTCAGCTGCAGTTG-1,four +CTCAGGCTCGTTGA-1,one +CTCAGGCTGCTAAC-1,four +CTCATTGACCTTAT-1,four +CTCATTGATGCTTT-1,one +CTCCACGAGAGATA-1,unassigned +CTCCATCTCTTAGG-1,four +CTCCATCTGACGAG-1,four +CTCCGAACAAGTGA-1,one +CTCCTACTGCCTTC-1,two +CTCGAAGATGTGGT-1,four +CTCGAAGATTAGGC-1,two +CTCGACTGCTCTAT-1,four +CTCGACTGGGTGAG-1,four +CTCGACTGGTTGAC-1,five +CTCGAGCTCTGGAT-1,five +CTCGCATGACTTTC-1,four +CTCGCATGCTTAGG-1,four +CTCTAAACCTCGAA-1,four +CTCTAAACGGCGAA-1,four +CTCTAATGTCCAAG-1,two +CTGAACGACAGTCA-1,four +CTGAACGATGAGGG-1,four +CTGAAGACCCAACA-1,four +CTGAAGACGTGCAT-1,three +CTGAAGTGAAGCCT-1,one +CTGAAGTGCAGCTA-1,three +CTGAAGTGGCTATG-1,four +CTGAAGTGTCCAGA-1,four +CTGAATCTGAATAG-1,four +CTGACAGAATCGTG-1,five +CTGACCACAGCAAA-1,four +CTGAGAACCGGGAA-1,four +CTGAGAACGTAAAG-1,five +CTGATACTAGTAGA-1,four +CTGATTTGGTGTTG-1,two +CTGCAGCTAACCGT-1,four +CTGCAGCTGACACT-1,four +CTGCAGCTGGATTC-1,two +CTGCAGCTTGGCAT-1,unassigned +CTGCCAACAGGAGC-1,five +CTGCCAACCAGCTA-1,four +CTGCCAACTAACCG-1,two +CTGCCAACTGCTCC-1,five +CTGCCAACTTGCAG-1,four +CTGCCAACTTGCTT-1,five +CTGCGACTCCACCT-1,four +CTGGAAACAAACGA-1,five +CTGGAAACATCGAC-1,four +CTGGATGACTGGAT-1,five +CTGGATGACTTGTT-1,four +CTGGATGATGTGAC-1,three +CTGGCACTCAAGCT-1,two +CTGTAACTAACCAC-1,four +CTGTAACTAGCGTT-1,four +CTGTATACGTAAAG-1,four +CTGTATACGTACGT-1,two +CTGTATACGTTGGT-1,four +CTGTGAGACAACCA-1,one +CTGTGAGACCTTGC-1,five +CTGTGAGACGAACT-1,two +CTGTGAGACTGTAG-1,two +CTTAAAGAACCTGA-1,four +CTTAACACCTGTAG-1,four +CTTAACACTATCGG-1,three +CTTAAGCTACCTAG-1,four +CTTAAGCTAGTACC-1,four +CTTAAGCTCATCAG-1,four +CTTAAGCTCCGCTT-1,three +CTTACAACTAACGC-1,four +CTTACAACTCCCGT-1,two +CTTACTGACGTACA-1,four +CTTAGACTAAACGA-1,four +CTTAGGGACTTGCC-1,four +CTTAGGGAGAATCC-1,four +CTTATCGACTCATT-1,four +CTTCACCTACCTGA-1,one +CTTCATGAAGCATC-1,three +CTTCATGAAGTACC-1,four +CTTCATGACCGAAT-1,five +CTTGAACTACGCAT-1,four +CTTGATTGAGGTTC-1,five +CTTGATTGATCTTC-1,two +CTTGATTGCATTCT-1,four +CTTGATTGTTTCGT-1,four +CTTGTATGACACCA-1,three +CTTGTATGCGCAAT-1,three +CTTTACGAGCGAAG-1,four +CTTTAGACCGTGAT-1,four +CTTTAGACGAGACG-1,four +CTTTAGACGATACC-1,four +CTTTAGACGTTGGT-1,four +CTTTAGACTCATTC-1,five +CTTTAGTGACGGGA-1,five +CTTTAGTGGGTGGA-1,four +CTTTCAGAGAAACA-1,four +CTTTGATGAGCACT-1,four +CTTTGATGTCTAGG-1,unassigned +CTTTGATGTGTCCC-1,four +CTTTGATGTGTGGT-1,four +GAAACAGAACTACG-1,four +GAAACAGAATCACG-1,four +GAAACAGACATTCT-1,four +GAAACCTGATCGTG-1,four +GAAACCTGATGCCA-1,four +GAAACCTGCTTATC-1,four +GAAACCTGGACTAC-1,one +GAAACCTGTGCTAG-1,four +GAAAGATGATTTCC-1,four +GAAAGATGCTGATG-1,one +GAAAGATGCTTCGC-1,two +GAAAGATGTAAGGA-1,four +GAAAGCCTACGTTG-1,three +GAAAGTGAAAGTGA-1,unassigned +GAAAGTGACCACAA-1,unassigned +GAAAGTGACTCAAG-1,four +GAAATACTACCAAC-1,two +GAAATACTCTTAGG-1,three +GAAATACTTCCTCG-1,four +GAACACACGTGCAT-1,two +GAACACACTGCCTC-1,four +GAACAGCTAACTGC-1,two +GAACAGCTCTCAGA-1,one +GAACCAACCACAAC-1,four +GAACCAACTTCCGC-1,five +GAACCTGAACGTGT-1,unassigned +GAACCTGAGAGACG-1,four +GAACCTGATGAACC-1,four +GAACGGGATACTTC-1,four +GAACTGTGACCTGA-1,one +GAACTGTGCCAGTA-1,two +GAAGAATGCAATCG-1,four +GAAGCGGACCTATT-1,two +GAAGCTACGAATGA-1,two +GAAGCTACGGTTTG-1,four +GAAGGGTGAAAGTG-1,four +GAAGGGTGCTTAGG-1,two +GAAGGTCTGAAAGT-1,four +GAAGGTCTGTTGCA-1,four +GAAGGTCTTAAAGG-1,unassigned +GAAGTAGACTCCCA-1,five +GAAGTAGATCCAAG-1,one +GAAGTCACCCTCGT-1,two +GAAGTCACCCTGTC-1,three +GAAGTCTGTCGCAA-1,three +GAAGTCTGTTCTGT-1,four +GAAGTGCTAAACGA-1,five +GAAGTGCTCCGCTT-1,two +GAAGTGCTTAACCG-1,four +GAATGCACCCTAAG-1,four +GAATGCACCTTCGC-1,five +GAATGCTGCGGTAT-1,four +GAATTAACGATAAG-1,four +GAATTAACGGTCAT-1,four +GAATTAACGTCGTA-1,five +GAATTAACTGAAGA-1,three +GACAACACAGGCGA-1,four +GACAACACATCGTG-1,four +GACAACACTCGCCT-1,four +GACAACTGAGGTTC-1,four +GACAGGGAAGAGTA-1,three +GACAGGGAATGCCA-1,four +GACAGTACGAGCTT-1,one +GACAGTACTTCGGA-1,one +GACAGTTGAGTAGA-1,three +GACATTCTCCACCT-1,unassigned +GACCAAACGACTAC-1,two +GACCAAACGTATCG-1,two +GACCCTACTAAAGG-1,three +GACCTAGACCTCAC-1,four +GACCTAGACGAGAG-1,four +GACCTCACAAGGTA-1,four +GACCTCACGTACGT-1,two +GACCTCTGCATCAG-1,four +GACGAACTCCCACT-1,four +GACGATTGCCAATG-1,three +GACGCCGACCTTCG-1,four +GACGCTCTCTCTCG-1,six +GACGGCACACGGGA-1,five +GACGGCACGAGATA-1,two +GACGTAACCTATGG-1,four +GACGTAACCTGTGA-1,four +GACGTAACTATGGC-1,four +GACGTATGTTGACG-1,four +GACGTATGTTTGCT-1,four +GACGTCCTACGGAG-1,four +GACGTCCTCTCAAG-1,two +GACGTCCTGATAAG-1,three +GACTACGATGGTCA-1,three +GACTCCTGCTCGCT-1,four +GACTCCTGGGTTAC-1,four +GACTCCTGTTATCC-1,four +GACTCCTGTTGGTG-1,two +GACTGAACCAATCG-1,two +GACTGATGTGATGC-1,four +GACTTTACATGCCA-1,four +GACTTTACGACAGG-1,three +GAGAAATGTTCTCA-1,three +GAGATAGAAAAAGC-1,two +GAGATCACGACAAA-1,one +GAGATGCTCTGGAT-1,five +GAGATGCTGAATGA-1,four +GAGCAGGATTCCCG-1,five +GAGCATACTTTGCT-1,four +GAGCGCACGCGTAT-1,two +GAGCGCACGGTGAG-1,four +GAGCGCTGAAGATG-1,five +GAGCGCTGTCTTAC-1,two +GAGCGGCTGGGAGT-1,three +GAGGACGACTCAGA-1,four +GAGGATCTGAAAGT-1,three +GAGGCAGACTTGCC-1,five +GAGGGAACACCAGT-1,four +GAGGGAACGAGGGT-1,four +GAGGGATGGGAAAT-1,four +GAGGGCCTTCACCC-1,four +GAGGGTGAAGAGTA-1,four +GAGGTACTACGGTT-1,four +GAGGTACTACTCAG-1,two +GAGGTACTGACACT-1,five +GAGGTACTGGGAGT-1,four +GAGGTACTTAGCGT-1,two +GAGGTGGAGTACGT-1,four +GAGGTGGATCCTCG-1,five +GAGGTTACTCGTTT-1,two +GAGGTTTGTAAGCC-1,two +GAGTCAACCATTCT-1,four +GAGTCAACGGGAGT-1,four +GAGTCTGATCGTGA-1,four +GAGTCTGATTTGGG-1,three +GAGTGACTCAGCTA-1,four +GAGTGACTCGGTAT-1,five +GAGTGACTCTTGCC-1,three +GAGTGACTGACTAC-1,three +GAGTGACTGTCTAG-1,four +GAGTGGGAGTCTTT-1,three +GAGTGGGATGCCCT-1,four +GAGTGGGATGCTGA-1,three +GAGTGTTGCTGTAG-1,four +GAGTGTTGTGGTCA-1,four +GAGTTGTGCATGGT-1,unassigned +GAGTTGTGCTGAGT-1,five +GAGTTGTGGCGAGA-1,two +GAGTTGTGGTAGCT-1,six +GAGTTGTGTATGCG-1,one +GATAAGGAGAAACA-1,two +GATAAGGATTCACT-1,two +GATACTCTATCGGT-1,four +GATACTCTTACTTC-1,four +GATACTCTTGACTG-1,two +GATAGAGAAGGGTG-1,five +GATAGAGACTGTGA-1,four +GATAGAGATCACGA-1,unassigned +GATAGCACCCATAG-1,four +GATAGCACGAAGGC-1,two +GATAGCACTTGTCT-1,four +GATATAACAAGGTA-1,four +GATATAACACGCAT-1,four +GATATATGCTGGAT-1,four +GATATATGTCCGTC-1,five +GATATATGTGGAGG-1,two +GATATCCTAGAAGT-1,four +GATATCCTCCCGTT-1,two +GATATTGACAGGAG-1,four +GATATTGACGAGTT-1,one +GATATTGAGCCAAT-1,four +GATCCCTGACCTTT-1,five +GATCCCTGTGTAGC-1,five +GATCCGCTGGTCAT-1,two +GATCGAACCGAGAG-1,unassigned +GATCGATGACTAGC-1,three +GATCGATGGTAAAG-1,three +GATCGATGTAAGGA-1,two +GATCGTGACACTAG-1,four +GATCGTGATTCACT-1,four +GATCTACTGGTGAG-1,four +GATCTACTTTGCAG-1,three +GATCTTACACCCAA-1,one +GATCTTACCCTACC-1,four +GATCTTACGAATAG-1,two +GATCTTACGAGATA-1,two +GATGCAACTCCAGA-1,four +GATGCCCTACGTAC-1,four +GATGCCCTCTCATT-1,four +GATGCCCTGGCAAG-1,two +GATGCCCTTTTGCT-1,four +GATTACCTTGTTCT-1,four +GATTCGGAACGACT-1,four +GATTCGGACAGGAG-1,two +GATTCGGAGAAGGC-1,one +GATTCTTGATTCGG-1,two +GATTCTTGCCGATA-1,four +GATTCTTGCGAGTT-1,three +GATTGGACCCGTTC-1,three +GATTGGACGGTGTT-1,three +GATTGGACTTTCGT-1,three +GATTGGTGTGTCAG-1,four +GATTTAGACACTCC-1,one +GATTTAGACTAAGC-1,three +GATTTAGATTCGTT-1,two +GATTTGCTAACGAA-1,four +GATTTGCTAACGGG-1,two +GCAACCCTCCTCGT-1,two +GCAACTGATTGCGA-1,four +GCAAGACTACTGGT-1,four +GCAAGACTAGGTCT-1,four +GCAAGACTCCCTTG-1,four +GCAATCGACTGCAA-1,four +GCAATCGAGACGTT-1,unassigned +GCAATCGATCCTTA-1,two +GCAATTCTCGTGTA-1,four +GCAATTCTTCTCCG-1,four +GCACAAACAATGCC-1,four +GCACAAACGGTACT-1,unassigned +GCACAATGGTGCAT-1,unassigned +GCACACCTGTGCTA-1,five +GCACCACTCATGAC-1,two +GCACCACTGTTTGG-1,two +GCACCACTTCCTTA-1,five +GCACCACTTTCGGA-1,two +GCACCTACGCGATT-1,four +GCACCTTGGCTGTA-1,four +GCACCTTGGGGAGT-1,two +GCACGGACCAGCTA-1,four +GCACGGTGACCTCC-1,four +GCACGGTGCTATGG-1,two +GCACTAGAACGGGA-1,four +GCACTAGAAGATGA-1,three +GCACTAGACCTTTA-1,one +GCACTAGACGTAAC-1,three +GCACTAGAGTCGTA-1,five +GCACTAGATGCAAC-1,four +GCACTGCTGAGGCA-1,five +GCAGATACAGCGTT-1,five +GCAGATACGACGGA-1,four +GCAGATACGCAGAG-1,four +GCAGCCGACAGTCA-1,four +GCAGCGTGCACTCC-1,four +GCAGCTCTCAATCG-1,four +GCAGCTCTGTTTCT-1,four +GCAGGGCTAAGAAC-1,four +GCAGGGCTAAGGGC-1,four +GCAGGGCTATCGAC-1,five +GCAGGGCTTGGGAG-1,five +GCAGTCCTAACTGC-1,four +GCAGTCCTCTCTTA-1,four +GCATCAGATGCGTA-1,unassigned +GCATGTGACAAGCT-1,four +GCATTGGAGAAGGC-1,four +GCCAAAACGAGGCA-1,five +GCCAAATGATCGAC-1,four +GCCAACCTACGGTT-1,four +GCCAACCTCGCCTT-1,four +GCCACGGAGGCGAA-1,two +GCCACGGATACTGG-1,four +GCCACTACGTCTTT-1,four +GCCCAACTACCGAT-1,four +GCCCAACTATGGTC-1,five +GCCCATACAGCGTT-1,unassigned +GCCGACGAACTCTT-1,two +GCCGAGTGCGTTGA-1,unassigned +GCCGGAACGAACTC-1,five +GCCGGAACGTTCTT-1,five +GCCGGAACTGCACA-1,five +GCCGGAACTTACTC-1,four +GCCGTACTACCTGA-1,four +GCCGTACTGGCAAG-1,two +GCCTACACAGTTCG-1,five +GCCTACACCACTGA-1,one +GCCTACACCTTGAG-1,four +GCCTAGCTACGGAG-1,three +GCCTAGCTCTATTC-1,four +GCCTAGCTTCTCAT-1,four +GCCTAGCTTCTCTA-1,one +GCCTCAACCATGGT-1,four +GCCTCAACTCTTTG-1,five +GCCTCATGTCTTAC-1,five +GCCTGACTCTCAAG-1,two +GCGAAGGAACTCTT-1,four +GCGAAGGATGCCAA-1,four +GCGAGAGAGGGACA-1,four +GCGAGCACTGTCGA-1,three +GCGAGCACTTGACG-1,four +GCGAGCACTTGCTT-1,unassigned +GCGATATGGTACGT-1,four +GCGATATGGTGTTG-1,four +GCGCACGAAGTCGT-1,four +GCGCACGACTTTAC-1,two +GCGCATCTAGGTCT-1,four +GCGCATCTGGTTAC-1,six +GCGCATCTTCGATG-1,four +GCGCATCTTGCTCC-1,five +GCGCATCTTTCTAC-1,four +GCGCGAACGTTCTT-1,three +GCGCGATGAACGGG-1,four +GCGCGATGGTGCAT-1,five +GCGGAGCTCCTGAA-1,four +GCGGCAACCCGATA-1,four +GCGGCAACGGAGGT-1,three +GCGGCAACTGTCGA-1,unassigned +GCGTAAACACGGTT-1,unassigned +GCGTAATGCACCAA-1,one +GCGTATGAACACCA-1,four +GCGTATGATGAGAA-1,two +GCTACAGAAAGGTA-1,four +GCTACAGAATCTTC-1,three +GCTACCTGAGAAGT-1,two +GCTACCTGATCACG-1,four +GCTACGCTAGAATG-1,four +GCTACGCTAGCTAC-1,four +GCTACGCTCCCTAC-1,four +GCTAGAACAGAGGC-1,three +GCTAGAACGGATCT-1,four +GCTAGAACTCCCGT-1,four +GCTAGATGAGCTCA-1,two +GCTAGATGGCGATT-1,five +GCTATACTAAGGCG-1,four +GCTATACTAGCGTT-1,five +GCTATACTCTCTTA-1,four +GCTATACTGGACGA-1,four +GCTCAAGAACCATG-1,three +GCTCAAGAAGTCAC-1,two +GCTCAGCTGTCTAG-1,four +GCTCCATGAGAAGT-1,five +GCTCCATGCCGAAT-1,four +GCTCGACTCTAGTG-1,two +GCTGATGAGGTATC-1,two +GCTTAACTACAGTC-1,two +GCTTAACTACTGGT-1,five +GCTTAACTGCTGAT-1,four +GCTTAACTTAGACC-1,two +GCTTAACTTCAGTG-1,four +GGAACACTCACTTT-1,two +GGAACACTTCAGAC-1,six +GGAACTACTACTTC-1,four +GGAACTTGAAGGTA-1,two +GGAACTTGAGAATG-1,four +GGAACTTGCTCCAC-1,two +GGAACTTGGGTAGG-1,three +GGAAGGACATCGGT-1,two +GGAAGGACCACTAG-1,four +GGAAGGACGAGGGT-1,four +GGAAGGACGCGAAG-1,four +GGAAGGTGGCGAGA-1,five +GGAATCTGAAGGGC-1,four +GGAATCTGAGGAGC-1,four +GGAATCTGCTTAGG-1,three +GGAATCTGCTTGTT-1,two +GGAATCTGGGAGGT-1,four +GGAATGCTTTCTAC-1,four +GGACAGGAAAGGGC-1,five +GGACAGGAGTGCTA-1,three +GGACAGGATCTCGC-1,four +GGACCCGAAGCTAC-1,two +GGACCGTGCTTACT-1,two +GGACCGTGGGAACG-1,unassigned +GGACCGTGTAACGC-1,four +GGACCTCTGTAAGA-1,two +GGACCTCTTTTCTG-1,five +GGACGAGAGTGTCA-1,two +GGACGCTGACGCAT-1,unassigned +GGACGCTGCTAGCA-1,four +GGACGCTGTCCTCG-1,five +GGAGAGACGTGAGG-1,three +GGAGCAGATTCAGG-1,three +GGAGCCACCTTCTA-1,three +GGAGCGCTACGCAT-1,two +GGAGCGCTCCGAAT-1,unassigned +GGAGGATGCCACCT-1,four +GGAGGATGGTTGAC-1,two +GGAGGATGTCAGTG-1,two +GGAGGCCTCGTTGA-1,five +GGAGGCCTTTCTTG-1,four +GGAGGTGATACGCA-1,four +GGAGGTGATCGCTC-1,four +GGATACTGCAGCTA-1,four +GGATACTGTCTAGG-1,four +GGATAGCTCGTCTC-1,four +GGATAGCTCTGAAC-1,five +GGATGTACCAAAGA-1,three +GGATGTACGCGAAG-1,four +GGATGTACGTCTTT-1,two +GGATGTACGTGTCA-1,four +GGATTTCTAGGTTC-1,four +GGATTTCTTTGTCT-1,two +GGCAAGGAAAAAGC-1,five +GGCAAGGAAGAAGT-1,four +GGCAAGGACTTGGA-1,unassigned +GGCAAGGAGGACTT-1,four +GGCAATACGCTAAC-1,three +GGCAATACGGCATT-1,five +GGCAATACGTTTCT-1,three +GGCACGTGGCTTAG-1,one +GGCACTCTTTTGTC-1,four +GGCATATGCTTATC-1,five +GGCATATGGGGAGT-1,six +GGCATATGTGTGAC-1,unassigned +GGCCACGACAGAGG-1,four +GGCCAGACTGGTTG-1,four +GGCCCAGAAAGTAG-1,two +GGCCGAACAACGAA-1,four +GGCCGAACGCAGAG-1,four +GGCCGAACGTAGGG-1,four +GGCCGAACTCTAGG-1,five +GGCCGATGCAGGAG-1,four +GGCCGATGCCGAAT-1,four +GGCCGATGTACTCT-1,five +GGCGACACTGCCCT-1,unassigned +GGCGACTGCGTAAC-1,four +GGCGCATGCCTAAG-1,one +GGCGCATGCTCCAC-1,four +GGCGCATGTGGAAA-1,four +GGCGGACTAGAGGC-1,four +GGCGGACTAGGAGC-1,five +GGCGGACTCTGACA-1,four +GGCGGACTCTTGGA-1,five +GGCGGACTTACTGG-1,four +GGCGGACTTGAACC-1,four +GGCTAAACACCTGA-1,five +GGCTAAACTCTTAC-1,four +GGCTAATGAGCACT-1,five +GGCTAATGGTCTAG-1,four +GGCTCACTACTCAG-1,two +GGGAACGAAGCTCA-1,one +GGGAACGACACAAC-1,three +GGGAACGAGTGTCA-1,two +GGGAAGTGTTGAGC-1,two +GGGACCACACGTTG-1,five +GGGACCACAGAACA-1,four +GGGACCACGAATAG-1,four +GGGACCACGTCATG-1,four +GGGACCACTCAAGC-1,four +GGGACCACTCGTGA-1,four +GGGACCACTGCATG-1,four +GGGACCTGACCCTC-1,two +GGGACCTGCTTGCC-1,four +GGGACCTGTGGAGG-1,three +GGGATGGACGACAT-1,three +GGGATGGATACTTC-1,four +GGGATGGATGGTTG-1,two +GGGATTACGTCTAG-1,four +GGGCAAGATGCATG-1,two +GGGCACACGGTGAG-1,four +GGGCACACGTTGCA-1,four +GGGCAGCTTGGGAG-1,five +GGGCAGCTTTTCTG-1,four +GGGCCAACCTTGGA-1,unassigned +GGGCCAACGCGTTA-1,three +GGGCCAACTACGCA-1,two +GGGCCAACTCCAAG-1,three +GGGCCATGATGGTC-1,unassigned +GGGCCATGTTGACG-1,four +GGGTAACTCAGCTA-1,four +GGGTAACTCTAGTG-1,three +GGGTAACTCTGGAT-1,four +GGGTTAACGTGCAT-1,five +GGTAAAGAGCTAAC-1,five +GGTACAACTGCAAC-1,four +GGTACATGAAAGCA-1,three +GGTACATGAGCTCA-1,four +GGTACATGCGGTAT-1,four +GGTACATGGTTACG-1,four +GGTACATGTGGGAG-1,four +GGTACTGAACTCTT-1,two +GGTAGTACACCACA-1,five +GGTAGTACACTAGC-1,two +GGTAGTACCCTGTC-1,four +GGTAGTACGCCATA-1,two +GGTAGTACTGTCTT-1,unassigned +GGTATCGAGACAAA-1,unassigned +GGTATCGATGAACC-1,three +GGTCAAACCAAAGA-1,four +GGTCTAGAGAAACA-1,two +GGTCTAGATAGCGT-1,two +GGTGATACCGACTA-1,three +GGTGATACGACTAC-1,two +GGTGATACTGTTTC-1,four +GGTGGAGAAACGGG-1,five +GGTGGAGAAGTAGA-1,four +GGTGGAGACAGATC-1,five +GGTGGAGATCGATG-1,four +GGTGGAGATCTCTA-1,two +GGTGGAGATTACTC-1,five +GGTTTACTACGCAT-1,five +GTAACGTGACCTCC-1,two +GTAACGTGATCGGT-1,three +GTAACGTGCAGCTA-1,three +GTAACGTGGTTGAC-1,three +GTAAGCACAACGGG-1,four +GTAAGCACTCATTC-1,three +GTAAGCTGGTACCA-1,two +GTAATAACCTTCTA-1,two +GTAATAACGTTGTG-1,four +GTACCCTGACAGTC-1,one +GTACCCTGGAGCTT-1,four +GTACCCTGTCCTTA-1,five +GTACCCTGTGAACC-1,four +GTACGTGAACGTTG-1,five +GTACTTTGTCGACA-1,five +GTAGACTGAGATGA-1,two +GTAGACTGTATTCC-1,four +GTAGCAACAGTCGT-1,two +GTAGCAACCATTTC-1,three +GTAGCAACGGTAGG-1,two +GTAGCATGCACTCC-1,three +GTAGCATGTAAGCC-1,two +GTAGCCCTGACGTT-1,four +GTAGCTGAAGCTAC-1,one +GTAGCTGAATTCGG-1,four +GTAGGTACACGGGA-1,two +GTAGTGACCTCATT-1,four +GTAGTGTGAGCGGA-1,five +GTAGTGTGAGGCGA-1,two +GTAGTGTGTGGTTG-1,four +GTATCACTGGTAGG-1,two +GTATCTACAGAAGT-1,five +GTATCTACGACGAG-1,five +GTATCTACGTTACG-1,four +GTATTAGAAACAGA-1,four +GTATTAGAGGTCTA-1,four +GTATTCACACAGCT-1,four +GTATTCACGGGTGA-1,five +GTCAACGACACTGA-1,four +GTCAACGAGTGTAC-1,two +GTCAATCTACACCA-1,five +GTCAATCTGTAGCT-1,five +GTCAATCTTGTGGT-1,four +GTCACCTGCCTCCA-1,four +GTCACCTGTCCCGT-1,four +GTCATACTAATCGC-1,three +GTCATACTGCGATT-1,five +GTCATACTTCGCCT-1,six +GTCATACTTTACCT-1,five +GTCATACTTTGACG-1,four +GTCCAAGAAAAACG-1,four +GTCCACTGACCTCC-1,four +GTCCACTGGGTACT-1,one +GTCCAGCTACGGGA-1,four +GTCCCATGTGGTGT-1,two +GTCGAATGAAGGCG-1,two +GTCGACCTGAATGA-1,five +GTCGACCTGTTCAG-1,four +GTCGCACTTGAGAA-1,three +GTCTAACTGGTCTA-1,two +GTCTAGGAGCTTCC-1,two +GTGAACACACTCTT-1,four +GTGAACACTCAGGT-1,four +GTGACCCTTAAGCC-1,unassigned +GTGATGACAAGTGA-1,four +GTGATGACCTGAGT-1,three +GTGATGACGGTTTG-1,five +GTGATTCTCATTTC-1,four +GTGATTCTCTCTCG-1,two +GTGATTCTGGTTCA-1,five +GTGATTCTGTCGAT-1,four +GTGATTCTTAGCGT-1,three +GTGCCACTCAGGAG-1,four +GTGGATTGCACTAG-1,one +GTGGATTGCGGAGA-1,four +GTGGATTGTAACGC-1,four +GTGTACGATCAGTG-1,four +GTGTAGTGGGTACT-1,four +GTGTATCTAGCCTA-1,three +GTGTATCTAGTAGA-1,five +GTGTATCTGTTACG-1,three +GTGTCAGAAGCGTT-1,five +GTTAAAACCGAGAG-1,five +GTTAAATGCTCGAA-1,four +GTTAAATGTCGACA-1,three +GTTAACCTAGCTAC-1,four +GTTAACCTTGCTTT-1,unassigned +GTTAGGTGCACTCC-1,four +GTTAGGTGCCAGTA-1,two +GTTAGGTGCCCAAA-1,one +GTTAGGTGGAACTC-1,three +GTTAGTCTAAGAAC-1,unassigned +GTTATAGAGGACAG-1,four +GTTATGCTTTCATC-1,four +GTTCAACTGGGACA-1,five +GTTCAACTTATGCG-1,four +GTTGACGAGCCCTT-1,five +GTTGACGATATCGG-1,five +GTTGAGTGGTCTTT-1,two +GTTGAGTGTGCTTT-1,four +GTTGATCTGGGACA-1,four +GTTGATCTTTTCAC-1,five +GTTGGATGTTTACC-1,one +GTTGTACTATTCCT-1,two +GTTGTACTTTTGGG-1,four +GTTTAAGACCATGA-1,two +GTTTAAGACTGTCC-1,five +TAAACAACCAACCA-1,four +TAAACAACGAATCC-1,three +TAAAGACTCAGGAG-1,unassigned +TAAATCGATGAGGG-1,four +TAACAATGTGCCCT-1,three +TAACACCTTCGCTC-1,two +TAACACCTTCGTAG-1,two +TAACACCTTGTTTC-1,six +TAACATGACACTAG-1,three +TAACCGGACTTACT-1,four +TAACGTCTCAACCA-1,two +TAACGTCTCATTGG-1,four +TAACTAGAATTTCC-1,four +TAACTAGACTTAGG-1,four +TAACTAGATCTGGA-1,four +TAACTCACGAGGAC-1,two +TAACTCACGTACAC-1,three +TAACTCACGTATCG-1,two +TAACTCACTCTACT-1,five +TAAGAACTGTGTCA-1,two +TAAGAGGACTAAGC-1,three +TAAGAGGACTTGTT-1,five +TAAGATACGGTTCA-1,two +TAAGATTGCGTAGT-1,four +TAAGATTGTTGCTT-1,four +TAAGCGTGAGGTTC-1,two +TAAGCGTGGACAAA-1,two +TAAGCGTGGGAAAT-1,four +TAAGCGTGTGCTCC-1,one +TAAGGCTGCCATGA-1,unassigned +TAAGGCTGCTGCTC-1,four +TAAGGCTGTCTCGC-1,two +TAAGGGCTGCTGTA-1,four +TAAGGGCTTTACTC-1,four +TAAGTAACCGAGAG-1,three +TAAGTAACCTCCAC-1,three +TAAGTAACCTGTAG-1,four +TAAGTAACTTGTCT-1,four +TAATGATGAGCGGA-1,four +TAATGCCTCATGAC-1,four +TAATGCCTCGTCTC-1,five +TAATGTGAAGATGA-1,four +TAATGTGACTGCAA-1,two +TAATGTGATTACTC-1,four +TACAAATGGGTACT-1,four +TACAATGAAAACAG-1,two +TACAATGACTTAGG-1,four +TACAATGATGCTAG-1,five +TACACACTCACACA-1,four +TACACACTCTTACT-1,four +TACATAGAACGCAT-1,five +TACATCACACGGGA-1,four +TACATCACCTGTTT-1,one +TACATCACGCTAAC-1,three +TACATCACTGAACC-1,one +TACCATTGAGGTTC-1,four +TACCATTGCGGGAA-1,two +TACCATTGGGGATG-1,two +TACCATTGTGAGGG-1,two +TACCGGCTGTTGGT-1,four +TACGAGTGATCTCT-1,four +TACGAGTGATGCTG-1,two +TACGAGTGCGGAGA-1,four +TACGAGTGGTTGGT-1,four +TACGATCTAGTGTC-1,four +TACGATCTCACTGA-1,two +TACGATCTTACGAC-1,four +TACGCAGAGAATCC-1,five +TACGCCACATTCCT-1,four +TACGCCACTCCCAC-1,one +TACGCCACTCCGAA-1,three +TACGGAACGCGTTA-1,three +TACGGCCTGGGACA-1,five +TACGTACTACGGAG-1,unassigned +TACGTACTCAGTTG-1,four +TACGTACTCCCGTT-1,three +TACGTTACAGAAGT-1,four +TACGTTACCAAGCT-1,two +TACTAAGAAAGGTA-1,three +TACTAAGAATCACG-1,four +TACTAAGATGATGC-1,two +TACTAAGATTGCGA-1,four +TACTACACGAGAGC-1,four +TACTACACTTACCT-1,four +TACTACTGAACCTG-1,two +TACTACTGATGTCG-1,five +TACTACTGATTCTC-1,four +TACTACTGTATGGC-1,four +TACTCAACGGTCTA-1,two +TACTCAACTGCTAG-1,three +TACTCCCTCAGTTG-1,four +TACTCTGAATCGAC-1,five +TACTCTGACGAGTT-1,four +TACTCTGATTGACG-1,three +TACTGGGATCGATG-1,four +TACTGTTGAAAGCA-1,four +TACTGTTGAGGCGA-1,five +TACTGTTGCTGAAC-1,three +TACTTGACTCCTCG-1,five +TACTTGACTGGTGT-1,three +TACTTTCTTTTGGG-1,four +TAGAAACTAATCGC-1,four +TAGAAACTGCTTCC-1,two +TAGAAACTGGGATG-1,five +TAGAATTGCGACAT-1,four +TAGAATTGTATCGG-1,four +TAGACGTGCTTGAG-1,four +TAGACGTGTCGCTC-1,four +TAGAGCACCTTACT-1,three +TAGATTGACTTGTT-1,three +TAGATTGAGGCATT-1,five +TAGCATCTCAGCTA-1,five +TAGCATCTCCCTCA-1,four +TAGCATCTGCTGTA-1,one +TAGCATCTGGGACA-1,four +TAGCATCTTGTCGA-1,four +TAGCCCACAAAAGC-1,four +TAGCCCACAGCCAT-1,five +TAGCCCACAGCTAC-1,five +TAGCCCACCCACAA-1,four +TAGCCCTGCGGAGA-1,five +TAGCCGCTTACGAC-1,four +TAGCCGCTTACTTC-1,five +TAGCCGCTTTCCAT-1,two +TAGCTACTGAATAG-1,three +TAGCTACTGTAGCT-1,four +TAGCTACTTTTGCT-1,five +TAGGACTGTGCTGA-1,four +TAGGAGCTAAGGCG-1,three +TAGGAGCTGAGGGT-1,four +TAGGAGCTTGCATG-1,three +TAGGCAACCGTCTC-1,three +TAGGCATGCTCTCG-1,unassigned +TAGGCATGGCGAGA-1,five +TAGGCTGATGCCTC-1,two +TAGGGACTGAACTC-1,five +TAGGTCGACACTGA-1,four +TAGGTCGAGGATCT-1,three +TAGGTGACACACTG-1,four +TAGGTGACACGTTG-1,three +TAGGTGTGTTCTGT-1,four +TAGGTTCTGAAGGC-1,four +TAGGTTCTTCTTAC-1,one +TAGGTTCTTGCTGA-1,two +TAGTAAACCTCGCT-1,four +TAGTAAACGTCACA-1,four +TAGTAATGAGATCC-1,four +TAGTACCTAAGAAC-1,two +TAGTATGATCTTAC-1,three +TAGTATGATTCTCA-1,four +TAGTCTTGGCTGTA-1,four +TAGTCTTGGGACTT-1,four +TAGTCTTGTGGAAA-1,five +TAGTGGTGAAGTGA-1,five +TAGTTAGAACCACA-1,five +TAGTTAGATGAACC-1,four +TATAAGACAACAGA-1,four +TATAAGACAGCTCA-1,one +TATAAGTGACACCA-1,four +TATAAGTGTATCGG-1,two +TATAAGTGTGGTGT-1,one +TATACAGAACCCTC-1,five +TATACAGAAGAACA-1,four +TATACAGATCCAGA-1,four +TATACCACCTGATG-1,three +TATACGCTACCAAC-1,four +TATAGATGGACGGA-1,four +TATAGATGTTCCGC-1,one +TATCCAACCAGCTA-1,two +TATCCAACTCTCTA-1,unassigned +TATCGACTACTAGC-1,five +TATCGACTCGATAC-1,two +TATCGTACAGATGA-1,unassigned +TATCGTACATTCCT-1,two +TATCTCGAGAGATA-1,four +TATCTGACAGGTTC-1,five +TATCTGACTGTTTC-1,five +TATCTTCTAAACAG-1,four +TATGAATGGAGGAC-1,five +TATGAATGTTTGCT-1,one +TATGCGGATAACCG-1,four +TATGGGTGCATCAG-1,five +TATGGGTGCTAGCA-1,three +TATGGTCTCTACCC-1,four +TATGTCACGGAACG-1,three +TATGTCACTAACCG-1,unassigned +TATGTCACTTCTCA-1,three +TATGTGCTCCGATA-1,two +TATGTGCTGGATTC-1,four +TATTGCTGAAGAAC-1,two +TATTGCTGCCGTTC-1,four +TATTGCTGTCTGGA-1,four +TATTGCTGTGCACA-1,five +TATTTCCTATTGGC-1,two +TATTTCCTGGAGGT-1,five +TATTTCCTGGTGTT-1,four +TCAACACTGTTTGG-1,five +TCAAGGACAGCGTT-1,four +TCAAGGACATTCTC-1,five +TCAAGGACGGTGTT-1,two +TCAATCACACTCTT-1,five +TCAATCACAGTCGT-1,one +TCACAACTATGTGC-1,four +TCACAACTTTGCTT-1,two +TCACATACACTTTC-1,four +TCACATACAGGGTG-1,four +TCACCCGAGACGGA-1,two +TCACCGTGCTCGCT-1,one +TCACCTCTACGACT-1,five +TCACCTCTTCCAAG-1,one +TCACGAGAGGAGGT-1,three +TCACTATGGGGCAA-1,five +TCACTATGGTTGTG-1,three +TCAGACGACGCTAA-1,four +TCAGACGACGTTAG-1,five +TCAGAGACTCCAGA-1,four +TCAGCAGACTCCAC-1,four +TCAGCGCTCTAGTG-1,five +TCAGCGCTGGATCT-1,three +TCAGCGCTGGTATC-1,five +TCAGGATGAAGTAG-1,four +TCAGGATGCCTTTA-1,one +TCAGTGGAAGATCC-1,five +TCAGTTACCTACGA-1,two +TCAGTTACTAGAAG-1,four +TCATCAACCCGATA-1,four +TCATCAACTGTTCT-1,two +TCATCATGCAGTTG-1,four +TCATCCCTTACTGG-1,unassigned +TCATTCGATACAGC-1,one +TCCACGTGGAAACA-1,unassigned +TCCACTCTACACTG-1,four +TCCACTCTGAGCTT-1,two +TCCACTCTTACTTC-1,three +TCCATAACAAAGTG-1,four +TCCATAACCGTAGT-1,three +TCCATAACGATGAA-1,four +TCCATAACTACGCA-1,four +TCCATCCTCCCTAC-1,five +TCCCACGATCATTC-1,four +TCCCATCTCAAAGA-1,two +TCCCGAACACAGTC-1,five +TCCCGAACTTCGCC-1,one +TCCCGATGAGATCC-1,four +TCCCGATGCCTGAA-1,two +TCCCTACTCAACTG-1,three +TCCGAAGACAATCG-1,two +TCCGAAGACGTTAG-1,four +TCCGGACTGAGGTG-1,four +TCCGGACTGTACGT-1,five +TCCTAAACATCGAC-1,four +TCCTAAACCGAGAG-1,four +TCCTAAACCGCATA-1,five +TCCTAATGGTTTGG-1,five +TCCTACCTGTCGTA-1,four +TCCTATGAAAAGCA-1,five +TCGAATCTCTGGTA-1,five +TCGACCTGCCGATA-1,one +TCGACGCTTCTATC-1,two +TCGACGCTTTGACG-1,four +TCGAGAACGACAGG-1,four +TCGAGAACGTTAGC-1,four +TCGAGCCTATCAGC-1,three +TCGAGCCTGCGAGA-1,five +TCGAGCCTTGTGAC-1,four +TCGATACTATTCCT-1,unassigned +TCGATACTTGCACA-1,four +TCGATTTGATGCCA-1,two +TCGATTTGCACTCC-1,four +TCGATTTGCAGCTA-1,five +TCGATTTGCCTACC-1,two +TCGATTTGTCGTGA-1,three +TCGCAGCTAGATCC-1,five +TCGCCATGAGACTC-1,four +TCGCCATGTGGTCA-1,three +TCGGACCTAACAGA-1,four +TCGGACCTATAAGG-1,three +TCGGACCTGTACAC-1,four +TCGGTAGAGTAGGG-1,four +TCGGTAGATCCCAC-1,three +TCGTAGGATCGACA-1,four +TCGTTATGGACAAA-1,four +TCTAACACCAGTTG-1,five +TCTAACACGAGCAG-1,unassigned +TCTAACTGAACCAC-1,four +TCTAAGCTAATGCC-1,four +TCTAAGCTTAGTCG-1,five +TCTAAGCTTCTAGG-1,one +TCTAAGCTTGTTCT-1,two +TCTAAGCTTTCGCC-1,three +TCTACAACGACTAC-1,two +TCTAGACTTAGAAG-1,two +TCTAGTTGCACCAA-1,two +TCTATGTGAAGAGT-1,two +TCTATGTGAGTCTG-1,five +TCTCAAACCTAAGC-1,three +TCTCTAGAATTTCC-1,four +TCTGATACACGTGT-1,four +TCTGATACTCGCCT-1,four +TCTTACGAACCTGA-1,two +TCTTCAGAGCTACA-1,two +TCTTGATGCGGAGA-1,five +TGAAATTGGTGAGG-1,two +TGAACCGAAAACGA-1,four +TGAACCGACTACTT-1,one +TGAACCGATTCGGA-1,one +TGAAGCACTCACGA-1,five +TGAAGCTGAACGAA-1,four +TGAAGCTGCATGGT-1,three +TGAAGCTGCGTAAC-1,four +TGAATAACCACTTT-1,four +TGAATAACTCCCAC-1,five +TGACACGACCTTAT-1,three +TGACCAGACAACCA-1,four +TGACCAGAGGATTC-1,four +TGACCGCTAAAAGC-1,four +TGACCGCTCTGCAA-1,four +TGACGATGCAAAGA-1,two +TGACGCCTGTACCA-1,unassigned +TGACGCCTTTACTC-1,four +TGACTGGAAGAGAT-1,two +TGACTGGACCGTAA-1,four +TGACTGGACGCAAT-1,four +TGACTGGAGGACAG-1,three +TGACTGGATTCTCA-1,four +TGACTTACACACCA-1,four +TGACTTACAGTCTG-1,two +TGACTTTGCGCATA-1,four +TGACTTTGTTTGTC-1,four +TGAGACACAAGGTA-1,unassigned +TGAGACACTCAAGC-1,four +TGAGCTGAATGCTG-1,one +TGAGCTGACTGGAT-1,two +TGAGCTGATGCTAG-1,unassigned +TGAGGACTCTCATT-1,two +TGAGGACTTCATTC-1,four +TGAGGTACGAACCT-1,one +TGAGTCGAGTTACG-1,three +TGAGTGACTGAGCT-1,four +TGATAAACGAATCC-1,four +TGATAAACTCCGTC-1,five +TGATAAACTTTCAC-1,four +TGATACCTCACTAG-1,two +TGATACCTGTTGGT-1,four +TGATACCTTATGCG-1,four +TGATACCTTGAAGA-1,four +TGATATGAACCTTT-1,four +TGATCACTAGCATC-1,five +TGATCACTCTCGCT-1,four +TGATCACTTCTACT-1,three +TGATCGGACTGACA-1,one +TGATCGGAGGAGCA-1,three +TGATCGGATATGCG-1,one +TGATTAGACATTGG-1,four +TGATTAGATGACTG-1,three +TGATTAGATGCTAG-1,four +TGATTCACTATGCG-1,four +TGATTCACTGTCAG-1,three +TGATTCTGCCGAAT-1,four +TGATTCTGCTCTTA-1,four +TGCAAGTGAGAACA-1,four +TGCAAGTGGGTAGG-1,two +TGCAATCTTCAGGT-1,three +TGCACAGACGACAT-1,five +TGCCAAGAGCAGTT-1,one +TGCCAAGATCTCTA-1,four +TGCCACTGAACGTC-1,four +TGCCACTGCGATAC-1,five +TGCCAGCTTGGCAT-1,four +TGCCCAACAGCAAA-1,four +TGCCCAACCGCATA-1,one +TGCCGACTCTCCCA-1,three +TGCGAAACAGTCAC-1,three +TGCGAAACGTTGCA-1,three +TGCGATGAACGGTT-1,four +TGCGATGACCTCGT-1,four +TGCGATGACTAGTG-1,three +TGCGATGACTGCTC-1,three +TGCGATGACTTGCC-1,three +TGCGATGAGTGCTA-1,two +TGCGCACTCTTGAG-1,two +TGCGTAGAATAAGG-1,two +TGCGTAGACGGGAA-1,three +TGCGTAGATGGTCA-1,one +TGCTAGGAAACCGT-1,five +TGCTAGGATAGTCG-1,four +TGCTATACGGTTCA-1,two +TGCTATACTGCTGA-1,five +TGCTGAGAGAGCAG-1,five +TGCTGAGATTATCC-1,four +TGGAAAGACTCTCG-1,three +TGGAAAGAGCGATT-1,four +TGGAAAGAGGTCAT-1,one +TGGAAAGATATGGC-1,four +TGGAACACAAACAG-1,five +TGGAACACGCTAAC-1,four +TGGAAGCTCAGATC-1,four +TGGACCCTACACTG-1,two +TGGACCCTCATGGT-1,five +TGGACCCTGGTACT-1,two +TGGACTGAGTATGC-1,four +TGGAGACTATCAGC-1,two +TGGAGACTTCAAGC-1,four +TGGAGACTTGACCA-1,five +TGGAGGGACGGAGA-1,two +TGGAGGGAGCTATG-1,four +TGGATCGATAAAGG-1,four +TGGATGTGACCTAG-1,three +TGGATGTGTGAAGA-1,three +TGGATTCTCATACG-1,five +TGGCAATGCTTGTT-1,three +TGGCACCTTCACGA-1,four +TGGCACCTTCAGTG-1,two +TGGGTATGAAGAGT-1,four +TGGGTATGCACAAC-1,four +TGGGTATGGTACGT-1,three +TGGGTATGTTTGGG-1,two +TGGTAGACATGCCA-1,four +TGGTAGACCCTCAC-1,five +TGGTAGACCTGATG-1,four +TGGTAGTGCACTGA-1,unassigned +TGGTATCTAAACAG-1,four +TGGTATCTCTTCCG-1,four +TGGTCAGACCCAAA-1,four +TGGTTACTGACGTT-1,two +TGGTTACTGTTCTT-1,four +TGTAACCTAGAGGC-1,four +TGTAACCTTGCCTC-1,two +TGTAATGACACAAC-1,five +TGTAATGAGGTAAA-1,five +TGTACTTGCTCTAT-1,one +TGTAGGTGCGAGAG-1,three +TGTAGGTGCTATGG-1,four +TGTAGGTGCTCTAT-1,unassigned +TGTAGGTGTGCTGA-1,four +TGTAGTCTTCCAGA-1,four +TGTAGTCTTGCACA-1,five +TGTATCTGTTAGGC-1,four +TGTATGCTCATGGT-1,three +TGTATGCTGTAGGG-1,four +TGTATGCTTTCATC-1,four +TGTCAGGAATACCG-1,four +TGTCAGGAGATGAA-1,one +TGTCTAACCCCTTG-1,four +TGTGACGATTCTCA-1,five +TGTGAGACTGTCAG-1,one +TGTGAGACTTGAGC-1,four +TGTGAGTGACCACA-1,four +TGTGAGTGAGTGCT-1,unassigned +TGTGAGTGGAGATA-1,five +TGTGATCTCTCTAT-1,three +TGTGATCTGACACT-1,four +TGTGGATGGCCAAT-1,three +TGTTAAGACAAAGA-1,two +TGTTAAGATAAGGA-1,two +TGTTACACCGCATA-1,unassigned +TGTTACACGACTAC-1,one +TGTTACTGGCTACA-1,five +TGTTACTGTAGTCG-1,five +TTAACCACCGTAAC-1,four +TTAACCACTAAGGA-1,five +TTAACCACTCAGAC-1,five +TTACACACGTGTTG-1,four +TTACACACTCCTAT-1,four +TTACCATGAATCGC-1,two +TTACCATGGTTGAC-1,four +TTACCATGTGTCTT-1,four +TTACCATGTTGTGG-1,three +TTACGACTGAGAGC-1,four +TTACGACTTGACAC-1,two +TTACGTACGTTCAG-1,six +TTACTCGAAGAATG-1,two +TTACTCGACGCAAT-1,unassigned +TTACTCGAGGGTGA-1,four +TTACTCGATCTACT-1,five +TTAGAATGTGGTGT-1,four +TTAGAATGTGTAGC-1,four +TTAGACCTCCTACC-1,four +TTAGACCTCCTTTA-1,two +TTAGCTACAACCGT-1,four +TTAGCTACTGTCCC-1,unassigned +TTAGCTACTTTCGT-1,four +TTAGGGACGCGAAG-1,five +TTAGGGTGCTGGAT-1,three +TTAGGGTGTCCTGC-1,two +TTAGGTCTACTTTC-1,two +TTAGTCACCAGTTG-1,four +TTAGTCTGAAAGCA-1,three +TTAGTCTGCCAACA-1,five +TTAGTCTGTGCACA-1,four +TTATCCGACTAGTG-1,three +TTATCCGAGAAAGT-1,one +TTATGAGAGATAAG-1,four +TTATGCACGTCACA-1,three +TTATGGCTTATGGC-1,unassigned +TTATTCCTATGCTG-1,two +TTATTCCTGGACAG-1,three +TTATTCCTGGTACT-1,three +TTATTCCTTCGTGA-1,four +TTCAAAGATAAAGG-1,five +TTCAACACAACAGA-1,four +TTCAACACCCCAAA-1,two +TTCAACACGGACGA-1,one +TTCAAGCTAAGAAC-1,two +TTCAAGCTAGATGA-1,four +TTCAAGCTGTTGAC-1,unassigned +TTCAAGCTTGATGC-1,four +TTCAAGCTTTCGCC-1,four +TTCACAACCCGTTC-1,two +TTCACAACGTCTGA-1,one +TTCAGACTACCCAA-1,four +TTCAGACTCTCGAA-1,four +TTCAGTACCGACTA-1,four +TTCAGTACTCAAGC-1,four +TTCAGTACTCCTAT-1,two +TTCAGTTGCCAAGT-1,four +TTCAGTTGTCCTTA-1,unassigned +TTCAGTTGTCTAGG-1,four +TTCAGTTGTCTCGC-1,three +TTCATCGAGGTGGA-1,two +TTCATGTGTGGTGT-1,four +TTCATTCTATGTCG-1,two +TTCATTCTTCTCTA-1,four +TTCCAAACCTATGG-1,two +TTCCAAACCTCCCA-1,three +TTCCAAACTCCCAC-1,five +TTCCAAACTTGACG-1,five +TTCCATGACGAGAG-1,two +TTCCATGACTGTCC-1,four +TTCCCACTTGAGGG-1,five +TTCCCACTTGTCTT-1,two +TTCCTAGAAAGTGA-1,four +TTCCTAGACTAGTG-1,one +TTCGAGGACTCTAT-1,four +TTCGAGGAGGGCAA-1,four +TTCGAGGATAGAAG-1,one +TTCGATTGAGCATC-1,four +TTCGGAGAATGCCA-1,two +TTCGGAGATGTGCA-1,four +TTCGTATGAAAAGC-1,five +TTCGTATGGATAGA-1,five +TTCGTATGGTCTGA-1,four +TTCGTATGTCCTTA-1,two +TTCTACGAACGTAC-1,four +TTCTACGAGTTGGT-1,four +TTCTAGTGACACGT-1,four +TTCTAGTGCATGAC-1,two +TTCTAGTGGAGAGC-1,five +TTCTAGTGGTCACA-1,unassigned +TTCTCAGAAGAGAT-1,four +TTCTCAGAAGCATC-1,three +TTCTCAGATGGAGG-1,four +TTCTGATGGAGACG-1,five +TTCTTACTCTGGAT-1,one +TTGAACCTCCTTGC-1,three +TTGAATGAACTACG-1,one +TTGAATGACTTACT-1,four +TTGAATGATCTCAT-1,three +TTGACACTCTGTAG-1,four +TTGACACTGATAAG-1,three +TTGAGGACAGAACA-1,two +TTGAGGACTACGCA-1,unassigned +TTGAGGTGGACGGA-1,four +TTGCATTGAGCTAC-1,five +TTGCATTGCTAAGC-1,two +TTGCATTGTGACTG-1,four +TTGCTAACACCAAC-1,five +TTGCTAACACGCTA-1,one +TTGCTAACCACTCC-1,four +TTGCTATGGTACGT-1,two +TTGCTATGGTAGGG-1,one +TTGGAGACCAATCG-1,unassigned +TTGGAGACGCTATG-1,two +TTGGAGACTATGGC-1,three +TTGGGAACTGAACC-1,four +TTGGTACTACTGGT-1,one +TTGGTACTCTTAGG-1,four +TTGGTACTGAATCC-1,three +TTGGTACTGGATTC-1,three +TTGTACACGTTGTG-1,three +TTGTACACTTGCAG-1,four +TTGTAGCTAGCTCA-1,four +TTGTAGCTCTCTTA-1,three +TTGTCATGGACGGA-1,five +TTTAGAGATCCTCG-1,three +TTTAGCTGATACCG-1,four +TTTAGCTGGATACC-1,two +TTTAGCTGTACTCT-1,unassigned +TTTAGGCTCCTTTA-1,two +TTTATCCTGTTGTG-1,two +TTTCACGAGGTTCA-1,four +TTTCAGTGGAAGGC-1,five +TTTCAGTGTCACGA-1,three +TTTCAGTGTCTATC-1,two +TTTCAGTGTGCAGT-1,three +TTTCCAGAGGTGAG-1,four +TTTCGAACACCTGA-1,unassigned +TTTCGAACTCTCAT-1,two +TTTCTACTGAGGCA-1,three +TTTCTACTTCCTCG-1,three +TTTGCATGAGAGGC-1,three +TTTGCATGCCTCAC-1,four diff --git a/local_server/test/fixtures/schema.json b/local_server/test/fixtures/schema.json new file mode 100644 index 00000000..eb3cc981 --- /dev/null +++ b/local_server/test/fixtures/schema.json @@ -0,0 +1,83 @@ +{ + "dataframe": { + "nObs": 2638, + "nVar": 1838, + "type": "float32" + }, + "annotations": { + "obs": { + "index": "name_0", + "columns": [ + { + "name": "name_0", + "type": "string", + "writable": false + }, + { + "name": "n_genes", + "type": "int32", + "writable": false + }, + { + "name": "percent_mito", + "type": "float32", + "writable": false + }, + { + "name": "n_counts", + "type": "float32", + "writable": false + }, + { + "name": "louvain", + "type": "categorical", + "categories": [ + "CD4 T cells", + "CD14+ Monocytes", + "B cells", + "CD8 T cells", + "NK cells", + "FCGR3A+ Monocytes", + "Dendritic cells", + "Megakaryocytes" + ], + "writable": false + } + ] + }, + "var": { + "index": "name_0", + "columns": [ + { + "name": "name_0", + "type": "string", + "writable": false + }, + { + "name": "n_cells", + "type": "int32", + "writable": false + } + ] + } + }, + "layout": { + "obs": [ + { + "name": "umap", + "type": "float32", + "dims": ["umap_0", "umap_1"] + }, + { + "name": "tsne", + "type": "float32", + "dims": ["tsne_0", "tsne_1"] + }, + { + "name": "pca", + "type": "float32", + "dims": ["pca_0", "pca_1"] + } + ] + } +} diff --git a/local_server/test/fixtures/schema_test_data/generate_test_data.sh b/local_server/test/fixtures/schema_test_data/generate_test_data.sh new file mode 100755 index 00000000..bb33d5a5 --- /dev/null +++ b/local_server/test/fixtures/schema_test_data/generate_test_data.sh @@ -0,0 +1,139 @@ +#!/bin/bash +wget "https://s3-us-west-2.amazonaws.com/10x.files/samples/cell/pbmc3k/pbmc3k_filtered_gene_bc_matrices.tar.gz" +tar xf "pbmc3k_filtered_gene_bc_matrices.tar.gz" + +python3 - < genes_tmp.tsv; mv genes_tmp.tsv merged/genes.tsv + +echo -e "\n\n\nRunning tutorial on original\n\n\n" +Rscript - < + -- Does initial render -- + 4. Concurrently load all /annotations/obs and all /layouts/obs + -- Fully initialized -- + """ + + # Users hit all of the init routes as fast as they can, subject to the ordering constraints and network latency. + wait_time = between(0.01, 0.1) + + def on_start(self): + self.dataset = self.parent.dataset + self.client.verify = False + self.api_less_client = HttpSession( + base_url=self.client.base_url.replace("api.", "").replace("cellxgene/", ""), + request_success=self.client.request_success, + request_failure=self.client.request_failure, + ) + + @task + def index(self): + self.api_less_client.get(f"{self.dataset}", stream=True) + + @task + def loadConfigAndSchema(self): + self.client.get(f"{self.dataset}/{API_SUFFIX}/schema", stream=True, catch_response=True) + self.client.get(f"{self.dataset}/{API_SUFFIX}/config", stream=True, catch_response=True) + + @task + def loadBootstrapData(self): + self.client.get( + f"{self.dataset}/{API_SUFFIX}/layout/obs", headers={"Accept": "application/octet-stream"}, stream=True + ) + self.client.get( + f"{self.dataset}/{API_SUFFIX}/annotations/var?annotation-name={self.parent.var_index_name()}", + headers={"Accept": "application/octet-stream"}, + catch_response=True, + ) + + @task + def loadObsAnnotationsAndLayouts(self): + obs_names = self.parent.obs_annotation_names() + for name in obs_names: + self.client.get( + f"{self.dataset}/{API_SUFFIX}/annotations/obs?annotation-name={name}", + headers={"Accept": "application/octet-stream"}, + stream=True, + ) + + layouts = self.parent.layout_names() + for name in layouts: + self.client.get( + f"{self.dataset}/{API_SUFFIX}/annotations/obs?layout-name={name}", + headers={"Accept": "application/octet-stream"}, + stream=True, + ) + + @task + def done(self): + self.interrupt() + + @task(1) + def load_expression(self): + """ + Simulate user occasionally loading some expression data for a gene + """ + + gene_name = random.choice(self.gene_names) + filter = {"filter": {"var": {"annotation_value": [{"name": self.var_index_name(), "values": [gene_name]}]}}} + self.client.put( + f"{self.dataset}/{API_SUFFIX}/data/var", + data=json.dumps(filter), + headers={"Content-Type": "application/json", "Accept": "application/octet-stream"}, + stream=True, + ).close() + + +class CellxgeneUser(HttpUser): + tasks = [CellXGeneTasks] + + # Most ops do not require back-end interaction, so slow cadence for users + wait_time = between(10, 60) diff --git a/local_server/test/locust/requirements-locust.txt b/local_server/test/locust/requirements-locust.txt new file mode 100644 index 00000000..e8a5c450 --- /dev/null +++ b/local_server/test/locust/requirements-locust.txt @@ -0,0 +1,2 @@ +locust +-r ../../requirements.txt diff --git a/local_server/test/performance/__init__.py b/local_server/test/performance/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/test/performance/create_test_matrix.py b/local_server/test/performance/create_test_matrix.py new file mode 100644 index 00000000..476b6387 --- /dev/null +++ b/local_server/test/performance/create_test_matrix.py @@ -0,0 +1,44 @@ +import anndata +import argparse +import random +import scipy +import numpy as np + + +def main(): + parser = argparse.ArgumentParser("A command to generate test h5ad files") + parser.add_argument("output", help="Name of the output file") + parser.add_argument("nobs", type=int, help="Number of observations (rows)") + parser.add_argument("nvar", type=int, help="Number of variables (columns)") + parser.add_argument("-n", "--nnz-percent", type=float, default=100, help="percent of non-zeros") + parser.add_argument("-c", "--col-shift", action="store_true", help="add a random value to each column") + parser.add_argument("--seed", type=int, default=None, help="add a random value to each column") + + args = parser.parse_args() + create_test_h5ad(args.output, args.nobs, args.nvar, args.nnz_percent, args.col_shift, args.seed) + + +def create_test_h5ad(outfile, nobs, nvar, nnz_percent=100, apply_col_shift=False, seed=None): + random.seed(seed) + np.random.seed(seed) + x = create_X_array(nobs, nvar, nnz_percent, apply_col_shift) + obsm = {"X_random": np.random.rand(nobs, 2).astype(np.float32)} + adata = anndata.AnnData(x, obsm=obsm) + adata.write(outfile) + + +def create_X_array(nobs, nvar, nnz_percent, apply_col_shift): + if nnz_percent < 100: + array = scipy.sparse.random(nobs, nvar, nnz_percent * 0.01, dtype=np.float32, format="csc") + else: + array = np.random.rand(nobs, nvar).astype(np.float32) + + if apply_col_shift: + col_shift = np.random.rand((nvar)) + array += col_shift + + return array + + +if __name__ == "__main__": + main() diff --git a/local_server/test/performance/run_diffexp.py b/local_server/test/performance/run_diffexp.py new file mode 100644 index 00000000..cd2634bd --- /dev/null +++ b/local_server/test/performance/run_diffexp.py @@ -0,0 +1,106 @@ +import sys +import argparse +import random +import time +import numpy as np + +import local_server.compute.diffexp_generic as diffexp_generic + +from local_server.common.config.app_config import AppConfig +from local_server.data_common.matrix_loader import MatrixDataLoader + + +def main(): + parser = argparse.ArgumentParser("A command to test diffexp") + parser.add_argument("dataset", help="name of a dataset to load") + parser.add_argument("-na", "--numA", type=int, help="number of rows in group A") + parser.add_argument("-nb", "--numB", type=int, help="number of rows in group B") + parser.add_argument("-va", "--varA", help="obs variable:value to use for group A") + parser.add_argument("-vb", "--varB", help="obs variable:value to use for group B") + parser.add_argument("-t", "--trials", default=1, type=int, help="number of trials") + parser.add_argument( + "-a", "--alg", choices=("default", "generic"), default="default", help="algorithm to use" + ) + parser.add_argument("-s", "--show", default=False, action="store_true", help="show the results") + parser.add_argument( + "-n", "--new-selection", default=False, action="store_true", help="change the selection between each trial" + ) + parser.add_argument("--seed", default=1, type=int, help="set the random seed") + + args = parser.parse_args() + + app_config = AppConfig() + app_config.update_server_config(single_dataset__datapath=args.dataset) + app_config.update_server_config(app__verbose=True) + app_config.complete_config() + + loader = MatrixDataLoader(args.dataset) + adaptor = loader.open(app_config) + + random.seed(args.seed) + np.random.seed(args.seed) + rows = adaptor.get_shape()[0] + + if args.numA: + filterA = random.sample(range(rows), args.numA) + elif args.varA: + vname, vval = args.varA.split(":") + filterA = get_filter_from_obs(adaptor, vname, vval) + else: + print("must supply numA or varA") + sys.exit(1) + + if args.numB: + filterB = random.sample(range(rows), args.numB) + elif args.varB: + vname, vval = args.varB.split(":") + filterB = get_filter_from_obs(adaptor, vname, vval) + else: + print("must supply numB or varB") + sys.exit(1) + + for i in range(args.trials): + if args.new_selection: + if args.numA: + filterA = random.sample(range(rows), args.numA) + if args.numB: + filterB = random.sample(range(rows), args.numB) + + maskA = np.zeros(rows, dtype=bool) + maskA[filterA] = True + maskB = np.zeros(rows, dtype=bool) + maskB[filterB] = True + + t1 = time.time() + if args.alg == "default": + results = adaptor.compute_diffexp_ttest(maskA, maskB) + elif args.alg == "generic": + results = diffexp_generic.diffexp_ttest(adaptor, maskA, maskB) + + t2 = time.time() + print("TIME=", t2 - t1) + + if args.show: + for res in results: + print(res) + + +def get_filter_from_obs(adaptor, obsname, obsval): + attrs = adaptor.get_obs_columns() + if obsname not in attrs: + print(f"Unknown obs attr {obsname}: expected on of {attrs}") + sys.exit(1) + obsvals = adaptor.query_obs_array(obsname)[:] + obsval = type(obsvals[0])(obsval) + + vfilter = np.where(obsvals == obsval)[0] + if len(vfilter) == 0: + u = np.unique(obsvals) + print(f"Unknown value in variable {obsname}:{obsval}: expected one of {list(u)}") + sys.exit(1) + + return vfilter + + +if __name__ == "__main__": + main() diff --git a/local_server/test/unit/__init__.py b/local_server/test/unit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/test/unit/auth/__init__.py b/local_server/test/unit/auth/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/test/unit/auth/test_auth.py b/local_server/test/unit/auth/test_auth.py new file mode 100644 index 00000000..dc44e30f --- /dev/null +++ b/local_server/test/unit/auth/test_auth.py @@ -0,0 +1,89 @@ +import unittest + +import requests + +from local_server.common.config.app_config import AppConfig +from local_server.test import H5AD_FIXTURE, test_server + + +class AuthTest(unittest.TestCase): + def setUp(self): + self.dataset_datapath = H5AD_FIXTURE + + def test_auth_none(self): + app_config = AppConfig() + app_config.update_server_config(app__flask_secret_key="secret") + app_config.update_server_config(authentication__type=None, single_dataset__datapath=self.dataset_datapath) + app_config.update_dataset_config(user_annotations__enable=False) + + app_config.complete_config() + + with test_server(app_config=app_config) as server: + session = requests.Session() + config = session.get(f"{server}/api/v0.2/config").json() + userinfo = session.get(f"{server}/api/v0.2/userinfo").json() + self.assertNotIn("authentication", config["config"]) + self.assertIsNone(userinfo) + + def test_auth_session(self): + app_config = AppConfig() + app_config.update_server_config(app__flask_secret_key="secret") + app_config.update_server_config(authentication__type="session", single_dataset__datapath=self.dataset_datapath) + app_config.update_dataset_config(user_annotations__enable=True) + app_config.complete_config() + + with test_server(app_config=app_config) as server: + session = requests.Session() + config = session.get(f"{server}/api/v0.2/config").json() + userinfo = session.get(f"{server}/api/v0.2/userinfo").json() + + self.assertFalse(config["config"]["authentication"]["requires_client_login"]) + self.assertTrue(userinfo["userinfo"]["is_authenticated"]) + self.assertEqual(userinfo["userinfo"]["username"], "anonymous") + + def test_auth_test_single(self): + app_config = AppConfig() + app_config.update_server_config(app__flask_secret_key="secret") + app_config.update_server_config( + authentication__type="test", + single_dataset__datapath=self.dataset_datapath, + authentication__insecure_test_environment=True, + ) + + app_config.complete_config() + + with test_server(app_config=app_config) as server: + session = requests.Session() + config = session.get(f"{server}/api/v0.2/config").json() + userinfo = session.get(f"{server}/api/v0.2/userinfo").json() + self.assertFalse(userinfo["userinfo"]["is_authenticated"]) + self.assertIsNone(userinfo["userinfo"]["username"]) + self.assertTrue(config["config"]["authentication"]["requires_client_login"]) + self.assertTrue(config["config"]["parameters"]["annotations"]) + + login_uri = config["config"]["authentication"]["login"] + logout_uri = config["config"]["authentication"]["logout"] + + self.assertEqual(login_uri, "/login") + self.assertEqual(logout_uri, "/logout") + + response = session.get(f"{server}/{login_uri}") + # check that the login redirect worked + self.assertEqual(response.history[0].status_code, 302) + self.assertEqual(response.url, f"{server}/") + + config = session.get(f"{server}/api/v0.2/config").json() + userinfo = session.get(f"{server}/api/v0.2/userinfo").json() + self.assertTrue(userinfo["userinfo"]["is_authenticated"]) + self.assertEqual(userinfo["userinfo"]["username"], "test_account") + self.assertTrue(config["config"]["parameters"]["annotations"]) + + response = session.get(f"{server}/{logout_uri}") + # check that the logout redirect worked + self.assertEqual(response.history[0].status_code, 302) + self.assertEqual(response.url, f"{server}/") + config = session.get(f"{server}/api/v0.2/config").json() + userinfo = session.get(f"{server}/api/v0.2/userinfo").json() + self.assertFalse(userinfo["userinfo"]["is_authenticated"]) + self.assertIsNone(userinfo["userinfo"]["username"]) + self.assertTrue(config["config"]["parameters"]["annotations"]) diff --git a/local_server/test/unit/cli/__init__.py b/local_server/test/unit/cli/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/test/unit/cli/test_launch.py b/local_server/test/unit/cli/test_launch.py new file mode 100644 index 00000000..d58638d8 --- /dev/null +++ b/local_server/test/unit/cli/test_launch.py @@ -0,0 +1,27 @@ +import filecmp +import os +import shutil +import unittest + +import yaml + +from local_server.default_config import default_config +from local_server.test import FIXTURES_ROOT + + +class CLIPLaunchTests(unittest.TestCase): + tmp_dir = os.path.join(FIXTURES_ROOT, "dump_configs") + + @classmethod + def setUpClass(cls) -> None: + os.mkdir(cls.tmp_dir) + + @classmethod + def tearDownClass(cls) -> None: + shutil.rmtree(cls.tmp_dir) + + def test_dump_default_config(self): + os.system(f"cellxgene launch --dump-default-config > {self.tmp_dir}/test_config_dump.txt") + with open(f"{self.tmp_dir}/expected_config_dump.txt", "w") as expected_config: + expected_config.write(yaml.dump(default_config)) + filecmp.cmp(f"{self.tmp_dir}/expected_config_dump.txt", f"{self.tmp_dir}/test_config_dump.txt") diff --git a/local_server/test/unit/cli/test_prepare.py b/local_server/test/unit/cli/test_prepare.py new file mode 100644 index 00000000..85ea4bd6 --- /dev/null +++ b/local_server/test/unit/cli/test_prepare.py @@ -0,0 +1,15 @@ +import unittest + +import pandas as pd + +from local_server.cli.prepare import make_index_unique + + +class CLIPrepareTests(unittest.TestCase): + """ Test cases for CLI prepare logic """ + + def test_make_index_unique(self): + index = pd.Index(["SNORD113", "SNORD113", "SNORD113-1"]) + result = make_index_unique(index) + expected = pd.Index(["SNORD113", "SNORD113-2", "SNORD113-1"]) + self.assertTrue(all(left == right for left, right in zip(result.values, expected.values))) diff --git a/local_server/test/unit/cli/test_upgrade.py b/local_server/test/unit/cli/test_upgrade.py new file mode 100644 index 00000000..9e5dae43 --- /dev/null +++ b/local_server/test/unit/cli/test_upgrade.py @@ -0,0 +1,28 @@ +import unittest + +from local_server.cli.upgrade import validate_version_str, split_version, version_gt + + +class CLIUpgradeTests(unittest.TestCase): + """ Test cases for CLI logic """ + + def test_validate_version_str(self): + self.assertTrue(validate_version_str("0.1.2")) + self.assertTrue(validate_version_str("0.1.2-RC", release_only=False)) + self.assertFalse(validate_version_str("0.1")) + self.assertFalse(validate_version_str("0.1.2.3")) + self.assertFalse(validate_version_str("0.1.2-RC")) + + def test_split_version_str(self): + self.assertEqual(split_version("0.1.2"), [0, 1, 2]) + with self.assertRaises(AttributeError): + split_version("0.1") + + def test_assert_verstion_gt(self): + self.assertTrue(version_gt("1.0.0", "0.1.1")) + self.assertTrue(version_gt("0.1.0", "0.0.1")) + self.assertTrue(version_gt("0.0.1", "0.0.0")) + self.assertFalse(version_gt("0.0.0", "0.0.0")) + self.assertFalse(version_gt("0.0.0", "0.0.1")) + self.assertFalse(version_gt("0.0.1", "0.1.0")) + self.assertFalse(version_gt("0.1.1", "1.0.0")) diff --git a/local_server/test/unit/common/__init__.py b/local_server/test/unit/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/test/unit/common/config/__init__.py b/local_server/test/unit/common/config/__init__.py new file mode 100644 index 00000000..e2f57bab --- /dev/null +++ b/local_server/test/unit/common/config/__init__.py @@ -0,0 +1,222 @@ +import os +import shutil +import unittest +import random +from unittest import mock +import yaml + +from local_server.test import FIXTURES_ROOT + + +def mockenv(**envvars): + return mock.patch.dict(os.environ, envvars) + + +class ConfigTests(unittest.TestCase): + tmp_fixtures_directory = os.path.join(FIXTURES_ROOT, "tmp_dir") + + @classmethod + def tearDownClass(cls) -> None: + shutil.rmtree(cls.tmp_fixtures_directory) + + @classmethod + def setUpClass(cls) -> None: + os.makedirs(cls.tmp_fixtures_directory) + + def custom_server_config( + self, + verbose="false", + debug="false", + host="localhost", + port="null", + open_browser="false", + force_https="false", + flask_secret_key="secret", + auth_type="session", + insecure_test_environment="false", + index="false", + allowed_matrix_types=[], + max_cached_datasets=5, + timelimit_s=5, + dataset_datapath="null", + obs_names="null", + var_names="null", + about="null", + title="null", + data_locater_region_name="us-east-1", + anndata_backed="false", + column_request_max=32, + diffexp_cellcount_max="null", + config_file_name="server_config.yaml", + ): + configfile = os.path.join(self.tmp_fixtures_directory, config_file_name) + server_config_outline_path = os.path.join(FIXTURES_ROOT, "server_config_outline.py") + with open(server_config_outline_path, "r") as config_skeleton: + config = config_skeleton.read() + server_config = eval(config) + with open(configfile, "w") as server_config_file: + server_config_file.write(server_config) + return configfile + + def custom_app_config( + self, + verbose="false", + debug="false", + host="localhost", + port="null", + open_browser="false", + force_https="false", + flask_secret_key="secret", + auth_type="session", + index="false", + allowed_matrix_types=[], + max_cached_datasets=5, + timelimit_s=5, + dataset_datapath="null", + obs_names="null", + var_names="null", + about="null", + title="null", + data_locater_region_name="us-east-1", + anndata_backed="false", + column_request_max=32, + diffexp_cellcount_max="null", + scripts=[], + inline_scripts=[], + authentication_enable="true", + max_categories=1000, + custom_colors="true", + enable_users_annotations="true", + annotation_type="local_file_csv", + db_uri="null", + hosted_file_directory="null", + local_file_csv_directory="null", + local_file_csv_file="null", + ontology_enabled="false", + obo_location="null", + embedding_names=[], + enable_reembedding="false", + enable_difexp="true", + lfc_cutoff=0.01, + top_n=10, + environment=None, + aws_secrets_manager_region=None, + aws_secrets_manager_secrets=[], + config_file_name="app_config.yml", + ): + random_num = random.randrange(999999) + configfile = os.path.join(self.tmp_fixtures_directory, config_file_name) + server_config = self.custom_server_config( + verbose=verbose, + debug=debug, + host=host, + port=port, + open_browser=open_browser, + force_https=force_https, + flask_secret_key=flask_secret_key, + auth_type=auth_type, + index=index, + allowed_matrix_types=allowed_matrix_types, + max_cached_datasets=max_cached_datasets, + timelimit_s=timelimit_s, + dataset_datapath=dataset_datapath, + obs_names=obs_names, + var_names=var_names, + about=about, + title=title, + data_locater_region_name=data_locater_region_name, + anndata_backed=anndata_backed, + column_request_max=column_request_max, + diffexp_cellcount_max=diffexp_cellcount_max, + config_file_name=f"temp_server_config_{random_num}.yml", + ) + dataset_config = self.custom_dataset_config( + scripts=scripts, + inline_scripts=inline_scripts, + authentication_enable=authentication_enable, + max_categories=max_categories, + custom_colors=custom_colors, + enable_users_annotations=enable_users_annotations, + annotation_type=annotation_type, + db_uri=db_uri, + hosted_file_directory=hosted_file_directory, + local_file_csv_directory=local_file_csv_directory, + local_file_csv_file=local_file_csv_file, + ontology_enabled=ontology_enabled, + obo_location=obo_location, + embedding_names=embedding_names, + enable_reembedding=enable_reembedding, + enable_difexp=enable_difexp, + lfc_cutoff=lfc_cutoff, + top_n=top_n, + config_file_name=f"temp_dataset_config_{random_num}.yml", + ) + external_config = self.custom_external_config( + environment=environment, + aws_secrets_manager_region=aws_secrets_manager_region, + aws_secrets_manager_secrets=aws_secrets_manager_secrets, + config_file_name=f"temp_external_config_{random_num}.yml", + ) + + with open(configfile, "w") as app_config_file: + app_config_file.write(open(server_config).read()) + app_config_file.write(open(dataset_config).read()) + app_config_file.write(open(external_config).read()) + + return configfile + + def custom_dataset_config( + self, + scripts=[], + inline_scripts=[], + authentication_enable="true", + max_categories=1000, + custom_colors="true", + enable_users_annotations="true", + annotation_type="local_file_csv", + db_uri="null", + hosted_file_directory="null", + local_file_csv_directory="null", + local_file_csv_file="null", + ontology_enabled="false", + obo_location="null", + embedding_names=[], + enable_reembedding="false", + enable_difexp="true", + lfc_cutoff=0.01, + top_n=10, + config_file_name="dataset_config.yml", + ): + configfile = os.path.join(self.tmp_fixtures_directory, config_file_name) + dataset_config_outline_path = os.path.join(FIXTURES_ROOT, "dataset_config_outline.py") + with open(dataset_config_outline_path, "r") as config_skeleton: + config = config_skeleton.read() + dataset_config = eval(config) + with open(configfile, "w") as dataset_config_file: + dataset_config_file.write(dataset_config) + + return configfile + + def custom_external_config( + self, + environment=None, + aws_secrets_manager_region=None, + aws_secrets_manager_secrets=[], + config_file_name="external_config.yaml", + ): + # set to the default if environment is None + if environment is None: + environment = [ + dict(name="CXG_SECRET_KEY", path=["server", "app", "flask_secret_key"], required=False), + ] + external_config = { + "external": { + "environment": environment, + "aws_secrets_manager": {"region": aws_secrets_manager_region, "secrets": aws_secrets_manager_secrets}, + } + } + + configfile = os.path.join(self.tmp_fixtures_directory, config_file_name) + with open(configfile, "w") as external_config_file: + yaml.dump(external_config, external_config_file) + return configfile diff --git a/local_server/test/unit/common/config/test_app_config.py b/local_server/test/unit/common/config/test_app_config.py new file mode 100644 index 00000000..b2280f86 --- /dev/null +++ b/local_server/test/unit/common/config/test_app_config.py @@ -0,0 +1,153 @@ +import os +import tempfile +import unittest + +import yaml + +from local_server.default_config import default_config +from local_server.common.config.app_config import AppConfig +from local_server.test.unit.common.config import ConfigTests +from local_server.common.errors import ConfigurationError +from local_server.test import FIXTURES_ROOT, H5AD_FIXTURE + + +class AppConfigTest(ConfigTests): + def setUp(self): + self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml" + self.config = AppConfig() + self.config.update_server_config(app__flask_secret_key="secret") + self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE) + self.server_config = self.config.server_config + self.config.complete_config() + + message_list = [] + + def noop(message): + message_list.append(message) + + messagefn = noop + self.context = dict(messagefn=messagefn, messages=message_list) + + def get_config(self, **kwargs): + file_name = self.custom_app_config( + dataset_datapath=H5AD_FIXTURE, config_file_name=self.config_file_name, **kwargs + ) + config = AppConfig() + config.update_from_config_file(file_name) + return config + + def test_get_default_config_correctly_reads_default_config_file(self): + app_default_config = AppConfig().default_config + + expected_config = yaml.load(default_config, Loader=yaml.Loader) + + server_config = app_default_config["server"] + dataset_config = app_default_config["dataset"] + + expected_server_config = expected_config["server"] + expected_dataset_config = expected_config["dataset"] + + self.assertDictEqual(app_default_config, expected_config) + self.assertDictEqual(server_config, expected_server_config) + self.assertDictEqual(dataset_config, expected_dataset_config) + + def test_get_dataset_config_returns_dataset_config_for_single_datasets(self): + datapath = f"{FIXTURES_ROOT}/1e4dfec4-c0b2-46ad-a04e-ff3ffb3c0a8f.h5ad" + file_name = self.custom_app_config(dataset_datapath=datapath, config_file_name=self.config_file_name) + config = AppConfig() + config.update_from_config_file(file_name) + + self.assertEqual(config.get_dataset_config(), config.dataset_config) + + def test_update_server_config_updates_server_config_and_config_status(self): + config = self.get_config() + config.complete_config() + config.check_config() + config.update_server_config(single_dataset__datapath=H5AD_FIXTURE) + with self.assertRaises(ConfigurationError): + config.server_config.check_config() + + def test_write_config_outputs_yaml_with_all_config_vars(self): + config = self.get_config() + config.write_config(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml") + with open(f"{FIXTURES_ROOT}/tmp_dir/{self.config_file_name}", "r") as default_config: + default_config_yml = yaml.safe_load(default_config) + + with open(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml", "r") as output_config: + output_config_yml = yaml.safe_load(output_config) + self.maxDiff = None + self.assertEqual(default_config_yml, output_config_yml) + + def test_update_app_config(self): + config = AppConfig() + config.update_server_config(app__verbose=True, single_dataset__datapath="datapath") + vars = config.server_config.changes_from_default() + self.assertCountEqual(vars, [("app__verbose", True, False), ("single_dataset__datapath", "datapath", None)]) + + config = AppConfig() + config.update_dataset_config(app__scripts=(), app__inline_scripts=()) + vars = config.server_config.changes_from_default() + self.assertCountEqual(vars, []) + + config = AppConfig() + config.update_dataset_config(app__scripts=[], app__inline_scripts=[]) + vars = config.dataset_config.changes_from_default() + self.assertCountEqual(vars, []) + + config = AppConfig() + config.update_dataset_config(app__scripts=("a", "b"), app__inline_scripts=["c", "d"]) + vars = config.dataset_config.changes_from_default() + self.assertCountEqual(vars, [("app__scripts", ["a", "b"], []), ("app__inline_scripts", ["c", "d"], [])]) + + def test_configfile_no_server_section(self): + # test a config file without a dataset section + + with tempfile.TemporaryDirectory() as tempdir: + configfile = os.path.join(tempdir, "config.yaml") + with open(configfile, "w") as fconfig: + config = """ + dataset: + user_annotations: + enable: false + """ + fconfig.write(config) + + app_config = AppConfig() + app_config.update_from_config_file(configfile) + server_changes = app_config.server_config.changes_from_default() + dataset_changes = app_config.dataset_config.changes_from_default() + self.assertEqual(server_changes, []) + self.assertEqual(dataset_changes, [("user_annotations__enable", False, True)]) + + def test_simple_update_single_config_from_path_and_value(self): + """Update a simple config parameter""" + + config = AppConfig() + config.server_config.single_dataset__datapath = "my/data/path" + + # test simple value in server + config.update_single_config_from_path_and_value(["server", "app", "flask_secret_key"], "mysecret") + self.assertEqual(config.server_config.app__flask_secret_key, "mysecret") + + # test simple value in default dataset + config.update_single_config_from_path_and_value( + ["dataset", "user_annotations", "ontology", "obo_location"], "dummy_location", + ) + self.assertEqual(config.dataset_config.user_annotations__ontology__obo_location, "dummy_location") + + # error checking + bad_paths = [ + ( + ["dataset", "does", "not", "exist"], + "unknown config parameter at path: '['dataset', 'does', 'not', 'exist']'", + ), + (["does", "not", "exist"], "path must start with 'server', or 'dataset'"), + ([], "path must start with 'server', or 'dataset'"), + ([1, 2, 3], "path must be a list of strings, got '[1, 2, 3]'"), + ("string", "path must be a list of strings, got 'string'"), + ] + for bad_path, error_message in bad_paths: + with self.assertRaises(ConfigurationError) as config_error: + config.update_single_config_from_path_and_value(bad_path, "value") + + self.assertEqual(config_error.exception.message, error_message) diff --git a/local_server/test/unit/common/config/test_base_config.py b/local_server/test/unit/common/config/test_base_config.py new file mode 100644 index 00000000..c289106a --- /dev/null +++ b/local_server/test/unit/common/config/test_base_config.py @@ -0,0 +1,63 @@ +import unittest + +from local_server.common.config.app_config import AppConfig +from local_server.test import H5AD_FIXTURE +from local_server.test.unit.common.config import ConfigTests +from local_server.common.errors import ConfigurationError + + +class BaseConfigTest(ConfigTests): + def setUp(self): + self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml" + self.config = AppConfig() + self.config.update_server_config(app__flask_secret_key="secret") + self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE) + self.server_config = self.config.server_config + self.config.complete_config() + + message_list = [] + + def noop(message): + message_list.append(message) + + messagefn = noop + self.context = dict(messagefn=messagefn, messages=message_list) + + def get_config(self, **kwargs): + file_name = self.custom_app_config( + dataset_datapath=f"{H5AD_FIXTURE}", config_file_name=self.config_file_name, **kwargs + ) + config = AppConfig() + config.update_from_config_file(file_name) + return config + + def test_mapping_creation_returns_map_of_server_and_dataset_config(self): + config = AppConfig() + mapping = config.dataset_config.create_mapping(config.default_config) + self.assertIsNotNone(mapping["server__app__verbose"]) + self.assertIsNotNone(mapping["dataset__presentation__max_categories"]) + self.assertIsNotNone(mapping["dataset__user_annotations__ontology__obo_location"]) + + def test_changes_from_default_returns_list_of_nondefault_config_values(self): + config = self.get_config(verbose="true", lfc_cutoff=0.05) + server_changes = config.server_config.changes_from_default() + dataset_changes = config.dataset_config.changes_from_default() + + self.assertEqual( + server_changes, + [ + ("app__verbose", True, False), + ("app__flask_secret_key", "secret", None), + ("single_dataset__datapath", H5AD_FIXTURE, None), + ('data_locator__s3__region_name', 'us-east-1', True) + ], + ) + self.assertEqual(dataset_changes, [("diffexp__lfc_cutoff", 0.05, 0.01)]) + + def test_check_config_throws_error_if_attr_has_not_been_checked(self): + config = self.get_config(verbose="true") + config.complete_config() + config.check_config() + config.update_server_config(app__verbose=False) + with self.assertRaises(ConfigurationError): + config.check_config() diff --git a/local_server/test/unit/common/config/test_dataset_config.py b/local_server/test/unit/common/config/test_dataset_config.py new file mode 100644 index 00000000..a5f47cba --- /dev/null +++ b/local_server/test/unit/common/config/test_dataset_config.py @@ -0,0 +1,157 @@ +import os +import tempfile + +import unittest +from unittest.mock import patch + +from local_server.common.annotations.local_file_csv import AnnotationsLocalFile +from local_server.common.config.app_config import AppConfig +from local_server.common.config.base_config import BaseConfig +from local_server.test import FIXTURES_ROOT, H5AD_FIXTURE + +from local_server.common.errors import ConfigurationError +from local_server.test.unit.common.config import ConfigTests + + +class TestDatasetConfig(ConfigTests): + def setUp(self): + self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml" + self.config = AppConfig() + self.config.update_server_config(app__flask_secret_key="secret") + self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE) + self.dataset_config = self.config.dataset_config + self.config.complete_config() + message_list = [] + + def noop(message): + message_list.append(message) + + messagefn = noop + self.context = dict(messagefn=messagefn, messages=message_list) + + def get_config(self, **kwargs): + file_name = self.custom_app_config(dataset_datapath=H5AD_FIXTURE, **kwargs) + config = AppConfig() + config.update_from_config_file(file_name) + return config + + def test_init_datatset_config_sets_vars_from_config(self): + config = AppConfig() + self.assertEqual(config.dataset_config.presentation__max_categories, 1000) + self.assertEqual(config.dataset_config.user_annotations__type, "local_file_csv") + self.assertEqual(config.dataset_config.diffexp__lfc_cutoff, 0.01) + self.assertIsNone(config.dataset_config.user_annotations__ontology__obo_location) + + @patch("local_server.common.config.dataset_config.BaseConfig.validate_correct_type_of_configuration_attribute") + def test_complete_config_checks_all_attr(self, mock_check_attrs): + mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute() + self.dataset_config.complete_config(self.context) + self.assertIsNotNone(self.config.server_config.data_adaptor) + self.assertEqual(mock_check_attrs.call_count, 17) + + def test_app_sets_script_vars(self): + config = self.get_config(scripts=["path/to/script"]) + config.dataset_config.handle_app() + + self.assertEqual(config.dataset_config.app__scripts, [{"src": "path/to/script"}]) + + config = self.get_config(scripts=[{"src": "path/to/script", "more": "different/script/path"}]) + config.dataset_config.handle_app() + self.assertEqual( + config.dataset_config.app__scripts, [{"src": "path/to/script", "more": "different/script/path"}] + ) + + config = self.get_config(scripts=["path/to/script", "different/script/path"]) + config.dataset_config.handle_app() + # TODO @madison -- is this the desired functionality? + self.assertEqual( + config.dataset_config.app__scripts, [{"src": "path/to/script"}, {"src": "different/script/path"}] + ) + + config = self.get_config(scripts=[{"more": "different/script/path"}]) + with self.assertRaises(ConfigurationError): + config.dataset_config.handle_app() + + def test_handle_user_annotations_ensures_auth_is_enabled_with_valid_auth_type(self): + config = self.get_config(enable_users_annotations="true", authentication_enable="false") + config.server_config.complete_config(self.context) + with self.assertRaises(ConfigurationError): + config.dataset_config.handle_user_annotations(self.context) + + config = self.get_config(enable_users_annotations="true", authentication_enable="true", auth_type="pretend") + with self.assertRaises(ConfigurationError): + config.server_config.complete_config(self.context) + + def test_handle_user_annotations__instantiates_user_annotations_class_correctly(self): + config = self.get_config( + enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv" + ) + config.server_config.complete_config(self.context) + config.dataset_config.handle_user_annotations(self.context) + self.assertIsInstance(config.dataset_config.user_annotations, AnnotationsLocalFile) + + config = self.get_config( + enable_users_annotations="true", authentication_enable="true", annotation_type="NOT_REAL" + ) + config.server_config.complete_config(self.context) + with self.assertRaises(ConfigurationError): + config.dataset_config.handle_user_annotations(self.context) + + def test_handle_local_file_csv_annotations__sets_dir_if_not_passed_in(self): + config = self.get_config( + enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv" + ) + config.server_config.complete_config(self.context) + config.dataset_config.handle_local_file_csv_annotations() + self.assertIsInstance(config.dataset_config.user_annotations, AnnotationsLocalFile) + cwd = os.getcwd() + self.assertEqual(config.dataset_config.user_annotations._get_output_dir(), cwd) + + def test_handle_embeddings__checks_data_file_types(self): + file_name = self.custom_app_config( + embedding_names=["name1", "name2"], + enable_reembedding="true", + dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", + anndata_backed="true", + config_file_name=self.config_file_name, + ) + config = AppConfig() + config.update_from_config_file(file_name) + config.server_config.complete_config(self.context) + with self.assertRaises(ConfigurationError): + config.dataset_config.handle_embeddings() + + def test_handle_diffexp__raises_warning_for_large_datasets(self): + config = self.get_config(lfc_cutoff=0.02, enable_difexp="true", top_n=15) + config.server_config.complete_config(self.context) + config.dataset_config.handle_diffexp(self.context) + self.assertEqual(len(self.context["messages"]), 1) + + def test_configfile_with_specialization(self): + # test that per_dataset_config config load the default config, then the specialized config + + with tempfile.TemporaryDirectory() as tempdir: + configfile = os.path.join(tempdir, "config.yaml") + with open(configfile, "w") as fconfig: + config = """ + server: + single_dataset: + datapath: fake_datapath + dataset: + user_annotations: + enable: false + type: local_file_csv + local_file_csv: + file: fake_file + directory: fake_dir + """ + fconfig.write(config) + + app_config = AppConfig() + app_config.update_from_config_file(configfile) + + test_config = app_config.dataset_config + + # test config from default + self.assertEqual(test_config.user_annotations__type, "local_file_csv") + self.assertEqual(test_config.user_annotations__local_file_csv__file, "fake_file") diff --git a/local_server/test/unit/common/config/test_external_config.py b/local_server/test/unit/common/config/test_external_config.py new file mode 100644 index 00000000..b0751c44 --- /dev/null +++ b/local_server/test/unit/common/config/test_external_config.py @@ -0,0 +1,215 @@ +import os +from unittest.mock import patch + +import requests + +from local_server.common.errors import ConfigurationError +from local_server.common.config.app_config import AppConfig +from local_server.test import test_server, FIXTURES_ROOT +from local_server.common.utils.type_conversion_utils import convert_string_to_value +from local_server.test.unit.common.config import ConfigTests + + +class TestExternalConfig(ConfigTests): + def test_type_convert(self): + # The values from environment variables and aws secrets are returned as strings. + # These values need to be converted to the proper types. + + self.assertEqual(convert_string_to_value("1"), int(1)) + self.assertEqual(convert_string_to_value("1.1"), float(1.1)) + self.assertEqual(convert_string_to_value("string"), "string") + self.assertEqual(convert_string_to_value("true"), True) + self.assertEqual(convert_string_to_value("True"), True) + self.assertEqual(convert_string_to_value("false"), False) + self.assertEqual(convert_string_to_value("False"), False) + self.assertEqual(convert_string_to_value("null"), None) + self.assertEqual(convert_string_to_value("None"), None) + self.assertEqual(convert_string_to_value("{'a':10, 'b':'string'}"), dict(a=int(10), b="string")) + + def test_environment_variable(self): + configfile = self.custom_external_config( + environment=[ + dict(name="DATAPATH", path=["server", "single_dataset", "datapath"], required=True), + dict(name="DIFFEXP", path=["dataset", "diffexp", "enable"], required=True), + ], + config_file_name="environment_external_config.yaml", + ) + + env = os.environ + env["DATAPATH"] = f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad" + env["DIFFEXP"] = "False" + with test_server(command_line_args=["-c", configfile], env=env) as server: + session = requests.Session() + response = session.get(f"{server}/api/v0.2/config") + data_config = response.json() + self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k-CSC-gz") + self.assertTrue(data_config["config"]["parameters"]["disable-diffexp"]) + + env["DATAPATH"] = f"{FIXTURES_ROOT}/a95c59b4-7f5d-4b80-ad53-a694834ca18b.h5ad" + env["DIFFEXP"] = "True" + with test_server(command_line_args=["-c", configfile], env=env) as server: + session = requests.Session() + response = session.get(f"{server}/api/v0.2/config") + data_config = response.json() + self.assertEqual(data_config["config"]["displayNames"]["dataset"], "a95c59b4-7f5d-4b80-ad53-a694834ca18b") + self.assertFalse(data_config["config"]["parameters"]["disable-diffexp"]) + + def test_environment_variable_errors(self): + + # no name + app_config = AppConfig() + app_config.external_config.environment = [dict(required=True, path=["this", "is", "a", "path"])] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "environment: 'name' is missing") + + # required has wrong type + app_config = AppConfig() + app_config.external_config.environment = [ + dict(name="myenvar", required="optional", path=["this", "is", "a", "path"]) + ] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "environment: 'required' must be a bool") + + # no path + app_config = AppConfig() + app_config.external_config.environment = [dict(name="myenvar", required=True)] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "environment: 'path' is missing") + + # required environment variable is not set + app_config = AppConfig() + app_config.external_config.environment = [ + dict(name="THIS_ENV_IS_NOT_SET", required=True, path=["this", "is", "a", "path"]) + ] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "required environment variable 'THIS_ENV_IS_NOT_SET' not set") + + @patch("local_server.common.config.external_config.get_secret_key") + def test_aws_secrets_manager(self, mock_get_secret_key): + mock_get_secret_key.return_value = { + "flask_secret_key": "mock_flask_secret_key", + } + configfile = self.custom_external_config( + aws_secrets_manager_region="us-west-2", + aws_secrets_manager_secrets=[ + dict( + name="my_secret", + values=[ + dict(key="flask_secret_key", path=["server", "app", "flask_secret_key"], required=True), + ], + ) + ], + config_file_name="secret_external_config.yaml", + ) + + app_config = AppConfig() + app_config.update_from_config_file(configfile) + app_config.server_config.single_dataset__datapath = f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad" + + app_config.complete_config() + + self.assertEqual(app_config.server_config.app__flask_secret_key, "mock_flask_secret_key") + + @patch("local_server.common.config.external_config.get_secret_key") + def test_aws_secrets_manager_error(self, mock_get_secret_key): + mock_get_secret_key.return_value = { + "db_uri": "mock_db_uri", + } + + # no region + app_config = AppConfig() + app_config.external_config.aws_secrets_manager__region = None + app_config.external_config.aws_secrets_manager__secrets = [ + dict(name="secret1", values=[dict(key="key1", required=True, path=["this", "is", "my", "path"])]) + ] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual( + config_error.exception.message, + "Invalid type for attribute: aws_secrets_manager__region, expected type str, got NoneType", + ) + + # missing secret name + app_config = AppConfig() + app_config.external_config.aws_secrets_manager__region = "us-west-2" + app_config.external_config.aws_secrets_manager__secrets = [ + dict(values=[dict(key="db_uri", required=True, path=["this", "is", "my", "path"])]) + ] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'name' is missing") + + # secret name wrong type + app_config = AppConfig() + app_config.external_config.aws_secrets_manager__region = "us-west-2" + app_config.external_config.aws_secrets_manager__secrets = [ + dict(name=1, values=[dict(key="db_uri", required=True, path=["this", "is", "my", "path"])]) + ] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'name' must be a string") + + # missing values name + app_config = AppConfig() + app_config.external_config.aws_secrets_manager__region = "us-west-2" + app_config.external_config.aws_secrets_manager__secrets = [dict(name="mysecret")] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'values' is missing") + + # values wrong type + app_config = AppConfig() + app_config.external_config.aws_secrets_manager__region = "us-west-2" + app_config.external_config.aws_secrets_manager__secrets = [ + dict(name="mysecret", values=dict(key="db_uri", required=True, path=["this", "is", "my", "path"])) + ] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'values' must be a list") + + # entry missing key + app_config = AppConfig() + app_config.external_config.aws_secrets_manager__region = "us-west-2" + app_config.external_config.aws_secrets_manager__secrets = [ + dict(name="mysecret", values=[dict(required=True, path=["this", "is", "my", "path"])]) + ] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "missing 'key' in secret values: mysecret") + + # entry required is wrong type + app_config = AppConfig() + app_config.external_config.aws_secrets_manager__region = "us-west-2" + app_config.external_config.aws_secrets_manager__secrets = [ + dict(name="mysecret", values=[dict(key="db_uri", required="optional", path=["this", "is", "my", "path"])]) + ] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "wrong type for 'required' in secret values: mysecret") + + # entry missing path + app_config = AppConfig() + app_config.external_config.aws_secrets_manager__region = "us-west-2" + app_config.external_config.aws_secrets_manager__secrets = [ + dict(name="mysecret", values=[dict(key="db_uri", required=True)]) + ] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "missing 'path' in secret values: mysecret") + + # secret missing required key + app_config = AppConfig() + app_config.external_config.aws_secrets_manager__region = "us-west-2" + app_config.external_config.aws_secrets_manager__secrets = [ + dict( + name="mysecret", + values=[dict(key="KEY_DOES_NOT_EXIST", required=True, path=["this", "is", "a", "path"])], + ) + ] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "required secret 'mysecret:KEY_DOES_NOT_EXIST' not set") diff --git a/local_server/test/unit/common/config/test_server_config.py b/local_server/test/unit/common/config/test_server_config.py new file mode 100644 index 00000000..f50430a5 --- /dev/null +++ b/local_server/test/unit/common/config/test_server_config.py @@ -0,0 +1,122 @@ +import os +import unittest +from unittest import mock +from unittest.mock import patch + +from local_server.common.config.base_config import BaseConfig +from local_server.test import H5AD_FIXTURE + +from local_server.common.config.app_config import AppConfig +from local_server.common.errors import ConfigurationError +from local_server.test.unit.common.config import ConfigTests + + +def mockenv(**envvars): + return mock.patch.dict(os.environ, envvars) + + +class TestServerConfig(ConfigTests): + def setUp(self): + self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml" + self.config = AppConfig() + self.config.update_server_config(app__flask_secret_key="secret") + self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE) + self.server_config = self.config.server_config + self.config.complete_config() + + message_list = [] + + def noop(message): + message_list.append(message) + + messagefn = noop + self.context = dict(messagefn=messagefn, messages=message_list) + + def get_config(self, **kwargs): + file_name = self.custom_app_config( + dataset_datapath=f"{H5AD_FIXTURE}", config_file_name=self.config_file_name, **kwargs + ) + config = AppConfig() + config.update_from_config_file(file_name) + return config + + def test_init_raises_error_if_default_config_is_invalid(self): + invalid_config = self.get_config(port="not_valid") + with self.assertRaises(ConfigurationError): + invalid_config.complete_config() + + @patch("local_server.common.config.server_config.BaseConfig.validate_correct_type_of_configuration_attribute") + def test_complete_config_checks_all_attr(self, mock_check_attrs): + mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute() + self.server_config.complete_config(self.context) + self.assertEqual(mock_check_attrs.call_count, 20) + + def test_handle_app__throws_error_if_port_doesnt_exist(self): + config = self.get_config(port=99999999) + with self.assertRaises(ConfigurationError): + config.server_config.handle_app(self.context) + + @patch("local_server.common.config.server_config.discover_s3_region_name") + def test_handle_data_locator_works_for_default_types(self, mock_discover_region_name): + mock_discover_region_name.return_value = None + # Default config + self.assertEqual(self.config.server_config.data_locator__s3__region_name, None) + # hard coded + config = self.get_config() + self.assertEqual(config.server_config.data_locator__s3__region_name, "us-east-1") + # incorrectly formatted + datapath = "s3://shouldnt/work" + file_name = self.custom_app_config( + dataset_datapath=datapath, config_file_name=self.config_file_name, data_locater_region_name="true" + ) + config = AppConfig() + config.update_from_config_file(file_name) + with self.assertRaises(ConfigurationError): + config.server_config.handle_data_locator() + + def test_handle_app___can_use_envar_port(self): + config = self.get_config(port=24) + self.assertEqual(config.server_config.app__port, 24) + + # Note if the port is set in the config file it will NOT be overwritten by a different envvar + os.environ["CXG_SERVER_PORT"] = "4008" + self.config = AppConfig() + self.config.update_server_config(app__flask_secret_key="secret") + self.config.server_config.handle_app(self.context) + self.assertEqual(self.config.server_config.app__port, 4008) + del os.environ["CXG_SERVER_PORT"] + + def test_handle_app__can_get_secret_key_from_envvar_or_config_file_with_envvar_given_preference(self): + config = self.get_config(flask_secret_key="KEY_FROM_FILE") + self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_FILE") + + os.environ["CXG_SECRET_KEY"] = "KEY_FROM_ENV" + config.external_config.handle_environment(self.context) + self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_ENV") + + def test_config_for_single_dataset(self): + file_name = self.custom_app_config( + config_file_name="single_dataset.yml", dataset_datapath=f"{H5AD_FIXTURE}" + ) + config = AppConfig() + config.update_from_config_file(file_name) + config.server_config.handle_single_dataset(self.context) + + file_name = self.custom_app_config( + config_file_name="single_dataset_with_about.yml", + about="www.cziscience.com", + dataset_datapath=f"{H5AD_FIXTURE}", + ) + config = AppConfig() + config.update_from_config_file(file_name) + with self.assertRaises(ConfigurationError): + config.server_config.handle_single_dataset(self.context) + + def test_test_auth_only_in_insecure(self): + + config = self.get_config(auth_type="test") + with self.assertRaises(ConfigurationError): + config.complete_config() + + config.update_server_config(authentication__insecure_test_environment=True) + config.complete_config() diff --git a/local_server/test/unit/common/test_api.py b/local_server/test/unit/common/test_api.py new file mode 100644 index 00000000..a1db48d7 --- /dev/null +++ b/local_server/test/unit/common/test_api.py @@ -0,0 +1,422 @@ +import shutil +import time +import unittest +import zlib +from http import HTTPStatus + +import pandas as pd +import requests + +import local_server.test.unit.decode_fbs as decode_fbs +from local_server.data_common.matrix_loader import MatrixDataType +from local_server.test import ( + data_with_tmp_annotations, + make_fbs, + PROJECT_ROOT, + start_test_server, + stop_test_server, +) +from local_server.test.fixtures.fixtures import pbmc3k_colors + +BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}} + + +# TODO (mweiden): remove ANNOTATIONS_ENABLED and Annotation subclasses when annotations are no longer experimental + + +class EndPoints(object): + ANNOTATIONS_ENABLED = True + + def test_initialize(self): + endpoint = "schema" + url = f"{self.URL_BASE}{endpoint}" + result = self.session.get(url) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/json") + result_data = result.json() + self.assertEqual(result_data["schema"]["dataframe"]["nObs"], 2638) + self.assertEqual(len(result_data["schema"]["annotations"]["obs"]), 2) + self.assertEqual( + len(result_data["schema"]["annotations"]["obs"]["columns"]), 6 if self.ANNOTATIONS_ENABLED else 5 + ) + + def test_config(self): + endpoint = "config" + url = f"{self.URL_BASE}{endpoint}" + result = self.session.get(url) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/json") + result_data = result.json() + self.assertIn("library_versions", result_data["config"]) + self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k") + + def test_get_layout_fbs(self): + endpoint = "layout/obs" + url = f"{self.URL_BASE}{endpoint}" + header = {"Accept": "application/octet-stream"} + result = self.session.get(url, headers=header) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertEqual(df["n_rows"], 2638) + self.assertEqual(df["n_cols"], 8) + self.assertIsNotNone(df["columns"]) + self.assertSetEqual( + set(df["col_idx"]), + {"pca_0", "pca_1", "tsne_0", "tsne_1", "umap_0", "umap_1", "draw_graph_fr_0", "draw_graph_fr_1"}, + ) + self.assertIsNone(df["row_idx"]) + self.assertEqual(len(df["columns"]), df["n_cols"]) + + def test_put_layout_fbs(self): + # first check that re-embedding is turned on + result = self.session.get(f"{self.URL_BASE}config") + config_data = result.json() + re_embed = config_data["config"]["parameters"]["enable-reembedding"] + if not re_embed: + return + # attempt to reembed with umap over 100 cells. + endpoint = "layout/obs" + url = f"{self.URL_BASE}{endpoint}" + data = {} + data["filter"] = {} + data["filter"]["obs"] = {} + data["filter"]["obs"]["index"] = list(range(100)) + data["method"] = "umap" + result = self.session.put(url, json=data) + + self.assertEqual(result.status_code, HTTPStatus.OK) + result_data = result.json() + self.assertIsInstance(result_data, dict) + self.assertEqual(result_data["type"], "float32") + self.assertTrue(result_data["name"].startswith("reembed:umap_")) + self.assertIsInstance(result_data["dims"], list) + self.assertEqual(len(result_data["dims"]), 2) + dims = result_data["dims"] + self.assertTrue(dims[0].startswith("reembed:umap_") and dims[0].endswith("_0")) + self.assertTrue(dims[1].startswith("reembed:umap_") and dims[1].endswith("_1")) + + def test_bad_filter(self): + endpoint = "data/var" + url = f"{self.URL_BASE}{endpoint}" + result = self.session.put(url, json=BAD_FILTER) + self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST) + + def test_get_annotations_obs_fbs(self): + endpoint = "annotations/obs" + url = f"{self.URL_BASE}{endpoint}" + header = {"Accept": "application/octet-stream"} + result = self.session.get(url, headers=header) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertEqual(df["n_rows"], 2638) + self.assertEqual(df["n_cols"], 6 if self.ANNOTATIONS_ENABLED else 5) + self.assertIsNotNone(df["columns"]) + self.assertIsNone(df["row_idx"]) + self.assertEqual(len(df["columns"]), df["n_cols"]) + obs_index_col_name = self.schema["schema"]["annotations"]["obs"]["index"] + self.assertCountEqual( + df["col_idx"], + [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"] + + (["cluster-test"] if self.ANNOTATIONS_ENABLED else []), + ) + + def test_get_annotations_obs_keys_fbs(self): + endpoint = "annotations/obs" + query = "annotation-name=n_genes&annotation-name=percent_mito" + url = f"{self.URL_BASE}{endpoint}?{query}" + header = {"Accept": "application/octet-stream"} + result = self.session.get(url, headers=header) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertEqual(df["n_rows"], 2638) + self.assertEqual(df["n_cols"], 2) + self.assertIsNotNone(df["columns"]) + self.assertIsNone(df["row_idx"]) + self.assertEqual(len(df["columns"]), df["n_cols"]) + self.assertCountEqual(df["col_idx"], ["n_genes", "percent_mito"]) + + def test_get_annotations_obs_error(self): + endpoint = "annotations/obs" + query = "annotation-name=notakey" + url = f"{self.URL_BASE}{endpoint}?{query}" + result = self.session.get(url) + self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST) + + def test_get_annotations_var_fbs(self): + endpoint = "annotations/var" + url = f"{self.URL_BASE}{endpoint}" + header = {"Accept": "application/octet-stream"} + result = self.session.get(url, headers=header) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertEqual(df["n_rows"], 1838) + self.assertEqual(df["n_cols"], 2) + self.assertIsNotNone(df["columns"]) + self.assertIsNone(df["row_idx"]) + self.assertEqual(len(df["columns"]), df["n_cols"]) + var_index_col_name = self.schema["schema"]["annotations"]["var"]["index"] + self.assertCountEqual(df["col_idx"], [var_index_col_name, "n_cells"]) + + def test_get_annotations_var_keys_fbs(self): + endpoint = "annotations/var" + query = "annotation-name=n_cells" + url = f"{self.URL_BASE}{endpoint}?{query}" + header = {"Accept": "application/octet-stream"} + result = self.session.get(url, headers=header) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertEqual(df["n_rows"], 1838) + self.assertEqual(df["n_cols"], 1) + self.assertIsNotNone(df["columns"]) + self.assertIsNone(df["row_idx"]) + self.assertEqual(len(df["columns"]), df["n_cols"]) + self.assertCountEqual(df["col_idx"], ["n_cells"]) + + def test_get_annotations_var_error(self): + endpoint = "annotations/var" + query = "annotation-name=notakey" + url = f"{self.URL_BASE}{endpoint}?{query}" + result = self.session.get(url) + self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST) + + def test_data_mimetype_error(self): + endpoint = "data/var" + header = {"Accept": "xxx"} + url = f"{self.URL_BASE}{endpoint}" + result = self.session.put(url, headers=header) + self.assertEqual(result.status_code, HTTPStatus.NOT_ACCEPTABLE) + + def test_fbs_default(self): + endpoint = "data/var" + url = f"{self.URL_BASE}{endpoint}" + result = self.session.put(url) + self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST) + + filter = {"filter": {"var": {"index": [0, 1, 4]}}} + result = self.session.put(url, json=filter) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + + def test_data_put_fbs(self): + endpoint = "data/var" + url = f"{self.URL_BASE}{endpoint}" + header = {"Accept": "application/octet-stream"} + result = self.session.put(url, headers=header) + self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST) + + def test_data_get_fbs(self): + endpoint = "data/var" + url = f"{self.URL_BASE}{endpoint}" + header = {"Accept": "application/octet-stream"} + result = self.session.get(url, headers=header) + self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST) + + def test_data_put_filter_fbs(self): + endpoint = "data/var" + url = f"{self.URL_BASE}{endpoint}" + header = {"Accept": "application/octet-stream"} + filter = {"filter": {"var": {"index": [0, 1, 4]}}} + result = self.session.put(url, headers=header, json=filter) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertEqual(df["n_rows"], 2638) + self.assertEqual(df["n_cols"], 3) + self.assertIsNotNone(df["columns"]) + self.assertIsNone(df["row_idx"]) + self.assertEqual(len(df["columns"]), df["n_cols"]) + self.assertListEqual(df["col_idx"].tolist(), [0, 1, 4]) + + def test_data_get_filter_fbs(self): + index_col_name = self.schema["schema"]["annotations"]["var"]["index"] + endpoint = "data/var" + query = f"var:{index_col_name}=SIK1" + url = f"{self.URL_BASE}{endpoint}?{query}" + header = {"Accept": "application/octet-stream"} + result = self.session.get(url, headers=header) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertEqual(df["n_rows"], 2638) + self.assertEqual(df["n_cols"], 1) + + def test_data_get_unknown_filter_fbs(self): + index_col_name = self.schema["schema"]["annotations"]["var"]["index"] + endpoint = "data/var" + query = f"var:{index_col_name}=UNKNOWN" + url = f"{self.URL_BASE}{endpoint}?{query}" + header = {"Accept": "application/octet-stream"} + result = self.session.get(url, headers=header) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertEqual(df["n_rows"], 2638) + self.assertEqual(df["n_cols"], 0) + + def test_data_put_single_var(self): + endpoint = "data/var" + url = f"{self.URL_BASE}{endpoint}" + header = {"Accept": "application/octet-stream"} + index_col_name = self.schema["schema"]["annotations"]["var"]["index"] + var_filter = {"filter": {"var": {"annotation_value": [{"name": index_col_name, "values": ["RER1"]}]}}} + result = self.session.put(url, headers=header, json=var_filter) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertEqual(df["n_rows"], 2638) + self.assertEqual(df["n_cols"], 1) + + def test_colors(self): + endpoint = "colors" + url = f"{self.URL_BASE}{endpoint}" + result = self.session.get(url) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/json") + result_data = result.json() + self.assertEqual(result_data, pbmc3k_colors) + + def test_static(self): + endpoint = "static" + file = "assets/favicon.ico" + url = f"{self.server}/{endpoint}/{file}" + result = self.session.get(url) + self.assertEqual(result.status_code, HTTPStatus.OK) + + def _setupClass(child_class, command_line): + child_class.ps, child_class.server = start_test_server(command_line) + child_class.URL_BASE = f"{child_class.server}/api/v0.2/" + child_class.session = requests.Session() + for i in range(90): + try: + result = child_class.session.get(f"{child_class.URL_BASE}schema") + child_class.schema = result.json() + except requests.exceptions.ConnectionError: + time.sleep(1) + + +class EndPointsAnnotations(EndPoints): + def test_get_schema_existing_writable(self): + self._test_get_schema_writable("cluster-test") + + def test_get_user_annotations_existing_obs_keys_fbs(self): + self._test_get_user_annotations_obs_keys_fbs( + "cluster-test", {"unassigned", "one", "two", "three", "four", "five", "six", "seven"}, + ) + + def test_put_user_annotations_obs_fbs(self): + endpoint = "annotations/obs" + query = "annotation-collection-name=test_annotations" + url = f"{self.URL_BASE}{endpoint}?{query}" + n_rows = self.data.get_shape()[0] + fbs = make_fbs({"cat_A": pd.Series(["label_A"] * n_rows, dtype="category")}) + result = self.session.put(url, data=zlib.compress(fbs)) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/json") + self.assertEqual(result.json(), {"status": "OK"}) + self._test_get_schema_writable("cat_A") + self._test_get_user_annotations_obs_keys_fbs("cat_A", {"label_A"}) + + def _test_get_user_annotations_obs_keys_fbs(self, annotation_name, columns): + endpoint = "annotations/obs" + query = f"annotation-name={annotation_name}" + url = f"{self.URL_BASE}{endpoint}?{query}" + header = {"Accept": "application/octet-stream"} + result = self.session.get(url, headers=header) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertEqual(df["n_rows"], 2638) + self.assertEqual(df["n_cols"], 1) + self.assertListEqual(df["col_idx"], [annotation_name]) + self.assertEqual(set(df["columns"][0]), columns) + self.assertIsNone(df["row_idx"]) + self.assertEqual(len(df["columns"]), df["n_cols"]) + + def _test_get_schema_writable(self, cluster_name): + endpoint = "schema" + url = f"{self.URL_BASE}{endpoint}" + result = self.session.get(url) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/json") + result_data = result.json() + columns = result_data["schema"]["annotations"]["obs"]["columns"] + matching_columns = [c for c in columns if c["name"] == cluster_name] + self.assertEqual(len(matching_columns), 1) + self.assertTrue(matching_columns[0]["writable"]) + + +class EndPointsAnndata(unittest.TestCase, EndPoints): + """Test Case for endpoints""" + + ANNOTATIONS_ENABLED = False + + @classmethod + def setUpClass(cls): + cls._setupClass( + cls, + [ + f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", + "--disable-annotations", + "--experimental-enable-reembedding", + ], + ) + + @classmethod + def tearDownClass(cls): + stop_test_server(cls.ps) + + @property + def annotations_enabled(self): + return False + + def test_diff_exp(self): + endpoint = "diffexp/obs" + url = f"{self.URL_BASE}{endpoint}" + params = { + "mode": "topN", + "set1": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["NK cells"]}]}}}, + "set2": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["CD8 T cells"]}]}}}, + "count": 7, + } + result = self.session.post(url, json=params) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/json") + result_data = result.json() + self.assertEqual(len(result_data), 7) + + def test_diff_exp_indices(self): + endpoint = "diffexp/obs" + url = f"{self.URL_BASE}{endpoint}" + params = { + "mode": "topN", + "count": 10, + "set1": {"filter": {"obs": {"index": [[0, 500]]}}}, + "set2": {"filter": {"obs": {"index": [[500, 1000]]}}}, + } + result = self.session.post(url, json=params) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/json") + result_data = result.json() + self.assertEqual(len(result_data), 10) + + +class EndPointsAnndataAnnotations(unittest.TestCase, EndPointsAnnotations): + """Test Case for endpoints""" + + ANNOTATIONS_ENABLED = True + + @classmethod + def setUpClass(cls): + cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations( + MatrixDataType.H5AD, annotations_fixture=True + ) + cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location()]) + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.tmp_dir) + stop_test_server(cls.ps) diff --git a/local_server/test/unit/common/test_colors.py b/local_server/test/unit/common/test_colors.py new file mode 100644 index 00000000..6e14637e --- /dev/null +++ b/local_server/test/unit/common/test_colors.py @@ -0,0 +1,40 @@ +import unittest + +import anndata +from local_server.common.colors import convert_color_to_hex_format, convert_anndata_category_colors_to_cxg_category_colors +from local_server.common.errors import ColorFormatException +from local_server.test import PROJECT_ROOT +from local_server.test.fixtures.fixtures import pbmc3k_colors + + +class ColorsTest(unittest.TestCase): + """ Test color helper functions """ + + def test_convert_color_to_hex_format(self): + self.assertEqual(convert_color_to_hex_format("wheat"), "#f5deb3") + self.assertEqual(convert_color_to_hex_format("WHEAT"), "#f5deb3") + self.assertEqual(convert_color_to_hex_format((245, 222, 179)), "#f5deb3") + self.assertEqual(convert_color_to_hex_format([245, 222, 179]), "#f5deb3") + self.assertEqual(convert_color_to_hex_format("#f5deb3"), "#f5deb3") + self.assertEqual( + convert_color_to_hex_format([0.9607843137254902, 0.8705882352941177, 0.7019607843137254]), "#f5deb3" + ) + for bad_input in ["foo", "BAR", "#AABB", "#AABBCCDD", "#AABBGG", (1, 2), [1, 2], (1, 2, 3, 4), [1, 2, 3, 4]]: + with self.assertRaises(ColorFormatException): + convert_color_to_hex_format(bad_input) + + def test_anndata_colors_to_cxg_colors(self): + # test standard behavior + adata = self._get_h5ad() + self.assertEqual(convert_anndata_category_colors_to_cxg_category_colors(adata), pbmc3k_colors) + # test that invalid color formats raise an exception + adata.uns["louvain_colors"][0] = "#NOTCOOL" + with self.assertRaises(ColorFormatException): + convert_anndata_category_colors_to_cxg_category_colors(adata) + # test that colors without a matching obs category are skipped + adata = self._get_h5ad() + del adata.obs["louvain"] + self.assertEqual(convert_anndata_category_colors_to_cxg_category_colors(adata), {}) + + def _get_h5ad(self): + return anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad") diff --git a/local_server/test/unit/common/test_corpora.py b/local_server/test/unit/common/test_corpora.py new file mode 100644 index 00000000..589795e4 --- /dev/null +++ b/local_server/test/unit/common/test_corpora.py @@ -0,0 +1,164 @@ +import json +import shutil +import tempfile +import unittest +from http import HTTPStatus + +import anndata +import requests + +from local_server.common.corpora import ( + corpora_get_versions_from_anndata, + corpora_is_version_supported, + corpora_get_props_from_anndata, +) +from local_server.test import PROJECT_ROOT, start_test_server, stop_test_server + +VERSION = "v0.2" + + +class CorporaAPITest(unittest.TestCase): + def test_corpora_get_versions_from_anndata(self): + adata = self._get_h5ad() + + if "version" in adata.uns: + del adata.uns["version"] + self.assertIsNone(corpora_get_versions_from_anndata(adata)) + + # something bogus + adata.uns["version"] = 99 + self.assertIsNone(corpora_get_versions_from_anndata(adata)) + + # something legit + adata.uns["version"] = {"corpora_schema_version": "0.0.0", "corpora_encoding_version": "9.9.9"} + self.assertEqual(corpora_get_versions_from_anndata(adata), ["0.0.0", "9.9.9"]) + + def test_corpora_is_version_supported(self): + self.assertTrue(corpora_is_version_supported("1.0.0", "0.1.0")) + self.assertFalse(corpora_is_version_supported("0.0.0", "0.1.0")) + self.assertFalse(corpora_is_version_supported("1.0.0", "0.0.0")) + + def test_corpora_get_props_from_anndata(self): + adata = self._get_h5ad() + + if "version" in adata.uns: + del adata.uns["version"] + self.assertIsNone(corpora_get_props_from_anndata(adata)) + + # something bogus + adata.uns["version"] = 99 + self.assertIsNone(corpora_get_props_from_anndata(adata)) + + # unsupported version, but missing required values + adata.uns["version"] = {"corpora_schema_version": "99.0.0", "corpora_encoding_version": "32.1.0"} + with self.assertRaises(ValueError): + corpora_get_props_from_anndata(adata) + + # legit version, but missing required values + adata.uns["version"] = {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"} + with self.assertRaises(KeyError): + corpora_get_props_from_anndata(adata) + + some_fields = { + "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"}, + "title": "title", + "layer_descriptions": "layer_descriptions", + "organism": "organism", + "organism_ontology_term_id": "organism_ontology_term_id", + "project_name": "project_name", + "project_description": "project_description", + "contributors": json.dumps([{"contributors": "contributors"}]), + "project_links": json.dumps([{"link_name": "link_name", "link_url": "link_url", "link_type": "SUMMARY"}]), + } + for k in some_fields: + adata.uns[k] = some_fields[k] + some_fields["contributors"] = json.loads(some_fields["contributors"]) + some_fields["project_links"] = json.loads(some_fields["project_links"]) + self.assertEqual(corpora_get_props_from_anndata(adata), some_fields) + + def test_corpora_get_props_from_anndata_v110(self): + adata = self._get_h5ad() + + if "version" in adata.uns: + del adata.uns["version"] + self.assertIsNone(corpora_get_props_from_anndata(adata)) + + # legit version, but missing required values + adata.uns["version"] = {"corpora_schema_version": "1.1.0", "corpora_encoding_version": "0.1.0"} + with self.assertRaises(KeyError): + corpora_get_props_from_anndata(adata) + + # Metadata following schema 1.1.0, which removes some fields relative to 1.1.0 + some_110_fields = { + "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"}, + "title": "title", + "layer_descriptions": "layer_descriptions", + "organism": "organism", + "organism_ontology_term_id": "organism_ontology_term_id", + } + for k in some_110_fields: + adata.uns[k] = some_110_fields[k] + self.assertEqual(corpora_get_props_from_anndata(adata), some_110_fields) + + def _get_h5ad(self): + return anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad") + + +class CorporaRESTAPITest(unittest.TestCase): + """ Confirm endpoints reflect Corpora-specific features """ + + @classmethod + def setCorporaFields(cls, path): + adata = anndata.read_h5ad(path) + corpora_props = { + "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"}, + "title": "PBMC3K", + "contributors": json.dumps([{"name": "name"}]), + "layer_descriptions": {"X": "raw counts"}, + "organism": "human", + "organism_ontology_term_id": "unknown", + "project_name": "test project", + "project_description": "test description", + "project_links": json.dumps( + [{"link_name": "test link", "link_type": "SUMMARY", "link_url": "https://a.u.r.l/"}] + ), + "default_embedding": "X_tsne", + } + adata.uns.update(corpora_props) + adata.write(path) + + @classmethod + def setUpClass(cls): + cls.tmp_dir = tempfile.TemporaryDirectory() + src = f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad" + dst = f"{cls.tmp_dir.name}/pbmc3k.h5ad" + shutil.copyfile(src, dst) + cls.setCorporaFields(dst) + cls.ps, cls.server = start_test_server([dst]) + + @classmethod + def tearDownClass(cls): + stop_test_server(cls.ps) + cls.tmp_dir.cleanup() + + def setUp(self): + self.session = requests.Session() + self.url_base = f"{self.server}/api/{VERSION}/" + + def test_config(self): + endpoint = "config" + url = f"{self.url_base}{endpoint}" + result = self.session.get(url) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/json") + + result_data = result.json() + self.assertIsInstance(result_data["config"]["corpora_props"], dict) + self.assertIsInstance(result_data["config"]["parameters"], dict) + + corpora_props = result_data["config"]["corpora_props"] + parameters = result_data["config"]["parameters"] + + self.assertEqual(corpora_props["version"]["corpora_schema_version"], "1.0.0") + self.assertEqual(corpora_props["organism"], "human") + self.assertEqual(parameters["default_embedding"], "tsne") diff --git a/local_server/test/unit/common/test_nan_rest.py b/local_server/test/unit/common/test_nan_rest.py new file mode 100644 index 00000000..7ec01464 --- /dev/null +++ b/local_server/test/unit/common/test_nan_rest.py @@ -0,0 +1,61 @@ +from http import HTTPStatus +import unittest +import math +from local_server.test import start_test_server, stop_test_server + +import local_server.test.unit.decode_fbs as decode_fbs + +import requests + +VERSION = "v0.2" +BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}} + + +class WithNaNs(unittest.TestCase): + """Test Case for endpoints""" + + @classmethod + def setUpClass(cls): + cls.ps, cls.server = start_test_server(["test/fixtures/nan.h5ad"]) + + @classmethod + def tearDownClass(cls): + stop_test_server(cls.ps) + + def setUp(self): + self.session = requests.Session() + self.url_base = f"{self.server}/api/{VERSION}/" + + def test_initialize(self): + endpoint = "schema" + url = f"{self.url_base}{endpoint}" + result = self.session.get(url) + self.assertEqual(result.status_code, HTTPStatus.OK) + + def test_data(self): + endpoint = "data/var" + url = f"{self.url_base}{endpoint}" + filter = {"filter": {"var": {"index": [[0, 20]]}}} + result = self.session.put(url, json=filter) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertTrue(math.isnan(df["columns"][3][3])) + + def test_annotation_obs(self): + endpoint = "annotations/obs" + url = f"{self.url_base}{endpoint}" + result = self.session.get(url) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertTrue(math.isnan(df["columns"][2][0])) + + def test_annotation_var(self): + endpoint = "annotations/var" + url = f"{self.url_base}{endpoint}" + result = self.session.get(url) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/octet-stream") + df = decode_fbs.decode_matrix_FBS(result.content) + self.assertTrue(math.isnan(df["columns"][2][0])) diff --git a/local_server/test/unit/common/test_rest.py b/local_server/test/unit/common/test_rest.py new file mode 100644 index 00000000..82ae3ad5 --- /dev/null +++ b/local_server/test/unit/common/test_rest.py @@ -0,0 +1,79 @@ +import unittest +from urllib.parse import parse_qs +from werkzeug.datastructures import MultiDict +from local_server.common.rest import _query_parameter_to_filter +from local_server.common.errors import FilterError + + +def _qsparse(qs): + """ emulate what Flask/Werkzeug do to our QS """ + return MultiDict(parse_qs(qs)) + + +class FilterParseTests(unittest.TestCase): + """ Test cases for various filter parsing """ + + def test_queryparam_to_filter_parse(self): + # categories + self.assertEqual( + _query_parameter_to_filter(_qsparse("obs:foo=bar&var:baz=133&var:baz=A&obs:baz=foo")), + { + "obs": {"annotation_value": [{"name": "foo", "values": ["bar"]}, {"name": "baz", "values": ["foo"]}]}, + "var": {"annotation_value": [{"name": "baz", "values": ["133", "A"]}]}, + }, + ) + + # ranges + self.assertEqual( + _query_parameter_to_filter(_qsparse("obs:A=1,99&obs:B=*,100&obs:C=0,*")), + { + "obs": { + "annotation_value": [ + {"name": "A", "min": 1, "max": 99.0}, + {"name": "B", "max": 100.0}, + {"name": "C", "min": 0.0}, + ] + }, + }, + ) + + # combo + self.assertEqual( + _query_parameter_to_filter(_qsparse("var:B=YES&var:A=1,99&var:B=NO")), + { + "var": { + "annotation_value": [ + {"name": "B", "values": ["YES", "NO"]}, + {"name": "A", "min": 1.0, "max": 99.0}, + ] + }, + }, + ) + + def test_queryparam_to_filter_escaping(self): + self.assertEqual( + _query_parameter_to_filter(_qsparse("obs:var=%2521%252C%253AOK%253D&obs:A%2521=YO")), + {"obs": {"annotation_value": [{"name": "var", "values": ["!,:OK="]}, {"name": "A!", "values": ["YO"]}]}}, + ) + + def test_queryparam_to_filter_errors(self): + + # should raise FilterError + filter_errors = [ + "foo=bar", # no axis + "X=&Y=3", # no value + "X&Y=3", # no value + "moo:foo=bar", # bad axis + "obs:x=1,A", # non-numeric range + "var:X=1,2&var:X=3,4", # duplicate ranges + "var:Y=,", + "var:Y=2,", + "var:Y=,5", + "var:Y=*,", + "var:Y=,*", + "var:Y=*,*", + ] + + for qs in filter_errors: + with self.assertRaises(FilterError): + _query_parameter_to_filter(_qsparse(qs)) diff --git a/local_server/test/unit/common/test_writable_annotation.py b/local_server/test/unit/common/test_writable_annotation.py new file mode 100644 index 00000000..74170b8c --- /dev/null +++ b/local_server/test/unit/common/test_writable_annotation.py @@ -0,0 +1,168 @@ +import json +import shutil +import unittest +from os import path, listdir + +import numpy as np +import pandas as pd + +import local_server.test.unit.decode_fbs as decode_fbs +from local_server.common.rest import schema_get_helper, annotations_put_fbs_helper +from local_server.data_common.matrix_loader import MatrixDataType +from local_server.test import data_with_tmp_annotations, make_fbs + + +class WritableAnnotationTest(unittest.TestCase): + def setUp(self): + self.data, self.tmp_dir, self.annotations = data_with_tmp_annotations(MatrixDataType.H5AD) + self.data.dataset_config.user_annotations = self.annotations + + def tearDown(self): + shutil.rmtree(self.tmp_dir) + + def annotation_put_fbs(self, fbs): + annotations_put_fbs_helper(self.data, fbs) + res = json.dumps({"status": "OK"}) + return res + + def test_error_checks(self): + # verify that the expected errors are generated + n_rows = self.data.get_shape()[0] + fbs_bad = make_fbs({"louvain": pd.Series(["undefined"] * n_rows, dtype="category")}) + + # ensure we catch attempt to overwrite non-writable data + with self.assertRaises(KeyError): + self.annotation_put_fbs(fbs_bad) + + def test_write_to_file(self): + # verify the file is written as expected + n_rows = self.data.get_shape()[0] + fbs = make_fbs( + { + "cat_A": pd.Series(["label_A"] * n_rows, dtype="category"), + "cat_B": pd.Series(["label_B"] * n_rows, dtype="category"), + } + ) + res = self.annotation_put_fbs(fbs) + self.assertEqual(res, json.dumps({"status": "OK"})) + self.assertTrue(path.exists(self.annotations.output_file)) + df = pd.read_csv(self.annotations.output_file, index_col=0, header=0, comment="#") + self.assertEqual(df.shape, (n_rows, 2)) + self.assertEqual(set(df.columns), {"cat_A", "cat_B"}) + self.assertTrue(self.data.original_obs_index.equals(df.index)) + self.assertTrue(np.all(df["cat_A"] == ["label_A"] * n_rows)) + self.assertTrue(np.all(df["cat_B"] == ["label_B"] * n_rows)) + + # verify complete overwrite on second attempt, AND rotation occurs + fbs = make_fbs( + { + "cat_A": pd.Series(["label_A1"] * n_rows, dtype="category"), + "cat_C": pd.Series(["label_C"] * n_rows, dtype="category"), + } + ) + res = self.annotation_put_fbs(fbs) + self.assertEqual(res, json.dumps({"status": "OK"})) + self.assertTrue(path.exists(self.annotations.output_file)) + df = pd.read_csv(self.annotations.output_file, index_col=0, header=0, comment="#") + self.assertEqual(set(df.columns), {"cat_A", "cat_C"}) + self.assertTrue(np.all(df["cat_A"] == ["label_A1"] * n_rows)) + self.assertTrue(np.all(df["cat_C"] == ["label_C"] * n_rows)) + + # rotation + name, ext = path.splitext(self.annotations.output_file) + backup_dir = f"{name}-backups" + self.assertTrue(path.isdir(backup_dir)) + found_files = listdir(backup_dir) + self.assertEqual(len(found_files), 1) + + def test_file_rotation_to_max_9(self): + # verify we stop rotation at 9 + n_rows = self.data.get_shape()[0] + fbs = make_fbs( + { + "cat_A": pd.Series(["label_A"] * n_rows, dtype="category"), + "cat_B": pd.Series(["label_B"] * n_rows, dtype="category"), + } + ) + for i in range(0, 11): + res = self.annotation_put_fbs(fbs) + self.assertEqual(res, json.dumps({"status": "OK"})) + + name, ext = path.splitext(self.annotations.output_file) + backup_dir = f"{name}-backups" + self.assertTrue(path.isdir(backup_dir)) + found_files = listdir(backup_dir) + self.assertTrue(len(found_files) <= 9) + + def test_put_get_roundtrip(self): + # verify that OBS PUTs (annotation_put_fbs) are accessible via + # GET (annotation_to_fbs_matrix) + + n_rows = self.data.get_shape()[0] + fbs = make_fbs( + { + "cat_A": pd.Series(["label_A"] * n_rows, dtype="category"), + "cat_B": pd.Series(["label_B"] * n_rows, dtype="category"), + } + ) + + # put + res = self.annotation_put_fbs(fbs) + self.assertEqual(res, json.dumps({"status": "OK"})) + + # get + labels = self.annotations.read_labels(None) + fbsAll = self.data.annotation_to_fbs_matrix("obs", None, labels) + schema = schema_get_helper(self.data) + annotations = decode_fbs.decode_matrix_FBS(fbsAll) + obs_index_col_name = schema["annotations"]["obs"]["index"] + self.assertEqual(annotations["n_rows"], n_rows) + self.assertEqual(annotations["n_cols"], 7) + self.assertIsNone(annotations["row_idx"]) + self.assertEqual( + annotations["col_idx"], + [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain", "cat_A", "cat_B"], + ) + col_idx = annotations["col_idx"] + self.assertEqual(annotations["columns"][col_idx.index("cat_A")], ["label_A"] * n_rows) + self.assertEqual(annotations["columns"][col_idx.index("cat_B")], ["label_B"] * n_rows) + + # verify the schema was updated + all_col_schema = {c["name"]: c for c in schema["annotations"]["obs"]["columns"]} + self.assertEqual( + all_col_schema["cat_A"], + {"name": "cat_A", "type": "categorical", "categories": ["label_A"], "writable": True}, + ) + self.assertEqual( + all_col_schema["cat_B"], + {"name": "cat_B", "type": "categorical", "categories": ["label_B"], "writable": True}, + ) + + def test_put_float_data(self): + # verify that OBS PUTs (annotation_put_fbs) are accessible via + # GET (annotation_to_fbs_matrix) + + n_rows = self.data.get_shape()[0] + + # verifies that floating point with decimals fail. + fbs = make_fbs({"cat_F_FAIL": pd.Series([1.1] * n_rows, dtype=np.dtype("float"))}) + with self.assertRaises(ValueError) as exception_context: + res = self.annotation_put_fbs(fbs) + self.assertEqual(str(exception_context.exception), "Columns may not have floating point types") + + # verifies that floating point that can be converted to int passes + fbs = make_fbs({"cat_F_PASS": pd.Series([1.0] * n_rows, dtype="float")}) + res = self.annotation_put_fbs(fbs) + self.assertEqual(res, json.dumps({"status": "OK"})) + + # check read_labels + labels = self.annotations.read_labels(None) + fbsAll = self.data.annotation_to_fbs_matrix("obs", None, labels) + schema = schema_get_helper(self.data) + annotations = decode_fbs.decode_matrix_FBS(fbsAll) + self.assertEqual(annotations["n_rows"], n_rows) + all_col_schema = {c["name"]: c for c in schema["annotations"]["obs"]["columns"]} + self.assertEqual( + all_col_schema["cat_F_PASS"], + {"name": "cat_F_PASS", "type": "int32", "writable": True}, + ) diff --git a/local_server/test/unit/common/utils/__init__.py b/local_server/test/unit/common/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/test/unit/common/utils/test_type_conversion_utils.py b/local_server/test/unit/common/utils/test_type_conversion_utils.py new file mode 100644 index 00000000..179f6a70 --- /dev/null +++ b/local_server/test/unit/common/utils/test_type_conversion_utils.py @@ -0,0 +1,217 @@ +import unittest +from time import time +from unittest.mock import patch + +import numpy as np +from pandas import Series, DataFrame + +from local_server.common.utils.type_conversion_utils import ( + can_cast_to_float32, + can_cast_to_int32, + get_dtype_of_array, + get_schema_type_hint_of_array, + get_dtypes_and_schemas_of_dataframe, + convert_pandas_series_to_numpy, +) + + +class TestTypeConversionUtils(unittest.TestCase): + def test__can_cast_to_float32__string_is_false(self): + array_to_convert = Series(data=["1", "2", "3"], dtype=str) + + can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert) + + self.assertFalse(can_cast) + + def test__can_cast_to_float32__float64_is_true_warning_outputted(self): + array_to_convert = Series(data=[1, 2, 3], dtype=np.dtype(np.float64)) + + with self.assertLogs(level="WARN") as logger: + can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert) + self.assertIn("may lose precision", logger.output[0]) + + self.assertTrue(can_cast) + + @patch("logging.warning") + def test__can_cast_to_float32__float32_is_false(self, mock_log_warning): + array_to_convert = Series(data=[1, 2, 3], dtype=np.dtype(np.float32)) + + can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert) + + self.assertTrue(can_cast) + assert not mock_log_warning.called + + def test__can_cast_to_float32__categorical_float64_is_false(self): + array_to_convert = Series(data=[1.1, 2.2, 3.3], dtype="category") + + can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert) + + self.assertFalse(can_cast) + + def test__can_cast_to_float32__categorical_int64_with_nans_is_true(self): + array_to_convert = Series(data=[1, 2, np.NaN], dtype="category") + + can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert) + + self.assertTrue(can_cast) + + def test__can_cast_to_float_32__float_32_with_nans_is_true(self): + array_to_convert = Series(data=[1, 2, np.NaN], dtype=np.dtype(np.float32)) + + can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert) + + self.assertTrue(can_cast) + + def test__can_cast_to_int32__string_is_false(self): + array_to_convert = Series(data=["1", "2", "3"], dtype=str) + + can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert) + + self.assertFalse(can_cast) + + def test__can_cast_to_int32__int64_is_true(self): + array_to_convert = Series(data=["1", "2", "3"], dtype=np.dtype(np.int64)) + + can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert) + + self.assertTrue(can_cast) + + def test__can_cast_to_int32__int16_is_true(self): + array_to_convert = Series(data=["1", "2", "3"], dtype=np.dtype(np.int16)) + + can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert) + + self.assertTrue(can_cast) + + def test__can_cast_to_int32__int64_with_large_value_is_false(self): + array_to_convert = Series(data=["3000000000", "2", "3"], dtype=np.dtype(np.int64)) + + can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert) + + self.assertFalse(can_cast) + + def test__can_cast_to_int32__int64_with_nans_is_false(self): + array_to_convert = Series(data=[np.NaN, "2", "3"], dtype="category") + + can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert) + + self.assertFalse(can_cast) + + def test__get_dtype_of_array__supported_dtypes_return_as_expected(self): + types = [np.float32, np.int32, np.bool_, str] + expected_dtypes = [np.float32, np.int32, np.uint8, str] + + for test_type_index in range(len(types)): + with self.subTest( + f"Testing get_dtype_of_array with type {types[test_type_index].__name__}", i=test_type_index + ): + array = Series(data=[], dtype=types[test_type_index]) + self.assertEqual(get_dtype_of_array(array), expected_dtypes[test_type_index]) + + def test__get_dtype_of_array__categories_return_as_expected(self): + array = Series(data=["a", "b", "c"], dtype="category") + expected_dtype = str + + actual_dtype = get_dtype_of_array(array) + + self.assertEqual(expected_dtype, actual_dtype) + + def test__get_dtype_of_array__unordered_integer_categories_return_as_expected(self): + array = Series(data=[2, 3, 1, 3, 1, 2], dtype="category") + expected_dtype = np.int32 + + actual_dtype = get_dtype_of_array(array) + + self.assertEqual(expected_dtype, actual_dtype) + + def test__get_dtype_of_array__castable_dtypes_return_as_expected(self): + types = [np.float64, np.int64] + expected_dtypes = [np.float32, np.int32] + + for test_type_index in range(len(types)): + with self.subTest( + f"Testing get_dtype_of_array with castable type {types[test_type_index].__name__}", i=test_type_index + ): + array = Series(data=[], dtype=types[test_type_index]) + self.assertEqual(get_dtype_of_array(array), expected_dtypes[test_type_index]) + + def test__get_dtype_of_array__unsupported_type_raises_exception(self): + unsupported_array = Series(list([time() for _ in range(2)]), dtype="datetime64[ns]") + + with self.assertRaises(TypeError) as exception_context: + get_dtype_of_array(unsupported_array) + + self.assertIn("unsupported", str(exception_context.exception)) + + def test__get_schema_type_hint_of_array__supported_dtypes_return_as_expected(self): + types = [np.float32, np.int32, np.bool_, str] + expected_schema_hints = [{"type": "float32"}, {"type": "int32"}, {"type": "boolean"}, {"type": "string"}] + + for test_type_index in range(len(types)): + with self.subTest( + f"Testing get_schema_type_hint_of_array with type {types[test_type_index].__name__}", i=test_type_index + ): + array = Series(data=[], dtype=types[test_type_index]) + self.assertEqual(get_schema_type_hint_of_array(array), expected_schema_hints[test_type_index]) + + def test__get_schema_type_hint_of_array__categories_return_as_expected(self): + array = Series(data=["a", "b", "b"], dtype="category") + expected_schema_hint = {"type": "categorical", "categories": ["a", "b"]} + + actual_schema_hint = get_schema_type_hint_of_array(array) + + self.assertEqual(expected_schema_hint, actual_schema_hint) + + def test__get_schema_type_hint_of_array__castable_dtypes_return_as_expected(self): + types = [np.float64, np.int64] + expected_schema_hints = [{"type": "float32"}, {"type": "int32"}] + + for test_type_index in range(len(types)): + with self.subTest( + f"Testing get_schema_type_hint_of_array with castable type {types[test_type_index].__name__}", + i=test_type_index, + ): + array = Series(data=[], dtype=types[test_type_index]) + self.assertEqual(get_schema_type_hint_of_array(array), expected_schema_hints[test_type_index]) + + def test__get_dtypes_and_schemas_of_dataframe__dtype_and_schema_returns_as_expected(self): + float_array = Series(data=[1, 2, 3], dtype=np.dtype(np.float64)) + category_array = Series(data=["a", "b", "b"], dtype="category") + dataframe = DataFrame({"float_array": float_array, "category_array": category_array}) + + expected_data_types_dict = {"float_array": np.float32, "category_array": str} + expected_schema_type_hints_dict = { + "float_array": {"type": "float32"}, + "category_array": {"type": "categorical", "categories": ["a", "b"]}, + } + + actual_dataframe_data_types, actual_dataframe_schema_type_hints = get_dtypes_and_schemas_of_dataframe(dataframe) + + self.assertEqual(expected_data_types_dict, actual_dataframe_data_types) + self.assertEqual(expected_schema_type_hints_dict, actual_dataframe_schema_type_hints) + + def test__convert_pandas_series_to_numpy__categorical_float64_to_float64_with_nans(self): + expected_float_array = np.array([1.1, 2.2, np.NaN], dtype=np.float64) + float_series = Series(data=[1.1, 2.2, np.NaN], dtype="category") + + actual_float_array = convert_pandas_series_to_numpy(float_series, np.float64) + + np.testing.assert_equal(expected_float_array, actual_float_array) + + def test__convert_pandas_series_to_numpy__float64_to_float64(self): + expected_float_array = np.array([1.1, 2.2], dtype=np.float64) + float_series = Series(data=[1.1, 2.2], dtype=np.dtype(np.float64)) + + actual_float_array = convert_pandas_series_to_numpy(float_series, np.float64) + + np.testing.assert_equal(expected_float_array, actual_float_array) + + def test__convert_pandas_series_to_numpy__int64_to_int32_with_nans_throws_error(self): + int_series = Series(data=[1, 2, np.NaN], dtype="category") + + with self.assertLogs(level="ERROR") as logger: + convert_pandas_series_to_numpy(int_series, np.int32) + + self.assertIn( + "Cannot convert a pandas Series object to an integer dtype if it contains NaNs", logger.output[0] + ) diff --git a/local_server/test/unit/common/utils/test_utils.py b/local_server/test/unit/common/utils/test_utils.py new file mode 100644 index 00000000..852c0969 --- /dev/null +++ b/local_server/test/unit/common/utils/test_utils.py @@ -0,0 +1,34 @@ +import os +import shutil +import unittest + +from local_server.common.utils.utils import import_plugins +from local_server.test import PROJECT_ROOT, random_string + + +class TestPlugins(unittest.TestCase): + """ Test plugin import functionality """ + + plugins_dir = f"{PROJECT_ROOT}/local_server/test/plugins" + test_plugin_path = f"{plugins_dir}/foo.py" + secret = random_string(8) + + @classmethod + def setUpClass(cls) -> None: + if not os.path.isdir(cls.plugins_dir): + os.mkdir(cls.plugins_dir) + with open(cls.test_plugin_path, "w") as fh: + fh.write(f'SECRET = "{cls.secret}"\n') + + @classmethod + def tearDownClass(cls) -> None: + if os.path.isdir(cls.plugins_dir): + shutil.rmtree(cls.plugins_dir) + + def test_import_plugins(self): + self.assertTrue(os.path.isfile(self.test_plugin_path)) + loaded_modules = import_plugins("local_server.test.plugins") + # test that import plugins found the file + self.assertEqual(["local_server.test.plugins.foo"], [ele.__name__ for ele in loaded_modules]) + # test that the module was properly executed + self.assertEqual(self.secret, loaded_modules[0].SECRET) diff --git a/local_server/test/unit/compute/__init__.py b/local_server/test/unit/compute/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/test/unit/compute/test_diffexp_cxg.py b/local_server/test/unit/compute/test_diffexp_cxg.py new file mode 100644 index 00000000..dd5020b4 --- /dev/null +++ b/local_server/test/unit/compute/test_diffexp_cxg.py @@ -0,0 +1,62 @@ +import unittest + +import numpy as np + +from local_server.data_common.matrix_loader import MatrixDataLoader +from local_server.test import PROJECT_ROOT, app_config + + +class DiffExpTest(unittest.TestCase): + """Tests the diffexp returns the expected results for one test case, using different + adaptor types and different algorithms.""" + + def load_dataset(self, path, extra_server_config={}, extra_dataset_config={}): + config = app_config(path, extra_server_config=extra_server_config, extra_dataset_config=extra_dataset_config) + loader = MatrixDataLoader(path) + adaptor = loader.open(config) + return adaptor + + def get_mask(self, adaptor, start, stride): + """Simple function to return a mask or rows""" + rows = adaptor.get_shape()[0] + sel = list(range(start, rows, stride)) + mask = np.zeros(rows, dtype=bool) + mask[sel] = True + return mask + + def compare_diffexp_results(self, results, expects): + self.assertEqual(len(results), len(expects)) + for result, expect in zip(results, expects): + self.assertEqual(result[0], expect[0]) + self.assertTrue(np.isclose(result[1], expect[1], 1e-6, 1e-4)) + self.assertTrue(np.isclose(result[2], expect[2], 1e-6, 1e-4)) + self.assertTrue(np.isclose(result[3], expect[3], 1e-6, 1e-4)) + + def check_1_10_2_10(self, results): + """Checks the results for a specific set of rows selections""" + expects = [ + [956, 0.016060986, 0.0008649321884808977, 1.0], + [1124, 0.96602094, 0.0011717216548271284, 1.0], + [1809, 1.1110606, 0.0019304405196777848, 1.0], + [1712, -0.5525154, 0.0051788902660723345, 1.0], + [1754, 0.5201581, 0.005691734062127954, 1.0], + [948, 1.6390722, 0.006622111055981219, 1.0], + [1810, 0.78618884, 0.007055917428377063, 1.0], + [779, 1.5241305, 0.007202934422407284, 1.0], + [1575, 1.0317602, 0.007830310753043345, 1.0], + [576, 0.97873515, 0.008272092578813124, 1.0], + ] + self.compare_diffexp_results(results, expects) + + def get_X_col(self, adaptor, cols): + varmask = np.zeros(adaptor.get_shape()[1], dtype=bool) + varmask[cols] = True + return adaptor.get_X_array(None, varmask) + + def test_anndata_default(self): + """Test an anndata adaptor with its default diffexp algorithm (diffexp_generic)""" + adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad") + maskA = self.get_mask(adaptor, 1, 10) + maskB = self.get_mask(adaptor, 2, 10) + results = adaptor.compute_diffexp_ttest(maskA, maskB, 10) + self.check_1_10_2_10(results) diff --git a/local_server/test/unit/converters/__init__.py b/local_server/test/unit/converters/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/test/unit/converters/schema/__init__.py b/local_server/test/unit/converters/schema/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/test/unit/converters/schema/test_gene_symbol.py b/local_server/test/unit/converters/schema/test_gene_symbol.py new file mode 100644 index 00000000..fcd400c8 --- /dev/null +++ b/local_server/test/unit/converters/schema/test_gene_symbol.py @@ -0,0 +1,61 @@ +import os +import unittest + +import pandas as pd + +from local_server.test import FIXTURES_ROOT +from local_server.converters.schema import gene_symbol + + +class TestHGNCSymbolChecker(unittest.TestCase): + + def setUp(self): + self.test_hgnc_path = os.path.join(FIXTURES_ROOT, "hgnc_example.txt.gz") + self.hgnc_checker = gene_symbol.HGNCSymbolChecker.from_hgnc_records(self.test_hgnc_path) + + def test_symbol_upgrade(self): + self.assertEqual(self.hgnc_checker.upgrade_symbol("SEPT1"), "SEPTIN1") + self.assertEqual(self.hgnc_checker.upgrade_symbol("ADRB2R"), "ADRB2") + self.assertEqual(self.hgnc_checker.upgrade_symbol("BAR"), "ADRB2") + self.assertEqual(self.hgnc_checker.upgrade_symbol("sept1"), "SEPTIN1") + self.assertEqual(self.hgnc_checker.upgrade_symbol("AdRb2R"), "ADRB2") + self.assertEqual(self.hgnc_checker.upgrade_symbol("bar"), "ADRB2") + + # Strip off seurat endings when appropriate + self.assertEqual(self.hgnc_checker.upgrade_symbol("SEPT1.1"), "SEPTIN1") + self.assertEqual(self.hgnc_checker.upgrade_symbol("ADRB2-1"), "ADRB2") + + # DIFF6 is ambiguous so don't upgrade it + self.assertEqual(self.hgnc_checker.upgrade_symbol("DIFF6"), "DIFF6") + self.assertEqual(self.hgnc_checker.upgrade_symbol("diff6"), "diff6") + + # ARG1 is approved + self.assertEqual(self.hgnc_checker.upgrade_symbol("ARG1"), "ARG1") + self.assertEqual(self.hgnc_checker.upgrade_symbol("arg1"), "ARG1") + + # HAP1 is both approved and withdrawn + self.assertEqual(self.hgnc_checker.upgrade_symbol("HAP1"), "HAP1") + self.assertEqual(self.hgnc_checker.upgrade_symbol("hap1"), "HAP1") + + # Leave unknown symbols alone + self.assertEqual(self.hgnc_checker.upgrade_symbol("NOTASYMBOL"), "NOTASYMBOL") + self.assertEqual(self.hgnc_checker.upgrade_symbol("notasymbol"), "notasymbol") + + # Upgrade HGNC ids unless you can't find it + self.assertEqual(self.hgnc_checker.upgrade_symbol("HGNC:286"), "ADRB2") + self.assertEqual(self.hgnc_checker.upgrade_symbol("HGNC:4812"), "HAP1") + self.assertEqual(self.hgnc_checker.upgrade_symbol("HGNC:123456"), "HGNC:123456") + + def test_check_symbol(self): + self.assertEqual(self.hgnc_checker.check_symbol("SEPT1"), gene_symbol.SymbolStatus.UPGRADABLE) + self.assertEqual(self.hgnc_checker.check_symbol("DIFF6"), gene_symbol.SymbolStatus.AMBIGUOUS) + self.assertEqual(self.hgnc_checker.check_symbol("NOTASYMBOL"), gene_symbol.SymbolStatus.UNKNOWN) + + # HAP1 is one of the approved and withdrawn symbols + self.assertEqual(self.hgnc_checker.check_symbol("HAP1"), gene_symbol.SymbolStatus.APPROVED) + + def test_upgrade_index(self): + index = pd.Index(["SEPT1", "DIFF6", "NOTASYMBOL", "bar", "SEPTIN1"]) + var_df = pd.DataFrame([[0] * len(index)], index=index) + upgraded_index = gene_symbol.get_upgraded_var_index(var_df, hgnc_path=self.test_hgnc_path) + self.assertEqual(upgraded_index.tolist(), ["SEPTIN1", "DIFF6", "NOTASYMBOL", "ADRB2", "SEPTIN1"]) diff --git a/local_server/test/unit/converters/schema/test_ontology.py b/local_server/test/unit/converters/schema/test_ontology.py new file mode 100644 index 00000000..25aa7f1e --- /dev/null +++ b/local_server/test/unit/converters/schema/test_ontology.py @@ -0,0 +1,129 @@ +import json + +import unittest +import unittest.mock + +from local_server.converters.schema import ontology + + +class TestOntologyParsing(unittest.TestCase): + def setUp(self): + + self.curies = ["UBERON:0002048", "HsapDv:0000174", "NCBITaxon:9606", "EFO:0008995"] + + self.names = ["UBERON", "HsapDv", "NCBITaxon", "EFO"] + + self.values = ["0002048", "0000174", "9606", "0008995"] + + self.iris = [ + "http://purl.obolibrary.org/obo/UBERON_0002048", + "http://purl.obolibrary.org/obo/HsapDv_0000174", + "http://purl.obolibrary.org/obo/NCBITaxon_9606", + "http://www.ebi.ac.uk/efo/EFO_0008995", + ] + + URL_ROOT = "http://www.ebi.ac.uk/ols/api/ontologies/" + self.urls = [ + URL_ROOT + "UBERON/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FUBERON_0002048", + URL_ROOT + "HsapDv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FHsapDv_0000174", + URL_ROOT + "NCBITaxon/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_9606", + URL_ROOT + "EFO/terms/http%253A%252F%252Fwww.ebi.ac.uk%252Fefo%252FEFO_0008995", + ] + + self.responses = { + "UBERON:0002048": { + "iri": "http://purl.obolibrary.org/obo/UBERON_0002048", + "description": ["Respiration organ that develops as an outpocketing of the esophagus."], + "label": "lung", + }, + "HsapDv:0000174": { + "iri": "http://purl.obolibrary.org/obo/HsapDv_0000174", + "description": ["Infant stage that refers to an infant who is over 1 and under 2 months old."], + "label": "1-month-old human stage", + }, + "NCBITaxon:9606": { + "iri": "http://purl.obolibrary.org/obo/NCBITaxon_9606", + "description": None, + "label": "Homo sapiens", + }, + "EFO:0008995": { + "iri": "http://www.ebi.ac.uk/efo/EFO_0008995", + "description": [ + ( + '10X is a "synthetic long-read" technology and works by capturing a barcoded oligo-coated ' + "gel-bead and 0.3x genome copies into a single emulsion droplet, processing the equivalent " + "of 1 million pipetting steps. Successive versions of the 10x chemistry use different " + "barcode locations to improve the sequencing yield and quality of 10x experiments." + ) + ], + "label": "10X sequencing", + }, + } + + def test_ontololgy_name(self): + for curie, expected_name in zip(self.curies, self.names): + self.assertEqual(ontology._ontology_name(curie), expected_name) + + def test_ontololgy_value(self): + for curie, expected_value in zip(self.curies, self.values): + self.assertEqual(ontology._ontology_value(curie), expected_value) + + def test_iri(self): + for curie, expected_iri in zip(self.curies, self.iris): + self.assertEqual(ontology._iri(curie), expected_iri) + + def test_ontology_info_url(self): + for curie, expected_url in zip(self.curies, self.urls): + self.assertEqual(ontology._ontology_info_url(curie), expected_url) + + def test_empty_ontology_info_url(self): + self.assertEqual(ontology._ontology_info_url(""), "") + + +class TestOntologyLookup(unittest.TestCase): + def setUp(self): + self.responses = { + "UBERON:0002048": { + "iri": "http://purl.obolibrary.org/obo/UBERON_0002048", + "description": ["Respiration organ that develops as an outpocketing of the esophagus."], + "label": "lung", + }, + "HsapDv:0000174": { + "iri": "http://purl.obolibrary.org/obo/HsapDv_0000174", + "description": ["Infant stage that refers to an infant who is over 1 and under 2 months old."], + "label": "1-month-old human stage", + }, + "NCBITaxon:9606": { + "iri": "http://purl.obolibrary.org/obo/NCBITaxon_9606", + "description": None, + "label": "Homo sapiens", + }, + "EFO:0008995": { + "iri": "http://www.ebi.ac.uk/efo/EFO_0008995", + "description": [ + ('10X is a "synthetic long-read" technology and works by capturing a barcoded oligo-coated ' + 'gel-bead and 0.3x genome copies into a single emulsion droplet, processing the equivalent ' + 'of 1 million pipetting steps. Successive versions of the 10x chemistry use different barcode ' + 'locations to improve the sequencing yield and quality of 10x experiments.') + ], + "label": "10X sequencing", + }, + } + + self.labels = { + "UBERON:0002048": "lung", + "HsapDv:0000174": "1-month-old human stage", + "NCBITaxon:9606": "Homo sapiens", + "EFO:0008995": "10X sequencing", + } + + @unittest.mock.patch("requests.get") + def test_lookup_label(self, mock_get): + + for curie, response in self.responses.items(): + mock_get.return_value.content = json.dumps(response) + mock_get.return_value.json.return_value = response + mock_get.return_value.status_code = 200 + + label = ontology.get_ontology_label(curie) + self.assertEqual(label, self.labels[curie]) diff --git a/local_server/test/unit/converters/schema/test_remix.py b/local_server/test/unit/converters/schema/test_remix.py new file mode 100644 index 00000000..49713d29 --- /dev/null +++ b/local_server/test/unit/converters/schema/test_remix.py @@ -0,0 +1,257 @@ +import json +import os +import unittest +import unittest.mock + +import anndata +import numpy +import pandas as pd +import scanpy as sc + +from local_server.converters.schema import remix + +PROJECT_ROOT = os.popen("git rev-parse --show-toplevel").read().strip() + + +class TestApplySchema(unittest.TestCase): + + def setUp(self): + self.source_h5ad_path = f"{PROJECT_ROOT}/local_server/test/fixtures/pbmc3k-CSC-gz.h5ad" + self.output_h5ad_path = f"{PROJECT_ROOT}/local_server/test/fixtures/test_remix.h5ad" + self.config_path = f"{PROJECT_ROOT}/local_server/test/fixtures/test_config.yaml" + self.bad_config_path = f"{PROJECT_ROOT}/local_server/test/fixtures/test_bad_config.yaml" + + def tearDown(self): + try: + os.remove(self.output_h5ad_path) + except OSError: + pass + + @unittest.mock.patch("local_server.converters.schema.ontology.get_ontology_label") + def test_apply_schema(self, mock_get_ontology_label): + mock_get_ontology_label.return_value = "test label" + remix.apply_schema(self.source_h5ad_path, self.config_path, self.output_h5ad_path) + new_adata = sc.read_h5ad(self.output_h5ad_path) + + self.assertIn("cell_type", new_adata.obs.columns) + self.assertListEqual(["test label"], new_adata.obs["cell_type"].unique().tolist()) + self.assertListEqual( + ["CL:00001", "CL:00002", "CL:00003", "CL:00004", "CL:00005", "CL:00006", "CL:00007", "CL:00008"], + sorted(new_adata.obs["cell_type_ontology_term_id"].unique().tolist()) + ) + + self.assertIn("version", new_adata.uns_keys()) + + @unittest.mock.patch("local_server.converters.schema.ontology.get_ontology_label") + def test_apply_bad_schema(self, mock_get_ontology_label): + mock_get_ontology_label.return_value = "test label" + remix.apply_schema(self.source_h5ad_path, self.bad_config_path, self.output_h5ad_path) + new_adata = sc.read_h5ad(self.output_h5ad_path) + + # Should refuse to write the version + self.assertNotIn("version", new_adata.uns_keys()) + +class TestFieldParsing(unittest.TestCase): + + def test_is_curie(self): + self.assertTrue(remix.is_curie("EFO:00001")) + self.assertTrue(remix.is_curie("UBERON:123456")) + self.assertTrue(remix.is_curie("HsapDv:0001")) + self.assertFalse(remix.is_curie("UBERON")) + self.assertFalse(remix.is_curie("UBERON:")) + self.assertFalse(remix.is_curie("123456")) + + def test_is_ontology_field(self): + self.assertTrue(remix.is_ontology_field("tissue_ontology_term_id")) + self.assertTrue(remix.is_ontology_field("cell_type_ontology_term_id")) + self.assertFalse(remix.is_ontology_field("cell_ontology")) + self.assertFalse(remix.is_ontology_field("method")) + + def test_get_label_field_name(self): + self.assertEqual("tissue", remix.get_label_field_name("tissue_ontology_term_id")) + self.assertEqual("cell_type", remix.get_label_field_name("cell_type_ontology_term_id")) + + def test_split_suffix(self): + self.assertEqual(("UBERON:1234", " (organoid)"), remix.split_suffix("UBERON:1234 (organoid)")) + self.assertEqual(("UBERON:1234", " (cell culture)"), remix.split_suffix("UBERON:1234 (cell culture)")) + self.assertEqual(("UBERON:1234", ""), remix.split_suffix("UBERON:1234")) + self.assertEqual(("UBERON:1234 (something)", ""), remix.split_suffix("UBERON:1234 (something)")) + + @unittest.mock.patch("local_server.converters.schema.ontology.get_ontology_label") + def test_get_curie_and_label(self, mock_get_ontology_label): + mock_get_ontology_label.return_value = "test label" + self.assertEqual( + remix.get_curie_and_label("UBERON:1234"), + ("UBERON:1234", "test label") + ) + self.assertEqual( + remix.get_curie_and_label("UBERON:1234 (cell culture)"), + ("UBERON:1234 (cell culture)", "test label (cell culture)") + ) + self.assertEqual( + remix.get_curie_and_label("whatever"), + ("", "whatever") + ) + + +class TestManipulateAnndata(unittest.TestCase): + + def setUp(self): + + self.cell_count = 20 + self.gene_count = 200 + X = numpy.random.randint(0, 1000, (self.cell_count, self.gene_count)) + uns = {"organism": "monkey", "experiment": "monkey experiment"} + obs = pd.DataFrame( + index=[f"Cell{d}" for d in range(self.cell_count)], + columns=["tissue", "CellType"], + data=[["lung", "epithelial"]] * (self.cell_count // 2) + [["lung", "endothelial"]] * (self.cell_count // 2) + ) + var = pd.DataFrame(index=[f"SEPT{d}" for d in range(self.gene_count)]) + + self.adata = anndata.AnnData(X=X, obs=obs, var=var, uns=uns) + + def test_safe_add_field(self): + + remix.safe_add_field(self.adata.obs, "tissue", ["monkey lung"] * self.cell_count) + self.assertEqual(self.adata.obs["tissue_original"].tolist(), ["lung"] * self.cell_count) + self.assertEqual(self.adata.obs["tissue"].tolist(), ["monkey lung"] * self.cell_count) + + remix.safe_add_field(self.adata.uns, "contributors", [{"name": "contributor1"}, {"name": "contributor2"}]) + self.assertEqual( + self.adata.uns["contributors"], + json.dumps([{"name": "contributor1"}, {"name": "contributor2"}]) + ) + + @unittest.mock.patch("local_server.converters.schema.ontology.get_ontology_label") + def test_remix_uns(self, mock_get_ontology_label): + mock_get_ontology_label.return_value = "Pan troglodytes" + uns_config = { + "version": { + "corpora_schema_version": "1.0.0", + "corpora_encoding_version": "0.1.0" + }, + "organism_ontology_term_id": "NCBITaxon:9598", + "contributors": [ + { + "name": "scientist", + "email": "scientist@science.com" + } + ] + } + + remix.remix_uns(self.adata, uns_config) + + self.assertEqual( + sorted(self.adata.uns_keys()), + sorted(["organism_original", "organism", "organism_ontology_term_id", + "contributors", "version", "experiment"]) + ) + + self.assertEqual(self.adata.uns['organism'], "Pan troglodytes") + self.assertEqual(self.adata.uns['organism_original'], "monkey") + self.assertEqual(self.adata.uns['organism_ontology_term_id'], "NCBITaxon:9598") + self.assertEqual(self.adata.uns['contributors'], + json.dumps([{"name": "scientist", "email": "scientist@science.com"}])) + + @unittest.mock.patch("local_server.converters.schema.ontology.get_ontology_label") + def test_remix_obs(self, mock_get_ontology_label): + mock_get_ontology_label.return_value = "lung (in a monkey)" + obs_config = { + "tissue_ontology_term_id": { + "tissue": { + "lung": "UBERON:00000" + } + }, + "cell_color": { + "CellType": { + "epithelial": "fuschia", + "endothelial": "khaki" + } + }, + "sex": "male" + } + + remix.remix_obs(self.adata, obs_config) + self.assertEqual( + sorted(self.adata.obs_keys()), + sorted(["tissue", "tissue_ontology_term_id", "tissue_original", "CellType", "cell_color", "sex"]) + ) + + self.assertTrue(all(v == "lung" for v in self.adata.obs.tissue_original)) + self.assertTrue(all(v == "UBERON:00000" for v in self.adata.obs.tissue_ontology_term_id)) + self.assertTrue(all(v == "lung (in a monkey)" for v in self.adata.obs.tissue)) + self.assertTrue(all(v == "male" for v in self.adata.obs.sex)) + self.assertTrue(all(v in (("epithelial", "fuschia"), ("endothelial", "khaki")) + for v in zip(self.adata.obs.CellType, self.adata.obs.cell_color))) + + +class TestFixupGeneSymbols(unittest.TestCase): + + def setUp(self): + self.seurat_path = f"{PROJECT_ROOT}/local_server/test/fixtures/schema_test_data/seurat_tutorial.h5ad" + self.seurat_merged_path = f"{PROJECT_ROOT}/local_server/test/fixtures/schema_test_data/seurat_tutorial_merged.h5ad" + self.sctransform_path = f"{PROJECT_ROOT}/local_server/test/fixtures/schema_test_data/sctransform.h5ad" + self.sctransform_merged_path = f"{PROJECT_ROOT}/local_server/test/fixtures/schema_test_data/sctransform_merged.h5ad" + + # There's lots of MALAT1, but it doesn't collide with any other names, + # so it shouldn't change during merging. + self.stable_gene = "MALAT1" + + def test_fixup_gene_symbols_seurat(self): + + if not os.path.isfile(self.seurat_path): + return unittest.skip( + "Skipping gene symbol conversion tests because test h5ads are not present. To create them, " + "run local_server/test/fixtures/schema_test_data/generate_test_data.sh" + ) + + original_adata = sc.read_h5ad(self.seurat_path) + merged_adata = sc.read_h5ad(self.seurat_merged_path) + + fixup_config = {"X": "log1p", "counts": "raw", "scale.data": "log1p"} + + fixed_adata = remix.fixup_gene_symbols(original_adata, fixup_config) + + self.assertEqual( + merged_adata.layers["counts"][:, merged_adata.var.index == self.stable_gene].sum(), + fixed_adata.raw.X[:, fixed_adata.var.index == self.stable_gene].sum() + ) + self.assertAlmostEqual( + merged_adata.X[:, merged_adata.var.index == self.stable_gene].sum(), + fixed_adata.X[:, fixed_adata.var.index == self.stable_gene].sum() + ) + + self.assertAlmostEqual( + merged_adata.layers["scale.data"][:, merged_adata.var.index == self.stable_gene].sum(), + fixed_adata.layers["scale.data"][:, fixed_adata.var.index == self.stable_gene].sum() + ) + + def test_fixup_gene_symbols_sctransform(self): + + if not os.path.isfile(self.sctransform_path): + return unittest.skip( + "Skipping gene symbol conversion tests because test h5ads are not present. To create them, " + "run local_server/test/fixtures/schema_test_data/generate_test_data.sh" + ) + + original_adata = sc.read_h5ad(self.sctransform_path) + merged_adata = sc.read_h5ad(self.sctransform_merged_path) + + fixup_config = {"X": "log1p", "counts": "raw"} + + fixed_adata = remix.fixup_gene_symbols(original_adata, fixup_config) + + # sctransform does a bunch of stuff, including slightly modifying the + # raw counts. So we can't assert for exact equality the way we do with + # the vanilla seurat tutorial. But, the results should still be very + # close. + merged_raw_stable = merged_adata.layers["counts"][:, merged_adata.var.index == self.stable_gene].sum() + fixed_raw_stable = fixed_adata.raw.X[:, fixed_adata.var.index == self.stable_gene].sum() + self.assertLess(abs(merged_raw_stable - fixed_raw_stable), .001 * merged_raw_stable) + + self.assertAlmostEqual( + merged_adata.X[:, merged_adata.var.index == self.stable_gene].sum(), + fixed_adata.X[:, fixed_adata.var.index == self.stable_gene].sum(), + 0 + ) diff --git a/local_server/test/unit/converters/schema/test_validate.py b/local_server/test/unit/converters/schema/test_validate.py new file mode 100644 index 00000000..269e629b --- /dev/null +++ b/local_server/test/unit/converters/schema/test_validate.py @@ -0,0 +1,435 @@ +import json +import os +import unittest + +import pandas as pd +import scanpy as sc + +from local_server.converters.schema import validate + +PROJECT_ROOT = os.popen("git rev-parse --show-toplevel").read().strip() + + +class TestFieldValidation(unittest.TestCase): + + def test_validate_stringified_list_of_dicts(self): + + good = json.dumps([{"a": 1}, {2: "x", "z": "y"}]) + not_stringified = [{"a": 1}, {2: "x", "z": "y"}] + not_a_list = json.dumps({"bad": "dict"}) + not_json = "oh hey!" + + self.assertTrue(validate._validate_stringified_list_of_dicts(good)) + + self.assertFalse(validate._validate_stringified_list_of_dicts(not_stringified)) + self.assertFalse(validate._validate_stringified_list_of_dicts(not_a_list)) + self.assertFalse(validate._validate_stringified_list_of_dicts(not_json)) + + def test_validate_human_readable_string(self): + + good = "oh hey!" + curie = "EFO:0001" + ensg = "ENSG000001234" + enst = "ENST000005678" + + self.assertTrue(validate._validate_human_readable_string(good)) + + self.assertFalse(validate._validate_human_readable_string(curie)) + self.assertFalse(validate._validate_human_readable_string(ensg)) + self.assertFalse(validate._validate_human_readable_string(enst)) + + def test_validate_curie(self): + + self.assertTrue(validate._validate_curie("UBERON:00001", ["UBERON", "EFO"])) + self.assertTrue(validate._validate_curie("HsapDv:00002", ["HsapDv"])) + + self.assertFalse(validate._validate_curie("HsapDv:00002", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_curie("EFO:00002 (organoid)", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_curie("EFO:00002 extra", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_curie("UBERON:ABCD", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_curie("Uberon:00002", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_curie("UBERON:", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_curie("UBERON", ["UBERON", "EFO"])) + + def test_validate_suffixed_curie(self): + + self.assertTrue(validate._validate_suffixed_curie("EFO:00001", ["UBERON", "EFO"])) + self.assertTrue(validate._validate_suffixed_curie("UBERON:00001 (cell culture)", ["UBERON", "EFO"])) + + self.assertFalse(validate._validate_suffixed_curie("HsapDv:00002 (organoid)", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_suffixed_curie("HsapDv:00002(organoid)", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_suffixed_curie("HsapDv:00002", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_suffixed_curie("EFO:00002 extra", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_suffixed_curie("UBERON:ABCD", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_suffixed_curie("Uberon:00002", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_suffixed_curie("UBERON:", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_suffixed_curie("UBERON", ["UBERON", "EFO"])) + + +class TestColumnValidation(unittest.TestCase): + + def test_validate_unique(self): + unique = pd.DataFrame([["abc", "def"], ["ghi", "jkl"], ["mnop", "qrs"]], + index=["X", "Y", "Z"], columns=["col1", "col2"]) + duped = pd.DataFrame([["abc", "def"], ["ghi", "qrs"], ["abc", "qrs"]], + index=["X", "Y", "X"], columns=["col1", "col2"]) + + schema_def = {"unique": True} + + errors = validate._validate_column(unique.index, "index", "unique_df", schema_def) + self.assertFalse(errors) + + errors = validate._validate_column(duped.index, "index", "duped_df", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("is not unique", errors[0]) + + errors = validate._validate_column(unique["col1"], "col1", "unique_df", schema_def) + self.assertFalse(errors) + + errors = validate._validate_column(duped["col1"], "col1", "duped_df", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("is not unique", errors[0]) + + schema_def = {"unique": False} + errors = validate._validate_column(duped["col1"], "col1", "duped_df", schema_def) + self.assertFalse(errors) + + def test_validate_nullable(self): + non_null = pd.DataFrame([["abc", "def"], ["ghi", "jkl"], ["mnop", "qrs"]], + index=["X", "Y", "Z"], columns=["col1", "col2"]) + has_null = pd.DataFrame([["abc", "", None], ["ghi", "jkl", 1], ["mnop", "qrs", 2]], + index=["X", "Y", "Z"], columns=["col1", "col2", "col3"]) + + schema_def = {"nullable": False} + errors = validate._validate_column(non_null["col1"], "col1", "nonnull_df", schema_def) + self.assertFalse(errors) + errors = validate._validate_column(has_null["col1"], "col1", "hasnull_df", schema_def) + self.assertFalse(errors) + + errors = validate._validate_column(has_null["col2"], "col2", "hasnull_df", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("contains empty values", errors[0]) + errors = validate._validate_column(has_null["col3"], "col3", "hasnull_df", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("contains empty values", errors[0]) + + schema_def = {"nullable": True} + errors = validate._validate_column(has_null["col2"], "col2", "hasnull_df", schema_def) + self.assertFalse(errors) + + def test_human_readable(self): + hr_df = pd.DataFrame( + [["for you, a human", "UBERON:12345", "UBERON:1234 (thundercat)"], + ["hope you're well", "bit of lungs", "brain"]], + index=["ENSG00001", "ENSG00002"], + columns=["good", "curie", "suffixed_curie"]) + + schema_def = {"type": "human-readable string"} + errors = validate._validate_column(hr_df["good"], "good", "hr", schema_def) + self.assertFalse(errors) + + errors = validate._validate_column(hr_df["curie"], "curie", "hr", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("non-human-readable", errors[0]) + + errors = validate._validate_column(hr_df["suffixed_curie"], "suffixed_curie", "hr", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("non-human-readable", errors[0]) + + errors = validate._validate_column(hr_df.index, "ensg", "hr", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("non-human-readable", errors[0]) + + def test_curie(self): + + curie_df = pd.DataFrame( + [["EFO:00001", "HsapDv:00001 (cell culture)", "EFO:", "MONDO:0001 cell culture"], + ["UBERON:00002", "HsapDv:00002 (organoid)", "EFO:12345", "MONDO:0002 (baba yaga)"], + ["EFO:0000000005", "HsapDv:000004 (humanzee)", "EFO:000002", "MONDO:0004 (TMNT)"]], + index=["X", "Y", "Z"], + columns=["good", "good_suffix", "bad", "bad_suffix"]) + + # Good + schema_def = {"type": "curie", "prefixes": ["EFO", "UBERON"]} + errors = validate._validate_column(curie_df["good"], "good", "curie_df", schema_def) + self.assertFalse(errors) + + # Good suffix + schema_def = {"type": "suffixed curie", "prefixes": ["HsapDv", "WHATEVER"]} + errors = validate._validate_column(curie_df["good_suffix"], "good_suffix", "curie_df", schema_def) + self.assertFalse(errors) + + # Bad prefix + schema_def = {"type": "curie", "prefixes": ["EFO"]} + errors = validate._validate_column(curie_df["good"], "good", "curie_df", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("invalid ontology", errors[0]) + self.assertIn("must be curies from one of these", errors[0]) + + # Bad curies + schema_def = {"type": "curie", "prefixes": ["EFO"]} + errors = validate._validate_column(curie_df["bad"], "bad", "curie_df", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("invalid ontology", errors[0]) + + # Bad suffixes + schema_def = {"type": "suffixed curie", "prefixes": ["EFO"]} + errors = validate._validate_column(curie_df["bad_suffix"], "bad_suffix", "curie_df", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("invalid ontology", errors[0]) + + def test_enum(self): + enum_df = pd.DataFrame( + [["abc", "ghi"], + ["def", "jkl"]], + index=["X", "Y"], + columns=["col1", "col2"]) + + # All match + schema_def = {"type": "string", "enum": ["abc", "def", "xyz"]} + errors = validate._validate_column(enum_df["col1"], "col1", "enum_df", schema_def) + self.assertFalse(errors) + + # Missing value + schema_def = {"type": "string", "enum": ["abc", "xyz"]} + errors = validate._validate_column(enum_df["col1"], "col1", "enum_df", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("unpermitted values", errors[0]) + + +class TestDictValidations(unittest.TestCase): + + + def test_key_presence(self): + + schema_def = {"keys": {"abc": None, "def": None}} + + dict_ = {"abc": "123", "def": "456"} + errors = validate._validate_dict(dict_, "d", schema_def) + self.assertFalse(errors) + + # Missing keys are bad + dict_ = {"abc": "123"} + errors = validate._validate_dict(dict_, "d", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("missing key", errors[0]) + + # Extra keys are okay + dict_ = {"abc": "123", "def": "456", "xyz": "789"} + errors = validate._validate_dict(dict_, "d", schema_def) + self.assertFalse(errors) + + # Better not be empty come on + dict_ = {} + errors = validate._validate_dict(dict_, "d", schema_def) + self.assertEqual(len(errors), 2) + + def test_nullable(self): + + schema_def = {"keys": {"abc": {"type": "string", "nullable": False}, + "def": {"type": "string", "nullable": True}}} + + dict_ = {"abc": "xyz", "def": ""} + errors = validate._validate_dict(dict_, "d", schema_def) + self.assertFalse(errors) + + dict_ = {"abc": "", "def": ""} + errors = validate._validate_dict(dict_, "d", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("empty value", errors[0]) + + def test_recurse(self): + + schema_def = { + "keys": { + "subdict": { + "type": "dict", + "keys": { + "subdict_key1": None, + "subdict_key2": None + } + }, + "ontology": { + "type": "curie", + "prefixes": ["ONTOLOGY"] + }, + "blob": { + "type": "stringified list of dicts" + } + } + } + + dict_ = { + "subdict": {"subdict_key1": "any", "subdict_key2": "any"}, + "ontology": "ONTOLOGY:123456", + "blob": json.dumps([{"abc": 123}, {"def": 456}]) + } + errors = validate._validate_dict(dict_, "d", schema_def) + self.assertFalse(errors) + + dict_ = { + "subdict": {"subdict_key1": "any"}, + "ontology": "ONTOLOGY:123456", + "blob": json.dumps([{"abc": 123}, {"def": 456}]) + } + errors = validate._validate_dict(dict_, "d", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("missing key", errors[0]) + + dict_ = { + "subdict": {"subdict_key1": "any", "subdict_key2": "any"}, + "ontology": "oh no not an ontology term", + "blob": json.dumps([{"abc": 123}, {"def": 456}]) + } + errors = validate._validate_dict(dict_, "d", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("invalid ontology", errors[0]) + + dict_ = { + "subdict": {"subdict_key1": "any", "subdict_key2": "any"}, + "ontology": "ONTOLOGY:123456", + "blob": [{"abc": 123}, {"def": 456}] + } + errors = validate._validate_dict(dict_, "d", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("JSON-encoded list of dicts", errors[0]) + + # Multiple errors + dict_ = { + "subdict": {"subdict_key1": "any"}, + "ontology": "oh no not an ontology term", + "blob": json.dumps([{"abc": 123}, {"def": 456}]) + } + errors = validate._validate_dict(dict_, "d", schema_def) + self.assertEqual(len(errors), 2) + + +class TestDataframeValidation(unittest.TestCase): + + def test_column_presence(self): + df = pd.DataFrame( + [["abc", "EFO:123"], + ["def", "UBERON:456"]], + columns=["hr_string", "ontology"], + index=["X", "Y"] + ) + + schema_def = { + "columns": { + "hr_string": {"type": "human-readable string"}, + "ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]} + } + } + errors = validate._validate_dataframe(df, "df", schema_def) + self.assertFalse(errors) + + schema_def = { + "columns": { + "hr_string": {"type": "human-readable string"}, + "another_hr_string": {"type": "human-readable string"}, + "ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]} + } + } + errors = validate._validate_dataframe(df, "df", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("missing column", errors[0]) + + # Extra is okay + df = pd.DataFrame( + [["abc", "EFO:123", "extra"], + ["def", "UBERON:456", "extra"]], + columns=["hr_string", "ontology", "extra"], + index=["X", "Y"] + ) + schema_def = { + "columns": { + "hr_string": {"type": "human-readable string"}, + "ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]} + } + } + errors = validate._validate_dataframe(df, "df", schema_def) + self.assertFalse(errors) + + + def test_index(self): + df = pd.DataFrame( + [["abc", "123"], + ["def", "456"]], + columns=["col1", "col2"], + index=["ENSG0001", "ENSG0002"] + ) + + schema_def = {"index": {"unique": True}} + errors = validate._validate_dataframe(df, "df", schema_def) + self.assertFalse(errors) + + schema_def = {"index": {"type": "human-readable string"}} + errors = validate._validate_dataframe(df, "df", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("non-human-readable", errors[0]) + + df = pd.DataFrame( + [["abc", "123"], + ["def", "456"]], + columns=["col1", "col2"], + index=["ENSG0001", "ENSG0001"] + ) + schema_def = {"index": {"unique": True}} + errors = validate._validate_dataframe(df, "df", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("is not unique", errors[0]) + + def test_recurse(self): + + df = pd.DataFrame( + [["abc", "HsapDv:0001"], + ["EFO:123", "UBERON:456"]], + columns=["hr_string", "ontology"], + index=["X", "Y"] + ) + schema_def = { + "columns": { + "hr_string": {"type": "human-readable string"}, + "ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]} + } + } + errors = validate._validate_dataframe(df, "df", schema_def) + self.assertEqual(len(errors), 2) + self.assertEqual(len([e for e in errors if "non-human-readable" in e]), 1) + self.assertEqual(len([e for e in errors if "invalid ontology" in e]), 1) + + +class TestGetSchema(unittest.TestCase): + + def test_get_schema(self): + self.assertIsInstance(validate.get_schema_definition("1.0.0"), dict) + + with self.assertRaises(ValueError): + validate.get_schema_definition("10.1.5") + + +class TestValidate(unittest.TestCase): + + def setUp(self): + self.source_h5ad_path = f"{PROJECT_ROOT}/local_server/test/fixtures/pbmc3k-CSC-gz.h5ad" + + def test_shallow(self): + + adata = sc.read_h5ad(self.source_h5ad_path) + self.assertFalse(validate.validate_adata(adata, True)) + + adata.uns["version"] = { + "corpora_schema_version": "1.0.0", + "corpora_encoding_version": "0.1.0" + } + self.assertTrue(validate.validate_adata(adata, True)) + + def test_deep(self): + adata = sc.read_h5ad(self.source_h5ad_path) + self.assertFalse(validate.validate_adata(adata, False)) + + adata.uns["version"] = { + "corpora_schema_version": "1.0.0", + "corpora_encoding_version": "0.1.0" + } + self.assertFalse(validate.validate_adata(adata, False)) diff --git a/local_server/test/unit/data_anndata/__init__.py b/local_server/test/unit/data_anndata/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/test/unit/data_anndata/test_anndata_adaptor.py b/local_server/test/unit/data_anndata/test_anndata_adaptor.py new file mode 100644 index 00000000..7aeae152 --- /dev/null +++ b/local_server/test/unit/data_anndata/test_anndata_adaptor.py @@ -0,0 +1,232 @@ +import json +import sys +import time +import unittest + +import numpy as np +import pandas as pd +import pytest +from parameterized import parameterized_class + +import local_server.test.unit.decode_fbs as decode_fbs +from local_server.common.data_locator import DataLocator +from local_server.common.errors import FilterError +from local_server.data_anndata.anndata_adaptor import AnndataAdaptor +from local_server.test import PROJECT_ROOT, app_config, FIXTURES_ROOT +from local_server.test.fixtures.fixtures import pbmc3k_colors + +""" +Test the anndata adaptor using the pbmc3k data set. +""" + + +@parameterized_class( + ("data_locator", "backed"), + [ + (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", False), + (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", False), + (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", False), + (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", True), + (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", True), + (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", True), + ], +) +class AdaptorTest(unittest.TestCase): + def setUp(self): + config = app_config(self.data_locator, self.backed) + self.data = AnndataAdaptor(DataLocator(self.data_locator), config) + + def test_init(self): + self.assertEqual(self.data.cell_count, 2638) + self.assertEqual(self.data.gene_count, 1838) + epsilon = 0.000_005 + self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon) + + def test_mandatory_annotations(self): + obs_index_col_name = self.data.get_schema()["annotations"]["obs"]["index"] + self.assertIn(obs_index_col_name, self.data.data.obs) + self.assertEqual(list(self.data.data.obs.index), list(range(2638))) + var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"] + self.assertIn(var_index_col_name, self.data.data.var) + self.assertEqual(list(self.data.data.var.index), list(range(1838))) + + @pytest.mark.filterwarnings("ignore:Anndata data matrix") + def test_data_type(self): + # don't run the test on the more exotic data types, as they don't + # support the astype() interface (used by this test, but not underlying app) + if isinstance(self.data.data.X, np.ndarray): + self.data.data.X = self.data.data.X.astype("float64") + with self.assertWarns(UserWarning): + self.data._validate_data_types() + + def test_filter_idx(self): + filter_ = {"filter": {"var": {"index": [1, 99, [200, 300]]}}} + fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") + data = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(data["n_rows"], 2638) + self.assertEqual(data["n_cols"], 102) + + def test_filter_complex(self): + filter_ = { + "filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 10}], "index": [1, 99, [200, 300]]}} + } + fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") + data = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(data["n_rows"], 2638) + self.assertEqual(data["n_cols"], 91) + + def test_obs_and_var_names(self): + self.assertEqual(np.sum(self.data.data.var[self.data.get_schema()["annotations"]["var"]["index"]].isna()), 0) + self.assertEqual(np.sum(self.data.data.obs[self.data.get_schema()["annotations"]["obs"]["index"]].isna()), 0) + + def test_get_colors(self): + self.assertEqual(self.data.get_colors(), pbmc3k_colors) + + def test_get_schema(self): + with open(f"{FIXTURES_ROOT}/schema.json") as fh: + schema = json.load(fh) + self.assertDictEqual(self.data.get_schema(), schema) + + def test_schema_produces_error(self): + self.data.data.obs["time"] = pd.Series( + list([time.time() for i in range(self.data.cell_count)]), dtype="datetime64[ns]", + ) + with pytest.raises(TypeError): + self.data._create_schema() + + def test_layout(self): + fbs = self.data.layout_to_fbs_matrix(fields=None) + layout = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(layout["n_cols"], 6) + self.assertEqual(layout["n_rows"], 2638) + + X = layout["columns"][0] + self.assertTrue((X >= 0).all() and (X <= 1).all()) + Y = layout["columns"][1] + self.assertTrue((Y >= 0).all() and (Y <= 1).all()) + + def test_layout_fields(self): + """ X_pca, X_tsne, X_umap are available """ + fbs = self.data.layout_to_fbs_matrix(["pca"]) + layout = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(layout["n_cols"], 2) + self.assertEqual(layout["n_rows"], 2638) + self.assertCountEqual(layout["col_idx"], ["pca_0", "pca_1"]) + + fbs = self.data.layout_to_fbs_matrix(["tsne", "pca"]) + layout = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(layout["n_cols"], 4) + self.assertEqual(layout["n_rows"], 2638) + self.assertCountEqual(layout["col_idx"], ["tsne_0", "tsne_1", "pca_0", "pca_1"]) + + def test_annotations(self): + fbs = self.data.annotation_to_fbs_matrix("obs") + annotations = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(annotations["n_rows"], 2638) + self.assertEqual(annotations["n_cols"], 5) + obs_index_col_name = self.data.get_schema()["annotations"]["obs"]["index"] + self.assertEqual( + annotations["col_idx"], [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"], + ) + + fbs = self.data.annotation_to_fbs_matrix("var") + annotations = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(annotations["n_rows"], 1838) + self.assertEqual(annotations["n_cols"], 2) + var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"] + self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells"]) + + def test_annotation_fields(self): + fbs = self.data.annotation_to_fbs_matrix("obs", ["n_genes", "n_counts"]) + annotations = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(annotations["n_rows"], 2638) + self.assertEqual(annotations["n_cols"], 2) + + var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"] + fbs = self.data.annotation_to_fbs_matrix("var", [var_index_col_name]) + annotations = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(annotations["n_rows"], 1838) + self.assertEqual(annotations["n_cols"], 1) + + def test_diffexp_topN(self): + f1 = {"filter": {"obs": {"index": [[0, 500]]}}} + f2 = {"filter": {"obs": {"index": [[500, 1000]]}}} + result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"])) + self.assertEqual(len(result), 10) + result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20)) + self.assertEqual(len(result), 20) + + def test_data_frame(self): + f1 = {"var": {"index": [[0, 10]]}} + fbs = self.data.data_frame_to_fbs_matrix(f1, "var") + data = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(data["n_rows"], 2638) + self.assertEqual(data["n_cols"], 10) + + with self.assertRaises(ValueError): + self.data.data_frame_to_fbs_matrix(None, "obs") + + def test_filtered_data_frame(self): + filter_ = {"filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 100}]}}} + fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") + data = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(data["n_rows"], 2638) + self.assertEqual(data["n_cols"], 1040) + + filter_ = {"filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}} + with self.assertRaises(FilterError): + self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") + + def test_data_named_gene(self): + var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"] + filter_ = {"filter": {"var": {"annotation_value": [{"name": var_index_col_name, "values": ["RER1"]}]}}} + fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") + data = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(data["n_rows"], 2638) + self.assertEqual(data["n_cols"], 1) + self.assertEqual(data["col_idx"], [4]) + + filter_ = { + "filter": {"var": {"annotation_value": [{"name": var_index_col_name, "values": ["SPEN", "TYMP", "PRMT2"]}]}} + } + fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") + data = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(data["n_rows"], 2638) + self.assertEqual(data["n_cols"], 3) + self.assertTrue((data["col_idx"] == [15, 1818, 1837]).all()) + + def test_compute_embedding(self): + filter = {"obs": {"index": [[0, 100]]}} + + # Verify that we correctly handle the case where we lack scanpy + import unittest.mock + + with unittest.mock.patch.dict(sys.modules, {"scanpy": None}): + with self.assertRaises(NotImplementedError): + self.data.compute_embedding("umap", filter) + + # if we happen to have scanpy, test the full API, else punt + import importlib + + scanpy_spec = importlib.util.find_spec("scanpy") + if scanpy_spec is None: + print("Skipping compute_embedding test as ScanPy not installed") + return + + # this feature is unsupported in backed mode, and we expect an error + if self.data.data.isbacked: + with self.assertRaises(NotImplementedError): + self.data.compute_embedding("umap", filter) + return + + schema = self.data.compute_embedding("umap", filter) + + self.assertIsInstance(schema["name"], str) + name = schema["name"] + self.assertEqual(schema["type"], "float32") + self.assertEqual(schema["dims"], [f"{name}_0", f"{name}_1"]) + + emb = self.data.data.obsm[f"X_{name}"] + self.assertEqual(emb.shape, (2638, 2)) + self.assertTrue(np.isfinite(emb[0:100]).all()) + self.assertTrue(np.isnan(emb[100:]).all()) diff --git a/local_server/test/unit/data_anndata/test_anndata_adaptor_data_load.py b/local_server/test/unit/data_anndata/test_anndata_adaptor_data_load.py new file mode 100644 index 00000000..dfef08da --- /dev/null +++ b/local_server/test/unit/data_anndata/test_anndata_adaptor_data_load.py @@ -0,0 +1,81 @@ +import unittest +import json + +from local_server.data_anndata.anndata_adaptor import AnndataAdaptor +from local_server.common.data_locator import DataLocator +from local_server.common.config.app_config import AppConfig +from local_server.test import PROJECT_ROOT + + +class DataLoadAdaptorTest(unittest.TestCase): + """ + Test file loading, including deferred loading/update. + """ + + def setUp(self): + self.data_file = DataLocator(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad") + config = AppConfig() + config.update_server_config(single_dataset__datapath=self.data_file.path) + config.update_server_config(app__flask_secret_key="secret") + config.complete_config() + self.data = AnndataAdaptor(self.data_file, config) + + def test_delayed_load_data(self): + self.data._create_schema() + self.assertEqual(self.data.cell_count, 2638) + self.assertEqual(self.data.gene_count, 1838) + epsilon = 0.000_005 + self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon) + + def test_diffexp_topN(self): + f1 = {"filter": {"obs": {"index": [[0, 500]]}}} + f2 = {"filter": {"obs": {"index": [[500, 1000]]}}} + result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"])) + self.assertEqual(len(result), 10) + result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20)) + self.assertEqual(len(result), 20) + + +class DataLocatorAdaptorTest(unittest.TestCase): + """ + Test various types of data locators we expect to consume + """ + + def get_basic_config(self): + config = AppConfig() + config.update_server_config( + single_dataset__obs_names=None, single_dataset__var_names=None, + ) + config.update_server_config(app__flask_secret_key="secret") + config.update_dataset_config( + embeddings__names=["umap"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01, + ) + return config + + def stdAsserts(self, data): + """ run these each time we load the data """ + self.assertIsNotNone(data) + self.assertEqual(data.cell_count, 2638) + self.assertEqual(data.gene_count, 1838) + + def test_posix_file(self): + locator = DataLocator("../example-dataset/pbmc3k.h5ad") + config = self.get_basic_config() + config.update_server_config(single_dataset__datapath=locator.path) + config.complete_config() + data = AnndataAdaptor(locator, config) + self.stdAsserts(data) + + def test_url_https(self): + url = "https://raw.githubusercontent.com/chanzuckerberg/cellxgene/main/example-dataset/pbmc3k.h5ad" + locator = DataLocator(url) + config = self.get_basic_config() + data = AnndataAdaptor(locator, config) + self.stdAsserts(data) + + def test_url_http(self): + url = "http://raw.githubusercontent.com/chanzuckerberg/cellxgene/main/example-dataset/pbmc3k.h5ad" + locator = DataLocator(url) + config = self.get_basic_config() + data = AnndataAdaptor(locator, config) + self.stdAsserts(data) diff --git a/local_server/test/unit/data_anndata/test_nan_anndata_adaptor.py b/local_server/test/unit/data_anndata/test_nan_anndata_adaptor.py new file mode 100644 index 00000000..2ab1c559 --- /dev/null +++ b/local_server/test/unit/data_anndata/test_nan_anndata_adaptor.py @@ -0,0 +1,64 @@ +import math +import unittest +import warnings + +import pytest + +import local_server.test.unit.decode_fbs as decode_fbs +from local_server.common.data_locator import DataLocator +from local_server.common.errors import FilterError +from local_server.data_anndata.anndata_adaptor import AnndataAdaptor +from local_server.test import app_config, FIXTURES_ROOT + + +class NaNTest(unittest.TestCase): + def setUp(self): + self.data_locator = DataLocator(f"{FIXTURES_ROOT}/nan.h5ad") + self.config = app_config(self.data_locator.path) + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=UserWarning) + self.data = AnndataAdaptor(self.data_locator, self.config) + self.data._create_schema() + + def test_load(self): + with self.assertLogs(level="WARN") as logger: + self.data = AnndataAdaptor(self.data_locator, self.config) + self.assertTrue(logger.output) + + def test_init(self): + self.assertEqual(self.data.cell_count, 100) + self.assertEqual(self.data.gene_count, 100) + epsilon = 0.000_005 + self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon) + + def test_dataframe(self): + data_frame_var = decode_fbs.decode_matrix_FBS(self.data.data_frame_to_fbs_matrix(None, "var")) + self.assertIsNotNone(data_frame_var) + self.assertEqual(data_frame_var["n_rows"], 100) + self.assertEqual(data_frame_var["n_cols"], 100) + self.assertTrue(math.isnan(data_frame_var["columns"][3][3])) + + with pytest.raises(FilterError): + self.data.data_frame_to_fbs_matrix("an erroneous filter", "var") + with pytest.raises(FilterError): + filter_ = {"filter": {"obs": {"index": [1, 99, [200, 300]]}}} + self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") + + def test_dataframe_obs_not_implemented(self): + with self.assertRaises(ValueError) as cm: + decode_fbs.decode_matrix_FBS(self.data.data_frame_to_fbs_matrix(None, "obs")) + self.assertIsNotNone(cm.exception) + + def test_annotation(self): + annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("obs")) + obs_index_col_name = self.data.schema["annotations"]["obs"]["index"] + self.assertEqual(annotations["col_idx"], [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"]) + self.assertEqual(annotations["n_rows"], 100) + self.assertTrue(math.isnan(annotations["columns"][2][0])) + + annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("var")) + var_index_col_name = self.data.schema["annotations"]["var"]["index"] + self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells", "var_with_nans"]) + self.assertEqual(annotations["n_rows"], 100) + self.assertTrue(math.isnan(annotations["columns"][2][0])) diff --git a/local_server/test/unit/data_common/__init__.py b/local_server/test/unit/data_common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/test/unit/data_common/fbs/__init__.py b/local_server/test/unit/data_common/fbs/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/local_server/test/unit/data_common/fbs/test_matrix.py b/local_server/test/unit/data_common/fbs/test_matrix.py new file mode 100644 index 00000000..aaaf12aa --- /dev/null +++ b/local_server/test/unit/data_common/fbs/test_matrix.py @@ -0,0 +1,82 @@ +import unittest +import pandas as pd +import numpy as np +from scipy import sparse + +import local_server.test.unit.decode_fbs as decode_fbs +from local_server.data_common.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs + + +class FbsTests(unittest.TestCase): + """Test Case for Matrix FBS data encode/decode """ + + def test_encode_boundary(self): + """ test various boundary checks """ + + # row indexing is unsupported + with self.assertRaises(ValueError): + encode_matrix_fbs(matrix=pd.DataFrame(), row_idx=[]) + + # matrix must be 2D + with self.assertRaises(ValueError): + encode_matrix_fbs(matrix=np.zeros((3, 2, 1))) + with self.assertRaises(ValueError): + encode_matrix_fbs(matrix=np.ones((10,))) + + def fbs_checks(self, fbs, dims, expected_types, expected_column_idx): + d = decode_fbs.decode_matrix_FBS(fbs) + self.assertEqual(d["n_rows"], dims[0]) + self.assertEqual(d["n_cols"], dims[1]) + self.assertIsNone(d["row_idx"]) + self.assertEqual(len(d["columns"]), dims[1]) + for i in range(0, len(d["columns"])): + self.assertEqual(len(d["columns"][i]), dims[0]) + self.assertIsInstance(d["columns"][i], expected_types[i][0]) + if expected_types[i][1] is not None: + self.assertEqual(d["columns"][i].dtype, expected_types[i][1]) + if expected_column_idx is not None: + self.assertSetEqual(set(expected_column_idx), set(d["col_idx"])) + + def test_encode_DataFrame(self): + df = pd.DataFrame( + data={ + "a": np.zeros((10,), dtype=np.float32), + "b": np.ones((10,), dtype=np.int64), + "c": np.array([i for i in range(0, 10)], dtype=np.uint16), + "d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"), + } + ) + expected_types = ((np.ndarray, np.float32), (np.ndarray, np.int32), (np.ndarray, np.uint32), (list, None)) + fbs = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns) + self.fbs_checks(fbs, (10, 4), expected_types, ["a", "b", "c", "d"]) + + def test_encode_ndarray(self): + arr = np.zeros((3, 2), dtype=np.float32) + expected_types = ((np.ndarray, np.float32), (np.ndarray, np.float32), (np.ndarray, np.float32)) + fbs = encode_matrix_fbs(matrix=arr, row_idx=None, col_idx=None) + self.fbs_checks(fbs, (3, 2), expected_types, None) + + def test_encode_sparse(self): + csc = sparse.csc_matrix(np.array([[0, 1, 2], [3, 0, 4]])) + expected_types = ((np.ndarray, np.int32), (np.ndarray, np.int32), (np.ndarray, np.int32)) + fbs = encode_matrix_fbs(matrix=csc, row_idx=None, col_idx=None) + self.fbs_checks(fbs, (2, 3), expected_types, None) + + def test_roundtrip(self): + dfSrc = pd.DataFrame( + data={ + "a": np.zeros((10,), dtype=np.float32), + "b": np.ones((10,), dtype=np.int64), + "c": np.array([i for i in range(0, 10)], dtype=np.uint16), + "d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"), + } + ) + dfDst = decode_matrix_fbs(encode_matrix_fbs(matrix=dfSrc, col_idx=dfSrc.columns)) + self.assertEqual(dfSrc.shape, dfDst.shape) + self.assertEqual(set(dfSrc.columns), set(dfDst.columns)) + for c in dfSrc.columns: + self.assertTrue(c in dfDst.columns) + if isinstance(dfSrc[c], pd.Series): + self.assertTrue(np.all(dfSrc[c] == dfDst[c])) + else: + self.assertEqual(dfSrc[c], dfDst[c]) diff --git a/local_server/test/unit/decode_fbs.py b/local_server/test/unit/decode_fbs.py new file mode 100644 index 00000000..0d5681fa --- /dev/null +++ b/local_server/test/unit/decode_fbs.py @@ -0,0 +1,31 @@ +""" +Code to decode, for testing purposes, the flatbuffer encoded blobs. + +This code will need to be updated if fbs/matrix.fbs changes. For more information, see fbs/matrix.fbs and +local_server/data_common/fbs/ +""" + +import local_server.data_common.fbs.NetEncoding.Matrix as Matrix +from local_server.data_common.fbs.matrix import deserialize_typed_array + + +def decode_matrix_FBS(buf): + """ + Given a FBS Matrix, return an decoded Python dict containing same info in native format. + NOTE / TODO: row_idx not currently implemented + """ + df = Matrix.Matrix.GetRootAsMatrix(buf, 0) + n_rows = df.NRows() + n_cols = df.NCols() + + columns_length = df.ColumnsLength() + + decoded_columns = [] + for col_idx in range(0, columns_length): + col = df.Columns(col_idx) + tarr = (col.UType(), col.U()) + decoded_columns.append(deserialize_typed_array(tarr)) + + cidx = deserialize_typed_array((df.ColIndexType(), df.ColIndex())) + + return {"n_rows": n_rows, "n_cols": n_cols, "columns": decoded_columns, "col_idx": cidx, "row_idx": None} diff --git a/requirements.txt b/requirements.txt index 815cac2f..58b2e504 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ --r ./server/requirements.txt +-r ./local_server/requirements.txt diff --git a/server/app/app.py b/server/app/app.py index 1396d676..8d8cab73 100644 --- a/server/app/app.py +++ b/server/app/app.py @@ -240,7 +240,7 @@ class DatasetResource(Resource): class SchemaAPI(DatasetResource): # TODO @mdunitz separate dataset schema and user schema - @cache_control(no_store=True) + @cache_control(public=True, max_age=ONE_WEEK) @rest_get_data_adaptor def get(self, data_adaptor): return common_rest.schema_get(data_adaptor) @@ -261,7 +261,7 @@ class UserInfoAPI(DatasetResource): class AnnotationsObsAPI(DatasetResource): - @cache_control(public=True, no_store=True) + @cache_control(public=True, max_age=ONE_WEEK) @rest_get_data_adaptor def get(self, data_adaptor): return common_rest.annotations_obs_get(request, data_adaptor) diff --git a/server/common/config/server_config.py b/server/common/config/server_config.py index 14eac613..48112b86 100644 --- a/server/common/config/server_config.py +++ b/server/common/config/server_config.py @@ -42,6 +42,9 @@ class ServerConfig(BaseConfig): self.app__web_base_url = default_config["app"]["web_base_url"] self.authentication__type = default_config["authentication"]["type"] + self.authentication__insecure_test_environment = default_config["authentication"][ + "insecure_test_environment" + ] self.authentication__params_oauth__oauth_api_base_url = default_config["authentication"]["params_oauth"][ "oauth_api_base_url" ] @@ -168,6 +171,10 @@ class ServerConfig(BaseConfig): def handle_authentication(self): self.validate_correct_type_of_configuration_attribute("authentication__type", (type(None), str)) + self.validate_correct_type_of_configuration_attribute("authentication__insecure_test_environment", bool) + + if self.authentication__type == "test" and not self.authentication__insecure_test_environment: + raise ConfigurationError("Test auth can only be used in an insecure test environment") # oauth ptypes = str if self.authentication__type == "oauth" else (type(None), str) diff --git a/server/default_config.py b/server/default_config.py index 20922ef9..d79afc4e 100644 --- a/server/default_config.py +++ b/server/default_config.py @@ -34,6 +34,7 @@ server: # session: A session based userid is automatically generated. (no params needed) # oauth: oauth2 is used for authentication; parameters are defined in params_oauth. type: session + insecure_test_environment: false params_oauth: # url to the oauth server diff --git a/server/test/__init__.py b/server/test/__init__.py index 1b96f7c1..ad8e7eef 100644 --- a/server/test/__init__.py +++ b/server/test/__init__.py @@ -34,7 +34,10 @@ def data_with_tmp_tiledb_annotations(ext: MatrixDataType): data_locator = DataLocator(fname) config = AppConfig() config.update_server_config( - app__flask_secret_key="secret", multi_dataset__dataroot=data_locator.path, authentication__type="test", + app__flask_secret_key="secret", + multi_dataset__dataroot=data_locator.path, + authentication__type="test", + authentication__insecure_test_environment=True, ) config.update_default_dataset_config( embeddings__names=["umap"], diff --git a/server/test/fixtures/server_config_outline.py b/server/test/fixtures/server_config_outline.py index 4bbb4675..a23f7d24 100644 --- a/server/test/fixtures/server_config_outline.py +++ b/server/test/fixtures/server_config_outline.py @@ -14,6 +14,7 @@ f"""server: web_base_url: {web_base_url} authentication: type: {auth_type} + insecure_test_environment: {insecure_test_environment} params_oauth: oauth_api_base_url: {oauth_api_base_url} client_id: {client_id} diff --git a/server/test/unit/auth/test_auth.py b/server/test/unit/auth/test_auth.py index c36b8c62..12f062e6 100644 --- a/server/test/unit/auth/test_auth.py +++ b/server/test/unit/auth/test_auth.py @@ -45,6 +45,7 @@ class AuthTest(unittest.TestCase): app_config = AppConfig() app_config.update_server_config(app__flask_secret_key="secret") app_config.update_server_config(authentication__type="test") + app_config.update_server_config(authentication__insecure_test_environment=True) app_config.update_server_config( multi_dataset__dataroot=dict( a1=dict(dataroot=self.dataset_dataroot, base_url="auth"), @@ -116,6 +117,7 @@ class AuthTest(unittest.TestCase): app_config.update_server_config( authentication__type="test", single_dataset__datapath=f"{self.dataset_dataroot}/pbmc3k.cxg" ) + app_config.update_server_config(authentication__insecure_test_environment=True) app_config.complete_config() diff --git a/server/test/unit/common/config/__init__.py b/server/test/unit/common/config/__init__.py index 8a48be5e..12fc7656 100644 --- a/server/test/unit/common/config/__init__.py +++ b/server/test/unit/common/config/__init__.py @@ -38,6 +38,7 @@ class ConfigTests(unittest.TestCase): api_base_url="null", web_base_url="null", auth_type="session", + insecure_test_environment="false", oauth_api_base_url="null", client_id="null", client_secret="null", diff --git a/server/test/unit/common/config/test_server_config.py b/server/test/unit/common/config/test_server_config.py index 8ad76b58..932a5d33 100644 --- a/server/test/unit/common/config/test_server_config.py +++ b/server/test/unit/common/config/test_server_config.py @@ -53,7 +53,7 @@ class TestServerConfig(ConfigTests): def test_complete_config_checks_all_attr(self, mock_check_attrs): mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute() self.server_config.complete_config(self.context) - self.assertEqual(mock_check_attrs.call_count, 40) + self.assertEqual(mock_check_attrs.call_count, 41) def test_handle_app__throws_error_if_port_doesnt_exist(self): config = self.get_config(port=99999999) @@ -312,3 +312,12 @@ class TestServerConfig(ConfigTests): mock_tiledb_context.assert_called_once_with( {"sm.tile_cache_size": 10, "sm.num_reader_threads": 2, "vfs.s3.region": "us-east-1"} ) + + def test_test_auth_only_in_insecure(self): + + config = self.get_config(auth_type="test") + with self.assertRaises(ConfigurationError): + config.complete_config() + + config.update_server_config(authentication__insecure_test_environment=True) + config.complete_config() diff --git a/setup.cfg b/setup.cfg index b240487d..3b07f60b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,4 +1,4 @@ [flake8] max-line-length = 120 ignore = E203, W503 -exclude = server/data_common/fbs/NetEncoding/,.git,__pycache__,venv,server/venv,old,build,dist,server/eb/artifact.dir/ +exclude = server/data_common/fbs/NetEncoding/,.git,__pycache__,venv,server/venv,old,build,dist,server/eb/artifact.dir/,local_server/data_common/fbs/NetEncoding diff --git a/setup.py b/setup.py index 56a2bd28..7f6d9206 100644 --- a/setup.py +++ b/setup.py @@ -3,10 +3,10 @@ from setuptools import setup, find_packages with open("README.md", "rb") as fh: long_description = fh.read().decode() -with open("server/requirements.txt") as fh: +with open("local_server/requirements.txt") as fh: requirements = fh.read().splitlines() -with open("server/requirements-prepare.txt") as fh: +with open("local_server/requirements-prepare.txt") as fh: requirements_prepare = fh.read().splitlines() setup( @@ -38,6 +38,6 @@ setup( "Programming Language :: Python :: 3 :: Only", "Topic :: Scientific/Engineering :: Bio-Informatics", ], - entry_points={"console_scripts": ["cellxgene = server.cli.cli:cli"]}, + entry_points={"console_scripts": ["cellxgene = local_server.cli.cli:cli"]}, extras_require=dict(prepare=requirements_prepare), ) diff --git a/setup_hosted.py b/setup_hosted.py new file mode 100644 index 00000000..56a2bd28 --- /dev/null +++ b/setup_hosted.py @@ -0,0 +1,43 @@ +from setuptools import setup, find_packages + +with open("README.md", "rb") as fh: + long_description = fh.read().decode() + +with open("server/requirements.txt") as fh: + requirements = fh.read().splitlines() + +with open("server/requirements-prepare.txt") as fh: + requirements_prepare = fh.read().splitlines() + +setup( + name="cellxgene", + version="0.16.0", + packages=find_packages(), + url="https://github.com/chanzuckerberg/cellxgene", + license="MIT", + author="Chan Zuckerberg Initiative", + author_email="cellxgene@chanzuckerberg.com", + description="Web application for exploration of large scale scRNA-seq datasets", + long_description=long_description, + long_description_content_type="text/markdown", + install_requires=requirements, + include_package_data=True, + zip_safe=False, + classifiers=[ + "Framework :: Flask", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Natural Language :: English", + "Operating System :: POSIX", + "Operating System :: Unix", + "Operating System :: MacOS :: MacOS X", + "Programming Language :: JavaScript", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3 :: Only", + "Topic :: Scientific/Engineering :: Bio-Informatics", + ], + entry_points={"console_scripts": ["cellxgene = server.cli.cli:cli"]}, + extras_require=dict(prepare=requirements_prepare), +)