Refactor czi_hosted and server into backend directory, pull common code into backend/common, refactor tests (#2102)

* move local_server -> backend/server server-> backend/czi_hosted, pull common code into backend/common update imports, tests and make commands

.dockerignore

@@ -2,4 +2,4 @@ bin
 client
 dist
 docs
-server
+backend

.github/workflows/compatibility_tests.yml

@@ -41,8 +41,8 @@ jobs:
       run: |
         # 1. only install the dev requirements on top of what is in the cellxgene pip package
         sudo apt-get update && sudo apt-get install -y libhdf5-serial-dev
-        sed -i 's/-r requirements.txt//' server/requirements-dev.txt
+        sed -i 's/-r requirements.txt//' backend/server/requirements-dev.txt
-        pip install -r server/requirements-dev.txt
+        pip install -r backend/server/requirements-dev.txt
         # 2. install cellxgene
         make pydist install-dist
         # 3. install anndata
@@ -73,8 +73,8 @@ jobs:
           cd cellxgene
           # 1. only install the dev requirements on top of what is in the cellxgene pip package
           make dev-env-client
-          sed -i 's/-r requirements.txt//' server/requirements-dev.txt
+          sed -i 's/-r requirements.txt//' backend/server/requirements-dev.txt
-          pip install -r server/requirements-dev.txt
+          pip install -r backend/server/requirements-dev.txt
           # 2. install cellxgene
           pip install --upgrade cellxgene
           # 3. install anndata
@@ -101,7 +101,7 @@ jobs:
       - name: Install dependencies
         run: |
           cd cellxgene
-          sed -i -E 's/^anndata[>=]=[0-9]+.[0-9]+.[0-9]+$/anndata/g' server/requirements.txt
+          sed -i -E 's/^anndata[>=]=[0-9]+.[0-9]+.[0-9]+$/anndata/g' backend/server/requirements.txt
           make pydist install-dist dev-env
           pip install git+https://github.com/theislab/anndata
       - name: Tests

.github/workflows/push_tests.yml

@@ -68,11 +68,11 @@ jobs:
         run: make pydist install-dist dev-env-server
       - name: Unit tests
         run: |
-          make unit-test
+          make unit-test-server
-          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k server -cF backend,python,unitTest
+          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k backend/server -cF backend,python,unitTest
           cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest
 
-  unit-test-local:
+  unit-test-czi-hosted:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -95,11 +95,11 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-node-
       - name: Install dependencies
-        run: make pydist-local install-dist dev-env-local-server
+        run: make pydist-czi-hosted install-dist dev-env-czi-hosted
       - name: Unit tests
         run: |
-          make unit-test-local
+          make unit-test-czi-hosted
-          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k local_server -cF backend,python,unitTest
+          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k backend/czi-hosted -cF backend,python,unitTest
           cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest
 
   smoke-tests:
@@ -126,7 +126,7 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-node-
       - name: Install dependencies
-        run: make pydist install-dist
+        run: make pydist-czi-hosted install-dist
       - name: Smoke tests (without annotations feature)
         run: |
           cd client && make smoke-test
@@ -156,7 +156,7 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-node-
       - name: Install dependencies
-        run: make pydist-local install-dist
+        run: make pydist install-dist
       - name: Smoke tests (with annotations feature)
         run: |
           cd client && make smoke-test-annotations

.github/workflows/scale-test.yml

@@ -15,10 +15,10 @@ jobs:
           python-version: 3.7
       - name: Install dependencies
         run: |
-          pip install -r server/test/locust/requirements-locust.txt
+          pip install -r backend/czi_hosted/test/locust/requirements-locust.txt
       - name: Dev Scale Test
         run: |
-          locust -f server/test/locust/locustfile.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt
+          locust -f backend/czi_hosted/test/locust/locustfile.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt
       - name: Slack success webhook
         env:
           SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}

.gitignore

@@ -19,9 +19,13 @@ venv/
 cellxgene/
 
 # client build
-server/common/web/static/*
+backend/server/common/web/static/*
-server/common/web/templates/
+backend/server/common/web/templates/
-server/common/web/csp-hashes.json
+backend/server/common/web/csp-hashes.json
+
+backend/czi_hosted/common/web/static/*
+backend/czi_hosted/common/web/templates/
+backend/czi_hosted/common/web/csp-hashes.json
 
 # eb build
 artifact.dir

MANIFEST.in

@@ -1,7 +1,7 @@
-recursive-include local_server/common/web/templates *
+recursive-include backend/server/common/web/templates *
-recursive-include local_server/common/web/static *
+recursive-include backend/server/common/web/static *
 
-include local_server/requirements.txt
+include backend/server/requirements.txt
-include local_server/requirements-prepare.txt
+include backend/server/requirements-prepare.txt
-include local_server/converters/schema/hgnc_complete_set.txt.gz
+include backend/server/converters/schema/hgnc_complete_set.txt.gz
-include local_server/converters/schema/schema_definitions/*
+include backend/server/converters/schema/schema_definitions/*

MANIFEST_hosted.in

@@ -1,7 +1,7 @@
-recursive-include server/common/web/templates *
+recursive-include backend/czi_hosted/common/web/templates *
-recursive-include server/common/web/static *
+recursive-include backend/czi_hosted/common/web/static *
 
-include server/requirements.txt
+include backend/czi_hosted/requirements.txt
-include server/requirements-prepare.txt
+include backend/czi_hosted/requirements-prepare.txt
-include server/converters/schema/hgnc_complete_set.txt.gz
+include backend/czi_hosted/converters/schema/hgnc_complete_set.txt.gz
-include server/converters/schema/schema_definitions/*
+include backend/czi_hosted/converters/schema/schema_definitions/*

Makefile

@@ -2,24 +2,32 @@ include common.mk
 
 BUILDDIR := build
 CLIENTBUILD := $(BUILDDIR)/client
-SERVERBUILD := $(BUILDDIR)/server
+CZIHOSTEDBUILD := $(BUILDDIR)/backend/czi_hosted
-LOCALSERVERBUILD := $(BUILDDIR)/local_server
+SERVERBUILD := $(BUILDDIR)/backend/server
 CLEANFILES :=  $(BUILDDIR)/ client/build build dist cellxgene.egg-info
 
 PART ?= patch
 
 # CLEANING
 .PHONY: clean
-clean: clean-lite clean-local-server clean-server clean-client
+clean: clean-lite clean-czi-hosted clean-server clean-client
 
 # cleaning the client's node_modules is the longest one, so we avoid that if possible
 .PHONY: clean-lite
 clean-lite:
 	rm -rf $(CLEANFILES)
 
-clean-%:
+.PHONY: clean-client
-	cd $(subst -,_,$*) && $(MAKE) clean
+clean-client:
+	cd client && $(MAKE) clean
 
+.PHONY: clean-server
+clean-server:
+	cd backend/server && $(MAKE) clean
+
+.PHONY: clean-czi-hosted
+clean-czi-hosted:
+	cd backend/czi_hosted && $(MAKE) clean
 
 # BUILDING PACKAGE
 
@@ -27,54 +35,73 @@ clean-%:
 build-client:
 	cd client && $(MAKE) ci build
 
-.PHONY: build-local
-build-local: clean build-client
-	git ls-files local_server/ | grep -v 'local_server/test/' | cpio -pdm $(BUILDDIR)
-	cp -r client/build/  $(CLIENTBUILD)
-	$(call copy_client_assets,$(CLIENTBUILD),$(LOCALSERVERBUILD))
-	cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR)
-
 .PHONY: build
 build: clean build-client
-	git ls-files server/ | grep -v 'server/test/' | cpio -pdm $(BUILDDIR)
+	git ls-files backend/server/ | grep -v 'backend/server/test/' | cpio -pdm $(BUILDDIR)
 	cp -r client/build/  $(CLIENTBUILD)
 	$(call copy_client_assets,$(CLIENTBUILD),$(SERVERBUILD))
+	cp backend/__init__.py $(BUILDDIR)
+	cp backend/__init__.py $(BUILDDIR)/backend
+	cp -r backend/common $(BUILDDIR)/backend/common
+	cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR)
+
+.PHONY: build-czi-hosted
+build-czi-hosted: clean build-client
+	git ls-files backend/czi_hosted/ | grep -v 'backend/czi_hosted/test/' | cpio -pdm $(BUILDDIR)
+	cp -r client/build/  $(CLIENTBUILD)
+	$(call copy_client_assets,$(CLIENTBUILD),$(CZIHOSTEDBUILD))
+	cp -r backend/common $(BUILDDIR)/backend/common
+	cp backend/__init__.py $(BUILDDIR)
+	cp backend/__init__.py $(BUILDDIR)/backend
 	cp MANIFEST_hosted.in README.md setup.cfg setup_hosted.py $(BUILDDIR)
 	mv $(BUILDDIR)/setup_hosted.py $(BUILDDIR)/setup.py
 	mv $(BUILDDIR)/MANIFEST_hosted.in $(BUILDDIR)/MANIFEST.in
 
 # If you are actively developing in the server folder use this, dirties the source tree
-.PHONY: build-for-server-dev-local
-build-for-server-dev-local: clean-local-server build-client
-	$(call copy_client_assets,client/build,local_server)
-
 .PHONY: build-for-server-dev
 build-for-server-dev: clean-server build-client
-	$(call copy_client_assets,client/build,server)
+	$(call copy_client_assets,client/build,backend/server)
 
-.PHONY: copy-client-assets-local
+.PHONY: build-for-czi-hosted-dev
-copy-client-assets-local:
+build-for-czi-hosted-dev: clean-czi-hosted build-client
-	$(call copy_client_assets,client/build,local_server)
+	$(call copy_client_assets,client/build,backend/czi_hosted)
 
 .PHONY: copy-client-assets
 copy-client-assets:
-	$(call copy_client_assets,client/build,server)
+	$(call copy_client_assets,client/build,backend/server)
+
+.PHONY: copy-client-assets-czi-hosted
+copy-client-assets-czi-hosted:
+	$(call copy_client_assets,client/build,backend/czi_hosted)
 
 # TESTING
 .PHONY: test
 test: unit-test smoke-test
 
-.PHONY: test-local
-test-local: unit-test-local smoke-test
-
-.PHONY: unit-test-local
-unit-test-local: unit-test-local-server unit-test-client
-
 .PHONY: unit-test
-unit-test: unit-test-server unit-test-client
+unit-test: unit-test-server unit-test-client unit-test-common
 
-unit-test-%:
+.PHONY: test-server
-	cd $(subst -,_,$*) && $(MAKE) unit-test
+test-server: unit-test-server smoke-test
+
+.PHONY: test-czi-hosted
+test-czi-hosted: unit-test-czi-hosted smoke-test
+
+.PHONY: unit-test-client
+unit-test-client:
+	cd client && $(MAKE) unit-test
+
+.PHONY: unit-test-czi-hosted
+unit-test-czi-hosted:
+	cd backend/czi_hosted && $(MAKE) unit-test
+
+.PHONY: unit-test-server
+unit-test-server:
+	cd backend/server && $(MAKE) unit-test
+
+.PHONY: unit-test-common
+unit-test-common:
+	cd backend/common && $(MAKE) unit-test
 
 .PHONY: smoke-test
 smoke-test:
@@ -86,7 +113,7 @@ smoke-test-annotations:
 
 .PHONY: test-db
 test-db:
-	cd server && $(MAKE) test-db
+	cd backend/czi_hosted && $(MAKE) test-db
 
 # FORMATTING CODE
 
@@ -105,15 +132,15 @@ fmt-py:
 lint: lint-servers lint-client
 
 .PHONY: lint-servers
-lint-servers: lint-local-server lint-server
+lint-servers: lint-server lint-czi-hosted-server
-
-.PHONY: lint-local-server
-lint-local-server: fmt-py
-	flake8 local_server --per-file-ignores='local_server/test/fixtures/dataset_config_outline.py:F821 local_server/test/fixtures/server_config_outline.py:F821 local_server/test/performance/scale_test_annotations.py:E501'
 
 .PHONY: lint-server
 lint-server: fmt-py
-	flake8 server --per-file-ignores='server/test/fixtures/dataset_config_outline.py:F821 server/test/fixtures/server_config_outline.py:F821 server/test/performance/scale_test_annotations.py:E501'
+	flake8 backend/server --per-file-ignores='backend/test/fixtures/dataset_config_outline.py:F821 backend/test/fixtures/server_config_outline.py:F821 backend/server/test/performance/scale_test_annotations.py:E501'
+
+.PHONY: lint-czi-hosted-server
+lint-czi-hosted-server: fmt-py
+	flake8 backend/czi_hosted --per-file-ignores='backend/test/fixtures/czi_hosted_dataset_config_outline.py:F821 backend/test/fixtures/czi_hosted_server_config_outline.py:F821 backend/test/performance/scale_test_annotations.py:E501'
 
 .PHONY: lint-client
 lint-client:
@@ -121,13 +148,13 @@ lint-client:
 
 # CREATING DISTRIBUTION RELEASE
 
-.PHONY: pydist-local
+.PHONY: pydist
-pydist-local: build-local
+pydist: build
 	cd $(BUILDDIR); python setup.py sdist -d ../dist
 	@echo "done"
 
 .PHONY: pydist
-pydist: build
+pydist-czi-hosted: build-czi-hosted
 	cd $(BUILDDIR); python setup.py sdist -d ../dist
 	@echo "done"
 
@@ -171,19 +198,19 @@ release-directly-to-prod: dev-env pydist twine-prod
 	@echo "    make install-release"
 
 .PHONY: dev-env
-dev-env: dev-env-client dev-env-local-server
+dev-env: dev-env-client dev-env-server
 
 .PHONY: dev-env-client
 dev-env-client:
 	cd client && $(MAKE) ci
 
-.PHONY: dev-env-local-server
-dev-env-local-server:
-	pip install -r local_server/requirements-dev.txt
-
 .PHONY: dev-env-server
 dev-env-server:
-	pip install -r server/requirements-dev.txt
+	pip install -r backend/server/requirements-dev.txt
+
+.PHONY: dev-env-czi-hosted
+dev-env-czi-hosted:
+	pip install -r backend/czi_hosted/requirements-dev.txt
 # Set PART=[major, minor, patch] as param to make bump.
 # This will create a release candidate. (i.e. 0.16.1 -> 0.16.2-rc.0 for a patch bump)
 .PHONY: bump-version

backend/common/Makefile

@@ -0,0 +1,11 @@
+
+.PHONY: unit-test
+unit-test:
+	PYTHONWARNINGS=ignore:ResourceWarning coverage run \
+		--source=fbs,utils \
+		--omit=.coverage,data_common/fbs/NetEncoding,venv \
+		-m unittest discover \
+		--start-directory ../test/test_common/unit \
+		--top-level-directory ../../ \
+		--verbose; test_result=$$?; \
+	exit $$test_result \

backend/common/colors.py

@@ -1,6 +1,6 @@
 import re
 
-from server.common.errors import ColorFormatException
+from backend.common.errors import ColorFormatException
 
 HEX_COLOR_FORMAT = re.compile("^#[a-fA-F0-9]{6,6}$")
 

backend/common/fbs/matrix.py

@@ -5,15 +5,14 @@ import pandas as pd
 from flatbuffers import Builder
 from scipy import sparse
 
-import server.data_common.fbs.NetEncoding.Column as Column
+import backend.common.fbs.NetEncoding.Column as Column
-import server.data_common.fbs.NetEncoding.Float32Array as Float32Array
+import backend.common.fbs.NetEncoding.Float32Array as Float32Array
-import server.data_common.fbs.NetEncoding.Float64Array as Float64Array
+import backend.common.fbs.NetEncoding.Float64Array as Float64Array
-import server.data_common.fbs.NetEncoding.Int32Array as Int32Array
+import backend.common.fbs.NetEncoding.Int32Array as Int32Array
-import server.data_common.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray
+import backend.common.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray
-import server.data_common.fbs.NetEncoding.Matrix as Matrix
+import backend.common.fbs.NetEncoding.Matrix as Matrix
-import server.data_common.fbs.NetEncoding.TypedArray as TypedArray
+import backend.common.fbs.NetEncoding.TypedArray as TypedArray
-import server.data_common.fbs.NetEncoding.Uint32Array as Uint32Array
+import backend.common.fbs.NetEncoding.Uint32Array as Uint32Array
-
 
 # Serialization helper
 def serialize_column(builder, typed_arr):

backend/common/utils/aws_secret_utils.py

@@ -3,7 +3,7 @@ import logging
 import boto3
 from flask import json
 
-from server.common.errors import SecretKeyRetrievalError
+from backend.common.errors import SecretKeyRetrievalError
 
 
 def get_secret_key(region_name, secret_name):

backend/common/utils/utils.py

@@ -10,7 +10,7 @@ from urllib.parse import urlsplit, urljoin
 import numpy as np
 from flask import json
 
-from server.common.errors import ConfigurationError
+from backend.common.errors import ConfigurationError
 
 
 def find_available_port(host, port=5005):

backend/czi_hosted/Makefile

@@ -1,4 +1,4 @@
-include ../common.mk
+include ../../common.mk
 
 .PHONY: clean
 clean:
@@ -9,11 +9,11 @@ clean:
 .PHONY: unit-test
 unit-test: create-test-db
 	PYTHONWARNINGS=ignore:ResourceWarning coverage run \
-		--source=app,cli,common,compute,converters,data_anndata,data_common,data_cxg \
+		--source=app,auth,cli,common,compute,converters,data_anndata,data_common,data_cxg,eb \
-		--omit=.coverage,data_common/fbs/NetEncoding,venv \
+		--omit=.coverage,venv \
 		-m unittest discover \
-		--start-directory test/ \
+		--start-directory ../test/test_czi_hosted/unit \
-		--top-level-directory ../ \
+		--top-level-directory ../.. \
 		--verbose; test_result=$$?; \
 	$(MAKE) clean-test-db; \
 	exit $$test_result \
@@ -22,11 +22,11 @@ unit-test: create-test-db
 .PHONY: test-db
 test-db: create-test-db
 	PYTHONWARNINGS=ignore:ResourceWarning coverage run \
-		--source=app,cli,common,compute,converters,data_anndata,data_common,data_cxg \
+		--source=db \
-		--omit=.coverage,data_common/fbs/NetEncoding,venv \
+		--omit=.coverage,venv \
 		-m unittest discover \
-		--start-directory test/test_database \
+		--start-directory ../test/test_czi_hosted/test_database \
-		--top-level-directory ../ \
+		--top-level-directory ../.. \
 		--verbose; test_result=$$?; \
 	$(MAKE) clean-test-db; \
 	exit $$test_result
@@ -42,8 +42,8 @@ clean-test-db:
 
 .PHONY: test-annotations-performance
 test-annotations-performance:
-	python test/performance/performance_test_annotations_backend.py
+	python ../test/test_czi_hosted/performance/performance_test_annotations_backend.py
 
 .PHONY: test-annotations-scale
 test-annotations-scale:
-	locust -f test/performance/scale_test_annotations.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt
+	locust -f ../test/test_czi_hosted/performance/scale_test_annotations.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt

backend/czi_hosted/__init__.py

@@ -1,12 +1,14 @@
 import logging
 import sys
-from server.common.utils.utils import import_plugins
+from backend.common.utils.utils import import_plugins
 
 __version__ = "0.16.7"
+
+
 display_version = "cellxgene v" + __version__
 
 try:
-    import_plugins("server.plugins")
+    import_plugins("backend.czi_hosted.plugins")
 except Exception as e:
     # Make sure to exit in this case, as the server may not be configured as expected.
     logging.critical(f"Error in import_plugins: {str(e)}")

backend/czi_hosted/__main__.py

@@ -5,8 +5,7 @@ if __package__ is None:
 
     PKG_PATH = Path(__file__).parent
     sys.path.insert(0, str(PKG_PATH.parent))
-    import server  # noqa F401
+    import backend.czi_hosted # noqa F401
-
     __package__ = PKG_PATH.name
 
 # Main thing

backend/czi_hosted/app/app.py

@@ -20,14 +20,14 @@ from flask import (
 from flask_restful import Api, Resource
 from server_timing import Timing as ServerTiming
 
-import server.common.rest as common_rest
+import backend.czi_hosted.common.rest as common_rest
-from server.common.data_locator import DataLocator
+from backend.common.utils.data_locator import DataLocator
-from server.common.errors import DatasetAccessError, RequestException
+from backend.common.errors import DatasetAccessError, RequestException
-from server.common.health import health_check
+from backend.czi_hosted.common.health import health_check
-from server.common.utils.utils import path_join, Float32JSONEncoder
+from backend.common.utils.utils import path_join, Float32JSONEncoder
-from server.data_common.matrix_loader import MatrixDataLoader
+from backend.czi_hosted.data_common.matrix_loader import MatrixDataLoader
 
-webbp = Blueprint("webapp", "server.common.web", template_folder="templates")
+webbp = Blueprint("webapp", "backend.czi_hosted.common.web", template_folder="templates")
 
 ONE_WEEK = 7 * 24 * 60 * 60
 

backend/czi_hosted/auth/__init__.py

@@ -0,0 +1,6 @@
+# import the built in auth types so they can be registered
+
+import backend.czi_hosted.auth.auth_test  # noqa: F401
+import backend.czi_hosted.auth.auth_session  # noqa: F401
+import backend.czi_hosted.auth.auth_oauth  # noqa: F401
+import backend.czi_hosted.auth.auth_none  # noqa: F401

backend/czi_hosted/auth/auth_none.py

@@ -1,4 +1,4 @@
-from local_server.auth.auth import AuthTypeBase, AuthTypeFactory
+from backend.czi_hosted.auth.auth import AuthTypeBase, AuthTypeFactory
 
 
 class AuthTypeNone(AuthTypeBase):

backend/czi_hosted/auth/auth_oauth.py

@@ -1,6 +1,6 @@
 from flask import session, request, redirect, current_app, after_this_request, has_request_context, g
-from server.auth.auth import AuthTypeClientBase, AuthTypeFactory
+from backend.czi_hosted.auth.auth import AuthTypeClientBase, AuthTypeFactory
-from server.common.errors import AuthenticationError, ConfigurationError
+from backend.common.errors import AuthenticationError, ConfigurationError
 from urllib.parse import urlencode, urlparse
 import json
 import requests

backend/czi_hosted/auth/auth_session.py

@@ -1,7 +1,8 @@
-from local_server.auth.auth import AuthTypeBase, AuthTypeFactory
 from flask import session
 from uuid import uuid4
 
+from backend.czi_hosted.auth.auth import AuthTypeBase, AuthTypeFactory
+
 
 class AuthTypeSession(AuthTypeBase):
     """Session based authentication.  The user is always logged.  The user id is a random number

backend/czi_hosted/auth/auth_test.py

@@ -1,6 +1,7 @@
-from server.auth.auth import AuthTypeClientBase, AuthTypeFactory
 from flask import session, request, redirect, current_app
 
+from backend.czi_hosted.auth.auth import AuthTypeClientBase, AuthTypeFactory
+
 
 class AuthTypeTest(AuthTypeClientBase):
     """An authentication type for testing client based logins.  When the login route is accessed

backend/czi_hosted/cli/convert_to_cxg.py

@@ -2,7 +2,7 @@ from os import path
 
 import click
 
-from server.converters.h5ad_data_file import H5ADDataFile
+from backend.czi_hosted.converters.h5ad_data_file import H5ADDataFile
 
 
 @click.command(

backend/czi_hosted/cli/launch.py

@@ -8,11 +8,12 @@ import click
 from flask_compress import Compress
 from flask_cors import CORS
 
-from server.default_config import default_config
+from backend.czi_hosted.default_config import default_config
-from server.app.app import Server
+from backend.czi_hosted.app.app import Server
-from server.common.config.app_config import AppConfig
+from backend.czi_hosted.common.config.app_config import AppConfig
-from server.common.errors import DatasetAccessError, ConfigurationError
+from backend.common.errors import DatasetAccessError, ConfigurationError
-from server.common.utils.utils import sort_options
+from backend.common.utils.utils import sort_options
+
 
 DEFAULT_CONFIG = AppConfig()
 
@@ -355,7 +356,6 @@ def launch(
     if dump_default_config:
         print(default_config)
         sys.exit(0)
-
     # Startup message
     click.echo("[cellxgene] Starting the CLI...")
 

backend/czi_hosted/cli/prepare.py

@@ -5,7 +5,7 @@ import pandas as pd
 from numpy import ndarray, unique
 from scipy.sparse.csc import csc_matrix
 
-from server.common.utils.utils import sort_options
+from backend.common.utils.utils import sort_options
 
 
 @sort_options

backend/czi_hosted/cli/schema.py

@@ -1,6 +1,6 @@
 import click
 
-from local_server.converters.schema import remix, validate
+from backend.czi_hosted.converters.schema import remix, validate
 
 
 @click.group(

backend/czi_hosted/common/annotations/annotations.py

@@ -3,8 +3,8 @@ from abc import ABCMeta, abstractmethod
 import fastobo
 import fsspec
 
-from server.common.errors import OntologyLoadFailure
+from backend.common.errors import OntologyLoadFailure
-from server.common.utils.type_conversion_utils import get_schema_type_hint_of_array
+from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
 
 
 class Annotations(metaclass=ABCMeta):

backend/czi_hosted/common/annotations/hosted_tiledb.py

@@ -7,11 +7,11 @@ import pandas as pd
 import tiledb
 from flask import current_app
 
-from server.common.annotations.annotations import Annotations
+from backend.czi_hosted.common.annotations.annotations import Annotations
-from server.common.errors import AnnotationCategoryNameError
+from backend.common.errors import AnnotationCategoryNameError
-from server.common.utils.sanitization_utils import sanitize_values_in_list
+from backend.czi_hosted.common.utils.sanitization_utils import sanitize_values_in_list
-from server.common.utils.type_conversion_utils import get_dtypes_and_schemas_of_dataframe, get_dtype_of_array
+from backend.common.utils.type_conversion_utils import get_dtypes_and_schemas_of_dataframe, get_dtype_of_array
-from server.db.cellxgene_orm import Annotation
+from backend.czi_hosted.db.cellxgene_orm import Annotation
 
 
 class AnnotationsHostedTileDB(Annotations):

backend/czi_hosted/common/annotations/local_file_csv.py

@@ -8,9 +8,9 @@ from hashlib import blake2b
 import pandas as pd
 from flask import session, has_request_context, current_app
 
-from server import __version__ as cellxgene_version
+from backend.czi_hosted import __version__ as cellxgene_version
-from server.common.annotations.annotations import Annotations
+from backend.czi_hosted.common.annotations.annotations import Annotations
-from server.common.errors import AnnotationsError
+from backend.common.errors import AnnotationsError
 
 
 class AnnotationsLocalFile(Annotations):

backend/czi_hosted/common/config/__init__.py

@@ -0,0 +1,4 @@
+from backend.common.utils.aws_secret_utils import get_secret_key  # noqa F504
+
+DEFAULT_SERVER_PORT = 5005
+BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20  # 100MB

backend/czi_hosted/common/config/app_config.py

@@ -1,11 +1,11 @@
 import yaml
 from flatten_dict import unflatten
 
-from server.default_config import get_default_config
+from backend.czi_hosted.common.config.external_config import ExternalConfig
-from server.common.config.dataset_config import DatasetConfig
+from backend.czi_hosted.common.config.dataset_config import DatasetConfig
-from server.common.config.server_config import ServerConfig
+from backend.czi_hosted.common.config.server_config import ServerConfig
-from server.common.config.external_config import ExternalConfig
+from backend.common.errors import ConfigurationError
-from server.common.errors import ConfigurationError
+from backend.czi_hosted.default_config import get_default_config
 
 
 class AppConfig(object):

backend/czi_hosted/common/config/base_config.py

@@ -1,7 +1,7 @@
 import copy
 
 from flatten_dict import flatten
-from server.common.errors import ConfigurationError
+from backend.common.errors import ConfigurationError
 
 
 class BaseConfig(object):

backend/czi_hosted/common/config/client_config.py

@@ -1,4 +1,4 @@
-from server import display_version as cellxgene_display_version
+from backend.czi_hosted import display_version as cellxgene_display_version
 
 
 def get_client_config(app_config, data_adaptor):

backend/czi_hosted/common/config/dataset_config.py

@@ -1,13 +1,13 @@
 import os
 from os.path import splitext, isdir
 
-from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
+from backend.czi_hosted.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
-from server.common.annotations.local_file_csv import AnnotationsLocalFile
+from backend.czi_hosted.common.annotations.local_file_csv import AnnotationsLocalFile
-from server.common.config.base_config import BaseConfig
+from backend.czi_hosted.common.config.base_config import BaseConfig
-from server.common.errors import ConfigurationError, OntologyLoadFailure
+from backend.common.errors import ConfigurationError, OntologyLoadFailure
-from server.compute.scanpy import get_scanpy_module
+from backend.czi_hosted.compute.scanpy import get_scanpy_module
-from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataType
+from backend.czi_hosted.data_common.matrix_loader import MatrixDataLoader, MatrixDataType
-from server.db.db_utils import DbUtils
+from backend.czi_hosted.db.db_utils import DbUtils
 
 
 class DatasetConfig(BaseConfig):

backend/czi_hosted/common/config/external_config.py

@@ -1,10 +1,9 @@
 import os
 
-from server.common.config.base_config import BaseConfig
+from backend.czi_hosted.common.config.base_config import BaseConfig
-from server.common.errors import ConfigurationError
+from backend.common.errors import ConfigurationError, SecretKeyRetrievalError
-from server.common.config import get_secret_key
+from backend.common.utils.aws_secret_utils import get_secret_key
-from server.common.errors import SecretKeyRetrievalError
+from backend.common.utils.type_conversion_utils import convert_string_to_value
-from server.common.utils.type_conversion_utils import convert_string_to_value
 
 
 class ExternalConfig(BaseConfig):

backend/czi_hosted/common/config/server_config.py

@@ -4,14 +4,14 @@ import warnings
 from os.path import basename
 from urllib.parse import urlparse, quote_plus
 
-from server.auth.auth import AuthTypeFactory
+from backend.czi_hosted.auth.auth import AuthTypeFactory
-from server.common.config.base_config import BaseConfig
+from backend.czi_hosted.common.config import DEFAULT_SERVER_PORT, BIG_FILE_SIZE_THRESHOLD
-from server.common.config import DEFAULT_SERVER_PORT, BIG_FILE_SIZE_THRESHOLD
+from backend.czi_hosted.common.config.base_config import BaseConfig
-from server.common.errors import ConfigurationError, DatasetAccessError
+from backend.common.utils.data_locator import discover_s3_region_name
-from server.common.data_locator import discover_s3_region_name
+from backend.common.errors import ConfigurationError, DatasetAccessError
-from server.common.utils.utils import is_port_available, find_available_port, custom_format_warning
+from backend.common.utils.utils import is_port_available, find_available_port, custom_format_warning
-from server.compute import diffexp_cxg as diffexp_tiledb
+from backend.czi_hosted.compute import diffexp_cxg as diffexp_tiledb
-from server.data_common.matrix_loader import MatrixDataCacheManager, MatrixDataLoader, MatrixDataType
+from backend.czi_hosted.data_common.matrix_loader import MatrixDataCacheManager, MatrixDataLoader, MatrixDataType
 
 
 class ServerConfig(BaseConfig):
@@ -353,7 +353,7 @@ class ServerConfig(BaseConfig):
             if type(self.data_locator__s3__region_name) == str:
                 self.adaptor__cxg_adaptor__tiledb_ctx[regionkey] = self.data_locator__s3__region_name
 
-        from server.data_cxg.cxg_adaptor import CxgAdaptor
+        from backend.czi_hosted.data_cxg.cxg_adaptor import CxgAdaptor
 
         CxgAdaptor.set_tiledb_context(self.adaptor__cxg_adaptor__tiledb_ctx)
 

backend/czi_hosted/common/corpora.py

@@ -8,8 +8,8 @@ https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/c
 import collections
 import json
 
-from local_server.cli.upgrade import validate_version_str
+from backend.czi_hosted.cli.upgrade import validate_version_str
-from local_server.common.utils.corpora_constants import CorporaConstants
+from backend.czi_hosted.common.utils.corpora_constants import CorporaConstants
 
 
 def corpora_get_versions_from_anndata(adata):

backend/czi_hosted/common/health.py

@@ -1,8 +1,8 @@
 from http import HTTPStatus
 from flask import make_response, jsonify
 
-from server import __version__ as cellxgene_version
+from backend.czi_hosted import __version__ as cellxgene_version
-from server.common.data_locator import DataLocator
+from backend.common.utils.data_locator import DataLocator
 
 
 def _is_accessible(path, config):

backend/czi_hosted/common/rest.py

@@ -7,9 +7,9 @@ import zlib
 from flask import make_response, jsonify, current_app, abort
 from werkzeug.urls import url_unquote
 
-from server.common.config.client_config import get_client_config, get_client_userinfo
+from backend.czi_hosted.common.config.client_config import get_client_config, get_client_userinfo
-from server.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg
+from backend.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg
-from server.common.errors import (
+from backend.common.errors import (
     FilterError,
     JSONEncodingValueError,
     PrepareError,
@@ -20,7 +20,7 @@ from server.common.errors import (
 )
 
 import json
-from server.data_common.fbs.matrix import decode_matrix_fbs
+from backend.common.fbs.matrix import decode_matrix_fbs
 
 
 def abort_and_log(code, logmsg, loglevel=logging.DEBUG, include_exc_info=False):

backend/czi_hosted/common/utils/cxg_generation_utils.py

@@ -3,7 +3,7 @@ import json
 import numpy as np
 import tiledb
 
-from server.common.utils.type_conversion_utils import get_dtype_of_array, get_dtype_and_schema_of_array
+from backend.common.utils.type_conversion_utils import get_dtype_of_array, get_dtype_and_schema_of_array
 
 
 def convert_dictionary_to_cxg_group(cxg_container, metadata_dict, group_metadata_name="cxg_group_metadata"):

backend/czi_hosted/compute/diffexp_cxg.py

@@ -1,10 +1,11 @@
 import concurrent.futures
 import numpy as np
-from server.compute.diffexp_generic import diffexp_ttest_from_mean_var, mean_var_n
+
-from server.data_cxg.cxg_util import pack_selector_from_indices
-from server.common.errors import ComputeError
 from numba import jit
 
+from backend.czi_hosted.data_cxg.cxg_util import pack_selector_from_indices
+from backend.czi_hosted.compute.diffexp_generic import diffexp_ttest_from_mean_var, mean_var_n
+from backend.common.errors import ComputeError
 
 """
 See the comments in diffexp_generic for a description of this algorithm

backend/czi_hosted/converters/h5ad_data_file.py

@@ -6,17 +6,17 @@ import anndata
 import numpy as np
 import tiledb
 
-from server.common.colors import convert_anndata_category_colors_to_cxg_category_colors
+from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
-from server.common.corpora import corpora_get_props_from_anndata
+from backend.czi_hosted.common.corpora import corpora_get_props_from_anndata
-from server.common.errors import ColorFormatException
+from backend.common.errors import ColorFormatException
-from server.common.utils.cxg_constants import CxgConstants
+from backend.czi_hosted.common.utils.cxg_constants import CxgConstants
-from server.common.utils.cxg_generation_utils import (
+from backend.czi_hosted.common.utils.cxg_generation_utils import (
     convert_dictionary_to_cxg_group,
     convert_dataframe_to_cxg_array,
     convert_ndarray_to_cxg_dense_array,
     convert_matrix_to_cxg_array,
 )
-from server.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix
+from backend.czi_hosted.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix
 
 
 class H5ADDataFile:

backend/czi_hosted/converters/to_sparse.py

@@ -9,8 +9,9 @@ import sys
 
 import tiledb
 
-from server.common.utils.cxg_generation_utils import convert_ndarray_to_cxg_dense_array, convert_matrix_to_cxg_array
+from backend.czi_hosted.common.utils.cxg_generation_utils import convert_ndarray_to_cxg_dense_array, \
-from server.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix
+    convert_matrix_to_cxg_array
+from backend.czi_hosted.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix
 
 
 def main():

backend/czi_hosted/data_anndata/anndata_adaptor.py

@@ -8,15 +8,15 @@ from pandas.core.dtypes.dtypes import CategoricalDtype
 from scipy import sparse
 from server_timing import Timing as ServerTiming
 
-import server.compute.diffexp_generic as diffexp_generic
+import backend.czi_hosted.compute.diffexp_generic as diffexp_generic
-from server.common.colors import convert_anndata_category_colors_to_cxg_category_colors
+from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
-from server.common.constants import Axis, MAX_LAYOUTS
+from backend.common.constants import Axis, MAX_LAYOUTS
-from server.common.corpora import corpora_get_props_from_anndata
+from backend.czi_hosted.common.corpora import corpora_get_props_from_anndata
-from server.common.errors import PrepareError, DatasetAccessError, FilterError
+from backend.common.errors import PrepareError, DatasetAccessError, FilterError
-from server.common.utils.type_conversion_utils import get_schema_type_hint_of_array
+from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
-from server.compute.scanpy import scanpy_umap
+from backend.czi_hosted.compute.scanpy import scanpy_umap
-from server.data_common.data_adaptor import DataAdaptor
+from backend.czi_hosted.data_common.data_adaptor import DataAdaptor
-from server.data_common.fbs.matrix import encode_matrix_fbs
+from backend.common.fbs.matrix import encode_matrix_fbs
 
 anndata_version = version.parse(str(anndata.__version__)).release
 

backend/czi_hosted/data_common/data_adaptor.py

@@ -5,11 +5,11 @@ import numpy as np
 import pandas as pd
 from server_timing import Timing as ServerTiming
 
-from server.common.config.app_config import AppConfig
+from backend.czi_hosted.common.config.app_config import AppConfig
-from server.common.constants import Axis
+from backend.common.constants import Axis
-from server.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError
+from backend.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError
-from server.common.utils.utils import jsonify_numpy
+from backend.common.utils.utils import jsonify_numpy
-from server.data_common.fbs.matrix import encode_matrix_fbs
+from backend.common.fbs.matrix import encode_matrix_fbs
 
 
 class DataAdaptor(metaclass=ABCMeta):

backend/czi_hosted/data_common/matrix_loader.py

@@ -1,12 +1,14 @@
 from enum import Enum
 import threading
 import time
-from server.data_common.rwlock import RWLock
+
-from server.common.errors import DatasetAccessError
+from backend.common.utils.data_locator import DataLocator
-from server.common.data_locator import DataLocator
+from backend.common.errors import DatasetAccessError
 from contextlib import contextmanager
 from http import HTTPStatus
 
+from backend.czi_hosted.data_common.rwlock import RWLock
+
 
 class MatrixDataCacheItem(object):
     """This class provides access and caching for a dataset.  The first time a dataset is accessed, it is
@@ -224,7 +226,7 @@ class MatrixDataLoader(object):
 
         # matrix_data_type is an enum value of type MatrixDataType
         self.matrix_data_type = matrix_data_type
-        # matrix_type is a DataAdaptor type, which corresonds to the matrix_data_type
+        # matrix_type is a DataAdaptor type, which corresponds to the matrix_data_type
         self.matrix_type = None
 
         if matrix_data_type is None:
@@ -234,11 +236,11 @@ class MatrixDataLoader(object):
             raise DatasetAccessError("Dataset does not have an allowed type.")
 
         if self.matrix_data_type == MatrixDataType.H5AD:
-            from server.data_anndata.anndata_adaptor import AnndataAdaptor
+            from backend.czi_hosted.data_anndata.anndata_adaptor import AnndataAdaptor
 
             self.matrix_type = AnndataAdaptor
         elif self.matrix_data_type == MatrixDataType.CXG:
-            from server.data_cxg.cxg_adaptor import CxgAdaptor
+            from backend.czi_hosted.data_cxg.cxg_adaptor import CxgAdaptor
 
             self.matrix_type = CxgAdaptor
 

backend/czi_hosted/data_cxg/cxg_adaptor.py

@@ -8,15 +8,15 @@ import pandas as pd
 import tiledb
 from server_timing import Timing as ServerTiming
 
-import server.compute.diffexp_cxg as diffexp_cxg
+from backend.common.constants import Axis
-from server.common.constants import Axis
+from backend.common.errors import DatasetAccessError, ConfigurationError
-from server.common.errors import DatasetAccessError, ConfigurationError
+from backend.czi_hosted.common.immutable_kvcache import ImmutableKVCache
-from server.common.immutable_kvcache import ImmutableKVCache
+from backend.common.utils.type_conversion_utils import get_schema_type_hint_from_dtype
-from server.common.utils.type_conversion_utils import get_schema_type_hint_from_dtype
+from backend.common.utils.utils import path_join
-from server.common.utils.utils import path_join
+from backend.czi_hosted.compute import diffexp_cxg
-from server.data_common.data_adaptor import DataAdaptor
+from backend.czi_hosted.data_common.data_adaptor import DataAdaptor
-from server.data_common.fbs.matrix import encode_matrix_fbs
+from backend.common.fbs.matrix import encode_matrix_fbs
-from server.data_cxg.cxg_util import pack_selector_from_mask
+from backend.czi_hosted.data_cxg.cxg_util import pack_selector_from_mask
 
 
 class CxgAdaptor(DataAdaptor):

backend/czi_hosted/db/create_db.py

@@ -3,7 +3,7 @@ Drops and recreates all tables for local testing according to cellxgene_orm.py
 """
 from sqlalchemy import create_engine
 
-from server.db.cellxgene_orm import Base
+from backend.czi_hosted.db.cellxgene_orm import Base
 
 
 def create_db(database_uri: str = "postgresql://postgres:test_pw@localhost:5432"):

backend/czi_hosted/db/db_utils.py

@@ -4,7 +4,7 @@ import uuid
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
 
-from server.db.cellxgene_orm import Base, CellxGeneDataset, CellxGeneUser
+from backend.czi_hosted.db.cellxgene_orm import Base, CellxGeneDataset, CellxGeneUser
 
 
 class DbUtils: