Refactor czi_hosted and server into backend directory, pull common code into backend/common, refactor tests (#2102)

* move local_server -> backend/server server-> backend/czi_hosted, pull common code into backend/common update imports, tests and make commands
2026-10-11 00:40:55 +08:00 · 2021-03-26 00:27:07 -05:00
parent e6e358ddc8
commit 78c9d24ed4
425 changed files with 734 additions and 5317 deletions
@@ -2,4 +2,4 @@ bin
 client
 dist
 docs
-server
+backend
@@ -41,8 +41,8 @@ jobs:
      run: |
        # 1. only install the dev requirements on top of what is in the cellxgene pip package
        sudo apt-get update && sudo apt-get install -y libhdf5-serial-dev
-        sed -i 's/-r requirements.txt//' server/requirements-dev.txt
-        pip install -r server/requirements-dev.txt
+        sed -i 's/-r requirements.txt//' backend/server/requirements-dev.txt
+        pip install -r backend/server/requirements-dev.txt
        # 2. install cellxgene
        make pydist install-dist
        # 3. install anndata
@@ -73,8 +73,8 @@ jobs:
          cd cellxgene
          # 1. only install the dev requirements on top of what is in the cellxgene pip package
          make dev-env-client
-          sed -i 's/-r requirements.txt//' server/requirements-dev.txt
-          pip install -r server/requirements-dev.txt
+          sed -i 's/-r requirements.txt//' backend/server/requirements-dev.txt
+          pip install -r backend/server/requirements-dev.txt
          # 2. install cellxgene
          pip install --upgrade cellxgene
          # 3. install anndata
@@ -101,7 +101,7 @@ jobs:
      - name: Install dependencies
        run: |
          cd cellxgene
-          sed -i -E 's/^anndata[>=]=[0-9]+.[0-9]+.[0-9]+$/anndata/g' server/requirements.txt
+          sed -i -E 's/^anndata[>=]=[0-9]+.[0-9]+.[0-9]+$/anndata/g' backend/server/requirements.txt
          make pydist install-dist dev-env
          pip install git+https://github.com/theislab/anndata
      - name: Tests
@@ -68,11 +68,11 @@ jobs:
        run: make pydist install-dist dev-env-server
      - name: Unit tests
        run: |
-          make unit-test
-          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k server -cF backend,python,unitTest
+          make unit-test-server
+          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k backend/server -cF backend,python,unitTest
          cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest

-  unit-test-local:
+  unit-test-czi-hosted:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
@@ -95,11 +95,11 @@ jobs:
          restore-keys: |
            ${{ runner.os }}-node-
      - name: Install dependencies
-        run: make pydist-local install-dist dev-env-local-server
+        run: make pydist-czi-hosted install-dist dev-env-czi-hosted
      - name: Unit tests
        run: |
-          make unit-test-local
-          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k local_server -cF backend,python,unitTest
+          make unit-test-czi-hosted
+          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k backend/czi-hosted -cF backend,python,unitTest
          cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest

  smoke-tests:
@@ -126,7 +126,7 @@ jobs:
          restore-keys: |
            ${{ runner.os }}-node-
      - name: Install dependencies
-        run: make pydist install-dist
+        run: make pydist-czi-hosted install-dist
      - name: Smoke tests (without annotations feature)
        run: |
          cd client && make smoke-test
@@ -156,7 +156,7 @@ jobs:
          restore-keys: |
            ${{ runner.os }}-node-
      - name: Install dependencies
-        run: make pydist-local install-dist
+        run: make pydist install-dist
      - name: Smoke tests (with annotations feature)
        run: |
          cd client && make smoke-test-annotations
@@ -15,10 +15,10 @@ jobs:
          python-version: 3.7
      - name: Install dependencies
        run: |
-          pip install -r server/test/locust/requirements-locust.txt
+          pip install -r backend/czi_hosted/test/locust/requirements-locust.txt
      - name: Dev Scale Test
        run: |
-          locust -f server/test/locust/locustfile.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt
+          locust -f backend/czi_hosted/test/locust/locustfile.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt
      - name: Slack success webhook
        env:
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
@@ -19,9 +19,13 @@ venv/
 cellxgene/

 # client build
-server/common/web/static/*
-server/common/web/templates/
-server/common/web/csp-hashes.json
+backend/server/common/web/static/*
+backend/server/common/web/templates/
+backend/server/common/web/csp-hashes.json
+
+backend/czi_hosted/common/web/static/*
+backend/czi_hosted/common/web/templates/
+backend/czi_hosted/common/web/csp-hashes.json

 # eb build
 artifact.dir
@@ -1,7 +1,7 @@
-recursive-include local_server/common/web/templates *
-recursive-include local_server/common/web/static *
+recursive-include backend/server/common/web/templates *
+recursive-include backend/server/common/web/static *

-include local_server/requirements.txt
-include local_server/requirements-prepare.txt
-include local_server/converters/schema/hgnc_complete_set.txt.gz
-include local_server/converters/schema/schema_definitions/*
+include backend/server/requirements.txt
+include backend/server/requirements-prepare.txt
+include backend/server/converters/schema/hgnc_complete_set.txt.gz
+include backend/server/converters/schema/schema_definitions/*
@@ -1,7 +1,7 @@
-recursive-include server/common/web/templates *
-recursive-include server/common/web/static *
+recursive-include backend/czi_hosted/common/web/templates *
+recursive-include backend/czi_hosted/common/web/static *

-include server/requirements.txt
-include server/requirements-prepare.txt
-include server/converters/schema/hgnc_complete_set.txt.gz
-include server/converters/schema/schema_definitions/*
+include backend/czi_hosted/requirements.txt
+include backend/czi_hosted/requirements-prepare.txt
+include backend/czi_hosted/converters/schema/hgnc_complete_set.txt.gz
+include backend/czi_hosted/converters/schema/schema_definitions/*
@@ -2,24 +2,32 @@ include common.mk

 BUILDDIR := build
 CLIENTBUILD := $(BUILDDIR)/client
-SERVERBUILD := $(BUILDDIR)/server
-LOCALSERVERBUILD := $(BUILDDIR)/local_server
+CZIHOSTEDBUILD := $(BUILDDIR)/backend/czi_hosted
+SERVERBUILD := $(BUILDDIR)/backend/server
 CLEANFILES :=  $(BUILDDIR)/ client/build build dist cellxgene.egg-info

 PART ?= patch

 # CLEANING
 .PHONY: clean
-clean: clean-lite clean-local-server clean-server clean-client
+clean: clean-lite clean-czi-hosted clean-server clean-client

 # cleaning the client's node_modules is the longest one, so we avoid that if possible
 .PHONY: clean-lite
 clean-lite:
 	rm -rf $(CLEANFILES)

-clean-%:
-	cd $(subst -,_,$*) && $(MAKE) clean
+.PHONY: clean-client
+clean-client:
+	cd client && $(MAKE) clean

+.PHONY: clean-server
+clean-server:
+	cd backend/server && $(MAKE) clean
+
+.PHONY: clean-czi-hosted
+clean-czi-hosted:
+	cd backend/czi_hosted && $(MAKE) clean

 # BUILDING PACKAGE

@@ -27,54 +35,73 @@ clean-%:
 build-client:
 	cd client && $(MAKE) ci build

-.PHONY: build-local
-build-local: clean build-client
-	git ls-files local_server/ | grep -v 'local_server/test/' | cpio -pdm $(BUILDDIR)
-	cp -r client/build/  $(CLIENTBUILD)
-	$(call copy_client_assets,$(CLIENTBUILD),$(LOCALSERVERBUILD))
-	cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR)
-
 .PHONY: build
 build: clean build-client
-	git ls-files server/ | grep -v 'server/test/' | cpio -pdm $(BUILDDIR)
+	git ls-files backend/server/ | grep -v 'backend/server/test/' | cpio -pdm $(BUILDDIR)
 	cp -r client/build/  $(CLIENTBUILD)
 	$(call copy_client_assets,$(CLIENTBUILD),$(SERVERBUILD))
+	cp backend/__init__.py $(BUILDDIR)
+	cp backend/__init__.py $(BUILDDIR)/backend
+	cp -r backend/common $(BUILDDIR)/backend/common
+	cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR)
+
+.PHONY: build-czi-hosted
+build-czi-hosted: clean build-client
+	git ls-files backend/czi_hosted/ | grep -v 'backend/czi_hosted/test/' | cpio -pdm $(BUILDDIR)
+	cp -r client/build/  $(CLIENTBUILD)
+	$(call copy_client_assets,$(CLIENTBUILD),$(CZIHOSTEDBUILD))
+	cp -r backend/common $(BUILDDIR)/backend/common
+	cp backend/__init__.py $(BUILDDIR)
+	cp backend/__init__.py $(BUILDDIR)/backend
 	cp MANIFEST_hosted.in README.md setup.cfg setup_hosted.py $(BUILDDIR)
 	mv $(BUILDDIR)/setup_hosted.py $(BUILDDIR)/setup.py
 	mv $(BUILDDIR)/MANIFEST_hosted.in $(BUILDDIR)/MANIFEST.in

 # If you are actively developing in the server folder use this, dirties the source tree
-.PHONY: build-for-server-dev-local
-build-for-server-dev-local: clean-local-server build-client
-	$(call copy_client_assets,client/build,local_server)
-
 .PHONY: build-for-server-dev
 build-for-server-dev: clean-server build-client
-	$(call copy_client_assets,client/build,server)
+	$(call copy_client_assets,client/build,backend/server)

-.PHONY: copy-client-assets-local
-copy-client-assets-local:
-	$(call copy_client_assets,client/build,local_server)
+.PHONY: build-for-czi-hosted-dev
+build-for-czi-hosted-dev: clean-czi-hosted build-client
+	$(call copy_client_assets,client/build,backend/czi_hosted)

 .PHONY: copy-client-assets
 copy-client-assets:
-	$(call copy_client_assets,client/build,server)
+	$(call copy_client_assets,client/build,backend/server)
+
+.PHONY: copy-client-assets-czi-hosted
+copy-client-assets-czi-hosted:
+	$(call copy_client_assets,client/build,backend/czi_hosted)

 # TESTING
 .PHONY: test
 test: unit-test smoke-test

-.PHONY: test-local
-test-local: unit-test-local smoke-test
-
-.PHONY: unit-test-local
-unit-test-local: unit-test-local-server unit-test-client
-
 .PHONY: unit-test
-unit-test: unit-test-server unit-test-client
+unit-test: unit-test-server unit-test-client unit-test-common

-unit-test-%:
-	cd $(subst -,_,$*) && $(MAKE) unit-test
+.PHONY: test-server
+test-server: unit-test-server smoke-test
+
+.PHONY: test-czi-hosted
+test-czi-hosted: unit-test-czi-hosted smoke-test
+
+.PHONY: unit-test-client
+unit-test-client:
+	cd client && $(MAKE) unit-test
+
+.PHONY: unit-test-czi-hosted
+unit-test-czi-hosted:
+	cd backend/czi_hosted && $(MAKE) unit-test
+
+.PHONY: unit-test-server
+unit-test-server:
+	cd backend/server && $(MAKE) unit-test
+
+.PHONY: unit-test-common
+unit-test-common:
+	cd backend/common && $(MAKE) unit-test

 .PHONY: smoke-test
 smoke-test:
@@ -86,7 +113,7 @@ smoke-test-annotations:

 .PHONY: test-db
 test-db:
-	cd server && $(MAKE) test-db
+	cd backend/czi_hosted && $(MAKE) test-db

 # FORMATTING CODE

@@ -105,15 +132,15 @@ fmt-py:
 lint: lint-servers lint-client

 .PHONY: lint-servers
-lint-servers: lint-local-server lint-server
-
-.PHONY: lint-local-server
-lint-local-server: fmt-py
-	flake8 local_server --per-file-ignores='local_server/test/fixtures/dataset_config_outline.py:F821 local_server/test/fixtures/server_config_outline.py:F821 local_server/test/performance/scale_test_annotations.py:E501'
+lint-servers: lint-server lint-czi-hosted-server

 .PHONY: lint-server
 lint-server: fmt-py
-	flake8 server --per-file-ignores='server/test/fixtures/dataset_config_outline.py:F821 server/test/fixtures/server_config_outline.py:F821 server/test/performance/scale_test_annotations.py:E501'
+	flake8 backend/server --per-file-ignores='backend/test/fixtures/dataset_config_outline.py:F821 backend/test/fixtures/server_config_outline.py:F821 backend/server/test/performance/scale_test_annotations.py:E501'
+
+.PHONY: lint-czi-hosted-server
+lint-czi-hosted-server: fmt-py
+	flake8 backend/czi_hosted --per-file-ignores='backend/test/fixtures/czi_hosted_dataset_config_outline.py:F821 backend/test/fixtures/czi_hosted_server_config_outline.py:F821 backend/test/performance/scale_test_annotations.py:E501'

 .PHONY: lint-client
 lint-client:
@@ -121,13 +148,13 @@ lint-client:

 # CREATING DISTRIBUTION RELEASE

-.PHONY: pydist-local
-pydist-local: build-local
+.PHONY: pydist
+pydist: build
 	cd $(BUILDDIR); python setup.py sdist -d ../dist
 	@echo "done"

 .PHONY: pydist
-pydist: build
+pydist-czi-hosted: build-czi-hosted
 	cd $(BUILDDIR); python setup.py sdist -d ../dist
 	@echo "done"

@@ -171,19 +198,19 @@ release-directly-to-prod: dev-env pydist twine-prod
 	@echo "    make install-release"

 .PHONY: dev-env
-dev-env: dev-env-client dev-env-local-server
+dev-env: dev-env-client dev-env-server

 .PHONY: dev-env-client
 dev-env-client:
 	cd client && $(MAKE) ci

-.PHONY: dev-env-local-server
-dev-env-local-server:
-	pip install -r local_server/requirements-dev.txt
-
 .PHONY: dev-env-server
 dev-env-server:
-	pip install -r server/requirements-dev.txt
+	pip install -r backend/server/requirements-dev.txt
+
+.PHONY: dev-env-czi-hosted
+dev-env-czi-hosted:
+	pip install -r backend/czi_hosted/requirements-dev.txt
 # Set PART=[major, minor, patch] as param to make bump.
 # This will create a release candidate. (i.e. 0.16.1 -> 0.16.2-rc.0 for a patch bump)
 .PHONY: bump-version
@@ -0,0 +1,11 @@
+
+.PHONY: unit-test
+unit-test:
+	PYTHONWARNINGS=ignore:ResourceWarning coverage run \
+		--source=fbs,utils \
+		--omit=.coverage,data_common/fbs/NetEncoding,venv \
+		-m unittest discover \
+		--start-directory ../test/test_common/unit \
+		--top-level-directory ../../ \
+		--verbose; test_result=$$?; \
+	exit $$test_result \
@@ -1,6 +1,6 @@
 import re

-from server.common.errors import ColorFormatException
+from backend.common.errors import ColorFormatException

 HEX_COLOR_FORMAT = re.compile("^#[a-fA-F0-9]{6,6}$")

@@ -5,15 +5,14 @@ import pandas as pd
 from flatbuffers import Builder
 from scipy import sparse

-import server.data_common.fbs.NetEncoding.Column as Column
-import server.data_common.fbs.NetEncoding.Float32Array as Float32Array
-import server.data_common.fbs.NetEncoding.Float64Array as Float64Array
-import server.data_common.fbs.NetEncoding.Int32Array as Int32Array
-import server.data_common.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray
-import server.data_common.fbs.NetEncoding.Matrix as Matrix
-import server.data_common.fbs.NetEncoding.TypedArray as TypedArray
-import server.data_common.fbs.NetEncoding.Uint32Array as Uint32Array
-
+import backend.common.fbs.NetEncoding.Column as Column
+import backend.common.fbs.NetEncoding.Float32Array as Float32Array
+import backend.common.fbs.NetEncoding.Float64Array as Float64Array
+import backend.common.fbs.NetEncoding.Int32Array as Int32Array
+import backend.common.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray
+import backend.common.fbs.NetEncoding.Matrix as Matrix
+import backend.common.fbs.NetEncoding.TypedArray as TypedArray
+import backend.common.fbs.NetEncoding.Uint32Array as Uint32Array

 # Serialization helper
 def serialize_column(builder, typed_arr):
@@ -3,7 +3,7 @@ import logging
 import boto3
 from flask import json

-from server.common.errors import SecretKeyRetrievalError
+from backend.common.errors import SecretKeyRetrievalError


 def get_secret_key(region_name, secret_name):
@@ -10,7 +10,7 @@ from urllib.parse import urlsplit, urljoin
 import numpy as np
 from flask import json

-from server.common.errors import ConfigurationError
+from backend.common.errors import ConfigurationError


 def find_available_port(host, port=5005):
@@ -1,4 +1,4 @@
-include ../common.mk
+include ../../common.mk

 .PHONY: clean
 clean:
@@ -9,11 +9,11 @@ clean:
 .PHONY: unit-test
 unit-test: create-test-db
 	PYTHONWARNINGS=ignore:ResourceWarning coverage run \
-		--source=app,cli,common,compute,converters,data_anndata,data_common,data_cxg \
-		--omit=.coverage,data_common/fbs/NetEncoding,venv \
+		--source=app,auth,cli,common,compute,converters,data_anndata,data_common,data_cxg,eb \
+		--omit=.coverage,venv \
 		-m unittest discover \
-		--start-directory test/ \
-		--top-level-directory ../ \
+		--start-directory ../test/test_czi_hosted/unit \
+		--top-level-directory ../.. \
 		--verbose; test_result=$$?; \
 	$(MAKE) clean-test-db; \
 	exit $$test_result \
@@ -22,11 +22,11 @@ unit-test: create-test-db
 .PHONY: test-db
 test-db: create-test-db
 	PYTHONWARNINGS=ignore:ResourceWarning coverage run \
-		--source=app,cli,common,compute,converters,data_anndata,data_common,data_cxg \
-		--omit=.coverage,data_common/fbs/NetEncoding,venv \
+		--source=db \
+		--omit=.coverage,venv \
 		-m unittest discover \
-		--start-directory test/test_database \
-		--top-level-directory ../ \
+		--start-directory ../test/test_czi_hosted/test_database \
+		--top-level-directory ../.. \
 		--verbose; test_result=$$?; \
 	$(MAKE) clean-test-db; \
 	exit $$test_result
@@ -42,8 +42,8 @@ clean-test-db:

 .PHONY: test-annotations-performance
 test-annotations-performance:
-	python test/performance/performance_test_annotations_backend.py
+	python ../test/test_czi_hosted/performance/performance_test_annotations_backend.py

 .PHONY: test-annotations-scale
 test-annotations-scale:
-	locust -f test/performance/scale_test_annotations.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt
+	locust -f ../test/test_czi_hosted/performance/scale_test_annotations.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt
@@ -1,12 +1,14 @@
 import logging
 import sys
-from server.common.utils.utils import import_plugins
+from backend.common.utils.utils import import_plugins

 __version__ = "0.16.7"
+
+
 display_version = "cellxgene v" + __version__

 try:
-    import_plugins("server.plugins")
+    import_plugins("backend.czi_hosted.plugins")
 except Exception as e:
    # Make sure to exit in this case, as the server may not be configured as expected.
    logging.critical(f"Error in import_plugins: {str(e)}")
@@ -5,8 +5,7 @@ if __package__ is None:

    PKG_PATH = Path(__file__).parent
    sys.path.insert(0, str(PKG_PATH.parent))
-    import server  # noqa F401
-
+    import backend.czi_hosted # noqa F401
    __package__ = PKG_PATH.name

 # Main thing
@@ -20,14 +20,14 @@ from flask import (
 from flask_restful import Api, Resource
 from server_timing import Timing as ServerTiming

-import server.common.rest as common_rest
-from server.common.data_locator import DataLocator
-from server.common.errors import DatasetAccessError, RequestException
-from server.common.health import health_check
-from server.common.utils.utils import path_join, Float32JSONEncoder
-from server.data_common.matrix_loader import MatrixDataLoader
+import backend.czi_hosted.common.rest as common_rest
+from backend.common.utils.data_locator import DataLocator
+from backend.common.errors import DatasetAccessError, RequestException
+from backend.czi_hosted.common.health import health_check
+from backend.common.utils.utils import path_join, Float32JSONEncoder
+from backend.czi_hosted.data_common.matrix_loader import MatrixDataLoader

-webbp = Blueprint("webapp", "server.common.web", template_folder="templates")
+webbp = Blueprint("webapp", "backend.czi_hosted.common.web", template_folder="templates")

 ONE_WEEK = 7 * 24 * 60 * 60

@@ -0,0 +1,6 @@
+# import the built in auth types so they can be registered
+
+import backend.czi_hosted.auth.auth_test  # noqa: F401
+import backend.czi_hosted.auth.auth_session  # noqa: F401
+import backend.czi_hosted.auth.auth_oauth  # noqa: F401
+import backend.czi_hosted.auth.auth_none  # noqa: F401
@@ -1,4 +1,4 @@
-from local_server.auth.auth import AuthTypeBase, AuthTypeFactory
+from backend.czi_hosted.auth.auth import AuthTypeBase, AuthTypeFactory


 class AuthTypeNone(AuthTypeBase):
@@ -1,6 +1,6 @@
 from flask import session, request, redirect, current_app, after_this_request, has_request_context, g
-from server.auth.auth import AuthTypeClientBase, AuthTypeFactory
-from server.common.errors import AuthenticationError, ConfigurationError
+from backend.czi_hosted.auth.auth import AuthTypeClientBase, AuthTypeFactory
+from backend.common.errors import AuthenticationError, ConfigurationError
 from urllib.parse import urlencode, urlparse
 import json
 import requests
@@ -1,7 +1,8 @@
-from local_server.auth.auth import AuthTypeBase, AuthTypeFactory
 from flask import session
 from uuid import uuid4

+from backend.czi_hosted.auth.auth import AuthTypeBase, AuthTypeFactory
+

 class AuthTypeSession(AuthTypeBase):
    """Session based authentication.  The user is always logged.  The user id is a random number
@@ -1,6 +1,7 @@
-from server.auth.auth import AuthTypeClientBase, AuthTypeFactory
 from flask import session, request, redirect, current_app

+from backend.czi_hosted.auth.auth import AuthTypeClientBase, AuthTypeFactory
+

 class AuthTypeTest(AuthTypeClientBase):
    """An authentication type for testing client based logins.  When the login route is accessed
@@ -2,7 +2,7 @@ from os import path

 import click

-from server.converters.h5ad_data_file import H5ADDataFile
+from backend.czi_hosted.converters.h5ad_data_file import H5ADDataFile


@click.command(
@@ -8,11 +8,12 @@ import click
 from flask_compress import Compress
 from flask_cors import CORS

-from server.default_config import default_config
-from server.app.app import Server
-from server.common.config.app_config import AppConfig
-from server.common.errors import DatasetAccessError, ConfigurationError
-from server.common.utils.utils import sort_options
+from backend.czi_hosted.default_config import default_config
+from backend.czi_hosted.app.app import Server
+from backend.czi_hosted.common.config.app_config import AppConfig
+from backend.common.errors import DatasetAccessError, ConfigurationError
+from backend.common.utils.utils import sort_options
+

 DEFAULT_CONFIG = AppConfig()

@@ -355,7 +356,6 @@ def launch(
    if dump_default_config:
        print(default_config)
        sys.exit(0)
-
    # Startup message
    click.echo("[cellxgene] Starting the CLI...")

@@ -5,7 +5,7 @@ import pandas as pd
 from numpy import ndarray, unique
 from scipy.sparse.csc import csc_matrix

-from server.common.utils.utils import sort_options
+from backend.common.utils.utils import sort_options


@sort_options
@@ -1,6 +1,6 @@
 import click

-from local_server.converters.schema import remix, validate
+from backend.czi_hosted.converters.schema import remix, validate


@click.group(
@@ -3,8 +3,8 @@ from abc import ABCMeta, abstractmethod
 import fastobo
 import fsspec

-from server.common.errors import OntologyLoadFailure
-from server.common.utils.type_conversion_utils import get_schema_type_hint_of_array
+from backend.common.errors import OntologyLoadFailure
+from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array


 class Annotations(metaclass=ABCMeta):
@@ -7,11 +7,11 @@ import pandas as pd
 import tiledb
 from flask import current_app

-from server.common.annotations.annotations import Annotations
-from server.common.errors import AnnotationCategoryNameError
-from server.common.utils.sanitization_utils import sanitize_values_in_list
-from server.common.utils.type_conversion_utils import get_dtypes_and_schemas_of_dataframe, get_dtype_of_array
-from server.db.cellxgene_orm import Annotation
+from backend.czi_hosted.common.annotations.annotations import Annotations
+from backend.common.errors import AnnotationCategoryNameError
+from backend.czi_hosted.common.utils.sanitization_utils import sanitize_values_in_list
+from backend.common.utils.type_conversion_utils import get_dtypes_and_schemas_of_dataframe, get_dtype_of_array
+from backend.czi_hosted.db.cellxgene_orm import Annotation


 class AnnotationsHostedTileDB(Annotations):
@@ -8,9 +8,9 @@ from hashlib import blake2b
 import pandas as pd
 from flask import session, has_request_context, current_app

-from server import __version__ as cellxgene_version
-from server.common.annotations.annotations import Annotations
-from server.common.errors import AnnotationsError
+from backend.czi_hosted import __version__ as cellxgene_version
+from backend.czi_hosted.common.annotations.annotations import Annotations
+from backend.common.errors import AnnotationsError


 class AnnotationsLocalFile(Annotations):
@@ -0,0 +1,4 @@
+from backend.common.utils.aws_secret_utils import get_secret_key  # noqa F504
+
+DEFAULT_SERVER_PORT = 5005
+BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20  # 100MB
@@ -1,11 +1,11 @@
 import yaml
 from flatten_dict import unflatten

-from server.default_config import get_default_config
-from server.common.config.dataset_config import DatasetConfig
-from server.common.config.server_config import ServerConfig
-from server.common.config.external_config import ExternalConfig
-from server.common.errors import ConfigurationError
+from backend.czi_hosted.common.config.external_config import ExternalConfig
+from backend.czi_hosted.common.config.dataset_config import DatasetConfig
+from backend.czi_hosted.common.config.server_config import ServerConfig
+from backend.common.errors import ConfigurationError
+from backend.czi_hosted.default_config import get_default_config


 class AppConfig(object):
@@ -1,7 +1,7 @@
 import copy

 from flatten_dict import flatten
-from server.common.errors import ConfigurationError
+from backend.common.errors import ConfigurationError


 class BaseConfig(object):
@@ -1,4 +1,4 @@
-from server import display_version as cellxgene_display_version
+from backend.czi_hosted import display_version as cellxgene_display_version


 def get_client_config(app_config, data_adaptor):
@@ -1,13 +1,13 @@
 import os
 from os.path import splitext, isdir

-from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
-from server.common.annotations.local_file_csv import AnnotationsLocalFile
-from server.common.config.base_config import BaseConfig
-from server.common.errors import ConfigurationError, OntologyLoadFailure
-from server.compute.scanpy import get_scanpy_module
-from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataType
-from server.db.db_utils import DbUtils
+from backend.czi_hosted.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
+from backend.czi_hosted.common.annotations.local_file_csv import AnnotationsLocalFile
+from backend.czi_hosted.common.config.base_config import BaseConfig
+from backend.common.errors import ConfigurationError, OntologyLoadFailure
+from backend.czi_hosted.compute.scanpy import get_scanpy_module
+from backend.czi_hosted.data_common.matrix_loader import MatrixDataLoader, MatrixDataType
+from backend.czi_hosted.db.db_utils import DbUtils


 class DatasetConfig(BaseConfig):
@@ -1,10 +1,9 @@
 import os

-from server.common.config.base_config import BaseConfig
-from server.common.errors import ConfigurationError
-from server.common.config import get_secret_key
-from server.common.errors import SecretKeyRetrievalError
-from server.common.utils.type_conversion_utils import convert_string_to_value
+from backend.czi_hosted.common.config.base_config import BaseConfig
+from backend.common.errors import ConfigurationError, SecretKeyRetrievalError
+from backend.common.utils.aws_secret_utils import get_secret_key
+from backend.common.utils.type_conversion_utils import convert_string_to_value


 class ExternalConfig(BaseConfig):
@@ -4,14 +4,14 @@ import warnings
 from os.path import basename
 from urllib.parse import urlparse, quote_plus

-from server.auth.auth import AuthTypeFactory
-from server.common.config.base_config import BaseConfig
-from server.common.config import DEFAULT_SERVER_PORT, BIG_FILE_SIZE_THRESHOLD
-from server.common.errors import ConfigurationError, DatasetAccessError
-from server.common.data_locator import discover_s3_region_name
-from server.common.utils.utils import is_port_available, find_available_port, custom_format_warning
-from server.compute import diffexp_cxg as diffexp_tiledb
-from server.data_common.matrix_loader import MatrixDataCacheManager, MatrixDataLoader, MatrixDataType
+from backend.czi_hosted.auth.auth import AuthTypeFactory
+from backend.czi_hosted.common.config import DEFAULT_SERVER_PORT, BIG_FILE_SIZE_THRESHOLD
+from backend.czi_hosted.common.config.base_config import BaseConfig
+from backend.common.utils.data_locator import discover_s3_region_name
+from backend.common.errors import ConfigurationError, DatasetAccessError
+from backend.common.utils.utils import is_port_available, find_available_port, custom_format_warning
+from backend.czi_hosted.compute import diffexp_cxg as diffexp_tiledb
+from backend.czi_hosted.data_common.matrix_loader import MatrixDataCacheManager, MatrixDataLoader, MatrixDataType


 class ServerConfig(BaseConfig):
@@ -353,7 +353,7 @@ class ServerConfig(BaseConfig):
            if type(self.data_locator__s3__region_name) == str:
                self.adaptor__cxg_adaptor__tiledb_ctx[regionkey] = self.data_locator__s3__region_name

-        from server.data_cxg.cxg_adaptor import CxgAdaptor
+        from backend.czi_hosted.data_cxg.cxg_adaptor import CxgAdaptor

        CxgAdaptor.set_tiledb_context(self.adaptor__cxg_adaptor__tiledb_ctx)

@@ -8,8 +8,8 @@ https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/c
 import collections
 import json

-from local_server.cli.upgrade import validate_version_str
-from local_server.common.utils.corpora_constants import CorporaConstants
+from backend.czi_hosted.cli.upgrade import validate_version_str
+from backend.czi_hosted.common.utils.corpora_constants import CorporaConstants


 def corpora_get_versions_from_anndata(adata):
@@ -1,8 +1,8 @@
 from http import HTTPStatus
 from flask import make_response, jsonify

-from server import __version__ as cellxgene_version
-from server.common.data_locator import DataLocator
+from backend.czi_hosted import __version__ as cellxgene_version
+from backend.common.utils.data_locator import DataLocator


 def _is_accessible(path, config):
@@ -7,9 +7,9 @@ import zlib
 from flask import make_response, jsonify, current_app, abort
 from werkzeug.urls import url_unquote

-from server.common.config.client_config import get_client_config, get_client_userinfo
-from server.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg
-from server.common.errors import (
+from backend.czi_hosted.common.config.client_config import get_client_config, get_client_userinfo
+from backend.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg
+from backend.common.errors import (
    FilterError,
    JSONEncodingValueError,
    PrepareError,
@@ -20,7 +20,7 @@ from server.common.errors import (
 )

 import json
-from server.data_common.fbs.matrix import decode_matrix_fbs
+from backend.common.fbs.matrix import decode_matrix_fbs


 def abort_and_log(code, logmsg, loglevel=logging.DEBUG, include_exc_info=False):
@@ -3,7 +3,7 @@ import json
 import numpy as np
 import tiledb

-from server.common.utils.type_conversion_utils import get_dtype_of_array, get_dtype_and_schema_of_array
+from backend.common.utils.type_conversion_utils import get_dtype_of_array, get_dtype_and_schema_of_array


 def convert_dictionary_to_cxg_group(cxg_container, metadata_dict, group_metadata_name="cxg_group_metadata"):
@@ -1,10 +1,11 @@
 import concurrent.futures
 import numpy as np
-from server.compute.diffexp_generic import diffexp_ttest_from_mean_var, mean_var_n
-from server.data_cxg.cxg_util import pack_selector_from_indices
-from server.common.errors import ComputeError
+
 from numba import jit

+from backend.czi_hosted.data_cxg.cxg_util import pack_selector_from_indices
+from backend.czi_hosted.compute.diffexp_generic import diffexp_ttest_from_mean_var, mean_var_n
+from backend.common.errors import ComputeError

 """
 See the comments in diffexp_generic for a description of this algorithm
@@ -6,17 +6,17 @@ import anndata
 import numpy as np
 import tiledb

-from server.common.colors import convert_anndata_category_colors_to_cxg_category_colors
-from server.common.corpora import corpora_get_props_from_anndata
-from server.common.errors import ColorFormatException
-from server.common.utils.cxg_constants import CxgConstants
-from server.common.utils.cxg_generation_utils import (
+from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
+from backend.czi_hosted.common.corpora import corpora_get_props_from_anndata
+from backend.common.errors import ColorFormatException
+from backend.czi_hosted.common.utils.cxg_constants import CxgConstants
+from backend.czi_hosted.common.utils.cxg_generation_utils import (
    convert_dictionary_to_cxg_group,
    convert_dataframe_to_cxg_array,
    convert_ndarray_to_cxg_dense_array,
    convert_matrix_to_cxg_array,
 )
-from server.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix
+from backend.czi_hosted.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix


 class H5ADDataFile:
@@ -9,8 +9,9 @@ import sys

 import tiledb

-from server.common.utils.cxg_generation_utils import convert_ndarray_to_cxg_dense_array, convert_matrix_to_cxg_array
-from server.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix
+from backend.czi_hosted.common.utils.cxg_generation_utils import convert_ndarray_to_cxg_dense_array, \
+    convert_matrix_to_cxg_array
+from backend.czi_hosted.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix


 def main():
@@ -8,15 +8,15 @@ from pandas.core.dtypes.dtypes import CategoricalDtype
 from scipy import sparse
 from server_timing import Timing as ServerTiming

-import server.compute.diffexp_generic as diffexp_generic
-from server.common.colors import convert_anndata_category_colors_to_cxg_category_colors
-from server.common.constants import Axis, MAX_LAYOUTS
-from server.common.corpora import corpora_get_props_from_anndata
-from server.common.errors import PrepareError, DatasetAccessError, FilterError
-from server.common.utils.type_conversion_utils import get_schema_type_hint_of_array
-from server.compute.scanpy import scanpy_umap
-from server.data_common.data_adaptor import DataAdaptor
-from server.data_common.fbs.matrix import encode_matrix_fbs
+import backend.czi_hosted.compute.diffexp_generic as diffexp_generic
+from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
+from backend.common.constants import Axis, MAX_LAYOUTS
+from backend.czi_hosted.common.corpora import corpora_get_props_from_anndata
+from backend.common.errors import PrepareError, DatasetAccessError, FilterError
+from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
+from backend.czi_hosted.compute.scanpy import scanpy_umap
+from backend.czi_hosted.data_common.data_adaptor import DataAdaptor
+from backend.common.fbs.matrix import encode_matrix_fbs

 anndata_version = version.parse(str(anndata.__version__)).release

@@ -5,11 +5,11 @@ import numpy as np
 import pandas as pd
 from server_timing import Timing as ServerTiming

-from server.common.config.app_config import AppConfig
-from server.common.constants import Axis
-from server.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError
-from server.common.utils.utils import jsonify_numpy
-from server.data_common.fbs.matrix import encode_matrix_fbs
+from backend.czi_hosted.common.config.app_config import AppConfig
+from backend.common.constants import Axis
+from backend.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError
+from backend.common.utils.utils import jsonify_numpy
+from backend.common.fbs.matrix import encode_matrix_fbs


 class DataAdaptor(metaclass=ABCMeta):
@@ -1,12 +1,14 @@
 from enum import Enum
 import threading
 import time
-from server.data_common.rwlock import RWLock
-from server.common.errors import DatasetAccessError
-from server.common.data_locator import DataLocator
+
+from backend.common.utils.data_locator import DataLocator
+from backend.common.errors import DatasetAccessError
 from contextlib import contextmanager
 from http import HTTPStatus

+from backend.czi_hosted.data_common.rwlock import RWLock
+

 class MatrixDataCacheItem(object):
    """This class provides access and caching for a dataset.  The first time a dataset is accessed, it is
@@ -224,7 +226,7 @@ class MatrixDataLoader(object):

        # matrix_data_type is an enum value of type MatrixDataType
        self.matrix_data_type = matrix_data_type
-        # matrix_type is a DataAdaptor type, which corresonds to the matrix_data_type
+        # matrix_type is a DataAdaptor type, which corresponds to the matrix_data_type
        self.matrix_type = None

        if matrix_data_type is None:
@@ -234,11 +236,11 @@ class MatrixDataLoader(object):
            raise DatasetAccessError("Dataset does not have an allowed type.")

        if self.matrix_data_type == MatrixDataType.H5AD:
-            from server.data_anndata.anndata_adaptor import AnndataAdaptor
+            from backend.czi_hosted.data_anndata.anndata_adaptor import AnndataAdaptor

            self.matrix_type = AnndataAdaptor
        elif self.matrix_data_type == MatrixDataType.CXG:
-            from server.data_cxg.cxg_adaptor import CxgAdaptor
+            from backend.czi_hosted.data_cxg.cxg_adaptor import CxgAdaptor

            self.matrix_type = CxgAdaptor

@@ -8,15 +8,15 @@ import pandas as pd
 import tiledb
 from server_timing import Timing as ServerTiming

-import server.compute.diffexp_cxg as diffexp_cxg
-from server.common.constants import Axis
-from server.common.errors import DatasetAccessError, ConfigurationError
-from server.common.immutable_kvcache import ImmutableKVCache
-from server.common.utils.type_conversion_utils import get_schema_type_hint_from_dtype
-from server.common.utils.utils import path_join
-from server.data_common.data_adaptor import DataAdaptor
-from server.data_common.fbs.matrix import encode_matrix_fbs
-from server.data_cxg.cxg_util import pack_selector_from_mask
+from backend.common.constants import Axis
+from backend.common.errors import DatasetAccessError, ConfigurationError
+from backend.czi_hosted.common.immutable_kvcache import ImmutableKVCache
+from backend.common.utils.type_conversion_utils import get_schema_type_hint_from_dtype
+from backend.common.utils.utils import path_join
+from backend.czi_hosted.compute import diffexp_cxg
+from backend.czi_hosted.data_common.data_adaptor import DataAdaptor
+from backend.common.fbs.matrix import encode_matrix_fbs
+from backend.czi_hosted.data_cxg.cxg_util import pack_selector_from_mask


 class CxgAdaptor(DataAdaptor):
@@ -3,7 +3,7 @@ Drops and recreates all tables for local testing according to cellxgene_orm.py
 """
 from sqlalchemy import create_engine

-from server.db.cellxgene_orm import Base
+from backend.czi_hosted.db.cellxgene_orm import Base


 def create_db(database_uri: str = "postgresql://postgres:test_pw@localhost:5432"):
@@ -4,7 +4,7 @@ import uuid
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker

-from server.db.cellxgene_orm import Base, CellxGeneDataset, CellxGeneUser
+from backend.czi_hosted.db.cellxgene_orm import Base, CellxGeneDataset, CellxGeneUser


 class DbUtils:
--- a/Show More
+++ b/Show More