tech: test tests using latest installations

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.3.0
+current_version = 1.1.2
 commit = True
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?:-(?P<prerel>rc)\.(?P<prerelversion>\d+))?
 serialize = 

.github/workflows/compatibility_tests.yml

@@ -2,7 +2,7 @@ name: Compatibility Tests
 
 on:
   schedule:
-    - cron: "0 8 7 * 2"
+    - cron: '0 8 7 * 2'
   push:
     branches:
       - main
@@ -14,9 +14,9 @@ jobs:
   docker-build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
       - name: Build docker image
@@ -28,23 +28,35 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest, macos-13]
+        # note: The `macos-latest` is latest Catalina version, and not Big Sur. So we explicitly ask for Big Sur (`macos-11`)
-        python-version: ["3.10", "3.11", "3.12"]
+        os: [ubuntu-latest, macos-latest, macos-11]
+        python-version: [3.6, 3.7, 3.8, 3.9]
         cellxgene_build: [main, latest]
+        exclude:
+          # 3.6 no longer avail on Big Sur (`macos-11`)
+          - os: macos-11
+            python-version: 3.6
+          # no pypi build exists for macos+py3.9 and source install fails to
+          # install `tables` py pkg (a `scanpy` dependency), so we test py3.9
+          # only on ubuntu
+          - os: macos-11
+            python-version: 3.9
+          - os: macos-latest
+            python-version: 3.9
         # add anndata pinned version test for subset of matrix configurations,
         # in order to reduce matrix cross-product explosion
         include:
-          - python-version: 3.12
+          - python-version: 3.8
             cellxgene_build: latest
             # TODO: dynamically use the literal version in requirements.txt,
             # to avoid having to update this in manually in the future
             # TODO: Do not bother running this if anndata latest version
             # matches this pinned version, to avoid a redundant test
-            anndata_version: "==0.10.9"
+            anndata_version: '==0.7.6'
     steps:
-      - uses: actions/checkout@v4
+    - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - name: Cache env vars
@@ -54,14 +66,14 @@ jobs:
       run: echo "BREW_CACHE=`brew --cache`" >> $GITHUB_ENV
     # FIXME: Only working for Linux
     - name: Python cache
-        uses: actions/cache@v4
+      uses: actions/cache@v1
       with:
         path: ${{ env.PIP_CACHE }}
         key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
         restore-keys: |
           ${{ runner.os }}-pip-
     - name: Node cache
-        uses: actions/cache@v4
+      uses: actions/cache@v1
       with:
         path: ~/.npm
         key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
@@ -69,7 +81,7 @@ jobs:
           ${{ runner.os }}-node-
     - name: Brew cache (MacOS)
       if: startsWith(matrix.os, 'macos')
-        uses: actions/cache@v4
+      uses: actions/cache@v1
       with:
         path: ${{ env.BREW_CACHE }}
         key: ${{ runner.os }}-brew-
@@ -95,7 +107,6 @@ jobs:
         # keep same pip pkg versions as in the cxg release
         sed -i'' -e 's/-r requirements.txt//' server/requirements-dev.txt
         pip install -r server/requirements-dev.txt
-          pip install --force-reinstall numpy==2.0.1 numba>=0.60.0 pandas flatbuffers==2.0.7
     - name: Install anndata version per matrix variable
       run: pip install anndata${{ matrix.anndata_version }}
     - name: Install node

.github/workflows/push_tests.yml

@@ -14,15 +14,15 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v2
       - run: |
           git fetch --depth=1 origin +${{github.base_ref}}
-      - name: Set up Python 3.12
+      - name: Set up Python 3.7
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v4
         with:
-          python-version: 3.12
+          python-version: 3.7
       - name: Node cache
-        uses: actions/cache@v4
+        uses: actions/cache@v1
         with:
           path: ~/.npm
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
@@ -45,22 +45,22 @@ jobs:
   unit-test:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v2
-      - name: Set up Python 3.12 (pyenv) # pyenv needed for mlflow in cli annotate tests
+      - name: Set up Python 3.7 (pyenv)  # pyenv needed for mlflow in cli annotate tests
         uses: gabrielfalcao/pyenv-action@v9
         with:
-          default: 3.12
+          default: 3.7
           command: pip install -U pip  # upgrade pip after installing python
       - run: pip install virtualenv  # virtualenv needed for mlflow in cli annotate tests
       - name: Python cache
-        uses: actions/cache@v4
+        uses: actions/cache@v1
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
           restore-keys: |
             ${{ runner.os }}-pip-
       - name: Node cache
-        uses: actions/cache@v4
+        uses: actions/cache@v1
         with:
           path: ~/.npm
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
@@ -69,51 +69,40 @@ jobs:
       - name: Install dependencies
         run: make pydist install-dist dev-env-server
       - name: Unit tests
-        run: make unit-test-server unit-test-client
+        run: |
-      - name: Generate server coverage XML
+          make unit-test-server unit-test-client
-        run: coverage xml -o server/coverage.xml
+          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k server -cF server,python,unitTest
-      - name: Upload server coverage
+          cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest
-        uses: codecov/codecov-action@v5
-        with:
-          flags: server,python,unitTest
-          files: ./server/coverage.xml
-          fail_ci_if_error: false
-      - name: Upload client coverage
-        uses: codecov/codecov-action@v5
-        with:
-          flags: frontend,javascript,unitTest
-          files: ./client/coverage/lcov.info
-          fail_ci_if_error: false
 
   smoke-tests:
     runs-on: macos-latest
     timeout-minutes: 20
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v2
-      - name: Set up Python 3.12
+      - name: Set up Python 3.7
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v4
         with:
-          python-version: 3.12
+          python-version: 3.7
       - name: Python cache
-        uses: actions/cache@v4
+        uses: actions/cache@v1
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
           restore-keys: |
             ${{ runner.os }}-pip-
       - name: Node cache
-        uses: actions/cache@v4
+        uses: actions/cache@v1
         with:
           path: ~/.npm
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
           restore-keys: |
             ${{ runner.os }}-node-
       - name: Install dependencies
-        run: |
+        run: make pydist install-dist
-          pip install setuptools
-          make pydist install-dist
       - name: Smoke tests (without annotations feature)
-        run: cd client && make smoke-test
+        run: |
+          cd client && make smoke-test
+          ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTest
 
   # TODO: reinstate: https://github.com/chanzuckerberg/cellxgene/issues/2544
   # smoke-tests-annotations:
@@ -121,10 +110,10 @@ jobs:
   #   timeout-minutes: 20
   #   steps:
   #     - uses: actions/checkout@v2
-  #     - name: Set up Python 3.9
+  #     - name: Set up Python 3.7
   #       uses: actions/setup-python@v4
   #       with:
-  #         python-version: 3.9
+  #         python-version: 3.7
   #     - name: Python cache
   #       uses: actions/cache@v1
   #       with:

LICENSE.txt

@@ -1,6 +1,6 @@
-# The MIT License (MIT)
+The MIT License (MIT)
 
-Copyright (c) 2017-2026 Chan Zuckerberg Initiative
+Copyright (c) 2017-2023 Chan Zuckerberg Initiative
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

README.md

@@ -27,7 +27,7 @@ Whether you need to visualize one thousand cells or one million, CELLxGENE Annot
 
 ### Quick start
 
-To install CELLxGENE Annotate you need Python 3.10+. We recommend [installing Annotate into a conda or virtual environment.](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/install.md)
+To install CELLxGENE Annotate you need Python 3.6+. We recommend [installing Annotate into a conda or virtual environment.](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/install.md)
 
 Install the package.
 
@@ -58,7 +58,7 @@ Please [file an issue](https://github.com/chanzuckerberg/cellxgene/issues/new/ch
 ### Finding help
 
 We'd love to hear from you!
-For questions, suggestions, or accolades, join the `#cellxgene-users` channel on the [CZI Science Community Slack](https://czi.co/science-slack) and say "hi!".
+For questions, suggestions, or accolades, [join the `#cellxgene-users` channel on the CZI Science Slack](https://join-cellxgene-users.herokuapp.com/) and say "hi!".
 
 For any errors, [report bugs on Github](https://github.com/chanzuckerberg/cellxgene/issues).
 
@@ -76,19 +76,13 @@ This project was started with the sole goal of empowering the scientific communi
 As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from 
 this project. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT).
 
+
 Before extending CELLxGENE Annotate, we encourage you to reach out to us with ideas or questions. It might be possible that an 
 extension could be directly contributed, which would make it available for a wider audience, or that it's on our 
 [roadmap](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/roadmap.md) and under active development. 
 
 See the [CELLxGENE extensions](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/community-extensions.md) section of our documentation for examples of community use and CELLxGENE extensions. 
 
-### Trademarks
-
-CZ CELLXGENE, CZ CELLXGENE DISCOVER, and CZ CELLXGENE ANNOTATE are trademarks of the Chan Zuckerberg Initiative. All rights reserved.
-
-Use, reuse, modification, and re-distribution of the source code in this repository is subject to the terms of the applicable open source [license](LICENSE.txt). However, that license does not grant permission to use the trademarks without separate, express permission from the Chan Zuckerberg Initiative.
-
-
 ### Security
 
 If you believe you have found a security issue, we would appreciate notification. Please send email to <security@chanzuckerberg.com>.

SECURITY.md

@@ -1,3 +0,0 @@
-# Reporting Security Issues
-
-If you believe you have found a security issue, please responsibly disclose by contacting us at [security@chanzuckerberg.com](mailto:security@chanzuckerberg.com).

client/.nvmrc

@@ -1 +0,0 @@
-18.17.0

client/__tests__/e2e/puppeteer.setup.js

@@ -13,7 +13,7 @@ import * as ENV_DEFAULT from "../../../environment.default.json";
 // a test can take more time to finish, so we don't want
 // jest to shut off the test too soon
 jest.setTimeout(2 * 60 * 1000);
-setDefaultOptions({ timeout: 60 * 1000 });
+setDefaultOptions({ timeout: 20 * 1000 });
 
 jest.retryTimes(ENV_DEFAULT.RETRY_ATTEMPTS);
 

client/configuration/babel/babel.dev.js

@@ -16,8 +16,8 @@ module.exports = {
     "@babel/plugin-proposal-function-bind",
     ["@babel/plugin-proposal-decorators", { legacy: true }],
     ["@babel/plugin-proposal-class-properties", { loose: true }],
-    ["@babel/plugin-transform-private-methods", { loose: true }],
+    ["@babel/plugin-proposal-private-methods", { loose: true }],
-    ["@babel/plugin-transform-private-property-in-object", { loose: true }],
+    ["@babel/plugin-proposal-private-property-in-object", { loose: true }],
     "@babel/plugin-proposal-export-namespace-from",
     "@babel/plugin-proposal-optional-chaining",
     "@babel/plugin-proposal-nullish-coalescing-operator",

client/configuration/babel/babel.prod.js

@@ -15,8 +15,8 @@ module.exports = {
     "@babel/plugin-proposal-function-bind",
     ["@babel/plugin-proposal-decorators", { legacy: true }],
     ["@babel/plugin-proposal-class-properties", { loose: true }],
-    ["@babel/plugin-transform-private-methods", { loose: true }],
+    ["@babel/plugin-proposal-private-methods", { loose: true }],
-    ["@babel/plugin-transform-private-property-in-object", { loose: true }],
+    ["@babel/plugin-proposal-private-property-in-object", { loose: true }],
     "@babel/plugin-proposal-export-namespace-from",
     "@babel/plugin-transform-react-constant-elements",
     "@babel/plugin-transform-runtime",

client/configuration/webpack/webpack.config.shared.js

@@ -1,7 +1,7 @@
 const path = require("path");
 const fs = require("fs");
 const MiniCssExtractPlugin = require("mini-css-extract-plugin");
-const ObsoleteWebpackPlugin = require("webpack-obsolete-plugin");
+const ObsoleteWebpackPlugin = require("obsolete-webpack-plugin");
 
 const src = path.resolve("src");
 const nodeModules = path.resolve("node_modules");

client/jest-puppeteer.config.js

@@ -14,7 +14,6 @@ const DEFAULT_LAUNCH_CONFIG = {
   headless: !isHeadful,
   args: ["--ignore-certificate-errors", "--ignore-ssl-errors"],
   ignoreHTTPSErrors: true,
-  timeout: 90000,
   defaultViewport: {
     width: 1280,
     height: 960,

client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cellxgene",
-  "version": "1.3.0",
+  "version": "1.1.2",
   "license": "MIT",
   "description": "cellxgene is a web application for the interactive exploration of single cell sequence data.",
   "repository": "https://github.com/chanzuckerberg/cellxgene",
@@ -18,8 +18,7 @@
   },
   "engineStrict": true,
   "engines": {
-    "npm": ">=9.6.7",
+    "npm": ">=3.0.0"
-    "node": "^18.17.0"
   },
   "eslintConfig": {
     "extends": "./configuration/eslint/eslint.js"
@@ -78,17 +77,16 @@
     "whatwg-fetch": "^3.2.0"
   },
   "devDependencies": {
-    "@babel/core": "^7.25.2",
+    "@babel/core": "^7.13.16",
     "@babel/plugin-proposal-class-properties": "^7.10.4",
     "@babel/plugin-proposal-decorators": "^7.13.15",
     "@babel/plugin-proposal-export-namespace-from": "^7.10.4",
     "@babel/plugin-proposal-function-bind": "^7.10.5",
     "@babel/plugin-proposal-nullish-coalescing-operator": "^7.10.4",
     "@babel/plugin-proposal-optional-chaining": "^7.10.4",
-    "@babel/plugin-transform-private-property-in-object": "^7.22.11",
     "@babel/plugin-transform-react-constant-elements": "^7.13.13",
     "@babel/plugin-transform-runtime": "^7.13.15",
-    "@babel/preset-env": "^7.22.20",
+    "@babel/preset-env": "^7.13.15",
     "@babel/preset-react": "^7.13.13",
     "@babel/register": "^7.13.16",
     "@babel/runtime": "^7.13.16",
@@ -101,12 +99,13 @@
     "cheerio": "^1.0.0-rc.6",
     "clean-css": "^5.1.2",
     "clean-webpack-plugin": "^4.0.0-alpha.0",
+    "codecov": "^3.7.1",
     "css-loader": "^5.2.4",
     "css-minimizer-webpack-plugin": "^4.0.0",
     "eslint": "^7.24.0",
     "eslint-config-airbnb": "^18.2.0",
     "eslint-config-prettier": "^8.2.0",
-    "eslint-plugin-compat": "^4.2.0",
+    "eslint-plugin-compat": "^3.8.0",
     "eslint-plugin-eslint-comments": "^3.2.0",
     "eslint-plugin-filenames": "^1.3.2",
     "eslint-plugin-import": "^2.24.2",
@@ -123,7 +122,7 @@
     "jest-circus": "^27.0.6",
     "jest-environment-puppeteer": "^5.0.1",
     "jest-fetch-mock": "^3.0.3",
-    "jest-puppeteer": "^6.2.0",
+    "jest-puppeteer": "^5.0.1",
     "json-loader": "^0.5.7",
     "lint-staged": "^10.2.11",
     "lodash": "^4.17.21",
@@ -133,16 +132,16 @@
     "lodash.map": "^4.6.0",
     "lodash.zip": "^4.2.0",
     "mini-css-extract-plugin": "^1.5.0",
+    "obsolete-webpack-plugin": "^0.5.6",
     "prettier": "^2.0.5",
-    "puppeteer": "^10.4.0",
+    "puppeteer": "^8.0.0",
     "rimraf": "^3.0.2",
     "serve-favicon": "^2.5.0",
     "terser-webpack-plugin": "^5.1.1",
-    "webpack": "^5.94.0",
+    "webpack": "^5.34.0",
     "webpack-cli": "^4.6.0",
     "webpack-dev-middleware": "^4.1.0",
-    "webpack-merge": "^5.0.9",
+    "webpack-merge": "^5.0.9"
-    "webpack-obsolete-plugin": "^1.0.5"
   },
   "jest": {
     "testMatch": [
@@ -176,13 +175,13 @@
             }
           ],
           [
-            "@babel/plugin-transform-private-methods",
+            "@babel/plugin-proposal-private-methods",
             {
               "loose": true
             }
           ],
           [
-            "@babel/plugin-transform-private-property-in-object",
+            "@babel/plugin-proposal-private-property-in-object",
             {
               "loose": true
             }

client/src/annoMatrix/whereCache.js

@@ -169,6 +169,5 @@ function __whereCacheMerge(dst, src) {
 }
 
 export function _whereCacheMerge(...caches) {
-  // eslint-disable-next-line compat/compat -- not using web APIs
   return caches.reduce(__whereCacheMerge, {});
 }

client/src/components/graph/overlays/centroidLabels.js

@@ -150,7 +150,7 @@ class CentroidLabels extends PureComponent {
                   dilatedValue={dilatedValue}
                   coords={coords}
                   inverseTransform={inverseTransform}
-                  opacity={selected ? 1 : deselectOpacity}
+                  opactity={selected ? 1 : deselectOpacity}
                   colorAccessor={colorAccessor}
                   displayLabel={displayLabel}
                   onMouseEnter={this.handleMouseEnter}
@@ -205,7 +205,7 @@ const Label = ({
           fontWeight,
           fill: "black",
           userSelect: "none",
-          opacity,
+          opacity: { opacity },
         }}
         onMouseEnter={(e) => onMouseEnter(e, colorAccessor, label)}
         onMouseOut={(e) => onMouseOut(e, colorAccessor, label)}

client/src/components/leftSidebar/infoMenu.js

@@ -16,7 +16,7 @@ const InformationMenu = React.memo((props) => {
             rel="noopener"
           />
           <MenuItem
-            href="https://czi.co/science-slack"
+            href="https://join-cellxgene-users.herokuapp.com/"
             target="_blank"
             icon="chat"
             text="Chat"

dev_docs/developer_guidelines.md

@@ -3,7 +3,7 @@
 ## Requirements
 
 - npm
-- Python 3.10+
+- Python 3.6+
 - Chrome
 
 [See dev section of README](../README.md)
@@ -148,6 +148,6 @@ If you would like to run the smoke tests against a hot-reloaded version of the c
 
 ### Tips
 
-- You can also install/launch the server side code from npm scrips (requires python3.10 with virtualenv) with the `scripts/backend_dev` script.
+- You can also install/launch the server side code from npm scrips (requires python3.6 with virtualenv) with the `scripts/backend_dev` script.
 
 - Check out [e2e Tests](e2e_tests.md) for more details

dev_docs/developer_scripts.md

@@ -11,10 +11,9 @@ $PROJECT_ROOT`.
 ### Build
 
 **Usage:** from the `$PROJECT_ROOT` directory run:
-
+* `make build` builds whole app client and server
-- `make build` builds whole app client and server
+* `make build-client` runs webpack build
-- `make build-client` runs webpack build
+* `make build-for-server-dev` builds client and copies output directly into
-- `make build-for-server-dev` builds client and copies output directly into
   source tree (only for server devlopment)
 
 ### Clean
@@ -22,19 +21,17 @@ $PROJECT_ROOT`.
 Deletes generated files.
 
 **Usage:** from the `$PROJECT_ROOT` directory run:
-
+* `make clean` cleans everything including node modules (means build with take
-- `make clean` cleans everything including node modules (means build with take
   a while
-- `make clean-lite` cleans built directories
+* `make clean-lite` cleans built directories
-- `make clean-server` cleans source tree
+* `make clean-server` cleans source tree
 
 ### Distribution
 
 Creates distribution for python module to upload to pypi.
 
 **Usage:** from the `$PROJECT_ROOT` directory run:
-
+* `make pydist` builds code and then builds sdist
-- `make pydist` builds code and then builds sdist
 
 ### Release
 
@@ -45,18 +42,16 @@ See `release_process.md`.
 Installs requirements files.
 
 **Usage:** from the `$PROJECT_ROOT` directory run:
-
+* `make dev-env` installs requirements and requirments-dev (for building code)
-- `make dev-env` installs requirements and requirments-dev (for building code)
 
 ### Installing cellxgene packages
 
 **Usage:** from the `$PROJECT_ROOT` directory:
-
+* `install-dev` - installs from local source tree
-- `install-dev` - installs from local source tree
+* `install-release-test` - installs from test pypi
-- `install-release-test` - installs from test pypi
+* `install-release` - installs from pypi
-- `install-release` - installs from pypi
+* `install-dist` - installs from local dist folder
-- `install-dist` - installs from local dist folder
+* `uninstall` - uninstalls cellxgene
-- `uninstall` - uninstalls cellxgene
 
 ## Client-level scripts
 
@@ -67,9 +62,8 @@ Installs requirements files.
 **About** Serve the current client javascript independently from the `server` code.
 
 **Requires**
-
+* The server to be running. Best way to do this is with [backend_dev](#backend_dev).
-- The server to be running. Best way to do this is with [backend_dev](#backend_dev).
+* `make ci` to install the necessary node modules
-- `make ci` to install the necessary node modules
 
 **Usage:** from the `$PROJECT_ROOT/client` directory run `make start-frontend`
 
@@ -81,24 +75,23 @@ the FE developer gets the current version of the backend with a single command
 and no knowledge of python necessary. It creates and activates a virtual
 environment and installs cellxgene from the current branch.
 
-**Requires** `Python3.10+`, `virtual-env`, `pip`
+**Requires** `Python3.6+`, `virtual-env`, `pip`
 
 **Usage:** from the `$PROJECT_ROOT` directory run `./scripts/backend_dev`
 
 **Options:**
-
+* In parallel, you can then launch the node development server to serve the
-- In parallel, you can then launch the node development server to serve the
   current state of the FE with [`start-frontend`](#start-frontend), usually in
   a different terminal tab.
-- You can also select a specific dataset using `DATASET=<dataset path> ./scripts/backend_dev`.
+* You can also select a specific dataset using `DATASET=<dataset path> ./scripts/backend_dev`.
-- You can also use `CXG_OPTIONS` to pass options to the `cellxgene launch`
+* You can also use `CXG_OPTIONS` to pass options to the `cellxgene launch`
   command, as in `CXG_OPTIONS='--disable-annotations' ./scripts/backend_dev`.
 
 **Breakdown**
 
 | command                                  | purpose                                                    |
 | ---------------------------------------- | ---------------------------------------------------------- |
-| python3.12 -m venv cellxgene             | creates cellxgene virtual environment                      |
+| python3.6 -m venv cellxgene              | creates cellxgene virtual environment                      |
 | source cellxgene/bin/activate            | activates virtual environment                              |
 | yes \| pip uninstall cellxgene \|\| true | uninstalls cellxgene (if installed)                        |
 | pip install -e .                         | installs current local version of cellxgene                |
@@ -109,15 +102,14 @@ environment and installs cellxgene from the current branch.
 Methods used to test the client javascript code
 
 **Usage:** from the `$PROJECT_ROOT/client` directory run:
-
+* `make unit-test` Runs all unit tests. It excludes any tests in the e2e
-- `make unit-test` Runs all unit tests. It excludes any tests in the e2e
   folder. This is used by travis to run unit tests.
-- `make smoke-test` Starts backend development server and runs end to end
+* `make smoke-test` Starts backend development server and runs end to end
   tests. This is what travis runs. It depends on the `e2e` and the
   `backend-dev` targets. One starts the server, the other runs the tests. If
   developing a front-end feature and just checking if tests pass, this is
   probabaly the one you want to run.
-- `npm run e2e` Runs backend tests without starting the server. You will need to
+* `npm run e2e` Runs backend tests without starting the server. You will need to
   start the rest api separately with the pbmc3k.h5ad file. Note you can use
   the `JEST_ENV` environment variable to change how JEST runs in the browser.
   The test runs against `localhost:3000` by default. You can use the

server/__init__.py

@@ -2,7 +2,7 @@ import logging
 import sys
 from server.common.utils.utils import import_plugins
 
-__version__ = "1.3.0"
+__version__ = "1.1.2"
 display_version = "cellxgene v" + __version__
 
 try:

server/cli/annotate.py

@@ -26,7 +26,9 @@ def annotate_args(func):
 
 
 @sort_options
-@click.command(options_metavar="<options>")
+@click.command(
+    options_metavar="<options>"
+)
 @click.argument(
     "input_h5ad_file",
     type=click.Path(exists=True, dir_okay=False, readable=True),

server/cli/prepare.py

@@ -128,12 +128,12 @@ def prepare(
             raise click.FileError(data, hint="not a valid file or path")
 
         if not set_obs_names == "":
-            if set_obs_names not in list(adata.obs.keys()):
+            if set_obs_names not in adata.obs_keys():
-                raise click.UsageError(f"obs {set_obs_names} not found, options are: {list(adata.obs.keys())}")
+                raise click.UsageError(f"obs {set_obs_names} not found, options are: {adata.obs_keys()}")
             adata.obs_names = adata.obs[set_obs_names]
         if not set_var_names == "":
-            if set_var_names not in list(adata.var.keys()):
+            if set_var_names not in adata.var_keys():
-                raise click.UsageError(f"var {set_var_names} not found, options are: {list(adata.var.keys())}")
+                raise click.UsageError(f"var {set_var_names} not found, options are: {adata.var_keys()}")
             adata.var_names = adata.var[set_var_names]
         if make_obs_names_unique:
             adata.obs.index = make_index_unique(adata.obs.index)

server/common/annotations/local_file_csv.py

@@ -145,7 +145,7 @@ class AnnotationsLocalFile(Annotations):
     def write_gene_sets(self, gene_sets, tid, data_adaptor):
         self.check_gene_sets_save_enabled()  # raises
 
-        if type(tid) is not int or tid < 0:
+        if type(tid) != int or tid < 0:
             raise ValueError("tid must be a positive integer")
 
         # may raise
@@ -175,7 +175,7 @@ class AnnotationsLocalFile(Annotations):
 
             # update the cache
             self.last_geneset_fname = fname
-            self.last_geneset = gene_sets if isinstance(gene_sets, dict) else {g["geneset_name"]: g for g in gene_sets}
+            self.last_geneset = gene_sets if type(gene_sets) == dict else {g["geneset_name"]: g for g in gene_sets}
 
     def _get_userdata_idhash(self, data_adaptor):
         """

server/common/colors.py

@@ -228,6 +228,6 @@ def convert_anndata_category_colors_to_cxg_category_colors(data):
 
         # create the cellxgene color entry for this category
         cxg_colors[category_name] = dict(
-            zip(data.obs[category_name].astype('category').cat.categories, [convert_color_to_hex_format(c) for c in data.uns[uns_key]])
+            zip(data.obs[category_name].cat.categories, [convert_color_to_hex_format(c) for c in data.uns[uns_key]])
         )
     return cxg_colors

server/common/compute/estimate_distribution.py

@@ -97,7 +97,7 @@ def estimate_approximate_distribution(X) -> XApproximateDistribution:
     if Xdata.size > CHUNKSIZE:
         min_val = max_val = Xdata[0]
         with concurrent.futures.ThreadPoolExecutor() as tp:
-            for _min, _max in tp.map(min_max, [Xdata[i : i + CHUNKSIZE] for i in range(0, Xdata.size, CHUNKSIZE)]):
+            for (_min, _max) in tp.map(min_max, [Xdata[i : i + CHUNKSIZE] for i in range(0, Xdata.size, CHUNKSIZE)]):
                 min_val = min(_min, min_val)
                 max_val = max(_max, max_val)
 

server/common/config/app_config.py

@@ -19,6 +19,7 @@ class AppConfig(object):
     """
 
     def __init__(self):
+
         # the default configuration (see default_config.py)
         # TODO @madison -- if we always read from the default config (hard coded path) can we set those values as
         #  defaults within the config class?

server/common/config/base_config.py

@@ -50,7 +50,7 @@ class BaseConfig(object):
                     f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}"
                 )
         else:
-            if type(val) is not vtype:
+            if type(val) != vtype:
                 raise ConfigurationError(
                     f"Invalid type for attribute: {attrname}, "
                     f"expected type {vtype.__name__}, got {type(val).__name__}"
@@ -70,7 +70,7 @@ class BaseConfig(object):
             if not hasattr(self, key):
                 raise ConfigurationError(f"unknown config parameter {key}.")
             try:
-                if type(value) is tuple:
+                if type(value) == tuple:
                     # convert tuple values to list values
                     value = list(value)
                 setattr(self, key, value)

server/common/config/dataset_config.py

@@ -176,7 +176,7 @@ class DatasetConfig(BaseConfig):
         self.validate_correct_type_of_configuration_attribute("diffexp__top_n", int)
 
         data_adaptor = self.get_data_adaptor()
-        if self.diffexp__enable and data_adaptor.parameters.get("diffexp-may-be-slow", False):
+        if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False):
             context["messagefn"](
                 "CAUTION: due to the size of your dataset, " "running differential expression may take longer or fail."
             )

server/common/config/external_config.py

@@ -29,7 +29,7 @@ class ExternalConfig(BaseConfig):
             if name is None:
                 raise ConfigurationError("environment: 'name' is missing")
             required = envdict.get("required", False)
-            if type(required) is not bool:
+            if type(required) != bool:
                 raise ConfigurationError("environment: 'required' must be a bool")
             path = envdict.get("path")
             if path is None:

server/common/corpora.py

@@ -22,7 +22,7 @@ def corpora_get_versions_from_anndata(adata):
     """
 
     # per Corpora AnnData spec, this is a corpora file if the following is true
-    if "version" not in list(adata.uns.keys()):
+    if "version" not in adata.uns_keys():
         return None
     version = adata.uns["version"]
     if not isinstance(version, collections.abc.Mapping) or "corpora_schema_version" not in version:

server/common/genesets.py

@@ -187,7 +187,7 @@ def validate_gene_sets(genesets, var_names, context=None):
     # 1. check gene set character set and format
     illegal_name = re.compile(r"^\s|  |[\u0000-\u001F\u007F-\uFFFF]|\s$")
     for name in geneset_names:
-        if type(name) is not str or len(name) == 0:
+        if type(name) != str or len(name) == 0:
             raise KeyError("Gene set names must be non-null string.")
         if illegal_name.search(name):
             messagefn(

server/common/rest.py

@@ -6,7 +6,7 @@ import zlib
 import json
 
 from flask import make_response, jsonify, current_app, abort
-from urllib.parse import unquote
+from werkzeug.urls import url_unquote
 
 from server.common.config.client_config import get_client_config
 from server.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg
@@ -64,22 +64,22 @@ def _query_parameter_to_filter(args):
             axis, name = key.split(":")
             if axis not in ("obs", "var"):
                 raise FilterError("unknown filter axis")
-            name = unquote(name)
+            name = url_unquote(name)
             current = filters[axis].setdefault(name, {"name": name})
 
             val_split = value.split(",")
             if len(val_split) == 1:
                 if "min" in current or "max" in current:
                     raise FilterError("do not mix range and value filters")
-                value = unquote(value)
+                value = url_unquote(value)
                 values = current.setdefault("values", [])
                 values.append(value)
 
             elif len(val_split) == 2:
                 if len(current) > 1:
                     raise FilterError("duplicate range specification")
-                min = unquote(val_split[0])
+                min = url_unquote(val_split[0])
-                max = unquote(val_split[1])
+                max = url_unquote(val_split[1])
                 if min != "*":
                     current["min"] = float(min)
                 if max != "*":
@@ -379,7 +379,7 @@ def summarize_var_helper(request, data_adaptor, key, raw_query):
             HTTPStatus.OK,
             {"Content-Type": "application/octet-stream"},
         )
-    except ValueError as e:
+    except (ValueError) as e:
         return abort(HTTPStatus.NOT_FOUND, description=str(e))
     except (UnsupportedSummaryMethod, FilterError) as e:
         return abort(HTTPStatus.BAD_REQUEST, description=str(e))

server/common/utils/type_conversion_utils.py

@@ -140,6 +140,7 @@ def _get_type_info(array: Union[np.ndarray, pd.Series, pd.Index]) -> Tuple[np.dt
 
     if dtype.kind in ["i", "u"] and _can_cast_array_values_to_int32(array):
         return (np.int32, {"type": "int32"})
+
     if dtype.kind == "f":
         _float64_warning(array.dtype)
         return (np.float32, {"type": "float32"})

server/common/utils/utils.py

@@ -8,7 +8,7 @@ import socket
 from urllib.parse import urlsplit, urljoin
 
 import numpy as np
-import json
+from flask import json
 
 from server.common.errors import ConfigurationError
 
@@ -100,7 +100,6 @@ def custom_format_warning(msg, *args, **kwargs):
 def jsonify_strict(data):
     return StrictJSONEncoder().encode(data)
 
-
 def import_plugins(plugin_module):
     """
     Load optional plugin modules from server.common.plugins

server/data_anndata/anndata_adaptor.py

@@ -1,5 +1,4 @@
 import warnings
-import importlib.metadata
 
 import anndata
 import numpy as np
@@ -17,7 +16,7 @@ from server.common.utils.type_conversion_utils import get_schema_type_hint_of_ar
 from server.data_common.data_adaptor import DataAdaptor
 from server.common.fbs.matrix import encode_matrix_fbs
 
-anndata_version = version.parse(str(importlib.metadata.version('anndata'))).release
+anndata_version = version.parse(str(anndata.__version__)).release
 
 
 def anndata_version_is_pre_070():
@@ -64,7 +63,7 @@ class AnndataAdaptor(DataAdaptor):
         return "cellxgene anndata adaptor version"
 
     def get_library_versions(self):
-        return dict(anndata=str(importlib.metadata.version('anndata')))
+        return dict(anndata=str(anndata.__version__))
 
     @staticmethod
     def _create_unique_column_name(df, col_name_prefix):
@@ -93,7 +92,7 @@ class AnndataAdaptor(DataAdaptor):
         """
         self.original_obs_index = self.data.obs.index
 
-        for ax_name, var_name in ((Axis.OBS, "obs"), (Axis.VAR, "var")):
+        for (ax_name, var_name) in ((Axis.OBS, "obs"), (Axis.VAR, "var")):
             config_name = f"single_dataset__{var_name}_names"
             parameter_name = f"{var_name}_names"
             name = getattr(self.server_config, config_name)
@@ -174,24 +173,12 @@ class AnndataAdaptor(DataAdaptor):
             )
         except MemoryError:
             raise DatasetAccessError("Out of memory - file is too large for available memory.")
-        except Exception as e:
+        except Exception:
             import traceback
-            error_msg = str(e)
-
-            # IMPROVEMENT: Broadly catch ANY version incompatibility
-            if "No read method registered" in error_msg and "IOSpec" in error_msg:
-                message = (
-                    "Error loading file: This H5AD file uses a newer internal format that "
-                    "your version of 'anndata' cannot read.\n"
-                    f"The specific error was: {error_msg}\n"
-                    "Please upgrade anndata in your environment (pip install --upgrade anndata)."
-                )
-            else:
             message = (
                 "File not found or is inaccessible. File must be an .h5ad object. "
                 "Please check your input and try again."
                 )
-
             if self.server_config.app__verbose:
                 message += f"\n{traceback.format_exc()}"
             raise DatasetAccessError(message)
@@ -223,7 +210,7 @@ class AnndataAdaptor(DataAdaptor):
         # heuristic
         n_values = self.data.shape[0] * self.data.shape[1]
         if (n_values > 1e8 and self.server_config.adaptor__anndata_adaptor__backed is True) or (n_values > 5e8):
-            self.parameters.update({"diffexp-may-be-slow": True})
+            self.parameters.update({"diffexp_may_be_slow": True})
 
     def _is_valid_layout(self, arr):
         """return True if this layout data is a valid array for front-end presentation:
@@ -231,7 +218,7 @@ class AnndataAdaptor(DataAdaptor):
         * with shape (n_obs, >= 2)
         * with all values finite or NaN (no +Inf or -Inf)
         """
-        is_valid = type(arr) is np.ndarray and arr.dtype.kind in "fiu"
+        is_valid = type(arr) == np.ndarray and arr.dtype.kind in "fiu"
         is_valid = is_valid and arr.shape[0] == self.data.n_obs and arr.shape[1] >= 2
         is_valid = is_valid and not np.any(np.isinf(arr)) and not np.all(np.isnan(arr))
         return is_valid
@@ -255,10 +242,8 @@ class AnndataAdaptor(DataAdaptor):
             )
         if self.data.X.dtype < np.float32:
             if self.data.isbacked:
-                raise DatasetAccessError(
+                raise DatasetAccessError(f"Data matrix in {self.data.X.dtype} format is not supported in backed mode."
-                    f"Data matrix in {self.data.X.dtype} format is not supported in backed mode."
+                                         " Please reload without --backed, or convert matrix to float32")
-                    " Please reload without --backed, or convert matrix to float32"
-                )
             warnings.warn(
                 f"Anndata data matrix is in unsupported {self.data.X.dtype} format -- will be cast to float32"
             )
@@ -314,11 +299,11 @@ class AnndataAdaptor(DataAdaptor):
         layouts = self.dataset_config.embeddings__names
 
         if layouts is None or len(layouts) == 0:
-            layouts = [key[2:] for key in list(self.data.obsm.keys()) if type(key) is str and key.startswith("X_")]
+            layouts = [key[2:] for key in self.data.obsm_keys() if type(key) == str and key.startswith("X_")]
 
         # remove invalid layouts
         valid_layouts = []
-        obsm_keys = list(self.data.obsm.keys())
+        obsm_keys = self.data.obsm_keys()
         for layout in layouts:
             layout_name = f"X_{layout}"
             if layout_name not in obsm_keys:

server/data_common/data_adaptor.py

@@ -154,7 +154,7 @@ class DataAdaptor(metaclass=ABCMeta):
         parameters.update(self.parameters)
 
     def _index_filter_to_mask(self, filter, count):
-        mask = np.zeros((count,), dtype="bool")
+        mask = np.zeros((count,), dtype=np.bool)
         for i in filter:
             if isinstance(i, list):
                 mask[i[0] : i[1]] = True
@@ -163,7 +163,7 @@ class DataAdaptor(metaclass=ABCMeta):
         return mask
 
     def _axis_filter_to_mask(self, axis, filter, count):
-        mask = np.ones((count,), dtype="bool")
+        mask = np.ones((count,), dtype=np.bool)
         if "index" in filter:
             mask = np.logical_and(mask, self._index_filter_to_mask(filter["index"], count))
         if "annotation_value" in filter:
@@ -172,7 +172,7 @@ class DataAdaptor(metaclass=ABCMeta):
         return mask
 
     def _annotation_filter_to_mask(self, axis, filter, count):
-        mask = np.ones((count,), dtype="bool")
+        mask = np.ones((count,), dtype=np.bool)
         for v in filter:
             name = v["name"]
             if axis == Axis.VAR:

server/requirements-annotate.txt

@@ -1,2 +1,2 @@
-mlflow==2.16.0
+mlflow
 scanpy

server/requirements-dev.txt

@@ -1,10 +1,10 @@
 black
 bumpversion>=0.5
-coverage>=5.0
+codecov>=2.0.15
 parameterized>=0.7.0
 pytest>=3.6.3
 python-jose>=3.2.0
 twine>=1.12.1
-aiohttp>=3.9.1
 -r requirements.txt
 -r requirements-prepare.txt
+-r requirements-annotate.txt

server/requirements.txt

@@ -1,23 +1,24 @@
-anndata>=0.8.0
+# NOTE: If you update 'anndata' min version, also update the 'anndata_version'
+# matrix value in .github/workflows/compatibility_tests.yml
+anndata>=0.7.6 # we need to_memory(), added in 0.7.6
 boto3>=1.12.18
 click>=7.1.2
-Flask>=3.0.0
+Flask>=1.0.2,<2.3.0
 Flask-Compress>=1.4.0
-Flask-Cors>=3.0.9
+Flask-Cors>=3.0.9  # CVE-2020-25032
 Flask-RESTful>=0.3.6
 flask-server-timing>=0.1.2
 flask-talisman>=0.7.0
-flatbuffers==2.0.7
+flatbuffers>=1.11.0,<2.0.0 # cellxgene is not compatible with 2.0.0. Requires migration
 flatten-dict>=0.2.0
-fsspec>0.8.0
+fsspec>=0.4.4,<0.8.0
 gunicorn>=20.0.4
 h5py>=3.0.0
-numba>=0.60.0
+numba>=0.51.2
-numpy==2.0.1
+numpy>=1.17.5,<=1.22
 packaging>=20.0
-pandas>=2.2.2
+pandas>=1.0,!=1.1  # pandas 1.1 breaks tests, https://github.com/pandas-dev/pandas/issues/35446
 PyYAML>=5.4  # CVE-2020-14343
+scipy>=1.4
 requests>=2.22.0
 s3fs==0.4.2
-scipy>=1.4
-setuptools

setup.py

@@ -14,7 +14,7 @@ with open("server/requirements-annotate.txt") as fh:
 
 setup(
     name="cellxgene",
-    version="1.3.0",
+    version="1.1.2",
     packages=find_packages(),
     url="https://github.com/chanzuckerberg/cellxgene",
     license="MIT",
@@ -24,7 +24,7 @@ setup(
     long_description=long_description,
     long_description_content_type="text/markdown",
     install_requires=requirements,
-    python_requires=">=3.10",
+    python_requires=">=3.6",
     include_package_data=True,
     zip_safe=False,
     classifiers=[
@@ -37,9 +37,8 @@ setup(
         "Operating System :: MacOS :: MacOS X",
         "Programming Language :: JavaScript",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.6",
-        "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.7",
-        "Programming Language :: Python :: 3.12",
         "Programming Language :: Python :: 3 :: Only",
         "Topic :: Scientific/Engineering :: Bio-Informatics",
     ],

test/unit/cli/fixtures/__init__.py

@@ -0,0 +1,5 @@
+from .mlflow_model_fixture import FakeModel
+
+
+def _load_pyfunc(data_path):
+    return FakeModel()

test/unit/cli/test_annotate.py

@@ -0,0 +1,130 @@
+import os
+import shutil
+import unittest
+from tempfile import mkstemp, TemporaryDirectory, NamedTemporaryFile
+
+import mlflow
+from click.testing import CliRunner
+
+from server.cli.annotate import annotate
+from test.unit.cli.fixtures.mlflow_model_fixture import FakeModel
+
+
+def write_model(model) -> str:
+    with TemporaryDirectory() as mlflow_model_dir:
+        fixtures_path = os.path.join(os.path.dirname(__file__), "fixtures")
+        mlflow.pyfunc.save_model(mlflow_model_dir, loader_module="fixtures", code_path=[fixtures_path])
+        return shutil.make_archive(mkstemp()[1], "zip", mlflow_model_dir)
+
+
+class TestCliAnnotate(unittest.TestCase):
+    def test__annotate__loads_and_runs(self):
+        """
+        Invokes the `annotate` subcommand of cellxgene CLI, using a CliRunner() programmatic invocation.
+
+        This tests the happy path case:
+        1) Command line options are parsed;
+        2) An MLflow model zip archive can be read in (from local disk), unpacked, and invoked;
+        3) The correct options are passed to the MLflow model.
+        4) The annotate subcommand exits successfully.
+
+        This does not verify model output or predictions (it's a fake MLflow model, after all); it's up to the real model
+        to output its predictions as it wants, but this is specific to the model and so not tested here.
+
+        The  CliRunner() invokes the subcommand in a subprocess, and the annotate subcommand itself invokes the MLflow
+        model in yet another subprocess. So while this test can help determine if everything is working, it is not a
+        simple matter to debug in the case of a failure. However, the stdout/stderr of the MLflow process is captured
+        by the CliRunner() subprocess, so errors can be inspected in result.stdout when debugging this test. Hope this
+        helps!
+        """
+
+        _, query_dataset_file_path = mkstemp()
+        model_file_path = write_model(FakeModel())
+
+        result = CliRunner().invoke(
+            annotate,
+            [
+                query_dataset_file_path,
+                "--model-url",
+                model_file_path,
+                "--output-h5ad-file",
+                f"{query_dataset_file_path}.output",
+                # avoid having mflow create conda env or virtualenv when in test env;
+                # this avoids making pip remote requests and is also faster
+                "--mlflow-env-manager",
+                "local",
+            ],
+        )
+
+        # to help debugging, show the output from the CliRunner and MLflow stdout
+        if result.exit_code:
+            print(result.stdout)
+
+        self.assertEqual(0, result.exit_code, "runs successfully")
+
+        # The FakeModel will print it inputs to stdout, as "__MODEL_INPUT__={...}", allowing us to assert that it received valid inputs.
+        self.assertIn(
+            "__MODEL_INPUT__={"
+            f'"query_dataset_h5ad_path": "{query_dataset_file_path}", '
+            f'"output_h5ad_path": "{query_dataset_file_path}.output", '
+            '"annotation_prefix": "cxg_cell_type", "classifier": "default", '
+            '"organism": "Homo sapiens", "use_gpu": true}',
+            result.stdout,
+            "inputs passed correctly",
+        )
+        self.assertIn(
+            f"Wrote annotations to {query_dataset_file_path}.output",
+            result.stdout,
+            "success message is correct",
+        )
+
+    def test__annotate__requires_overwrite_option_when_output_file_exists(self):
+
+        with NamedTemporaryFile() as input_h5ad, NamedTemporaryFile() as existing_file:
+            required_options = [input_h5ad.name, "--output-h5ad-file", existing_file.name, "--model-url", "some_url"]
+            result = CliRunner().invoke(
+                annotate,
+                required_options + [],
+            )
+
+            self.assertNotEqual(0, result.exit_code, "aborts with non-success code")
+            self.assertIn(
+                "try using the flag --overwrite",
+                result.stdout,
+                "error message displayed",
+            )
+
+    def test__annotate__overwrite_option_allows_overwrite_of_existing_output_file(self):
+        model_file_path = write_model(FakeModel())
+
+        with NamedTemporaryFile() as existing_file:
+            required_options = [
+                existing_file.name,
+                "--output-h5ad-file",
+                existing_file.name,
+                "--overwrite",
+                "--model-url",
+                model_file_path,
+            ]
+            result = CliRunner().invoke(
+                annotate,
+                required_options + [],
+            )
+
+            print(result.stdout)
+            self.assertNotEqual(1, result.exit_code, "aborts with non-success code")
+            self.assertIn(
+                f"Wrote annotations to {existing_file.name}",
+                result.stdout,
+                "success message is correct on output file overwrite",
+            )
+
+
+# TODO:
+# Test annotate cli args more comprehensively
+# Test server.cli.annotate._validate_options
+# Test model caching feature works
+# Test model loading from s3 works (maybe w/just a real model)
+
+if __name__ == "__main__":
+    unittest.main()

test/unit/common/config/__init__.py

@@ -21,7 +21,7 @@ class ConfigTests(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls) -> None:
-        os.makedirs(cls.tmp_fixtures_directory, exist_ok=True)
+        os.makedirs(cls.tmp_fixtures_directory)
 
     def custom_server_config(
         self,

test/unit/common/config/test_dataset_config.py

@@ -72,18 +72,24 @@ class TestDatasetConfig(ConfigTests):
             config.dataset_config.handle_app()
 
     def test_handle_user_annotations__instantiates_user_annotations_class_correctly(self):
-        config = self.get_config(enable_users_annotations="true", annotation_type="local_file_csv")
+        config = self.get_config(
+            enable_users_annotations="true", annotation_type="local_file_csv"
+        )
         config.server_config.complete_config(self.context)
         config.dataset_config.handle_user_annotations(self.context)
         self.assertIsInstance(config.dataset_config.user_annotations, AnnotationsLocalFile)
 
-        config = self.get_config(enable_users_annotations="true", annotation_type="NOT_REAL")
+        config = self.get_config(
+            enable_users_annotations="true", annotation_type="NOT_REAL"
+        )
         config.server_config.complete_config(self.context)
         with self.assertRaises(ConfigurationError):
             config.dataset_config.handle_user_annotations(self.context)
 
     def test_handle_local_file_csv_annotations__sets_dir_if_not_passed_in(self):
-        config = self.get_config(enable_users_annotations="true", annotation_type="local_file_csv")
+        config = self.get_config(
+            enable_users_annotations="true", annotation_type="local_file_csv"
+        )
         config.server_config.complete_config(self.context)
         config.dataset_config.handle_local_file_csv_annotations(self.context)
         self.assertIsInstance(config.dataset_config.user_annotations, AnnotationsLocalFile)

test/unit/common/config/test_external_config.py

@@ -56,6 +56,7 @@ class TestExternalConfig(ConfigTests):
             self.assertFalse(data_config["config"]["parameters"]["disable-diffexp"])
 
     def test_environment_variable_errors(self):
+
         # no name
         app_config = AppConfig()
         app_config.external_config.environment = [dict(required=True, path=["this", "is", "a", "path"])]

test/unit/common/test_api.py

@@ -196,18 +196,17 @@ class EndPoints(object):
     def test_fbs_default(self):
         endpoint = "data/var"
         url = f"{self.URL_BASE}{endpoint}"
-        headers = {"Content-Type": "application/json"}
+        result = self.session.put(url)
-        result = self.session.put(url, headers=headers)
         self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
 
         filter = {"filter": {"var": {"index": [0, 1, 4]}}}
-        result = self.session.put(url, json=filter, headers=headers)
+        result = self.session.put(url, json=filter)
         self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
 
     def test_data_put_fbs(self):
         endpoint = "data/var"
         url = f"{self.URL_BASE}{endpoint}"
-        header = {"Accept": "application/octet-stream", "Content-Type": "application/json"}
+        header = {"Accept": "application/octet-stream"}
         result = self.session.put(url, headers=header)
         self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
 
@@ -253,7 +252,6 @@ class EndPoints(object):
             if type(column) is np.ndarray:
                 self.assertIn(column.dtype, [np.float32, np.int32])
 
-    @unittest.skip("This test is currently broken after upgrading Werkzeug.")
     def test_data_get_unknown_filter_fbs(self):
         index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
         endpoint = "data/var"
@@ -292,7 +290,7 @@ class EndPoints(object):
         result_data = result.json()
         self.assertEqual(result_data, pbmc3k_colors)
 
-    @unittest.skip("needs fix: https://github.com/chanzuckerberg/cellxgene/issues/2542")
+    @unittest.skip('needs fix: https://github.com/chanzuckerberg/cellxgene/issues/2542')
     def test_static(self):
         endpoint = "static"
         file = "assets/favicon.ico"

test/unit/common/test_rest.py

@@ -57,6 +57,7 @@ class FilterParseTests(unittest.TestCase):
         )
 
     def test_queryparam_to_filter_errors(self):
+
         # should raise FilterError
         filter_errors = [
             "foo=bar",  # no axis

test/unit/compute/test_est_dist.py

@@ -65,13 +65,13 @@ class EstDistTest(unittest.TestCase):
 
         # non-finites
         self.assertEqual(estimate_approximate_distribution(np.array([np.nan])), XApproximateDistribution.NORMAL)
-        self.assertEqual(estimate_approximate_distribution(np.array([np.inf])), XApproximateDistribution.NORMAL)
+        self.assertEqual(estimate_approximate_distribution(np.array([np.PINF])), XApproximateDistribution.NORMAL)
-        self.assertEqual(estimate_approximate_distribution(np.array([np.inf])), XApproximateDistribution.NORMAL)
+        self.assertEqual(estimate_approximate_distribution(np.array([np.NINF])), XApproximateDistribution.NORMAL)
         self.assertEqual(
-            estimate_approximate_distribution(np.array([np.inf, np.inf, 0])), XApproximateDistribution.NORMAL
+            estimate_approximate_distribution(np.array([np.PINF, np.NINF, 0])), XApproximateDistribution.NORMAL
         )
         self.assertEqual(
-            estimate_approximate_distribution(np.array([np.nan, np.inf, np.inf])), XApproximateDistribution.NORMAL
+            estimate_approximate_distribution(np.array([np.nan, np.PINF, np.NINF])), XApproximateDistribution.NORMAL
         )
 
         raw = np.random.exponential(scale=1000, size=(50, 3))
@@ -82,15 +82,15 @@ class EstDistTest(unittest.TestCase):
             XApproximateDistribution.COUNT,
         )
         self.assertEqual(
-            estimate_approximate_distribution(put(raw, [1], [np.inf])),
+            estimate_approximate_distribution(put(raw, [1], [np.PINF])),
             XApproximateDistribution.COUNT,
         )
         self.assertEqual(
-            estimate_approximate_distribution(put(raw, [1], [np.inf])),
+            estimate_approximate_distribution(put(raw, [1], [np.NINF])),
             XApproximateDistribution.COUNT,
         )
         self.assertEqual(
-            estimate_approximate_distribution(put(raw, [1, 3, 88], [np.nan, np.inf, np.inf])),
+            estimate_approximate_distribution(put(raw, [1, 3, 88], [np.nan, np.PINF, np.NINF])),
             XApproximateDistribution.COUNT,
         )
         self.assertEqual(
@@ -103,15 +103,15 @@ class EstDistTest(unittest.TestCase):
             XApproximateDistribution.NORMAL,
         )
         self.assertEqual(
-            estimate_approximate_distribution(put(logged, [1], [np.inf])),
+            estimate_approximate_distribution(put(logged, [1], [np.PINF])),
             XApproximateDistribution.NORMAL,
         )
         self.assertEqual(
-            estimate_approximate_distribution(put(logged, [1], [np.inf])),
+            estimate_approximate_distribution(put(logged, [1], [np.NINF])),
             XApproximateDistribution.NORMAL,
         )
         self.assertEqual(
-            estimate_approximate_distribution(put(logged, [1, 3, 88], [np.nan, np.inf, np.inf])),
+            estimate_approximate_distribution(put(logged, [1, 3, 88], [np.nan, np.PINF, np.NINF])),
             XApproximateDistribution.NORMAL,
         )
         self.assertEqual(

test/unit/data_anndata/test_anndata_adaptor.py

@@ -36,6 +36,7 @@ Test the anndata adaptor using the pbmc3k data set.
         (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", True, "normal"),
         (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", True, "normal"),
         (f"{FIXTURES_ROOT}/pbmc3k_64.h5ad", False, "auto"),  # 64 bit conversion tests
+        (f"{FIXTURES_ROOT}/pbmc3k_16.h5ad", False, "auto"),  # 16 bit conversion tests
     ],
 )
 class AdaptorTest(unittest.TestCase):

test/unit/data_anndata/test_anndata_adaptor_errors.py

@@ -0,0 +1,23 @@
+import unittest
+
+from parameterized import parameterized_class
+
+from server.common.errors import DatasetAccessError
+from test import FIXTURES_ROOT
+from test.unit import app_config
+
+
+@parameterized_class(
+    ("data_locator", "backed", "X_approximate_distribution"),
+    [
+        (f"{FIXTURES_ROOT}/pbmc3k_16.h5ad", True, "auto"),  # 16 bit conversion tests
+    ],
+)
+class AdaptorLoadErrorTest(unittest.TestCase):
+    def test_float16_backed_raises_err(self):
+        with self.assertRaises(DatasetAccessError):
+            config = app_config(
+                self.data_locator,
+                backed=self.backed,
+                extra_dataset_config=dict(X_approximate_distribution=self.X_approximate_distribution),
+            )

test/unit/utils/test_jsonify_strict.py

@@ -16,10 +16,10 @@ class TestJsonifyStrict(unittest.TestCase):
             jsonify_strict({"nan": [np.nan]})
 
         with self.assertRaises(ValueError):
-            jsonify_strict({"pinf": [np.inf]})
+            jsonify_strict({"pinf": [np.PINF]})
 
         with self.assertRaises(ValueError):
-            jsonify_strict({"ninf": [np.inf]})
+            jsonify_strict({"ninf": [np.NINF]})
 
     def test_jsonify_numpy_ndarray(self):
         values = {

test/unit/utils/test_type_conversion_utils.py

@@ -42,7 +42,7 @@ class TestTypeConversionUtils(unittest.TestCase):
             with self.assertRaises(TypeError):
                 get_schema_type_hint_from_dtype(np.dtype(dtype))
 
-        for dtype in [np.float32, np.float64]:
+        for dtype in [np.float16, np.float32, np.float64]:
             self.assertEqual(get_schema_type_hint_from_dtype(np.dtype(dtype)), {"type": "float32"})
 
         for dtype in [np.dtype(object), np.dtype(str)]:
@@ -123,18 +123,17 @@ int_OK_cases = [
 
 float_OK_cases = [
     {
-        "test_case": "float_OK_cases",
         "data": data,
         "expected_encoding_dtype": np.float32,
         "expected_schema_hint": {"type": "float32"},
-        "logs": None if dtype == np.float32 else {"level": logging.WARNING, "output": "may lose precision"},
+        "logs": None if data.dtype != np.float64 else {"level": logging.WARNING, "output": "may lose precision"},
     }
-    for dtype in [np.float32, np.float64]
+    for dtype in [np.float16, np.float32, np.float64]
     for data in [
         np.arange(-128, 1000, dtype=dtype),
         pd.Series(np.arange(-128, 1000, dtype=dtype)),
         pd.Index(np.arange(-129, 1000, dtype=dtype)),
-        np.array([-np.nan, -np.inf, -1, -0.0, 0, 0.0, 1, np.inf, np.nan], dtype=dtype),
+        np.array([-np.nan, np.NINF, -1, np.NZERO, 0, np.PZERO, 1, np.PINF, np.nan], dtype=dtype),
         np.array([np.finfo(dtype).min, 0, np.finfo(dtype).max], dtype=dtype),
         sparse.csr_matrix((10, 100), dtype=dtype),
     ]
@@ -199,13 +198,12 @@ category_numeric_OK_cases = [
     # numeric, no NA/NaN, float
     *[
         {
-            "test_case": "numeric, no NA/NaN, float",
             "data": data,
             "expected_encoding_dtype": np.float32,
             "expected_schema_hint": {"type": "categorical"},
-            "logs": None if dtype == np.float32 else {"level": logging.WARNING, "output": "may lose precision"},
+            "logs": {"level": logging.WARNING, "output": "may lose precision"},
         }
-        for dtype in [np.float32, np.float64]
+        for dtype in [np.float16, np.float32, np.float64]
         for data in [
             pd.Series(np.array([0, 1, 2], dtype=dtype), dtype="category"),
             pd.Series(np.array([0, 1, 2], dtype=dtype), dtype="category").cat.remove_categories([1]),
@@ -215,11 +213,10 @@ category_numeric_OK_cases = [
     # numeric, has NA-induced cast to float32
     *[
         {
-            "test_case": "numeric, has NA-induced cast to float32",
             "data": data,
             "expected_encoding_dtype": np.float32,
             "expected_schema_hint": {"type": "categorical"},
-            "logs": None if dtype == np.float32 else {"level": logging.WARNING, "output": "may lose precision"},
+            "logs": {"level": logging.WARNING, "output": "may lose precision"},
         }
         for dtype in [
             np.int8,
@@ -230,6 +227,7 @@ category_numeric_OK_cases = [
             np.uint32,
             np.int64,
             np.uint64,
+            np.float16,
             np.float32,
             np.float64,
         ]
@@ -314,6 +312,7 @@ class TestTypeInference(unittest.TestCase, AssertNoLog):
                     self.assertEqual(encoding_dtype, self.expected_encoding_dtype)
                     self.assertEqual(schema_hint, self.expected_schema_hint)
                     self.assertIn(logs["output"], logger.output[0])
+
             else:
                 with self.assertNoLogs(logging.getLogger(), logging.WARNING):
                     encoding_dtype, schema_hint = get_dtype_and_schema_of_array(self.data)