debug

* chore: Fix compatibility tests

* DEBUGGGG

* fix: update deps, fix unit tests

* fix: FE deps

* chore: update compatibility matrix

---------

Co-authored-by: kaloster <rkalo@contractor.chanzuckerberg.com>

.github/workflows/compatibility_tests.yml

@@ -7,6 +7,10 @@ on:
     branches:
       - main
 
+  ### For debugging purposes - uncomment below to run on all PRs
+  pull_request:
+    branches: "*"
+
 env:
   JEST_ENV: prod
 
@@ -14,9 +18,9 @@ jobs:
   docker-build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Build docker image
@@ -29,23 +33,23 @@ jobs:
       fail-fast: false
       matrix:
         # note: The `macos-latest` is latest Catalina version, and not Big Sur. So we explicitly ask for Big Sur (`macos-11`)
-        os: [ubuntu-latest, macos-latest, macos-11]
-        python-version: [3.8, 3.9, 3.10, 3.11]
+        os: [ubuntu-latest, macos-latest, macos-13]
+        python-version: ["3.10", "3.11", "3.12"]
         cellxgene_build: [main, latest]
         # add anndata pinned version test for subset of matrix configurations,
         # in order to reduce matrix cross-product explosion
         include:
-          - python-version: 3.9
+          - python-version: 3.12
             cellxgene_build: latest
             # TODO: dynamically use the literal version in requirements.txt,
             # to avoid having to update this in manually in the future
             # TODO: Do not bother running this if anndata latest version
             # matches this pinned version, to avoid a redundant test
-            anndata_version: "==0.10.3"
+            anndata_version: "==0.10.9"
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Cache env vars
@@ -55,14 +59,14 @@ jobs:
         run: echo "BREW_CACHE=`brew --cache`" >> $GITHUB_ENV
       # FIXME: Only working for Linux
       - name: Python cache
-        uses: actions/cache@v1
+        uses: actions/cache@v4
         with:
           path: ${{ env.PIP_CACHE }}
           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
           restore-keys: |
             ${{ runner.os }}-pip-
       - name: Node cache
-        uses: actions/cache@v1
+        uses: actions/cache@v4
         with:
           path: ~/.npm
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
@@ -70,7 +74,7 @@ jobs:
             ${{ runner.os }}-node-
       - name: Brew cache (MacOS)
         if: startsWith(matrix.os, 'macos')
-        uses: actions/cache@v1
+        uses: actions/cache@v4
         with:
           path: ${{ env.BREW_CACHE }}
           key: ${{ runner.os }}-brew-
@@ -96,6 +100,7 @@ jobs:
           # keep same pip pkg versions as in the cxg release
           sed -i'' -e 's/-r requirements.txt//' server/requirements-dev.txt
           pip install -r server/requirements-dev.txt
+          pip install --force-reinstall numpy==2.0.1 numba>=0.60.0 pandas
       - name: Install anndata version per matrix variable
         run: pip install anndata${{ matrix.anndata_version }}
       - name: Install node

.github/workflows/push_tests.yml

@@ -14,25 +14,28 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - run: |
           git fetch --depth=1 origin +${{github.base_ref}}
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v4
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: 3.12
       - name: Node cache
-        uses: actions/cache@v1
+        uses: actions/cache@v4
         with:
           path: ~/.npm
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
           restore-keys: |
             ${{ runner.os }}-node-
-      - name: Install dependencies
+      - name: Install dependencies now
         run: |
           pip install flake8
           pip install black
-          cd client
+          pip install setuptools
+      - name: Install client dependencies
+        run: |
+          cd client 
           npm install
       - name: Format with black and lint with flake8
         run: |
@@ -45,22 +48,22 @@ jobs:
   unit-test:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.9 (pyenv) # pyenv needed for mlflow in cli annotate tests
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.12 (pyenv) # pyenv needed for mlflow in cli annotate tests
         uses: gabrielfalcao/pyenv-action@v9
         with:
-          default: 3.9
+          default: 3.12
           command: pip install -U pip # upgrade pip after installing python
       - run: pip install virtualenv # virtualenv needed for mlflow in cli annotate tests
       - name: Python cache
-        uses: actions/cache@v1
+        uses: actions/cache@v4
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
           restore-keys: |
             ${{ runner.os }}-pip-
       - name: Node cache
-        uses: actions/cache@v1
+        uses: actions/cache@v4
         with:
           path: ~/.npm
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
@@ -78,20 +81,20 @@ jobs:
     runs-on: macos-latest
     timeout-minutes: 20
     steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: 3.12
       - name: Python cache
-        uses: actions/cache@v1
+        uses: actions/cache@v4
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
           restore-keys: |
             ${{ runner.os }}-pip-
       - name: Node cache
-        uses: actions/cache@v1
+        uses: actions/cache@v4
         with:
           path: ~/.npm
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}

README.md

@@ -58,7 +58,7 @@ Please [file an issue](https://github.com/chanzuckerberg/cellxgene/issues/new/ch
 ### Finding help
 
 We'd love to hear from you!
-For questions, suggestions, or accolades, [join the `#cellxgene-users` channel on the CZI Science Slack](https://join-cellxgene-users.herokuapp.com/) and say "hi!".
+For questions, suggestions, or accolades, join the `#cellxgene-users` channel on the [CZI Science Community Slack](https://czi.co/science-slack) and say "hi!".
 
 For any errors, [report bugs on Github](https://github.com/chanzuckerberg/cellxgene/issues).
 

client/.nvmrc

@@ -0,0 +1 @@
+16.20.0

client/__tests__/e2e/puppeteer.setup.js

@@ -13,7 +13,7 @@ import * as ENV_DEFAULT from "../../../environment.default.json";
 // a test can take more time to finish, so we don't want
 // jest to shut off the test too soon
 jest.setTimeout(2 * 60 * 1000);
-setDefaultOptions({ timeout: 20 * 1000 });
+setDefaultOptions({ timeout: 60 * 1000 });
 
 jest.retryTimes(ENV_DEFAULT.RETRY_ATTEMPTS);
 

client/package.json

@@ -18,7 +18,8 @@
   },
   "engineStrict": true,
   "engines": {
-    "npm": ">=3.0.0"
+    "npm": ">=3.0.0",
+    "node": "^16.0.0"
   },
   "eslintConfig": {
     "extends": "./configuration/eslint/eslint.js"
@@ -77,7 +78,7 @@
     "whatwg-fetch": "^3.2.0"
   },
   "devDependencies": {
-    "@babel/core": "^7.13.16",
+    "@babel/core": "^7.25.2",
     "@babel/plugin-proposal-class-properties": "^7.10.4",
     "@babel/plugin-proposal-decorators": "^7.13.15",
     "@babel/plugin-proposal-export-namespace-from": "^7.10.4",
@@ -123,7 +124,7 @@
     "jest-circus": "^27.0.6",
     "jest-environment-puppeteer": "^5.0.1",
     "jest-fetch-mock": "^3.0.3",
-    "jest-puppeteer": "^5.0.1",
+    "jest-puppeteer": "^6.2.0",
     "json-loader": "^0.5.7",
     "lint-staged": "^10.2.11",
     "lodash": "^4.17.21",
@@ -134,11 +135,11 @@
     "lodash.zip": "^4.2.0",
     "mini-css-extract-plugin": "^1.5.0",
     "prettier": "^2.0.5",
-    "puppeteer": "^8.0.0",
+    "puppeteer": "^10.4.0",
     "rimraf": "^3.0.2",
     "serve-favicon": "^2.5.0",
     "terser-webpack-plugin": "^5.1.1",
-    "webpack": "^5.88.2",
+    "webpack": "^5.94.0",
     "webpack-cli": "^4.6.0",
     "webpack-dev-middleware": "^4.1.0",
     "webpack-merge": "^5.0.9",

client/src/components/leftSidebar/infoMenu.js

@@ -16,7 +16,7 @@ const InformationMenu = React.memo((props) => {
             rel="noopener"
           />
           <MenuItem
-            href="https://join-cellxgene-users.herokuapp.com/"
+            href="https://czi.co/science-slack"
             target="_blank"
             icon="chat"
             text="Chat"

server/common/config/dataset_config.py

@@ -176,7 +176,7 @@ class DatasetConfig(BaseConfig):
         self.validate_correct_type_of_configuration_attribute("diffexp__top_n", int)
 
         data_adaptor = self.get_data_adaptor()
-        if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False):
+        if self.diffexp__enable and data_adaptor.parameters.get("diffexp-may-be-slow", False):
             context["messagefn"](
                 "CAUTION: due to the size of your dataset, " "running differential expression may take longer or fail."
             )

server/common/utils/type_conversion_utils.py

@@ -116,7 +116,7 @@ def _get_type_info(array: Union[np.ndarray, pd.Series, pd.Index]) -> Tuple[np.dt
         raise TypeError("Unsupported data type.")
 
     dtype = array.dtype
-
+    
     res = _get_type_info_from_dtype(dtype)
     if res is not None:
         return res
@@ -140,7 +140,6 @@ def _get_type_info(array: Union[np.ndarray, pd.Series, pd.Index]) -> Tuple[np.dt
 
     if dtype.kind in ["i", "u"] and _can_cast_array_values_to_int32(array):
         return (np.int32, {"type": "int32"})
-
     if dtype.kind == "f":
         _float64_warning(array.dtype)
         return (np.float32, {"type": "float32"})

server/data_anndata/anndata_adaptor.py

@@ -211,7 +211,7 @@ class AnndataAdaptor(DataAdaptor):
         # heuristic
         n_values = self.data.shape[0] * self.data.shape[1]
         if (n_values > 1e8 and self.server_config.adaptor__anndata_adaptor__backed is True) or (n_values > 5e8):
-            self.parameters.update({"diffexp_may_be_slow": True})
+            self.parameters.update({"diffexp-may-be-slow": True})
 
     def _is_valid_layout(self, arr):
         """return True if this layout data is a valid array for front-end presentation:

server/requirements-annotate.txt

@@ -1,2 +1,2 @@
-mlflow==1.27.0
+mlflow==2.16.0
 scanpy

server/requirements.txt

@@ -7,16 +7,17 @@ Flask-Cors>=3.0.9
 Flask-RESTful>=0.3.6
 flask-server-timing>=0.1.2
 flask-talisman>=0.7.0
-flatbuffers>=1.11.0,<2.0.0 # cellxgene is not compatible with 2.0.0. Requires migration
+flatbuffers==2.0.7
 flatten-dict>=0.2.0
 fsspec>0.8.0
 gunicorn>=20.0.4
 h5py>=3.0.0
-numba>=0.51.2
-numpy>1.22
+numba>=0.60.0
+numpy==2.0.1
 packaging>=20.0
-pandas<2.0.0
+pandas>=2.2.2
 PyYAML>=5.4  # CVE-2020-14343
 requests>=2.22.0
 s3fs==0.4.2
-scipy>=1.4
+scipy>=1.4
+setuptools

setup.py

@@ -24,7 +24,7 @@ setup(
     long_description=long_description,
     long_description_content_type="text/markdown",
     install_requires=requirements,
-    python_requires=">=3.6",
+    python_requires=">=3.10",
     include_package_data=True,
     zip_safe=False,
     classifiers=[

test/unit/compute/test_est_dist.py

@@ -65,13 +65,13 @@ class EstDistTest(unittest.TestCase):
 
         # non-finites
         self.assertEqual(estimate_approximate_distribution(np.array([np.nan])), XApproximateDistribution.NORMAL)
-        self.assertEqual(estimate_approximate_distribution(np.array([np.PINF])), XApproximateDistribution.NORMAL)
-        self.assertEqual(estimate_approximate_distribution(np.array([np.NINF])), XApproximateDistribution.NORMAL)
+        self.assertEqual(estimate_approximate_distribution(np.array([np.inf])), XApproximateDistribution.NORMAL)
+        self.assertEqual(estimate_approximate_distribution(np.array([np.inf])), XApproximateDistribution.NORMAL)
         self.assertEqual(
-            estimate_approximate_distribution(np.array([np.PINF, np.NINF, 0])), XApproximateDistribution.NORMAL
+            estimate_approximate_distribution(np.array([np.inf, np.inf, 0])), XApproximateDistribution.NORMAL
         )
         self.assertEqual(
-            estimate_approximate_distribution(np.array([np.nan, np.PINF, np.NINF])), XApproximateDistribution.NORMAL
+            estimate_approximate_distribution(np.array([np.nan, np.inf, np.inf])), XApproximateDistribution.NORMAL
         )
 
         raw = np.random.exponential(scale=1000, size=(50, 3))
@@ -82,15 +82,15 @@ class EstDistTest(unittest.TestCase):
             XApproximateDistribution.COUNT,
         )
         self.assertEqual(
-            estimate_approximate_distribution(put(raw, [1], [np.PINF])),
+            estimate_approximate_distribution(put(raw, [1], [np.inf])),
             XApproximateDistribution.COUNT,
         )
         self.assertEqual(
-            estimate_approximate_distribution(put(raw, [1], [np.NINF])),
+            estimate_approximate_distribution(put(raw, [1], [np.inf])),
             XApproximateDistribution.COUNT,
         )
         self.assertEqual(
-            estimate_approximate_distribution(put(raw, [1, 3, 88], [np.nan, np.PINF, np.NINF])),
+            estimate_approximate_distribution(put(raw, [1, 3, 88], [np.nan, np.inf, np.inf])),
             XApproximateDistribution.COUNT,
         )
         self.assertEqual(
@@ -103,15 +103,15 @@ class EstDistTest(unittest.TestCase):
             XApproximateDistribution.NORMAL,
         )
         self.assertEqual(
-            estimate_approximate_distribution(put(logged, [1], [np.PINF])),
+            estimate_approximate_distribution(put(logged, [1], [np.inf])),
             XApproximateDistribution.NORMAL,
         )
         self.assertEqual(
-            estimate_approximate_distribution(put(logged, [1], [np.NINF])),
+            estimate_approximate_distribution(put(logged, [1], [np.inf])),
             XApproximateDistribution.NORMAL,
         )
         self.assertEqual(
-            estimate_approximate_distribution(put(logged, [1, 3, 88], [np.nan, np.PINF, np.NINF])),
+            estimate_approximate_distribution(put(logged, [1, 3, 88], [np.nan, np.inf, np.inf])),
             XApproximateDistribution.NORMAL,
         )
         self.assertEqual(

test/unit/utils/test_jsonify_strict.py

@@ -16,10 +16,10 @@ class TestJsonifyStrict(unittest.TestCase):
             jsonify_strict({"nan": [np.nan]})
 
         with self.assertRaises(ValueError):
-            jsonify_strict({"pinf": [np.PINF]})
+            jsonify_strict({"pinf": [np.inf]})
 
         with self.assertRaises(ValueError):
-            jsonify_strict({"ninf": [np.NINF]})
+            jsonify_strict({"ninf": [np.inf]})
 
     def test_jsonify_numpy_ndarray(self):
         values = {

test/unit/utils/test_type_conversion_utils.py

@@ -42,7 +42,7 @@ class TestTypeConversionUtils(unittest.TestCase):
             with self.assertRaises(TypeError):
                 get_schema_type_hint_from_dtype(np.dtype(dtype))
 
-        for dtype in [np.float16, np.float32, np.float64]:
+        for dtype in [np.float32, np.float64]:
             self.assertEqual(get_schema_type_hint_from_dtype(np.dtype(dtype)), {"type": "float32"})
 
         for dtype in [np.dtype(object), np.dtype(str)]:
@@ -123,17 +123,18 @@ int_OK_cases = [
 
 float_OK_cases = [
     {
+        "test_case": "float_OK_cases",
         "data": data,
         "expected_encoding_dtype": np.float32,
         "expected_schema_hint": {"type": "float32"},
-        "logs": None if data.dtype != np.float64 else {"level": logging.WARNING, "output": "may lose precision"},
+        "logs": None if dtype == np.float32 else {"level": logging.WARNING, "output": "may lose precision"},
     }
-    for dtype in [np.float16, np.float32, np.float64]
+    for dtype in [np.float32, np.float64]
     for data in [
         np.arange(-128, 1000, dtype=dtype),
         pd.Series(np.arange(-128, 1000, dtype=dtype)),
         pd.Index(np.arange(-129, 1000, dtype=dtype)),
-        np.array([-np.nan, np.NINF, -1, np.NZERO, 0, np.PZERO, 1, np.PINF, np.nan], dtype=dtype),
+        np.array([-np.nan, np.inf, -1, 0.0, 0, 0.0, 1, np.inf, np.nan], dtype=dtype),
         np.array([np.finfo(dtype).min, 0, np.finfo(dtype).max], dtype=dtype),
         sparse.csr_matrix((10, 100), dtype=dtype),
     ]
@@ -198,12 +199,13 @@ category_numeric_OK_cases = [
     # numeric, no NA/NaN, float
     *[
         {
+            "test_case": "numeric, no NA/NaN, float",
             "data": data,
             "expected_encoding_dtype": np.float32,
             "expected_schema_hint": {"type": "categorical"},
-            "logs": {"level": logging.WARNING, "output": "may lose precision"},
+            "logs": None if dtype == np.float32 else {"level": logging.WARNING, "output": "may lose precision"},
         }
-        for dtype in [np.float16, np.float32, np.float64]
+        for dtype in [np.float32, np.float64]
         for data in [
             pd.Series(np.array([0, 1, 2], dtype=dtype), dtype="category"),
             pd.Series(np.array([0, 1, 2], dtype=dtype), dtype="category").cat.remove_categories([1]),
@@ -213,10 +215,11 @@ category_numeric_OK_cases = [
     # numeric, has NA-induced cast to float32
     *[
         {
+            "test_case": "numeric, has NA-induced cast to float32",
             "data": data,
             "expected_encoding_dtype": np.float32,
             "expected_schema_hint": {"type": "categorical"},
-            "logs": {"level": logging.WARNING, "output": "may lose precision"},
+            "logs": None if dtype == np.float32 else {"level": logging.WARNING, "output": "may lose precision"},
         }
         for dtype in [
             np.int8,
@@ -227,7 +230,6 @@ category_numeric_OK_cases = [
             np.uint32,
             np.int64,
             np.uint64,
-            np.float16,
             np.float32,
             np.float64,
         ]
@@ -312,7 +314,6 @@ class TestTypeInference(unittest.TestCase, AssertNoLog):
                     self.assertEqual(encoding_dtype, self.expected_encoding_dtype)
                     self.assertEqual(schema_hint, self.expected_schema_hint)
                     self.assertIn(logs["output"], logger.output[0])
-
             else:
                 with self.assertNoLogs(logging.getLogger(), logging.WARNING):
                     encoding_dtype, schema_hint = get_dtype_and_schema_of_array(self.data)