Add README-visium

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.3.0
+current_version = 1.0.0
 commit = True
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?:-(?P<prerel>rc)\.(?P<prerelversion>\d+))?
 serialize = 

.github/ISSUE_TEMPLATE/tech-issue.md

@@ -1,23 +0,0 @@
----
-name: Tech Issue
-about: Engineering-specific technical work that is not product-specific. Engineering team "owns" these issues.
-title: ""
-labels: tech
-assignees: ""
----
-
-## Motivation
-
-Why is this work important to engineers?
-
-## Definition of Done
-
-What should the end result look like? What will have been changed?
-
-## Tasks
-
-Detail the specific tasks that can be used to accomplish the desired changes.
-If detailed steps cannot be provided at this time, please file a [Tech Proposal](https://docs.google.com/document/d/1o2vuvl-kXwRJN1nBoPzJS_MAQgDGYnjmPZWa4qRDi-I/edit#heading=h.7dvzhm7gqc3v) instead.
-
-- [ ]
-- [ ]

.github/workflows/close-stale-prs.yml

@@ -1,23 +0,0 @@
-name: Close inactive pull requests
-on:
-  schedule:
-    - cron: "30 1 * * *"
-
-jobs:
-  close-issues:
-    runs-on: ubuntu-latest
-    permissions:
-      issues: write
-      pull-requests: write
-    steps:
-      - uses: actions/stale@v5
-        with:
-          days-before-issue-stale: -1 # Do not mark any issues as stale
-          days-before-pr-stale: 14
-          days-before-pr-close: 3
-          stale-pr-message: "This PR has not seen any activity in the past 2 weeks; if no one comments or reviews it in the next 3 days, this PR will be closed."
-          close-pr-message: "This PR was closed because it has been inactive for 17 days, 3 days since being marked as stale. Please re-open if you still need this to be addressed."
-          stale-pr-label: "stale"
-          close-pr-label: "autoclosed"
-          exempt-draft-pr: true
-          repo-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/compatibility_tests.yml

@@ -2,7 +2,7 @@ name: Compatibility Tests
 
 on:
   schedule:
-    - cron: "0 8 7 * 2"
+    - cron: '0 8 7 * 2'
   push:
     branches:
       - main
@@ -14,9 +14,9 @@ jobs:
   docker-build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
       - name: Build docker image
@@ -28,85 +28,96 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest, macos-13]
+        # note: The `macos-latest` is latest Catalina version, and not Big Sur. So we explicitly ask for Big Sur (`macos-11`)
-        python-version: ["3.10", "3.11", "3.12"]
+        os: [ubuntu-latest, macos-latest, macos-11]
+        python-version: [3.6, 3.7, 3.8, 3.9]
         cellxgene_build: [main, latest]
+        exclude:
+          # 3.6 no longer avail on Big Sur (`macos-11`)
+          - os: macos-11
+            python-version: 3.6
+          # no pypi build exists for macos+py3.9 and source install fails to
+          # install `tables` py pkg (a `scanpy` dependency), so we test py3.9
+          # only on ubuntu
+          - os: macos-11
+            python-version: 3.9
+          - os: macos-latest
+            python-version: 3.9
         # add anndata pinned version test for subset of matrix configurations,
         # in order to reduce matrix cross-product explosion
         include:
-          - python-version: 3.12
+          - python-version: 3.8
             cellxgene_build: latest
             # TODO: dynamically use the literal version in requirements.txt,
             # to avoid having to update this in manually in the future
             # TODO: Do not bother running this if anndata latest version
             # matches this pinned version, to avoid a redundant test
-            anndata_version: "==0.10.9"
+            anndata_version: '==0.7.6'
     steps:
-      - uses: actions/checkout@v4
+    - uses: actions/checkout@v2
-      - name: Set up Python ${{ matrix.python-version }}
+    - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
+      uses: actions/setup-python@v1
-        with:
+      with:
-          python-version: ${{ matrix.python-version }}
+        python-version: ${{ matrix.python-version }}
-      - name: Cache env vars
+    - name: Cache env vars
-        run: echo "PIP_CACHE=`python -m pip cache dir`" >> $GITHUB_ENV
+      run: echo "PIP_CACHE=`python -m pip cache dir`" >> $GITHUB_ENV
-      - name: Cache env vars (MacOS)
+    - name: Cache env vars (MacOS)
-        if: startsWith(matrix.os, 'macos')
+      if: startsWith(matrix.os, 'macos')
-        run: echo "BREW_CACHE=`brew --cache`" >> $GITHUB_ENV
+      run: echo "BREW_CACHE=`brew --cache`" >> $GITHUB_ENV
-      # FIXME: Only working for Linux
+    # FIXME: Only working for Linux
-      - name: Python cache
+    - name: Python cache
-        uses: actions/cache@v4
+      uses: actions/cache@v1
-        with:
+      with:
-          path: ${{ env.PIP_CACHE }}
+        path: ${{ env.PIP_CACHE }}
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
-          restore-keys: |
+        restore-keys: |
-            ${{ runner.os }}-pip-
+          ${{ runner.os }}-pip-
-      - name: Node cache
+    - name: Node cache
-        uses: actions/cache@v4
+      uses: actions/cache@v1
-        with:
+      with:
-          path: ~/.npm
+        path: ~/.npm
-          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+        key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
-          restore-keys: |
+        restore-keys: |
-            ${{ runner.os }}-node-
+          ${{ runner.os }}-node-
-      - name: Brew cache (MacOS)
+    - name: Brew cache (MacOS)
-        if: startsWith(matrix.os, 'macos')
+      if: startsWith(matrix.os, 'macos')
-        uses: actions/cache@v4
+      uses: actions/cache@v1
-        with:
+      with:
-          path: ${{ env.BREW_CACHE }}
+        path: ${{ env.BREW_CACHE }}
-          key: ${{ runner.os }}-brew-
+        key: ${{ runner.os }}-brew-
-      - name: Install dependencies (Ubuntu Linux)
+    - name: Install dependencies (Ubuntu Linux)
-        if: startsWith(matrix.os, 'ubuntu')
+      if: startsWith(matrix.os, 'ubuntu')
-        run: |
+      run: |
-          sudo apt-get update
+        sudo apt-get update
-          sudo apt-get install -y libhdf5-serial-dev
+        sudo apt-get install -y libhdf5-serial-dev
-      - name: Install dependencies (MacOS)
+    - name: Install dependencies (MacOS)
-        if: startsWith(matrix.os, 'macos')
+      if: startsWith(matrix.os, 'macos')
-        run: brew install hdf5
+      run: brew install hdf5
-      - name: Install cellxgene from `main` branch
+    - name: Install cellxgene from `main` branch
-        if: matrix.cellxgene_build == 'main'
+      if: matrix.cellxgene_build == 'main'
-        run: |
+      run: |
-          pip install -r server/requirements-dev.txt
+        pip install -r server/requirements-dev.txt
-          make pydist install-dist
+        make pydist install-dist
-      - name: Install cellxgene from latest release (pypi.org)
+    - name: Install cellxgene from latest release (pypi.org)
-        if: matrix.cellxgene_build == 'latest'
+      if: matrix.cellxgene_build == 'latest'
-        run: |
+      run: |
-          pip install --upgrade cellxgene
+        pip install --upgrade cellxgene
-          # install the additional dev requirements on top of what is in the
+        # install the additional dev requirements on top of what is in the
-          # cellxgene pip package, which are needed for testing, but otherwise
+        # cellxgene pip package, which are needed for testing, but otherwise
-          # keep same pip pkg versions as in the cxg release
+        # keep same pip pkg versions as in the cxg release
-          sed -i'' -e 's/-r requirements.txt//' server/requirements-dev.txt
+        sed -i'' -e 's/-r requirements.txt//' server/requirements-dev.txt
-          pip install -r server/requirements-dev.txt
+        pip install -r server/requirements-dev.txt
-          pip install --force-reinstall numpy==2.0.1 numba>=0.60.0 pandas flatbuffers==2.0.7
+    - name: Install anndata version per matrix variable
-      - name: Install anndata version per matrix variable
+      run: pip install anndata${{ matrix.anndata_version }}
-        run: pip install anndata${{ matrix.anndata_version }}
+    - name: Install node
-      - name: Install node
+      run: make dev-env-client
-        run: make dev-env-client
+    # Run different types of test separately, to facilitate troubleshooting
-      # Run different types of test separately, to facilitate troubleshooting
+    - name: Unit Tests - client
-      - name: Unit Tests - client
+      run: make unit-test-client
-        run: make unit-test-client
+    - name: Unit Tests - server
-      - name: Unit Tests - server
+      run: make unit-test-server
-        run: make unit-test-server
+    - name: Smoke Tests
-      - name: Smoke Tests
+      run: make smoke-test
-        run: make smoke-test
     # FIXME: Fails intermittently. See https://app.zenhub.com/workspaces/single-cell-5e2a191dad828d52cc78b028/issues/chanzuckerberg/cellxgene/2415
     # - name: Smoke Tests with Annotations
     #   run: make smoke-test-annotations

.github/workflows/lint-pr-commit-message.yml

@@ -1,19 +0,0 @@
-
-name: "Lint PR commit message"
-
-on:
-  pull_request_target:
-    types:
-      - opened
-      - edited
-      - synchronize
-
-jobs:
-  main:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: amannn/action-semantic-pull-request@v3.4.1
-        with:
-          validateSingleCommit: true
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/push_tests.yml

@@ -14,15 +14,15 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v2
       - run: |
           git fetch --depth=1 origin +${{github.base_ref}}
-      - name: Set up Python 3.12
+      - name: Set up Python 3.7
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v1
         with:
-          python-version: 3.12
+          python-version: 3.7
       - name: Node cache
-        uses: actions/cache@v4
+        uses: actions/cache@v1
         with:
           path: ~/.npm
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
@@ -45,22 +45,20 @@ jobs:
   unit-test:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v2
-      - name: Set up Python 3.12 (pyenv) # pyenv needed for mlflow in cli annotate tests
+      - name: Set up Python 3.7
-        uses: gabrielfalcao/pyenv-action@v9
+        uses: actions/setup-python@v1
         with:
-          default: 3.12
+          python-version: 3.7
-          command: pip install -U pip # upgrade pip after installing python
-      - run: pip install virtualenv # virtualenv needed for mlflow in cli annotate tests
       - name: Python cache
-        uses: actions/cache@v4
+        uses: actions/cache@v1
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
           restore-keys: |
             ${{ runner.os }}-pip-
       - name: Node cache
-        uses: actions/cache@v4
+        uses: actions/cache@v1
         with:
           path: ~/.npm
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
@@ -69,79 +67,67 @@ jobs:
       - name: Install dependencies
         run: make pydist install-dist dev-env-server
       - name: Unit tests
-        run: make unit-test-server unit-test-client
+        run: |
-      - name: Generate server coverage XML
+          make unit-test-server unit-test-client
-        run: coverage xml -o server/coverage.xml
+          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k server -cF server,python,unitTest
-      - name: Upload server coverage
+          cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest
-        uses: codecov/codecov-action@v5
-        with:
-          flags: server,python,unitTest
-          files: ./server/coverage.xml
-          fail_ci_if_error: false
-      - name: Upload client coverage
-        uses: codecov/codecov-action@v5
-        with:
-          flags: frontend,javascript,unitTest
-          files: ./client/coverage/lcov.info
-          fail_ci_if_error: false
 
   smoke-tests:
     runs-on: macos-latest
     timeout-minutes: 20
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v2
-      - name: Set up Python 3.12
+      - name: Set up Python 3.7
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v1
         with:
-          python-version: 3.12
+          python-version: 3.7
       - name: Python cache
-        uses: actions/cache@v4
+        uses: actions/cache@v1
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
           restore-keys: |
             ${{ runner.os }}-pip-
       - name: Node cache
-        uses: actions/cache@v4
+        uses: actions/cache@v1
         with:
           path: ~/.npm
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
           restore-keys: |
             ${{ runner.os }}-node-
       - name: Install dependencies
-        run: |
+        run: make pydist install-dist
-          pip install setuptools
-          make pydist install-dist
       - name: Smoke tests (without annotations feature)
-        run: cd client && make smoke-test
+        run: |
+          cd client && make smoke-test
+          ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTest
 
-  # TODO: reinstate: https://github.com/chanzuckerberg/cellxgene/issues/2544
+  smoke-tests-annotations:
-  # smoke-tests-annotations:
+    runs-on: ubuntu-latest
-  #   runs-on: ubuntu-latest
+    timeout-minutes: 20
-  #   timeout-minutes: 20
+    steps:
-  #   steps:
+      - uses: actions/checkout@v2
-  #     - uses: actions/checkout@v2
+      - name: Set up Python 3.7
-  #     - name: Set up Python 3.9
+        uses: actions/setup-python@v1
-  #       uses: actions/setup-python@v4
+        with:
-  #       with:
+          python-version: 3.7
-  #         python-version: 3.9
+      - name: Python cache
-  #     - name: Python cache
+        uses: actions/cache@v1
-  #       uses: actions/cache@v1
+        with:
-  #       with:
+          path: ~/.cache/pip
-  #         path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
-  #         key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+          restore-keys: |
-  #         restore-keys: |
+            ${{ runner.os }}-pip-
-  #           ${{ runner.os }}-pip-
+      - name: Node cache
-  #     - name: Node cache
+        uses: actions/cache@v1
-  #       uses: actions/cache@v1
+        with:
-  #       with:
+          path: ~/.npm
-  #         path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
-  #         key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
-  #         restore-keys: |
+            ${{ runner.os }}-node-
-  #           ${{ runner.os }}-node-
+      - name: Install dependencies
-  #     - name: Install dependencies
+        run: make pydist install-dist
-  #       run: make pydist install-dist
+      - name: Smoke tests (with annotations feature)
-  #     - name: Smoke tests (with annotations feature)
+        run: |
-  #       run: |
+          cd client && make smoke-test-annotations
-  #         cd client && make smoke-test-annotations
+          ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTestAnnotations
-  #         ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTestAnnotations

.gitignore

@@ -54,6 +54,3 @@ client/.eslintcache
 
 # E2E Testing
 ignoreE2E*
-
-# annotate subcmd
-.models_cache

LICENSE.txt

@@ -1,6 +1,6 @@
-# The MIT License (MIT)
+The MIT License (MIT)
 
-Copyright (c) 2017-2026 Chan Zuckerberg Initiative
+Copyright (c) 2017-2021 Chan Zuckerberg Initiative
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

MANIFEST.in

@@ -3,6 +3,5 @@ recursive-include server/common/web/static *
 
 include server/requirements.txt
 include server/requirements-prepare.txt
-include server/requirements-annotate.txt
 include server/converters/schema/hgnc_complete_set.txt.gz
 include server/converters/schema/schema_definitions/*

README-visium.md

@@ -0,0 +1,20 @@
+# Cellxgene Visium Beta
+
+## How it works
+1. Launch `cellxgene` as normal. 
+1. If the loaded dataset has spatial information available, the image data will be loaded on startup.
+1. On the toolbar, next to the Zoom icon, a `Toggle image` button will now appear. Click on it and the image will be added as an underlay.
+1. You can now use any `cellxgene` functionality and the image will still be present. If you pan and zoom, the image will also be panned and zoomed.
+1. If you want to hide the image, you can click on `Toggle image` again
+
+In order for the image to be displayed with the correct size and alignment, the H5AD needs to have a few requirements. See the following section to learn more.
+
+## h5ad requirements
+1. The spatial embedding layer should be contained in `obsm` and be named `X_spatial`. Other layers can exist, but only this one will have the spatial feature enabled.
+2. A `spatial` dict needs to be defined in the `uns` dictionary.
+3. Inside the `spatial` dict, an `images` dict must be defined. 
+4. The `images` dict must contain a `hires` key, which should reference an image encoded as an RGB matrix (i.e., a three-dimensional matrix of size `height x width x 3` where the final dimension has the RGB values for each pixel)
+5. The `images` dict must contain a `scalefactors` dict. This should in turn contain a `tissue_hires_scalef` key, which should reference a floating point number.
+
+Moreover, in order to have the image correctly aligned with the dots, the following must be true:
+1. `tissue_hires_scalef` should represent the ratio between the embedding layer `X_spatial` and the image matrix. In particular, if you multiply `X_spatial` by `tissue_hires_scalef`, you should obtain an array of points that ovelap the tissue image if you plot them in a plane.

README.md

@@ -7,27 +7,27 @@ _an interactive explorer for single-cell transcriptomics data_
 [![Compatibility Tests](https://github.com/chanzuckerberg/cellxgene/workflows/Compatibility%20Tests/badge.svg)](https://github.com/chanzuckerberg/cellxgene/actions?query=workflow%3A%22Compatibility+Tests%22)
 ![Code Coverage](https://codecov.io/gh/chanzuckerberg/cellxgene/branch/main/graph/badge.svg)
 
-CZ CELLxGENE Annotate (pronounced "cell-by-gene") is an interactive data explorer for single-cell datasets, such as those coming from the [Human Cell Atlas](https://humancellatlas.org). Leveraging modern web development techniques to enable fast visualizations of at least 1 million cells, we hope to enable biologists and computational researchers to explore their data.
+cellxgene Desktop (pronounced "cell-by-gene") is an interactive data explorer for single-cell datasets, such as those coming from the [Human Cell Atlas](https://humancellatlas.org). Leveraging modern web development techniques to enable fast visualizations of at least 1 million cells, we hope to enable biologists and computational researchers to explore their data.
 
-Whether you need to visualize one thousand cells or one million, CELLxGENE Annotate helps you gain insight into your single-cell data.
+Whether you need to visualize one thousand cells or one million, cellxgene Desktop helps you gain insight into your single-cell data.
 
 <img src="https://github.com/chanzuckerberg/cellxgene/raw/main/docs/images/crossfilter.gif" width="350" height="200" hspace="30"><img src="https://github.com/chanzuckerberg/cellxgene/raw/main/docs/images/category-breakdown.gif" width="350" height="200" hspace="30">
 
 # Getting started
 
-### The comprehensive guide to CZ CELLxGENE Annotate
+### The comprehensive guide to cellxgene Desktop
 
-[The CZ CELLxGENE Annotate documentation is your one-stop-shop for information about CELLxGENE Annotate](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/README.md)! You may be particularly interested in:
+[The cellxgene documentation is your one-stop-shop for information about cellxgene Desktop](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/README.md)! You may be particularly interested in:
 
-- Seeing [what Annotate can do](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/explore-data/explorer-tutorials.md)
+- Seeing [what cellxgene Desktop can do](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/explore-data/explorer-tutorials.md)
-- Learning more about Annotate [installation](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/install.md) and [usage](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/quick-start.md#quick-start-1)
+- Learning more about cellxgene [installation](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/install.md) and [usage](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/quick-start.md#quick-start-1)
-- [Preparing your own data](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/data-reqs.md) for use in Annotate
+- [Preparing your own data](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/data-reqs.md) for use in cellxgene Desktop
 - Checking out [our roadmap](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/roadmap.md) for future development
-- [Contributing](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/contribute.md) to Annotate
+- [Contributing](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/contribute.md) to cellxgene Desktop
 
 ### Quick start
 
-To install CELLxGENE Annotate you need Python 3.10+. We recommend [installing Annotate into a conda or virtual environment.](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/install.md)
+To install cellxgene Desktop you need Python 3.6+. We recommend [installing cellxgene Desktop into a conda or virtual environment.](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/install.md)
 
 Install the package.
 
@@ -35,19 +35,19 @@ Install the package.
 pip install cellxgene
 ```
 
-Launch Annotate with an example [anndata](https://anndata.readthedocs.io/en/latest/) file
+Launch cellxgene Desktop with an example [anndata](https://anndata.readthedocs.io/en/latest/) file
 
 ```bash
 cellxgene launch https://cellxgene-example-data.czi.technology/pbmc3k.h5ad
 ```
 
-To explore more datasets already formatted for Annotate, check out the [Demo data](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/quick-start.md#example-datasets) or
+To explore more datasets already formatted for cellxgene Desktop, check out the [Demo data](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/quick-start.md#example-datasets) or
 see [Preparing your data](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/data-reqs.md) to learn more about formatting your own
-data for CELLxGENE Annotate.
+data for cellxgene Desktop.
 
 ### Supported browsers
 
-CELLxGENE Annotate currently supports the following browsers:
+cellxgene Desktop currently supports the following browsers:
 
 - Google Chrome 61+
 - Edge 15+
@@ -58,36 +58,30 @@ Please [file an issue](https://github.com/chanzuckerberg/cellxgene/issues/new/ch
 ### Finding help
 
 We'd love to hear from you!
-For questions, suggestions, or accolades, join the `#cellxgene-users` channel on the [CZI Science Community Slack](https://czi.co/science-slack) and say "hi!".
+For questions, suggestions, or accolades, [join the `#cellxgene-users` channel on the CZI Science Slack](https://join-cellxgene-users.herokuapp.com/) and say "hi!".
 
 For any errors, [report bugs on Github](https://github.com/chanzuckerberg/cellxgene/issues).
 
-# Developing with CZ CELLxGENE Annotate
+# Developing with cellxgene Desktop
 
 ### Contributing
 
-We warmly welcome contributions from the community! Please see our [contributing guide](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/contribute.md) and don't hesitate to open an issue or send a pull request to improve CELLxGENE Annotate. Please see the [dev_docs](https://github.com/chanzuckerberg/cellxgene/tree/main/dev_docs) for pull request suggestions, unit test details, local documentation preview, and other development specifics.
+We warmly welcome contributions from the community! Please see our [contributing guide](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/contribute.md) and don't hesitate to open an issue or send a pull request to improve cellxgene Desktop. Please see the [dev_docs](https://github.com/chanzuckerberg/cellxgene/tree/main/dev_docs) for pull request suggestions, unit test details, local documentation preview, and other development specifics. 
 
 This project adheres to the Contributor Covenant [code of conduct](https://github.com/chanzuckerberg/.github/blob/master/CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to opensource@chanzuckerberg.com.
 
 ### Reuse
 
-This project was started with the sole goal of empowering the scientific community to explore and understand their data.
+This project was started with the sole goal of empowering the scientific community to explore and understand their data. 
-As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from
+As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from 
 this project. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT).
 
-Before extending CELLxGENE Annotate, we encourage you to reach out to us with ideas or questions. It might be possible that an
-extension could be directly contributed, which would make it available for a wider audience, or that it's on our
-[roadmap](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/roadmap.md) and under active development.
 
-See the [CELLxGENE extensions](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/community-extensions.md) section of our documentation for examples of community use and CELLxGENE extensions.
+Before extending cellxgene, we encourage you to reach out to us with ideas or questions. It might be possible that an 
-
+extension could be directly contributed, which would make it available for a wider audience, or that it's on our 
-### Trademarks
+[roadmap](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/roadmap.md) and under active development. 
-
-CZ CELLXGENE, CZ CELLXGENE DISCOVER, and CZ CELLXGENE ANNOTATE are trademarks of the Chan Zuckerberg Initiative. All rights reserved.
-
-Use, reuse, modification, and re-distribution of the source code in this repository is subject to the terms of the applicable open source [license](LICENSE.txt). However, that license does not grant permission to use the trademarks without separate, express permission from the Chan Zuckerberg Initiative.
 
+See the [cellxgene extensions](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/community-extensions.md) section of our documentation for examples of community use and cellxgene extensions. 
 
 ### Security
 

SECURITY.md

@@ -1,3 +0,0 @@
-# Reporting Security Issues
-
-If you believe you have found a security issue, please responsibly disclose by contacting us at [security@chanzuckerberg.com](mailto:security@chanzuckerberg.com).

client/.nvmrc

@@ -1 +0,0 @@
-18.17.0

client/__tests__/e2e/__snapshots__/e2e.test.js.snap

@@ -2,4 +2,4 @@
 
 exports[`did launch page launched 1`] = `"<span style=\\"max-width: 155px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">pbm</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">c3k</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">c3k</span></span></span>"`;
 
-exports[`metadata loads categories and values from dataset appear 1`] = `"<div style=\\"display: flex; justify-content: space-between; align-items: baseline;\\"><div style=\\"display: flex; justify-content: flex-start; align-items: flex-start;\\"><label class=\\"bp3-control bp3-checkbox\\" for=\\"category-select-louvain\\"><input id=\\"category-select-louvain\\" data-testclass=\\"category-select\\" data-testid=\\"louvain:category-select\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span role=\\"menuitem\\" tabindex=\\"0\\" data-testclass=\\"category-expand\\" data-testid=\\"louvain:category-expand\\" style=\\"cursor: pointer;\\"><span aria-haspopup=\\"true\\" class=\\"bp3-popover2-target\\"><span data-testid=\\"louvain:category-label\\" tabindex=\\"-1\\" aria-label=\\"louvain\\" class=\\"\\" style=\\"max-width: 265px;\\"><span style=\\"max-width: 265px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">lou</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">vain</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">vain</span></span></span></span></span><svg stroke=\\"currentColor\\" fill=\\"currentColor\\" stroke-width=\\"0\\" viewBox=\\"0 0 320 512\\" data-testclass=\\"category-expand-is-not-expanded\\" height=\\"1em\\" width=\\"1em\\" xmlns=\\"http://www.w3.org/2000/svg\\" style=\\"font-size: 10px; margin-left: 5px;\\"><path d=\\"M285.476 272.971L91.132 467.314c-9.373 9.373-24.569 9.373-33.941 0l-22.667-22.667c-9.357-9.357-9.375-24.522-.04-33.901L188.505 256 34.484 101.255c-9.335-9.379-9.317-24.544.04-33.901l22.667-22.667c9.373-9.373 24.569-9.373 33.941 0L285.475 239.03c9.373 9.372 9.373 24.568.001 33.941z\\"></path></svg></span></div><div><span class=\\"bp3-popover-wrapper\\"><span aria-haspopup=\\"true\\" class=\\"bp3-popover-target\\"><a role=\\"button\\" data-testclass=\\"colorby\\" data-testid=\\"colorby-louvain\\" class=\\"bp3-button\\" tabindex=\\"0\\"><span icon=\\"tint\\" aria-hidden=\\"true\\" tabindex=\\"0\\" class=\\"bp3-icon bp3-icon-tint\\"><svg data-icon=\\"tint\\" width=\\"16\\" height=\\"16\\" viewBox=\\"0 0 16 16\\"><path d=\\"M7.88 1s-4.9 6.28-4.9 8.9c.01 2.82 2.34 5.1 4.99 5.1 2.65-.01 5.03-2.3 5.03-5.13C12.99 7.17 7.88 1 7.88 1z\\" fill-rule=\\"evenodd\\"></path></svg></span></a></span></span></div></div><div style=\\"margin-left: 26px;\\"></div>"`;
+exports[`metadata loads categories and values from dataset appear 1`] = `"<div style=\\"display: flex; justify-content: space-between; align-items: baseline;\\"><div style=\\"display: flex; justify-content: flex-start; align-items: flex-start;\\"><label class=\\"bp3-control bp3-checkbox\\" for=\\"category-select-louvain\\"><input id=\\"category-select-louvain\\" data-testclass=\\"category-select\\" data-testid=\\"louvain:category-select\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span role=\\"menuitem\\" tabindex=\\"0\\" data-testclass=\\"category-expand\\" data-testid=\\"louvain:category-expand\\" style=\\"cursor: pointer;\\"><span aria-haspopup=\\"true\\" class=\\"bp3-popover2-target\\"><span data-testid=\\"louvain:category-label\\" tabindex=\\"-1\\" aria-label=\\"louvain\\" class=\\"\\" style=\\"max-width: 265px;\\"><span style=\\"max-width: 265px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">lou</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">vain</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">vain</span></span></span></span></span><svg stroke=\\"currentColor\\" fill=\\"currentColor\\" stroke-width=\\"0\\" viewBox=\\"0 0 320 512\\" data-testclass=\\"category-expand-is-not-expanded\\" height=\\"1em\\" width=\\"1em\\" xmlns=\\"http://www.w3.org/2000/svg\\" style=\\"font-size: 10px; margin-left: 5px;\\"><path d=\\"M285.476 272.971L91.132 467.314c-9.373 9.373-24.569 9.373-33.941 0l-22.667-22.667c-9.357-9.357-9.375-24.522-.04-33.901L188.505 256 34.484 101.255c-9.335-9.379-9.317-24.544.04-33.901l22.667-22.667c9.373-9.373 24.569-9.373 33.941 0L285.475 239.03c9.373 9.372 9.373 24.568.001 33.941z\\"></path></svg></span></div><div><span class=\\"bp3-popover-wrapper\\"><span aria-haspopup=\\"true\\" class=\\"bp3-popover-target\\"><a role=\\"button\\" data-testclass=\\"colorby\\" data-testid=\\"colorby-louvain\\" class=\\"bp3-button\\" tabindex=\\"0\\"><span icon=\\"tint\\" class=\\"bp3-icon bp3-icon-tint\\"><svg data-icon=\\"tint\\" width=\\"16\\" height=\\"16\\" viewBox=\\"0 0 16 16\\"><desc>tint</desc><path d=\\"M7.88 1s-4.9 6.28-4.9 8.9c.01 2.82 2.34 5.1 4.99 5.1 2.65-.01 5.03-2.3 5.03-5.13C12.99 7.17 7.88 1 7.88 1z\\" fill-rule=\\"evenodd\\"></path></svg></span></a></span></span></div></div><div style=\\"margin-left: 26px;\\"></div>"`;

client/__tests__/e2e/puppeteer.setup.js

@@ -13,7 +13,7 @@ import * as ENV_DEFAULT from "../../../environment.default.json";
 // a test can take more time to finish, so we don't want
 // jest to shut off the test too soon
 jest.setTimeout(2 * 60 * 1000);
-setDefaultOptions({ timeout: 60 * 1000 });
+setDefaultOptions({ timeout: 20 * 1000 });
 
 jest.retryTimes(ENV_DEFAULT.RETRY_ATTEMPTS);
 

client/configuration/babel/babel.dev.js

@@ -16,8 +16,8 @@ module.exports = {
     "@babel/plugin-proposal-function-bind",
     ["@babel/plugin-proposal-decorators", { legacy: true }],
     ["@babel/plugin-proposal-class-properties", { loose: true }],
-    ["@babel/plugin-transform-private-methods", { loose: true }],
+    ["@babel/plugin-proposal-private-methods", { loose: true }],
-    ["@babel/plugin-transform-private-property-in-object", { loose: true }],
+    ["@babel/plugin-proposal-private-property-in-object", { loose: true }],
     "@babel/plugin-proposal-export-namespace-from",
     "@babel/plugin-proposal-optional-chaining",
     "@babel/plugin-proposal-nullish-coalescing-operator",

client/configuration/babel/babel.prod.js

@@ -15,8 +15,8 @@ module.exports = {
     "@babel/plugin-proposal-function-bind",
     ["@babel/plugin-proposal-decorators", { legacy: true }],
     ["@babel/plugin-proposal-class-properties", { loose: true }],
-    ["@babel/plugin-transform-private-methods", { loose: true }],
+    ["@babel/plugin-proposal-private-methods", { loose: true }],
-    ["@babel/plugin-transform-private-property-in-object", { loose: true }],
+    ["@babel/plugin-proposal-private-property-in-object", { loose: true }],
     "@babel/plugin-proposal-export-namespace-from",
     "@babel/plugin-transform-react-constant-elements",
     "@babel/plugin-transform-runtime",

client/configuration/webpack/webpack.config.dev.js

@@ -1,6 +1,8 @@
 const path = require("path");
 const webpack = require("webpack");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
+const FaviconsWebpackPlugin = require("favicons-webpack-plugin");
+const ScriptExtHtmlWebpackPlugin = require("script-ext-html-webpack-plugin");
 const MiniCssExtractPlugin = require("mini-css-extract-plugin");
 
 const { merge } = require("webpack-merge");
@@ -9,7 +11,6 @@ const sharedConfig = require("./webpack.config.shared");
 const babelOptions = require("../babel/babel.dev");
 
 const fonts = path.resolve("src/fonts");
-const images = path.resolve("src/images");
 const nodeModules = path.resolve("node_modules");
 
 const devConfig = {
@@ -29,11 +30,11 @@ const devConfig = {
       {
         test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2|otf)$/i,
         loader: "file-loader",
-        include: [nodeModules, fonts, images],
+        include: [nodeModules, fonts],
         options: {
           name: "static/assets/[name].[ext]",
           // (thuang): This is needed to make sure @font url path is '/static/assets/'
-          publicPath: "..",
+          publicPath: "/",
         },
       },
     ],
@@ -43,6 +44,21 @@ const devConfig = {
       inject: true,
       template: path.resolve("index.html"),
     }),
+    new FaviconsWebpackPlugin({
+      logo: "./favicon.png",
+      prefix: "static/img/",
+      favicons: {
+        icons: {
+          android: false,
+          appleIcon: false,
+          appleStartup: false,
+          coast: false,
+          firefox: false,
+          windows: false,
+          yandex: false,
+        },
+      },
+    }),
     new MiniCssExtractPlugin({
       filename: "static/[name].css",
     }),
@@ -57,6 +73,9 @@ const devConfig = {
         CXG_SERVER_PORT: process.env.CXG_SERVER_PORT || "5005",
       }),
     }),
+    new ScriptExtHtmlWebpackPlugin({
+      async: "obsolete",
+    }),
   ],
   infrastructureLogging: {
     level: "warn",

client/configuration/webpack/webpack.config.prod.js

@@ -3,7 +3,9 @@ const webpack = require("webpack");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
 const { CleanWebpackPlugin } = require("clean-webpack-plugin");
 const TerserJSPlugin = require("terser-webpack-plugin");
-const CssMinimizerPlugin = require("css-minimizer-webpack-plugin");
+const CleanCss = require("clean-css");
+const OptimizeCSSAssetsPlugin = require("optimize-css-assets-webpack-plugin");
+const FaviconsWebpackPlugin = require("favicons-webpack-plugin");
 const MiniCssExtractPlugin = require("mini-css-extract-plugin");
 
 const { merge } = require("webpack-merge");
@@ -14,7 +16,6 @@ const CspHashPlugin = require("./cspHashPlugin");
 const sharedConfig = require("./webpack.config.shared");
 
 const fonts = path.resolve("src/fonts");
-const images = path.resolve("src/images");
 const nodeModules = path.resolve("node_modules");
 
 const prodConfig = {
@@ -28,8 +29,8 @@ const prodConfig = {
     minimize: true,
     minimizer: [
       new TerserJSPlugin({}),
-      new CssMinimizerPlugin({
+      new OptimizeCSSAssetsPlugin({
-        minify: CssMinimizerPlugin.cleanCssMinify,
+        cssProcessor: CleanCss,
       }),
     ],
   },
@@ -44,11 +45,11 @@ const prodConfig = {
       {
         test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2|otf)$/i,
         loader: "file-loader",
-        include: [nodeModules, fonts, images],
+        include: [nodeModules, fonts],
         options: {
           name: "static/assets/[name]-[contenthash].[ext]",
           // (thuang): This is needed to make sure @font url path is '../static/assets/'
-          publicPath: "..",
+          publicPath: "static/",
         },
       },
     ],
@@ -65,6 +66,21 @@ const prodConfig = {
       protectWebpackAssets: false,
       cleanAfterEveryBuildPatterns: ["main.js", "main.css"],
     }),
+    new FaviconsWebpackPlugin({
+      logo: "./favicon.png",
+      prefix: "static/assets/",
+      favicons: {
+        icons: {
+          android: false,
+          appleIcon: false,
+          appleStartup: false,
+          coast: false,
+          firefox: false,
+          windows: false,
+          yandex: false,
+        },
+      },
+    }),
     new MiniCssExtractPlugin({
       filename: "static/[name]-[contenthash].css",
     }),

client/configuration/webpack/webpack.config.shared.js

@@ -1,7 +1,9 @@
 const path = require("path");
 const fs = require("fs");
 const MiniCssExtractPlugin = require("mini-css-extract-plugin");
-const ObsoleteWebpackPlugin = require("webpack-obsolete-plugin");
+const ObsoleteWebpackPlugin = require("obsolete-webpack-plugin");
+// eslint-disable-next-line @blueprintjs/classes-constants -- incorrect match
+const ScriptExtHtmlWebpackPlugin = require("script-ext-html-webpack-plugin");
 
 const src = path.resolve("src");
 const nodeModules = path.resolve("node_modules");
@@ -65,5 +67,8 @@ module.exports = {
       template: obsoleteHTMLTemplate,
       promptOnNonTargetBrowser: false,
     }),
+    new ScriptExtHtmlWebpackPlugin({
+      async: "obsolete",
+    }),
   ],
 };

client/index.html

@@ -3,7 +3,7 @@
   <head>
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1" />
-    <title>CELL&times;GENE | Annotate</title>
+    <title>cell&times;gene</title>
     <style>
       html,
       body,

client/index_template.html

@@ -3,7 +3,7 @@
   <head>
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1" />
-    <title>CELL&times;GENE | Annotate</title>
+    <title>cell&times;gene</title>
     <style>
       html,
       body,

client/jest-puppeteer.config.js

@@ -14,7 +14,6 @@ const DEFAULT_LAUNCH_CONFIG = {
   headless: !isHeadful,
   args: ["--ignore-certificate-errors", "--ignore-ssl-errors"],
   ignoreHTTPSErrors: true,
-  timeout: 90000,
   defaultViewport: {
     width: 1280,
     height: 960,

client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cellxgene",
-  "version": "1.3.0",
+  "version": "1.0.0",
   "license": "MIT",
   "description": "cellxgene is a web application for the interactive exploration of single cell sequence data.",
   "repository": "https://github.com/chanzuckerberg/cellxgene",
@@ -18,8 +18,7 @@
   },
   "engineStrict": true,
   "engines": {
-    "npm": ">=9.6.7",
+    "npm": ">=3.0.0"
-    "node": "^18.17.0"
   },
   "eslintConfig": {
     "extends": "./configuration/eslint/eslint.js"
@@ -78,17 +77,16 @@
     "whatwg-fetch": "^3.2.0"
   },
   "devDependencies": {
-    "@babel/core": "^7.25.2",
+    "@babel/core": "^7.13.16",
     "@babel/plugin-proposal-class-properties": "^7.10.4",
     "@babel/plugin-proposal-decorators": "^7.13.15",
     "@babel/plugin-proposal-export-namespace-from": "^7.10.4",
     "@babel/plugin-proposal-function-bind": "^7.10.5",
     "@babel/plugin-proposal-nullish-coalescing-operator": "^7.10.4",
     "@babel/plugin-proposal-optional-chaining": "^7.10.4",
-    "@babel/plugin-transform-private-property-in-object": "^7.22.11",
     "@babel/plugin-transform-react-constant-elements": "^7.13.13",
     "@babel/plugin-transform-runtime": "^7.13.15",
-    "@babel/preset-env": "^7.22.20",
+    "@babel/preset-env": "^7.13.15",
     "@babel/preset-react": "^7.13.13",
     "@babel/register": "^7.13.16",
     "@babel/runtime": "^7.13.16",
@@ -101,12 +99,12 @@
     "cheerio": "^1.0.0-rc.6",
     "clean-css": "^5.1.2",
     "clean-webpack-plugin": "^4.0.0-alpha.0",
+    "codecov": "^3.7.1",
     "css-loader": "^5.2.4",
-    "css-minimizer-webpack-plugin": "^4.0.0",
     "eslint": "^7.24.0",
     "eslint-config-airbnb": "^18.2.0",
     "eslint-config-prettier": "^8.2.0",
-    "eslint-plugin-compat": "^4.2.0",
+    "eslint-plugin-compat": "^3.8.0",
     "eslint-plugin-eslint-comments": "^3.2.0",
     "eslint-plugin-filenames": "^1.3.2",
     "eslint-plugin-import": "^2.24.2",
@@ -116,6 +114,8 @@
     "eslint-plugin-react-hooks": "^4.0.8",
     "expect-puppeteer": "^5.0.0",
     "express": "^4.17.1",
+    "favicons": "^6.2.2",
+    "favicons-webpack-plugin": "^5.0.2",
     "file-loader": "^6.0.0",
     "html-webpack-plugin": "^5.3.1",
     "husky": "^7.0.2",
@@ -123,7 +123,7 @@
     "jest-circus": "^27.0.6",
     "jest-environment-puppeteer": "^5.0.1",
     "jest-fetch-mock": "^3.0.3",
-    "jest-puppeteer": "^6.2.0",
+    "jest-puppeteer": "^5.0.1",
     "json-loader": "^0.5.7",
     "lint-staged": "^10.2.11",
     "lodash": "^4.17.21",
@@ -133,16 +133,18 @@
     "lodash.map": "^4.6.0",
     "lodash.zip": "^4.2.0",
     "mini-css-extract-plugin": "^1.5.0",
+    "obsolete-webpack-plugin": "^0.5.6",
+    "optimize-css-assets-webpack-plugin": "^5.0.3",
     "prettier": "^2.0.5",
-    "puppeteer": "^10.4.0",
+    "puppeteer": "^8.0.0",
     "rimraf": "^3.0.2",
+    "script-ext-html-webpack-plugin": "^2.1.4",
     "serve-favicon": "^2.5.0",
     "terser-webpack-plugin": "^5.1.1",
-    "webpack": "^5.94.0",
+    "webpack": "^5.34.0",
     "webpack-cli": "^4.6.0",
     "webpack-dev-middleware": "^4.1.0",
-    "webpack-merge": "^5.0.9",
+    "webpack-merge": "^5.0.9"
-    "webpack-obsolete-plugin": "^1.0.5"
   },
   "jest": {
     "testMatch": [
@@ -176,13 +178,13 @@
             }
           ],
           [
-            "@babel/plugin-transform-private-methods",
+            "@babel/plugin-proposal-private-methods",
             {
               "loose": true
             }
           ],
           [
-            "@babel/plugin-transform-private-property-in-object",
+            "@babel/plugin-proposal-private-property-in-object",
             {
               "loose": true
             }

client/src/actions/index.js

@@ -8,6 +8,7 @@ import {
 import { loadUserColorConfig } from "../util/stateManager/colorHelpers";
 import * as selnActions from "./selection";
 import * as annoActions from "./annotation";
+import * as spatialActions from "./spatial";
 import * as viewActions from "./viewStack";
 import * as embActions from "./embedding";
 import * as genesetActions from "./geneset";
@@ -272,4 +273,5 @@ export default {
   genesetDelete: genesetActions.genesetDelete,
   genesetAddGenes: genesetActions.genesetAddGenes,
   genesetDeleteGenes: genesetActions.genesetDeleteGenes,
+  requestSpatialMetadata: spatialActions.requestSpatialMetadata,
 };

client/src/actions/spatial.js

@@ -0,0 +1,35 @@
+import * as globals from "../globals";
+
+export const requestSpatialMetadata = () => async (dispatch) => {
+  dispatch({ type: "request spatial metadata started" });
+  try {
+    const res = await fetch(
+      `${globals.API.prefix}${globals.API.version}spatial/meta`,
+      {
+        method: "GET",
+        headers: new Headers({
+          Accept: "application/json",
+          "Content-Type": "application/json",
+        }),
+        credentials: "include",
+      }
+    );
+
+    if (!res.ok || res.headers.get("Content-Type") !== "application/json") {
+      return null; // TODO need a dispatch //dispatchDiffExpErrors(dispatch, res);
+    }
+
+    const response = await res.json();
+
+    /* then send the success case action through */
+    return dispatch({
+      type: "request spatial metadata success",
+      data: response,
+    });
+  } catch (error) {
+    return dispatch({
+      type: "request spatial metadata error",
+      error,
+    });
+  }
+};

client/src/annoMatrix/viewCreators.js

@@ -58,11 +58,10 @@ function _maskToList(mask) {
   if (!mask) {
     return null;
   }
-  const [...m] = mask;
+  const list = new Int32Array(mask.length);
-  const list = new Int32Array(m.length);
   let elems = 0;
-  for (let i = 0, l = m.length; i < l; i += 1) {
+  for (let i = 0, l = mask.length; i < l; i += 1) {
-    if (m[i]) {
+    if (mask[i]) {
       list[elems] = i;
       elems += 1;
     }

client/src/annoMatrix/whereCache.js

@@ -52,7 +52,7 @@ import { _getColumnDimensionNames } from "./schema";
 import { _hashStringValues } from "./query";
 
 export function _whereCacheGet(whereCache, schema, field, query) {
-  /*
+  /* 
 	query will either be an where query (object) or a column name (string).
 
 	Return array of column labels or undefined.
@@ -91,9 +91,12 @@ export function _whereCacheCreate(field, query, columnLabels) {
 	*/
   if (typeof query !== "object") return null;
 
-  const { where, summarize } = query;
+  if (query.where) {
-  if (where) {
+    const {
-    const { field: queryField, column: queryColumn, value: queryValue } = where;
+      field: queryField,
+      column: queryColumn,
+      value: queryValue,
+    } = query.where;
     return {
       where: {
         [field]: {
@@ -104,13 +107,13 @@ export function _whereCacheCreate(field, query, columnLabels) {
       },
     };
   }
-  if (summarize) {
+  if (query.summarize) {
     const {
       method,
       field: queryField,
       column: queryColumn,
       values: queryValues,
-    } = summarize;
+    } = query.summarize;
     const queryValueHash = _hashStringValues(queryValues);
     return {
       summarize: {
@@ -169,6 +172,5 @@ function __whereCacheMerge(dst, src) {
 }
 
 export function _whereCacheMerge(...caches) {
-  // eslint-disable-next-line compat/compat -- not using web APIs
   return caches.reduce(__whereCacheMerge, {});
 }

client/src/components/app.js

@@ -23,6 +23,8 @@ class App extends React.Component {
   componentDidMount() {
     const { dispatch } = this.props;
 
+    dispatch(actions.requestSpatialMetadata());
+
     /* listen for url changes, fire one when we start the app up */
     window.addEventListener("popstate", this._onURLChanged);
     this._onURLChanged();
@@ -41,7 +43,7 @@ class App extends React.Component {
     const { loading, error, graphRenderCounter } = this.props;
     return (
       <Container>
-        <Helmet title="CELL&times;GENE | Annotate" />
+        <Helmet title="cellxgene" />
         {loading ? (
           <div
             style={{

client/src/components/embedding/index.js

@@ -16,10 +16,11 @@ import actions from "../../actions";
 import { getDiscreteCellEmbeddingRowIndex } from "../../util/stateManager/viewStackHelpers";
 
 @connect((state) => ({
-    layoutChoice: state.layoutChoice, // TODO: really should clean up naming, s/layout/embedding/g
+  imageUnderlay: state.imageUnderlay,
-    schema: state.annoMatrix?.schema,
+  layoutChoice: state.layoutChoice, // TODO: really should clean up naming, s/layout/embedding/g
-    crossfilter: state.obsCrossfilter,
+  schema: state.annoMatrix?.schema,
-  }))
+  crossfilter: state.obsCrossfilter,
+}))
 class Embedding extends React.PureComponent {
   constructor(props) {
     super(props);
@@ -27,8 +28,18 @@ class Embedding extends React.PureComponent {
   }
 
   handleLayoutChoiceChange = (e) => {
-    const { dispatch } = this.props;
+    const { dispatch, imageUnderlay } = this.props;
     dispatch(actions.layoutChoiceAction(e.currentTarget.value));
+
+    // if we just switched off spatial, if the image is on, turn it off
+    if (
+      imageUnderlay.isActive &&
+      e.target.value !== globals.spatialEmbeddingKeyword
+    ) {
+      dispatch({
+        type: "toggle image underlay",
+      });
+    }
   };
 
   render() {

client/src/components/framework/logo.js

@@ -1,15 +1,17 @@
 import React from "react";
-import icon from "../../images/icon.png";
+import * as globals from "../../globals";
 
 const Logo = (props) => {
   const { size } = props;
   return (
-    <img
+    <svg width={size} height={size} viewBox="0 0 48 48" fill="none">
-      src={icon}
+      <rect width="48" height="48" fill="white" />
-      height={size}
+      <rect width="48" height="48" fill={globals.logoColor} />
-      width={size}
+      <rect x="19" y="19" width="22" height="22" fill="white" />
-      alt="CELLxGENE Annotate Logo"
+      <rect x="24" y="24" width="12" height="12" fill={globals.logoColor} />
-     />
+      <rect x="7" y="19" width="7" height="22" fill="white" />
+      <rect x="19" y="7" width="22" height="7" fill="white" />
+    </svg>
   );
 };
 

client/src/components/graph/drawSpatialImageRegl.js

@@ -0,0 +1,62 @@
+export default function drawSpatialImageRegl(regl) {
+  return regl({
+    frag: `
+        precision mediump float;
+
+        // our texture
+        uniform sampler2D u_image;
+
+        // the texCoords passed in from the vertex shader.
+        varying vec2 v_texCoord;
+
+        void main() {
+            gl_FragColor = texture2D(u_image, v_texCoord);
+        }`,
+
+    vert: `
+        attribute vec2 a_position;
+        attribute vec2 a_texCoord;
+
+        uniform vec2 u_resolution;
+
+        uniform mat3 projView;
+
+        varying vec2 v_texCoord;
+
+        void main() {
+            // convert the rectangle from pixels to 0.0 to 1.0
+            vec3 pos = vec3(a_position, 1.);
+            vec2 zeroToOne = pos.xy / u_resolution;
+
+            // convert from 0->1 to 0->2
+            vec2 zeroToTwo = zeroToOne * 2.0;
+
+            // convert from 0->2 to -1->+1 (clipspace)
+            vec2 clipSpace = zeroToTwo - 1.0;
+
+            vec3 pos2 = projView * vec3(clipSpace, 1.);
+
+            gl_Position = vec4(pos2.xy , 0, 1);
+
+            // pass the texCoord to the fragment shader
+            // The GPU will interpolate this value between points.
+            v_texCoord = a_texCoord;
+        }`,
+
+    attributes: {
+      a_texCoord: [0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 1.0, 1.0, 0.0, 1.0, 1.0],
+      a_position: regl.prop("rectCoords"),
+    },
+
+    uniforms: {
+      projView: regl.prop("projView"),
+      u_image: regl.prop("spatialImageAsTexture"),
+      color: [1, 0, 0, 1],
+      u_resolution: [regl.prop("imageWidth"), regl.prop("imageHeight")],
+      image_width: regl.prop("imageWidth"),
+      //   translate:
+    },
+
+    count: 6,
+  });
+}

client/src/components/graph/graph.js

@@ -14,6 +14,7 @@ import {
   createColorTable,
   createColorQuery,
 } from "../../util/stateManager/colorHelpers";
+import _drawSpatialImage from "./drawSpatialImageRegl";
 import * as globals from "../../globals";
 
 import GraphOverlayLayer from "./overlays/graphOverlayLayer";
@@ -77,6 +78,8 @@ function createModelTF() {
   colors: state.colors,
   pointDilation: state.pointDilation,
   genesets: state.genesets.genesets,
+  spatial: state.spatial.metadata,
+  imageUnderlay: state.imageUnderlay,
 }))
 class Graph extends React.Component {
   static createReglState(canvas) {
@@ -87,6 +90,7 @@ class Graph extends React.Component {
     const camera = _camera(canvas);
     const regl = _regl(canvas);
     const drawPoints = _drawPoints(regl);
+    const drawSpatialImage = _drawSpatialImage(regl);
 
     // preallocate webgl buffers
     const pointBuffer = regl.buffer();
@@ -100,6 +104,7 @@ class Graph extends React.Component {
       pointBuffer,
       colorBuffer,
       flagBuffer,
+      drawSpatialImage,
     };
   }
 
@@ -232,6 +237,8 @@ class Graph extends React.Component {
       pointBuffer: null,
       colorBuffer: null,
       flagBuffer: null,
+      drawSpatialImage: null,
+      spatial: null,
 
       // component rendering derived state - these must stay synchronized
       // with the reducer state they were generated from.
@@ -317,7 +324,10 @@ class Graph extends React.Component {
     if (e.type !== "wheel") e.preventDefault();
     if (camera.handleEvent(e, projectionTF)) {
       this.renderCanvas();
-      this.setState((state) => ({ ...state, updateOverlay: !state.updateOverlay }));
+      this.setState((state) => ({
+        ...state,
+        updateOverlay: !state.updateOverlay,
+      }));
     }
   };
 
@@ -509,6 +519,14 @@ class Graph extends React.Component {
     return { toolSVG: newToolSVG, tool, container };
   };
 
+  loadTextureFromUrl = (src) =>
+    new Promise((resolve, reject) => {
+      const img = new Image();
+      img.onload = () => resolve(img);
+      img.onerror = reject;
+      img.src = src;
+    });
+
   fetchAsyncProps = async (props) => {
     const {
       annoMatrix,
@@ -517,6 +535,8 @@ class Graph extends React.Component {
       crossfilter,
       pointDilation,
       viewport,
+      spatial,
+      imageUnderlay,
     } = props.watchProps;
     const { modelTF } = this.state;
 
@@ -524,7 +544,8 @@ class Graph extends React.Component {
       annoMatrix,
       layoutChoice,
       colorsProp,
-      pointDilation
+      pointDilation,
+      imageUnderlay
     );
 
     const { currentDimNames } = layoutChoice;
@@ -551,6 +572,10 @@ class Graph extends React.Component {
       pointDilationLabel
     );
 
+    this.spatialImage = await this.loadTextureFromUrl(
+      "/api/v0.2/spatial/image"
+    );
+
     const { width, height } = viewport;
     return {
       positions,
@@ -558,6 +583,8 @@ class Graph extends React.Component {
       flags,
       width,
       height,
+      spatial,
+      imageUnderlay,
     };
   };
 
@@ -721,6 +748,7 @@ class Graph extends React.Component {
       flagBuffer,
       camera,
       projectionTF,
+      drawSpatialImage,
     } = this.state;
     this.renderPoints(
       regl,
@@ -729,12 +757,14 @@ class Graph extends React.Component {
       pointBuffer,
       flagBuffer,
       camera,
-      projectionTF
+      projectionTF,
+      drawSpatialImage
     );
   });
 
   updateReglAndRender(asyncProps, prevAsyncProps) {
-    const { positions, colors, flags, height, width } = asyncProps;
+    const { positions, colors, flags, height, width, imageUnderlay } =
+      asyncProps;
     this.cachedAsyncProps = asyncProps;
     const { pointBuffer, colorBuffer, flagBuffer } = this.state;
     let needToRenderCanvas = false;
@@ -754,6 +784,9 @@ class Graph extends React.Component {
       flagBuffer({ data: flags, dimension: 1 });
       needToRenderCanvas = true;
     }
+    if (imageUnderlay !== prevAsyncProps?.imageUnderlay) {
+      needToRenderCanvas = true;
+    }
     if (needToRenderCanvas) this.renderCanvas();
   }
 
@@ -797,20 +830,25 @@ class Graph extends React.Component {
     pointBuffer,
     flagBuffer,
     camera,
-    projectionTF
+    projectionTF,
+    drawSpatialImage
   ) {
-    const { annoMatrix } = this.props;
+    const { annoMatrix, spatial, imageUnderlay } = this.props;
     if (!this.reglCanvas || !annoMatrix) return;
 
     const { schema } = annoMatrix;
     const cameraTF = camera.view();
     const projView = mat3.multiply(mat3.create(), projectionTF, cameraTF);
     const { width, height } = this.reglCanvas;
+    const imW = spatial.data.imageWidth;
+    const imH = spatial.data.imageHeight;
+
     regl.poll();
     regl.clear({
       depth: 1,
-      color: [1, 1, 1, 1],
+      color: [0, 0, 0, 0],
     });
+
     drawPoints({
       distance: camera.distance(),
       color: colorBuffer,
@@ -821,6 +859,19 @@ class Graph extends React.Component {
       nPoints: schema.dataframe.nObs,
       minViewportDimension: Math.min(width, height),
     });
+    if (imageUnderlay?.isActive) {
+      drawSpatialImage({
+        projView,
+        imageWidth: imW,
+        imageHeight: imH,
+        rectCoords: [0, 0, imW, 0, 0, imH, 0, imH, imW, 0, imW, imH],
+        spatialImageAsTexture: regl.texture({
+          data: this.spatialImage,
+          wrapS: "clamp",
+          wrapT: "clamp",
+        }),
+      });
+    }
     regl._gl.flush();
   }
 
@@ -832,6 +883,8 @@ class Graph extends React.Component {
       layoutChoice,
       pointDilation,
       crossfilter,
+      spatial,
+      imageUnderlay,
     } = this.props;
     const { modelTF, projectionTF, camera, viewport, regl } = this.state;
     const cameraTF = camera?.view()?.slice();
@@ -902,6 +955,8 @@ class Graph extends React.Component {
             pointDilation,
             crossfilter,
             viewport,
+            spatial,
+            imageUnderlay,
           }}
         >
           <Async.Pending initial>
@@ -951,32 +1006,29 @@ const ErrorLoading = ({ displayName, error, width, height }) => {
   );
 };
 
-const StillLoading = ({ displayName, width, height }) => 
+const StillLoading = ({ displayName, width, height }) => (
   /*
   Render a busy/loading indicator
   */
-   (
+  <div
+    style={{
+      position: "fixed",
+      fontWeight: 500,
+      top: height / 2,
+      width,
+    }}
+  >
     <div
       style={{
-        position: "fixed",
+        display: "flex",
-        fontWeight: 500,
+        justifyContent: "center",
-        top: height / 2,
+        justifyItems: "center",
-        width,
+        alignItems: "center",
       }}
     >
-      <div
+      <Button minimal loading intent="primary" />
-        style={{
+      <span style={{ fontStyle: "italic" }}>Loading {displayName}</span>
-          display: "flex",
-          justifyContent: "center",
-          justifyItems: "center",
-          alignItems: "center",
-        }}
-      >
-        <Button minimal loading intent="primary" />
-        <span style={{ fontStyle: "italic" }}>Loading {displayName}</span>
-      </div>
     </div>
-  )
+  </div>
-;
+);
-
 export default Graph;

client/src/components/graph/overlays/centroidLabels.js

@@ -150,7 +150,7 @@ class CentroidLabels extends PureComponent {
                   dilatedValue={dilatedValue}
                   coords={coords}
                   inverseTransform={inverseTransform}
-                  opacity={selected ? 1 : deselectOpacity}
+                  opactity={selected ? 1 : deselectOpacity}
                   colorAccessor={colorAccessor}
                   displayLabel={displayLabel}
                   onMouseEnter={this.handleMouseEnter}
@@ -205,7 +205,7 @@ const Label = ({
           fontWeight,
           fill: "black",
           userSelect: "none",
-          opacity,
+          opacity: { opacity },
         }}
         onMouseEnter={(e) => onMouseEnter(e, colorAccessor, label)}
         onMouseOut={(e) => onMouseOut(e, colorAccessor, label)}

client/src/components/leftSidebar/infoMenu.js

@@ -16,7 +16,7 @@ const InformationMenu = React.memo((props) => {
             rel="noopener"
           />
           <MenuItem
-            href="https://czi.co/science-slack"
+            href="https://join-cellxgene-users.herokuapp.com/"
             target="_blank"
             icon="chat"
             text="Chat"

client/src/components/menubar/index.js

@@ -28,6 +28,8 @@ import { getEmbSubsetView } from "../../util/stateManager/viewStackHelpers";
     subsetPossible,
     subsetResetPossible,
     graphInteractionMode: state.controls.graphInteractionMode,
+    imageUnderlay: state.imageUnderlay,
+    layoutChoice: state.layoutChoice, // TODO: really should clean up naming, s/layout/embedding/g
     clipPercentileMin: Math.round(100 * (annoMatrix?.clipRange?.[0] ?? 0)),
     clipPercentileMax: Math.round(100 * (annoMatrix?.clipRange?.[1] ?? 1)),
     userDefinedGenes: state.controls.userDefinedGenes,
@@ -206,6 +208,8 @@ class MenuBar extends React.PureComponent {
       colorAccessor,
       subsetPossible,
       subsetResetPossible,
+      imageUnderlay,
+      layoutChoice,
     } = this.props;
     const { pendingClipPercentiles } = this.state;
 
@@ -268,6 +272,29 @@ class MenuBar extends React.PureComponent {
             disabled={!isColoredByCategorical}
           />
         </Tooltip>
+        {layoutChoice?.available?.includes(globals.spatialEmbeddingKeyword) && (
+          <ButtonGroup className={styles.menubarButton}>
+            <Tooltip
+              content={"Toggle image"}
+              position="bottom"
+              hoverOpenDelay={globals.tooltipHoverOpenDelay}
+            >
+              <AnchorButton
+                type="button"
+                data-testid="toggle-image-underlay"
+                icon={"media"}
+                intent={imageUnderlay.isActive ? "primary" : "none"}
+                active={imageUnderlay.isActive}
+                onClick={() => {
+                  dispatch({
+                    type: "toggle image underlay",
+                  });
+                }}
+              />
+            </Tooltip>
+          </ButtonGroup>
+        )}
+
         <ButtonGroup className={styles.menubarButton}>
           <Tooltip
             content={selectionTooltip}

client/src/globals.js

@@ -2,6 +2,9 @@ import { Colors } from "@blueprintjs/core";
 import { dispatchNetworkErrorMessageToUser } from "./util/actionHelpers";
 import ENV_DEFAULT from "../../environment.default.json";
 
+// visium embedding word, spatial image underlay
+export const spatialEmbeddingKeyword = "spatial";
+
 /* overflow category values are created  using this string */
 export const overflowCategoryLabel = ": all other labels";
 

client/src/reducers/imageUnderlay.js

@@ -0,0 +1,14 @@
+const imageUnderlay = (state = { isActive: false }, action) => {
+  switch (action.type) {
+    case "toggle image underlay":
+      return {
+        ...state,
+        isActive: !state.isActive,
+      };
+
+    default:
+      return state;
+  }
+};
+
+export default imageUnderlay;

client/src/reducers/index.js

@@ -11,6 +11,7 @@ import continuousSelection from "./continuousSelection";
 import graphSelection from "./graphSelection";
 import colors from "./colors";
 import differential from "./differential";
+import spatial from "./spatial";
 import layoutChoice from "./layoutChoice";
 import controls from "./controls";
 import annotations from "./annotations";
@@ -19,6 +20,7 @@ import genesetsUI from "./genesetsUI";
 import autosave from "./autosave";
 import centroidLabels from "./centroidLabels";
 import pointDialation from "./pointDilation";
+import imageUnderlay from "./imageUnderlay";
 import { gcMiddleware as annoMatrixGC } from "../annoMatrix";
 
 import undoableConfig from "./undoableConfig";
@@ -38,7 +40,9 @@ const Reducer = undoable(
     ["colors", colors],
     ["controls", controls],
     ["differential", differential],
+    ["spatial", spatial],
     ["centroidLabels", centroidLabels],
+    ["imageUnderlay", imageUnderlay],
     ["pointDilation", pointDialation],
     ["autosave", autosave],
   ]),
@@ -51,6 +55,7 @@ const Reducer = undoable(
     "colors",
     "controls",
     "differential",
+    "spatial",
     "layoutChoice",
     "centroidLabels",
     "genesets",

client/src/reducers/spatial.js

@@ -0,0 +1,34 @@
+const Spatial = (
+  state = {
+    loading: null,
+    error: null,
+    metadata: null,
+  },
+  action
+) => {
+  switch (action.type) {
+    case "request spatial metadata started":
+      return {
+        ...state,
+        loading: true,
+        error: null,
+      };
+    case "request spatial metadata success":
+      return {
+        ...state,
+        error: null,
+        loading: false,
+        metadata: action,
+      };
+    case "request spatial metadata error":
+      return {
+        ...state,
+        loading: false,
+        error: action.data,
+      };
+    default:
+      return state;
+  }
+};
+
+export default Spatial;

client/src/reducers/undoableConfig.js

@@ -52,6 +52,9 @@ const skipOnActions = new Set([
   "geneset: disable add new genes mode",
   "geneset: activate rename geneset mode",
   "geneset: disable rename geneset mode",
+
+  /* spatial */
+  "toggle image underlay",
 ]);
 
 /*

client/src/util/stateManager/viewStackHelpers.js

@@ -137,13 +137,10 @@ function _getEmbeddingRowOffsets(baseRowIndex, embeddingDf) {
     - if the embedding contains NaN coordinates, return a rowIndex
       that contains only the rows with discrete valued coordinates.
 
-  Currently assumes that there will be only two dimensions in the embedding.
+  Currently assumes that there will be onl two dimensions in the embedding.
   */
-  // eslint-disable-next-line react/destructuring-assignment -- destructuring fails
   const X = embeddingDf.icol(0).asArray();
-  // eslint-disable-next-line react/destructuring-assignment -- destructuring fails
   const Y = embeddingDf.icol(1).asArray();
-
   const offsets = new Int32Array(X.length);
   let numOffsets = 0;
 

dev_docs/developer_guidelines.md

@@ -3,7 +3,7 @@
 ## Requirements
 
 - npm
-- Python 3.10+
+- Python 3.6+
 - Chrome
 
 [See dev section of README](../README.md)
@@ -148,6 +148,6 @@ If you would like to run the smoke tests against a hot-reloaded version of the c
 
 ### Tips
 
-- You can also install/launch the server side code from npm scrips (requires python3.10 with virtualenv) with the `scripts/backend_dev` script.
+- You can also install/launch the server side code from npm scrips (requires python3.6 with virtualenv) with the `scripts/backend_dev` script.
 
-- Check out [e2e Tests](e2e_tests.md) for more details
+- Check out [e2e Tests](e2e_tests.md) for more details

dev_docs/developer_scripts.md

@@ -11,10 +11,9 @@ $PROJECT_ROOT`.
 ### Build
 
 **Usage:** from the `$PROJECT_ROOT` directory run:
-
+* `make build` builds whole app client and server
-- `make build` builds whole app client and server
+* `make build-client` runs webpack build
-- `make build-client` runs webpack build
+* `make build-for-server-dev` builds client and copies output directly into
-- `make build-for-server-dev` builds client and copies output directly into
   source tree (only for server devlopment)
 
 ### Clean
@@ -22,19 +21,17 @@ $PROJECT_ROOT`.
 Deletes generated files.
 
 **Usage:** from the `$PROJECT_ROOT` directory run:
-
+* `make clean` cleans everything including node modules (means build with take
-- `make clean` cleans everything including node modules (means build with take
   a while
-- `make clean-lite` cleans built directories
+* `make clean-lite` cleans built directories
-- `make clean-server` cleans source tree
+* `make clean-server` cleans source tree
 
 ### Distribution
 
 Creates distribution for python module to upload to pypi.
 
 **Usage:** from the `$PROJECT_ROOT` directory run:
-
+* `make pydist` builds code and then builds sdist
-- `make pydist` builds code and then builds sdist
 
 ### Release
 
@@ -45,18 +42,16 @@ See `release_process.md`.
 Installs requirements files.
 
 **Usage:** from the `$PROJECT_ROOT` directory run:
-
+* `make dev-env` installs requirements and requirments-dev (for building code)
-- `make dev-env` installs requirements and requirments-dev (for building code)
 
 ### Installing cellxgene packages
 
 **Usage:** from the `$PROJECT_ROOT` directory:
-
+* `install-dev` - installs from local source tree
-- `install-dev` - installs from local source tree
+* `install-release-test` - installs from test pypi
-- `install-release-test` - installs from test pypi
+* `install-release` - installs from pypi
-- `install-release` - installs from pypi
+* `install-dist` - installs from local dist folder
-- `install-dist` - installs from local dist folder
+* `uninstall` - uninstalls cellxgene
-- `uninstall` - uninstalls cellxgene
 
 ## Client-level scripts
 
@@ -67,9 +62,8 @@ Installs requirements files.
 **About** Serve the current client javascript independently from the `server` code.
 
 **Requires**
-
+* The server to be running. Best way to do this is with [backend_dev](#backend_dev).
-- The server to be running. Best way to do this is with [backend_dev](#backend_dev).
+* `make ci` to install the necessary node modules
-- `make ci` to install the necessary node modules
 
 **Usage:** from the `$PROJECT_ROOT/client` directory run `make start-frontend`
 
@@ -81,24 +75,23 @@ the FE developer gets the current version of the backend with a single command
 and no knowledge of python necessary. It creates and activates a virtual
 environment and installs cellxgene from the current branch.
 
-**Requires** `Python3.10+`, `virtual-env`, `pip`
+**Requires** `Python3.6+`, `virtual-env`, `pip`
 
 **Usage:** from the `$PROJECT_ROOT` directory run `./scripts/backend_dev`
 
 **Options:**
-
+* In parallel, you can then launch the node development server to serve the
-- In parallel, you can then launch the node development server to serve the
   current state of the FE with [`start-frontend`](#start-frontend), usually in
   a different terminal tab.
-- You can also select a specific dataset using `DATASET=<dataset path> ./scripts/backend_dev`.
+* You can also select a specific dataset using `DATASET=<dataset path> ./scripts/backend_dev`.
-- You can also use `CXG_OPTIONS` to pass options to the `cellxgene launch`
+* You can also use `CXG_OPTIONS` to pass options to the `cellxgene launch`
   command, as in `CXG_OPTIONS='--disable-annotations' ./scripts/backend_dev`.
 
 **Breakdown**
 
 | command                                  | purpose                                                    |
 | ---------------------------------------- | ---------------------------------------------------------- |
-| python3.12 -m venv cellxgene             | creates cellxgene virtual environment                      |
+| python3.6 -m venv cellxgene              | creates cellxgene virtual environment                      |
 | source cellxgene/bin/activate            | activates virtual environment                              |
 | yes \| pip uninstall cellxgene \|\| true | uninstalls cellxgene (if installed)                        |
 | pip install -e .                         | installs current local version of cellxgene                |
@@ -109,15 +102,14 @@ environment and installs cellxgene from the current branch.
 Methods used to test the client javascript code
 
 **Usage:** from the `$PROJECT_ROOT/client` directory run:
-
+* `make unit-test` Runs all unit tests. It excludes any tests in the e2e
-- `make unit-test` Runs all unit tests. It excludes any tests in the e2e
   folder. This is used by travis to run unit tests.
-- `make smoke-test` Starts backend development server and runs end to end
+* `make smoke-test` Starts backend development server and runs end to end
   tests. This is what travis runs. It depends on the `e2e` and the
   `backend-dev` targets. One starts the server, the other runs the tests. If
   developing a front-end feature and just checking if tests pass, this is
   probabaly the one you want to run.
-- `npm run e2e` Runs backend tests without starting the server. You will need to
+* `npm run e2e` Runs backend tests without starting the server. You will need to
   start the rest api separately with the pbmc3k.h5ad file. Note you can use
   the `JEST_ENV` environment variable to change how JEST runs in the browser.
   The test runs against `localhost:3000` by default. You can use the

dev_docs/release_process.md

@@ -26,14 +26,14 @@ Steps must be run from the project directory and in a virtual env with all the d
 3. In the release branch, run `make create-release-candidate PART=[major | minor | patch]`. This will bump the version and create a release *candidate* version (e.g. `0.3.0-rc.0`).
 4. Commit changes, push the new branch to origin and open a `DO NOT MERGE` draft PR, which will run tests on your branch. We will use this PR later
 5. Upload the release candidate to Test PyPI by running the command `make release-candidate-to-test-pypi`. (Make sure you are registered for PyPI and Test PyPI and you have write access to the cellxgene PyPI package for both).
-6. Verify the release candidate in a fresh virtual environment by running `VERSION=<X>.<Y>.<Z>rc.<#> make install-release-test` which installs the cellxgene build you just uploaded to Test PyPI (note that the version value does not include a dash `-`!). The PM should do this too. Note that you may need to run `hash -r` to ensure the cellxgene executable that was just installed is found in your shell path.
+6. Verify the release candidate in a fresh virtual environment by running `make install-release-test` which installs the cellxgene build you just uploaded to Test PyPI. The PM should do this too.
 7. If you find errors with the release candidate, fix them in main, rebase, and run `make recreate-release-candidate` to increment the release candidate version (i.e. `0.3.0-rc.0` -> `0.3.0-rc.1`). Then go back to Steps 5 and 6 to re-upload and re-test the new release candidate.
 8. If everything looks good, push the release to Test PyPI without the release candidate tag by running the command `make release-final-to-test-pypi` (i.e. `0.3.0-rc.1` -> `0.3.0`).
     - **NOTE:** Once you push the final release version to Test PyPI, you cannot ever re-upload the build again. If you need to make changes to the build, you will have to "burn" the version number and bump the part again and go back to step 1 with a brand new version number. For example, if you upload `0.3.0` to Test PyPI and realize there's a bug, you will have to create a new version `0.4.0` and there will be no `0.3.0` version of cellxgene. This is why testing the release candidate is very important.
 9. Publish the open draft PR for the release and conduct a PR review.
 10. Merge to the `main` branch.
 11. Publish to PyPI (prod) (assuming you that you have registered for PyPI, and that you have write access to the cellxgene pypi package) by running `make release-final`.
-12. Test the installation in a fresh virtual environment by running `pip install --no-cache-dir cellxgene`. Note that you may need to run `hash -r` to ensure the cellxgene executable that was just installed is found in your shell path.
+12. Test the installation in a fresh virtual environment by running `pip install --no-cache-dir cellxgene`.
 13. Create Github release using the version number and release notes ([instructions](https://help.github.com/articles/creating-releases/)):
      - Draft new release
      - Type version name matching release version number from (1)

scripts/launch_and_open

@@ -1,16 +0,0 @@
-#!/usr/bin/expect -f
-
-# Mac only! (depends upon `open` command)
-
-set h5ad [lindex $argv 0]
-puts "$h5ad"
-
-spawn cellxgene launch $h5ad
-
-set timeout 10
-expect -indices -re "Please go to (http:\/\/localhost:\[0-9\]+)" {
-    set url $expect_out(1,string)
-    exec >@stdout 2>@stderr open $url
-}
-
-interact

server/__init__.py

@@ -2,7 +2,7 @@ import logging
 import sys
 from server.common.utils.utils import import_plugins
 
-__version__ = "1.3.0"
+__version__ = "1.0.0"
 display_version = "cellxgene v" + __version__
 
 try:

server/annotate/annotation_types.py

@@ -1,5 +0,0 @@
-from enum import Enum
-
-
-class AnnotationType(Enum):
-    CELL_TYPE = "cell_type"

server/app/app.py

@@ -48,12 +48,12 @@ def _cache_control(always, **cache_kwargs):
 
 
 def cache_control(**cache_kwargs):
-    """config driven"""
+    """ config driven """
     return _cache_control(False, **cache_kwargs)
 
 
 def cache_control_always(**cache_kwargs):
-    """always generate headers, regardless of the config"""
+    """ always generate headers, regardless of the config """
     return _cache_control(True, **cache_kwargs)
 
 
@@ -190,6 +190,16 @@ class SummarizeVarAPI(Resource):
     def post(self, data_adaptor):
         return common_rest.summarize_var_post(request, data_adaptor)
 
+class SpatialImageAPI(Resource):
+    @rest_get_data_adaptor
+    def get(self, data_adaptor):
+        return common_rest.spatial_image_get(request, data_adaptor)
+
+class SpatialMetaAPI(Resource):
+    @rest_get_data_adaptor
+    def get(self, data_adaptor):
+        return data_adaptor.get_spatial_metadata()
+
 
 def get_api_base_resources(bp_base):
     """Add resources that are accessed from the api url"""
@@ -222,13 +232,16 @@ def get_api_dataroot_resources(bp_dataroot):
     # Computation routes
     add_resource(DiffExpObsAPI, "/diffexp/obs")
     add_resource(LayoutObsAPI, "/layout/obs")
+    # Spatial routes
+    add_resource(SpatialImageAPI, "/spatial/image")
+    add_resource(SpatialMetaAPI, "/spatial/meta")
     return api
 
 
 class Server:
     @staticmethod
     def _before_adding_routes(app, app_config):
-        """will be called before routes are added, during __init__.  Subclass protocol"""
+        """ will be called before routes are added, during __init__.  Subclass protocol """
         pass
 
     def __init__(self, app_config):

server/app/session.py

@@ -6,7 +6,7 @@ CXGUID = "cxguid"
 
 
 def get_user_id(session: SessionMixin) -> str:
-    """Gets a session-persistent user id. Creates one in the Flask session if non-extant"""
+    """ Gets a session-persistent user id. Creates one in the Flask session if non-extant """
     if CXGUID not in session:
         session[CXGUID] = uuid4().hex
         session.permanent = True

server/cli/annotate.py

@@ -1,240 +0,0 @@
-import functools
-import json
-import os.path
-import shlex
-import shutil
-import subprocess
-from os.path import isfile
-from subprocess import STDOUT, PIPE
-from tempfile import NamedTemporaryFile
-
-import click
-import pandas as pd
-from click import BadParameter
-
-from server.annotate.annotation_types import AnnotationType
-from server.common.utils.data_locator import DataLocator
-from server.common.utils.utils import sort_options
-
-
-def annotate_args(func):
-    @functools.wraps(func)
-    def wrapper(*args, **kwargs):
-        return func(*args, **kwargs)
-
-    return wrapper
-
-
-@sort_options
-@click.command(options_metavar="<options>")
-@click.argument(
-    "input_h5ad_file",
-    type=click.Path(exists=True, dir_okay=False, readable=True),
-    nargs=1,
-    metavar="<path to H5AD input file>",
-    required=True,
-)
-@click.option(
-    "-m",
-    "--model-url",
-    # Making this a required "option", rather than an "argument", since we support automatic model selection in the
-    # future, in which case the user would not need to specify this option at all and we can make it optional at
-    # that time.
-    required=True,
-    help="The URL of the model used to prediction annotated labels. May be a local filesystem directory "
-    "or S3 path (s3://)",
-)
-@click.option(
-    "-o",
-    "--output-h5ad-file",
-    default="",
-    help="The output H5AD file that will contain the generated annotation values. If this option is not provided, "
-    "the input file will be overwritten to include the new annotations; in this case you must specify "
-    "--overwrite.",
-    metavar="<filename>",
-)
-@click.option(
-    "--overwrite",
-    default=False,
-    is_flag=True,
-    help="Allow overwriting of the specified H5AD output file, if it exists. For safety, you must specify this "
-    "flag if the specified output file already exists or if the --output-h5ad-file option is not provided.",
-    show_default=True,
-)
-@click.option(
-    "-l",
-    "--counts-layer",
-    help="If specified, raw counts will be read from the AnnData layer of the specified name. If unspecified, "
-    "raw counts will be read from `X` matrix, unless 'raw.X' exists, in which case that will be used.",
-)
-@click.option(
-    "-g",
-    "--gene-column-name",
-    help="The name of the `var` column that contains gene names. The values in this column will be used to match "
-    "genes between the query and reference datasets. If not specified, the gene names are expected to exist "
-    "in `var.index`.",
-)
-# TODO: Useful if we want to support discoverability of models
-# @click.option(
-#     "-r",
-#     "--model-repository",
-#     help="The base URL of the model repository. Maybe a local filesystem directory or S3 path (s3://)"
-# )
-# TODO: Useful if we want to support other, future annotation types, beyond "Cell Type". Currently hidden
-@click.option(
-    "-a",
-    "--annotation-type",
-    type=click.Choice([t.value for t in AnnotationType]),
-    default=AnnotationType.CELL_TYPE.value,
-    show_default=True,
-    hidden=True,  # Remove if we add support for more annotation types
-    help="The type of annotation to perform. This model to be used will be inferred from the annotation type.",
-)
-@click.option(
-    "-c",
-    "--annotation-prefix",
-    type=str,
-    default="cxg",
-    show_default=True,
-    help="An optional prefix used to form the names of: 1) new `obs` annotation columns that will store the predicted "
-    "annotation values and confidence scores, 2) `obsm` embeddings (reference and umap embedding), and "
-    "3) `uns` metadata for the prediction operation",
-)
-@click.option(
-    "-n",
-    "--run-name",
-    type=str,
-    help="An optional run name that will be used as a suffix to form the names of new `obs` annotation columns that "
-    "will store the predicted annotation values and confidence scores. This can be used to allow multiple "
-    "annotation predictions to be run on a single AnnData object.",
-)
-@click.option("--use-model-cache/--no-use-model-cache", default=True)
-@click.option(
-    "--use-gpu/--no-use-gpu",
-    default=True,
-    help="Whether to use a GPU for annotation operations (highly recommended, if available).",
-)
-# TODO: This is a cell type model-specific arg, so not ideal to specify here as a hardcoded option
-@click.option(
-    "--classifier",
-    default="default",
-    help="For cell type annotation, the classifier level to use. The classifier is model-dependent, so refer to "
-    "documentation for the specified model for valid values.",
-)
-# TODO: This is a cell type model-specific arg, so not ideal to specify here as a hardcoded option
-@click.option(
-    "--organism",
-    type=click.Choice(["Homo sapiens", "Mus musculus"], case_sensitive=True),
-    default="Homo sapiens",
-    help="For cell type annotation, the organism of the dataset. Used to normalize gene names to HGLC conventions when "
-    "an annotation model has been trained using data from different organism.",
-)
-@click.option(
-    "--model-cache-dir",
-    default=".models_cache",
-    help="Local directory used to store model files that are retrieved from a remote location. Model files will "
-    "be read from this directory first, if they exist, to avoid repeating large downloads.",
-)
-@click.option(
-    "--mlflow-env-manager",
-    type=click.Choice(["virtualenv", "conda", "local"]),
-    default="virtualenv",
-    help="Annotation model prediction will be installed and executed in the specified type of environment. MacOS users "
-    "on Apple Silicon (arm64, M1, M2, etc.) are recommended to use 'conda' to avoid Python package installation "
-    "errors. If 'conda' is specified then cellxgene must also have been installed within a conda environment",
-)
-@click.help_option("--help", "-h", help="Show this message and exit.")
-def annotate(**cli_args):
-    """
-    Add predicted annotations to an H5AD file. Run `cellxgene annotate --help` for more information.
-    """
-    _validate_options(cli_args)
-
-    print(f"Reading query dataset {cli_args['input_h5ad_file']}...")
-
-    annotation_prefix = "_".join(
-        filter(None, [cli_args.get("annotation_prefix"), cli_args.get("annotation_type"), cli_args.get("run_name")])
-    )
-
-    output_h5ad_file = (
-        cli_args["input_h5ad_file"]
-        if cli_args["overwrite"] and not cli_args["output_h5ad_file"]
-        else cli_args["output_h5ad_file"]
-    )
-
-    model_url = cli_args.get("model_url")
-    local_model_path = _retrieve_model(cli_args.get("model_cache_dir"), model_url, cli_args.get("use_model_cache"))
-
-    print(f"Annotating {cli_args.get('input_h5ad_file')} with {cli_args.get('annotation_type')}...")
-
-    if cli_args["annotation_type"] == AnnotationType.CELL_TYPE.value:
-        predict_args = dict(
-            query_dataset_h5ad_path=cli_args.get("input_h5ad_file"),
-            output_h5ad_path=output_h5ad_file,
-            annotation_prefix=annotation_prefix,
-            counts_layer=cli_args.get("counts_layer"),
-            gene_column_name=cli_args.get("gene_column_name"),
-            classifier=cli_args.get("classifier"),
-            organism=cli_args.get("organism"),
-            use_gpu=cli_args.get("use_gpu"),
-        )
-        # Drop args that have values of `None` as these will cause problems when passing into MLflow predict, since it
-        # ultimately gets converted into 1-row Pandas DataFrame (None is interpreted as a float type column!)
-        predict_args = dict([(k, v) for k, v in predict_args.items() if v is not None])
-
-        # Invoke prediction using MLflow cli, as a separate process.
-        # This fully prepares the Python environment that is needed for executing the model.
-        # The Python environment will be reused after it is setup once.
-        with NamedTemporaryFile(buffering=0) as predict_args_file:
-            # write the mlflow predict arguments to a csv file, which will be passed to mlflow cmd
-            pd.DataFrame([json.dumps(predict_args)]).to_csv(predict_args_file, index=None)
-            predict_args_file.seek(0)
-
-            # run mlflow prediction in subprocess
-            predict_cmd = (
-                f"mlflow models predict "
-                f"--env-manager {cli_args['mlflow_env_manager']} "
-                f"--model-uri {local_model_path} "
-                f"--content-type csv --input-path {predict_args_file.name}"
-            )
-            p = subprocess.Popen(
-                args=shlex.split(predict_cmd), stdin=predict_args_file, text=True, bufsize=0, stdout=PIPE, stderr=STDOUT
-            )
-
-            # display mlflow process output as it runs
-            for line in p.stdout:
-                print(line.rstrip())
-
-            p.wait()
-            if p.returncode == 0:
-                print(f"Wrote annotations to {output_h5ad_file}")
-            else:
-                print("Annotation failed!")
-    else:
-        raise BadParameter(f"unknown annotation type {cli_args['annotation_type']}")
-
-
-def _retrieve_model(model_cache_dir, model_url, use_cache=True):
-    local_cache_model_path = os.path.join(model_cache_dir, os.path.splitext(os.path.basename(model_url))[0])
-    if not os.path.exists(local_cache_model_path) or not use_cache:
-        print(f"Retrieving model from {model_url}")
-        # download from remote source
-        with DataLocator(model_url).local_handle() as model_archive_local_path:
-            # unpack archive to local cache dir
-            shutil.unpack_archive(model_archive_local_path, local_cache_model_path)
-    else:
-        print(f"Using cached model at {local_cache_model_path}")
-
-    return local_cache_model_path
-
-
-def _validate_options(cli_args):
-    output = cli_args["output_h5ad_file"]
-    overwrite = cli_args["overwrite"]
-
-    if isfile(output) and not overwrite:
-        raise click.UsageError(f"Cannot overwrite existing file {output}, try using the flag --overwrite")
-
-
-if __name__ == "__main__":
-    annotate()

server/cli/cli.py

@@ -1,6 +1,5 @@
 import click
 
-from .annotate import annotate
 from .launch import launch
 from .prepare import prepare
 from .upgrade import log_upgrade_check
@@ -32,5 +31,4 @@ def cli(upgrade_check):
 
 
 cli.add_command(launch)
-cli.add_command(annotate)
 cli.add_command(prepare)

server/cli/prepare.py

@@ -128,12 +128,12 @@ def prepare(
             raise click.FileError(data, hint="not a valid file or path")
 
         if not set_obs_names == "":
-            if set_obs_names not in list(adata.obs.keys()):
+            if set_obs_names not in adata.obs_keys():
-                raise click.UsageError(f"obs {set_obs_names} not found, options are: {list(adata.obs.keys())}")
+                raise click.UsageError(f"obs {set_obs_names} not found, options are: {adata.obs_keys()}")
             adata.obs_names = adata.obs[set_obs_names]
         if not set_var_names == "":
-            if set_var_names not in list(adata.var.keys()):
+            if set_var_names not in adata.var_keys():
-                raise click.UsageError(f"var {set_var_names} not found, options are: {list(adata.var.keys())}")
+                raise click.UsageError(f"var {set_var_names} not found, options are: {adata.var_keys()}")
             adata.var_names = adata.var[set_var_names]
         if make_obs_names_unique:
             adata.obs.index = make_index_unique(adata.obs.index)

server/common/annotations/annotations.py

@@ -57,7 +57,7 @@ class Annotations(metaclass=ABCMeta):
         pass
 
     @abstractmethod
-    def write_gene_sets(self, gs, tid, data_adaptor):
+    def write_gene_sets(self, gs, data_adaptor):
         """Write the gene sets (gs) to a persistent storage such that it can later be read"""
         pass
 

server/common/annotations/local_file_csv.py

@@ -7,7 +7,6 @@ from hashlib import blake2b
 
 import pandas as pd
 from flask import session
-from fsspec import AbstractFileSystem
 
 from server import __version__ as cellxgene_version
 from server.app.session import get_user_id
@@ -63,27 +62,21 @@ class AnnotationsLocalFile(Annotations):
         self.check_user_annotations_enabled()  # raises
 
         fname = self._get_celllabels_filename(data_adaptor)
-        empty_labels = pd.DataFrame()
-        if fname is None:
-            return empty_labels
-
         with self.label_lock:
-            locator = DataLocator(fname)
+            if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0:
-            if not locator.exists() or locator.size() == 0:
+                # returned the cached labels if possible, otherwise read them from the file
-                return empty_labels
+                if fname == self.last_label_fname:
-
+                    return self.last_labels
-            # return the cached labels if possible
+                else:
-            if fname == self.last_label_fname:
+                    labels = pd.read_csv(
-                return self.last_labels
+                        fname, dtype="category", index_col=0, header=0, comment="#", keep_default_na=False
-
+                    )
-            # otherwise, read labels from file
+                    # update the cache
-            with locator.open() as f:
+                    self.last_label_fname = fname
-                labels = pd.read_csv(f, dtype="category", index_col=0, header=0, comment="#", keep_default_na=False)
+                    self.last_labels = labels
-
+                    return labels
-            # update the cache
+            else:
-            self.last_label_fname = fname
+                return pd.DataFrame()
-            self.last_labels = labels
-            return labels
 
     def write_labels(self, df, data_adaptor):
         self.check_user_annotations_enabled()  # raises
@@ -102,12 +95,13 @@ class AnnotationsLocalFile(Annotations):
 
             fname = self._get_celllabels_filename(data_adaptor)
             self._backup(fname)
-            locator = DataLocator(fname)
+            if not df.empty:
-            with locator.open("w") as f:
+                with open(fname, "w", newline="") as f:
-                if not df.empty:
                     if header is not None:
                         f.write(header)
                     df.to_csv(f)
+            else:
+                open(fname, "w").close()
 
             # update the cache
             self.last_label_fname = fname
@@ -115,37 +109,31 @@ class AnnotationsLocalFile(Annotations):
 
     def read_gene_sets(self, data_adaptor, context=None):
         fname = self._get_genesets_filename(data_adaptor)
-        empty_gene_sets = {}
+        gene_sets = {}
-
+        tid = None
         with self.gene_sets_lock:
             tid = self.last_geneset_tid  # inside the critical section
-            if fname is None:
+            if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0:
-                return (empty_gene_sets, tid)
+                # return the cached genesets if possible, otherwise read from file and validate them
+                if fname == self.last_geneset_fname:
+                    gene_sets = self.last_geneset
+                else:
+                    # read
+                    gene_sets = read_gene_sets_tidycsv(DataLocator(fname), context)
 
-            locator = DataLocator(fname)
+                    # validate
-            if not locator.exists() or locator.size() == 0:
+                    gene_sets = data_adaptor.check_new_gene_sets(gene_sets, context)
-                return (empty_gene_sets, tid)
 
-            # return the cached genesets if possible, otherwise read from file and validate them
+                    # update cache
-            if fname == self.last_geneset_fname:
+                    self.last_geneset_fname = fname
-                return (self.last_geneset, tid)
+                    self.last_geneset = gene_sets
 
-            # read
+        return (gene_sets, tid)
-            gene_sets = read_gene_sets_tidycsv(locator, context)
-
-            # validate
-            gene_sets = data_adaptor.check_new_gene_sets(gene_sets, context)
-
-            # update cache
-            self.last_geneset_fname = fname
-            self.last_geneset = gene_sets
-
-            return (gene_sets, tid)
 
     def write_gene_sets(self, gene_sets, tid, data_adaptor):
         self.check_gene_sets_save_enabled()  # raises
 
-        if type(tid) is not int or tid < 0:
+        if type(tid) != int or tid < 0:
             raise ValueError("tid must be a positive integer")
 
         # may raise
@@ -169,13 +157,13 @@ class AnnotationsLocalFile(Annotations):
 
             fname = self._get_genesets_filename(data_adaptor)
             self._backup(fname)
-            locator = DataLocator(fname)
+            with open(fname, "w", newline="") as f:
-            with locator.open("w", newline="") as f:
+                f.write(header)
-                f.write(header + self.gene_sets_to_csv(gene_sets))
+                f.write(self.gene_sets_to_csv(gene_sets))
 
             # update the cache
             self.last_geneset_fname = fname
-            self.last_geneset = gene_sets if isinstance(gene_sets, dict) else {g["geneset_name"]: g for g in gene_sets}
+            self.last_geneset = gene_sets if type(gene_sets) == dict else {g["geneset_name"]: g for g in gene_sets}
 
     def _get_userdata_idhash(self, data_adaptor):
         """
@@ -193,7 +181,7 @@ class AnnotationsLocalFile(Annotations):
 
         output_file = self.label_output_file or self.gene_sets_output_file
         if output_file:
-            return os.path.dirname(DataLocator(output_file).abspath())
+            return os.path.dirname(os.path.abspath(output_file))
 
         return os.getcwd()
 
@@ -232,37 +220,34 @@ class AnnotationsLocalFile(Annotations):
             1. fname -> backup_dir/fname-TIME
             2. delete excess files in backup_dir
         """
-        locator = DataLocator(fname)
+        root, ext = os.path.splitext(fname)
-        fs: AbstractFileSystem = locator.fs  # Handle to underlying fsspec file system
-
-        # Make sure there is work to do
-        if not locator.exists():
-            return
-
-        root, ext = os.path.splitext(locator.abspath())
         backup_dir = f"{root}-backups"
 
+        # Make sure there is work to do
+        if not os.path.exists(fname):
+            return
+
         # Ensure backup_dir exists
-        fs.mkdirs(backup_dir, exist_ok=True)
+        if not os.path.exists(backup_dir):
+            os.mkdir(backup_dir)
 
         # Save current file to backup_dir
         fname_base = os.path.basename(fname)
         fname_base_root, fname_base_ext = os.path.splitext(fname_base)
-        # don't use ISO standard time format, as it contains characters illegal on some filesystems.
+        # don't use ISO standard time format, as it contains characters illegal on some filesytems.
         nowish = datetime.now().strftime("%Y-%m-%dT%H-%M-%S")
         backup_fname = os.path.join(backup_dir, f"{fname_base_root}-{nowish}{fname_base_ext}")
-        if fs.exists(backup_fname):
+        if os.path.exists(backup_fname):
-            fs.delete(backup_fname)
+            os.remove(backup_fname)
-        fs.rename(fname, backup_fname)
+        os.rename(fname, backup_fname)
 
         # prune the backup_dir to max number of backup files, keeping the most recent backups
-        backup_path_prefix = DataLocator.strip_protocol(os.path.join(backup_dir, fname_base_root + "-"))
+        backups = list(filter(lambda s: s.startswith(fname_base_root), os.listdir(backup_dir)))
-        backups = list(filter(lambda s: s.startswith(backup_path_prefix), fs.ls(backup_dir)))
+        excess_count = len(backups) - max_backups
-
+        if excess_count > 0:
-        # sorting to drop the oldest
+            backups.sort()
-        excess_backups = list(sorted(backups, reverse=True))[max_backups:]
+            for bu in backups[0:excess_count]:
-        for bu in excess_backups:
+                os.remove(os.path.join(backup_dir, bu))
-            fs.delete(bu)
 
     def update_parameters(self, parameters, data_adaptor):
         params = {}

server/common/colors.py

@@ -228,6 +228,6 @@ def convert_anndata_category_colors_to_cxg_category_colors(data):
 
         # create the cellxgene color entry for this category
         cxg_colors[category_name] = dict(
-            zip(data.obs[category_name].astype('category').cat.categories, [convert_color_to_hex_format(c) for c in data.uns[uns_key]])
+            zip(data.obs[category_name].cat.categories, [convert_color_to_hex_format(c) for c in data.uns[uns_key]])
         )
     return cxg_colors

server/common/compute/diffexp_generic.py

@@ -56,7 +56,7 @@ def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp
 
     # degrees of freedom for Welch's t-test
     with np.errstate(divide="ignore", invalid="ignore"):
-        dof = sum_vn**2 / (vnA**2 / (nA - 1) + vnB**2 / (nB - 1))
+        dof = sum_vn ** 2 / (vnA ** 2 / (nA - 1) + vnB ** 2 / (nB - 1))
     dof[np.isnan(dof)] = 1
 
     # Welch's t-test score calculation

server/common/compute/estimate_distribution.py

@@ -97,7 +97,7 @@ def estimate_approximate_distribution(X) -> XApproximateDistribution:
     if Xdata.size > CHUNKSIZE:
         min_val = max_val = Xdata[0]
         with concurrent.futures.ThreadPoolExecutor() as tp:
-            for _min, _max in tp.map(min_max, [Xdata[i : i + CHUNKSIZE] for i in range(0, Xdata.size, CHUNKSIZE)]):
+            for (_min, _max) in tp.map(min_max, [Xdata[i : i + CHUNKSIZE] for i in range(0, Xdata.size, CHUNKSIZE)]):
                 min_val = min(_min, min_val)
                 max_val = max(_max, max_val)
 

server/common/config/__init__.py

@@ -1,2 +1,2 @@
 DEFAULT_SERVER_PORT = 5005
-BIG_FILE_SIZE_THRESHOLD = 100 * 2**20  # 100MB
+BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20  # 100MB

server/common/config/app_config.py

@@ -19,6 +19,7 @@ class AppConfig(object):
     """
 
     def __init__(self):
+
         # the default configuration (see default_config.py)
         # TODO @madison -- if we always read from the default config (hard coded path) can we set those values as
         #  defaults within the config class?

server/common/config/base_config.py

@@ -50,7 +50,7 @@ class BaseConfig(object):
                     f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}"
                 )
         else:
-            if type(val) is not vtype:
+            if type(val) != vtype:
                 raise ConfigurationError(
                     f"Invalid type for attribute: {attrname}, "
                     f"expected type {vtype.__name__}, got {type(val).__name__}"
@@ -70,7 +70,7 @@ class BaseConfig(object):
             if not hasattr(self, key):
                 raise ConfigurationError(f"unknown config parameter {key}.")
             try:
-                if type(value) is tuple:
+                if type(value) == tuple:
                     # convert tuple values to list values
                     value = list(value)
                 setattr(self, key, value)

server/common/config/dataset_config.py

@@ -4,7 +4,6 @@ from os.path import splitext, isdir
 from server.common.annotations.local_file_csv import AnnotationsLocalFile
 from server.common.config.base_config import BaseConfig
 from server.common.errors import ConfigurationError, AnnotationsError
-from server.common.utils.data_locator import DataLocator
 from server.data_common.matrix_loader import MatrixDataLoader
 
 
@@ -128,15 +127,11 @@ class DatasetConfig(BaseConfig):
             if lf_ext and lf_ext != ".csv":
                 raise ConfigurationError(f"genesets file type must be .csv: {genesets_filename}")
 
-        if dirname is not None:
+        if dirname is not None and not isdir(dirname):
-            if not DataLocator(dirname).islocal():
+            try:
-                # remote object stores only support objects but not directories, do nothing
+                os.mkdir(dirname)
-                pass
+            except OSError:
-            elif not isdir(dirname):
+                raise ConfigurationError("Unable to create directory specified by --user-generated-data-dir")
-                try:
-                    os.mkdir(dirname)
-                except OSError:
-                    raise ConfigurationError("Unable to create directory specified by --user-generated-data-dir")
 
         anno_config = {
             "user-annotations": self.user_annotations__enable,
@@ -176,7 +171,7 @@ class DatasetConfig(BaseConfig):
         self.validate_correct_type_of_configuration_attribute("diffexp__top_n", int)
 
         data_adaptor = self.get_data_adaptor()
-        if self.diffexp__enable and data_adaptor.parameters.get("diffexp-may-be-slow", False):
+        if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False):
             context["messagefn"](
                 "CAUTION: due to the size of your dataset, " "running differential expression may take longer or fail."
             )

server/common/config/external_config.py

@@ -29,7 +29,7 @@ class ExternalConfig(BaseConfig):
             if name is None:
                 raise ConfigurationError("environment: 'name' is missing")
             required = envdict.get("required", False)
-            if type(required) is not bool:
+            if type(required) != bool:
                 raise ConfigurationError("environment: 'required' must be a bool")
             path = envdict.get("path")
             if path is None:

server/common/corpora.py

@@ -22,7 +22,7 @@ def corpora_get_versions_from_anndata(adata):
     """
 
     # per Corpora AnnData spec, this is a corpora file if the following is true
-    if "version" not in list(adata.uns.keys()):
+    if "version" not in adata.uns_keys():
         return None
     version = adata.uns["version"]
     if not isinstance(version, collections.abc.Mapping) or "corpora_schema_version" not in version:

server/common/fbs/matrix.py

@@ -19,7 +19,7 @@ import server.common.fbs.NetEncoding.Uint32Array as Uint32Array
 
 # Serialization helper
 def serialize_column(builder, typed_arr):
-    """Serialize NetEncoding.Column"""
+    """ Serialize NetEncoding.Column """
 
     (u_type, u_value) = typed_arr
     Column.ColumnStart(builder)
@@ -30,7 +30,7 @@ def serialize_column(builder, typed_arr):
 
 # Serialization helper
 def serialize_matrix(builder, n_rows, n_cols, columns, col_idx):
-    """Serialize NetEncoding.Matrix"""
+    """ Serialize NetEncoding.Matrix """
 
     Matrix.MatrixStart(builder)
     Matrix.MatrixAddNRows(builder, n_rows)

server/common/genesets.py

@@ -136,7 +136,7 @@ def write_gene_sets_tidycsv(f, genesets):
 
 
 def summarizeQueryHash(raw_query):
-    """generate a cache key (hash) from the raw query string"""
+    """ generate a cache key (hash) from the raw query string """
     return hashlib.sha1(raw_query).hexdigest()
 
 
@@ -187,7 +187,7 @@ def validate_gene_sets(genesets, var_names, context=None):
     # 1. check gene set character set and format
     illegal_name = re.compile(r"^\s|  |[\u0000-\u001F\u007F-\uFFFF]|\s$")
     for name in geneset_names:
-        if type(name) is not str or len(name) == 0:
+        if type(name) != str or len(name) == 0:
             raise KeyError("Gene set names must be non-null string.")
         if illegal_name.search(name):
             messagefn(

server/common/rest.py

@@ -4,9 +4,10 @@ import sys
 from http import HTTPStatus
 import zlib
 import json
+import numpy as np
 
-from flask import make_response, jsonify, current_app, abort
+from flask import make_response, jsonify, current_app, abort, send_file
-from urllib.parse import unquote
+from werkzeug.urls import url_unquote
 
 from server.common.config.client_config import get_client_config
 from server.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg
@@ -64,22 +65,22 @@ def _query_parameter_to_filter(args):
             axis, name = key.split(":")
             if axis not in ("obs", "var"):
                 raise FilterError("unknown filter axis")
-            name = unquote(name)
+            name = url_unquote(name)
             current = filters[axis].setdefault(name, {"name": name})
 
             val_split = value.split(",")
             if len(val_split) == 1:
                 if "min" in current or "max" in current:
                     raise FilterError("do not mix range and value filters")
-                value = unquote(value)
+                value = url_unquote(value)
                 values = current.setdefault("values", [])
                 values.append(value)
 
             elif len(val_split) == 2:
                 if len(current) > 1:
                     raise FilterError("duplicate range specification")
-                min = unquote(val_split[0])
+                min = url_unquote(val_split[0])
-                max = unquote(val_split[1])
+                max = url_unquote(val_split[1])
                 if min != "*":
                     current["min"] = float(min)
                 if max != "*":
@@ -293,7 +294,7 @@ def layout_obs_get(request, data_adaptor):
 
     try:
         return make_response(
-            data_adaptor.layout_to_fbs_matrix(fields), HTTPStatus.OK, {"Content-Type": "application/octet-stream"}
+            data_adaptor.layout_to_fbs_matrix(fields, data_adaptor.get_spatial()), HTTPStatus.OK, {"Content-Type": "application/octet-stream"}
         )
     except (KeyError, DatasetAccessError) as e:
         return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
@@ -379,7 +380,7 @@ def summarize_var_helper(request, data_adaptor, key, raw_query):
             HTTPStatus.OK,
             {"Content-Type": "application/octet-stream"},
         )
-    except ValueError as e:
+    except (ValueError) as e:
         return abort(HTTPStatus.NOT_FOUND, description=str(e))
     except (UnsupportedSummaryMethod, FilterError) as e:
         return abort(HTTPStatus.BAD_REQUEST, description=str(e))
@@ -397,3 +398,40 @@ def summarize_var_post(request, data_adaptor):
 
     key = request.args.get("key", default=None)
     return summarize_var_helper(request, data_adaptor, key, request.get_data())
+
+def spatial_image_get(request, data_adaptor):
+    import io
+    import matplotlib.pyplot    
+
+    resolution = "hires"
+    spatial = data_adaptor.get_spatial()
+
+    if len(list(spatial)) == 0:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, "uns does not have spatial information")
+
+    library_id = list(spatial)[0]
+    if len(spatial) > 1:
+        current_app.logger.warning(f"More than one library found under uns.spatial, using library '{library_id}'")
+
+    if "images" not in spatial[library_id]:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, "spatial information does not contain images")
+
+    if resolution not in spatial[library_id]["images"]:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, f"spatial information does not contain requested resolution '{resolution}'")
+
+    response_image = io.BytesIO()
+    img = spatial[library_id]["images"][resolution]
+    matplotlib.pyplot.imsave(response_image, img)
+    response_image.seek(0)
+
+    try:
+        return send_file(response_image, attachment_filename=f"{library_id}-{resolution}.png", mimetype="image/png")
+    except (KeyError, DatasetAccessError) as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+    except PrepareError:
+        return abort_and_log(
+            HTTPStatus.NOT_IMPLEMENTED,
+            f"No spatial image available {request.path}",
+            loglevel=logging.ERROR,
+            include_exc_info=True,
+        ) 

server/common/utils/data_locator.py

@@ -52,10 +52,8 @@ class DataLocator:
             self.fs = fsspec.filesystem(self.protocol)
 
     def __repr__(self):
-        return (
+        return f"DataLocator(protocol={self.protocol}, cname={self.cname}, "
-            f"DataLocator(protocol={self.protocol}, cname={self.cname}, "
+        f"path={self.path}, uri_or_path={self.uri_or_path})"
-            f"path={self.path}, uri_or_path={self.uri_or_path})"
-        )
 
     @staticmethod
     def _get_protocol_and_path(uri_or_path):
@@ -67,10 +65,6 @@ class DataLocator:
                 return protocol, path
         return None, uri_or_path
 
-    @staticmethod
-    def strip_protocol(uri_or_path):
-        return DataLocator._get_protocol_and_path(uri_or_path)[1]
-
     def exists(self):
         return self.fs.exists(self.cname)
 
@@ -78,7 +72,7 @@ class DataLocator:
         return self.fs.size(self.cname)
 
     def lastmodtime(self):
-        """return datetime object representing last modification time, or None if unavailable"""
+        """ return datetime object representing last modification time, or None if unavailable """
         info = self.fs.info(self.cname)
         if self.islocal() and info is not None:
             return datetime.fromtimestamp(info["mtime"])
@@ -98,8 +92,8 @@ class DataLocator:
     def isfile(self):
         return self.fs.isfile(self.cname)
 
-    def open(self, *args, **kwargs):
+    def open(self, *args):
-        return self.fs.open(self.uri_or_path, *args, **kwargs)
+        return self.fs.open(self.uri_or_path, *args)
 
     def islocal(self):
         return self.protocol is None or self.protocol == "file"
@@ -113,9 +107,10 @@ class DataLocator:
         # do our best to create a file with the same.
         ext = os.path.splitext(self.path)
         suffix = None if ext[1] == "" else ext[1]
-        with tempfile.NamedTemporaryFile(prefix="cellxgene_", suffix=suffix, delete=False) as tmp:
+        with self.open() as src, tempfile.NamedTemporaryFile(prefix="cellxgene_", suffix=suffix, delete=False) as tmp:
-            self.fs.download(self.uri_or_path, tmp.name)
+            tmp.write(src.read())
             tmp.close()
+            src.close()
             tmp_path = tmp.name
             return LocalFilePath(tmp_path, delete=True)
 

server/common/utils/type_conversion_utils.py

@@ -116,7 +116,7 @@ def _get_type_info(array: Union[np.ndarray, pd.Series, pd.Index]) -> Tuple[np.dt
         raise TypeError("Unsupported data type.")
 
     dtype = array.dtype
-    
+
     res = _get_type_info_from_dtype(dtype)
     if res is not None:
         return res
@@ -140,6 +140,7 @@ def _get_type_info(array: Union[np.ndarray, pd.Series, pd.Index]) -> Tuple[np.dt
 
     if dtype.kind in ["i", "u"] and _can_cast_array_values_to_int32(array):
         return (np.int32, {"type": "int32"})
+
     if dtype.kind == "f":
         _float64_warning(array.dtype)
         return (np.float32, {"type": "float32"})

server/common/utils/utils.py

@@ -8,7 +8,7 @@ import socket
 from urllib.parse import urlsplit, urljoin
 
 import numpy as np
-import json
+from flask import json
 
 from server.common.errors import ConfigurationError
 
@@ -98,7 +98,7 @@ def custom_format_warning(msg, *args, **kwargs):
 
 
 def jsonify_strict(data):
-    return StrictJSONEncoder().encode(data)
+    return json.dumps(data, cls=StrictJSONEncoder, allow_nan=False)
 
 
 def import_plugins(plugin_module):

server/data_anndata/anndata_adaptor.py

@@ -1,5 +1,4 @@
 import warnings
-import importlib.metadata
 
 import anndata
 import numpy as np
@@ -17,7 +16,7 @@ from server.common.utils.type_conversion_utils import get_schema_type_hint_of_ar
 from server.data_common.data_adaptor import DataAdaptor
 from server.common.fbs.matrix import encode_matrix_fbs
 
-anndata_version = version.parse(str(importlib.metadata.version('anndata'))).release
+anndata_version = version.parse(str(anndata.__version__)).release
 
 
 def anndata_version_is_pre_070():
@@ -64,7 +63,7 @@ class AnndataAdaptor(DataAdaptor):
         return "cellxgene anndata adaptor version"
 
     def get_library_versions(self):
-        return dict(anndata=str(importlib.metadata.version('anndata')))
+        return dict(anndata=str(anndata.__version__))
 
     @staticmethod
     def _create_unique_column_name(df, col_name_prefix):
@@ -93,7 +92,7 @@ class AnndataAdaptor(DataAdaptor):
         """
         self.original_obs_index = self.data.obs.index
 
-        for ax_name, var_name in ((Axis.OBS, "obs"), (Axis.VAR, "var")):
+        for (ax_name, var_name) in ((Axis.OBS, "obs"), (Axis.VAR, "var")):
             config_name = f"single_dataset__{var_name}_names"
             parameter_name = f"{var_name}_names"
             name = getattr(self.server_config, config_name)
@@ -174,27 +173,11 @@ class AnndataAdaptor(DataAdaptor):
             )
         except MemoryError:
             raise DatasetAccessError("Out of memory - file is too large for available memory.")
-        except Exception as e:
+        except Exception:
-            import traceback
+            raise DatasetAccessError(
-            error_msg = str(e)
+                "File not found or is inaccessible. File must be an .h5ad object. "
-
+                "Please check your input and try again."
-            # IMPROVEMENT: Broadly catch ANY version incompatibility
+            )
-            if "No read method registered" in error_msg and "IOSpec" in error_msg:
-                message = (
-                    "Error loading file: This H5AD file uses a newer internal format that "
-                    "your version of 'anndata' cannot read.\n"
-                    f"The specific error was: {error_msg}\n"
-                    "Please upgrade anndata in your environment (pip install --upgrade anndata)."
-                )
-            else:
-                message = (
-                    "File not found or is inaccessible. File must be an .h5ad object. "
-                    "Please check your input and try again."
-                )
-
-            if self.server_config.app__verbose:
-                message += f"\n{traceback.format_exc()}"
-            raise DatasetAccessError(message)
 
     def _validate_and_initialize(self):
         if anndata_version_is_pre_070():
@@ -223,7 +206,7 @@ class AnndataAdaptor(DataAdaptor):
         # heuristic
         n_values = self.data.shape[0] * self.data.shape[1]
         if (n_values > 1e8 and self.server_config.adaptor__anndata_adaptor__backed is True) or (n_values > 5e8):
-            self.parameters.update({"diffexp-may-be-slow": True})
+            self.parameters.update({"diffexp_may_be_slow": True})
 
     def _is_valid_layout(self, arr):
         """return True if this layout data is a valid array for front-end presentation:
@@ -231,7 +214,7 @@ class AnndataAdaptor(DataAdaptor):
         * with shape (n_obs, >= 2)
         * with all values finite or NaN (no +Inf or -Inf)
         """
-        is_valid = type(arr) is np.ndarray and arr.dtype.kind in "fiu"
+        is_valid = type(arr) == np.ndarray and arr.dtype.kind in "fiu"
         is_valid = is_valid and arr.shape[0] == self.data.n_obs and arr.shape[1] >= 2
         is_valid = is_valid and not np.any(np.isinf(arr)) and not np.all(np.isnan(arr))
         return is_valid
@@ -253,16 +236,6 @@ class AnndataAdaptor(DataAdaptor):
             warnings.warn(
                 f"Anndata data matrix is in {self.data.X.dtype} format not float32. " f"Precision may be truncated."
             )
-        if self.data.X.dtype < np.float32:
-            if self.data.isbacked:
-                raise DatasetAccessError(
-                    f"Data matrix in {self.data.X.dtype} format is not supported in backed mode."
-                    " Please reload without --backed, or convert matrix to float32"
-                )
-            warnings.warn(
-                f"Anndata data matrix is in unsupported {self.data.X.dtype} format -- will be cast to float32"
-            )
-            self.data.X = self.data.X.astype(np.float32)
         for ax in Axis:
             curr_axis = getattr(self.data, str(ax))
             for ann in curr_axis:
@@ -301,6 +274,43 @@ class AnndataAdaptor(DataAdaptor):
             df = df[fields]
         return encode_matrix_fbs(df, col_idx=df.columns)
 
+    def get_spatial(self):
+        return self.data.uns["spatial"]
+
+    def get_spatial_metadata(self):
+        spatial = self.get_spatial()
+
+        resolution = "hires"
+
+        if len(list(spatial)) == 0:
+            raise Exception("uns does not have spatial information")
+
+        library_id = list(spatial)[0]
+
+        if "images" not in spatial[library_id]:
+            raise Exception("spatial information does not contain images")
+
+        if resolution not in spatial[library_id]["images"]:
+            raise Exception(f"spatial information does not contain requested resolution '{resolution}'")
+
+        scaleref = spatial[library_id]["scalefactors"][f"tissue_{resolution}_scalef"]
+        (h, w, _) = spatial[library_id]["images"][resolution].shape
+
+        A = self.data.obsm["X_spatial"]
+        min = np.nanmin(A, axis=0)
+        max = np.nanmax(A, axis=0)
+        scale = np.amax(max - min)
+        translate = 0.5 - ((max - min) / scale / 2)
+
+        return {
+            "imageWidth": w,
+            "imageHeight": h,
+            "scaleref": scaleref,
+            "inverseScale": int(scale),
+            "inverseTranslate": translate.tolist(),
+            "inverseMin": min.tolist(),
+        }
+
     def get_embedding_names(self):
         """
         Return pre-computed embeddings.
@@ -314,11 +324,11 @@ class AnndataAdaptor(DataAdaptor):
         layouts = self.dataset_config.embeddings__names
 
         if layouts is None or len(layouts) == 0:
-            layouts = [key[2:] for key in list(self.data.obsm.keys()) if type(key) is str and key.startswith("X_")]
+            layouts = [key[2:] for key in self.data.obsm_keys() if type(key) == str and key.startswith("X_")]
 
         # remove invalid layouts
         valid_layouts = []
-        obsm_keys = list(self.data.obsm.keys())
+        obsm_keys = self.data.obsm_keys()
         for layout in layouts:
             layout_name = f"X_{layout}"
             if layout_name not in obsm_keys:

server/data_common/data_adaptor.py

@@ -154,7 +154,7 @@ class DataAdaptor(metaclass=ABCMeta):
         parameters.update(self.parameters)
 
     def _index_filter_to_mask(self, filter, count):
-        mask = np.zeros((count,), dtype="bool")
+        mask = np.zeros((count,), dtype=np.bool)
         for i in filter:
             if isinstance(i, list):
                 mask[i[0] : i[1]] = True
@@ -163,7 +163,7 @@ class DataAdaptor(metaclass=ABCMeta):
         return mask
 
     def _axis_filter_to_mask(self, axis, filter, count):
-        mask = np.ones((count,), dtype="bool")
+        mask = np.ones((count,), dtype=np.bool)
         if "index" in filter:
             mask = np.logical_and(mask, self._index_filter_to_mask(filter["index"], count))
         if "annotation_value" in filter:
@@ -172,7 +172,7 @@ class DataAdaptor(metaclass=ABCMeta):
         return mask
 
     def _annotation_filter_to_mask(self, axis, filter, count):
-        mask = np.ones((count,), dtype="bool")
+        mask = np.ones((count,), dtype=np.bool)
         for v in filter:
             name = v["name"]
             if axis == Axis.VAR:
@@ -340,31 +340,57 @@ class DataAdaptor(metaclass=ABCMeta):
         pass
 
     @staticmethod
-    def normalize_embedding(embedding):
+    def normalize_embedding(embedding, spatial = None):
         """Normalize embedding layout to meet client assumptions.
-        Embedding is an ndarray, shape (n_obs, n)., where n is normally 2
+        Embedding is an ndarray, shape (n_obs, n)., where n is normally 2.
+        Note: if spatial data is available, the normalization will be done
+        according to the size of the underlying image
         """
 
-        # scale isotropically
+        if spatial is not None:
-        try:
-            min = np.nanmin(embedding, axis=0)
-            max = np.nanmax(embedding, axis=0)
-        except RuntimeError:
-            # indicates entire array was NaN, which should propagate
-            min = np.NaN
-            max = np.NaN
 
-        scale = np.amax(max - min)
+            # TODO: sync with the code in spatial_data_get
-        normalized_layout = (embedding - min) / scale
+            resolution = "hires"
 
-        # translate to center on both axis
+            if len(list(spatial)) == 0:
-        translate = 0.5 - ((max - min) / scale / 2)
+                raise Exception("uns does not have spatial information")
-        normalized_layout = normalized_layout + translate
+
+            library_id = list(spatial)[0]
+
+            if "images" not in spatial[library_id]:
+                raise Exception("spatial information does not contain images")
+
+            if resolution not in spatial[library_id]["images"]:
+                raise Exception(f"spatial information does not contain requested resolution '{resolution}'")
+
+            scaleref = spatial[library_id]["scalefactors"][f"tissue_{resolution}_scalef"]
+            (h, w, _) = spatial[library_id]["images"][resolution].shape
+
+            A = embedding * scaleref
+            A = np.column_stack([A[:, 0] / w, A[:, 1] / h])
+            normalized_layout = A.astype(dtype=np.float32)
+
+        else:
+
+            # scale isotropically
+            try:
+                min = np.nanmin(embedding, axis=0)
+                max = np.nanmax(embedding, axis=0)
+            except RuntimeError:
+                # indicates entire array was NaN, which should propagate
+                min = np.NaN
+                max = np.NaN
+
+            scale = np.amax(max - min)
+            normalized_layout = (embedding - min) / scale
+
+            # translate to center on both axis
+            translate = 0.5 - ((max - min) / scale / 2)
+            normalized_layout = normalized_layout + translate
 
-        normalized_layout = normalized_layout.astype(dtype=np.float32)
         return normalized_layout
 
-    def layout_to_fbs_matrix(self, fields):
+    def layout_to_fbs_matrix(self, fields, spatial = None):
         """
         return specified embeddings as a flatbuffer, using the cellxgene matrix fbs encoding.
 
@@ -380,7 +406,7 @@ class DataAdaptor(metaclass=ABCMeta):
         with ServerTiming.time("layout.query"):
             for ename in embeddings:
                 embedding = self.get_embedding_array(ename, 2)
-                normalized_layout = DataAdaptor.normalize_embedding(embedding)
+                normalized_layout = DataAdaptor.normalize_embedding(embedding, ename == "spatial" and spatial)
                 layout_data.append(pd.DataFrame(normalized_layout, columns=[f"{ename}_0", f"{ename}_1"]))
 
         with ServerTiming.time("layout.encode"):

server/data_common/matrix_loader.py

@@ -12,7 +12,7 @@ class MatrixDataType(Enum):
 
 class MatrixDataLoader(object):
     def __init__(self, location, matrix_data_type=None, app_config=None):
-        """location can be a string or DataLocator"""
+        """ location can be a string or DataLocator """
         region_name = None if app_config is None else app_config.server_config.data_locator__s3__region_name
         self.location = DataLocator(location, region_name=region_name)
         if not self.location.exists():

server/requirements-annotate.txt

@@ -1,2 +0,0 @@
-mlflow==2.16.0
-scanpy

server/requirements-dev.txt

@@ -1,10 +1,10 @@
 black
 bumpversion>=0.5
-coverage>=5.0
+codecov>=2.0.15
 parameterized>=0.7.0
+psycopg2-binary>=2.8.5
 pytest>=3.6.3
 python-jose>=3.2.0
 twine>=1.12.1
-aiohttp>=3.9.1
 -r requirements.txt
 -r requirements-prepare.txt

server/requirements.txt

@@ -1,23 +1,25 @@
-anndata>=0.8.0
+# NOTE: If you update 'anndata' min version, also update the 'anndata_version'
+# matrix value in .github/workflows/compatibility_tests.yml
+anndata>=0.7.6 # we need to_memory(), added in 0.7.6
 boto3>=1.12.18
 click>=7.1.2
-Flask>=3.0.0
+Flask>=1.0.2
 Flask-Compress>=1.4.0
-Flask-Cors>=3.0.9
+Flask-Cors>=3.0.9  # CVE-2020-25032
 Flask-RESTful>=0.3.6
 flask-server-timing>=0.1.2
 flask-talisman>=0.7.0
-flatbuffers==2.0.7
+flatbuffers>=1.11.0,<2.0.0 # cellxgene is not compatible with 2.0.0. Requires migration
 flatten-dict>=0.2.0
-fsspec>0.8.0
+fsspec>=0.4.4,<0.8.0
 gunicorn>=20.0.4
 h5py>=3.0.0
-numba>=0.60.0
+matplotlib>=3.5.0
-numpy==2.0.1
+numba>=0.51.2
+numpy>=1.17.5
 packaging>=20.0
-pandas>=2.2.2
+pandas>=1.0,!=1.1  # pandas 1.1 breaks tests, https://github.com/pandas-dev/pandas/issues/35446
 PyYAML>=5.4  # CVE-2020-14343
+scipy>=1.4
 requests>=2.22.0
 s3fs==0.4.2
-scipy>=1.4
-setuptools

setup.py

@@ -9,12 +9,9 @@ with open("server/requirements.txt") as fh:
 with open("server/requirements-prepare.txt") as fh:
     requirements_prepare = fh.read().splitlines()
 
-with open("server/requirements-annotate.txt") as fh:
-    requirements_annotate = fh.read().splitlines()
-
 setup(
     name="cellxgene",
-    version="1.3.0",
+    version="1.0.0",
     packages=find_packages(),
     url="https://github.com/chanzuckerberg/cellxgene",
     license="MIT",
@@ -24,7 +21,7 @@ setup(
     long_description=long_description,
     long_description_content_type="text/markdown",
     install_requires=requirements,
-    python_requires=">=3.10",
+    python_requires=">=3.6",
     include_package_data=True,
     zip_safe=False,
     classifiers=[
@@ -37,12 +34,11 @@ setup(
         "Operating System :: MacOS :: MacOS X",
         "Programming Language :: JavaScript",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.6",
-        "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.7",
-        "Programming Language :: Python :: 3.12",
         "Programming Language :: Python :: 3 :: Only",
         "Topic :: Scientific/Engineering :: Bio-Informatics",
     ],
     entry_points={"console_scripts": ["cellxgene = server.cli.cli:cli"]},
-    extras_require=dict(prepare=requirements_prepare, annotate=requirements_annotate),
+    extras_require=dict(prepare=requirements_prepare),
 )

test/unit/__init__.py

@@ -113,7 +113,7 @@ def start_test_server(command_line_args=[], app_config=None, env=None):
     elif "--port" in command_line_args:
         port = int(command_line_args[command_line_args.index("--port") + 1])
     else:
-        start = random.randint(DEFAULT_SERVER_PORT, 2**16 - 1)
+        start = random.randint(DEFAULT_SERVER_PORT, 2 ** 16 - 1)
         port = int(os.environ.get("CXG_SERVER_PORT", start))
         port = find_available_port("localhost", port)
         command += ["--port=%d" % port]

test/unit/cli/fixtures/mlflow_model_fixture.py

@@ -1,11 +0,0 @@
-import mlflow
-
-
-class FakeModel(mlflow.pyfunc.PythonModel):
-    def __init__(self, input_to_output: dict = {}):
-        self.input_to_output = input_to_output
-
-    def predict(self, model_input) -> None:
-        # this stdout output is useful for validating the input in a test, noting that this model will be invoked in a
-        # subprocess, so stdout is one means of communicating information back to the test code
-        print(f"__MODEL_INPUT__={model_input.iloc[0][0]}")

test/unit/cli/test_prepare.py

@@ -6,7 +6,7 @@ from server.cli.prepare import make_index_unique
 
 
 class CLIPrepareTests(unittest.TestCase):
-    """Test cases for CLI prepare logic"""
+    """ Test cases for CLI prepare logic """
 
     def test_make_index_unique(self):
         index = pd.Index(["SNORD113", "SNORD113", "SNORD113-1"])

test/unit/cli/test_upgrade.py

@@ -4,7 +4,7 @@ from server.cli.upgrade import validate_version_str, split_version, version_gt
 
 
 class CLIUpgradeTests(unittest.TestCase):
-    """Test cases for CLI logic"""
+    """ Test cases for CLI logic """
 
     def test_validate_version_str(self):
         self.assertTrue(validate_version_str("0.1.2"))

test/unit/common/config/__init__.py

@@ -21,7 +21,7 @@ class ConfigTests(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls) -> None:
-        os.makedirs(cls.tmp_fixtures_directory, exist_ok=True)
+        os.makedirs(cls.tmp_fixtures_directory)
 
     def custom_server_config(
         self,

test/unit/common/config/test_dataset_config.py

@@ -72,18 +72,24 @@ class TestDatasetConfig(ConfigTests):
             config.dataset_config.handle_app()
 
     def test_handle_user_annotations__instantiates_user_annotations_class_correctly(self):
-        config = self.get_config(enable_users_annotations="true", annotation_type="local_file_csv")
+        config = self.get_config(
+            enable_users_annotations="true", annotation_type="local_file_csv"
+        )
         config.server_config.complete_config(self.context)
         config.dataset_config.handle_user_annotations(self.context)
         self.assertIsInstance(config.dataset_config.user_annotations, AnnotationsLocalFile)
 
-        config = self.get_config(enable_users_annotations="true", annotation_type="NOT_REAL")
+        config = self.get_config(
+            enable_users_annotations="true", annotation_type="NOT_REAL"
+        )
         config.server_config.complete_config(self.context)
         with self.assertRaises(ConfigurationError):
             config.dataset_config.handle_user_annotations(self.context)
 
     def test_handle_local_file_csv_annotations__sets_dir_if_not_passed_in(self):
-        config = self.get_config(enable_users_annotations="true", annotation_type="local_file_csv")
+        config = self.get_config(
+            enable_users_annotations="true", annotation_type="local_file_csv"
+        )
         config.server_config.complete_config(self.context)
         config.dataset_config.handle_local_file_csv_annotations(self.context)
         self.assertIsInstance(config.dataset_config.user_annotations, AnnotationsLocalFile)

test/unit/common/config/test_external_config.py

@@ -56,6 +56,7 @@ class TestExternalConfig(ConfigTests):
             self.assertFalse(data_config["config"]["parameters"]["disable-diffexp"])
 
     def test_environment_variable_errors(self):
+
         # no name
         app_config = AppConfig()
         app_config.external_config.environment = [dict(required=True, path=["this", "is", "a", "path"])]

test/unit/common/test_api.py

@@ -196,18 +196,17 @@ class EndPoints(object):
     def test_fbs_default(self):
         endpoint = "data/var"
         url = f"{self.URL_BASE}{endpoint}"
-        headers = {"Content-Type": "application/json"}
+        result = self.session.put(url)
-        result = self.session.put(url, headers=headers)
         self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
 
         filter = {"filter": {"var": {"index": [0, 1, 4]}}}
-        result = self.session.put(url, json=filter, headers=headers)
+        result = self.session.put(url, json=filter)
         self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
 
     def test_data_put_fbs(self):
         endpoint = "data/var"
         url = f"{self.URL_BASE}{endpoint}"
-        header = {"Accept": "application/octet-stream", "Content-Type": "application/json"}
+        header = {"Accept": "application/octet-stream"}
         result = self.session.put(url, headers=header)
         self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
 
@@ -253,7 +252,6 @@ class EndPoints(object):
             if type(column) is np.ndarray:
                 self.assertIn(column.dtype, [np.float32, np.int32])
 
-    @unittest.skip("This test is currently broken after upgrading Werkzeug.")
     def test_data_get_unknown_filter_fbs(self):
         index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
         endpoint = "data/var"
@@ -292,7 +290,6 @@ class EndPoints(object):
         result_data = result.json()
         self.assertEqual(result_data, pbmc3k_colors)
 
-    @unittest.skip("needs fix: https://github.com/chanzuckerberg/cellxgene/issues/2542")
     def test_static(self):
         endpoint = "static"
         file = "assets/favicon.ico"

test/unit/common/test_corpora.py

@@ -106,7 +106,7 @@ class CorporaAPITest(unittest.TestCase):
 
 
 class CorporaRESTAPITest(unittest.TestCase):
-    """Confirm endpoints reflect Corpora-specific features"""
+    """ Confirm endpoints reflect Corpora-specific features """
 
     @classmethod
     def setCorporaFields(cls, path):

test/unit/common/test_rest.py

@@ -6,12 +6,12 @@ from server.common.rest import _query_parameter_to_filter
 
 
 def _qsparse(qs):
-    """emulate what Flask/Werkzeug do to our QS"""
+    """ emulate what Flask/Werkzeug do to our QS """
     return MultiDict(parse_qs(qs))
 
 
 class FilterParseTests(unittest.TestCase):
-    """Test cases for various filter parsing"""
+    """ Test cases for various filter parsing """
 
     def test_queryparam_to_filter_parse(self):
         # categories
@@ -57,6 +57,7 @@ class FilterParseTests(unittest.TestCase):
         )
 
     def test_queryparam_to_filter_errors(self):
+
         # should raise FilterError
         filter_errors = [
             "foo=bar",  # no axis

test/unit/common/utils/test_utils.py

@@ -7,7 +7,7 @@ from test import PROJECT_ROOT, random_string
 
 
 class TestPlugins(unittest.TestCase):
-    """Test plugin import functionality"""
+    """ Test plugin import functionality """
 
     plugins_dir = f"{PROJECT_ROOT}/test/plugins"
     test_plugin_path = f"{plugins_dir}/foo.py"

test/unit/compute/test_est_dist.py

@@ -65,13 +65,13 @@ class EstDistTest(unittest.TestCase):
 
         # non-finites
         self.assertEqual(estimate_approximate_distribution(np.array([np.nan])), XApproximateDistribution.NORMAL)
-        self.assertEqual(estimate_approximate_distribution(np.array([np.inf])), XApproximateDistribution.NORMAL)
+        self.assertEqual(estimate_approximate_distribution(np.array([np.PINF])), XApproximateDistribution.NORMAL)
-        self.assertEqual(estimate_approximate_distribution(np.array([np.inf])), XApproximateDistribution.NORMAL)
+        self.assertEqual(estimate_approximate_distribution(np.array([np.NINF])), XApproximateDistribution.NORMAL)
         self.assertEqual(
-            estimate_approximate_distribution(np.array([np.inf, np.inf, 0])), XApproximateDistribution.NORMAL
+            estimate_approximate_distribution(np.array([np.PINF, np.NINF, 0])), XApproximateDistribution.NORMAL
         )
         self.assertEqual(
-            estimate_approximate_distribution(np.array([np.nan, np.inf, np.inf])), XApproximateDistribution.NORMAL
+            estimate_approximate_distribution(np.array([np.nan, np.PINF, np.NINF])), XApproximateDistribution.NORMAL
         )
 
         raw = np.random.exponential(scale=1000, size=(50, 3))
@@ -82,15 +82,15 @@ class EstDistTest(unittest.TestCase):
             XApproximateDistribution.COUNT,
         )
         self.assertEqual(
-            estimate_approximate_distribution(put(raw, [1], [np.inf])),
+            estimate_approximate_distribution(put(raw, [1], [np.PINF])),
             XApproximateDistribution.COUNT,
         )
         self.assertEqual(
-            estimate_approximate_distribution(put(raw, [1], [np.inf])),
+            estimate_approximate_distribution(put(raw, [1], [np.NINF])),
             XApproximateDistribution.COUNT,
         )
         self.assertEqual(
-            estimate_approximate_distribution(put(raw, [1, 3, 88], [np.nan, np.inf, np.inf])),
+            estimate_approximate_distribution(put(raw, [1, 3, 88], [np.nan, np.PINF, np.NINF])),
             XApproximateDistribution.COUNT,
         )
         self.assertEqual(
@@ -103,15 +103,15 @@ class EstDistTest(unittest.TestCase):
             XApproximateDistribution.NORMAL,
         )
         self.assertEqual(
-            estimate_approximate_distribution(put(logged, [1], [np.inf])),
+            estimate_approximate_distribution(put(logged, [1], [np.PINF])),
             XApproximateDistribution.NORMAL,
         )
         self.assertEqual(
-            estimate_approximate_distribution(put(logged, [1], [np.inf])),
+            estimate_approximate_distribution(put(logged, [1], [np.NINF])),
             XApproximateDistribution.NORMAL,
         )
         self.assertEqual(
-            estimate_approximate_distribution(put(logged, [1, 3, 88], [np.nan, np.inf, np.inf])),
+            estimate_approximate_distribution(put(logged, [1, 3, 88], [np.nan, np.PINF, np.NINF])),
             XApproximateDistribution.NORMAL,
         )
         self.assertEqual(

test/unit/data_anndata/test_anndata_adaptor.py

@@ -1,4 +1,5 @@
 import json
+import sys
 import time
 import unittest
 

test/unit/data_anndata/test_anndata_adaptor_data_load.py

@@ -58,7 +58,7 @@ class DataLocatorAdaptorTest(unittest.TestCase):
         return config
 
     def stdAsserts(self, data):
-        """run these each time we load the data"""
+        """ run these each time we load the data """
         self.assertIsNotNone(data)
         self.assertEqual(data.cell_count, 2638)
         self.assertEqual(data.gene_count, 1838)

test/unit/test_colors.py

@@ -9,7 +9,7 @@ from test.fixtures.fixtures import pbmc3k_colors
 
 
 class ColorsTest(unittest.TestCase):
-    """Test color helper functions"""
+    """ Test color helper functions """
 
     def test_convert_color_to_hex_format(self):
         self.assertEqual(convert_color_to_hex_format("wheat"), "#f5deb3")

test/unit/utils/test_jsonify_strict.py

@@ -16,10 +16,10 @@ class TestJsonifyStrict(unittest.TestCase):
             jsonify_strict({"nan": [np.nan]})
 
         with self.assertRaises(ValueError):
-            jsonify_strict({"pinf": [np.inf]})
+            jsonify_strict({"pinf": [np.PINF]})
 
         with self.assertRaises(ValueError):
-            jsonify_strict({"ninf": [np.inf]})
+            jsonify_strict({"ninf": [np.NINF]})
 
     def test_jsonify_numpy_ndarray(self):
         values = {
@@ -54,5 +54,5 @@ class TestJsonifyStrict(unittest.TestCase):
         # the actual test!
         self.assertEqual(
             jsonify_strict(values),
-            '{"integer": [0, 1, 2, 3, 4, 5, 6, 7], "floating": [100.0, 101.0, 102.0]}',
+            '{"floating": [100.0, 101.0, 102.0], "integer": [0, 1, 2, 3, 4, 5, 6, 7]}',
         )

test/unit/utils/test_type_conversion_utils.py

@@ -42,7 +42,7 @@ class TestTypeConversionUtils(unittest.TestCase):
             with self.assertRaises(TypeError):
                 get_schema_type_hint_from_dtype(np.dtype(dtype))
 
-        for dtype in [np.float32, np.float64]:
+        for dtype in [np.float16, np.float32, np.float64]:
             self.assertEqual(get_schema_type_hint_from_dtype(np.dtype(dtype)), {"type": "float32"})
 
         for dtype in [np.dtype(object), np.dtype(str)]:
@@ -123,18 +123,17 @@ int_OK_cases = [
 
 float_OK_cases = [
     {
-        "test_case": "float_OK_cases",
         "data": data,
         "expected_encoding_dtype": np.float32,
         "expected_schema_hint": {"type": "float32"},
-        "logs": None if dtype == np.float32 else {"level": logging.WARNING, "output": "may lose precision"},
+        "logs": None if data.dtype != np.float64 else {"level": logging.WARNING, "output": "may lose precision"},
     }
-    for dtype in [np.float32, np.float64]
+    for dtype in [np.float16, np.float32, np.float64]
     for data in [
         np.arange(-128, 1000, dtype=dtype),
         pd.Series(np.arange(-128, 1000, dtype=dtype)),
         pd.Index(np.arange(-129, 1000, dtype=dtype)),
-        np.array([-np.nan, -np.inf, -1, -0.0, 0, 0.0, 1, np.inf, np.nan], dtype=dtype),
+        np.array([-np.nan, np.NINF, -1, np.NZERO, 0, np.PZERO, 1, np.PINF, np.nan], dtype=dtype),
         np.array([np.finfo(dtype).min, 0, np.finfo(dtype).max], dtype=dtype),
         sparse.csr_matrix((10, 100), dtype=dtype),
     ]
@@ -199,13 +198,12 @@ category_numeric_OK_cases = [
     # numeric, no NA/NaN, float
     *[
         {
-            "test_case": "numeric, no NA/NaN, float",
             "data": data,
             "expected_encoding_dtype": np.float32,
             "expected_schema_hint": {"type": "categorical"},
-            "logs": None if dtype == np.float32 else {"level": logging.WARNING, "output": "may lose precision"},
+            "logs": {"level": logging.WARNING, "output": "may lose precision"},
         }
-        for dtype in [np.float32, np.float64]
+        for dtype in [np.float16, np.float32, np.float64]
         for data in [
             pd.Series(np.array([0, 1, 2], dtype=dtype), dtype="category"),
             pd.Series(np.array([0, 1, 2], dtype=dtype), dtype="category").cat.remove_categories([1]),
@@ -215,11 +213,10 @@ category_numeric_OK_cases = [
     # numeric, has NA-induced cast to float32
     *[
         {
-            "test_case": "numeric, has NA-induced cast to float32",
             "data": data,
             "expected_encoding_dtype": np.float32,
             "expected_schema_hint": {"type": "categorical"},
-            "logs": None if dtype == np.float32 else {"level": logging.WARNING, "output": "may lose precision"},
+            "logs": {"level": logging.WARNING, "output": "may lose precision"},
         }
         for dtype in [
             np.int8,
@@ -230,6 +227,7 @@ category_numeric_OK_cases = [
             np.uint32,
             np.int64,
             np.uint64,
+            np.float16,
             np.float32,
             np.float64,
         ]
@@ -314,6 +312,7 @@ class TestTypeInference(unittest.TestCase, AssertNoLog):
                     self.assertEqual(encoding_dtype, self.expected_encoding_dtype)
                     self.assertEqual(schema_hint, self.expected_schema_hint)
                     self.assertIn(logs["output"], logger.output[0])
+
             else:
                 with self.assertNoLogs(logging.getLogger(), logging.WARNING):
                     encoding_dtype, schema_hint = get_dtype_and_schema_of_array(self.data)