Add README-visium

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.0.1
+current_version = 1.0.0
 commit = True
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?:-(?P<prerel>rc)\.(?P<prerelversion>\d+))?
 serialize = 

.github/workflows/compatibility_tests.yml

@@ -16,7 +16,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
       - name: Build docker image
@@ -56,7 +56,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v1
       with:
         python-version: ${{ matrix.python-version }}
     - name: Cache env vars

.github/workflows/push_tests.yml

@@ -18,7 +18,7 @@ jobs:
       - run: |
           git fetch --depth=1 origin +${{github.base_ref}}
       - name: Set up Python 3.7
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v1
         with:
           python-version: 3.7
       - name: Node cache
@@ -46,12 +46,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Python 3.7 (pyenv)  # pyenv needed for mlflow in cli annotate tests
+      - name: Set up Python 3.7
-        uses: gabrielfalcao/pyenv-action@v9
+        uses: actions/setup-python@v1
         with:
-          default: 3.7
+          python-version: 3.7
-          command: pip install -U pip  # upgrade pip after installing python
-      - run: pip install virtualenv  # virtualenv needed for mlflow in cli annotate tests
       - name: Python cache
         uses: actions/cache@v1
         with:
@@ -80,7 +78,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python 3.7
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v1
         with:
           python-version: 3.7
       - name: Python cache
@@ -104,33 +102,32 @@ jobs:
           cd client && make smoke-test
           ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTest
 
-  # TODO: reinstate: https://github.com/chanzuckerberg/cellxgene/issues/2544
+  smoke-tests-annotations:
-  # smoke-tests-annotations:
+    runs-on: ubuntu-latest
-  #   runs-on: ubuntu-latest
+    timeout-minutes: 20
-  #   timeout-minutes: 20
+    steps:
-  #   steps:
+      - uses: actions/checkout@v2
-  #     - uses: actions/checkout@v2
+      - name: Set up Python 3.7
-  #     - name: Set up Python 3.7
+        uses: actions/setup-python@v1
-  #       uses: actions/setup-python@v4
+        with:
-  #       with:
+          python-version: 3.7
-  #         python-version: 3.7
+      - name: Python cache
-  #     - name: Python cache
+        uses: actions/cache@v1
-  #       uses: actions/cache@v1
+        with:
-  #       with:
+          path: ~/.cache/pip
-  #         path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
-  #         key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+          restore-keys: |
-  #         restore-keys: |
+            ${{ runner.os }}-pip-
-  #           ${{ runner.os }}-pip-
+      - name: Node cache
-  #     - name: Node cache
+        uses: actions/cache@v1
-  #       uses: actions/cache@v1
+        with:
-  #       with:
+          path: ~/.npm
-  #         path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
-  #         key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
-  #         restore-keys: |
+            ${{ runner.os }}-node-
-  #           ${{ runner.os }}-node-
+      - name: Install dependencies
-  #     - name: Install dependencies
+        run: make pydist install-dist
-  #       run: make pydist install-dist
+      - name: Smoke tests (with annotations feature)
-  #     - name: Smoke tests (with annotations feature)
+        run: |
-  #       run: |
+          cd client && make smoke-test-annotations
-  #         cd client && make smoke-test-annotations
+          ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTestAnnotations
-  #         ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTestAnnotations

.gitignore

@@ -54,6 +54,3 @@ client/.eslintcache
 
 # E2E Testing
 ignoreE2E*
-
-# annotate subcmd
-.models_cache

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2017-2022 Chan Zuckerberg Initiative
+Copyright (c) 2017-2021 Chan Zuckerberg Initiative
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

MANIFEST.in

@@ -3,6 +3,5 @@ recursive-include server/common/web/static *
 
 include server/requirements.txt
 include server/requirements-prepare.txt
-include server/requirements-annotate.txt
 include server/converters/schema/hgnc_complete_set.txt.gz
 include server/converters/schema/schema_definitions/*

README-visium.md

@@ -0,0 +1,20 @@
+# Cellxgene Visium Beta
+
+## How it works
+1. Launch `cellxgene` as normal. 
+1. If the loaded dataset has spatial information available, the image data will be loaded on startup.
+1. On the toolbar, next to the Zoom icon, a `Toggle image` button will now appear. Click on it and the image will be added as an underlay.
+1. You can now use any `cellxgene` functionality and the image will still be present. If you pan and zoom, the image will also be panned and zoomed.
+1. If you want to hide the image, you can click on `Toggle image` again
+
+In order for the image to be displayed with the correct size and alignment, the H5AD needs to have a few requirements. See the following section to learn more.
+
+## h5ad requirements
+1. The spatial embedding layer should be contained in `obsm` and be named `X_spatial`. Other layers can exist, but only this one will have the spatial feature enabled.
+2. A `spatial` dict needs to be defined in the `uns` dictionary.
+3. Inside the `spatial` dict, an `images` dict must be defined. 
+4. The `images` dict must contain a `hires` key, which should reference an image encoded as an RGB matrix (i.e., a three-dimensional matrix of size `height x width x 3` where the final dimension has the RGB values for each pixel)
+5. The `images` dict must contain a `scalefactors` dict. This should in turn contain a `tissue_hires_scalef` key, which should reference a floating point number.
+
+Moreover, in order to have the image correctly aligned with the dots, the following must be true:
+1. `tissue_hires_scalef` should represent the ratio between the embedding layer `X_spatial` and the image matrix. In particular, if you multiply `X_spatial` by `tissue_hires_scalef`, you should obtain an array of points that ovelap the tissue image if you plot them in a plane.

client/__tests__/e2e/__snapshots__/e2e.test.js.snap

@@ -2,4 +2,4 @@
 
 exports[`did launch page launched 1`] = `"<span style=\\"max-width: 155px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">pbm</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">c3k</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">c3k</span></span></span>"`;
 
-exports[`metadata loads categories and values from dataset appear 1`] = `"<div style=\\"display: flex; justify-content: space-between; align-items: baseline;\\"><div style=\\"display: flex; justify-content: flex-start; align-items: flex-start;\\"><label class=\\"bp3-control bp3-checkbox\\" for=\\"category-select-louvain\\"><input id=\\"category-select-louvain\\" data-testclass=\\"category-select\\" data-testid=\\"louvain:category-select\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span role=\\"menuitem\\" tabindex=\\"0\\" data-testclass=\\"category-expand\\" data-testid=\\"louvain:category-expand\\" style=\\"cursor: pointer;\\"><span aria-haspopup=\\"true\\" class=\\"bp3-popover2-target\\"><span data-testid=\\"louvain:category-label\\" tabindex=\\"-1\\" aria-label=\\"louvain\\" class=\\"\\" style=\\"max-width: 265px;\\"><span style=\\"max-width: 265px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">lou</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">vain</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">vain</span></span></span></span></span><svg stroke=\\"currentColor\\" fill=\\"currentColor\\" stroke-width=\\"0\\" viewBox=\\"0 0 320 512\\" data-testclass=\\"category-expand-is-not-expanded\\" height=\\"1em\\" width=\\"1em\\" xmlns=\\"http://www.w3.org/2000/svg\\" style=\\"font-size: 10px; margin-left: 5px;\\"><path d=\\"M285.476 272.971L91.132 467.314c-9.373 9.373-24.569 9.373-33.941 0l-22.667-22.667c-9.357-9.357-9.375-24.522-.04-33.901L188.505 256 34.484 101.255c-9.335-9.379-9.317-24.544.04-33.901l22.667-22.667c9.373-9.373 24.569-9.373 33.941 0L285.475 239.03c9.373 9.372 9.373 24.568.001 33.941z\\"></path></svg></span></div><div><span class=\\"bp3-popover-wrapper\\"><span aria-haspopup=\\"true\\" class=\\"bp3-popover-target\\"><a role=\\"button\\" data-testclass=\\"colorby\\" data-testid=\\"colorby-louvain\\" class=\\"bp3-button\\" tabindex=\\"0\\"><span icon=\\"tint\\" aria-hidden=\\"true\\" tabindex=\\"0\\" class=\\"bp3-icon bp3-icon-tint\\"><svg data-icon=\\"tint\\" width=\\"16\\" height=\\"16\\" viewBox=\\"0 0 16 16\\"><path d=\\"M7.88 1s-4.9 6.28-4.9 8.9c.01 2.82 2.34 5.1 4.99 5.1 2.65-.01 5.03-2.3 5.03-5.13C12.99 7.17 7.88 1 7.88 1z\\" fill-rule=\\"evenodd\\"></path></svg></span></a></span></span></div></div><div style=\\"margin-left: 26px;\\"></div>"`;
+exports[`metadata loads categories and values from dataset appear 1`] = `"<div style=\\"display: flex; justify-content: space-between; align-items: baseline;\\"><div style=\\"display: flex; justify-content: flex-start; align-items: flex-start;\\"><label class=\\"bp3-control bp3-checkbox\\" for=\\"category-select-louvain\\"><input id=\\"category-select-louvain\\" data-testclass=\\"category-select\\" data-testid=\\"louvain:category-select\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span role=\\"menuitem\\" tabindex=\\"0\\" data-testclass=\\"category-expand\\" data-testid=\\"louvain:category-expand\\" style=\\"cursor: pointer;\\"><span aria-haspopup=\\"true\\" class=\\"bp3-popover2-target\\"><span data-testid=\\"louvain:category-label\\" tabindex=\\"-1\\" aria-label=\\"louvain\\" class=\\"\\" style=\\"max-width: 265px;\\"><span style=\\"max-width: 265px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">lou</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">vain</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">vain</span></span></span></span></span><svg stroke=\\"currentColor\\" fill=\\"currentColor\\" stroke-width=\\"0\\" viewBox=\\"0 0 320 512\\" data-testclass=\\"category-expand-is-not-expanded\\" height=\\"1em\\" width=\\"1em\\" xmlns=\\"http://www.w3.org/2000/svg\\" style=\\"font-size: 10px; margin-left: 5px;\\"><path d=\\"M285.476 272.971L91.132 467.314c-9.373 9.373-24.569 9.373-33.941 0l-22.667-22.667c-9.357-9.357-9.375-24.522-.04-33.901L188.505 256 34.484 101.255c-9.335-9.379-9.317-24.544.04-33.901l22.667-22.667c9.373-9.373 24.569-9.373 33.941 0L285.475 239.03c9.373 9.372 9.373 24.568.001 33.941z\\"></path></svg></span></div><div><span class=\\"bp3-popover-wrapper\\"><span aria-haspopup=\\"true\\" class=\\"bp3-popover-target\\"><a role=\\"button\\" data-testclass=\\"colorby\\" data-testid=\\"colorby-louvain\\" class=\\"bp3-button\\" tabindex=\\"0\\"><span icon=\\"tint\\" class=\\"bp3-icon bp3-icon-tint\\"><svg data-icon=\\"tint\\" width=\\"16\\" height=\\"16\\" viewBox=\\"0 0 16 16\\"><desc>tint</desc><path d=\\"M7.88 1s-4.9 6.28-4.9 8.9c.01 2.82 2.34 5.1 4.99 5.1 2.65-.01 5.03-2.3 5.03-5.13C12.99 7.17 7.88 1 7.88 1z\\" fill-rule=\\"evenodd\\"></path></svg></span></a></span></span></div></div><div style=\\"margin-left: 26px;\\"></div>"`;

client/configuration/webpack/webpack.config.dev.js

@@ -1,6 +1,8 @@
 const path = require("path");
 const webpack = require("webpack");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
+const FaviconsWebpackPlugin = require("favicons-webpack-plugin");
+const ScriptExtHtmlWebpackPlugin = require("script-ext-html-webpack-plugin");
 const MiniCssExtractPlugin = require("mini-css-extract-plugin");
 
 const { merge } = require("webpack-merge");
@@ -42,6 +44,21 @@ const devConfig = {
       inject: true,
       template: path.resolve("index.html"),
     }),
+    new FaviconsWebpackPlugin({
+      logo: "./favicon.png",
+      prefix: "static/img/",
+      favicons: {
+        icons: {
+          android: false,
+          appleIcon: false,
+          appleStartup: false,
+          coast: false,
+          firefox: false,
+          windows: false,
+          yandex: false,
+        },
+      },
+    }),
     new MiniCssExtractPlugin({
       filename: "static/[name].css",
     }),
@@ -56,6 +73,9 @@ const devConfig = {
         CXG_SERVER_PORT: process.env.CXG_SERVER_PORT || "5005",
       }),
     }),
+    new ScriptExtHtmlWebpackPlugin({
+      async: "obsolete",
+    }),
   ],
   infrastructureLogging: {
     level: "warn",

client/configuration/webpack/webpack.config.prod.js

@@ -3,7 +3,9 @@ const webpack = require("webpack");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
 const { CleanWebpackPlugin } = require("clean-webpack-plugin");
 const TerserJSPlugin = require("terser-webpack-plugin");
-const CssMinimizerPlugin = require("css-minimizer-webpack-plugin");
+const CleanCss = require("clean-css");
+const OptimizeCSSAssetsPlugin = require("optimize-css-assets-webpack-plugin");
+const FaviconsWebpackPlugin = require("favicons-webpack-plugin");
 const MiniCssExtractPlugin = require("mini-css-extract-plugin");
 
 const { merge } = require("webpack-merge");
@@ -27,8 +29,8 @@ const prodConfig = {
     minimize: true,
     minimizer: [
       new TerserJSPlugin({}),
-      new CssMinimizerPlugin({
+      new OptimizeCSSAssetsPlugin({
-        minify: CssMinimizerPlugin.cleanCssMinify,
+        cssProcessor: CleanCss,
       }),
     ],
   },
@@ -64,6 +66,21 @@ const prodConfig = {
       protectWebpackAssets: false,
       cleanAfterEveryBuildPatterns: ["main.js", "main.css"],
     }),
+    new FaviconsWebpackPlugin({
+      logo: "./favicon.png",
+      prefix: "static/assets/",
+      favicons: {
+        icons: {
+          android: false,
+          appleIcon: false,
+          appleStartup: false,
+          coast: false,
+          firefox: false,
+          windows: false,
+          yandex: false,
+        },
+      },
+    }),
     new MiniCssExtractPlugin({
       filename: "static/[name]-[contenthash].css",
     }),

client/configuration/webpack/webpack.config.shared.js

@@ -2,6 +2,8 @@ const path = require("path");
 const fs = require("fs");
 const MiniCssExtractPlugin = require("mini-css-extract-plugin");
 const ObsoleteWebpackPlugin = require("obsolete-webpack-plugin");
+// eslint-disable-next-line @blueprintjs/classes-constants -- incorrect match
+const ScriptExtHtmlWebpackPlugin = require("script-ext-html-webpack-plugin");
 
 const src = path.resolve("src");
 const nodeModules = path.resolve("node_modules");
@@ -65,5 +67,8 @@ module.exports = {
       template: obsoleteHTMLTemplate,
       promptOnNonTargetBrowser: false,
     }),
+    new ScriptExtHtmlWebpackPlugin({
+      async: "obsolete",
+    }),
   ],
 };

client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cellxgene",
-  "version": "1.0.1",
+  "version": "1.0.0",
   "license": "MIT",
   "description": "cellxgene is a web application for the interactive exploration of single cell sequence data.",
   "repository": "https://github.com/chanzuckerberg/cellxgene",
@@ -101,7 +101,6 @@
     "clean-webpack-plugin": "^4.0.0-alpha.0",
     "codecov": "^3.7.1",
     "css-loader": "^5.2.4",
-    "css-minimizer-webpack-plugin": "^4.0.0",
     "eslint": "^7.24.0",
     "eslint-config-airbnb": "^18.2.0",
     "eslint-config-prettier": "^8.2.0",
@@ -115,6 +114,8 @@
     "eslint-plugin-react-hooks": "^4.0.8",
     "expect-puppeteer": "^5.0.0",
     "express": "^4.17.1",
+    "favicons": "^6.2.2",
+    "favicons-webpack-plugin": "^5.0.2",
     "file-loader": "^6.0.0",
     "html-webpack-plugin": "^5.3.1",
     "husky": "^7.0.2",
@@ -133,9 +134,11 @@
     "lodash.zip": "^4.2.0",
     "mini-css-extract-plugin": "^1.5.0",
     "obsolete-webpack-plugin": "^0.5.6",
+    "optimize-css-assets-webpack-plugin": "^5.0.3",
     "prettier": "^2.0.5",
     "puppeteer": "^8.0.0",
     "rimraf": "^3.0.2",
+    "script-ext-html-webpack-plugin": "^2.1.4",
     "serve-favicon": "^2.5.0",
     "terser-webpack-plugin": "^5.1.1",
     "webpack": "^5.34.0",

client/src/actions/index.js

@@ -8,6 +8,7 @@ import {
 import { loadUserColorConfig } from "../util/stateManager/colorHelpers";
 import * as selnActions from "./selection";
 import * as annoActions from "./annotation";
+import * as spatialActions from "./spatial";
 import * as viewActions from "./viewStack";
 import * as embActions from "./embedding";
 import * as genesetActions from "./geneset";
@@ -272,4 +273,5 @@ export default {
   genesetDelete: genesetActions.genesetDelete,
   genesetAddGenes: genesetActions.genesetAddGenes,
   genesetDeleteGenes: genesetActions.genesetDeleteGenes,
+  requestSpatialMetadata: spatialActions.requestSpatialMetadata,
 };

client/src/actions/spatial.js

@@ -0,0 +1,35 @@
+import * as globals from "../globals";
+
+export const requestSpatialMetadata = () => async (dispatch) => {
+  dispatch({ type: "request spatial metadata started" });
+  try {
+    const res = await fetch(
+      `${globals.API.prefix}${globals.API.version}spatial/meta`,
+      {
+        method: "GET",
+        headers: new Headers({
+          Accept: "application/json",
+          "Content-Type": "application/json",
+        }),
+        credentials: "include",
+      }
+    );
+
+    if (!res.ok || res.headers.get("Content-Type") !== "application/json") {
+      return null; // TODO need a dispatch //dispatchDiffExpErrors(dispatch, res);
+    }
+
+    const response = await res.json();
+
+    /* then send the success case action through */
+    return dispatch({
+      type: "request spatial metadata success",
+      data: response,
+    });
+  } catch (error) {
+    return dispatch({
+      type: "request spatial metadata error",
+      error,
+    });
+  }
+};

client/src/annoMatrix/viewCreators.js

@@ -58,11 +58,10 @@ function _maskToList(mask) {
   if (!mask) {
     return null;
   }
-  const [...m] = mask;
+  const list = new Int32Array(mask.length);
-  const list = new Int32Array(m.length);
   let elems = 0;
-  for (let i = 0, l = m.length; i < l; i += 1) {
+  for (let i = 0, l = mask.length; i < l; i += 1) {
-    if (m[i]) {
+    if (mask[i]) {
       list[elems] = i;
       elems += 1;
     }

client/src/annoMatrix/whereCache.js

@@ -91,9 +91,12 @@ export function _whereCacheCreate(field, query, columnLabels) {
 	*/
   if (typeof query !== "object") return null;
 
-  const { where, summarize } = query;
+  if (query.where) {
-  if (where) {
+    const {
-    const { field: queryField, column: queryColumn, value: queryValue } = where;
+      field: queryField,
+      column: queryColumn,
+      value: queryValue,
+    } = query.where;
     return {
       where: {
         [field]: {
@@ -104,13 +107,13 @@ export function _whereCacheCreate(field, query, columnLabels) {
       },
     };
   }
-  if (summarize) {
+  if (query.summarize) {
     const {
       method,
       field: queryField,
       column: queryColumn,
       values: queryValues,
-    } = summarize;
+    } = query.summarize;
     const queryValueHash = _hashStringValues(queryValues);
     return {
       summarize: {

client/src/components/app.js

@@ -23,6 +23,8 @@ class App extends React.Component {
   componentDidMount() {
     const { dispatch } = this.props;
 
+    dispatch(actions.requestSpatialMetadata());
+
     /* listen for url changes, fire one when we start the app up */
     window.addEventListener("popstate", this._onURLChanged);
     this._onURLChanged();

client/src/components/embedding/index.js

@@ -16,10 +16,11 @@ import actions from "../../actions";
 import { getDiscreteCellEmbeddingRowIndex } from "../../util/stateManager/viewStackHelpers";
 
 @connect((state) => ({
-    layoutChoice: state.layoutChoice, // TODO: really should clean up naming, s/layout/embedding/g
+  imageUnderlay: state.imageUnderlay,
-    schema: state.annoMatrix?.schema,
+  layoutChoice: state.layoutChoice, // TODO: really should clean up naming, s/layout/embedding/g
-    crossfilter: state.obsCrossfilter,
+  schema: state.annoMatrix?.schema,
-  }))
+  crossfilter: state.obsCrossfilter,
+}))
 class Embedding extends React.PureComponent {
   constructor(props) {
     super(props);
@@ -27,8 +28,18 @@ class Embedding extends React.PureComponent {
   }
 
   handleLayoutChoiceChange = (e) => {
-    const { dispatch } = this.props;
+    const { dispatch, imageUnderlay } = this.props;
     dispatch(actions.layoutChoiceAction(e.currentTarget.value));
+
+    // if we just switched off spatial, if the image is on, turn it off
+    if (
+      imageUnderlay.isActive &&
+      e.target.value !== globals.spatialEmbeddingKeyword
+    ) {
+      dispatch({
+        type: "toggle image underlay",
+      });
+    }
   };
 
   render() {

client/src/components/graph/drawSpatialImageRegl.js

@@ -0,0 +1,62 @@
+export default function drawSpatialImageRegl(regl) {
+  return regl({
+    frag: `
+        precision mediump float;
+
+        // our texture
+        uniform sampler2D u_image;
+
+        // the texCoords passed in from the vertex shader.
+        varying vec2 v_texCoord;
+
+        void main() {
+            gl_FragColor = texture2D(u_image, v_texCoord);
+        }`,
+
+    vert: `
+        attribute vec2 a_position;
+        attribute vec2 a_texCoord;
+
+        uniform vec2 u_resolution;
+
+        uniform mat3 projView;
+
+        varying vec2 v_texCoord;
+
+        void main() {
+            // convert the rectangle from pixels to 0.0 to 1.0
+            vec3 pos = vec3(a_position, 1.);
+            vec2 zeroToOne = pos.xy / u_resolution;
+
+            // convert from 0->1 to 0->2
+            vec2 zeroToTwo = zeroToOne * 2.0;
+
+            // convert from 0->2 to -1->+1 (clipspace)
+            vec2 clipSpace = zeroToTwo - 1.0;
+
+            vec3 pos2 = projView * vec3(clipSpace, 1.);
+
+            gl_Position = vec4(pos2.xy , 0, 1);
+
+            // pass the texCoord to the fragment shader
+            // The GPU will interpolate this value between points.
+            v_texCoord = a_texCoord;
+        }`,
+
+    attributes: {
+      a_texCoord: [0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 1.0, 1.0, 0.0, 1.0, 1.0],
+      a_position: regl.prop("rectCoords"),
+    },
+
+    uniforms: {
+      projView: regl.prop("projView"),
+      u_image: regl.prop("spatialImageAsTexture"),
+      color: [1, 0, 0, 1],
+      u_resolution: [regl.prop("imageWidth"), regl.prop("imageHeight")],
+      image_width: regl.prop("imageWidth"),
+      //   translate:
+    },
+
+    count: 6,
+  });
+}

client/src/components/graph/graph.js

@@ -14,6 +14,7 @@ import {
   createColorTable,
   createColorQuery,
 } from "../../util/stateManager/colorHelpers";
+import _drawSpatialImage from "./drawSpatialImageRegl";
 import * as globals from "../../globals";
 
 import GraphOverlayLayer from "./overlays/graphOverlayLayer";
@@ -77,6 +78,8 @@ function createModelTF() {
   colors: state.colors,
   pointDilation: state.pointDilation,
   genesets: state.genesets.genesets,
+  spatial: state.spatial.metadata,
+  imageUnderlay: state.imageUnderlay,
 }))
 class Graph extends React.Component {
   static createReglState(canvas) {
@@ -87,6 +90,7 @@ class Graph extends React.Component {
     const camera = _camera(canvas);
     const regl = _regl(canvas);
     const drawPoints = _drawPoints(regl);
+    const drawSpatialImage = _drawSpatialImage(regl);
 
     // preallocate webgl buffers
     const pointBuffer = regl.buffer();
@@ -100,6 +104,7 @@ class Graph extends React.Component {
       pointBuffer,
       colorBuffer,
       flagBuffer,
+      drawSpatialImage,
     };
   }
 
@@ -232,6 +237,8 @@ class Graph extends React.Component {
       pointBuffer: null,
       colorBuffer: null,
       flagBuffer: null,
+      drawSpatialImage: null,
+      spatial: null,
 
       // component rendering derived state - these must stay synchronized
       // with the reducer state they were generated from.
@@ -317,7 +324,10 @@ class Graph extends React.Component {
     if (e.type !== "wheel") e.preventDefault();
     if (camera.handleEvent(e, projectionTF)) {
       this.renderCanvas();
-      this.setState((state) => ({ ...state, updateOverlay: !state.updateOverlay }));
+      this.setState((state) => ({
+        ...state,
+        updateOverlay: !state.updateOverlay,
+      }));
     }
   };
 
@@ -509,6 +519,14 @@ class Graph extends React.Component {
     return { toolSVG: newToolSVG, tool, container };
   };
 
+  loadTextureFromUrl = (src) =>
+    new Promise((resolve, reject) => {
+      const img = new Image();
+      img.onload = () => resolve(img);
+      img.onerror = reject;
+      img.src = src;
+    });
+
   fetchAsyncProps = async (props) => {
     const {
       annoMatrix,
@@ -517,6 +535,8 @@ class Graph extends React.Component {
       crossfilter,
       pointDilation,
       viewport,
+      spatial,
+      imageUnderlay,
     } = props.watchProps;
     const { modelTF } = this.state;
 
@@ -524,7 +544,8 @@ class Graph extends React.Component {
       annoMatrix,
       layoutChoice,
       colorsProp,
-      pointDilation
+      pointDilation,
+      imageUnderlay
     );
 
     const { currentDimNames } = layoutChoice;
@@ -551,6 +572,10 @@ class Graph extends React.Component {
       pointDilationLabel
     );
 
+    this.spatialImage = await this.loadTextureFromUrl(
+      "/api/v0.2/spatial/image"
+    );
+
     const { width, height } = viewport;
     return {
       positions,
@@ -558,6 +583,8 @@ class Graph extends React.Component {
       flags,
       width,
       height,
+      spatial,
+      imageUnderlay,
     };
   };
 
@@ -721,6 +748,7 @@ class Graph extends React.Component {
       flagBuffer,
       camera,
       projectionTF,
+      drawSpatialImage,
     } = this.state;
     this.renderPoints(
       regl,
@@ -729,12 +757,14 @@ class Graph extends React.Component {
       pointBuffer,
       flagBuffer,
       camera,
-      projectionTF
+      projectionTF,
+      drawSpatialImage
     );
   });
 
   updateReglAndRender(asyncProps, prevAsyncProps) {
-    const { positions, colors, flags, height, width } = asyncProps;
+    const { positions, colors, flags, height, width, imageUnderlay } =
+      asyncProps;
     this.cachedAsyncProps = asyncProps;
     const { pointBuffer, colorBuffer, flagBuffer } = this.state;
     let needToRenderCanvas = false;
@@ -754,6 +784,9 @@ class Graph extends React.Component {
       flagBuffer({ data: flags, dimension: 1 });
       needToRenderCanvas = true;
     }
+    if (imageUnderlay !== prevAsyncProps?.imageUnderlay) {
+      needToRenderCanvas = true;
+    }
     if (needToRenderCanvas) this.renderCanvas();
   }
 
@@ -797,20 +830,25 @@ class Graph extends React.Component {
     pointBuffer,
     flagBuffer,
     camera,
-    projectionTF
+    projectionTF,
+    drawSpatialImage
   ) {
-    const { annoMatrix } = this.props;
+    const { annoMatrix, spatial, imageUnderlay } = this.props;
     if (!this.reglCanvas || !annoMatrix) return;
 
     const { schema } = annoMatrix;
     const cameraTF = camera.view();
     const projView = mat3.multiply(mat3.create(), projectionTF, cameraTF);
     const { width, height } = this.reglCanvas;
+    const imW = spatial.data.imageWidth;
+    const imH = spatial.data.imageHeight;
+
     regl.poll();
     regl.clear({
       depth: 1,
-      color: [1, 1, 1, 1],
+      color: [0, 0, 0, 0],
     });
+
     drawPoints({
       distance: camera.distance(),
       color: colorBuffer,
@@ -821,6 +859,19 @@ class Graph extends React.Component {
       nPoints: schema.dataframe.nObs,
       minViewportDimension: Math.min(width, height),
     });
+    if (imageUnderlay?.isActive) {
+      drawSpatialImage({
+        projView,
+        imageWidth: imW,
+        imageHeight: imH,
+        rectCoords: [0, 0, imW, 0, 0, imH, 0, imH, imW, 0, imW, imH],
+        spatialImageAsTexture: regl.texture({
+          data: this.spatialImage,
+          wrapS: "clamp",
+          wrapT: "clamp",
+        }),
+      });
+    }
     regl._gl.flush();
   }
 
@@ -832,6 +883,8 @@ class Graph extends React.Component {
       layoutChoice,
       pointDilation,
       crossfilter,
+      spatial,
+      imageUnderlay,
     } = this.props;
     const { modelTF, projectionTF, camera, viewport, regl } = this.state;
     const cameraTF = camera?.view()?.slice();
@@ -902,6 +955,8 @@ class Graph extends React.Component {
             pointDilation,
             crossfilter,
             viewport,
+            spatial,
+            imageUnderlay,
           }}
         >
           <Async.Pending initial>
@@ -951,32 +1006,29 @@ const ErrorLoading = ({ displayName, error, width, height }) => {
   );
 };
 
-const StillLoading = ({ displayName, width, height }) => 
+const StillLoading = ({ displayName, width, height }) => (
   /*
   Render a busy/loading indicator
   */
-   (
+  <div
+    style={{
+      position: "fixed",
+      fontWeight: 500,
+      top: height / 2,
+      width,
+    }}
+  >
     <div
       style={{
-        position: "fixed",
+        display: "flex",
-        fontWeight: 500,
+        justifyContent: "center",
-        top: height / 2,
+        justifyItems: "center",
-        width,
+        alignItems: "center",
       }}
     >
-      <div
+      <Button minimal loading intent="primary" />
-        style={{
+      <span style={{ fontStyle: "italic" }}>Loading {displayName}</span>
-          display: "flex",
-          justifyContent: "center",
-          justifyItems: "center",
-          alignItems: "center",
-        }}
-      >
-        <Button minimal loading intent="primary" />
-        <span style={{ fontStyle: "italic" }}>Loading {displayName}</span>
-      </div>
     </div>
-  )
+  </div>
-;
+);
-
 export default Graph;

client/src/components/menubar/index.js

@@ -28,6 +28,8 @@ import { getEmbSubsetView } from "../../util/stateManager/viewStackHelpers";
     subsetPossible,
     subsetResetPossible,
     graphInteractionMode: state.controls.graphInteractionMode,
+    imageUnderlay: state.imageUnderlay,
+    layoutChoice: state.layoutChoice, // TODO: really should clean up naming, s/layout/embedding/g
     clipPercentileMin: Math.round(100 * (annoMatrix?.clipRange?.[0] ?? 0)),
     clipPercentileMax: Math.round(100 * (annoMatrix?.clipRange?.[1] ?? 1)),
     userDefinedGenes: state.controls.userDefinedGenes,
@@ -206,6 +208,8 @@ class MenuBar extends React.PureComponent {
       colorAccessor,
       subsetPossible,
       subsetResetPossible,
+      imageUnderlay,
+      layoutChoice,
     } = this.props;
     const { pendingClipPercentiles } = this.state;
 
@@ -268,6 +272,29 @@ class MenuBar extends React.PureComponent {
             disabled={!isColoredByCategorical}
           />
         </Tooltip>
+        {layoutChoice?.available?.includes(globals.spatialEmbeddingKeyword) && (
+          <ButtonGroup className={styles.menubarButton}>
+            <Tooltip
+              content={"Toggle image"}
+              position="bottom"
+              hoverOpenDelay={globals.tooltipHoverOpenDelay}
+            >
+              <AnchorButton
+                type="button"
+                data-testid="toggle-image-underlay"
+                icon={"media"}
+                intent={imageUnderlay.isActive ? "primary" : "none"}
+                active={imageUnderlay.isActive}
+                onClick={() => {
+                  dispatch({
+                    type: "toggle image underlay",
+                  });
+                }}
+              />
+            </Tooltip>
+          </ButtonGroup>
+        )}
+
         <ButtonGroup className={styles.menubarButton}>
           <Tooltip
             content={selectionTooltip}

client/src/globals.js

@@ -2,6 +2,9 @@ import { Colors } from "@blueprintjs/core";
 import { dispatchNetworkErrorMessageToUser } from "./util/actionHelpers";
 import ENV_DEFAULT from "../../environment.default.json";
 
+// visium embedding word, spatial image underlay
+export const spatialEmbeddingKeyword = "spatial";
+
 /* overflow category values are created  using this string */
 export const overflowCategoryLabel = ": all other labels";
 

client/src/reducers/imageUnderlay.js

@@ -0,0 +1,14 @@
+const imageUnderlay = (state = { isActive: false }, action) => {
+  switch (action.type) {
+    case "toggle image underlay":
+      return {
+        ...state,
+        isActive: !state.isActive,
+      };
+
+    default:
+      return state;
+  }
+};
+
+export default imageUnderlay;

client/src/reducers/index.js

@@ -11,6 +11,7 @@ import continuousSelection from "./continuousSelection";
 import graphSelection from "./graphSelection";
 import colors from "./colors";
 import differential from "./differential";
+import spatial from "./spatial";
 import layoutChoice from "./layoutChoice";
 import controls from "./controls";
 import annotations from "./annotations";
@@ -19,6 +20,7 @@ import genesetsUI from "./genesetsUI";
 import autosave from "./autosave";
 import centroidLabels from "./centroidLabels";
 import pointDialation from "./pointDilation";
+import imageUnderlay from "./imageUnderlay";
 import { gcMiddleware as annoMatrixGC } from "../annoMatrix";
 
 import undoableConfig from "./undoableConfig";
@@ -38,7 +40,9 @@ const Reducer = undoable(
     ["colors", colors],
     ["controls", controls],
     ["differential", differential],
+    ["spatial", spatial],
     ["centroidLabels", centroidLabels],
+    ["imageUnderlay", imageUnderlay],
     ["pointDilation", pointDialation],
     ["autosave", autosave],
   ]),
@@ -51,6 +55,7 @@ const Reducer = undoable(
     "colors",
     "controls",
     "differential",
+    "spatial",
     "layoutChoice",
     "centroidLabels",
     "genesets",

client/src/reducers/spatial.js

@@ -0,0 +1,34 @@
+const Spatial = (
+  state = {
+    loading: null,
+    error: null,
+    metadata: null,
+  },
+  action
+) => {
+  switch (action.type) {
+    case "request spatial metadata started":
+      return {
+        ...state,
+        loading: true,
+        error: null,
+      };
+    case "request spatial metadata success":
+      return {
+        ...state,
+        error: null,
+        loading: false,
+        metadata: action,
+      };
+    case "request spatial metadata error":
+      return {
+        ...state,
+        loading: false,
+        error: action.data,
+      };
+    default:
+      return state;
+  }
+};
+
+export default Spatial;

client/src/reducers/undoableConfig.js

@@ -52,6 +52,9 @@ const skipOnActions = new Set([
   "geneset: disable add new genes mode",
   "geneset: activate rename geneset mode",
   "geneset: disable rename geneset mode",
+
+  /* spatial */
+  "toggle image underlay",
 ]);
 
 /*

client/src/util/stateManager/viewStackHelpers.js

@@ -137,13 +137,10 @@ function _getEmbeddingRowOffsets(baseRowIndex, embeddingDf) {
     - if the embedding contains NaN coordinates, return a rowIndex
       that contains only the rows with discrete valued coordinates.
 
-  Currently assumes that there will be only two dimensions in the embedding.
+  Currently assumes that there will be onl two dimensions in the embedding.
   */
-  // eslint-disable-next-line react/destructuring-assignment -- destructuring fails
   const X = embeddingDf.icol(0).asArray();
-  // eslint-disable-next-line react/destructuring-assignment -- destructuring fails
   const Y = embeddingDf.icol(1).asArray();
-
   const offsets = new Int32Array(X.length);
   let numOffsets = 0;
 

scripts/launch_and_open

@@ -1,16 +0,0 @@
-#!/usr/bin/expect -f
-
-# Mac only! (depends upon `open` command)
-
-set h5ad [lindex $argv 0]
-puts "$h5ad"
-
-spawn cellxgene launch $h5ad
-
-set timeout 10
-expect -indices -re "Please go to (http:\/\/localhost:\[0-9\]+)" {
-    set url $expect_out(1,string)
-    exec >@stdout 2>@stderr open $url
-}
-
-interact

server/__init__.py

@@ -2,7 +2,7 @@ import logging
 import sys
 from server.common.utils.utils import import_plugins
 
-__version__ = "1.0.1"
+__version__ = "1.0.0"
 display_version = "cellxgene v" + __version__
 
 try:

server/annotate/annotation_types.py

@@ -1,5 +0,0 @@
-from enum import Enum
-
-
-class AnnotationType(Enum):
-    CELL_TYPE = "cell_type"

server/app/app.py

@@ -190,6 +190,16 @@ class SummarizeVarAPI(Resource):
     def post(self, data_adaptor):
         return common_rest.summarize_var_post(request, data_adaptor)
 
+class SpatialImageAPI(Resource):
+    @rest_get_data_adaptor
+    def get(self, data_adaptor):
+        return common_rest.spatial_image_get(request, data_adaptor)
+
+class SpatialMetaAPI(Resource):
+    @rest_get_data_adaptor
+    def get(self, data_adaptor):
+        return data_adaptor.get_spatial_metadata()
+
 
 def get_api_base_resources(bp_base):
     """Add resources that are accessed from the api url"""
@@ -222,6 +232,9 @@ def get_api_dataroot_resources(bp_dataroot):
     # Computation routes
     add_resource(DiffExpObsAPI, "/diffexp/obs")
     add_resource(LayoutObsAPI, "/layout/obs")
+    # Spatial routes
+    add_resource(SpatialImageAPI, "/spatial/image")
+    add_resource(SpatialMetaAPI, "/spatial/meta")
     return api
 
 

server/cli/annotate.py

@@ -1,231 +0,0 @@
-import functools
-import json
-import os.path
-import shlex
-import shutil
-import subprocess
-import sys
-from subprocess import STDOUT, PIPE
-from tempfile import NamedTemporaryFile
-
-import click
-import pandas as pd
-from click import BadParameter
-
-from server.annotate.annotation_types import AnnotationType
-from server.common.utils.data_locator import DataLocator
-from server.common.utils.utils import sort_options
-
-
-def annotate_args(func):
-    @functools.wraps(func)
-    def wrapper(*args, **kwargs):
-        return func(*args, **kwargs)
-
-    return wrapper
-
-
-@sort_options
-@click.command(
-    short_help="Annotate H5AD file columns. Run `cellxgene annotation --help` for more information.",
-    options_metavar="<options>",
-)
-@click.option(
-    "-i",
-    "--input-h5ad-file",
-    required=True,
-    type=str,
-    help="The input H5AD file containing the missing annotations.",
-)
-@click.option(
-    "-m",
-    "--model-url",
-    required=True,
-    help="The URL of the model used to prediction annotated labels. May be a local filesystem directory "
-    "or S3 path (s3://)",
-)
-@click.option(
-    "-l",
-    "--counts-layer",
-    help="If specified, raw counts will be read from the AnnData layer of the specified name. If unspecified, "
-    "raw counts will be read from `X` matrix, unless 'raw.X' exists, in which case that will be used.",
-)
-@click.option(
-    "-g",
-    "--gene-column-name",
-    help="The name of the `var` column that contains gene identifiers. The values in this column will be used to match "
-    "genes between the query and reference datasets. If not specified, the gene identifiers are expected to exist "
-    "in `var.index`.",
-)
-# TODO: Useful if we want to support discoverability of models
-# @click.option(
-#     "-r",
-#     "--model-repository",
-#     help="The base URL of the model repository. Maybe a local filesystem directory or S3 path (s3://)"
-# )
-# TODO: Useful if we want to support other, future annotation types, beyond "Cell Type". Currently hidden
-@click.option(
-    "-a",
-    "--annotation-type",
-    type=click.Choice([t.value for t in AnnotationType]),
-    default=AnnotationType.CELL_TYPE.value,
-    show_default=True,
-    hidden=True,  # Remove if we add support for more annotation types
-    help="The type of annotation to perform. This model to be used will be inferred from the annotation type.",
-)
-@click.option(
-    "-c",
-    "--annotation-prefix",
-    type=str,
-    default="cxg",
-    show_default=True,
-    help="An optional prefix used to form the names of: 1) new `obs` annotation columns that will store the predicted "
-    "annotation values and confidence scores, 2) `obsm` embeddings (reference and umap embedding), and "
-    "3) `uns` metadata for the prediction operation",
-)
-@click.option(
-    "-n",
-    "--run-name",
-    type=str,
-    help="An optional run name that will be used as a suffix to form the names of new `obs` annotation columns that "
-    "will store the predicted annotation values and confidence scores. This can be used to allow multiple "
-    "annotation predictions to be run on a single AnnData object.",
-)
-@click.option(
-    "-u",
-    "--update-h5ad-file",
-    is_flag=True,
-    help="Flag indicating whether to update the input h5ad file with annotation values.  This option is mutually "
-    "exclusive with --output-h5ad-file.",
-)
-@click.option(
-    "-o",
-    "--output-h5ad-file",
-    help="The output H5AD file that will contain the generated annotation values. This option is mutually "
-    "exclusive with --update-h5ad-file.",
-)
-@click.option("--use-model-cache/--no-use-model-cache", default=True)
-@click.option(
-    "--use-gpu/--no-use-gpu",
-    default=True,
-    help="Whether to use a GPU for annotation operations (highly recommended, if available).",
-)
-# TODO: This is a cell type model-specific arg, so not ideal to specify here as a hardcoded option
-@click.option(
-    "--classifier",
-    default="default",
-    help="For cell type annotation, the classifier level to use. The classifier is model-dependent, so refer to "
-    "documentation for the specified model for valid values.",
-)
-# TODO: This is a cell type model-specific arg, so not ideal to specify here as a hardcoded option
-@click.option(
-    "--organism",
-    type=click.Choice(["Homo sapiens", "Mus musculus"], case_sensitive=True),
-    default="Homo sapiens",
-    help="For cell type annotation, the organism of the dataset. Used to normalize gene names to HGLC conventions when "
-    "an annotation model has been trained using data from different organism.",
-)
-@click.option(
-    "--model-cache-dir",
-    default=".models_cache",
-    help="Local directory used to store model files that are retrieved from a remote location. Model files will "
-    "be read from this directory first, if they exist, to avoid repeating large downloads.",
-)
-@click.option(
-    "--mlflow-env-manager",
-    type=click.Choice(["virtualenv", "conda", "local"]),
-    default="virtualenv",
-    help="Annotation model prediction will be installed and executed in the specified type of environment. MacOS users "
-    "on Apple Silicon (arm64, M1, M2, etc.) are recommended to use 'conda' to avoid Python package installation "
-    "errors. If 'conda' is specified then cellxgene must also have been installed within a conda environment",
-)
-@click.help_option("--help", "-h", help="Show this message and exit.")
-def annotate(**cli_args):
-    _validate_options(cli_args)
-
-    print(f"Reading query dataset {cli_args['input_h5ad_file']}...")
-
-    annotation_prefix = "_".join(
-        filter(None, [cli_args.get("annotation_prefix"), cli_args.get("annotation_type"), cli_args.get("run_name")])
-    )
-
-    output_h5ad_file = cli_args["input_h5ad_file"] if cli_args["update_h5ad_file"] else cli_args["output_h5ad_file"]
-
-    model_url = cli_args.get("model_url")
-    local_model_path = _retrieve_model(cli_args.get("model_cache_dir"), model_url, cli_args.get("use_model_cache"))
-
-    print(f"Annotating {cli_args.get('input_h5ad_file')} with {cli_args.get('annotation_type')}...")
-
-    if cli_args["annotation_type"] == AnnotationType.CELL_TYPE.value:
-        predict_args = dict(
-            query_dataset_h5ad_path=cli_args.get("input_h5ad_file"),
-            output_h5ad_path=output_h5ad_file,
-            annotation_prefix=annotation_prefix,
-            counts_layer=cli_args.get("counts_layer"),
-            gene_column_name=cli_args.get("gene_column_name"),
-            classifier=cli_args.get("classifier"),
-            organism=cli_args.get("organism"),
-            use_gpu=cli_args.get("use_gpu"),
-        )
-        # Drop args that have values of `None` as these will cause problems when passing into MLflow predict, since it
-        # ultimately gets converted into 1-row Pandas DataFrame (None is interpreted as a float type column!)
-        predict_args = dict([(k, v) for k, v in predict_args.items() if v is not None])
-
-        # Invoke prediction using MLflow cli, as a separate process.
-        # This fully prepares the Python environment that is needed for executing the model.
-        # The Python environment will be reused after it is setup once.
-        with NamedTemporaryFile(buffering=0) as predict_args_file:
-            # write the mlflow predict arguments to a csv file, which will be passed to mlflow cmd
-            pd.DataFrame([json.dumps(predict_args)]).to_csv(predict_args_file, index=None)
-            predict_args_file.seek(0)
-
-            # run mlflow prediction in subprocess
-            predict_cmd = (
-                f"mlflow models predict "
-                f"--env-manager {cli_args['mlflow_env_manager']} "
-                f"--model-uri {local_model_path} "
-                f"--content-type csv --input-path {predict_args_file.name}"
-            )
-            p = subprocess.Popen(
-                args=shlex.split(predict_cmd), stdin=predict_args_file, text=True, bufsize=0, stdout=PIPE, stderr=STDOUT
-            )
-
-            # display mlflow process output as it runs
-            for line in p.stdout:
-                print(line.rstrip())
-
-            p.wait()
-            if p.returncode == 0:
-                print(f"Wrote annotations to {cli_args.get('output_h5ad_file')}")
-            else:
-                print("Annotation failed!")
-    else:
-        raise BadParameter(f"unknown annotation type {cli_args['annotation_type']}")
-
-
-def _retrieve_model(model_cache_dir, model_url, use_cache=True):
-    local_cache_model_path = os.path.join(model_cache_dir, os.path.splitext(os.path.basename(model_url))[0])
-    if not os.path.exists(local_cache_model_path) or not use_cache:
-        print(f"Retrieving model from {model_url}")
-        # download from remote source
-        with DataLocator(model_url).local_handle() as model_archive_local_path:
-            # unpack archive to local cache dir
-            shutil.unpack_archive(model_archive_local_path, local_cache_model_path)
-    else:
-        print(f"Using cached model at {local_cache_model_path}")
-
-    return local_cache_model_path
-
-
-def _validate_options(cli_args):
-    # TODO(atolopko): Use cloup library for this logic
-    if cli_args["update_h5ad_file"] and cli_args["output_h5ad_file"]:
-        click.echo("--update_h5ad_file and --output_h5ad_file are mutually exclusive")
-        sys.exit(1)
-    if not (cli_args["update_h5ad_file"] or cli_args["output_h5ad_file"]):
-        click.echo("--update_h5ad_file or --output_h5ad_file must be specified")
-        sys.exit(1)
-
-
-if __name__ == "__main__":
-    annotate()

server/cli/cli.py

@@ -1,6 +1,5 @@
 import click
 
-from .annotate import annotate
 from .launch import launch
 from .prepare import prepare
 from .upgrade import log_upgrade_check
@@ -32,5 +31,4 @@ def cli(upgrade_check):
 
 
 cli.add_command(launch)
-cli.add_command(annotate)
 cli.add_command(prepare)

server/common/annotations/annotations.py

@@ -57,7 +57,7 @@ class Annotations(metaclass=ABCMeta):
         pass
 
     @abstractmethod
-    def write_gene_sets(self, gs, tid, data_adaptor):
+    def write_gene_sets(self, gs, data_adaptor):
         """Write the gene sets (gs) to a persistent storage such that it can later be read"""
         pass
 

server/common/annotations/local_file_csv.py

@@ -7,7 +7,6 @@ from hashlib import blake2b
 
 import pandas as pd
 from flask import session
-from fsspec import AbstractFileSystem
 
 from server import __version__ as cellxgene_version
 from server.app.session import get_user_id
@@ -63,27 +62,21 @@ class AnnotationsLocalFile(Annotations):
         self.check_user_annotations_enabled()  # raises
 
         fname = self._get_celllabels_filename(data_adaptor)
-        empty_labels = pd.DataFrame()
-        if fname is None:
-            return empty_labels
-
         with self.label_lock:
-            locator = DataLocator(fname)
+            if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0:
-            if not locator.exists() or locator.size() == 0:
+                # returned the cached labels if possible, otherwise read them from the file
-                return empty_labels
+                if fname == self.last_label_fname:
-
+                    return self.last_labels
-            # return the cached labels if possible
+                else:
-            if fname == self.last_label_fname:
+                    labels = pd.read_csv(
-                return self.last_labels
+                        fname, dtype="category", index_col=0, header=0, comment="#", keep_default_na=False
-
+                    )
-            # otherwise, read labels from file
+                    # update the cache
-            with locator.open() as f:
+                    self.last_label_fname = fname
-                labels = pd.read_csv(f, dtype="category", index_col=0, header=0, comment="#", keep_default_na=False)
+                    self.last_labels = labels
-
+                    return labels
-            # update the cache
+            else:
-            self.last_label_fname = fname
+                return pd.DataFrame()
-            self.last_labels = labels
-            return labels
 
     def write_labels(self, df, data_adaptor):
         self.check_user_annotations_enabled()  # raises
@@ -102,12 +95,13 @@ class AnnotationsLocalFile(Annotations):
 
             fname = self._get_celllabels_filename(data_adaptor)
             self._backup(fname)
-            locator = DataLocator(fname)
+            if not df.empty:
-            with locator.open("w") as f:
+                with open(fname, "w", newline="") as f:
-                if not df.empty:
                     if header is not None:
                         f.write(header)
                     df.to_csv(f)
+            else:
+                open(fname, "w").close()
 
             # update the cache
             self.last_label_fname = fname
@@ -115,32 +109,26 @@ class AnnotationsLocalFile(Annotations):
 
     def read_gene_sets(self, data_adaptor, context=None):
         fname = self._get_genesets_filename(data_adaptor)
-        empty_gene_sets = {}
+        gene_sets = {}
-
+        tid = None
         with self.gene_sets_lock:
             tid = self.last_geneset_tid  # inside the critical section
-            if fname is None:
+            if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0:
-                return (empty_gene_sets, tid)
+                # return the cached genesets if possible, otherwise read from file and validate them
+                if fname == self.last_geneset_fname:
+                    gene_sets = self.last_geneset
+                else:
+                    # read
+                    gene_sets = read_gene_sets_tidycsv(DataLocator(fname), context)
 
-            locator = DataLocator(fname)
+                    # validate
-            if not locator.exists() or locator.size() == 0:
+                    gene_sets = data_adaptor.check_new_gene_sets(gene_sets, context)
-                return (empty_gene_sets, tid)
 
-            # return the cached genesets if possible, otherwise read from file and validate them
+                    # update cache
-            if fname == self.last_geneset_fname:
+                    self.last_geneset_fname = fname
-                return (self.last_geneset, tid)
+                    self.last_geneset = gene_sets
 
-            # read
+        return (gene_sets, tid)
-            gene_sets = read_gene_sets_tidycsv(locator, context)
-
-            # validate
-            gene_sets = data_adaptor.check_new_gene_sets(gene_sets, context)
-
-            # update cache
-            self.last_geneset_fname = fname
-            self.last_geneset = gene_sets
-
-            return (gene_sets, tid)
 
     def write_gene_sets(self, gene_sets, tid, data_adaptor):
         self.check_gene_sets_save_enabled()  # raises
@@ -169,9 +157,9 @@ class AnnotationsLocalFile(Annotations):
 
             fname = self._get_genesets_filename(data_adaptor)
             self._backup(fname)
-            locator = DataLocator(fname)
+            with open(fname, "w", newline="") as f:
-            with locator.open("w", newline="") as f:
+                f.write(header)
-                f.write(header + self.gene_sets_to_csv(gene_sets))
+                f.write(self.gene_sets_to_csv(gene_sets))
 
             # update the cache
             self.last_geneset_fname = fname
@@ -193,7 +181,7 @@ class AnnotationsLocalFile(Annotations):
 
         output_file = self.label_output_file or self.gene_sets_output_file
         if output_file:
-            return os.path.dirname(DataLocator(output_file).abspath())
+            return os.path.dirname(os.path.abspath(output_file))
 
         return os.getcwd()
 
@@ -232,37 +220,34 @@ class AnnotationsLocalFile(Annotations):
             1. fname -> backup_dir/fname-TIME
             2. delete excess files in backup_dir
         """
-        locator = DataLocator(fname)
+        root, ext = os.path.splitext(fname)
-        fs: AbstractFileSystem = locator.fs  # Handle to underlying fsspec file system
-
-        # Make sure there is work to do
-        if not locator.exists():
-            return
-
-        root, ext = os.path.splitext(locator.abspath())
         backup_dir = f"{root}-backups"
 
+        # Make sure there is work to do
+        if not os.path.exists(fname):
+            return
+
         # Ensure backup_dir exists
-        fs.mkdirs(backup_dir, exist_ok=True)
+        if not os.path.exists(backup_dir):
+            os.mkdir(backup_dir)
 
         # Save current file to backup_dir
         fname_base = os.path.basename(fname)
         fname_base_root, fname_base_ext = os.path.splitext(fname_base)
-        # don't use ISO standard time format, as it contains characters illegal on some filesystems.
+        # don't use ISO standard time format, as it contains characters illegal on some filesytems.
         nowish = datetime.now().strftime("%Y-%m-%dT%H-%M-%S")
         backup_fname = os.path.join(backup_dir, f"{fname_base_root}-{nowish}{fname_base_ext}")
-        if fs.exists(backup_fname):
+        if os.path.exists(backup_fname):
-            fs.delete(backup_fname)
+            os.remove(backup_fname)
-        fs.rename(fname, backup_fname)
+        os.rename(fname, backup_fname)
 
         # prune the backup_dir to max number of backup files, keeping the most recent backups
-        backup_path_prefix = DataLocator.strip_protocol(os.path.join(backup_dir, fname_base_root + "-"))
+        backups = list(filter(lambda s: s.startswith(fname_base_root), os.listdir(backup_dir)))
-        backups = list(filter(lambda s: s.startswith(backup_path_prefix), fs.ls(backup_dir)))
+        excess_count = len(backups) - max_backups
-
+        if excess_count > 0:
-        # sorting to drop the oldest
+            backups.sort()
-        excess_backups = list(sorted(backups, reverse=True))[max_backups:]
+            for bu in backups[0:excess_count]:
-        for bu in excess_backups:
+                os.remove(os.path.join(backup_dir, bu))
-            fs.delete(bu)
 
     def update_parameters(self, parameters, data_adaptor):
         params = {}

server/common/config/dataset_config.py

@@ -4,7 +4,6 @@ from os.path import splitext, isdir
 from server.common.annotations.local_file_csv import AnnotationsLocalFile
 from server.common.config.base_config import BaseConfig
 from server.common.errors import ConfigurationError, AnnotationsError
-from server.common.utils.data_locator import DataLocator
 from server.data_common.matrix_loader import MatrixDataLoader
 
 
@@ -128,15 +127,11 @@ class DatasetConfig(BaseConfig):
             if lf_ext and lf_ext != ".csv":
                 raise ConfigurationError(f"genesets file type must be .csv: {genesets_filename}")
 
-        if dirname is not None:
+        if dirname is not None and not isdir(dirname):
-            if not DataLocator(dirname).islocal():
+            try:
-                # remote object stores only support objects but not directories, do nothing
+                os.mkdir(dirname)
-                pass
+            except OSError:
-            elif not isdir(dirname):
+                raise ConfigurationError("Unable to create directory specified by --user-generated-data-dir")
-                try:
-                    os.mkdir(dirname)
-                except OSError:
-                    raise ConfigurationError("Unable to create directory specified by --user-generated-data-dir")
 
         anno_config = {
             "user-annotations": self.user_annotations__enable,

server/common/rest.py

@@ -4,8 +4,9 @@ import sys
 from http import HTTPStatus
 import zlib
 import json
+import numpy as np
 
-from flask import make_response, jsonify, current_app, abort
+from flask import make_response, jsonify, current_app, abort, send_file
 from werkzeug.urls import url_unquote
 
 from server.common.config.client_config import get_client_config
@@ -293,7 +294,7 @@ def layout_obs_get(request, data_adaptor):
 
     try:
         return make_response(
-            data_adaptor.layout_to_fbs_matrix(fields), HTTPStatus.OK, {"Content-Type": "application/octet-stream"}
+            data_adaptor.layout_to_fbs_matrix(fields, data_adaptor.get_spatial()), HTTPStatus.OK, {"Content-Type": "application/octet-stream"}
         )
     except (KeyError, DatasetAccessError) as e:
         return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
@@ -397,3 +398,40 @@ def summarize_var_post(request, data_adaptor):
 
     key = request.args.get("key", default=None)
     return summarize_var_helper(request, data_adaptor, key, request.get_data())
+
+def spatial_image_get(request, data_adaptor):
+    import io
+    import matplotlib.pyplot    
+
+    resolution = "hires"
+    spatial = data_adaptor.get_spatial()
+
+    if len(list(spatial)) == 0:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, "uns does not have spatial information")
+
+    library_id = list(spatial)[0]
+    if len(spatial) > 1:
+        current_app.logger.warning(f"More than one library found under uns.spatial, using library '{library_id}'")
+
+    if "images" not in spatial[library_id]:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, "spatial information does not contain images")
+
+    if resolution not in spatial[library_id]["images"]:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, f"spatial information does not contain requested resolution '{resolution}'")
+
+    response_image = io.BytesIO()
+    img = spatial[library_id]["images"][resolution]
+    matplotlib.pyplot.imsave(response_image, img)
+    response_image.seek(0)
+
+    try:
+        return send_file(response_image, attachment_filename=f"{library_id}-{resolution}.png", mimetype="image/png")
+    except (KeyError, DatasetAccessError) as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+    except PrepareError:
+        return abort_and_log(
+            HTTPStatus.NOT_IMPLEMENTED,
+            f"No spatial image available {request.path}",
+            loglevel=logging.ERROR,
+            include_exc_info=True,
+        ) 

server/common/utils/data_locator.py

@@ -52,10 +52,8 @@ class DataLocator:
             self.fs = fsspec.filesystem(self.protocol)
 
     def __repr__(self):
-        return (
+        return f"DataLocator(protocol={self.protocol}, cname={self.cname}, "
-            f"DataLocator(protocol={self.protocol}, cname={self.cname}, "
+        f"path={self.path}, uri_or_path={self.uri_or_path})"
-            f"path={self.path}, uri_or_path={self.uri_or_path})"
-        )
 
     @staticmethod
     def _get_protocol_and_path(uri_or_path):
@@ -67,10 +65,6 @@ class DataLocator:
                 return protocol, path
         return None, uri_or_path
 
-    @staticmethod
-    def strip_protocol(uri_or_path):
-        return DataLocator._get_protocol_and_path(uri_or_path)[1]
-
     def exists(self):
         return self.fs.exists(self.cname)
 
@@ -78,7 +72,7 @@ class DataLocator:
         return self.fs.size(self.cname)
 
     def lastmodtime(self):
-        """return datetime object representing last modification time, or None if unavailable"""
+        """ return datetime object representing last modification time, or None if unavailable """
         info = self.fs.info(self.cname)
         if self.islocal() and info is not None:
             return datetime.fromtimestamp(info["mtime"])
@@ -98,8 +92,8 @@ class DataLocator:
     def isfile(self):
         return self.fs.isfile(self.cname)
 
-    def open(self, *args, **kwargs):
+    def open(self, *args):
-        return self.fs.open(self.uri_or_path, *args, **kwargs)
+        return self.fs.open(self.uri_or_path, *args)
 
     def islocal(self):
         return self.protocol is None or self.protocol == "file"
@@ -113,9 +107,10 @@ class DataLocator:
         # do our best to create a file with the same.
         ext = os.path.splitext(self.path)
         suffix = None if ext[1] == "" else ext[1]
-        with tempfile.NamedTemporaryFile(prefix="cellxgene_", suffix=suffix, delete=False) as tmp:
+        with self.open() as src, tempfile.NamedTemporaryFile(prefix="cellxgene_", suffix=suffix, delete=False) as tmp:
-            self.fs.download(self.uri_or_path, tmp.name)
+            tmp.write(src.read())
             tmp.close()
+            src.close()
             tmp_path = tmp.name
             return LocalFilePath(tmp_path, delete=True)
 

server/data_anndata/anndata_adaptor.py

@@ -174,14 +174,10 @@ class AnndataAdaptor(DataAdaptor):
         except MemoryError:
             raise DatasetAccessError("Out of memory - file is too large for available memory.")
         except Exception:
-            import traceback
+            raise DatasetAccessError(
-            message = (
                 "File not found or is inaccessible. File must be an .h5ad object. "
                 "Please check your input and try again."
-                )
+            )
-            if self.server_config.app__verbose:
-                message += f"\n{traceback.format_exc()}"
-            raise DatasetAccessError(message)
 
     def _validate_and_initialize(self):
         if anndata_version_is_pre_070():
@@ -240,14 +236,6 @@ class AnndataAdaptor(DataAdaptor):
             warnings.warn(
                 f"Anndata data matrix is in {self.data.X.dtype} format not float32. " f"Precision may be truncated."
             )
-        if self.data.X.dtype < np.float32:
-            if self.data.isbacked:
-                raise DatasetAccessError(f"Data matrix in {self.data.X.dtype} format is not supported in backed mode."
-                                         " Please reload without --backed, or convert matrix to float32")
-            warnings.warn(
-                f"Anndata data matrix is in unsupported {self.data.X.dtype} format -- will be cast to float32"
-            )
-            self.data.X = self.data.X.astype(np.float32)
         for ax in Axis:
             curr_axis = getattr(self.data, str(ax))
             for ann in curr_axis:
@@ -286,6 +274,43 @@ class AnndataAdaptor(DataAdaptor):
             df = df[fields]
         return encode_matrix_fbs(df, col_idx=df.columns)
 
+    def get_spatial(self):
+        return self.data.uns["spatial"]
+
+    def get_spatial_metadata(self):
+        spatial = self.get_spatial()
+
+        resolution = "hires"
+
+        if len(list(spatial)) == 0:
+            raise Exception("uns does not have spatial information")
+
+        library_id = list(spatial)[0]
+
+        if "images" not in spatial[library_id]:
+            raise Exception("spatial information does not contain images")
+
+        if resolution not in spatial[library_id]["images"]:
+            raise Exception(f"spatial information does not contain requested resolution '{resolution}'")
+
+        scaleref = spatial[library_id]["scalefactors"][f"tissue_{resolution}_scalef"]
+        (h, w, _) = spatial[library_id]["images"][resolution].shape
+
+        A = self.data.obsm["X_spatial"]
+        min = np.nanmin(A, axis=0)
+        max = np.nanmax(A, axis=0)
+        scale = np.amax(max - min)
+        translate = 0.5 - ((max - min) / scale / 2)
+
+        return {
+            "imageWidth": w,
+            "imageHeight": h,
+            "scaleref": scaleref,
+            "inverseScale": int(scale),
+            "inverseTranslate": translate.tolist(),
+            "inverseMin": min.tolist(),
+        }
+
     def get_embedding_names(self):
         """
         Return pre-computed embeddings.

server/data_common/data_adaptor.py

@@ -340,31 +340,57 @@ class DataAdaptor(metaclass=ABCMeta):
         pass
 
     @staticmethod
-    def normalize_embedding(embedding):
+    def normalize_embedding(embedding, spatial = None):
         """Normalize embedding layout to meet client assumptions.
-        Embedding is an ndarray, shape (n_obs, n)., where n is normally 2
+        Embedding is an ndarray, shape (n_obs, n)., where n is normally 2.
+        Note: if spatial data is available, the normalization will be done
+        according to the size of the underlying image
         """
 
-        # scale isotropically
+        if spatial is not None:
-        try:
-            min = np.nanmin(embedding, axis=0)
-            max = np.nanmax(embedding, axis=0)
-        except RuntimeError:
-            # indicates entire array was NaN, which should propagate
-            min = np.NaN
-            max = np.NaN
 
-        scale = np.amax(max - min)
+            # TODO: sync with the code in spatial_data_get
-        normalized_layout = (embedding - min) / scale
+            resolution = "hires"
 
-        # translate to center on both axis
+            if len(list(spatial)) == 0:
-        translate = 0.5 - ((max - min) / scale / 2)
+                raise Exception("uns does not have spatial information")
-        normalized_layout = normalized_layout + translate
+
+            library_id = list(spatial)[0]
+
+            if "images" not in spatial[library_id]:
+                raise Exception("spatial information does not contain images")
+
+            if resolution not in spatial[library_id]["images"]:
+                raise Exception(f"spatial information does not contain requested resolution '{resolution}'")
+
+            scaleref = spatial[library_id]["scalefactors"][f"tissue_{resolution}_scalef"]
+            (h, w, _) = spatial[library_id]["images"][resolution].shape
+
+            A = embedding * scaleref
+            A = np.column_stack([A[:, 0] / w, A[:, 1] / h])
+            normalized_layout = A.astype(dtype=np.float32)
+
+        else:
+
+            # scale isotropically
+            try:
+                min = np.nanmin(embedding, axis=0)
+                max = np.nanmax(embedding, axis=0)
+            except RuntimeError:
+                # indicates entire array was NaN, which should propagate
+                min = np.NaN
+                max = np.NaN
+
+            scale = np.amax(max - min)
+            normalized_layout = (embedding - min) / scale
+
+            # translate to center on both axis
+            translate = 0.5 - ((max - min) / scale / 2)
+            normalized_layout = normalized_layout + translate
 
-        normalized_layout = normalized_layout.astype(dtype=np.float32)
         return normalized_layout
 
-    def layout_to_fbs_matrix(self, fields):
+    def layout_to_fbs_matrix(self, fields, spatial = None):
         """
         return specified embeddings as a flatbuffer, using the cellxgene matrix fbs encoding.
 
@@ -380,7 +406,7 @@ class DataAdaptor(metaclass=ABCMeta):
         with ServerTiming.time("layout.query"):
             for ename in embeddings:
                 embedding = self.get_embedding_array(ename, 2)
-                normalized_layout = DataAdaptor.normalize_embedding(embedding)
+                normalized_layout = DataAdaptor.normalize_embedding(embedding, ename == "spatial" and spatial)
                 layout_data.append(pd.DataFrame(normalized_layout, columns=[f"{ename}_0", f"{ename}_1"]))
 
         with ServerTiming.time("layout.encode"):

server/requirements-annotate.txt

@@ -1,2 +0,0 @@
-mlflow
-scanpy

server/requirements-dev.txt

@@ -2,9 +2,9 @@ black
 bumpversion>=0.5
 codecov>=2.0.15
 parameterized>=0.7.0
+psycopg2-binary>=2.8.5
 pytest>=3.6.3
 python-jose>=3.2.0
 twine>=1.12.1
 -r requirements.txt
 -r requirements-prepare.txt
--r requirements-annotate.txt

server/requirements.txt

@@ -14,8 +14,9 @@ flatten-dict>=0.2.0
 fsspec>=0.4.4,<0.8.0
 gunicorn>=20.0.4
 h5py>=3.0.0
+matplotlib>=3.5.0
 numba>=0.51.2
-numpy>=1.17.5,<=1.22
+numpy>=1.17.5
 packaging>=20.0
 pandas>=1.0,!=1.1  # pandas 1.1 breaks tests, https://github.com/pandas-dev/pandas/issues/35446
 PyYAML>=5.4  # CVE-2020-14343

setup.py

@@ -9,12 +9,9 @@ with open("server/requirements.txt") as fh:
 with open("server/requirements-prepare.txt") as fh:
     requirements_prepare = fh.read().splitlines()
 
-with open("server/requirements-annotate.txt") as fh:
-    requirements_annotate = fh.read().splitlines()
-
 setup(
     name="cellxgene",
-    version="1.0.1",
+    version="1.0.0",
     packages=find_packages(),
     url="https://github.com/chanzuckerberg/cellxgene",
     license="MIT",
@@ -43,5 +40,5 @@ setup(
         "Topic :: Scientific/Engineering :: Bio-Informatics",
     ],
     entry_points={"console_scripts": ["cellxgene = server.cli.cli:cli"]},
-    extras_require=dict(prepare=requirements_prepare, annotate=requirements_annotate),
+    extras_require=dict(prepare=requirements_prepare),
 )

test/unit/cli/fixtures/__init__.py

@@ -1,5 +0,0 @@
-from .mlflow_model_fixture import FakeModel
-
-
-def _load_pyfunc(data_path):
-    return FakeModel()

test/unit/cli/fixtures/mlflow_model_fixture.py

@@ -1,11 +0,0 @@
-import mlflow
-
-
-class FakeModel(mlflow.pyfunc.PythonModel):
-    def __init__(self, input_to_output: dict = {}):
-        self.input_to_output = input_to_output
-
-    def predict(self, model_input) -> None:
-        # this stdout output is useful for validating the input in a test, noting that this model will be invoked in a
-        # subprocess, so stdout is one means of communicating information back to the test code
-        print(f"__MODEL_INPUT__={model_input.iloc[0][0]}")

test/unit/cli/test_annotate.py

@@ -1,111 +0,0 @@
-import os
-import shutil
-import unittest
-from tempfile import mkstemp, TemporaryDirectory
-
-import mlflow
-from click.testing import CliRunner
-
-from server.cli.annotate import annotate
-from test.unit.cli.fixtures.mlflow_model_fixture import FakeModel
-
-
-def write_model(model) -> str:
-    with TemporaryDirectory() as mlflow_model_dir:
-        fixtures_path = os.path.join(os.path.dirname(__file__), 'fixtures')
-        mlflow.pyfunc.save_model(mlflow_model_dir,
-                                 loader_module='fixtures',
-                                 code_path=[fixtures_path])
-        return shutil.make_archive(mkstemp()[1], "zip", mlflow_model_dir)
-
-
-class TestCliAnnotate(unittest.TestCase):
-    def test__annotate__loads_and_runs(self):
-        """
-        Invokes the `annotate` subcommand of cellxgene CLI, using a CliRunner() programmatic invocation.
-
-        This tests the happy path case:
-        1) Command line options are parsed;
-        2) An MLflow model zip archive can be read in (from local disk), unpacked, and invoked;
-        3) The correct options are passed to the MLflow model.
-        4) The annotate subcommand exits successfully.
-
-        This does not verify model output or predictions (it's a fake MLflow model, after all); it's up to the real model
-        to output its predictions as it wants, but this is specific to the model and so not tested here.
-
-        The  CliRunner() invokes the subcommand in a subprocess, and the annotate subcommand itself invokes the MLflow
-        model in yet another subprocess. So while this test can help determine if everything is working, it is not a
-        simple matter to debug in the case of a failure. However, the stdout/stderr of the MLflow process is captured
-        by the CliRunner() subprocess, so errors can be inspected in result.stdout when debugging this test. Hope this
-        helps!
-        """
-
-        _, query_dataset_file_path = mkstemp()
-        model_file_path = write_model(FakeModel())
-
-        result = CliRunner().invoke(
-            annotate,
-            [
-                "--input-h5ad-file",
-                query_dataset_file_path,
-                "--model-url",
-                model_file_path,
-                "--output-h5ad-file",
-                f"{query_dataset_file_path}.output",
-                # avoid having mflow create conda env or virtualenv when in test env;
-                # this avoids making pip remote requests and is also faster
-                "--mlflow-env-manager", "local"
-            ],
-        )
-
-        # to help debugging, show the output from the CliRunner and MLflow stdout
-        if result.exit_code:
-            print(result.stdout)
-
-        self.assertEqual(0, result.exit_code, "runs successfully")
-
-        # The FakeModel will print it inputs to stdout, as "__MODEL_INPUT__={...}", allowing us to assert that it received valid inputs.
-        self.assertIn(
-            "__MODEL_INPUT__={"
-            f'"query_dataset_h5ad_path": "{query_dataset_file_path}", '
-            f'"output_h5ad_path": "{query_dataset_file_path}.output", '
-            '"annotation_prefix": "cxg_cell_type", "classifier": "default", '
-            '"organism": "Homo sapiens", "use_gpu": true}',
-            result.stdout,
-            "inputs passed correctly",
-        )
-
-    def test__annotate__verifies_mutually_exclusive_options(self):
-        required_options = ["--input-h5ad-file", "some.h5ad", "--model-url", "some_url"]
-        result = CliRunner().invoke(
-            annotate,
-            required_options + [],
-        )
-
-        self.assertNotEqual(0, result.exit_code, "aborts with non-success code")
-        self.assertIn(
-            "--update_h5ad_file or --output_h5ad_file must be specified",
-            result.stdout,
-            "error message displayed",
-        )
-
-        result = CliRunner().invoke(
-            annotate, required_options + ["--output-h5ad-file", "some_arg", "--update-h5ad-file"]
-        )
-
-        self.assertNotEqual(0, result.exit_code, "aborts with non-success code")
-        self.assertIn(
-            "--update_h5ad_file and --output_h5ad_file are mutually exclusive",
-            result.stdout,
-            "error message displayed",
-        )
-
-
-# TODO:
-# Test annotate cli args more comprehensively
-# Test server.cli.annotate._validate_options
-# Test model caching feature works
-# Test model loading from s3 works (maybe w/just a real model)
-
-if __name__ == "__main__":
-    unittest.main()

test/unit/common/test_api.py

@@ -290,7 +290,6 @@ class EndPoints(object):
         result_data = result.json()
         self.assertEqual(result_data, pbmc3k_colors)
 
-    @unittest.skip('needs fix: https://github.com/chanzuckerberg/cellxgene/issues/2542')
     def test_static(self):
         endpoint = "static"
         file = "assets/favicon.ico"

test/unit/data_anndata/test_anndata_adaptor.py

@@ -1,4 +1,5 @@
 import json
+import sys
 import time
 import unittest
 
@@ -36,7 +37,6 @@ Test the anndata adaptor using the pbmc3k data set.
         (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", True, "normal"),
         (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", True, "normal"),
         (f"{FIXTURES_ROOT}/pbmc3k_64.h5ad", False, "auto"),  # 64 bit conversion tests
-        (f"{FIXTURES_ROOT}/pbmc3k_16.h5ad", False, "auto"),  # 16 bit conversion tests
     ],
 )
 class AdaptorTest(unittest.TestCase):

test/unit/data_anndata/test_anndata_adaptor_errors.py

@@ -1,23 +0,0 @@
-import unittest
-
-from parameterized import parameterized_class
-
-from server.common.errors import DatasetAccessError
-from test import FIXTURES_ROOT
-from test.unit import app_config
-
-
-@parameterized_class(
-    ("data_locator", "backed", "X_approximate_distribution"),
-    [
-        (f"{FIXTURES_ROOT}/pbmc3k_16.h5ad", True, "auto"),  # 16 bit conversion tests
-    ],
-)
-class AdaptorLoadErrorTest(unittest.TestCase):
-    def test_float16_backed_raises_err(self):
-        with self.assertRaises(DatasetAccessError):
-            config = app_config(
-                self.data_locator,
-                backed=self.backed,
-                extra_dataset_config=dict(X_approximate_distribution=self.X_approximate_distribution),
-            )