Add README-visium

README-visium.md

@@ -0,0 +1,20 @@
+# Cellxgene Visium Beta
+
+## How it works
+1. Launch `cellxgene` as normal. 
+1. If the loaded dataset has spatial information available, the image data will be loaded on startup.
+1. On the toolbar, next to the Zoom icon, a `Toggle image` button will now appear. Click on it and the image will be added as an underlay.
+1. You can now use any `cellxgene` functionality and the image will still be present. If you pan and zoom, the image will also be panned and zoomed.
+1. If you want to hide the image, you can click on `Toggle image` again
+
+In order for the image to be displayed with the correct size and alignment, the H5AD needs to have a few requirements. See the following section to learn more.
+
+## h5ad requirements
+1. The spatial embedding layer should be contained in `obsm` and be named `X_spatial`. Other layers can exist, but only this one will have the spatial feature enabled.
+2. A `spatial` dict needs to be defined in the `uns` dictionary.
+3. Inside the `spatial` dict, an `images` dict must be defined. 
+4. The `images` dict must contain a `hires` key, which should reference an image encoded as an RGB matrix (i.e., a three-dimensional matrix of size `height x width x 3` where the final dimension has the RGB values for each pixel)
+5. The `images` dict must contain a `scalefactors` dict. This should in turn contain a `tissue_hires_scalef` key, which should reference a floating point number.
+
+Moreover, in order to have the image correctly aligned with the dots, the following must be true:
+1. `tissue_hires_scalef` should represent the ratio between the embedding layer `X_spatial` and the image matrix. In particular, if you multiply `X_spatial` by `tissue_hires_scalef`, you should obtain an array of points that ovelap the tissue image if you plot them in a plane.

client/src/actions/index.js

@@ -8,6 +8,7 @@ import {
 import { loadUserColorConfig } from "../util/stateManager/colorHelpers";
 import * as selnActions from "./selection";
 import * as annoActions from "./annotation";
+import * as spatialActions from "./spatial";
 import * as viewActions from "./viewStack";
 import * as embActions from "./embedding";
 import * as genesetActions from "./geneset";
@@ -272,4 +273,5 @@ export default {
   genesetDelete: genesetActions.genesetDelete,
   genesetAddGenes: genesetActions.genesetAddGenes,
   genesetDeleteGenes: genesetActions.genesetDeleteGenes,
+  requestSpatialMetadata: spatialActions.requestSpatialMetadata,
 };

client/src/actions/spatial.js

@@ -0,0 +1,35 @@
+import * as globals from "../globals";
+
+export const requestSpatialMetadata = () => async (dispatch) => {
+  dispatch({ type: "request spatial metadata started" });
+  try {
+    const res = await fetch(
+      `${globals.API.prefix}${globals.API.version}spatial/meta`,
+      {
+        method: "GET",
+        headers: new Headers({
+          Accept: "application/json",
+          "Content-Type": "application/json",
+        }),
+        credentials: "include",
+      }
+    );
+
+    if (!res.ok || res.headers.get("Content-Type") !== "application/json") {
+      return null; // TODO need a dispatch //dispatchDiffExpErrors(dispatch, res);
+    }
+
+    const response = await res.json();
+
+    /* then send the success case action through */
+    return dispatch({
+      type: "request spatial metadata success",
+      data: response,
+    });
+  } catch (error) {
+    return dispatch({
+      type: "request spatial metadata error",
+      error,
+    });
+  }
+};

client/src/components/app.js

@@ -23,6 +23,8 @@ class App extends React.Component {
   componentDidMount() {
     const { dispatch } = this.props;
 
+    dispatch(actions.requestSpatialMetadata());
+
     /* listen for url changes, fire one when we start the app up */
     window.addEventListener("popstate", this._onURLChanged);
     this._onURLChanged();

client/src/components/embedding/index.js

@@ -16,10 +16,11 @@ import actions from "../../actions";
 import { getDiscreteCellEmbeddingRowIndex } from "../../util/stateManager/viewStackHelpers";
 
 @connect((state) => ({
-    layoutChoice: state.layoutChoice, // TODO: really should clean up naming, s/layout/embedding/g
+  imageUnderlay: state.imageUnderlay,
-    schema: state.annoMatrix?.schema,
+  layoutChoice: state.layoutChoice, // TODO: really should clean up naming, s/layout/embedding/g
-    crossfilter: state.obsCrossfilter,
+  schema: state.annoMatrix?.schema,
-  }))
+  crossfilter: state.obsCrossfilter,
+}))
 class Embedding extends React.PureComponent {
   constructor(props) {
     super(props);
@@ -27,8 +28,18 @@ class Embedding extends React.PureComponent {
   }
 
   handleLayoutChoiceChange = (e) => {
-    const { dispatch } = this.props;
+    const { dispatch, imageUnderlay } = this.props;
     dispatch(actions.layoutChoiceAction(e.currentTarget.value));
+
+    // if we just switched off spatial, if the image is on, turn it off
+    if (
+      imageUnderlay.isActive &&
+      e.target.value !== globals.spatialEmbeddingKeyword
+    ) {
+      dispatch({
+        type: "toggle image underlay",
+      });
+    }
   };
 
   render() {

client/src/components/graph/drawSpatialImageRegl.js

@@ -0,0 +1,62 @@
+export default function drawSpatialImageRegl(regl) {
+  return regl({
+    frag: `
+        precision mediump float;
+
+        // our texture
+        uniform sampler2D u_image;
+
+        // the texCoords passed in from the vertex shader.
+        varying vec2 v_texCoord;
+
+        void main() {
+            gl_FragColor = texture2D(u_image, v_texCoord);
+        }`,
+
+    vert: `
+        attribute vec2 a_position;
+        attribute vec2 a_texCoord;
+
+        uniform vec2 u_resolution;
+
+        uniform mat3 projView;
+
+        varying vec2 v_texCoord;
+
+        void main() {
+            // convert the rectangle from pixels to 0.0 to 1.0
+            vec3 pos = vec3(a_position, 1.);
+            vec2 zeroToOne = pos.xy / u_resolution;
+
+            // convert from 0->1 to 0->2
+            vec2 zeroToTwo = zeroToOne * 2.0;
+
+            // convert from 0->2 to -1->+1 (clipspace)
+            vec2 clipSpace = zeroToTwo - 1.0;
+
+            vec3 pos2 = projView * vec3(clipSpace, 1.);
+
+            gl_Position = vec4(pos2.xy , 0, 1);
+
+            // pass the texCoord to the fragment shader
+            // The GPU will interpolate this value between points.
+            v_texCoord = a_texCoord;
+        }`,
+
+    attributes: {
+      a_texCoord: [0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 1.0, 1.0, 0.0, 1.0, 1.0],
+      a_position: regl.prop("rectCoords"),
+    },
+
+    uniforms: {
+      projView: regl.prop("projView"),
+      u_image: regl.prop("spatialImageAsTexture"),
+      color: [1, 0, 0, 1],
+      u_resolution: [regl.prop("imageWidth"), regl.prop("imageHeight")],
+      image_width: regl.prop("imageWidth"),
+      //   translate:
+    },
+
+    count: 6,
+  });
+}

client/src/components/graph/graph.js

@@ -14,6 +14,7 @@ import {
   createColorTable,
   createColorQuery,
 } from "../../util/stateManager/colorHelpers";
+import _drawSpatialImage from "./drawSpatialImageRegl";
 import * as globals from "../../globals";
 
 import GraphOverlayLayer from "./overlays/graphOverlayLayer";
@@ -77,6 +78,8 @@ function createModelTF() {
   colors: state.colors,
   pointDilation: state.pointDilation,
   genesets: state.genesets.genesets,
+  spatial: state.spatial.metadata,
+  imageUnderlay: state.imageUnderlay,
 }))
 class Graph extends React.Component {
   static createReglState(canvas) {
@@ -87,6 +90,7 @@ class Graph extends React.Component {
     const camera = _camera(canvas);
     const regl = _regl(canvas);
     const drawPoints = _drawPoints(regl);
+    const drawSpatialImage = _drawSpatialImage(regl);
 
     // preallocate webgl buffers
     const pointBuffer = regl.buffer();
@@ -100,6 +104,7 @@ class Graph extends React.Component {
       pointBuffer,
       colorBuffer,
       flagBuffer,
+      drawSpatialImage,
     };
   }
 
@@ -232,6 +237,8 @@ class Graph extends React.Component {
       pointBuffer: null,
       colorBuffer: null,
       flagBuffer: null,
+      drawSpatialImage: null,
+      spatial: null,
 
       // component rendering derived state - these must stay synchronized
       // with the reducer state they were generated from.
@@ -317,7 +324,10 @@ class Graph extends React.Component {
     if (e.type !== "wheel") e.preventDefault();
     if (camera.handleEvent(e, projectionTF)) {
       this.renderCanvas();
-      this.setState((state) => ({ ...state, updateOverlay: !state.updateOverlay }));
+      this.setState((state) => ({
+        ...state,
+        updateOverlay: !state.updateOverlay,
+      }));
     }
   };
 
@@ -509,6 +519,14 @@ class Graph extends React.Component {
     return { toolSVG: newToolSVG, tool, container };
   };
 
+  loadTextureFromUrl = (src) =>
+    new Promise((resolve, reject) => {
+      const img = new Image();
+      img.onload = () => resolve(img);
+      img.onerror = reject;
+      img.src = src;
+    });
+
   fetchAsyncProps = async (props) => {
     const {
       annoMatrix,
@@ -517,6 +535,8 @@ class Graph extends React.Component {
       crossfilter,
       pointDilation,
       viewport,
+      spatial,
+      imageUnderlay,
     } = props.watchProps;
     const { modelTF } = this.state;
 
@@ -524,7 +544,8 @@ class Graph extends React.Component {
       annoMatrix,
       layoutChoice,
       colorsProp,
-      pointDilation
+      pointDilation,
+      imageUnderlay
     );
 
     const { currentDimNames } = layoutChoice;
@@ -551,6 +572,10 @@ class Graph extends React.Component {
       pointDilationLabel
     );
 
+    this.spatialImage = await this.loadTextureFromUrl(
+      "/api/v0.2/spatial/image"
+    );
+
     const { width, height } = viewport;
     return {
       positions,
@@ -558,6 +583,8 @@ class Graph extends React.Component {
       flags,
       width,
       height,
+      spatial,
+      imageUnderlay,
     };
   };
 
@@ -721,6 +748,7 @@ class Graph extends React.Component {
       flagBuffer,
       camera,
       projectionTF,
+      drawSpatialImage,
     } = this.state;
     this.renderPoints(
       regl,
@@ -729,12 +757,14 @@ class Graph extends React.Component {
       pointBuffer,
       flagBuffer,
       camera,
-      projectionTF
+      projectionTF,
+      drawSpatialImage
     );
   });
 
   updateReglAndRender(asyncProps, prevAsyncProps) {
-    const { positions, colors, flags, height, width } = asyncProps;
+    const { positions, colors, flags, height, width, imageUnderlay } =
+      asyncProps;
     this.cachedAsyncProps = asyncProps;
     const { pointBuffer, colorBuffer, flagBuffer } = this.state;
     let needToRenderCanvas = false;
@@ -754,6 +784,9 @@ class Graph extends React.Component {
       flagBuffer({ data: flags, dimension: 1 });
       needToRenderCanvas = true;
     }
+    if (imageUnderlay !== prevAsyncProps?.imageUnderlay) {
+      needToRenderCanvas = true;
+    }
     if (needToRenderCanvas) this.renderCanvas();
   }
 
@@ -797,20 +830,25 @@ class Graph extends React.Component {
     pointBuffer,
     flagBuffer,
     camera,
-    projectionTF
+    projectionTF,
+    drawSpatialImage
   ) {
-    const { annoMatrix } = this.props;
+    const { annoMatrix, spatial, imageUnderlay } = this.props;
     if (!this.reglCanvas || !annoMatrix) return;
 
     const { schema } = annoMatrix;
     const cameraTF = camera.view();
     const projView = mat3.multiply(mat3.create(), projectionTF, cameraTF);
     const { width, height } = this.reglCanvas;
+    const imW = spatial.data.imageWidth;
+    const imH = spatial.data.imageHeight;
+
     regl.poll();
     regl.clear({
       depth: 1,
-      color: [1, 1, 1, 1],
+      color: [0, 0, 0, 0],
     });
+
     drawPoints({
       distance: camera.distance(),
       color: colorBuffer,
@@ -821,6 +859,19 @@ class Graph extends React.Component {
       nPoints: schema.dataframe.nObs,
       minViewportDimension: Math.min(width, height),
     });
+    if (imageUnderlay?.isActive) {
+      drawSpatialImage({
+        projView,
+        imageWidth: imW,
+        imageHeight: imH,
+        rectCoords: [0, 0, imW, 0, 0, imH, 0, imH, imW, 0, imW, imH],
+        spatialImageAsTexture: regl.texture({
+          data: this.spatialImage,
+          wrapS: "clamp",
+          wrapT: "clamp",
+        }),
+      });
+    }
     regl._gl.flush();
   }
 
@@ -832,6 +883,8 @@ class Graph extends React.Component {
       layoutChoice,
       pointDilation,
       crossfilter,
+      spatial,
+      imageUnderlay,
     } = this.props;
     const { modelTF, projectionTF, camera, viewport, regl } = this.state;
     const cameraTF = camera?.view()?.slice();
@@ -902,6 +955,8 @@ class Graph extends React.Component {
             pointDilation,
             crossfilter,
             viewport,
+            spatial,
+            imageUnderlay,
           }}
         >
           <Async.Pending initial>
@@ -951,32 +1006,29 @@ const ErrorLoading = ({ displayName, error, width, height }) => {
   );
 };
 
-const StillLoading = ({ displayName, width, height }) => 
+const StillLoading = ({ displayName, width, height }) => (
   /*
   Render a busy/loading indicator
   */
-   (
+  <div
+    style={{
+      position: "fixed",
+      fontWeight: 500,
+      top: height / 2,
+      width,
+    }}
+  >
     <div
       style={{
-        position: "fixed",
+        display: "flex",
-        fontWeight: 500,
+        justifyContent: "center",
-        top: height / 2,
+        justifyItems: "center",
-        width,
+        alignItems: "center",
       }}
     >
-      <div
+      <Button minimal loading intent="primary" />
-        style={{
+      <span style={{ fontStyle: "italic" }}>Loading {displayName}</span>
-          display: "flex",
-          justifyContent: "center",
-          justifyItems: "center",
-          alignItems: "center",
-        }}
-      >
-        <Button minimal loading intent="primary" />
-        <span style={{ fontStyle: "italic" }}>Loading {displayName}</span>
-      </div>
     </div>
-  )
+  </div>
-;
+);
-
 export default Graph;

client/src/components/menubar/index.js

@@ -28,6 +28,8 @@ import { getEmbSubsetView } from "../../util/stateManager/viewStackHelpers";
     subsetPossible,
     subsetResetPossible,
     graphInteractionMode: state.controls.graphInteractionMode,
+    imageUnderlay: state.imageUnderlay,
+    layoutChoice: state.layoutChoice, // TODO: really should clean up naming, s/layout/embedding/g
     clipPercentileMin: Math.round(100 * (annoMatrix?.clipRange?.[0] ?? 0)),
     clipPercentileMax: Math.round(100 * (annoMatrix?.clipRange?.[1] ?? 1)),
     userDefinedGenes: state.controls.userDefinedGenes,
@@ -206,6 +208,8 @@ class MenuBar extends React.PureComponent {
       colorAccessor,
       subsetPossible,
       subsetResetPossible,
+      imageUnderlay,
+      layoutChoice,
     } = this.props;
     const { pendingClipPercentiles } = this.state;
 
@@ -268,6 +272,29 @@ class MenuBar extends React.PureComponent {
             disabled={!isColoredByCategorical}
           />
         </Tooltip>
+        {layoutChoice?.available?.includes(globals.spatialEmbeddingKeyword) && (
+          <ButtonGroup className={styles.menubarButton}>
+            <Tooltip
+              content={"Toggle image"}
+              position="bottom"
+              hoverOpenDelay={globals.tooltipHoverOpenDelay}
+            >
+              <AnchorButton
+                type="button"
+                data-testid="toggle-image-underlay"
+                icon={"media"}
+                intent={imageUnderlay.isActive ? "primary" : "none"}
+                active={imageUnderlay.isActive}
+                onClick={() => {
+                  dispatch({
+                    type: "toggle image underlay",
+                  });
+                }}
+              />
+            </Tooltip>
+          </ButtonGroup>
+        )}
+
         <ButtonGroup className={styles.menubarButton}>
           <Tooltip
             content={selectionTooltip}

client/src/globals.js

@@ -2,6 +2,9 @@ import { Colors } from "@blueprintjs/core";
 import { dispatchNetworkErrorMessageToUser } from "./util/actionHelpers";
 import ENV_DEFAULT from "../../environment.default.json";
 
+// visium embedding word, spatial image underlay
+export const spatialEmbeddingKeyword = "spatial";
+
 /* overflow category values are created  using this string */
 export const overflowCategoryLabel = ": all other labels";
 

client/src/reducers/imageUnderlay.js

@@ -0,0 +1,14 @@
+const imageUnderlay = (state = { isActive: false }, action) => {
+  switch (action.type) {
+    case "toggle image underlay":
+      return {
+        ...state,
+        isActive: !state.isActive,
+      };
+
+    default:
+      return state;
+  }
+};
+
+export default imageUnderlay;

client/src/reducers/index.js

@@ -11,6 +11,7 @@ import continuousSelection from "./continuousSelection";
 import graphSelection from "./graphSelection";
 import colors from "./colors";
 import differential from "./differential";
+import spatial from "./spatial";
 import layoutChoice from "./layoutChoice";
 import controls from "./controls";
 import annotations from "./annotations";
@@ -19,6 +20,7 @@ import genesetsUI from "./genesetsUI";
 import autosave from "./autosave";
 import centroidLabels from "./centroidLabels";
 import pointDialation from "./pointDilation";
+import imageUnderlay from "./imageUnderlay";
 import { gcMiddleware as annoMatrixGC } from "../annoMatrix";
 
 import undoableConfig from "./undoableConfig";
@@ -38,7 +40,9 @@ const Reducer = undoable(
     ["colors", colors],
     ["controls", controls],
     ["differential", differential],
+    ["spatial", spatial],
     ["centroidLabels", centroidLabels],
+    ["imageUnderlay", imageUnderlay],
     ["pointDilation", pointDialation],
     ["autosave", autosave],
   ]),
@@ -51,6 +55,7 @@ const Reducer = undoable(
     "colors",
     "controls",
     "differential",
+    "spatial",
     "layoutChoice",
     "centroidLabels",
     "genesets",

client/src/reducers/spatial.js

@@ -0,0 +1,34 @@
+const Spatial = (
+  state = {
+    loading: null,
+    error: null,
+    metadata: null,
+  },
+  action
+) => {
+  switch (action.type) {
+    case "request spatial metadata started":
+      return {
+        ...state,
+        loading: true,
+        error: null,
+      };
+    case "request spatial metadata success":
+      return {
+        ...state,
+        error: null,
+        loading: false,
+        metadata: action,
+      };
+    case "request spatial metadata error":
+      return {
+        ...state,
+        loading: false,
+        error: action.data,
+      };
+    default:
+      return state;
+  }
+};
+
+export default Spatial;

client/src/reducers/undoableConfig.js

@@ -52,6 +52,9 @@ const skipOnActions = new Set([
   "geneset: disable add new genes mode",
   "geneset: activate rename geneset mode",
   "geneset: disable rename geneset mode",
+
+  /* spatial */
+  "toggle image underlay",
 ]);
 
 /*

server/app/app.py

@@ -190,6 +190,16 @@ class SummarizeVarAPI(Resource):
     def post(self, data_adaptor):
         return common_rest.summarize_var_post(request, data_adaptor)
 
+class SpatialImageAPI(Resource):
+    @rest_get_data_adaptor
+    def get(self, data_adaptor):
+        return common_rest.spatial_image_get(request, data_adaptor)
+
+class SpatialMetaAPI(Resource):
+    @rest_get_data_adaptor
+    def get(self, data_adaptor):
+        return data_adaptor.get_spatial_metadata()
+
 
 def get_api_base_resources(bp_base):
     """Add resources that are accessed from the api url"""
@@ -222,6 +232,9 @@ def get_api_dataroot_resources(bp_dataroot):
     # Computation routes
     add_resource(DiffExpObsAPI, "/diffexp/obs")
     add_resource(LayoutObsAPI, "/layout/obs")
+    # Spatial routes
+    add_resource(SpatialImageAPI, "/spatial/image")
+    add_resource(SpatialMetaAPI, "/spatial/meta")
     return api
 
 

server/common/rest.py

@@ -4,8 +4,9 @@ import sys
 from http import HTTPStatus
 import zlib
 import json
+import numpy as np
 
-from flask import make_response, jsonify, current_app, abort
+from flask import make_response, jsonify, current_app, abort, send_file
 from werkzeug.urls import url_unquote
 
 from server.common.config.client_config import get_client_config
@@ -293,7 +294,7 @@ def layout_obs_get(request, data_adaptor):
 
     try:
         return make_response(
-            data_adaptor.layout_to_fbs_matrix(fields), HTTPStatus.OK, {"Content-Type": "application/octet-stream"}
+            data_adaptor.layout_to_fbs_matrix(fields, data_adaptor.get_spatial()), HTTPStatus.OK, {"Content-Type": "application/octet-stream"}
         )
     except (KeyError, DatasetAccessError) as e:
         return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
@@ -397,3 +398,40 @@ def summarize_var_post(request, data_adaptor):
 
     key = request.args.get("key", default=None)
     return summarize_var_helper(request, data_adaptor, key, request.get_data())
+
+def spatial_image_get(request, data_adaptor):
+    import io
+    import matplotlib.pyplot    
+
+    resolution = "hires"
+    spatial = data_adaptor.get_spatial()
+
+    if len(list(spatial)) == 0:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, "uns does not have spatial information")
+
+    library_id = list(spatial)[0]
+    if len(spatial) > 1:
+        current_app.logger.warning(f"More than one library found under uns.spatial, using library '{library_id}'")
+
+    if "images" not in spatial[library_id]:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, "spatial information does not contain images")
+
+    if resolution not in spatial[library_id]["images"]:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, f"spatial information does not contain requested resolution '{resolution}'")
+
+    response_image = io.BytesIO()
+    img = spatial[library_id]["images"][resolution]
+    matplotlib.pyplot.imsave(response_image, img)
+    response_image.seek(0)
+
+    try:
+        return send_file(response_image, attachment_filename=f"{library_id}-{resolution}.png", mimetype="image/png")
+    except (KeyError, DatasetAccessError) as e:
+        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
+    except PrepareError:
+        return abort_and_log(
+            HTTPStatus.NOT_IMPLEMENTED,
+            f"No spatial image available {request.path}",
+            loglevel=logging.ERROR,
+            include_exc_info=True,
+        ) 

server/data_anndata/anndata_adaptor.py

@@ -274,6 +274,43 @@ class AnndataAdaptor(DataAdaptor):
             df = df[fields]
         return encode_matrix_fbs(df, col_idx=df.columns)
 
+    def get_spatial(self):
+        return self.data.uns["spatial"]
+
+    def get_spatial_metadata(self):
+        spatial = self.get_spatial()
+
+        resolution = "hires"
+
+        if len(list(spatial)) == 0:
+            raise Exception("uns does not have spatial information")
+
+        library_id = list(spatial)[0]
+
+        if "images" not in spatial[library_id]:
+            raise Exception("spatial information does not contain images")
+
+        if resolution not in spatial[library_id]["images"]:
+            raise Exception(f"spatial information does not contain requested resolution '{resolution}'")
+
+        scaleref = spatial[library_id]["scalefactors"][f"tissue_{resolution}_scalef"]
+        (h, w, _) = spatial[library_id]["images"][resolution].shape
+
+        A = self.data.obsm["X_spatial"]
+        min = np.nanmin(A, axis=0)
+        max = np.nanmax(A, axis=0)
+        scale = np.amax(max - min)
+        translate = 0.5 - ((max - min) / scale / 2)
+
+        return {
+            "imageWidth": w,
+            "imageHeight": h,
+            "scaleref": scaleref,
+            "inverseScale": int(scale),
+            "inverseTranslate": translate.tolist(),
+            "inverseMin": min.tolist(),
+        }
+
     def get_embedding_names(self):
         """
         Return pre-computed embeddings.

server/data_common/data_adaptor.py

@@ -340,31 +340,57 @@ class DataAdaptor(metaclass=ABCMeta):
         pass
 
     @staticmethod
-    def normalize_embedding(embedding):
+    def normalize_embedding(embedding, spatial = None):
         """Normalize embedding layout to meet client assumptions.
-        Embedding is an ndarray, shape (n_obs, n)., where n is normally 2
+        Embedding is an ndarray, shape (n_obs, n)., where n is normally 2.
+        Note: if spatial data is available, the normalization will be done
+        according to the size of the underlying image
         """
 
-        # scale isotropically
+        if spatial is not None:
-        try:
-            min = np.nanmin(embedding, axis=0)
-            max = np.nanmax(embedding, axis=0)
-        except RuntimeError:
-            # indicates entire array was NaN, which should propagate
-            min = np.NaN
-            max = np.NaN
 
-        scale = np.amax(max - min)
+            # TODO: sync with the code in spatial_data_get
-        normalized_layout = (embedding - min) / scale
+            resolution = "hires"
 
-        # translate to center on both axis
+            if len(list(spatial)) == 0:
-        translate = 0.5 - ((max - min) / scale / 2)
+                raise Exception("uns does not have spatial information")
-        normalized_layout = normalized_layout + translate
+
+            library_id = list(spatial)[0]
+
+            if "images" not in spatial[library_id]:
+                raise Exception("spatial information does not contain images")
+
+            if resolution not in spatial[library_id]["images"]:
+                raise Exception(f"spatial information does not contain requested resolution '{resolution}'")
+
+            scaleref = spatial[library_id]["scalefactors"][f"tissue_{resolution}_scalef"]
+            (h, w, _) = spatial[library_id]["images"][resolution].shape
+
+            A = embedding * scaleref
+            A = np.column_stack([A[:, 0] / w, A[:, 1] / h])
+            normalized_layout = A.astype(dtype=np.float32)
+
+        else:
+
+            # scale isotropically
+            try:
+                min = np.nanmin(embedding, axis=0)
+                max = np.nanmax(embedding, axis=0)
+            except RuntimeError:
+                # indicates entire array was NaN, which should propagate
+                min = np.NaN
+                max = np.NaN
+
+            scale = np.amax(max - min)
+            normalized_layout = (embedding - min) / scale
+
+            # translate to center on both axis
+            translate = 0.5 - ((max - min) / scale / 2)
+            normalized_layout = normalized_layout + translate
 
-        normalized_layout = normalized_layout.astype(dtype=np.float32)
         return normalized_layout
 
-    def layout_to_fbs_matrix(self, fields):
+    def layout_to_fbs_matrix(self, fields, spatial = None):
         """
         return specified embeddings as a flatbuffer, using the cellxgene matrix fbs encoding.
 
@@ -380,7 +406,7 @@ class DataAdaptor(metaclass=ABCMeta):
         with ServerTiming.time("layout.query"):
             for ename in embeddings:
                 embedding = self.get_embedding_array(ename, 2)
-                normalized_layout = DataAdaptor.normalize_embedding(embedding)
+                normalized_layout = DataAdaptor.normalize_embedding(embedding, ename == "spatial" and spatial)
                 layout_data.append(pd.DataFrame(normalized_layout, columns=[f"{ename}_0", f"{ename}_1"]))
 
         with ServerTiming.time("layout.encode"):

server/requirements.txt

@@ -14,6 +14,7 @@ flatten-dict>=0.2.0
 fsspec>=0.4.4,<0.8.0
 gunicorn>=20.0.4
 h5py>=3.0.0
+matplotlib>=3.5.0
 numba>=0.51.2
 numpy>=1.17.5
 packaging>=20.0