import React from "react"; import * as d3 from "d3"; import { connect, shallowEqual } from "react-redux"; import { mat3, vec2 } from "gl-matrix"; import _regl from "regl"; import memoize from "memoize-one"; import Async from "react-async"; import { Button } from "@blueprintjs/core"; import setupSVGandBrushElements from "./setupSVGandBrush"; import _camera from "../../util/camera"; import _drawPoints from "./drawPointsRegl"; import { createColorTable, createColorQuery, } from "../../util/stateManager/colorHelpers"; import _drawSpatialImage from "./drawSpatialImageRegl"; import * as globals from "../../globals"; import GraphOverlayLayer from "./overlays/graphOverlayLayer"; import CentroidLabels from "./overlays/centroidLabels"; import actions from "../../actions"; import renderThrottle from "../../util/renderThrottle"; import { flagBackground, flagSelected, flagHighlight, } from "../../util/glHelpers"; /* Simple 2D transforms control all point painting. There are three: * model - convert from underlying per-point coordinate to a layout. Currently used to move from data to webgl coordinate system. * camera - apply a 2D camera transformation (pan, zoom) * projection - apply any transformation required for screen size and layout */ function createProjectionTF(viewportWidth, viewportHeight) { /* the projection transform accounts for the screen size & other layout */ const fractionToUse = 0.95; // fraction of min dimension to use const topGutterSizePx = 32; // top gutter for tools const bottomGutterSizePx = 32; // bottom gutter for tools const heightMinusGutter = viewportHeight - topGutterSizePx - bottomGutterSizePx; const minDim = Math.min(viewportWidth, heightMinusGutter); const aspectScale = [ (fractionToUse * minDim) / viewportWidth, (fractionToUse * minDim) / viewportHeight, ]; const m = mat3.create(); mat3.fromTranslation(m, [ 0, (bottomGutterSizePx - topGutterSizePx) / viewportHeight / aspectScale[1], ]); mat3.scale(m, m, aspectScale); return m; } function createModelTF() { /* preallocate coordinate system transformation between data and gl. Data arrives in a [0,1] range, and we operate elsewhere in [-1,1]. */ const m = mat3.fromScaling(mat3.create(), [2, 2]); mat3.translate(m, m, [-0.5, -0.5]); return m; } @connect((state) => ({ annoMatrix: state.annoMatrix, crossfilter: state.obsCrossfilter, selectionTool: state.graphSelection.tool, currentSelection: state.graphSelection.selection, layoutChoice: state.layoutChoice, graphInteractionMode: state.controls.graphInteractionMode, colors: state.colors, pointDilation: state.pointDilation, genesets: state.genesets.genesets, spatial: state.spatial.metadata, imageUnderlay: state.imageUnderlay, })) class Graph extends React.Component { static createReglState(canvas) { /* Must be created for each canvas */ // setup canvas, webgl draw function and camera const camera = _camera(canvas); const regl = _regl(canvas); const drawPoints = _drawPoints(regl); const drawSpatialImage = _drawSpatialImage(regl); // preallocate webgl buffers const pointBuffer = regl.buffer(); const colorBuffer = regl.buffer(); const flagBuffer = regl.buffer(); return { camera, regl, drawPoints, pointBuffer, colorBuffer, flagBuffer, drawSpatialImage, }; } static watchAsync(props, prevProps) { return !shallowEqual(props.watchProps, prevProps.watchProps); } computePointPositions = memoize( (X, Y, modelTF, spatialMetadata, imageUnderlay) => { /* compute the model coordinate for each point */ console.log({ X }, { Y }); const positions = new Float32Array(2 * X.length); for (let i = 0, len = X.length; i < len; i += 1) { const p = imageUnderlay?.isActive ? this.rescalePointForSpatial(X[i], Y[i], spatialMetadata) : vec2.fromValues(X[i], Y[i]); vec2.transformMat3(p, p, modelTF); positions[2 * i] = p[0]; positions[2 * i + 1] = p[1]; } return positions; } ); computePointColors = memoize((rgb) => { /* compute webgl colors for each point */ const colors = new Float32Array(3 * rgb.length); for (let i = 0, len = rgb.length; i < len; i += 1) { colors.set(rgb[i], 3 * i); } return colors; }); computeSelectedFlags = memoize( (crossfilter, _flagSelected, _flagUnselected) => { const x = crossfilter.fillByIsSelected( new Float32Array(crossfilter.size()), _flagSelected, _flagUnselected ); return x; } ); computeHighlightFlags = memoize( (nObs, pointDilationData, pointDilationLabel) => { const flags = new Float32Array(nObs); if (pointDilationData) { for (let i = 0, len = flags.length; i < len; i += 1) { if (pointDilationData[i] === pointDilationLabel) { flags[i] = flagHighlight; } } } return flags; } ); computeColorByFlags = memoize((nObs, colorByData) => { const flags = new Float32Array(nObs); if (colorByData) { for (let i = 0, len = flags.length; i < len; i += 1) { const val = colorByData[i]; if (typeof val === "number" && !Number.isFinite(val)) { flags[i] = flagBackground; } } } return flags; }); computePointFlags = memoize( (crossfilter, colorByData, pointDilationData, pointDilationLabel) => { /* We communicate with the shader using three flags: - isNaN -- the value is a NaN. Only makes sense when we have a colorAccessor - isSelected -- the value is selected - isHightlighted -- the value is highlighted in the UI (orthogonal from selection highlighting) Due to constraints in webgl vertex shader attributes, these are encoded in a float, "kinda" like bitmasks. We also have separate code paths for generating flags for categorical and continuous metadata, as they rely on different tests, and some of the flags (eg, isNaN) are meaningless in the face of categorical metadata. */ const nObs = crossfilter.size(); const flags = new Float32Array(nObs); const selectedFlags = this.computeSelectedFlags( crossfilter, flagSelected, 0 ); const highlightFlags = this.computeHighlightFlags( nObs, pointDilationData, pointDilationLabel ); const colorByFlags = this.computeColorByFlags(nObs, colorByData); for (let i = 0; i < nObs; i += 1) { flags[i] = selectedFlags[i] + highlightFlags[i] + colorByFlags[i]; } return flags; } ); constructor(props) { super(props); const viewport = this.getViewportDimensions(); this.reglCanvas = null; this.cachedAsyncProps = null; const modelTF = createModelTF(); this.state = { toolSVG: null, tool: null, container: null, viewport, // projection camera: null, modelTF, modelInvTF: mat3.invert([], modelTF), projectionTF: createProjectionTF(viewport.width, viewport.height), // regl state regl: null, drawPoints: null, pointBuffer: null, colorBuffer: null, flagBuffer: null, drawSpatialImage: null, spatial: null, // component rendering derived state - these must stay synchronized // with the reducer state they were generated from. layoutState: { layoutDf: null, layoutChoice: null, }, colorState: { colors: null, colorDf: null, colorTable: null, }, pointDilationState: { pointDilation: null, pointDilationDf: null, }, }; } componentDidMount() { window.addEventListener("resize", this.handleResize); } componentDidUpdate(prevProps, prevState) { const { selectionTool, currentSelection, graphInteractionMode } = this.props; const { toolSVG, viewport } = this.state; const hasResized = prevState.viewport.height !== viewport.height || prevState.viewport.width !== viewport.width; let stateChanges = {}; if ( (viewport.height && viewport.width && !toolSVG) || // first time init hasResized || // window size has changed we want to recreate all SVGs selectionTool !== prevProps.selectionTool || // change of selection tool prevProps.graphInteractionMode !== graphInteractionMode // lasso/zoom mode is switched ) { stateChanges = { ...stateChanges, ...this.createToolSVG(), }; } /* if the selection tool or state has changed, ensure that the selection tool correctly reflects the underlying selection. */ if ( currentSelection !== prevProps.currentSelection || graphInteractionMode !== prevProps.graphInteractionMode || stateChanges.toolSVG ) { const { tool, container } = this.state; this.selectionToolUpdate( stateChanges.tool ? stateChanges.tool : tool, stateChanges.container ? stateChanges.container : container ); } if (Object.keys(stateChanges).length > 0) { // eslint-disable-next-line react/no-did-update-set-state --- Preventing update loop via stateChanges and diff checks this.setState(stateChanges); } } componentWillUnmount() { window.removeEventListener("resize", this.handleResize); } handleResize = () => { const { state } = this.state; const viewport = this.getViewportDimensions(); const projectionTF = createProjectionTF(viewport.width, viewport.height); this.setState({ ...state, viewport, projectionTF, }); }; handleCanvasEvent = (e) => { const { camera, projectionTF } = this.state; if (e.type !== "wheel") e.preventDefault(); if (camera.handleEvent(e, projectionTF)) { this.renderCanvas(); this.setState((state) => ({ ...state, updateOverlay: !state.updateOverlay, })); } }; handleBrushDragAction() { /* event describing brush position: @-------| | | | | |-------@ */ // ignore programatically generated events if (d3.event.sourceEvent === null || !d3.event.selection) return; const { dispatch, layoutChoice } = this.props; const s = d3.event.selection; const northwest = this.mapScreenToPoint(s[0]); const southeast = this.mapScreenToPoint(s[1]); const [minX, maxY] = northwest; const [maxX, minY] = southeast; dispatch( actions.graphBrushChangeAction(layoutChoice.current, { minX, minY, maxX, maxY, northwest, southeast, }) ); } handleBrushStartAction() { // Ignore programatically generated events. if (!d3.event.sourceEvent) return; const { dispatch } = this.props; dispatch(actions.graphBrushStartAction()); } handleBrushEndAction() { // Ignore programatically generated events. if (!d3.event.sourceEvent) return; /* coordinates will be included if selection made, null if selection cleared. */ const { dispatch, layoutChoice } = this.props; const s = d3.event.selection; if (s) { const northwest = this.mapScreenToPoint(s[0]); const southeast = this.mapScreenToPoint(s[1]); const [minX, maxY] = northwest; const [maxX, minY] = southeast; dispatch( actions.graphBrushEndAction(layoutChoice.current, { minX, minY, maxX, maxY, northwest, southeast, }) ); } else { dispatch(actions.graphBrushDeselectAction(layoutChoice.current)); } } handleBrushDeselectAction() { const { dispatch, layoutChoice } = this.props; dispatch(actions.graphBrushDeselectAction(layoutChoice.current)); } handleLassoStart() { const { dispatch, layoutChoice } = this.props; dispatch(actions.graphLassoStartAction(layoutChoice.current)); } // when a lasso is completed, filter to the points within the lasso polygon handleLassoEnd(polygon) { const minimumPolygonArea = 10; const { dispatch, layoutChoice } = this.props; if ( polygon.length < 3 || Math.abs(d3.polygonArea(polygon)) < minimumPolygonArea ) { // if less than three points, or super small area, treat as a clear selection. dispatch(actions.graphLassoDeselectAction(layoutChoice.current)); } else { dispatch( actions.graphLassoEndAction( layoutChoice.current, polygon.map((xy) => this.mapScreenToPoint(xy)) ) ); } } handleLassoCancel() { const { dispatch, layoutChoice } = this.props; dispatch(actions.graphLassoCancelAction(layoutChoice.current)); } handleLassoDeselectAction() { const { dispatch, layoutChoice } = this.props; dispatch(actions.graphLassoDeselectAction(layoutChoice.current)); } handleDeselectAction() { const { selectionTool } = this.props; if (selectionTool === "brush") this.handleBrushDeselectAction(); if (selectionTool === "lasso") this.handleLassoDeselectAction(); } handleOpacityRangeChange(e) { const { dispatch } = this.props; dispatch({ type: "change opacity deselected cells in 2d graph background", data: e.target.value, }); } rescalePointForSpatial = (x, y, spatialMetadata) => { // console.log({spatialMetadata}); const translate = vec2.fromValues( spatialMetadata.inverseTranslate[0], spatialMetadata.inverseTranslate[1] ); const min = vec2.fromValues( spatialMetadata.inverseMin[0], spatialMetadata.inverseMin[1] ); const scalefactor = spatialMetadata.scaleref; const wh = vec2.fromValues( spatialMetadata.imageWidth, spatialMetadata.imageHeight ); // Apply the inverse transform const p = vec2.fromValues(x, y); vec2.sub(p, p, translate); vec2.scale(p, p, spatialMetadata.inverseScale); vec2.add(p, p, min); vec2.scale(p, p, scalefactor); vec2.div(p, p, wh); return p; }; setReglCanvas = (canvas) => { this.reglCanvas = canvas; this.setState({ ...Graph.createReglState(canvas), }); }; getViewportDimensions = () => { const { viewportRef } = this.props; return { height: viewportRef.clientHeight, width: viewportRef.clientWidth, }; }; createToolSVG = () => { /* Called from componentDidUpdate. Create the tool SVG, and return any state changes that should be passed to setState(). */ const { selectionTool, graphInteractionMode } = this.props; const { viewport } = this.state; /* clear out whatever was on the div, even if nothing, but usually the brushes etc */ const lasso = d3.select("#lasso-layer"); if (lasso.empty()) return {}; // still initializing lasso.selectAll(".lasso-group").remove(); // Don't render or recreate toolSVG if currently in zoom mode if (graphInteractionMode !== "select") { // don't return "change" of state unless we are really changing it! const { toolSVG } = this.state; if (toolSVG === undefined) return {}; return { toolSVG: undefined }; } let handleStart; let handleDrag; let handleEnd; let handleCancel; if (selectionTool === "brush") { handleStart = this.handleBrushStartAction.bind(this); handleDrag = this.handleBrushDragAction.bind(this); handleEnd = this.handleBrushEndAction.bind(this); } else { handleStart = this.handleLassoStart.bind(this); handleEnd = this.handleLassoEnd.bind(this); handleCancel = this.handleLassoCancel.bind(this); } const { svg: newToolSVG, tool, container, } = setupSVGandBrushElements( selectionTool, handleStart, handleDrag, handleEnd, handleCancel, viewport ); return { toolSVG: newToolSVG, tool, container }; }; loadTextureFromUrl = (src) => new Promise((resolve, reject) => { const img = new Image(); img.onload = () => resolve(img); img.onerror = reject; img.src = src; }); fetchAsyncProps = async (props) => { const { annoMatrix, colors: colorsProp, layoutChoice, crossfilter, pointDilation, viewport, spatial, imageUnderlay, } = props.watchProps; const { modelTF } = this.state; console.log({ spatial }); const [layoutDf, colorDf, pointDilationDf] = await this.fetchData( annoMatrix, layoutChoice, colorsProp, pointDilation, imageUnderlay ); const { currentDimNames } = layoutChoice; const X = layoutDf.col(currentDimNames[0]).asArray(); const Y = layoutDf.col(currentDimNames[1]).asArray(); const positions = this.computePointPositions( X, Y, modelTF, spatial.data, imageUnderlay ); const colorTable = this.updateColorTable(colorsProp, colorDf); const colors = this.computePointColors(colorTable.rgb); const { colorAccessor } = colorsProp; const colorByData = colorDf?.col(colorAccessor)?.asArray(); const { metadataField: pointDilationCategory, categoryField: pointDilationLabel, } = pointDilation; const pointDilationData = pointDilationDf ?.col(pointDilationCategory) ?.asArray(); const flags = this.computePointFlags( crossfilter, colorByData, pointDilationData, pointDilationLabel ); this.spatialImage = await this.loadTextureFromUrl( "/api/v0.2/spatial/image" ); const { width, height } = viewport; return { positions, colors, flags, width, height, spatial, imageUnderlay, }; }; async fetchData(annoMatrix, layoutChoice, colors, pointDilation) { /* fetch all data needed. Includes: - the color by dataframe - the layout dataframe - the point dilation dataframe */ const { metadataField: pointDilationAccessor } = pointDilation; const promises = []; // layout promises.push(annoMatrix.fetch("emb", layoutChoice.current)); // color const query = this.createColorByQuery(colors); if (query) { promises.push(annoMatrix.fetch(...query)); } else { promises.push(Promise.resolve(null)); } // point highlighting if (pointDilationAccessor) { promises.push(annoMatrix.fetch("obs", pointDilationAccessor)); } else { promises.push(Promise.resolve(null)); } return Promise.all(promises); } brushToolUpdate(tool, container) { /* this is called from componentDidUpdate(), so be very careful using anything from this.state, which may be updated asynchronously. */ const { currentSelection } = this.props; if (container) { const toolCurrentSelection = d3.brushSelection(container.node()); if (currentSelection.mode === "within-rect") { /* if there is a selection, make sure the brush tool matches */ const screenCoords = [ this.mapPointToScreen(currentSelection.brushCoords.northwest), this.mapPointToScreen(currentSelection.brushCoords.southeast), ]; if (!toolCurrentSelection) { /* tool is not selected, so just move the brush */ container.call(tool.move, screenCoords); } else { /* there is an active selection and a brush - make sure they match */ /* this just sums the difference of each dimension, of each point */ let delta = 0; for (let x = 0; x < 2; x += 1) { for (let y = 0; y < 2; y += 1) { delta += Math.abs( screenCoords[x][y] - toolCurrentSelection[x][y] ); } } if (delta > 0) { container.call(tool.move, screenCoords); } } } else if (toolCurrentSelection) { /* no selection, so clear the brush tool if it is set */ container.call(tool.move, null); } } } lassoToolUpdate(tool) { /* this is called from componentDidUpdate(), so be very careful using anything from this.state, which may be updated asynchronously. */ const { currentSelection } = this.props; if (currentSelection.mode === "within-polygon") { /* if there is a current selection, make sure the lasso tool matches */ const polygon = currentSelection.polygon.map((p) => this.mapPointToScreen(p) ); tool.move(polygon); } else { tool.reset(); } } selectionToolUpdate(tool, container) { /* this is called from componentDidUpdate(), so be very careful using anything from this.state, which may be updated asynchronously. */ const { selectionTool } = this.props; switch (selectionTool) { case "brush": this.brushToolUpdate(tool, container); break; case "lasso": this.lassoToolUpdate(tool, container); break; default: /* punt? */ break; } } mapScreenToPoint(pin) { /* Map an XY coordinates from screen domain to cell/point range, accounting for current pan/zoom camera. */ const { camera, projectionTF, modelInvTF, viewport } = this.state; const cameraInvTF = camera.invView(); /* screen -> gl */ const x = (2 * pin[0]) / viewport.width - 1; const y = 2 * (1 - pin[1] / viewport.height) - 1; const xy = vec2.fromValues(x, y); const projectionInvTF = mat3.invert(mat3.create(), projectionTF); vec2.transformMat3(xy, xy, projectionInvTF); vec2.transformMat3(xy, xy, cameraInvTF); vec2.transformMat3(xy, xy, modelInvTF); return xy; } mapPointToScreen(xyCell) { /* Map an XY coordinate from cell/point domain to screen range. Inverse of mapScreenToPoint() */ const { camera, projectionTF, modelTF, viewport } = this.state; const cameraTF = camera.view(); const xy = vec2.transformMat3(vec2.create(), xyCell, modelTF); vec2.transformMat3(xy, xy, cameraTF); vec2.transformMat3(xy, xy, projectionTF); return [ Math.round(((xy[0] + 1) * viewport.width) / 2), Math.round(-((xy[1] + 1) / 2 - 1) * viewport.height), ]; } renderCanvas = renderThrottle(() => { const { regl, drawPoints, colorBuffer, pointBuffer, flagBuffer, camera, projectionTF, drawSpatialImage, } = this.state; this.renderPoints( regl, drawPoints, colorBuffer, pointBuffer, flagBuffer, camera, projectionTF, drawSpatialImage ); }); updateReglAndRender(asyncProps, prevAsyncProps) { const { positions, colors, flags, height, width, imageUnderlay } = asyncProps; this.cachedAsyncProps = asyncProps; const { pointBuffer, colorBuffer, flagBuffer } = this.state; let needToRenderCanvas = false; console.log("updateReglAndRender"); if (height !== prevAsyncProps?.height || width !== prevAsyncProps?.width) { needToRenderCanvas = true; } if (positions !== prevAsyncProps?.positions) { pointBuffer({ data: positions, dimension: 2 }); needToRenderCanvas = true; } if (colors !== prevAsyncProps?.colors) { colorBuffer({ data: colors, dimension: 3 }); needToRenderCanvas = true; } if (flags !== prevAsyncProps?.flags) { flagBuffer({ data: flags, dimension: 1 }); needToRenderCanvas = true; } if (imageUnderlay !== prevAsyncProps?.imageUnderlay) { needToRenderCanvas = true; } if (needToRenderCanvas) this.renderCanvas(); } updateColorTable(colors, colorDf) { const { annoMatrix } = this.props; const { schema } = annoMatrix; /* update color table state */ if (!colors || !colorDf) { return createColorTable( null, // default mode null, null, schema, null ); } const { colorAccessor, userColors, colorMode } = colors; return createColorTable( colorMode, colorAccessor, colorDf, schema, userColors ); } createColorByQuery(colors) { const { annoMatrix, genesets } = this.props; const { schema } = annoMatrix; const { colorMode, colorAccessor } = colors; return createColorQuery(colorMode, colorAccessor, schema, genesets); } renderPoints( regl, drawPoints, colorBuffer, pointBuffer, flagBuffer, camera, projectionTF, drawSpatialImage ) { const { annoMatrix, spatial, imageUnderlay } = this.props; if (!this.reglCanvas || !annoMatrix) return; const { schema } = annoMatrix; const cameraTF = camera.view(); const projView = mat3.multiply(mat3.create(), projectionTF, cameraTF); const { width, height } = this.reglCanvas; const imW = spatial.data.imageWidth; const imH = spatial.data.imageHeight; console.log({ imageUnderlay }); regl.poll(); regl.clear({ depth: 1, color: [0, 0, 0, 0], }); drawPoints({ distance: camera.distance(), color: colorBuffer, position: pointBuffer, flag: flagBuffer, count: annoMatrix.nObs, projView, nPoints: schema.dataframe.nObs, minViewportDimension: Math.min(width, height), }); if (imageUnderlay?.isActive) { drawSpatialImage({ projView, imageWidth: imW, imageHeight: imH, rectCoords: [0, 0, imW, 0, 0, imH, 0, imH, imW, 0, imW, imH], spatialImageAsTexture: regl.texture({ data: this.spatialImage, wrapS: "clamp", wrapT: "clamp", }), }); } regl._gl.flush(); } render() { const { graphInteractionMode, annoMatrix, colors, layoutChoice, pointDilation, crossfilter, spatial, imageUnderlay, } = this.props; const { modelTF, projectionTF, camera, viewport, regl } = this.state; const cameraTF = camera?.view()?.slice(); console.log("---RENDER"); console.log({ props: this.props }); return (
{(error) => ( )} {(asyncProps) => { if (regl && !shallowEqual(asyncProps, this.cachedAsyncProps)) { this.updateReglAndRender(asyncProps, this.cachedAsyncProps); } return null; }}
); } } const ErrorLoading = ({ displayName, error, width, height }) => { console.log(error); // log to console as this is an unepected error return (
{`Failure loading ${displayName}`}
); }; const StillLoading = ({ displayName, width, height }) => ( /* Render a busy/loading indicator */
); export default Graph;