Make Imaginate into a node (#878)

* Simplify document node input defenitions

* Remove imaginate layer

* Imaginate node properties

* Fix serde feature gate

* Add Proc Macro for Protonode implementation

* Fix incorrect type

* Add cargo.toml metadata

* Send imaginate params to frontend

* Fix image_creativity range

* Finish imaginate implementation

* Fix the imaginate draw tool

* Remove node-graph/rpco-macro

* Cargo fmt

* Fix missing workspace member

* Changes to the resolution

* Add checkbox for Imaginate auto resolution; improve Properties panel layouts

And fix bugs in panel resizing

* Implement the Rescale button

* Reorder imports

* Update Rust deps

Co-authored-by: Keavon Chambers <keavon@keavon.com>

frontend/assets/icon-12px-solid/edit.svg

@@ -0,0 +1,8 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 12 12">
+	<!-- TODO: Replace this with a proper icon design -->
+	<g transform="scale(0.5)">
+		<path d="M18.5,5.5c-1-1-2.5-1.5-3-1L5,15c0.39,0.24,1.23,0.1,1.59,0.38c0.36,0.28,0.3,1.06,0.63,1.4c0.33,0.33,1.11,0.28,1.4,0.63C8.9,17.77,8.76,18.61,9,19L19.5,8.5C20,8,19.5,6.5,18.5,5.5z M16.5,7.5l-7,7c-0.14,0.14-0.32,0.21-0.5,0.21s-0.36-0.07-0.5-0.21c-0.28-0.28-0.28-0.72,0-1l7-7c0.28-0.28,0.72-0.28,1,0C16.78,6.78,16.78,7.22,16.5,7.5z" />
+		<path d="M7.35,18.47l-2.39,1.71c-0.1-0.27-0.22-0.51-0.42-0.71c-0.2-0.2-0.44-0.32-0.71-0.42l1.71-2.39C5.14,16.61,4.5,16.42,4,16l-3,7l7-3C7.58,19.5,7.39,18.86,7.35,18.47z" />
+		<path d="M21.99,2.01C21,1,19.5,0.5,19,1l-2,2c2,0,4,2,4,4l2-2C23.5,4.5,22.98,3.02,21.99,2.01z" />
+	</g>
+</svg>

frontend/src/App.vue

@@ -83,6 +83,13 @@ body,
 	user-select: none;
 }
 
+// The default value of `auto` from the CSS spec is a footgun with flexbox layouts:
+// https://stackoverflow.com/questions/36247140/why-dont-flex-items-shrink-past-content-size
+* {
+	min-width: 0;
+	min-height: 0;
+}
+
 html,
 body,
 input,

frontend/src/components/layout/FloatingMenu.vue

@@ -267,6 +267,7 @@ export default defineComponent({
 
 			if (!inParentFloatingMenu) {
 				// Required to correctly position content when scrolled (it has a `position: fixed` to prevent clipping)
+				// We use `.style` on a ref (instead of a `:style` Vue binding) because the binding causes the `updated()` hook to call the function we're in recursively forever
 				const tailOffset = this.type === "Popover" ? 10 : 0;
 				if (this.direction === "Bottom") floatingMenuContent.style.top = `${tailOffset + this.floatingMenuBounds.top}px`;
 				if (this.direction === "Top") floatingMenuContent.style.bottom = `${tailOffset + this.floatingMenuBounds.bottom}px`;
@@ -274,7 +275,7 @@ export default defineComponent({
 				if (this.direction === "Left") floatingMenuContent.style.right = `${tailOffset + this.floatingMenuBounds.right}px`;
 
 				// Required to correctly position tail when scrolled (it has a `position: fixed` to prevent clipping)
-				// We use a ref here, instead of a `:style` binding, because that causes the `updated()` hook to call the function we're in recursively forever
+				// We use `.style` on a ref (instead of a `:style` Vue binding) because the binding causes the `updated()` hook to call the function we're in recursively forever
 				const tail = this.$refs.tail as HTMLElement;
 				if (tail && this.direction === "Bottom") tail.style.top = `${this.floatingMenuBounds.top}px`;
 				if (tail && this.direction === "Top") tail.style.bottom = `${this.floatingMenuBounds.bottom}px`;
@@ -289,6 +290,7 @@ export default defineComponent({
 			if (this.direction === "Top" || this.direction === "Bottom") {
 				zeroedBorderVertical = this.direction === "Top" ? "Bottom" : "Top";
 
+				// We use `.style` on a ref (instead of a `:style` Vue binding) because the binding causes the `updated()` hook to call the function we're in recursively forever
 				if (this.floatingMenuContentBounds.left - this.windowEdgeMargin <= this.workspaceBounds.left) {
 					floatingMenuContent.style.left = `${this.windowEdgeMargin}px`;
 					if (this.workspaceBounds.left + floatingMenuContainerBounds.left === 12) zeroedBorderHorizontal = "Left";
@@ -301,6 +303,7 @@ export default defineComponent({
 			if (this.direction === "Left" || this.direction === "Right") {
 				zeroedBorderHorizontal = this.direction === "Left" ? "Right" : "Left";
 
+				// We use `.style` on a ref (instead of a `:style` Vue binding) because the binding causes the `updated()` hook to call the function we're in recursively forever
 				if (this.floatingMenuContentBounds.top - this.windowEdgeMargin <= this.workspaceBounds.top) {
 					floatingMenuContent.style.top = `${this.windowEdgeMargin}px`;
 					if (this.workspaceBounds.top + floatingMenuContainerBounds.top === 12) zeroedBorderVertical = "Top";
@@ -313,6 +316,7 @@ export default defineComponent({
 
 			// Remove the rounded corner from the content where the tail perfectly meets the corner
 			if (this.type === "Popover" && this.windowEdgeMargin === 6 && zeroedBorderVertical && zeroedBorderHorizontal) {
+				// We use `.style` on a ref (instead of a `:style` Vue binding) because the binding causes the `updated()` hook to call the function we're in recursively forever
 				switch (`${zeroedBorderVertical}${zeroedBorderHorizontal}`) {
 					case "TopLeft":
 						floatingMenuContent.style.borderTopLeftRadius = "0";

frontend/src/components/layout/LayoutCol.vue

@@ -15,8 +15,6 @@
 	display: flex;
 	flex-direction: column;
 	flex-grow: 1;
-	min-width: 0;
-	min-height: 0;
 
 	.spacer {
 		flex: 1 1 100%;

frontend/src/components/layout/LayoutRow.vue

@@ -15,8 +15,6 @@
 	display: flex;
 	flex-direction: row;
 	flex-grow: 1;
-	min-width: 0;
-	min-height: 0;
 
 	.spacer {
 		flex: 1 1 100%;

frontend/src/components/panels/Document.vue

@@ -133,8 +133,6 @@
 				flex: 0 0 auto;
 
 				.widget-row {
-					min-height: 0;
-
 					.swatch-pair {
 						margin: 0;
 					}

frontend/src/components/panels/LayerTree.vue

@@ -77,8 +77,6 @@
 
 <style lang="scss">
 .layer-tree {
-	min-height: 0;
-
 	// Options bar
 	.options-bar {
 		height: 32px;

frontend/src/components/panels/NodeGraph.vue

@@ -409,6 +409,7 @@ export default defineComponent({
 		nodeIcon(nodeName: string): IconName {
 			const iconMap: Record<string, IconName> = {
 				Output: "NodeOutput",
+				Imaginate: "NodeImaginate",
 				"Hue Shift Image": "NodeColorCorrection",
 				"Brighten Image": "NodeColorCorrection",
 				"Grayscale Image": "NodeColorCorrection",

frontend/src/components/widgets/WidgetRow.vue

@@ -76,6 +76,7 @@
 .widget-row {
 	flex: 0 0 auto;
 	display: flex;
+	overflow: hidden;
 	min-height: 32px;
 
 	> * {

frontend/src/components/widgets/buttons/ParameterExposeButton.vue

@@ -9,6 +9,7 @@
 	display: flex;
 	align-items: center;
 	flex: 0 0 auto;
+	max-height: 24px;
 
 	button {
 		flex: 0 0 auto;

frontend/src/components/widgets/groups/WidgetSection.vue

@@ -96,12 +96,12 @@
 			}
 
 			> .text-label:first-of-type {
-				flex: 0 0 30%;
-				text-align: right;
+				flex: 0 0 25%;
+				margin-left: 16px;
 			}
 
 			> .parameter-expose-button ~ .text-label:first-of-type {
-				text-align: left;
+				margin-left: 0;
 			}
 
 			> .text-button {

frontend/src/components/widgets/inputs/FieldInput.vue

@@ -113,7 +113,7 @@
 	}
 
 	textarea {
-		min-height: calc(18px * 4);
+		min-height: calc(18px * 2);
 		margin: 3px;
 		padding: 0 5px;
 		box-sizing: border-box;

frontend/src/components/window/workspace/Panel.vue

@@ -168,7 +168,6 @@
 		background: var(--color-3-darkgray);
 		flex: 1 1 100%;
 		flex-direction: column;
-		min-height: 0;
 
 		.empty-panel {
 			background: var(--color-2-mildblack);

frontend/src/components/window/workspace/Workspace.vue

@@ -1,8 +1,8 @@
 <template>
 	<LayoutRow class="workspace" data-workspace>
-		<LayoutRow class="workspace-grid-subdivision">
-			<LayoutCol class="workspace-grid-subdivision">
-				<LayoutRow class="workspace-grid-subdivision">
+		<LayoutRow class="workspace-grid-subdivision" :style="{ 'flex-grow': panelSizes['root'] }" data-subdivision-name="root">
+			<LayoutCol class="workspace-grid-subdivision" :style="{ 'flex-grow': panelSizes['content'] }" data-subdivision-name="content">
+				<LayoutRow class="workspace-grid-subdivision" :style="{ 'flex-grow': panelSizes['document'] }" data-subdivision-name="document">
 					<Panel
 						:panelType="portfolio.state.documents.length > 0 ? 'Document' : undefined"
 						:tabCloseButtons="true"
@@ -15,17 +15,17 @@
 					/>
 				</LayoutRow>
 				<LayoutRow class="workspace-grid-resize-gutter" data-gutter-vertical @pointerdown="(e: PointerEvent) => resizePanel(e)" v-if="nodeGraphVisible"></LayoutRow>
-				<LayoutRow class="workspace-grid-subdivision" v-if="nodeGraphVisible">
+				<LayoutRow class="workspace-grid-subdivision" v-if="nodeGraphVisible" :style="{ 'flex-grow': panelSizes['graph'] }" data-subdivision-name="graph">
 					<Panel :panelType="'NodeGraph'" :tabLabels="[{ name: 'Node Graph' }]" :tabActiveIndex="0" />
 				</LayoutRow>
 			</LayoutCol>
 			<LayoutCol class="workspace-grid-resize-gutter" data-gutter-horizontal @pointerdown="(e: PointerEvent) => resizePanel(e)"></LayoutCol>
-			<LayoutCol class="workspace-grid-subdivision" style="flex-grow: 0.2">
-				<LayoutRow class="workspace-grid-subdivision" style="flex-grow: 402">
+			<LayoutCol class="workspace-grid-subdivision" :style="{ 'flex-grow': panelSizes['details'] }" data-subdivision-name="details">
+				<LayoutRow class="workspace-grid-subdivision" :style="{ 'flex-grow': panelSizes['properties'] }" data-subdivision-name="properties">
 					<Panel :panelType="'Properties'" :tabLabels="[{ name: 'Properties' }]" :tabActiveIndex="0" />
 				</LayoutRow>
 				<LayoutRow class="workspace-grid-resize-gutter" data-gutter-vertical @pointerdown="(e: PointerEvent) => resizePanel(e)"></LayoutRow>
-				<LayoutRow class="workspace-grid-subdivision" style="flex-grow: 590">
+				<LayoutRow class="workspace-grid-subdivision" :style="{ 'flex-grow': panelSizes['layers'] }" data-subdivision-name="layers">
 					<Panel :panelType="'LayerTree'" :tabLabels="[{ name: 'Layer Tree' }]" :tabActiveIndex="0" />
 				</LayoutRow>
 			</LayoutCol>
@@ -72,9 +72,23 @@ import LayoutRow from "@/components/layout/LayoutRow.vue";
 import Panel from "@/components/window/workspace/Panel.vue";
 
 const MIN_PANEL_SIZE = 100;
+const PANEL_SIZES = {
+	/**/ root: 100,
+	/*   ├── */ content: 80,
+	/*   │      ├── */ document: 60,
+	/*   │      └── */ graph: 40,
+	/*   └── */ details: 20,
+	/*          ├── */ properties: 45,
+	/*          └── */ layers: 55,
+};
 
 export default defineComponent({
 	inject: ["workspace", "portfolio", "dialog", "editor"],
+	data() {
+		return {
+			panelSizes: PANEL_SIZES,
+		};
+	},
 	computed: {
 		activeDocumentIndex() {
 			return this.portfolio.state.activeDocumentIndex;
@@ -97,16 +111,26 @@ export default defineComponent({
 		resizePanel(event: PointerEvent) {
 			const gutter = (event.target || undefined) as HTMLDivElement | undefined;
 			const nextSibling = (gutter?.nextElementSibling || undefined) as HTMLDivElement | undefined;
-			const previousSibling = (gutter?.previousElementSibling || undefined) as HTMLDivElement | undefined;
+			const prevSibling = (gutter?.previousElementSibling || undefined) as HTMLDivElement | undefined;
+			const parentElement = (gutter?.parentElement || undefined) as HTMLDivElement | undefined;
 
-			if (!gutter || !nextSibling || !previousSibling) return;
+			const nextSiblingName = (nextSibling?.getAttribute("data-subdivision-name") || undefined) as keyof typeof PANEL_SIZES;
+			const prevSiblingName = (prevSibling?.getAttribute("data-subdivision-name") || undefined) as keyof typeof PANEL_SIZES;
+
+			if (!gutter || !nextSibling || !prevSibling || !parentElement || !nextSiblingName || !prevSiblingName) return;
 
 			// Are we resizing horizontally?
 			const isHorizontal = gutter.getAttribute("data-gutter-horizontal") !== null;
 
-			// Get the current size in px of the panels being resized
+			// Get the current size in px of the panels being resized and the gutter
+			const gutterSize = isHorizontal ? gutter.getBoundingClientRect().width : gutter.getBoundingClientRect().height;
 			const nextSiblingSize = isHorizontal ? nextSibling.getBoundingClientRect().width : nextSibling.getBoundingClientRect().height;
-			const previousSiblingSize = isHorizontal ? previousSibling.getBoundingClientRect().width : previousSibling.getBoundingClientRect().height;
+			const prevSiblingSize = isHorizontal ? prevSibling.getBoundingClientRect().width : prevSibling.getBoundingClientRect().height;
+			const parentElementSize = isHorizontal ? parentElement.getBoundingClientRect().width : parentElement.getBoundingClientRect().height;
+
+			// Measure the resizing panels as a percentage of all sibling panels
+			const totalResizingSpaceOccupied = gutterSize + nextSiblingSize + prevSiblingSize;
+			const proportionBeingResized = totalResizingSpaceOccupied / parentElementSize;
 
 			// Prevent cursor flicker as mouse temporarily leaves the gutter
 			gutter.setPointerCapture(event.pointerId);
@@ -118,10 +142,10 @@ export default defineComponent({
 				let mouseDelta = mouseStart - mouseCurrent;
 
 				mouseDelta = Math.max(nextSiblingSize + mouseDelta, MIN_PANEL_SIZE) - nextSiblingSize;
-				mouseDelta = previousSiblingSize - Math.max(previousSiblingSize - mouseDelta, MIN_PANEL_SIZE);
+				mouseDelta = prevSiblingSize - Math.max(prevSiblingSize - mouseDelta, MIN_PANEL_SIZE);
 
-				nextSibling.style.flexGrow = (nextSiblingSize + mouseDelta).toString();
-				previousSibling.style.flexGrow = (previousSiblingSize - mouseDelta).toString();
+				this.panelSizes[nextSiblingName] = ((nextSiblingSize + mouseDelta) / totalResizingSpaceOccupied) * proportionBeingResized * 100;
+				this.panelSizes[prevSiblingName] = ((prevSiblingSize - mouseDelta) / totalResizingSpaceOccupied) * proportionBeingResized * 100;
 
 				window.dispatchEvent(new CustomEvent("resize"));
 			};

frontend/src/state-providers/portfolio.ts

@@ -2,7 +2,7 @@
 import { reactive, readonly } from "vue";
 
 import { downloadFileText, downloadFileBlob, upload } from "@/utility-functions/files";
-import { imaginateGenerate, imaginateCheckConnection, imaginateTerminate, preloadAndSetImaginateBlobURL } from "@/utility-functions/imaginate";
+import { imaginateGenerate, imaginateCheckConnection, imaginateTerminate, updateBackendImage } from "@/utility-functions/imaginate";
 import { rasterizeSVG, rasterizeSVGCanvas } from "@/utility-functions/rasterization";
 import { type Editor } from "@/wasm-communication/editor";
 import {
@@ -68,15 +68,15 @@ export function createPortfolioState(editor: Editor) {
 		imaginateCheckConnection(hostname, editor);
 	});
 	editor.subscriptions.subscribeJsMessage(TriggerImaginateGenerate, async (triggerImaginateGenerate) => {
-		const { documentId, layerPath, hostname, refreshFrequency, baseImage, maskImage, maskPaintMode, maskBlurPx, maskFillContent, parameters } = triggerImaginateGenerate;
+		const { documentId, layerPath, nodePath, hostname, refreshFrequency, baseImage, maskImage, maskPaintMode, maskBlurPx, maskFillContent, parameters } = triggerImaginateGenerate;
 
 		// Handle img2img mode
 		let image: Blob | undefined;
 		if (parameters.denoisingStrength !== undefined && baseImage !== undefined) {
-			// Rasterize the SVG to an image file
-			image = await rasterizeSVG(baseImage.svg, baseImage.size[0], baseImage.size[1], "image/png");
+			const buffer = new Uint8Array(baseImage.imageData.values()).buffer;
 
-			preloadAndSetImaginateBlobURL(editor, image, documentId, layerPath, baseImage.size[0], baseImage.size[1]);
+			image = new Blob([buffer], { type: baseImage.mime });
+			updateBackendImage(editor, image, documentId, layerPath, nodePath);
 		}
 
 		// Handle layer mask
@@ -86,12 +86,12 @@ export function createPortfolioState(editor: Editor) {
 			mask = await rasterizeSVG(maskImage.svg, maskImage.size[0], maskImage.size[1], "image/png");
 		}
 
-		imaginateGenerate(parameters, image, mask, maskPaintMode, maskBlurPx, maskFillContent, hostname, refreshFrequency, documentId, layerPath, editor);
+		imaginateGenerate(parameters, image, mask, maskPaintMode, maskBlurPx, maskFillContent, hostname, refreshFrequency, documentId, layerPath, nodePath, editor);
 	});
 	editor.subscriptions.subscribeJsMessage(TriggerImaginateTerminate, async (triggerImaginateTerminate) => {
-		const { documentId, layerPath, hostname } = triggerImaginateTerminate;
+		const { documentId, layerPath, nodePath, hostname } = triggerImaginateTerminate;
 
-		imaginateTerminate(hostname, documentId, layerPath, editor);
+		imaginateTerminate(hostname, documentId, layerPath, nodePath, editor);
 	});
 	editor.subscriptions.subscribeJsMessage(UpdateImageData, (updateImageData) => {
 		updateImageData.imageData.forEach(async (element) => {
@@ -109,12 +109,12 @@ export function createPortfolioState(editor: Editor) {
 		});
 	});
 	editor.subscriptions.subscribeJsMessage(TriggerNodeGraphFrameGenerate, async (triggerNodeGraphFrameGenerate) => {
-		const { documentId, layerPath, svg, size } = triggerNodeGraphFrameGenerate;
+		const { documentId, layerPath, svg, size, imaginateNode } = triggerNodeGraphFrameGenerate;
 
 		// Rasterize the SVG to an image file
 		const imageData = (await rasterizeSVGCanvas(svg, size[0], size[1])).getContext("2d")?.getImageData(0, 0, size[0], size[1]);
 
-		if (imageData) editor.instance.processNodeGraphFrame(documentId, layerPath, new Uint8Array(imageData.data), imageData.width, imageData.height);
+		if (imageData) editor.instance.processNodeGraphFrame(documentId, layerPath, new Uint8Array(imageData.data), imageData.width, imageData.height, imaginateNode);
 	});
 	editor.subscriptions.subscribeJsMessage(TriggerRevokeBlobUrl, async (triggerRevokeBlobUrl) => {
 		URL.revokeObjectURL(triggerRevokeBlobUrl.url);

frontend/src/utility-functions/files.ts

@@ -26,7 +26,6 @@ export async function upload<T extends "text" | "data">(acceptedExtensions: stri
 	return new Promise<UploadResult<T>>((resolve, _) => {
 		const element = document.createElement("input");
 		element.type = "file";
-		element.style.display = "none";
 		element.accept = acceptedExtensions;
 
 		element.addEventListener(

frontend/src/utility-functions/icons.ts

@@ -12,6 +12,7 @@ import Add from "@/../assets/icon-12px-solid/add.svg";
 import Checkmark from "@/../assets/icon-12px-solid/checkmark.svg";
 import CloseX from "@/../assets/icon-12px-solid/close-x.svg";
 import DropdownArrow from "@/../assets/icon-12px-solid/dropdown-arrow.svg";
+import Edit from "@/../assets/icon-12px-solid/edit.svg";
 import Empty12px from "@/../assets/icon-12px-solid/empty-12px.svg";
 import FullscreenEnter from "@/../assets/icon-12px-solid/fullscreen-enter.svg";
 import FullscreenExit from "@/../assets/icon-12px-solid/fullscreen-exit.svg";
@@ -47,6 +48,7 @@ const SOLID_12PX = {
 	Checkmark: { component: Checkmark, size: 12 },
 	CloseX: { component: CloseX, size: 12 },
 	DropdownArrow: { component: DropdownArrow, size: 12 },
+	Edit: { component: Edit, size: 12 },
 	Empty12px: { component: Empty12px, size: 12 },
 	FullscreenEnter: { component: FullscreenEnter, size: 12 },
 	FullscreenExit: { component: FullscreenExit, size: 12 },

frontend/src/utility-functions/imaginate.ts

@@ -31,6 +31,7 @@ export async function imaginateGenerate(
 	refreshFrequency: number,
 	documentId: bigint,
 	layerPath: BigUint64Array,
+	nodePath: BigUint64Array,
 	editor: Editor
 ): Promise<void> {
 	// Ignore a request to generate a new image while another is already being generated
@@ -39,11 +40,11 @@ export async function imaginateGenerate(
 	terminated = false;
 
 	// Immediately set the progress to 0% so the backend knows to update its layout
-	editor.instance.setImaginateGeneratingStatus(documentId, layerPath, 0, "Beginning");
+	editor.instance.setImaginateGeneratingStatus(documentId, layerPath, nodePath, 0, "Beginning");
 
 	// Initiate a request to the computation server
 	const discloseUploadingProgress = (progress: number): void => {
-		editor.instance.setImaginateGeneratingStatus(documentId, layerPath, progress * 100, "Uploading");
+		editor.instance.setImaginateGeneratingStatus(documentId, layerPath, nodePath, progress * 100, "Uploading");
 	};
 	const { uploaded, result, xhr } = await generate(discloseUploadingProgress, hostname, image, mask, maskPaintMode, maskBlurPx, maskFillContent, parameters);
 	generatingAbortRequest = xhr;
@@ -51,13 +52,13 @@ export async function imaginateGenerate(
 	try {
 		// Wait until the request is fully uploaded, which could be slow if the img2img source is large and the user is on a slow connection
 		await uploaded;
-		editor.instance.setImaginateGeneratingStatus(documentId, layerPath, 0, "Generating");
+		editor.instance.setImaginateGeneratingStatus(documentId, layerPath, nodePath, 0, "Generating");
 
 		// Begin polling for updates to the in-progress image generation at the specified interval
 		// Don't poll if the chosen interval is 0, or if the chosen sampling method does not support polling
 		if (refreshFrequency > 0) {
 			const interval = Math.max(refreshFrequency * 1000, 500);
-			scheduleNextPollingUpdate(interval, Date.now(), 0, editor, hostname, documentId, layerPath, parameters.resolution);
+			scheduleNextPollingUpdate(interval, Date.now(), 0, editor, hostname, documentId, layerPath, nodePath, parameters.resolution);
 		}
 
 		// Wait for the final image to be returned by the initial request containing either the full image or the last frame if it was terminated by the user
@@ -75,16 +76,12 @@ export async function imaginateGenerate(
 		// Send the backend an updated status
 		const percent = terminated ? undefined : 100;
 		const newStatus = terminated ? "Terminated" : "Idle";
-		editor.instance.setImaginateGeneratingStatus(documentId, layerPath, percent, newStatus);
+		editor.instance.setImaginateGeneratingStatus(documentId, layerPath, nodePath, percent, newStatus);
 
 		// Send the backend a blob URL for the final image
-		preloadAndSetImaginateBlobURL(editor, blob, documentId, layerPath, parameters.resolution.x, parameters.resolution.y);
-
-		// Send the backend the blob data to be stored persistently in the layer
-		const u8Array = new Uint8Array(await blob.arrayBuffer());
-		editor.instance.setImaginateImageData(documentId, layerPath, u8Array);
+		updateBackendImage(editor, blob, documentId, layerPath, nodePath);
 	} catch {
-		editor.instance.setImaginateGeneratingStatus(documentId, layerPath, undefined, "Terminated");
+		editor.instance.setImaginateGeneratingStatus(documentId, layerPath, nodePath, undefined, "Terminated");
 
 		await imaginateCheckConnection(hostname, editor);
 	}
@@ -93,19 +90,19 @@ export async function imaginateGenerate(
 	abortAndResetPolling();
 }
 
-export async function imaginateTerminate(hostname: string, documentId: bigint, layerPath: BigUint64Array, editor: Editor): Promise<void> {
+export async function imaginateTerminate(hostname: string, documentId: bigint, layerPath: BigUint64Array, nodePath: BigUint64Array, editor: Editor): Promise<void> {
 	terminated = true;
 	abortAndResetPolling();
 
 	try {
 		await terminate(hostname);
 
-		editor.instance.setImaginateGeneratingStatus(documentId, layerPath, undefined, "Terminating");
+		editor.instance.setImaginateGeneratingStatus(documentId, layerPath, nodePath, undefined, "Terminating");
 	} catch {
 		abortAndResetGenerating();
 		abortAndResetPolling();
 
-		editor.instance.setImaginateGeneratingStatus(documentId, layerPath, undefined, "Terminated");
+		editor.instance.setImaginateGeneratingStatus(documentId, layerPath, nodePath, undefined, "Terminated");
 
 		await imaginateCheckConnection(hostname, editor);
 	}
@@ -116,15 +113,21 @@ export async function imaginateCheckConnection(hostname: string, editor: Editor)
 	editor.instance.setImaginateServerStatus(serverReached);
 }
 
-export async function preloadAndSetImaginateBlobURL(editor: Editor, blob: Blob, documentId: bigint, layerPath: BigUint64Array, width: number, height: number): Promise<void> {
-	const blobURL = URL.createObjectURL(blob);
+// Converts the blob image into a list of pixels using an invisible canvas.
+export async function updateBackendImage(editor: Editor, blob: Blob, documentId: bigint, layerPath: BigUint64Array, nodePath: BigUint64Array): Promise<void> {
+	const image = await createImageBitmap(blob);
+	const canvas = document.createElement("canvas");
+	canvas.width = image.width;
+	canvas.height = image.height;
+	const ctx = canvas.getContext("2d");
+	if (!ctx) throw new Error("Could not create canvas context");
+	ctx.drawImage(image, 0, 0);
 
-	// Pre-decode the image so it is ready to be drawn instantly once it's placed into the viewport SVG
-	const image = new Image();
-	image.src = blobURL;
-	await image.decode();
+	// Send the backend the blob data to be stored persistently in the layer
+	const imageData = ctx.getImageData(0, 0, image.width, image.height);
+	const u8Array = new Uint8Array(imageData.data);
 
-	editor.instance.setImaginateBlobURL(documentId, layerPath, blobURL, width, height);
+	editor.instance.setImaginateImageData(documentId, layerPath, nodePath, u8Array, imageData.width, imageData.height);
 }
 
 // ABORTING AND RESETTING HELPERS
@@ -150,6 +153,7 @@ function scheduleNextPollingUpdate(
 	hostname: string,
 	documentId: bigint,
 	layerPath: BigUint64Array,
+	nodePath: BigUint64Array,
 	resolution: XY
 ): void {
 	// Pick a future time that keeps to the user-requested interval if possible, but on slower connections will go as fast as possible without overlapping itself
@@ -165,10 +169,10 @@ function scheduleNextPollingUpdate(
 			// After waiting for the polling result back from the server, if during that intervening time the user has terminated the generation, exit so we don't overwrite that terminated status
 			if (terminated) return;
 
-			if (blob) preloadAndSetImaginateBlobURL(editor, blob, documentId, layerPath, resolution.x, resolution.y);
-			editor.instance.setImaginateGeneratingStatus(documentId, layerPath, percentComplete, "Generating");
+			if (blob) updateBackendImage(editor, blob, documentId, layerPath, nodePath);
+			editor.instance.setImaginateGeneratingStatus(documentId, layerPath, nodePath, percentComplete, "Generating");
 
-			scheduleNextPollingUpdate(interval, nextTimeoutBegan, 0, editor, hostname, documentId, layerPath, resolution);
+			scheduleNextPollingUpdate(interval, nextTimeoutBegan, 0, editor, hostname, documentId, layerPath, nodePath, resolution);
 		} catch {
 			if (generatingAbortRequest === undefined) return;
 
@@ -178,7 +182,7 @@ function scheduleNextPollingUpdate(
 
 				await imaginateCheckConnection(hostname, editor);
 			} else {
-				scheduleNextPollingUpdate(interval, nextTimeoutBegan, pollingRetries + 1, editor, hostname, documentId, layerPath, resolution);
+				scheduleNextPollingUpdate(interval, nextTimeoutBegan, pollingRetries + 1, editor, hostname, documentId, layerPath, nodePath, resolution);
 			}
 		}
 	}, timeFromNow);

frontend/src/wasm-communication/editor.ts

@@ -41,11 +41,12 @@ export async function fetchImage(path: BigUint64Array, mime: string, documentId:
 }
 
 // export async function dispatchTauri(message: string): Promise<string> {
-export async function dispatchTauri(message: any): Promise<void> {
+export async function dispatchTauri(message: unknown): Promise<void> {
 	try {
 		const response = await invoke("handle_message", { message });
 		editorInstance?.tauriResponse(response);
 	} catch {
+		// eslint-disable-next-line no-console
 		console.error("Failed to dispatch Tauri message");
 	}
 }

frontend/src/wasm-communication/messages.ts

@@ -519,8 +519,8 @@ export class TriggerImaginateGenerate extends JsMessage {
 	@Type(() => ImaginateBaseImage)
 	readonly baseImage!: ImaginateBaseImage | undefined;
 
-	@Type(() => ImaginateBaseImage)
-	readonly maskImage: ImaginateBaseImage | undefined;
+	@Type(() => ImaginateMaskImage)
+	readonly maskImage: ImaginateMaskImage | undefined;
 
 	readonly maskPaintMode!: string;
 
@@ -535,11 +535,22 @@ export class TriggerImaginateGenerate extends JsMessage {
 	readonly documentId!: bigint;
 
 	readonly layerPath!: BigUint64Array;
+
+	readonly nodePath!: BigUint64Array;
+}
+
+export class ImaginateMaskImage {
+	readonly svg!: string;
+
+	readonly size!: [number, number];
 }
 
 export class ImaginateBaseImage {
-	readonly svg!: string;
+	readonly mime!: string;
 
+	readonly imageData!: Uint8Array;
+
+	@TupleToVec2
 	readonly size!: [number, number];
 }
 
@@ -571,6 +582,8 @@ export class TriggerImaginateTerminate extends JsMessage {
 
 	readonly layerPath!: BigUint64Array;
 
+	readonly nodePath!: BigUint64Array;
+
 	readonly hostname!: string;
 }
 
@@ -582,6 +595,8 @@ export class TriggerNodeGraphFrameGenerate extends JsMessage {
 	readonly svg!: string;
 
 	readonly size!: [number, number];
+
+	readonly imaginateNode!: BigUint64Array | undefined;
 }
 
 export class TriggerRefreshBoundsOfViewports extends JsMessage {}

frontend/wasm/src/editor_api.rs

@@ -12,8 +12,8 @@ use editor::messages::input_mapper::utility_types::input_keyboard::ModifierKeys;
 use editor::messages::input_mapper::utility_types::input_mouse::{EditorMouseState, ScrollDelta, ViewportBounds};
 use editor::messages::portfolio::utility_types::{ImaginateServerStatus, Platform};
 use editor::messages::prelude::*;
+use graph_craft::document::NodeId;
 use graphene::color::Color;
-use graphene::layers::imaginate_layer::ImaginateStatus;
 use graphene::LayerId;
 
 use serde::Serialize;
@@ -508,27 +508,23 @@ impl JsEditorHandle {
 
 	/// Sends the blob URL generated by JS to the Imaginate layer in the respective document
 	#[wasm_bindgen(js_name = setImaginateImageData)]
-	pub fn set_imaginate_image_data(&self, document_id: u64, layer_path: Vec<LayerId>, image_data: Vec<u8>) {
-		let message = PortfolioMessage::ImaginateSetImageData { document_id, layer_path, image_data };
-		self.dispatch(message);
-	}
-
-	/// Sends the blob URL generated by JS to the Imaginate layer in the respective document
-	#[wasm_bindgen(js_name = setImaginateBlobURL)]
-	pub fn set_imaginate_blob_url(&self, document_id: u64, layer_path: Vec<LayerId>, blob_url: String, width: f64, height: f64) {
-		let resolution = (width, height);
-		let message = PortfolioMessage::ImaginateSetBlobUrl {
+	pub fn set_imaginate_image_data(&self, document_id: u64, layer_path: Vec<LayerId>, node_path: Vec<NodeId>, image_data: Vec<u8>, width: u32, height: u32) {
+		let message = PortfolioMessage::ImaginateSetImageData {
 			document_id,
+			node_path,
 			layer_path,
-			blob_url,
-			resolution,
+			image_data,
+			width,
+			height,
 		};
 		self.dispatch(message);
 	}
 
 	/// Notifies the Imaginate layer of a new percentage of completion and whether or not it's currently generating
 	#[wasm_bindgen(js_name = setImaginateGeneratingStatus)]
-	pub fn set_imaginate_generating_status(&self, document_id: u64, path: Vec<LayerId>, percent: Option<f64>, status: String) {
+	pub fn set_imaginate_generating_status(&self, document_id: u64, layer_path: Vec<LayerId>, node_path: Vec<NodeId>, percent: Option<f64>, status: String) {
+		use graph_craft::imaginate_input::ImaginateStatus;
+
 		let status = match status.as_str() {
 			"Idle" => ImaginateStatus::Idle,
 			"Beginning" => ImaginateStatus::Beginning,
@@ -541,7 +537,13 @@ impl JsEditorHandle {
 
 		let percent = if matches!(status, ImaginateStatus::Uploading(_)) { None } else { percent };
 
-		let message = PortfolioMessage::ImaginateSetGeneratingStatus { document_id, path, percent, status };
+		let message = PortfolioMessage::ImaginateSetGeneratingStatus {
+			document_id,
+			layer_path,
+			node_path,
+			percent,
+			status,
+		};
 		self.dispatch(message);
 	}
 
@@ -563,12 +565,13 @@ impl JsEditorHandle {
 
 	/// Sends the blob URL generated by JS to the Imaginate layer in the respective document
 	#[wasm_bindgen(js_name = processNodeGraphFrame)]
-	pub fn process_node_graph_frame(&self, document_id: u64, layer_path: Vec<LayerId>, image_data: Vec<u8>, width: u32, height: u32) {
+	pub fn process_node_graph_frame(&self, document_id: u64, layer_path: Vec<LayerId>, image_data: Vec<u8>, width: u32, height: u32, imaginate_node: Option<Vec<NodeId>>) {
 		let message = PortfolioMessage::ProcessNodeGraphFrame {
 			document_id,
 			layer_path,
 			image_data,
 			size: (width, height),
+			imaginate_node,
 		};
 		self.dispatch(message);
 	}