Thumbnails for the layer node (#1210)

* Thumbnails for the layer node

* Raster node graph frames

* Downscale to a random resolution

* Cleanup and bug fixes

* Generate paths before duplicating outputs

* Fix stable id test

* Code review changes

* Code review pass with minor changes

---------

Co-authored-by: Keavon Chambers <keavon@keavon.com>

frontend/src-tauri/src/main.rs

@@ -102,6 +102,7 @@ fn handle_message(message: String) -> String {
 				images.insert(format!("{:?}_{}", &image.path, document_id), image);
 				stub_data.push(FrontendImageData {
 					path,
+					node_id: None,
 					mime,
 					image_data: Arc::new(Vec::new()),
 					transform,

frontend/src/components/panels/NodeGraph.svelte

@@ -548,7 +548,11 @@
 								</div>
 							{/if}
 						</div>
-						<IconLabel icon={nodeIcon(node.displayName)} />
+						{#if node.thumbnailSvg}
+							{@html node.thumbnailSvg}
+						{:else}
+							<IconLabel icon={nodeIcon(node.displayName)} />
+						{/if}
 						<TextLabel>{node.displayName}</TextLabel>
 					</div>
 					{#if exposedInputsOutputs.length > 0}

frontend/src/state-providers/portfolio.ts

@@ -128,7 +128,7 @@ export function createPortfolioState(editor: Editor) {
 			image.src = blobURL;
 			await image.decode();
 
-			editor.instance.setImageBlobURL(updateImageData.documentId, element.path, blobURL, image.naturalWidth, image.naturalHeight, element.transform);
+			editor.instance.setImageBlobURL(updateImageData.documentId, element.path, element.nodeId, blobURL, image.naturalWidth, image.naturalHeight, element.transform);
 		});
 	});
 	editor.subscriptions.subscribeJsMessage(TriggerRasterizeRegionBelowLayer, async (triggerRasterizeRegionBelowLayer) => {

frontend/src/wasm-communication/editor.ts

@@ -11,7 +11,7 @@ export type Editor = Readonly<ReturnType<typeof createEditor>>;
 // `wasmImport` starts uninitialized because its initialization needs to occur asynchronously, and thus needs to occur by manually calling and awaiting `initWasm()`
 let wasmImport: WebAssembly.Memory | undefined;
 
-export async function updateImage(path: BigUint64Array, mime: string, imageData: Uint8Array, transform: Float64Array, documentId: bigint): Promise<void> {
+export async function updateImage(path: BigUint64Array, nodeId: bigint, mime: string, imageData: Uint8Array, transform: Float64Array, documentId: bigint): Promise<void> {
 	const blob = new Blob([imageData], { type: mime });
 
 	const blobURL = URL.createObjectURL(blob);
@@ -21,10 +21,10 @@ export async function updateImage(path: BigUint64Array, mime: string, imageData:
 	image.src = blobURL;
 	await image.decode();
 
-	window["editorInstance"]?.setImageBlobURL(documentId, path, blobURL, image.naturalWidth, image.naturalHeight, transform);
+	window["editorInstance"]?.setImageBlobURL(documentId, path, nodeId, blobURL, image.naturalWidth, image.naturalHeight, transform);
 }
 
-export async function fetchImage(path: BigUint64Array, mime: string, documentId: bigint, url: string): Promise<void> {
+export async function fetchImage(path: BigUint64Array, nodeId: bigint, mime: string, documentId: bigint, url: string): Promise<void> {
 	const data = await fetch(url);
 	const blob = await data.blob();
 
@@ -35,7 +35,7 @@ export async function fetchImage(path: BigUint64Array, mime: string, documentId:
 	image.src = blobURL;
 	await image.decode();
 
-	window["editorInstance"]?.setImageBlobURL(documentId, path, blobURL, image.naturalWidth, image.naturalHeight, undefined);
+	window["editorInstance"]?.setImageBlobURL(documentId, path, nodeId, blobURL, image.naturalWidth, image.naturalHeight, undefined);
 }
 
 const tauri = "__TAURI_METADATA__" in window && import("@tauri-apps/api");

frontend/src/wasm-communication/messages.ts

@@ -105,6 +105,8 @@ export class FrontendNode {
 	readonly previewed!: boolean;
 
 	readonly disabled!: boolean;
+
+	readonly thumbnailSvg!: string | undefined;
 }
 
 export class FrontendNodeLink {
@@ -771,6 +773,8 @@ export type LayerTypeData = {
 export class ImaginateImageData {
 	readonly path!: BigUint64Array;
 
+	readonly nodeId!: bigint;
+
 	readonly mime!: string;
 
 	readonly imageData!: Uint8Array;

frontend/wasm/src/editor_api.rs

@@ -32,8 +32,8 @@ pub fn set_random_seed(seed: u64) {
 /// This avoids creating a json with a list millions of numbers long.
 #[wasm_bindgen(module = "/../src/wasm-communication/editor.ts")]
 extern "C" {
-	fn updateImage(path: Vec<u64>, mime: String, imageData: &[u8], transform: js_sys::Float64Array, document_id: u64);
-	fn fetchImage(path: Vec<u64>, mime: String, document_id: u64, identifier: String);
+	fn updateImage(path: Vec<u64>, nodeId: Option<u64>, mime: String, imageData: &[u8], transform: js_sys::Float64Array, document_id: u64);
+	fn fetchImage(path: Vec<u64>, nodeId: Option<u64>, mime: String, document_id: u64, identifier: String);
 	//fn dispatchTauri(message: String) -> String;
 	fn dispatchTauri(message: String);
 }
@@ -122,10 +122,13 @@ impl JsEditorHandle {
 					} else {
 						js_sys::Float64Array::default()
 					};
-					updateImage(image.path, image.mime, &image.image_data, transform, document_id);
+					updateImage(image.path, image.node_id, image.mime, &image.image_data, transform, document_id);
 				}
 				#[cfg(feature = "tauri")]
-				fetchImage(image.path.clone(), image.mime, document_id, format!("http://localhost:3001/image/{:?}_{}", &image.path, document_id));
+				{
+					let identifier = format!("http://localhost:3001/image/{:?}_{}", image.path, document_id);
+					fetchImage(image.path, image.node_id, image.mime, document_id, identifier);
+				}
 			}
 			return;
 		}
@@ -504,11 +507,12 @@ impl JsEditorHandle {
 
 	/// Sends the blob URL generated by JS to the Image layer
 	#[wasm_bindgen(js_name = setImageBlobURL)]
-	pub fn set_image_blob_url(&self, document_id: u64, layer_path: Vec<LayerId>, blob_url: String, width: f64, height: f64, transform: Option<js_sys::Float64Array>) {
+	pub fn set_image_blob_url(&self, document_id: u64, layer_path: Vec<LayerId>, node_id: Option<NodeId>, blob_url: String, width: f64, height: f64, transform: Option<js_sys::Float64Array>) {
 		let resolution = (width, height);
 		let message = PortfolioMessage::SetImageBlobUrl {
 			document_id,
 			layer_path: layer_path.clone(),
+			node_id,
 			blob_url,
 			resolution,
 		};