Fix the Imaginate node from crashing (#1512)

* Allow generic node input for type inference

* Make imaginate resolution picking depend on the image resolution instead of the transform

* Remove dead code

* Fix console spam after crash

* Fix crash when disconnecting Imaginate node input

* Update Imaginate tool tooltip

---------

Co-authored-by: Dennis Kobert <dennis@kobert.dev>

frontend/src/state-providers/portfolio.ts

@@ -4,7 +4,7 @@ import { writable } from "svelte/store";
 
 import { copyToClipboardFileURL } from "@graphite/io-managers/clipboard";
 import { downloadFileText, downloadFileBlob, upload } from "@graphite/utility-functions/files";
-import { extractPixelData, imageToPNG, rasterizeSVG, rasterizeSVGCanvas } from "@graphite/utility-functions/rasterization";
+import { extractPixelData, imageToPNG, rasterizeSVG } from "@graphite/utility-functions/rasterization";
 import { type Editor } from "@graphite/wasm-communication/editor";
 import {
 	type FrontendDocumentDetails,
@@ -15,7 +15,6 @@ import {
 	TriggerDownloadTextFile,
 	TriggerImport,
 	TriggerOpenDocument,
-	TriggerRasterizeRegionBelowLayer,
 	TriggerRevokeBlobUrl,
 	UpdateActiveDocument,
 	UpdateImageData,
@@ -116,23 +115,6 @@ export function createPortfolioState(editor: Editor) {
 			editor.instance.setImageBlobURL(updateImageData.documentId, element.path, element.nodeId, blobURL, image.naturalWidth, image.naturalHeight, element.transform);
 		});
 	});
-	editor.subscriptions.subscribeJsMessage(TriggerRasterizeRegionBelowLayer, async (triggerRasterizeRegionBelowLayer) => {
-		const { documentId, layerPath, svg, size } = triggerRasterizeRegionBelowLayer;
-
-		// Rasterize the SVG to an image file
-		try {
-			if (size[0] >= 1 && size[1] >= 1) {
-				const imageData = (await rasterizeSVGCanvas(svg, size[0], size[1])).getContext("2d")?.getImageData(0, 0, size[0], size[1]);
-				if (!imageData) return;
-
-				editor.instance.renderGraphUsingRasterizedRegionBelowLayer(documentId, layerPath, new Uint8Array(imageData.data), imageData.width, imageData.height);
-			}
-		} catch (e) {
-			// getImageData may throw an exception if the resolution is too high
-			// eslint-disable-next-line no-console
-			console.error("Failed to rasterize the SVG canvas in JS to be sent back to Rust:", e);
-		}
-	});
 	editor.subscriptions.subscribeJsMessage(TriggerRevokeBlobUrl, async (triggerRevokeBlobUrl) => {
 		URL.revokeObjectURL(triggerRevokeBlobUrl.url);
 	});

frontend/src/wasm-communication/messages.ts

@@ -560,16 +560,6 @@ export class TriggerDownloadTextFile extends JsMessage {
 	readonly name!: string;
 }
 
-export class TriggerRasterizeRegionBelowLayer extends JsMessage {
-	readonly documentId!: bigint;
-
-	readonly layerPath!: BigUint64Array;
-
-	readonly svg!: string;
-
-	readonly size!: [number, number];
-}
-
 export class TriggerRefreshBoundsOfViewports extends JsMessage {}
 
 export class TriggerRevokeBlobUrl extends JsMessage {
@@ -672,8 +662,8 @@ export class DisplayEditableTextboxTransform extends JsMessage {
 export class UpdateImageData extends JsMessage {
 	readonly documentId!: bigint;
 
-	@Type(() => ImaginateImageData)
-	readonly imageData!: ImaginateImageData[];
+	@Type(() => RenderedImageData)
+	readonly imageData!: RenderedImageData[];
 }
 
 export class DisplayRemoveEditableTextbox extends JsMessage {}
@@ -710,7 +700,7 @@ export class LayerMetadata {
 
 export type LayerType = "Folder" | "Layer" | "Artboard";
 
-export class ImaginateImageData {
+export class RenderedImageData {
 	readonly path!: BigUint64Array;
 
 	readonly nodeId!: bigint;
@@ -1415,7 +1405,6 @@ export const messageMakers: Record<string, MessageMaker> = {
 	TriggerLoadPreferences,
 	TriggerOpenDocument,
 	TriggerPaste,
-	TriggerRasterizeRegionBelowLayer,
 	TriggerRefreshBoundsOfViewports,
 	TriggerRevokeBlobUrl,
 	TriggerSavePreferences,

frontend/wasm/src/editor_api.rs

@@ -2,8 +2,8 @@
 //! It serves as a thin wrapper over the editor backend API that relies
 //! on the dispatcher messaging system and more complex Rust data types.
 
-use crate::helpers::{translate_key, Error};
-use crate::{EDITOR_HAS_CRASHED, EDITOR_INSTANCES, JS_EDITOR_HANDLES};
+use crate::helpers::translate_key;
+use crate::{Error, EDITOR_HAS_CRASHED, EDITOR_INSTANCES, JS_EDITOR_HANDLES};
 
 use document_legacy::document_metadata::LayerNodeIdentifier;
 use document_legacy::LayerId;
@@ -596,13 +596,6 @@ impl JsEditorHandle {
 		}
 	}
 
-	/// Sends the blob URL generated by JS to the Imaginate layer in the respective document
-	#[wasm_bindgen(js_name = renderGraphUsingRasterizedRegionBelowLayer)]
-	pub fn render_graph_using_rasterized_region_below_layer(&self, document_id: u64, layer_path: Vec<LayerId>, _input_image_data: Vec<u8>, _width: u32, _height: u32) {
-		let message = PortfolioMessage::SubmitGraphRender { document_id, layer_path };
-		self.dispatch(message);
-	}
-
 	/// Notifies the backend that the user connected a node's primary output to one of another node's inputs
 	#[wasm_bindgen(js_name = connectNodesByLink)]
 	pub fn connect_nodes_by_link(&self, output_node: u64, output_node_connector_index: usize, input_node: u64, input_node_connector_index: usize) {

frontend/wasm/src/helpers.rs

@@ -1,68 +1,4 @@
-use crate::JS_EDITOR_HANDLES;
-
 use editor::messages::input_mapper::utility_types::input_keyboard::Key;
-use editor::messages::prelude::*;
-
-use std::panic;
-use wasm_bindgen::prelude::*;
-
-/// When a panic occurs, notify the user and log the error to the JS console before the backend dies
-pub fn panic_hook(info: &panic::PanicInfo) {
-	error!("{info}");
-
-	JS_EDITOR_HANDLES.with(|instances| {
-		instances
-			.borrow_mut()
-			.values_mut()
-			.for_each(|instance| instance.send_frontend_message_to_js_rust_proxy(FrontendMessage::DisplayDialogPanic { panic_info: info.to_string() }))
-	});
-}
-
-/// The JavaScript `Error` type
-#[wasm_bindgen]
-extern "C" {
-	#[derive(Clone, Debug)]
-	pub type Error;
-
-	#[wasm_bindgen(constructor)]
-	pub fn new(msg: &str) -> Error;
-}
-
-/// Logging to the JS console
-#[wasm_bindgen]
-extern "C" {
-	#[wasm_bindgen(js_namespace = console)]
-	fn log(msg: &str, format: &str);
-	#[wasm_bindgen(js_namespace = console)]
-	fn info(msg: &str, format: &str);
-	#[wasm_bindgen(js_namespace = console)]
-	fn warn(msg: &str, format: &str);
-	#[wasm_bindgen(js_namespace = console)]
-	fn error(msg: &str, format: &str);
-}
-
-#[derive(Default)]
-pub struct WasmLog;
-
-impl log::Log for WasmLog {
-	fn enabled(&self, metadata: &log::Metadata) -> bool {
-		metadata.level() <= log::Level::Info
-	}
-
-	fn log(&self, record: &log::Record) {
-		let (log, name, color): (fn(&str, &str), &str, &str) = match record.level() {
-			log::Level::Trace => (log, "trace", "color:plum"),
-			log::Level::Debug => (log, "debug", "color:cyan"),
-			log::Level::Warn => (warn, "warn", "color:goldenrod"),
-			log::Level::Info => (info, "info", "color:mediumseagreen"),
-			log::Level::Error => (error, "error", "color:red"),
-		};
-		let msg = &format!("%c{}\t{}", name, record.args());
-		log(msg, color)
-	}
-
-	fn flush(&self) {}
-}
 
 /// Translate a keyboard key from its JS name to its Rust `Key` enum
 pub fn translate_key(name: &str) -> Key {

frontend/wasm/src/lib.rs

@@ -7,12 +7,12 @@ extern crate log;
 pub mod editor_api;
 pub mod helpers;
 
-use helpers::{panic_hook, WasmLog};
+use editor::messages::prelude::*;
 
 use std::cell::RefCell;
 use std::collections::HashMap;
 use std::panic;
-use std::sync::atomic::AtomicBool;
+use std::sync::atomic::{AtomicBool, Ordering};
 use wasm_bindgen::prelude::*;
 
 // Set up the persistent editor backend state
@@ -33,3 +33,63 @@ pub fn init_graphite() {
 	log::set_logger(&LOGGER).expect("Failed to set logger");
 	log::set_max_level(log::LevelFilter::Debug);
 }
+
+/// When a panic occurs, notify the user and log the error to the JS console before the backend dies
+pub fn panic_hook(info: &panic::PanicInfo) {
+	EDITOR_HAS_CRASHED.store(true, Ordering::SeqCst);
+
+	error!("{info}");
+
+	JS_EDITOR_HANDLES.with(|instances| {
+		instances
+			.borrow_mut()
+			.values_mut()
+			.for_each(|instance| instance.send_frontend_message_to_js_rust_proxy(FrontendMessage::DisplayDialogPanic { panic_info: info.to_string() }))
+	});
+}
+
+/// The JavaScript `Error` type
+#[wasm_bindgen]
+extern "C" {
+	#[derive(Clone, Debug)]
+	pub type Error;
+
+	#[wasm_bindgen(constructor)]
+	pub fn new(msg: &str) -> Error;
+}
+
+/// Logging to the JS console
+#[wasm_bindgen]
+extern "C" {
+	#[wasm_bindgen(js_namespace = console)]
+	fn log(msg: &str, format: &str);
+	#[wasm_bindgen(js_namespace = console)]
+	fn info(msg: &str, format: &str);
+	#[wasm_bindgen(js_namespace = console)]
+	fn warn(msg: &str, format: &str);
+	#[wasm_bindgen(js_namespace = console)]
+	fn error(msg: &str, format: &str);
+}
+
+#[derive(Default)]
+pub struct WasmLog;
+
+impl log::Log for WasmLog {
+	fn enabled(&self, metadata: &log::Metadata) -> bool {
+		metadata.level() <= log::Level::Info
+	}
+
+	fn log(&self, record: &log::Record) {
+		let (log, name, color): (fn(&str, &str), &str, &str) = match record.level() {
+			log::Level::Trace => (log, "trace", "color:plum"),
+			log::Level::Debug => (log, "debug", "color:cyan"),
+			log::Level::Warn => (warn, "warn", "color:goldenrod"),
+			log::Level::Info => (info, "info", "color:mediumseagreen"),
+			log::Level::Error => (error, "error", "color:red"),
+		};
+		let msg = &format!("%c{}\t{}", name, record.args());
+		log(msg, color)
+	}
+
+	fn flush(&self) {}
+}