Desktop: Execute editor and node graph natively (#2955)

* Desktop: Execute editor and node graph natively

* Remove decouple execution feature

* Disable feature gate for native communication functions

* Avoid ininite message loop on an infinite canvas

* Add any lint exception

* Build evaluation loop

* Fix texture passing message

* Cleanup

* More cleanup

---------

Co-authored-by: Timon Schelling <me@timon.zip>

frontend/package.json

@@ -12,6 +12,7 @@
 		"production": "npm run setup && npm run wasm:build-production && concurrently -k -n \"VITE,RUST\" \"vite\" \"npm run wasm:watch-production\"",
 		"---------- BUILDS ----------": "",
 		"build-dev": "npm run wasm:build-dev && vite build",
+		"build-native": "npm run native:build-dev && vite build",
 		"build-profiling": "npm run wasm:build-profiling && vite build",
 		"build": "npm run wasm:build-production && vite build",
 		"---------- UTILITIES ----------": "",
@@ -19,6 +20,7 @@
 		"lint-fix": "eslint . --fix && tsc --noEmit",
 		"---------- INTERNAL ----------": "",
 		"setup": "node package-installer.js",
+		"native:build-dev": "wasm-pack build ./wasm --dev --target=web --features native",
 		"wasm:build-dev": "wasm-pack build ./wasm --dev --target=web",
 		"wasm:build-profiling": "wasm-pack build ./wasm --profiling --target=web",
 		"wasm:build-production": "wasm-pack build ./wasm --release --target=web",

frontend/src/editor.ts

@@ -1,7 +1,7 @@
 // import { panicProxy } from "@graphite/utility-functions/panic-proxy";
 import { type JsMessageType } from "@graphite/messages";
 import { createSubscriptionRouter, type SubscriptionRouter } from "@graphite/subscription-router";
-import init, { setRandomSeed, wasmMemory, EditorHandle } from "@graphite-frontend/wasm/pkg/graphite_wasm.js";
+import init, { setRandomSeed, wasmMemory, EditorHandle, receiveNativeMessage } from "@graphite-frontend/wasm/pkg/graphite_wasm.js";
 
 export type Editor = {
 	raw: WebAssembly.Memory;
@@ -27,6 +27,8 @@ export async function initWasm() {
 	wasmImport = await wasmMemory();
 	// eslint-disable-next-line @typescript-eslint/no-explicit-any
 	(window as any).imageCanvases = {};
+	// eslint-disable-next-line @typescript-eslint/no-explicit-any
+	(window as any).receiveNativeMessage = receiveNativeMessage;
 
 	// Provide a random starter seed which must occur after initializing the WASM module, since WASM can't generate its own random numbers
 	const randomSeedFloat = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER);

frontend/wasm/Cargo.toml

@@ -13,6 +13,7 @@ license = "Apache-2.0"
 [features]
 default = ["gpu"]
 gpu = ["editor/gpu"]
+native = []
 
 [lib]
 crate-type = ["cdylib", "rlib"]
@@ -37,6 +38,7 @@ wasm-bindgen-futures = { workspace = true }
 math-parser = { workspace = true }
 wgpu = { workspace = true }
 web-sys = { workspace = true }
+ron = { workspace = true }
 
 [package.metadata.wasm-pack.profile.dev]
 wasm-opt = false

frontend/wasm/src/editor_api.rs

@@ -154,6 +154,7 @@ impl EditorHandle {
 	}
 
 	// Sends a message to the dispatcher in the Editor Backend
+	#[cfg(not(feature = "native"))]
 	fn dispatch<T: Into<Message>>(&self, message: T) {
 		// Process no further messages after a crash to avoid spamming the console
 		if EDITOR_HAS_CRASHED.load(Ordering::SeqCst) {
@@ -169,6 +170,16 @@ impl EditorHandle {
 		}
 	}
 
+	#[cfg(feature = "native")]
+	fn dispatch<T: Into<Message>>(&self, message: T) {
+		let message: Message = message.into();
+		let Ok(serialized_message) = ron::to_string(&message) else {
+			log::error!("Failed to serialize message");
+			return;
+		};
+		crate::native_communcation::send_message_to_cef(serialized_message)
+	}
+
 	// Sends a FrontendMessage to JavaScript
 	fn send_frontend_message_to_js(&self, mut message: FrontendMessage) {
 		if let FrontendMessage::UpdateImageData { ref image_data } = message {
@@ -228,22 +239,15 @@ impl EditorHandle {
 				wasm_bindgen_futures::spawn_local(poll_node_graph_evaluation());
 
 				if !EDITOR_HAS_CRASHED.load(Ordering::SeqCst) {
-					editor_and_handle(|editor, handle| {
-						for message in editor.handle_message(InputPreprocessorMessage::CurrentTime {
+					editor_and_handle(|_, handle| {
+						handle.dispatch(InputPreprocessorMessage::CurrentTime {
 							timestamp: js_sys::Date::now() as u64,
-						}) {
-							handle.send_frontend_message_to_js(message);
-						}
-
-						for message in editor.handle_message(AnimationMessage::IncrementFrameCounter) {
-							handle.send_frontend_message_to_js(message);
-						}
+						});
+						handle.dispatch(AnimationMessage::IncrementFrameCounter);
 
 						// Used by auto-panning, but this could possibly be refactored in the future, see:
 						// <https://github.com/GraphiteEditor/Graphite/pull/2562#discussion_r2041102786>
-						for message in editor.handle_message(BroadcastMessage::TriggerEvent(BroadcastEvent::AnimationFrame)) {
-							handle.send_frontend_message_to_js(message);
-						}
+						handle.dispatch(BroadcastMessage::TriggerEvent(BroadcastEvent::AnimationFrame));
 					});
 				}
 
@@ -910,7 +914,7 @@ fn editor<T: Default>(callback: impl FnOnce(&mut editor::application::Editor) ->
 }
 
 /// Provides access to the `Editor` and its `EditorHandle` by calling the given closure with them as arguments.
-pub(crate) fn editor_and_handle(mut callback: impl FnMut(&mut Editor, &mut EditorHandle)) {
+pub(crate) fn editor_and_handle(callback: impl FnOnce(&mut Editor, &mut EditorHandle)) {
 	EDITOR_HANDLE.with(|editor_handle| {
 		editor(|editor| {
 			let mut guard = editor_handle.try_lock();
@@ -964,9 +968,7 @@ fn auto_save_all_documents() {
 		return;
 	}
 
-	editor_and_handle(|editor, handle| {
-		for message in editor.handle_message(PortfolioMessage::AutoSaveAllDocuments) {
-			handle.send_frontend_message_to_js(message);
-		}
+	editor_and_handle(|_, handle| {
+		handle.dispatch(PortfolioMessage::AutoSaveAllDocuments);
 	});
 }

frontend/wasm/src/lib.rs

@@ -6,6 +6,7 @@ extern crate log;
 
 pub mod editor_api;
 pub mod helpers;
+pub mod native_communcation;
 
 use editor::messages::prelude::*;
 use std::panic;

frontend/wasm/src/native_communcation.rs

@@ -0,0 +1,35 @@
+use editor::{application::Editor, messages::prelude::FrontendMessage};
+use js_sys::{ArrayBuffer, Uint8Array};
+use wasm_bindgen::prelude::*;
+
+use crate::editor_api::{self, EditorHandle};
+
+#[wasm_bindgen(js_name = "receiveNativeMessage")]
+pub fn receive_native_message(buffer: ArrayBuffer) {
+	let buffer = Uint8Array::new(buffer.as_ref()).to_vec();
+	match ron::from_str::<Vec<FrontendMessage>>(str::from_utf8(buffer.as_slice()).unwrap()) {
+		Ok(messages) => {
+			let callback = move |_: &mut Editor, handle: &mut EditorHandle| {
+				for message in messages {
+					handle.send_frontend_message_to_js_rust_proxy(message);
+				}
+			};
+			editor_api::editor_and_handle(callback);
+		}
+		Err(e) => log::error!("Failed to deserialize frontend messages: {e:?}"),
+	}
+}
+
+pub fn send_message_to_cef(message: String) {
+	let global = js_sys::global();
+
+	// Get the function by name
+	let func = js_sys::Reflect::get(&global, &JsValue::from_str("sendNativeMessage")).expect("Function not found");
+
+	let func = func.dyn_into::<js_sys::Function>().expect("Not a function");
+	let array = Uint8Array::from(message.as_bytes());
+	let buffer = array.buffer();
+
+	// Call it with argument
+	func.call1(&JsValue::NULL, &JsValue::from(buffer)).expect("Function call failed");
+}