Display images in the SVG viewport renderer via canvases instead of base64 PNGs (#2903)

* add: move images as rendered canvases to node_graph_executor

* add: added the frontend message

* fix: bytemuck stuff

* fix: canvas element breaking

* fix: width issues

* fix: remove the old message

* npm: run lint-fix

* fix

* works finally

* fix transforms

* Fix self closing tag

* fix: reuse id

* fix: have it working with repeat instance

* cargo: fmt

* fix

* Avoid "canvas" prefix to IDs

* fix

* fix: vello issue from 6111440

* fix: gpu stuff

* fix: vello bbox

* Code review

---------

Co-authored-by: hypercube <0hypercube@gmail.com>
Co-authored-by: Keavon Chambers <keavon@keavon.com>

node-graph/gcore/src/blending.rs

@@ -56,6 +56,10 @@ impl AlphaBlending {
 			clip: if t < 0.5 { self.clip } else { other.clip },
 		}
 	}
+
+	pub fn opacity(&self, mask: bool) -> f32 {
+		self.opacity * if mask { 1. } else { self.fill }
+	}
 }
 
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]

node-graph/gcore/src/raster.rs

@@ -1,12 +1,3 @@
-use crate::GraphicGroupTable;
-pub use crate::color::*;
-use crate::raster_types::{CPU, RasterDataTable};
-use crate::vector::VectorDataTable;
-use std::fmt::Debug;
-
-#[cfg(target_arch = "spirv")]
-use spirv_std::num_traits::float::Float;
-
 /// as to not yet rename all references
 pub mod color {
 	pub use super::*;
@@ -14,7 +5,14 @@ pub mod color {
 
 pub mod image;
 
-pub use self::image::Image;
+pub use self::image::{Image, TransformImage};
+use crate::GraphicGroupTable;
+pub use crate::color::*;
+use crate::raster_types::{CPU, RasterDataTable};
+use crate::vector::VectorDataTable;
+#[cfg(target_arch = "spirv")]
+use spirv_std::num_traits::float::Float;
+use std::fmt::Debug;
 
 pub trait Bitmap {
 	type Pixel: Pixel;

node-graph/gcore/src/raster/image.rs

@@ -50,6 +50,13 @@ pub struct Image<P: Pixel> {
 	// TODO: Currently it is always anchored at the top left corner at (0, 0). The bottom right corner of the new origin field would correspond to (1, 1).
 }
 
+#[derive(Debug, Clone, dyn_any::DynAny, Default, PartialEq, serde::Serialize, serde::Deserialize, specta::Type)]
+pub struct TransformImage(pub DAffine2);
+
+impl Hash for TransformImage {
+	fn hash<H: std::hash::Hasher>(&self, _: &mut H) {}
+}
+
 impl<P: Pixel + Debug> Debug for Image<P> {
 	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
 		let length = self.data.len();

node-graph/gcore/src/vector/vector_nodes.rs

@@ -417,7 +417,6 @@ where
 	// Create and add mirrored instance
 	for mut instance in instance.instance_iter() {
 		instance.transform = reflected_transform * instance.transform;
-		instance.source_node_id = None;
 		result_table.push(instance);
 	}
 
@@ -448,6 +447,7 @@ async fn round_corners(
 		.map(|source| {
 			let source_transform = *source.transform;
 			let source_transform_inverse = source_transform.inverse();
+			let source_node_id = source.source_node_id;
 			let source = source.instance;
 
 			let upstream_graphic_group = source.upstream_graphic_group.clone();
@@ -542,7 +542,7 @@ async fn round_corners(
 				instance: result,
 				transform: source_transform,
 				alpha_blending: Default::default(),
-				source_node_id: None,
+				source_node_id: *source_node_id,
 			}
 		})
 		.collect()
@@ -645,7 +645,6 @@ async fn box_warp(_: impl Ctx, vector_data: VectorDataTable, #[expose] rectangle
 			// Add this to the table and reset the transform since we've applied it directly to the points
 			vector_data_instance.instance = result;
 			vector_data_instance.transform = DAffine2::IDENTITY;
-			vector_data_instance.source_node_id = None;
 			vector_data_instance
 		})
 		.collect()
@@ -917,7 +916,6 @@ async fn bounding_box(_: impl Ctx, vector_data: VectorDataTable) -> VectorDataTa
 			result.style.set_stroke_transform(DAffine2::IDENTITY);
 
 			vector_data_instance.instance = result;
-			vector_data_instance.source_node_id = None;
 			vector_data_instance
 		})
 		.collect()
@@ -1013,7 +1011,6 @@ async fn offset_path(_: impl Ctx, vector_data: VectorDataTable, distance: f64, j
 			}
 
 			vector_data_instance.instance = result;
-			vector_data_instance.source_node_id = None;
 			vector_data_instance
 		})
 		.collect()
@@ -1068,7 +1065,6 @@ async fn solidify_stroke(_: impl Ctx, vector_data: VectorDataTable) -> VectorDat
 			}
 
 			vector_data_instance.instance = result;
-			vector_data_instance.source_node_id = None;
 			vector_data_instance
 		})
 		.collect()

node-graph/graph-craft/src/document/value.rs

@@ -7,6 +7,7 @@ pub use glam::{DAffine2, DVec2, IVec2, UVec2};
 use graphene_application_io::SurfaceFrame;
 use graphene_brush::brush_cache::BrushCache;
 use graphene_brush::brush_stroke::BrushStroke;
+use graphene_core::raster::Image;
 use graphene_core::raster_types::CPU;
 use graphene_core::transform::ReferencePoint;
 use graphene_core::uuid::NodeId;
@@ -424,10 +425,10 @@ pub struct RenderOutput {
 	pub metadata: RenderMetadata,
 }
 
-#[derive(Debug, Clone, PartialEq, dyn_any::DynAny, Hash, serde::Serialize, serde::Deserialize)]
+#[derive(Debug, Clone, Hash, PartialEq, dyn_any::DynAny, serde::Serialize, serde::Deserialize)]
 pub enum RenderOutputType {
 	CanvasFrame(SurfaceFrame),
-	Svg(String),
+	Svg { svg: String, image_data: Vec<(u64, Image<Color>)> },
 	Image(Vec<u8>),
 }
 

node-graph/graph-craft/src/wasm_application_io.rs

@@ -3,6 +3,7 @@ use graphene_application_io::{ApplicationError, ApplicationIo, ResourceFuture, S
 #[cfg(target_arch = "wasm32")]
 use js_sys::{Object, Reflect};
 use std::collections::HashMap;
+use std::hash::Hash;
 use std::sync::Arc;
 #[cfg(target_arch = "wasm32")]
 use std::sync::atomic::AtomicU64;
@@ -39,7 +40,7 @@ impl Drop for WindowWrapper {
 		let wrapper = || {
 			if let Ok(canvases) = Reflect::get(&window, &image_canvases_key) {
 				// Convert key and value to JsValue
-				let js_key = JsValue::from_str(format!("canvas{}", self.window.window_id).as_str());
+				let js_key = JsValue::from_str(self.window.window_id.to_string().as_str());
 
 				// Use Reflect API to set property
 				Reflect::delete_property(&canvases.into(), &js_key)?;
@@ -200,7 +201,7 @@ impl ApplicationIo for WasmApplicationIo {
 			}
 
 			// Convert key and value to JsValue
-			let js_key = JsValue::from_str(format!("canvas{}", id).as_str());
+			let js_key = JsValue::from_str(id.to_string().as_str());
 			let js_value = JsValue::from(canvas.clone());
 
 			let canvases = Object::from(canvases.unwrap());
@@ -253,7 +254,7 @@ impl ApplicationIo for WasmApplicationIo {
 		let wrapper = || {
 			if let Ok(canvases) = Reflect::get(&window, &image_canvases_key) {
 				// Convert key and value to JsValue
-				let js_key = JsValue::from_str(format!("canvas{}", surface_id.0).as_str());
+				let js_key = JsValue::from_str(surface_id.0.to_string().as_str());
 
 				// Use Reflect API to set property
 				Reflect::delete_property(&canvases.into(), &js_key)?;

node-graph/graster-nodes/src/dehaze.rs

@@ -31,7 +31,6 @@ async fn dehaze(_: impl Ctx, image_frame: RasterDataTable<CPU>, strength: Percen
 			};
 
 			image_frame_instance.instance = Raster::new_cpu(dehazed_image);
-			image_frame_instance.source_node_id = None;
 			image_frame_instance
 		})
 		.collect()

node-graph/graster-nodes/src/filter.rs

@@ -36,7 +36,6 @@ async fn blur(
 			};
 
 			image_instance.instance = blurred_image;
-			image_instance.source_node_id = None;
 			image_instance
 		})
 		.collect()

node-graph/graster-nodes/src/std_nodes.rs

@@ -88,7 +88,6 @@ pub fn sample_image(ctx: impl ExtractFootprint + Clone + Send, image_frame: Rast
 			let new_transform = image_frame_transform * DAffine2::from_translation(offset) * DAffine2::from_scale(size);
 
 			image_frame_instance.transform = new_transform;
-			image_frame_instance.source_node_id = None;
 			image_frame_instance.instance = Raster::new_cpu(image);
 			Some(image_frame_instance)
 		})
@@ -121,9 +120,10 @@ pub fn combine_channels(
 			let alpha = alpha.filter(|i| i.instance.width > 0 && i.instance.height > 0);
 
 			// Get this instance's transform and alpha blending mode from the first non-empty channel
-			let Some((transform, alpha_blending)) = [&red, &green, &blue, &alpha].iter().find_map(|i| i.as_ref()).map(|i| (i.transform, i.alpha_blending)) else {
-				return None;
-			};
+			let (transform, alpha_blending, source_node_id) = [&red, &green, &blue, &alpha]
+				.iter()
+				.find_map(|i| i.as_ref())
+				.map(|i| (i.transform, i.alpha_blending, i.source_node_id))?;
 
 			// Get the common width and height of the channels, which must have equal dimensions
 			let channel_dimensions = [
@@ -140,9 +140,7 @@ pub fn combine_channels(
 			{
 				return None;
 			}
-			let Some(&(width, height)) = channel_dimensions.iter().flatten().next() else {
-				return None;
-			};
+			let &(width, height) = channel_dimensions.iter().flatten().next()?;
 
 			// Create a new image for this instance output
 			let mut image = Image::new(width, height, Color::TRANSPARENT);
@@ -179,7 +177,7 @@ pub fn combine_channels(
 				instance: Raster::new_cpu(image),
 				transform,
 				alpha_blending,
-				source_node_id: None,
+				source_node_id,
 			})
 		})
 		.collect()
@@ -276,7 +274,6 @@ pub fn extend_image_to_bounds(_: impl Ctx, image: RasterDataTable<CPU>, bounds:
 
 			image_instance.instance = Raster::new_cpu(new_image);
 			image_instance.transform = new_texture_to_layer_space;
-			image_instance.source_node_id = None;
 			image_instance
 		})
 		.collect()

node-graph/gstd/src/wasm_application_io.rs

@@ -30,35 +30,6 @@ async fn create_surface<'a: 'n>(_: impl Ctx, editor: &'a WasmEditorApi) -> Arc<W
 	Arc::new(editor.application_io.as_ref().unwrap().create_window())
 }
 
-// TODO: Fix and reenable in order to get the 'Draw Canvas' node working again.
-// #[cfg(target_arch = "wasm32")]
-// use wasm_bindgen::Clamped;
-//
-// #[node_macro::node(category("Debug: GPU"))]
-// #[cfg(target_arch = "wasm32")]
-// async fn draw_image_frame(
-// 	_: impl Ctx,
-// 	image: RasterDataTable<graphene_core::raster::SRGBA8>,
-// 	surface_handle: Arc<WasmSurfaceHandle>,
-// ) -> graphene_core::application_io::SurfaceHandleFrame<HtmlCanvasElement> {
-// 	let image = image.instance_ref_iter().next().unwrap().instance;
-// 	let image_data = image.image.data;
-// 	let array: Clamped<&[u8]> = Clamped(bytemuck::cast_slice(image_data.as_slice()));
-// 	if image.image.width > 0 && image.image.height > 0 {
-// 		let canvas = &surface_handle.surface;
-// 		canvas.set_width(image.image.width);
-// 		canvas.set_height(image.image.height);
-// 		// TODO: replace "2d" with "bitmaprenderer" once we switch to ImageBitmap (lives on gpu) from RasterData (lives on cpu)
-// 		let context = canvas.get_context("2d").unwrap().unwrap().dyn_into::<CanvasRenderingContext2d>().unwrap();
-// 		let image_data = web_sys::ImageData::new_with_u8_clamped_array_and_sh(array, image.image.width, image.image.height).expect("Failed to construct RasterData");
-// 		context.put_image_data(&image_data, 0., 0.).unwrap();
-// 	}
-// 	graphene_core::application_io::SurfaceHandleFrame {
-// 		surface_handle,
-// 		transform: image.transform,
-// 	}
-// }
-
 #[node_macro::node(category("Web Request"))]
 async fn get_request(_: impl Ctx, _primary: (), #[name("URL")] url: String, discard_result: bool) -> String {
 	#[cfg(target_arch = "wasm32")]
@@ -187,7 +158,10 @@ fn render_svg(data: impl GraphicElementRendered, mut render: SvgRender, render_p
 
 	render.wrap_with_transform(footprint.transform, Some(footprint.resolution.as_dvec2()));
 
-	RenderOutputType::Svg(render.svg.to_svg_string())
+	RenderOutputType::Svg {
+		svg: render.svg.to_svg_string(),
+		image_data: render.image_data,
+	}
 }
 
 #[cfg(feature = "vello")]

node-graph/gsvg-renderer/src/renderer.rs

@@ -20,6 +20,7 @@ use graphene_core::{Artboard, ArtboardGroupTable, GraphicElement, GraphicGroupTa
 use num_traits::Zero;
 use std::collections::{HashMap, HashSet};
 use std::fmt::Write;
+use std::hash::{DefaultHasher, Hash, Hasher};
 use std::ops::Deref;
 use std::sync::{Arc, LazyLock};
 #[cfg(feature = "vello")]
@@ -175,6 +176,10 @@ impl RenderParams {
 		let alignment_parent_transform = Some(transform);
 		Self { alignment_parent_transform, ..*self }
 	}
+
+	pub fn to_canvas(&self) -> bool {
+		!self.for_export && !self.thumbnail && !self.for_mask
+	}
 }
 
 pub fn format_transform_matrix(transform: DAffine2) -> String {
@@ -243,8 +248,7 @@ impl GraphicElementRendered for GraphicGroupTable {
 						attributes.push("transform", matrix);
 					}
 
-					let factor = if render_params.for_mask { 1. } else { instance.alpha_blending.fill };
-					let opacity = instance.alpha_blending.opacity * factor;
+					let opacity = instance.alpha_blending.opacity(render_params.for_mask);
 					if opacity < 1. {
 						attributes.push("opacity", opacity.to_string());
 					}
@@ -299,8 +303,7 @@ impl GraphicElementRendered for GraphicGroupTable {
 			};
 			let mut bounds = None;
 
-			let factor = if render_params.for_mask { 1. } else { alpha_blending.fill };
-			let opacity = alpha_blending.opacity * factor;
+			let opacity = instance.alpha_blending.opacity(render_params.for_mask);
 			if opacity < 1. || (render_params.view_mode != ViewMode::Outline && alpha_blending.blend_mode != BlendMode::default()) {
 				bounds = self
 					.instance_ref_iter()
@@ -503,8 +506,7 @@ impl GraphicElementRendered for VectorDataTable {
 				}
 				attributes.push_val(fill_and_stroke);
 
-				let factor = if render_params.for_mask { 1. } else { instance.alpha_blending.fill };
-				let opacity = instance.alpha_blending.opacity * factor;
+				let opacity = instance.alpha_blending.opacity(render_params.for_mask);
 				if opacity < 1. {
 					attributes.push("opacity", opacity.to_string());
 				}
@@ -545,10 +547,13 @@ impl GraphicElementRendered for VectorDataTable {
 				_ => instance.alpha_blending.blend_mode.to_peniko(),
 			};
 			let mut layer = false;
-			let factor = if render_params.for_mask { 1. } else { instance.alpha_blending.fill };
-			let opacity = instance.alpha_blending.opacity * factor;
+
+			let opacity = instance.alpha_blending.opacity(render_params.for_mask);
 			if opacity < 1. || instance.alpha_blending.blend_mode != BlendMode::default() {
 				layer = true;
+				let weight = instance.instance.style.stroke().unwrap().weight;
+				let quad = Quad::from_box(layer_bounds).inflate(weight * element_transform.matrix2.determinant());
+				let layer_bounds = quad.bounding_box();
 				scene.push_layer(
 					peniko::BlendMode::new(blend_mode, peniko::Compose::SrcOver),
 					opacity,
@@ -945,56 +950,122 @@ impl GraphicElementRendered for RasterDataTable<CPU> {
 	fn render_svg(&self, render: &mut SvgRender, render_params: &RenderParams) {
 		for instance in self.instance_ref_iter() {
 			let transform = *instance.transform;
-
 			let image = &instance.instance;
+
 			if image.data.is_empty() {
-				return;
+				continue;
 			}
 
-			let base64_string = image.base64_string.clone().unwrap_or_else(|| {
-				use base64::Engine;
+			if render_params.to_canvas() {
+				let id = instance.source_node_id.map(|x| x.0).unwrap_or_else(|| {
+					let mut state = DefaultHasher::new();
+					image.data().hash(&mut state);
+					state.finish()
+				});
+				if !render.image_data.iter().any(|(old_id, _)| *old_id == id) {
+					render.image_data.push((id, image.data().clone()));
+				}
+				render.parent_tag(
+					"foreignObject",
+					|attributes| {
+						let mut transform_values = transform.to_scale_angle_translation();
+						let size = DVec2::new(image.width as f64, image.height as f64);
+						transform_values.0 /= size;
 
-				let output = image.to_png();
-				let preamble = "data:image/png;base64,";
-				let mut base64_string = String::with_capacity(preamble.len() + output.len() * 4);
-				base64_string.push_str(preamble);
-				base64::engine::general_purpose::STANDARD.encode_string(output, &mut base64_string);
-				base64_string
-			});
-			render.leaf_tag("image", |attributes| {
-				attributes.push("width", 1.to_string());
-				attributes.push("height", 1.to_string());
-				attributes.push("preserveAspectRatio", "none");
-				attributes.push("href", base64_string);
-				let matrix = format_transform_matrix(transform);
-				if !matrix.is_empty() {
-					attributes.push("transform", matrix);
-				}
-				let factor = if render_params.for_mask { 1. } else { instance.alpha_blending.fill };
-				let opacity = instance.alpha_blending.opacity * factor;
-				if opacity < 1. {
-					attributes.push("opacity", opacity.to_string());
-				}
-				if instance.alpha_blending.blend_mode != BlendMode::default() {
-					attributes.push("style", instance.alpha_blending.blend_mode.render());
-				}
-			});
+						let matrix = DAffine2::from_scale_angle_translation(transform_values.0, transform_values.1, transform_values.2);
+						let matrix = format_transform_matrix(matrix);
+						if !matrix.is_empty() {
+							attributes.push("transform", matrix);
+						}
+
+						attributes.push("width", size.x.to_string());
+						attributes.push("height", size.y.to_string());
+
+						let opacity = instance.alpha_blending.opacity(render_params.for_mask);
+						if opacity < 1. {
+							attributes.push("opacity", opacity.to_string());
+						}
+
+						if instance.alpha_blending.blend_mode != BlendMode::default() {
+							attributes.push("style", instance.alpha_blending.blend_mode.render());
+						}
+					},
+					|render| {
+						render.leaf_tag(
+							"img", // Must be a self-closing (void element) tag, so we can't use `div` or `span`, for example
+							|attributes| {
+								attributes.push("data-canvas-placeholder", id.to_string());
+							},
+						)
+					},
+				);
+			} else {
+				let base64_string = image.base64_string.clone().unwrap_or_else(|| {
+					use base64::Engine;
+
+					let output = image.to_png();
+					let preamble = "data:image/png;base64,";
+					let mut base64_string = String::with_capacity(preamble.len() + output.len() * 4);
+					base64_string.push_str(preamble);
+					base64::engine::general_purpose::STANDARD.encode_string(output, &mut base64_string);
+					base64_string
+				});
+
+				render.leaf_tag("image", |attributes| {
+					attributes.push("width", "1");
+					attributes.push("height", "1");
+					attributes.push("preserveAspectRatio", "none");
+					attributes.push("href", base64_string);
+					let matrix = format_transform_matrix(transform);
+					if !matrix.is_empty() {
+						attributes.push("transform", matrix);
+					}
+
+					let opacity = instance.alpha_blending.opacity(render_params.for_mask);
+					if opacity < 1. {
+						attributes.push("opacity", opacity.to_string());
+					}
+					if instance.alpha_blending.blend_mode != BlendMode::default() {
+						attributes.push("style", instance.alpha_blending.blend_mode.render());
+					}
+				});
+			}
 		}
 	}
 
 	#[cfg(feature = "vello")]
-	fn render_to_vello(&self, scene: &mut Scene, transform: DAffine2, _: &mut RenderContext, _render_params: &RenderParams) {
+	fn render_to_vello(&self, scene: &mut Scene, transform: DAffine2, _: &mut RenderContext, render_params: &RenderParams) {
 		use vello::peniko;
 
 		for instance in self.instance_ref_iter() {
 			let image = &instance.instance;
 			if image.data.is_empty() {
-				return;
+				continue;
 			}
-			let image = peniko::Image::new(image.to_flat_u8().0.into(), peniko::ImageFormat::Rgba8, image.width, image.height).with_extend(peniko::Extend::Repeat);
-			let transform = transform * *instance.transform * DAffine2::from_scale(1. / DVec2::new(image.width as f64, image.height as f64));
 
-			scene.draw_image(&image, kurbo::Affine::new(transform.to_cols_array()));
+			let alpha_blending = *instance.alpha_blending;
+			let blend_mode = alpha_blending.blend_mode.to_peniko();
+
+			let opacity = alpha_blending.opacity(render_params.for_mask);
+			let mut layer = false;
+
+			if opacity < 1. || alpha_blending.blend_mode != BlendMode::default() {
+				if let Some(bounds) = self.bounding_box(transform, false) {
+					let blending = peniko::BlendMode::new(blend_mode, peniko::Compose::SrcOver);
+					let rect = kurbo::Rect::new(bounds[0].x, bounds[0].y, bounds[1].x, bounds[1].y);
+					scene.push_layer(blending, opacity, kurbo::Affine::IDENTITY, &rect);
+					layer = true;
+				}
+			}
+
+			let image = peniko::Image::new(image.to_flat_u8().0.into(), peniko::ImageFormat::Rgba8, image.width, image.height).with_extend(peniko::Extend::Repeat);
+			let image_transform = transform * *instance.transform * DAffine2::from_scale(1. / DVec2::new(image.width as f64, image.height as f64));
+
+			scene.draw_image(&image, kurbo::Affine::new(image_transform.to_cols_array()));
+
+			if layer {
+				scene.pop_layer();
+			}
 		}
 	}
 
@@ -1029,12 +1100,14 @@ impl GraphicElementRendered for RasterDataTable<GPU> {
 
 		for instance in self.instance_ref_iter() {
 			let blend_mode = *instance.alpha_blending;
-			let layer = blend_mode != Default::default();
-			if layer {
-				let Some(bounds) = self.bounding_box(transform, true) else { return };
-				let blending = peniko::BlendMode::new(blend_mode.blend_mode.to_peniko(), peniko::Compose::SrcOver);
-				let rect = kurbo::Rect::new(bounds[0].x, bounds[0].y, bounds[1].x, bounds[1].y);
-				scene.push_layer(blending, blend_mode.opacity, kurbo::Affine::IDENTITY, &rect);
+			let mut layer = false;
+			if blend_mode != Default::default() {
+				if let Some(bounds) = self.bounding_box(transform, true) {
+					let blending = peniko::BlendMode::new(blend_mode.blend_mode.to_peniko(), peniko::Compose::SrcOver);
+					let rect = kurbo::Rect::new(bounds[0].x, bounds[0].y, bounds[1].x, bounds[1].y);
+					scene.push_layer(blending, blend_mode.opacity, kurbo::Affine::IDENTITY, &rect);
+					layer = true;
+				}
 			}
 
 			let image = peniko::Image::new(