Integrate raw WGPU textures into Vello rendering pipeline (#1897)

* WIP

* Introduce `Raster` enum and plumb texture injection to vello

* Use correct texture formas for usage in vello

* Add missing cache implementations

* Update vello image override api

* Use git version of vello

* Fix type for upload texture node

* Improve crash resiliance

* Fix warnings

* Remove unused node impls

node-graph/gcore/Cargo.toml

@@ -14,7 +14,8 @@ nightly = []
 alloc = ["dyn-any", "bezier-rs"]
 type_id_logging = []
 wasm = ["web-sys"]
-vello = ["dep:vello", "bezier-rs/kurbo"]
+wgpu = ["dep:wgpu"]
+vello = ["dep:vello", "bezier-rs/kurbo", "wgpu"]
 std = [
 	"dyn-any",
 	"dyn-any/std",
@@ -59,6 +60,7 @@ bezier-rs = { workspace = true, optional = true }
 kurbo = { workspace = true, optional = true }
 base64 = { workspace = true, optional = true }
 vello = { workspace  = true, optional = true }
+wgpu = { workspace = true, optional = true }
 specta = { workspace = true, optional = true }
 rustybuzz = { workspace = true, optional = true }
 wasm-bindgen = { workspace = true, optional = true }

node-graph/gcore/src/application_io.rs

@@ -63,6 +63,55 @@ impl Size for web_sys::HtmlCanvasElement {
 	}
 }
 
+#[derive(Debug, Clone)]
+pub struct TextureFrame {
+	#[cfg(feature = "wgpu")]
+	pub texture: Arc<wgpu::Texture>,
+	#[cfg(not(feature = "wgpu"))]
+	pub texture: (),
+	pub transform: DAffine2,
+}
+
+impl Hash for TextureFrame {
+	fn hash<H: Hasher>(&self, state: &mut H) {
+		self.transform.to_cols_array().iter().for_each(|x| x.to_bits().hash(state));
+		#[cfg(feature = "wgpu")]
+		self.texture.global_id().hash(state);
+	}
+}
+
+impl PartialEq for TextureFrame {
+	fn eq(&self, other: &Self) -> bool {
+		#[cfg(feature = "wgpu")]
+		return self.transform.eq(&other.transform) && self.texture.global_id() == other.texture.global_id();
+
+		#[cfg(not(feature = "wgpu"))]
+		self.transform.eq(&other.transform)
+	}
+}
+
+impl Transform for TextureFrame {
+	fn transform(&self) -> DAffine2 {
+		self.transform
+	}
+}
+impl TransformMut for TextureFrame {
+	fn transform_mut(&mut self) -> &mut DAffine2 {
+		&mut self.transform
+	}
+}
+
+unsafe impl StaticType for TextureFrame {
+	type Static = TextureFrame;
+}
+
+#[cfg(feature = "wgpu")]
+impl Size for TextureFrame {
+	fn size(&self) -> UVec2 {
+		UVec2::new(self.texture.width(), self.texture.height())
+	}
+}
+
 impl<S: Size> From<SurfaceHandleFrame<S>> for SurfaceFrame {
 	fn from(x: SurfaceHandleFrame<S>) -> Self {
 		Self {

node-graph/gcore/src/graphic_element.rs

@@ -1,15 +1,14 @@
-use crate::application_io::SurfaceHandleFrame;
+use crate::application_io::TextureFrame;
 use crate::raster::{BlendMode, ImageFrame};
-use crate::transform::Footprint;
+use crate::transform::{Footprint, Transform, TransformMut};
 use crate::vector::VectorData;
-use crate::{Color, Node, SurfaceFrame};
+use crate::{Color, Node};
 
 use dyn_any::{DynAny, StaticType};
 use node_macro::node_fn;
 
 use core::ops::{Deref, DerefMut};
-use glam::{DAffine2, IVec2, UVec2};
-use web_sys::HtmlCanvasElement;
+use glam::{DAffine2, IVec2};
 
 pub mod renderer;
 
@@ -65,10 +64,7 @@ pub enum GraphicElement {
 	GraphicGroup(GraphicGroup),
 	/// A vector shape, equivalent to the SVG <path> tag: https://developer.mozilla.org/en-US/docs/Web/SVG/Element/path
 	VectorData(Box<VectorData>),
-	/// A bitmap image with a finite position and extent, equivalent to the SVG <image> tag: https://developer.mozilla.org/en-US/docs/Web/SVG/Element/image
-	ImageFrame(ImageFrame<Color>),
-	/// A Canvas element
-	Surface(SurfaceFrame),
+	Raster(Raster),
 }
 
 // TODO: Can this be removed? It doesn't necessarily make that much sense to have a default when, instead, the entire GraphicElement just shouldn't exist if there's no specific content to assign it.
@@ -78,6 +74,58 @@ impl Default for GraphicElement {
 	}
 }
 
+#[derive(Clone, Debug, Hash, PartialEq, DynAny)]
+pub enum Raster {
+	/// A bitmap image with a finite position and extent, equivalent to the SVG <image> tag: https://developer.mozilla.org/en-US/docs/Web/SVG/Element/image
+	ImageFrame(ImageFrame<Color>),
+	Texture(TextureFrame),
+}
+
+impl<'de> serde::Deserialize<'de> for Raster {
+	fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+	where
+		D: serde::Deserializer<'de>,
+	{
+		let frame = ImageFrame::deserialize(deserializer)?;
+		Ok(Raster::ImageFrame(frame))
+	}
+}
+
+impl serde::Serialize for Raster {
+	fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+	where
+		S: serde::Serializer,
+	{
+		match self {
+			Raster::ImageFrame(_) => self.serialize(serializer),
+			Raster::Texture(_) => todo!(),
+		}
+	}
+}
+
+impl Transform for Raster {
+	fn transform(&self) -> DAffine2 {
+		match self {
+			Raster::ImageFrame(frame) => frame.transform(),
+			Raster::Texture(frame) => frame.transform(),
+		}
+	}
+	fn local_pivot(&self, pivot: glam::DVec2) -> glam::DVec2 {
+		match self {
+			Raster::ImageFrame(frame) => frame.local_pivot(pivot),
+			Raster::Texture(frame) => frame.local_pivot(pivot),
+		}
+	}
+}
+impl TransformMut for Raster {
+	fn transform_mut(&mut self) -> &mut DAffine2 {
+		match self {
+			Raster::ImageFrame(frame) => frame.transform_mut(),
+			Raster::Texture(frame) => frame.transform_mut(),
+		}
+	}
+}
+
 /// Some [`ArtboardData`] with some optional clipping bounds that can be exported.
 /// Similar to an Inkscape page: https://media.inkscape.org/media/doc/release_notes/1.2/Inkscape_1.2.html#Page_tool
 #[derive(Clone, Debug, Hash, PartialEq, DynAny)]
@@ -202,7 +250,12 @@ async fn add_artboard<Data: Into<Artboard> + Send>(footprint: Footprint, artboar
 
 impl From<ImageFrame<Color>> for GraphicElement {
 	fn from(image_frame: ImageFrame<Color>) -> Self {
-		GraphicElement::ImageFrame(image_frame)
+		GraphicElement::Raster(Raster::ImageFrame(image_frame))
+	}
+}
+impl From<TextureFrame> for GraphicElement {
+	fn from(texture: TextureFrame) -> Self {
+		GraphicElement::Raster(Raster::Texture(texture))
 	}
 }
 impl From<VectorData> for GraphicElement {
@@ -215,39 +268,6 @@ impl From<GraphicGroup> for GraphicElement {
 		GraphicElement::GraphicGroup(graphic_group)
 	}
 }
-impl From<SurfaceFrame> for GraphicElement {
-	fn from(surface: SurfaceFrame) -> Self {
-		GraphicElement::Surface(surface)
-	}
-}
-impl From<alloc::sync::Arc<SurfaceHandleFrame<HtmlCanvasElement>>> for GraphicElement {
-	fn from(surface: alloc::sync::Arc<SurfaceHandleFrame<HtmlCanvasElement>>) -> Self {
-		let surface_id = surface.surface_handle.window_id;
-		let transform = surface.transform;
-		GraphicElement::Surface(SurfaceFrame {
-			surface_id,
-			transform,
-			resolution: UVec2 {
-				x: surface.surface_handle.surface.width(),
-				y: surface.surface_handle.surface.height(),
-			},
-		})
-	}
-}
-impl From<SurfaceHandleFrame<HtmlCanvasElement>> for GraphicElement {
-	fn from(surface: SurfaceHandleFrame<HtmlCanvasElement>) -> Self {
-		let surface_id = surface.surface_handle.window_id;
-		let transform = surface.transform;
-		GraphicElement::Surface(SurfaceFrame {
-			surface_id,
-			transform,
-			resolution: UVec2 {
-				x: surface.surface_handle.surface.width(),
-				y: surface.surface_handle.surface.height(),
-			},
-		})
-	}
-}
 
 impl Deref for GraphicGroup {
 	type Target = Vec<GraphicElement>;

node-graph/gcore/src/graphic_element/renderer.rs

@@ -3,13 +3,12 @@ mod rect;
 pub use quad::Quad;
 pub use rect::Rect;
 
-use crate::raster::bbox::Bbox;
 use crate::raster::{BlendMode, Image, ImageFrame};
 use crate::transform::Transform;
 use crate::uuid::generate_uuid;
 use crate::vector::style::{Fill, Stroke, ViewMode};
 use crate::vector::PointId;
-use crate::SurfaceFrame;
+use crate::Raster;
 use crate::{vector::VectorData, Artboard, Color, GraphicElement, GraphicGroup};
 
 use bezier_rs::Subpath;
@@ -219,6 +218,12 @@ pub enum ImageRenderMode {
 	Base64,
 }
 
+#[derive(Clone, Debug, Default)]
+pub struct RenderContext {
+	#[cfg(feature = "wgpu")]
+	pub ressource_overrides: std::collections::HashMap<u64, alloc::sync::Arc<wgpu::Texture>>,
+}
+
 /// Static state used whilst rendering
 #[derive(Default)]
 pub struct RenderParams {
@@ -268,13 +273,13 @@ pub trait GraphicElementRendered {
 	fn bounding_box(&self, transform: DAffine2) -> Option<[DVec2; 2]>;
 	fn add_click_targets(&self, click_targets: &mut Vec<ClickTarget>);
 	#[cfg(feature = "vello")]
-	fn to_vello_scene(&self, transform: DAffine2) -> Scene {
+	fn to_vello_scene(&self, transform: DAffine2, context: &mut RenderContext) -> Scene {
 		let mut scene = vello::Scene::new();
-		self.render_to_vello(&mut scene, transform);
+		self.render_to_vello(&mut scene, transform, context);
 		scene
 	}
 	#[cfg(feature = "vello")]
-	fn render_to_vello(&self, _scene: &mut Scene, _transform: DAffine2) {}
+	fn render_to_vello(&self, _scene: &mut Scene, _transform: DAffine2, _render_condext: &mut RenderContext) {}
 
 	fn contains_artboard(&self) -> bool {
 		false
@@ -323,7 +328,7 @@ impl GraphicElementRendered for GraphicGroup {
 	}
 
 	#[cfg(feature = "vello")]
-	fn render_to_vello(&self, scene: &mut Scene, transform: DAffine2) {
+	fn render_to_vello(&self, scene: &mut Scene, transform: DAffine2, context: &mut RenderContext) {
 		let child_transform = transform * self.transform;
 
 		let Some(bounds) = self.bounding_box(transform) else { return };
@@ -340,7 +345,7 @@ impl GraphicElementRendered for GraphicGroup {
 			);
 		}
 		for element in self.iter() {
-			element.render_to_vello(scene, child_transform);
+			element.render_to_vello(scene, child_transform, context);
 		}
 		if layer {
 			scene.pop_layer();
@@ -406,7 +411,7 @@ impl GraphicElementRendered for VectorData {
 	}
 
 	#[cfg(feature = "vello")]
-	fn render_to_vello(&self, scene: &mut Scene, transform: DAffine2) {
+	fn render_to_vello(&self, scene: &mut Scene, transform: DAffine2, _: &mut RenderContext) {
 		use crate::vector::style::GradientType;
 		use vello::peniko;
 
@@ -593,7 +598,7 @@ impl GraphicElementRendered for Artboard {
 	}
 
 	#[cfg(feature = "vello")]
-	fn render_to_vello(&self, scene: &mut Scene, transform: DAffine2) {
+	fn render_to_vello(&self, scene: &mut Scene, transform: DAffine2, context: &mut RenderContext) {
 		use vello::peniko;
 
 		// Render background
@@ -608,7 +613,7 @@ impl GraphicElementRendered for Artboard {
 		if self.clip {
 			scene.push_layer(blend_mode, 1., kurbo::Affine::new(transform.to_cols_array()), &rect);
 		}
-		self.graphic_group.render_to_vello(scene, transform);
+		self.graphic_group.render_to_vello(scene, transform, context);
 		if self.clip {
 			scene.pop_layer();
 		}
@@ -643,9 +648,9 @@ impl GraphicElementRendered for crate::ArtboardGroup {
 	}
 
 	#[cfg(feature = "vello")]
-	fn render_to_vello(&self, scene: &mut Scene, transform: DAffine2) {
+	fn render_to_vello(&self, scene: &mut Scene, transform: DAffine2, context: &mut RenderContext) {
 		for artboard in &self.artboards {
-			artboard.render_to_vello(scene, transform)
+			artboard.render_to_vello(scene, transform, context)
 		}
 	}
 
@@ -654,42 +659,6 @@ impl GraphicElementRendered for crate::ArtboardGroup {
 	}
 }
 
-impl GraphicElementRendered for SurfaceFrame {
-	fn render_svg(&self, render: &mut SvgRender, _render_params: &RenderParams) {
-		let transform = self.transform;
-
-		let (width, height) = (transform.transform_vector2(DVec2::new(1., 0.)).length(), transform.transform_vector2(DVec2::new(0., 1.)).length());
-
-		let matrix = format_transform_matrix(transform * DAffine2::from_scale((width, height).into()).inverse());
-		let transform = if matrix.is_empty() { String::new() } else { format!(r#" transform="{}""#, matrix) };
-
-		let canvas = format!(
-			r#"<foreignObject width="{}" height="{}"{transform}><div data-canvas-placeholder="canvas{}"></div></foreignObject>"#,
-			width.abs(),
-			height.abs(),
-			self.surface_id
-		);
-		render.svg.push(canvas.into())
-	}
-
-	#[cfg(feature = "vello")]
-	fn render_to_vello(&self, _scene: &mut Scene, _transform: DAffine2) {
-		todo!()
-	}
-
-	fn bounding_box(&self, transform: DAffine2) -> Option<[DVec2; 2]> {
-		let bbox = Bbox::from_transform(transform);
-		let aabb = bbox.to_axis_aligned_bbox();
-		Some([aabb.start, aabb.end])
-	}
-
-	fn add_click_targets(&self, _click_targets: &mut Vec<ClickTarget>) {}
-
-	fn contains_artboard(&self) -> bool {
-		false
-	}
-}
-
 impl GraphicElementRendered for ImageFrame<Color> {
 	fn render_svg(&self, render: &mut SvgRender, render_params: &RenderParams) {
 		let transform = self.transform * render.transform;
@@ -737,7 +706,7 @@ impl GraphicElementRendered for ImageFrame<Color> {
 	}
 
 	#[cfg(feature = "vello")]
-	fn render_to_vello(&self, scene: &mut Scene, transform: DAffine2) {
+	fn render_to_vello(&self, scene: &mut Scene, transform: DAffine2, _: &mut RenderContext) {
 		use vello::peniko;
 
 		let image = &self.image;
@@ -756,51 +725,135 @@ impl GraphicElementRendered for ImageFrame<Color> {
 		scene.draw_image(&image, vello::kurbo::Affine::new(transform.to_cols_array()));
 	}
 }
+impl GraphicElementRendered for Raster {
+	fn render_svg(&self, render: &mut SvgRender, render_params: &RenderParams) {
+		let transform = self.transform() * render.transform;
+
+		match render_params.image_render_mode {
+			ImageRenderMode::Base64 => {
+				let image = match self {
+					Raster::ImageFrame(ref image) => image,
+					Raster::Texture(_) => return,
+				};
+				let (image, blending) = (&image.image, image.alpha_blending);
+				if image.data.is_empty() {
+					return;
+				}
+
+				let base64_string = image.base64_string.clone().unwrap_or_else(|| {
+					let output = image.to_png();
+					let preamble = "data:image/png;base64,";
+					let mut base64_string = String::with_capacity(preamble.len() + output.len() * 4);
+					base64_string.push_str(preamble);
+					base64::engine::general_purpose::STANDARD.encode_string(output, &mut base64_string);
+					base64_string
+				});
+				render.leaf_tag("image", |attributes| {
+					attributes.push("width", 1.to_string());
+					attributes.push("height", 1.to_string());
+					attributes.push("preserveAspectRatio", "none");
+					attributes.push("href", base64_string);
+					let matrix = format_transform_matrix(transform);
+					if !matrix.is_empty() {
+						attributes.push("transform", matrix);
+					}
+					if blending.blend_mode != BlendMode::default() {
+						attributes.push("style", blending.blend_mode.render());
+					}
+				});
+			}
+		}
+	}
+
+	fn bounding_box(&self, transform: DAffine2) -> Option<[DVec2; 2]> {
+		let transform = self.transform() * transform;
+		(transform.matrix2 != glam::DMat2::ZERO).then(|| (transform * Quad::from_box([DVec2::ZERO, DVec2::ONE])).bounding_box())
+	}
+
+	fn add_click_targets(&self, click_targets: &mut Vec<ClickTarget>) {
+		let subpath = Subpath::new_rect(DVec2::ZERO, DVec2::ONE);
+		click_targets.push(ClickTarget::new(subpath, 0.));
+	}
+
+	#[cfg(feature = "vello")]
+	fn render_to_vello(&self, scene: &mut Scene, transform: DAffine2, context: &mut RenderContext) {
+		use vello::peniko;
+
+		match self {
+			Raster::ImageFrame(image_frame) => {
+				let image = &image_frame.image;
+				if image.data.is_empty() {
+					return;
+				}
+				let image = vello::peniko::Image {
+					data: image.to_flat_u8().0.into(),
+					width: image.width,
+					height: image.height,
+					format: peniko::Format::Rgba8,
+					extend: peniko::Extend::Repeat,
+				};
+				let transform = transform * self.transform() * DAffine2::from_scale(1. / DVec2::new(image.width as f64, image.height as f64));
+
+				scene.draw_image(&image, vello::kurbo::Affine::new(transform.to_cols_array()));
+			}
+			Raster::Texture(texture) => {
+				let image = vello::peniko::Image {
+					data: vec![].into(),
+					width: texture.texture.width(),
+					height: texture.texture.height(),
+					format: peniko::Format::Rgba8,
+					extend: peniko::Extend::Repeat,
+				};
+				let id = image.data.id();
+				context.ressource_overrides.insert(id, texture.texture.clone());
+
+				let transform = transform * self.transform() * DAffine2::from_scale(1. / DVec2::new(image.width as f64, image.height as f64));
+
+				scene.draw_image(&image, vello::kurbo::Affine::new(transform.to_cols_array()));
+			}
+		};
+	}
+}
 
 impl GraphicElementRendered for GraphicElement {
 	fn render_svg(&self, render: &mut SvgRender, render_params: &RenderParams) {
 		match self {
 			GraphicElement::VectorData(vector_data) => vector_data.render_svg(render, render_params),
-			GraphicElement::ImageFrame(image_frame) => image_frame.render_svg(render, render_params),
+			GraphicElement::Raster(raster) => raster.render_svg(render, render_params),
 			GraphicElement::GraphicGroup(graphic_group) => graphic_group.render_svg(render, render_params),
-			GraphicElement::Surface(surface) => surface.render_svg(render, render_params),
 		}
 	}
 
 	fn bounding_box(&self, transform: DAffine2) -> Option<[DVec2; 2]> {
 		match self {
 			GraphicElement::VectorData(vector_data) => GraphicElementRendered::bounding_box(&**vector_data, transform),
-			GraphicElement::ImageFrame(image_frame) => image_frame.bounding_box(transform),
+			GraphicElement::Raster(raster) => raster.bounding_box(transform),
 			GraphicElement::GraphicGroup(graphic_group) => graphic_group.bounding_box(transform),
-			GraphicElement::Surface(surface) => surface.bounding_box(transform),
 		}
 	}
 
 	fn add_click_targets(&self, click_targets: &mut Vec<ClickTarget>) {
 		match self {
 			GraphicElement::VectorData(vector_data) => vector_data.add_click_targets(click_targets),
-			GraphicElement::ImageFrame(image_frame) => image_frame.add_click_targets(click_targets),
+			GraphicElement::Raster(raster) => raster.add_click_targets(click_targets),
 			GraphicElement::GraphicGroup(graphic_group) => graphic_group.add_click_targets(click_targets),
-			GraphicElement::Surface(surface) => surface.add_click_targets(click_targets),
 		}
 	}
 
 	#[cfg(feature = "vello")]
-	fn render_to_vello(&self, scene: &mut Scene, transform: DAffine2) {
+	fn render_to_vello(&self, scene: &mut Scene, transform: DAffine2, context: &mut RenderContext) {
 		match self {
-			GraphicElement::VectorData(vector_data) => vector_data.render_to_vello(scene, transform),
-			GraphicElement::ImageFrame(image_frame) => image_frame.render_to_vello(scene, transform),
-			GraphicElement::GraphicGroup(graphic_group) => graphic_group.render_to_vello(scene, transform),
-			GraphicElement::Surface(surface) => surface.render_to_vello(scene, transform),
+			GraphicElement::VectorData(vector_data) => vector_data.render_to_vello(scene, transform, context),
+			GraphicElement::GraphicGroup(graphic_group) => graphic_group.render_to_vello(scene, transform, context),
+			GraphicElement::Raster(raster) => raster.render_to_vello(scene, transform, context),
 		}
 	}
 
 	fn contains_artboard(&self) -> bool {
 		match self {
 			GraphicElement::VectorData(vector_data) => vector_data.contains_artboard(),
-			GraphicElement::ImageFrame(image_frame) => image_frame.contains_artboard(),
 			GraphicElement::GraphicGroup(graphic_group) => graphic_group.contains_artboard(),
-			GraphicElement::Surface(surface) => surface.contains_artboard(),
+			GraphicElement::Raster(raster) => raster.contains_artboard(),
 		}
 	}
 }

node-graph/gcore/src/transform.rs

@@ -65,17 +65,15 @@ impl Transform for GraphicElement {
 	fn transform(&self) -> DAffine2 {
 		match self {
 			GraphicElement::VectorData(vector_shape) => vector_shape.transform(),
-			GraphicElement::ImageFrame(image_frame) => image_frame.transform(),
 			GraphicElement::GraphicGroup(graphic_group) => graphic_group.transform(),
-			GraphicElement::Surface(surface) => surface.transform(),
+			GraphicElement::Raster(raster) => raster.transform(),
 		}
 	}
 	fn local_pivot(&self, pivot: DVec2) -> DVec2 {
 		match self {
 			GraphicElement::VectorData(vector_shape) => vector_shape.local_pivot(pivot),
-			GraphicElement::ImageFrame(image_frame) => image_frame.local_pivot(pivot),
 			GraphicElement::GraphicGroup(graphic_group) => graphic_group.local_pivot(pivot),
-			GraphicElement::Surface(surface) => surface.local_pivot(pivot),
+			GraphicElement::Raster(raster) => raster.local_pivot(pivot),
 		}
 	}
 }
@@ -83,9 +81,8 @@ impl TransformMut for GraphicElement {
 	fn transform_mut(&mut self) -> &mut DAffine2 {
 		match self {
 			GraphicElement::VectorData(vector_shape) => vector_shape.transform_mut(),
-			GraphicElement::ImageFrame(image_frame) => image_frame.transform_mut(),
 			GraphicElement::GraphicGroup(graphic_group) => graphic_group.transform_mut(),
-			GraphicElement::Surface(surface) => surface.transform_mut(),
+			GraphicElement::Raster(raster) => raster.transform_mut(),
 		}
 	}
 }