Integrate the node graph as a Node Graph Frame layer type (#812)

* Add node graph frame tool

* Add a brighten

* Use the node graph

* Fix topological_sort

* Update UI

* Add icons for the tool and layer type

* Avoid serde & use bitmaps to improve performance

* Allow serialising a node graph

* Fix missing ..Default::default()

* Fix incorrect comments

* Cache node graph output image

* Suppress no-cycle import warning

Co-authored-by: Keavon Chambers <keavon@keavon.com>

graphene/src/document.rs

@@ -4,6 +4,7 @@ use crate::layers::folder_layer::FolderLayer;
 use crate::layers::image_layer::ImageLayer;
 use crate::layers::imaginate_layer::{ImaginateImageData, ImaginateLayer, ImaginateStatus};
 use crate::layers::layer_info::{Layer, LayerData, LayerDataType, LayerDataTypeDiscriminant};
+use crate::layers::nodegraph_layer::NodeGraphFrameLayer;
 use crate::layers::shape_layer::ShapeLayer;
 use crate::layers::style::RenderData;
 use crate::layers::text_layer::{Font, FontCache, TextLayer};
@@ -595,6 +596,22 @@ impl Document {
 
 				Some([vec![DocumentChanged, CreatedLayer { path: path.clone() }], update_thumbnails_upstream(&path)].concat())
 			}
+			Operation::AddNodeGraphFrame { path, insert_index, transform } => {
+				let layer = Layer::new(LayerDataType::NodeGraphFrame(NodeGraphFrameLayer::default()), transform);
+
+				self.set_layer(&path, layer, insert_index)?;
+
+				Some([vec![DocumentChanged, CreatedLayer { path: path.clone() }], update_thumbnails_upstream(&path)].concat())
+			}
+			Operation::SetNodeGraphFrameImageData { layer_path, image_data } => {
+				let layer = self.layer_mut(&layer_path).expect("Setting NodeGraphFrame image data for invalid layer");
+				if let LayerDataType::NodeGraphFrame(node_graph_frame) = &mut layer.data {
+					node_graph_frame.image_data = Some(crate::layers::nodegraph_layer::ImageData { image_data });
+				} else {
+					panic!("Incorrectly trying to set image data for a layer that is not an NodeGraphFrame layer type");
+				}
+				Some(vec![LayerChanged { path: layer_path.clone() }])
+			}
 			Operation::SetTextEditability { path, editable } => {
 				self.layer_mut(&path)?.as_text_mut()?.editable = editable;
 				self.mark_as_dirty(&path)?;
@@ -797,11 +814,15 @@ impl Document {
 						image.blob_url = Some(blob_url);
 						image.dimensions = resolution.into();
 					}
+					LayerDataType::NodeGraphFrame(node_graph_frame) => {
+						node_graph_frame.blob_url = Some(blob_url);
+						node_graph_frame.dimensions = resolution.into();
+					}
 					LayerDataType::Imaginate(imaginate) => {
 						imaginate.blob_url = Some(blob_url);
 						imaginate.dimensions = resolution.into();
 					}
-					_ => panic!("Incorrectly trying to set the image blob URL for a layer that is not an Image or Imaginate layer type"),
+					_ => panic!("Incorrectly trying to set the image blob URL for a layer that is not an Image, NodeGraphFrame or Imaginate layer type"),
 				}
 
 				self.mark_as_dirty(&layer_path)?;
@@ -833,15 +854,20 @@ impl Document {
 				}
 				Some(vec![LayerChanged { path: path.clone() }])
 			}
-			Operation::ImaginateClear { path } => {
+			Operation::ClearBlobURL { path } => {
 				let layer = self.layer_mut(&path).expect("Clearing Imaginate image for invalid layer");
-				if let LayerDataType::Imaginate(imaginate) = &mut layer.data {
-					imaginate.image_data = None;
-					imaginate.blob_url = None;
-					imaginate.status = ImaginateStatus::Idle;
-					imaginate.percent_complete = 0.;
-				} else {
-					panic!("Incorrectly trying to clear the blob URL for a layer that is not an Imaginate layer type");
+				match &mut layer.data {
+					LayerDataType::Imaginate(imaginate) => {
+						imaginate.image_data = None;
+						imaginate.blob_url = None;
+						imaginate.status = ImaginateStatus::Idle;
+						imaginate.percent_complete = 0.;
+					}
+					LayerDataType::NodeGraphFrame(node_graph) => {
+						node_graph.image_data = None;
+						node_graph.blob_url = None;
+					}
+					e => panic!("Incorrectly trying to clear the blob URL for layer of type {}", LayerDataTypeDiscriminant::from(&*e)),
 				}
 				self.mark_as_dirty(&path)?;
 				Some([vec![DocumentChanged, LayerChanged { path: path.clone() }], update_thumbnails_upstream(&path)].concat())

graphene/src/layers/layer_info.rs

@@ -2,6 +2,7 @@ use super::blend_mode::BlendMode;
 use super::folder_layer::FolderLayer;
 use super::image_layer::ImageLayer;
 use super::imaginate_layer::ImaginateLayer;
+use super::nodegraph_layer::NodeGraphFrameLayer;
 use super::shape_layer::ShapeLayer;
 use super::style::{PathStyle, RenderData};
 use super::text_layer::TextLayer;
@@ -27,8 +28,10 @@ pub enum LayerDataType {
 	Text(TextLayer),
 	/// A layer that wraps an [ImageLayer] struct.
 	Image(ImageLayer),
-	/// A layer that wraps an [ImageLayer] struct.
+	/// A layer that wraps an [ImaginateLayer] struct.
 	Imaginate(ImaginateLayer),
+	/// A layer that wraps an [NodeGraphFrameLayer] struct.
+	NodeGraphFrame(NodeGraphFrameLayer),
 }
 
 impl LayerDataType {
@@ -39,6 +42,7 @@ impl LayerDataType {
 			LayerDataType::Text(t) => t,
 			LayerDataType::Image(i) => i,
 			LayerDataType::Imaginate(a) => a,
+			LayerDataType::NodeGraphFrame(n) => n,
 		}
 	}
 
@@ -49,6 +53,7 @@ impl LayerDataType {
 			LayerDataType::Text(t) => t,
 			LayerDataType::Image(i) => i,
 			LayerDataType::Imaginate(a) => a,
+			LayerDataType::NodeGraphFrame(n) => n,
 		}
 	}
 }
@@ -60,6 +65,7 @@ pub enum LayerDataTypeDiscriminant {
 	Text,
 	Image,
 	Imaginate,
+	NodeGraphFrame,
 }
 
 impl fmt::Display for LayerDataTypeDiscriminant {
@@ -70,6 +76,7 @@ impl fmt::Display for LayerDataTypeDiscriminant {
 			LayerDataTypeDiscriminant::Text => write!(f, "Text"),
 			LayerDataTypeDiscriminant::Image => write!(f, "Image"),
 			LayerDataTypeDiscriminant::Imaginate => write!(f, "Imaginate"),
+			LayerDataTypeDiscriminant::NodeGraphFrame => write!(f, "NodeGraphFrame"),
 		}
 	}
 }
@@ -84,6 +91,7 @@ impl From<&LayerDataType> for LayerDataTypeDiscriminant {
 			Text(_) => LayerDataTypeDiscriminant::Text,
 			Image(_) => LayerDataTypeDiscriminant::Image,
 			Imaginate(_) => LayerDataTypeDiscriminant::Imaginate,
+			NodeGraphFrame(_) => LayerDataTypeDiscriminant::NodeGraphFrame,
 		}
 	}
 }

graphene/src/layers/mod.rs

@@ -27,6 +27,8 @@ pub mod image_layer;
 pub mod imaginate_layer;
 /// Contains the base [Layer](layer_info::Layer) type, an abstraction over the different types of layers.
 pub mod layer_info;
+/// Contains the [NodegraphLayer](nodegraph_layer::NodegraphLayer) type that contains a node graph.
+pub mod nodegraph_layer;
 /// Contains the [ShapeLayer](shape_layer::ShapeLayer) type, a generic SVG element defined using Bezier paths.
 pub mod shape_layer;
 pub mod style;

graphene/src/layers/nodegraph_layer.rs

@@ -0,0 +1,152 @@
+use super::base64_serde;
+use super::layer_info::LayerData;
+use super::style::{RenderData, ViewMode};
+use crate::intersection::{intersect_quad_bez_path, Quad};
+use crate::layers::text_layer::FontCache;
+use crate::LayerId;
+
+use glam::{DAffine2, DMat2, DVec2};
+use kurbo::{Affine, BezPath, Shape as KurboShape};
+use serde::{Deserialize, Serialize};
+use std::fmt::Write;
+
+#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
+pub struct NodeGraphFrameLayer {
+	// Image stored in layer after generation completes
+	pub mime: String,
+
+	/// The document node network that this layer contains
+	pub network: graph_craft::document::NodeNetwork,
+
+	// TODO: Have the browser dispose of this blob URL when this is dropped (like when the layer is deleted)
+	#[serde(skip)]
+	pub blob_url: Option<String>,
+	#[serde(skip)]
+	pub dimensions: DVec2,
+	pub image_data: Option<ImageData>,
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Deserialize, Serialize)]
+pub struct ImageData {
+	#[serde(serialize_with = "base64_serde::as_base64", deserialize_with = "base64_serde::from_base64")]
+	pub image_data: Vec<u8>,
+}
+
+impl LayerData for NodeGraphFrameLayer {
+	fn render(&mut self, svg: &mut String, _svg_defs: &mut String, transforms: &mut Vec<DAffine2>, render_data: RenderData) {
+		let transform = self.transform(transforms, render_data.view_mode);
+		let inverse = transform.inverse();
+
+		let (width, height) = (transform.transform_vector2(DVec2::new(1., 0.)).length(), transform.transform_vector2(DVec2::new(0., 1.)).length());
+
+		if !inverse.is_finite() {
+			let _ = write!(svg, "<!-- SVG shape has an invalid transform -->");
+			return;
+		}
+
+		let _ = writeln!(svg, r#"<g transform="matrix("#);
+		inverse.to_cols_array().iter().enumerate().for_each(|(i, entry)| {
+			let _ = svg.write_str(&(entry.to_string() + if i == 5 { "" } else { "," }));
+		});
+		let _ = svg.write_str(r#")">"#);
+
+		let matrix = (transform * DAffine2::from_scale((width, height).into()).inverse())
+			.to_cols_array()
+			.iter()
+			.enumerate()
+			.fold(String::new(), |val, (i, entry)| val + &(entry.to_string() + if i == 5 { "" } else { "," }));
+
+		if let Some(blob_url) = &self.blob_url {
+			let _ = write!(
+				svg,
+				r#"<image width="{}" height="{}" preserveAspectRatio="none" href="{}" transform="matrix({})" />"#,
+				width.abs(),
+				height.abs(),
+				blob_url,
+				matrix
+			);
+		}
+		let _ = write!(
+			svg,
+			r#"<rect width="{}" height="{}" fill="none" stroke="var(--color-data-vector)" stroke-width="3" stroke-dasharray="8" transform="matrix({})" />"#,
+			width.abs(),
+			height.abs(),
+			matrix,
+		);
+
+		let _ = svg.write_str(r#"</g>"#);
+	}
+
+	fn bounding_box(&self, transform: glam::DAffine2, _font_cache: &FontCache) -> Option<[DVec2; 2]> {
+		let mut path = self.bounds();
+
+		if transform.matrix2 == DMat2::ZERO {
+			return None;
+		}
+		path.apply_affine(glam_to_kurbo(transform));
+
+		let kurbo::Rect { x0, y0, x1, y1 } = path.bounding_box();
+		Some([(x0, y0).into(), (x1, y1).into()])
+	}
+
+	fn intersects_quad(&self, quad: Quad, path: &mut Vec<LayerId>, intersections: &mut Vec<Vec<LayerId>>, _font_cache: &FontCache) {
+		if intersect_quad_bez_path(quad, &self.bounds(), true) {
+			intersections.push(path.clone());
+		}
+	}
+}
+
+impl NodeGraphFrameLayer {
+	pub fn transform(&self, transforms: &[DAffine2], mode: ViewMode) -> DAffine2 {
+		let start = match mode {
+			ViewMode::Outline => 0,
+			_ => (transforms.len() as i32 - 1).max(0) as usize,
+		};
+		transforms.iter().skip(start).cloned().reduce(|a, b| a * b).unwrap_or(DAffine2::IDENTITY)
+	}
+
+	fn bounds(&self) -> BezPath {
+		kurbo::Rect::from_origin_size(kurbo::Point::ZERO, kurbo::Size::new(1., 1.)).to_path(0.)
+	}
+}
+
+fn glam_to_kurbo(transform: DAffine2) -> Affine {
+	Affine::new(transform.to_cols_array())
+}
+
+impl Default for NodeGraphFrameLayer {
+	fn default() -> Self {
+		use graph_craft::document::*;
+		use graph_craft::proto::NodeIdentifier;
+		Self {
+			mime: String::new(),
+			network: NodeNetwork {
+				inputs: vec![1],
+				output: 1,
+				nodes: [
+					(
+						0,
+						DocumentNode {
+							name: "grayscale".into(),
+							inputs: vec![NodeInput::Network],
+							implementation: DocumentNodeImplementation::Unresolved(NodeIdentifier::new("graphene_core::raster::GrayscaleNode", &[])),
+						},
+					),
+					(
+						1,
+						DocumentNode {
+							name: "map image".into(),
+							inputs: vec![NodeInput::Network, NodeInput::Node(0)],
+							implementation: DocumentNodeImplementation::Unresolved(NodeIdentifier::new("graphene_std::raster::MapImageNode", &[])),
+						},
+					),
+				]
+				.into_iter()
+				.collect(),
+			},
+			blob_url: None,
+			dimensions: DVec2::ZERO,
+			image_data: None,
+		}
+	}
+}

graphene/src/operation.rs

@@ -57,6 +57,15 @@ pub enum Operation {
 		insert_index: isize,
 		transform: [f64; 6],
 	},
+	AddNodeGraphFrame {
+		path: Vec<LayerId>,
+		insert_index: isize,
+		transform: [f64; 6],
+	},
+	SetNodeGraphFrameImageData {
+		layer_path: Vec<LayerId>,
+		image_data: Vec<u8>,
+	},
 	/// Sets a blob URL as the image source for an Image or Imaginate layer type.
 	/// **Be sure to call `FrontendMessage::TriggerRevokeBlobUrl` together with this.**
 	SetLayerBlobUrl {
@@ -64,9 +73,9 @@ pub enum Operation {
 		blob_url: String,
 		resolution: (f64, f64),
 	},
-	/// Clears the image to leave the Imaginate layer un-rendered.
+	/// Clears the image to leave the layer un-rendered.
 	/// **Be sure to call `FrontendMessage::TriggerRevokeBlobUrl` together with this.**
-	ImaginateClear {
+	ClearBlobURL {
 		path: Vec<LayerId>,
 	},
 	ImaginateSetGeneratingStatus {