Implement Infrastructure to reuse previous frames for brush drawing

Implement Infrastructuro to reuse the previous evaluation of the
node graph to blend the new stroke with instead of drawing the
entire trace from scratch.
This does not transition to a blending based approach because that still
caused regressions but allows the brush node to work with input data
natively.

Test Plan:
- Use the brush tool in the editor and check for regressions
- Evaluate the performance

Reviewers: Keavon

Pull Request: https://github.com/GraphiteEditor/Graphite/pull/1190

editor/src/dispatcher.rs

@@ -170,6 +170,7 @@ impl Dispatcher {
 								self.message_handlers.portfolio_message_handler.active_document_id().unwrap(),
 								&self.message_handlers.input_preprocessor_message_handler,
 								&self.message_handlers.portfolio_message_handler.persistent_data,
+								&self.message_handlers.portfolio_message_handler.executor,
 							),
 						);
 					} else {

editor/src/messages/portfolio/document/node_graph/node_graph_message_handler/document_node_types.rs

@@ -106,6 +106,18 @@ fn static_nodes() -> Vec<DocumentNodeType> {
 			outputs: vec![DocumentOutputType::new("Out", FrontendGraphDataType::General)],
 			properties: |_document_node, _node_id, _context| node_properties::string_properties("The identity node simply returns the input"),
 		},
+		DocumentNodeType {
+			name: "Monitor",
+			category: "General",
+			identifier: NodeImplementation::proto("graphene_core::ops::IdNode"),
+			inputs: vec![DocumentInputType {
+				name: "In",
+				data_type: FrontendGraphDataType::General,
+				default: NodeInput::value(TaggedValue::None, true),
+			}],
+			outputs: vec![DocumentOutputType::new("Out", FrontendGraphDataType::General)],
+			properties: |_document_node, _node_id, _context| node_properties::string_properties("The Monitor node stores the value of its last evaluation"),
+		},
 		DocumentNodeType {
 			name: "Downres",
 			category: "Ignore",
@@ -457,6 +469,7 @@ fn static_nodes() -> Vec<DocumentNodeType> {
 			identifier: NodeImplementation::proto("graphene_std::brush::BrushNode"),
 			inputs: vec![
 				DocumentInputType::value("None", TaggedValue::None, false),
+				DocumentInputType::value("Background", TaggedValue::ImageFrame(ImageFrame::empty()), true),
 				DocumentInputType::value("Trace", TaggedValue::VecDVec2((0..2).map(|x| DVec2::new(x as f64 * 10., 0.)).collect()), true),
 				DocumentInputType::value("Diameter", TaggedValue::F64(40.), false),
 				DocumentInputType::value("Hardness", TaggedValue::F64(50.), false),

editor/src/messages/portfolio/document/node_graph/node_graph_message_handler/node_properties.rs

@@ -588,11 +588,11 @@ pub fn blur_image_properties(document_node: &DocumentNode, node_id: NodeId, _con
 }
 
 pub fn brush_node_properties(document_node: &DocumentNode, node_id: NodeId, _context: &mut NodePropertiesContext) -> Vec<LayoutGroup> {
-	let color = color_widget(document_node, node_id, 5, "Color", ColorInput::default(), true);
+	let color = color_widget(document_node, node_id, 6, "Color", ColorInput::default(), true);
 
-	let size = number_widget(document_node, node_id, 2, "Diameter", NumberInput::default().min(1.).max(100.).unit(" px"), true);
-	let hardness = number_widget(document_node, node_id, 3, "Hardness", NumberInput::default().min(0.).max(100.).unit("%"), true);
-	let flow = number_widget(document_node, node_id, 4, "Flow", NumberInput::default().min(1.).max(100.).unit("%"), true);
+	let size = number_widget(document_node, node_id, 3, "Diameter", NumberInput::default().min(1.).max(100.).unit(" px"), true);
+	let hardness = number_widget(document_node, node_id, 4, "Hardness", NumberInput::default().min(0.).max(100.).unit("%"), true);
+	let flow = number_widget(document_node, node_id, 5, "Flow", NumberInput::default().min(1.).max(100.).unit("%"), true);
 
 	vec![color, LayoutGroup::Row { widgets: size }, LayoutGroup::Row { widgets: hardness }, LayoutGroup::Row { widgets: flow }]
 }

editor/src/messages/portfolio/portfolio_message_handler.rs

@@ -1,3 +1,5 @@
+use std::sync::Arc;
+
 use super::utility_types::PersistentData;
 use crate::application::generate_uuid;
 use crate::consts::{DEFAULT_DOCUMENT_NAME, GRAPHITE_DOCUMENT_VERSION};
@@ -25,7 +27,7 @@ pub struct PortfolioMessageHandler {
 	menu_bar_message_handler: MenuBarMessageHandler,
 	documents: HashMap<u64, DocumentMessageHandler>,
 	document_ids: Vec<u64>,
-	executor: NodeGraphExecutor,
+	pub executor: NodeGraphExecutor,
 	active_document_id: Option<u64>,
 	copy_buffer: [Vec<CopyBufferEntry>; INTERNAL_CLIPBOARD_COUNT as usize],
 	pub persistent_data: PersistentData,
@@ -564,7 +566,7 @@ impl MessageHandler<PortfolioMessage, (&InputPreprocessorMessageHandler, &Prefer
 }
 
 impl PortfolioMessageHandler {
-	pub fn introspect_node(&self, node_path: &[NodeId]) -> Option<String> {
+	pub fn introspect_node(&self, node_path: &[NodeId]) -> Option<Arc<dyn std::any::Any>> {
 		self.executor.introspect_node(node_path)
 	}
 

editor/src/messages/tool/tool_message_handler.rs

@@ -7,6 +7,7 @@ use crate::messages::layout::utility_types::misc::LayoutTarget;
 use crate::messages::portfolio::utility_types::PersistentData;
 use crate::messages::prelude::*;
 use crate::messages::tool::utility_types::ToolType;
+use crate::node_graph_executor::NodeGraphExecutor;
 
 use document_legacy::layers::style::RenderData;
 use graphene_core::raster::color::Color;
@@ -19,13 +20,13 @@ pub struct ToolMessageHandler {
 	pub shape_editor: ShapeState,
 }
 
-impl MessageHandler<ToolMessage, (&DocumentMessageHandler, u64, &InputPreprocessorMessageHandler, &PersistentData)> for ToolMessageHandler {
+impl MessageHandler<ToolMessage, (&DocumentMessageHandler, u64, &InputPreprocessorMessageHandler, &PersistentData, &NodeGraphExecutor)> for ToolMessageHandler {
 	#[remain::check]
 	fn process_message(
 		&mut self,
 		message: ToolMessage,
 		responses: &mut VecDeque<Message>,
-		(document, document_id, input, persistent_data): (&DocumentMessageHandler, u64, &InputPreprocessorMessageHandler, &PersistentData),
+		(document, document_id, input, persistent_data, node_graph): (&DocumentMessageHandler, u64, &InputPreprocessorMessageHandler, &PersistentData, &NodeGraphExecutor),
 	) {
 		let render_data = RenderData::new(&persistent_data.font_cache, document.view_mode, None);
 
@@ -94,6 +95,7 @@ impl MessageHandler<ToolMessage, (&DocumentMessageHandler, u64, &InputPreprocess
 							render_data: &render_data,
 							shape_overlay: &mut self.shape_overlay,
 							shape_editor: &mut self.shape_editor,
+							node_graph,
 						};
 						if let Some(tool_abort_message) = tool.event_to_message_map().tool_abort {
 							tool.process_message(tool_abort_message, responses, &mut data);
@@ -173,6 +175,7 @@ impl MessageHandler<ToolMessage, (&DocumentMessageHandler, u64, &InputPreprocess
 					render_data: &render_data,
 					shape_overlay: &mut self.shape_overlay,
 					shape_editor: &mut self.shape_editor,
+					node_graph,
 				};
 
 				// Set initial hints and cursor
@@ -243,6 +246,7 @@ impl MessageHandler<ToolMessage, (&DocumentMessageHandler, u64, &InputPreprocess
 							render_data: &render_data,
 							shape_overlay: &mut self.shape_overlay,
 							shape_editor: &mut self.shape_editor,
+							node_graph,
 						};
 						if matches!(tool_message, ToolMessage::UpdateHints) {
 							if self.transform_layer_handler.is_transforming() {

editor/src/messages/tool/tool_messages/brush_tool.rs

@@ -8,14 +8,18 @@ use crate::messages::prelude::*;
 use crate::messages::tool::common_functionality::graph_modification_utils;
 use crate::messages::tool::utility_types::{DocumentToolData, EventToMessageMap, Fsm, ToolActionHandlerData, ToolMetadata, ToolTransition, ToolType};
 use crate::messages::tool::utility_types::{HintData, HintGroup, HintInfo};
+use crate::node_graph_executor::NodeGraphExecutor;
 
 use document_legacy::LayerId;
+use dyn_any::downcast_ref;
 use graph_craft::document::value::TaggedValue;
 use graph_craft::document::{DocumentNode, DocumentNodeImplementation, NodeInput, NodeNetwork};
-use graph_craft::{concrete, Type, TypeDescriptor};
-use graphene_core::Cow;
+
+use graphene_core::Color;
 
 use glam::DVec2;
+use graphene_core::raster::ImageFrame;
+
 use serde::{Deserialize, Serialize};
 
 #[derive(Default)]
@@ -172,7 +176,7 @@ impl ToolTransition for BrushTool {
 
 #[derive(Clone, Debug, Default)]
 struct BrushToolData {
-	points: Vec<DVec2>,
+	points: Vec<Vec<DVec2>>,
 	diameter: f64,
 	hardness: f64,
 	flow: f64,
@@ -181,12 +185,28 @@ struct BrushToolData {
 
 impl BrushToolData {
 	fn update_points(&self, responses: &mut VecDeque<Message>) {
+		if let Some(layer_path) = self.path.clone() {
+			let points = self.points.iter().flatten().cloned().collect();
+			responses.add(NodeGraphMessage::SetQualifiedInputValue {
+				layer_path,
+				node_path: vec![0],
+				input_index: 2,
+				value: TaggedValue::VecDVec2(points),
+			});
+		}
+	}
+	fn update_image(&self, node_graph: &NodeGraphExecutor, responses: &mut VecDeque<Message>) {
+		let Some(image) = node_graph.introspect_node(&[1]) else { return; };
+		let image: &ImageFrame<Color> = image.downcast_ref().unwrap();
+		self.set_image(image.clone(), responses)
+	}
+	fn set_image(&self, image_frame: ImageFrame<Color>, responses: &mut VecDeque<Message>) {
 		if let Some(layer_path) = self.path.clone() {
 			responses.add(NodeGraphMessage::SetQualifiedInputValue {
 				layer_path,
 				node_path: vec![0],
 				input_index: 1,
-				value: TaggedValue::VecDVec2(self.points.clone()),
+				value: TaggedValue::ImageFrame(image_frame),
 			});
 		}
 	}
@@ -201,7 +221,11 @@ impl Fsm for BrushToolFsmState {
 		event: ToolMessage,
 		tool_data: &mut Self::ToolData,
 		ToolActionHandlerData {
-			document, global_tool_data, input, ..
+			document,
+			global_tool_data,
+			input,
+			node_graph,
+			..
 		}: &mut ToolActionHandlerData,
 		tool_options: &Self::ToolOptions,
 		responses: &mut VecDeque<Message>,
@@ -217,9 +241,12 @@ impl Fsm for BrushToolFsmState {
 					responses.add(DocumentMessage::StartTransaction);
 					let existing_points = load_existing_points(document);
 					let new_layer = existing_points.is_none();
-					if let Some((layer_path, points)) = existing_points {
+					if let Some((layer_path, points, image)) = existing_points {
 						tool_data.path = Some(layer_path);
-						tool_data.points = points;
+						//tool_data.set_image(image, responses);
+						if tool_data.points.is_empty() {
+							tool_data.points.push(points);
+						}
 					} else {
 						responses.add(DocumentMessage::DeselectAllLayers);
 						tool_data.path = Some(document.get_path_for_new_layer());
@@ -227,7 +254,7 @@ impl Fsm for BrushToolFsmState {
 
 					let pos = transform.inverse().transform_point2(input.mouse.position);
 
-					tool_data.points.push(pos);
+					tool_data.points.push(vec![pos]);
 
 					tool_data.diameter = tool_options.diameter;
 					tool_data.hardness = tool_options.hardness;
@@ -236,6 +263,7 @@ impl Fsm for BrushToolFsmState {
 					if new_layer {
 						add_brush_render(tool_data, global_tool_data, responses);
 					} else {
+						//tool_data.update_image(node_graph, responses);
 						tool_data.update_points(responses);
 					}
 
@@ -244,15 +272,21 @@ impl Fsm for BrushToolFsmState {
 				(Drawing, PointerMove) => {
 					let pos = transform.inverse().transform_point2(input.mouse.position);
 
-					if tool_data.points.last() != Some(&pos) {
+					if tool_data.points.last().and_then(|x| x.last()) != Some(&pos) {
 						// Linear interpolation for when the mouse has moved a lot between frames
-						if let Some(&last_point) = tool_data.points.last() {
+						if let Some(&last_point) = tool_data.points.last().and_then(|x| x.last()) {
 							let distance = (last_point - pos).length();
 							let extra_points = (distance / (tool_data.diameter / 2.)).floor() as usize;
-							tool_data.points.extend((0..extra_points).map(|i| last_point.lerp(pos, (i as f64 + 1.) / (extra_points as f64 + 1.))));
+							tool_data
+								.points
+								.last_mut()
+								.unwrap()
+								.extend((0..extra_points).map(|i| last_point.lerp(pos, (i as f64 + 1.) / (extra_points as f64 + 1.))));
 						}
 
-						tool_data.points.push(pos);
+						if let Some(x) = tool_data.points.last_mut() {
+							x.push(pos)
+						}
 					}
 
 					tool_data.update_points(responses);
@@ -266,8 +300,8 @@ impl Fsm for BrushToolFsmState {
 						responses.add(DocumentMessage::AbortTransaction);
 					}
 
-					tool_data.path = None;
 					tool_data.points.clear();
+					tool_data.path = None;
 
 					Ready
 				}
@@ -294,11 +328,13 @@ impl Fsm for BrushToolFsmState {
 
 fn add_brush_render(data: &BrushToolData, tool_data: &DocumentToolData, responses: &mut VecDeque<Message>) {
 	let layer_path = data.path.clone().unwrap();
+
 	let brush_node = DocumentNode {
 		name: "Brush".to_string(),
 		inputs: vec![
-			NodeInput::ShortCircut(concrete!(())),
-			NodeInput::value(TaggedValue::VecDVec2(data.points.clone()), false),
+			NodeInput::value(TaggedValue::None, false),
+			NodeInput::value(TaggedValue::ImageFrame(ImageFrame::empty()), true),
+			NodeInput::value(TaggedValue::VecDVec2(data.points.last().cloned().unwrap_or_default()), false),
 			// Diameter
 			NodeInput::value(TaggedValue::F64(data.diameter), false),
 			// Hardness
@@ -312,12 +348,18 @@ fn add_brush_render(data: &BrushToolData, tool_data: &DocumentToolData, response
 		metadata: graph_craft::document::DocumentNodeMetadata { position: (8, 4).into() },
 		..Default::default()
 	};
+	let monitor_node = DocumentNode {
+		name: "Monitor".to_string(),
+		implementation: DocumentNodeImplementation::Unresolved("graphene_std::memo::MonitorNode<_>".into()),
+		..Default::default()
+	};
 	let mut network = NodeNetwork::value_network(brush_node);
+	//network.push_node(monitor_node, true);
 	network.push_output_node();
 	graph_modification_utils::new_custom_layer(network, layer_path, responses);
 }
 
-fn load_existing_points(document: &DocumentMessageHandler) -> Option<(Vec<LayerId>, Vec<DVec2>)> {
+fn load_existing_points(document: &DocumentMessageHandler) -> Option<(Vec<LayerId>, Vec<DVec2>, ImageFrame<Color>)> {
 	if document.selected_layers().count() != 1 {
 		return None;
 	}
@@ -327,11 +369,16 @@ fn load_existing_points(document: &DocumentMessageHandler) -> Option<(Vec<LayerI
 	if brush_node.implementation != DocumentNodeImplementation::Unresolved("graphene_std::brush::BrushNode".into()) {
 		return None;
 	}
-	let points_input = brush_node.inputs.get(1)?;
+	let image_input = brush_node.inputs.get(1)?;
+	let NodeInput::Value {
+		tagged_value: TaggedValue::ImageFrame(image_frame),
+		..
+	} = image_input else { return None };
+	let points_input = brush_node.inputs.get(2)?;
 	let NodeInput::Value {
 		tagged_value: TaggedValue::VecDVec2(points),
 		..
 	} = points_input else { return None };
 
-	Some((layer_path, points.clone()))
+	Some((layer_path, points.clone(), image_frame.clone()))
 }

editor/src/messages/tool/utility_types.rs

@@ -12,6 +12,7 @@ use crate::messages::layout::utility_types::widgets::button_widgets::IconButton;
 use crate::messages::layout::utility_types::widgets::input_widgets::SwatchPairInput;
 use crate::messages::layout::utility_types::widgets::label_widgets::{Separator, SeparatorDirection, SeparatorType};
 use crate::messages::prelude::*;
+use crate::node_graph_executor::NodeGraphExecutor;
 
 use document_legacy::layers::style::RenderData;
 use graphene_core::raster::color::Color;
@@ -27,6 +28,7 @@ pub struct ToolActionHandlerData<'a> {
 	pub render_data: &'a RenderData<'a>,
 	pub shape_overlay: &'a mut OverlayRenderer,
 	pub shape_editor: &'a mut ShapeState,
+	pub node_graph: &'a NodeGraphExecutor,
 }
 impl<'a> ToolActionHandlerData<'a> {
 	pub fn new(
@@ -37,6 +39,7 @@ impl<'a> ToolActionHandlerData<'a> {
 		render_data: &'a RenderData<'a>,
 		shape_overlay: &'a mut OverlayRenderer,
 		shape_editor: &'a mut ShapeState,
+		node_graph: &'a NodeGraphExecutor,
 	) -> Self {
 		Self {
 			document,
@@ -46,6 +49,7 @@ impl<'a> ToolActionHandlerData<'a> {
 			render_data,
 			shape_overlay,
 			shape_editor,
+			node_graph,
 		}
 	}
 }

editor/src/node_graph_executor.rs

@@ -17,6 +17,7 @@ use interpreted_executor::executor::DynamicExecutor;
 
 use glam::{DAffine2, DVec2};
 use std::borrow::Cow;
+use std::sync::Arc;
 
 #[derive(Debug, Clone, Default)]
 pub struct NodeGraphExecutor {
@@ -37,6 +38,7 @@ impl NodeGraphExecutor {
 		assert_eq!(scoped_network.outputs.len(), 1, "Graph with multiple outputs not yet handled");
 		let c = Compiler {};
 		let proto_network = c.compile_single(scoped_network, true)?;
+
 		assert_ne!(proto_network.nodes.len(), 0, "No protonodes exist?");
 		if let Err(e) = self.executor.update(proto_network) {
 			error!("Failed to update executor:\n{}", e);
@@ -53,7 +55,7 @@ impl NodeGraphExecutor {
 		}
 	}
 
-	pub fn introspect_node(&self, path: &[NodeId]) -> Option<String> {
+	pub fn introspect_node(&self, path: &[NodeId]) -> Option<Arc<dyn std::any::Any>> {
 		self.executor.introspect(path).flatten()
 	}