Desktop: Add Eyedropper tool support with native Vello (#3684)

* mostly done

* fix

* kinda works but tilt and flip broken

* fix footprint

Co-authored-by: James Lindsay <78500760+0HyperCube@users.noreply.github.com>

* Code review

* fix cursor hiding

* Remove console.log

---------

Co-authored-by: James Lindsay <78500760+0HyperCube@users.noreply.github.com>
Co-authored-by: Keavon Chambers <keavon@keavon.com>

editor/src/consts.rs

@@ -124,6 +124,9 @@ pub const LINE_ROTATE_SNAP_ANGLE: f64 = 15.;
 pub const BRUSH_SIZE_CHANGE_KEYBOARD: f64 = 5.;
 pub const DEFAULT_BRUSH_SIZE: f64 = 20.;
 
+// EYEDROPPER TOOL
+pub const EYEDROPPER_PREVIEW_AREA_RESOLUTION: u32 = 11;
+
 // GIZMOS
 pub const POINT_RADIUS_HANDLE_SNAP_THRESHOLD: f64 = 8.;
 pub const POINT_RADIUS_HANDLE_SEGMENT_THRESHOLD: f64 = 7.9;

editor/src/messages/frontend/frontend_message.rs

@@ -1,5 +1,6 @@
 use super::utility_types::{DocumentDetails, MouseCursorIcon, OpenDocument};
 use crate::messages::app_window::app_window_message_handler::AppWindowPlatform;
+use crate::messages::frontend::utility_types::EyedropperPreviewImage;
 use crate::messages::input_mapper::utility_types::misc::ActionShortcut;
 use crate::messages::layout::utility_types::widget_prelude::*;
 use crate::messages::portfolio::document::node_graph::utility_types::{
@@ -253,6 +254,7 @@ pub enum FrontendMessage {
 		multiplier: (f64, f64),
 	},
 	UpdateEyedropperSamplingState {
+		image: Option<EyedropperPreviewImage>,
 		#[serde(rename = "mousePosition")]
 		mouse_position: Option<(f64, f64)>,
 		#[serde(rename = "primaryColor")]

editor/src/messages/frontend/utility_types.rs

@@ -62,3 +62,10 @@ pub enum ExportBounds {
 	Selection,
 	Artboard(LayerNodeIdentifier),
 }
+
+#[derive(Clone, Debug, Default, Eq, PartialEq, serde::Serialize, serde::Deserialize, specta::Type)]
+pub struct EyedropperPreviewImage {
+	pub data: Vec<u8>,
+	pub width: u32,
+	pub height: u32,
+}

editor/src/messages/portfolio/portfolio_message.rs

@@ -148,6 +148,7 @@ pub enum PortfolioMessage {
 		document_id: DocumentId,
 		ignore_hash: bool,
 	},
+	SubmitEyedropperPreviewRender,
 	ToggleResetNodesToDefinitionsOnOpen,
 	ToggleFocusDocument,
 	ToggleDataPanelOpen,

editor/src/messages/portfolio/portfolio_message_handler.rs

@@ -1178,6 +1178,41 @@ impl MessageHandler<PortfolioMessage, PortfolioMessageContext<'_>> for Portfolio
 					Ok(message) => responses.add_front(message),
 				}
 			}
+			#[cfg(not(target_family = "wasm"))]
+			PortfolioMessage::SubmitEyedropperPreviewRender => {
+				use crate::consts::EYEDROPPER_PREVIEW_AREA_RESOLUTION;
+
+				let Some(document_id) = self.active_document_id else { return };
+				let Some(document) = self.documents.get_mut(&document_id) else { return };
+
+				let resolution = glam::UVec2::splat(EYEDROPPER_PREVIEW_AREA_RESOLUTION);
+
+				let preview_offset_in_viewport = ipp.mouse.position - (glam::DVec2::splat(EYEDROPPER_PREVIEW_AREA_RESOLUTION as f64 / 2.));
+				let preview_offset_in_viewport = DAffine2::from_translation(preview_offset_in_viewport);
+
+				let document_to_viewport = document.metadata().document_to_viewport;
+
+				let preview_transform = preview_offset_in_viewport.inverse() * document_to_viewport;
+				let pointer_position = document_to_viewport.inverse().transform_point2(ipp.mouse.position);
+
+				let result = self
+					.executor
+					.submit_eyedropper_preview(document_id, preview_transform, pointer_position, resolution, timing_information);
+
+				match result {
+					Err(description) => {
+						responses.add(DialogMessage::DisplayDialogError {
+							title: "Unable to update node graph".to_string(),
+							description,
+						});
+					}
+					Ok(message) => responses.add_front(message),
+				}
+			}
+			#[cfg(target_family = "wasm")]
+			PortfolioMessage::SubmitEyedropperPreviewRender => {
+				// TODO: Currently for Wasm, this is implemented through SVG rendering but the Eyedropper tool doesn't work at all when Vello is enabled as the renderer
+			}
 			PortfolioMessage::ToggleFocusDocument => {
 				self.focus_document = !self.focus_document;
 				responses.add(MenuBarMessage::SendLayout);

editor/src/messages/tool/tool_messages/eyedropper_tool.rs

@@ -1,4 +1,5 @@
 use super::tool_prelude::*;
+use crate::messages::frontend::utility_types::EyedropperPreviewImage;
 use crate::messages::tool::utility_types::DocumentToolData;
 
 #[derive(Default, ExtractField)]
@@ -19,6 +20,8 @@ pub enum EyedropperToolMessage {
 	PointerMove,
 	SampleSecondaryColorBegin,
 	SampleSecondaryColorEnd,
+
+	PreviewImage { data: Vec<u8>, width: u32, height: u32 },
 }
 
 impl ToolMetadata for EyedropperTool {
@@ -42,6 +45,17 @@ impl LayoutHolder for EyedropperTool {
 #[message_handler_data]
 impl<'a> MessageHandler<ToolMessage, &mut ToolActionMessageContext<'a>> for EyedropperTool {
 	fn process_message(&mut self, message: ToolMessage, responses: &mut VecDeque<Message>, context: &mut ToolActionMessageContext<'a>) {
+		if let ToolMessage::Eyedropper(EyedropperToolMessage::PreviewImage { data, width, height }) = message {
+			let image = EyedropperPreviewImage { data, width, height };
+
+			update_cursor_preview_common(responses, Some(image), context.input, context.global_tool_data, self.data.color_choice.clone());
+
+			if !self.data.preview {
+				disable_cursor_preview(responses, &mut self.data);
+			}
+			return;
+		}
+
 		self.fsm_state.process_event(message, &mut self.data, context, &(), responses, true);
 	}
 
@@ -74,13 +88,16 @@ enum EyedropperToolFsmState {
 }
 
 #[derive(Clone, Debug, Default)]
-struct EyedropperToolData {}
+struct EyedropperToolData {
+	preview: bool,
+	color_choice: Option<String>,
+}
 
 impl Fsm for EyedropperToolFsmState {
 	type ToolData = EyedropperToolData;
 	type ToolOptions = ();
 
-	fn transition(self, event: ToolMessage, _tool_data: &mut Self::ToolData, tool_action_data: &mut ToolActionMessageContext, _tool_options: &(), responses: &mut VecDeque<Message>) -> Self {
+	fn transition(self, event: ToolMessage, tool_data: &mut Self::ToolData, tool_action_data: &mut ToolActionMessageContext, _tool_options: &(), responses: &mut VecDeque<Message>) -> Self {
 		let ToolActionMessageContext {
 			global_tool_data, input, viewport, ..
 		} = tool_action_data;
@@ -89,7 +106,7 @@ impl Fsm for EyedropperToolFsmState {
 		match (self, event) {
 			// Ready -> Sampling
 			(EyedropperToolFsmState::Ready, mouse_down) if matches!(mouse_down, EyedropperToolMessage::SamplePrimaryColorBegin | EyedropperToolMessage::SampleSecondaryColorBegin) => {
-				update_cursor_preview(responses, input, global_tool_data, None);
+				update_cursor_preview(responses, tool_data, input, global_tool_data, None);
 
 				if mouse_down == EyedropperToolMessage::SamplePrimaryColorBegin {
 					EyedropperToolFsmState::SamplingPrimary
@@ -101,9 +118,9 @@ impl Fsm for EyedropperToolFsmState {
 			(EyedropperToolFsmState::SamplingPrimary | EyedropperToolFsmState::SamplingSecondary, EyedropperToolMessage::PointerMove) => {
 				let mouse_position = viewport.logical(input.mouse.position);
 				if viewport.is_in_bounds(mouse_position + viewport.offset()) {
-					update_cursor_preview(responses, input, global_tool_data, None);
+					update_cursor_preview(responses, tool_data, input, global_tool_data, None);
 				} else {
-					disable_cursor_preview(responses);
+					disable_cursor_preview(responses, tool_data);
 				}
 
 				self
@@ -111,14 +128,14 @@ impl Fsm for EyedropperToolFsmState {
 			// Sampling -> Ready
 			(EyedropperToolFsmState::SamplingPrimary, EyedropperToolMessage::SamplePrimaryColorEnd) | (EyedropperToolFsmState::SamplingSecondary, EyedropperToolMessage::SampleSecondaryColorEnd) => {
 				let set_color_choice = if self == EyedropperToolFsmState::SamplingPrimary { "Primary" } else { "Secondary" }.to_string();
-				update_cursor_preview(responses, input, global_tool_data, Some(set_color_choice));
-				disable_cursor_preview(responses);
+				update_cursor_preview(responses, tool_data, input, global_tool_data, Some(set_color_choice));
+				disable_cursor_preview(responses, tool_data);
 
 				EyedropperToolFsmState::Ready
 			}
 			// Any -> Ready
 			(_, EyedropperToolMessage::Abort) => {
-				disable_cursor_preview(responses);
+				disable_cursor_preview(responses, tool_data);
 
 				EyedropperToolFsmState::Ready
 			}
@@ -151,8 +168,10 @@ impl Fsm for EyedropperToolFsmState {
 	}
 }
 
-fn disable_cursor_preview(responses: &mut VecDeque<Message>) {
+fn disable_cursor_preview(responses: &mut VecDeque<Message>, tool_data: &mut EyedropperToolData) {
+	tool_data.preview = false;
 	responses.add(FrontendMessage::UpdateEyedropperSamplingState {
+		image: None,
 		mouse_position: None,
 		primary_color: "".into(),
 		secondary_color: "".into(),
@@ -160,8 +179,42 @@ fn disable_cursor_preview(responses: &mut VecDeque<Message>) {
 	});
 }
 
-fn update_cursor_preview(responses: &mut VecDeque<Message>, input: &InputPreprocessorMessageHandler, global_tool_data: &DocumentToolData, set_color_choice: Option<String>) {
+#[cfg(not(target_family = "wasm"))]
+fn update_cursor_preview(
+	responses: &mut VecDeque<Message>,
+	tool_data: &mut EyedropperToolData,
+	_input: &InputPreprocessorMessageHandler,
+	_global_tool_data: &DocumentToolData,
+	set_color_choice: Option<String>,
+) {
+	tool_data.preview = true;
+	tool_data.color_choice = set_color_choice;
+	responses.add(PortfolioMessage::SubmitEyedropperPreviewRender);
+}
+
+#[cfg(target_family = "wasm")]
+fn update_cursor_preview(
+	responses: &mut VecDeque<Message>,
+	tool_data: &mut EyedropperToolData,
+	input: &InputPreprocessorMessageHandler,
+	global_tool_data: &DocumentToolData,
+	set_color_choice: Option<String>,
+) {
+	tool_data.preview = true;
+	tool_data.color_choice = set_color_choice.clone();
+
+	update_cursor_preview_common(responses, None, input, global_tool_data, set_color_choice);
+}
+
+fn update_cursor_preview_common(
+	responses: &mut VecDeque<Message>,
+	image: Option<EyedropperPreviewImage>,
+	input: &InputPreprocessorMessageHandler,
+	global_tool_data: &DocumentToolData,
+	set_color_choice: Option<String>,
+) {
 	responses.add(FrontendMessage::UpdateEyedropperSamplingState {
+		image,
 		mouse_position: Some(input.mouse.position.into()),
 		primary_color: "#".to_string() + global_tool_data.primary_color.to_rgb_hex_srgb().as_str(),
 		secondary_color: "#".to_string() + global_tool_data.secondary_color.to_rgb_hex_srgb().as_str(),

editor/src/node_graph_executor.rs

@@ -59,6 +59,7 @@ pub struct NodeGraphExecutor {
 
 #[derive(Debug, Clone)]
 struct ExecutionContext {
+	render_config: RenderConfig,
 	export_config: Option<ExportConfig>,
 	document_id: DocumentId,
 }
@@ -157,12 +158,20 @@ impl NodeGraphExecutor {
 			render_mode: document.render_mode,
 			hide_artboards: false,
 			for_export: false,
+			for_eyedropper: false,
 		};
 
 		// Execute the node graph
 		let execution_id = self.queue_execution(render_config);
 
-		self.futures.push_back((execution_id, ExecutionContext { export_config: None, document_id }));
+		self.futures.push_back((
+			execution_id,
+			ExecutionContext {
+				render_config,
+				export_config: None,
+				document_id,
+			},
+		));
 
 		Ok(DeferMessage::SetGraphSubmissionIndex { execution_id }.into())
 	}
@@ -184,6 +193,40 @@ impl NodeGraphExecutor {
 		self.submit_current_node_graph_evaluation(document, document_id, viewport_resolution, viewport_scale, time, pointer)
 	}
 
+	#[cfg(not(target_family = "wasm"))]
+	pub(crate) fn submit_eyedropper_preview(&mut self, document_id: DocumentId, transform: DAffine2, pointer: DVec2, resolution: UVec2, time: TimingInformation) -> Result<Message, String> {
+		let viewport = Footprint {
+			transform,
+			resolution,
+			..Default::default()
+		};
+		let render_config = RenderConfig {
+			viewport,
+			scale: 1.,
+			time,
+			pointer,
+			export_format: graphene_std::application_io::ExportFormat::Raster,
+			render_mode: graphene_std::vector::style::RenderMode::Normal,
+			hide_artboards: false,
+			for_export: false,
+			for_eyedropper: true,
+		};
+
+		// Execute the node graph
+		let execution_id = self.queue_execution(render_config);
+
+		self.futures.push_back((
+			execution_id,
+			ExecutionContext {
+				render_config,
+				export_config: None,
+				document_id,
+			},
+		));
+
+		Ok(DeferMessage::SetGraphSubmissionIndex { execution_id }.into())
+	}
+
 	/// Evaluates a node graph for export
 	pub fn submit_document_export(&mut self, document: &mut DocumentMessageHandler, document_id: DocumentId, mut export_config: ExportConfig) -> Result<(), String> {
 		let network = document.network_interface.document_network().clone();
@@ -217,6 +260,7 @@ impl NodeGraphExecutor {
 			render_mode: document.render_mode,
 			hide_artboards: export_config.transparent_background,
 			for_export: true,
+			for_eyedropper: false,
 		};
 		export_config.size = resolution.as_dvec2();
 
@@ -225,97 +269,14 @@ impl NodeGraphExecutor {
 			.send(GraphRuntimeRequest::GraphUpdate(GraphUpdate { network, node_to_inspect: None }))
 			.map_err(|e| e.to_string())?;
 		let execution_id = self.queue_execution(render_config);
-		let execution_context = ExecutionContext {
-			export_config: Some(export_config),
-			document_id,
-		};
-		self.futures.push_back((execution_id, execution_context));
-
-		Ok(())
-	}
-
-	fn export(&self, node_graph_output: TaggedValue, export_config: ExportConfig, responses: &mut VecDeque<Message>) -> Result<(), String> {
-		let ExportConfig {
-			file_type,
-			name,
-			size,
-			scale_factor,
-			#[cfg(feature = "gpu")]
-			transparent_background,
-			artboard_name,
-			artboard_count,
-			..
-		} = export_config;
-
-		let file_extension = match file_type {
-			FileType::Svg => "svg",
-			FileType::Png => "png",
-			FileType::Jpg => "jpg",
-		};
-		let base_name = match (artboard_name, artboard_count) {
-			(Some(artboard_name), count) if count > 1 => format!("{name} - {artboard_name}"),
-			_ => name,
-		};
-		let name = format!("{base_name}.{file_extension}");
-
-		match node_graph_output {
-			TaggedValue::RenderOutput(RenderOutput {
-				data: RenderOutputType::Svg { svg, .. },
-				..
-			}) => {
-				if file_type == FileType::Svg {
-					responses.add(FrontendMessage::TriggerSaveFile { name, content: svg.into_bytes() });
-				} else {
-					let mime = file_type.to_mime().to_string();
-					let size = (size * scale_factor).into();
-					responses.add(FrontendMessage::TriggerExportImage { svg, name, mime, size });
-				}
-			}
-			#[cfg(feature = "gpu")]
-			TaggedValue::RenderOutput(RenderOutput {
-				data: RenderOutputType::Buffer { data, width, height },
-				..
-			}) if file_type != FileType::Svg => {
-				use image::buffer::ConvertBuffer;
-				use image::{ImageFormat, RgbImage, RgbaImage};
-
-				let Some(image) = RgbaImage::from_raw(width, height, data) else {
-					return Err("Failed to create image buffer for export".to_string());
-				};
-
-				let mut encoded = Vec::new();
-				let mut cursor = std::io::Cursor::new(&mut encoded);
-
-				match file_type {
-					FileType::Png => {
-						let result = if transparent_background {
-							image.write_to(&mut cursor, ImageFormat::Png)
-						} else {
-							let image: RgbImage = image.convert();
-							image.write_to(&mut cursor, ImageFormat::Png)
-						};
-						if let Err(err) = result {
-							return Err(format!("Failed to encode PNG: {err}"));
-						}
-					}
-					FileType::Jpg => {
-						let image: RgbImage = image.convert();
-						let result = image.write_to(&mut cursor, ImageFormat::Jpeg);
-						if let Err(err) = result {
-							return Err(format!("Failed to encode JPG: {err}"));
-						}
-					}
-					FileType::Svg => {
-						return Err("SVG cannot be exported from an image buffer".to_string());
-					}
-				}
-
-				responses.add(FrontendMessage::TriggerSaveFile { name, content: encoded });
-			}
-			_ => {
-				return Err(format!("Incorrect render type for exporting to an SVG ({file_type:?}, {node_graph_output})"));
-			}
-		};
+		self.futures.push_back((
+			execution_id,
+			ExecutionContext {
+				render_config,
+				export_config: Some(export_config),
+				document_id,
+			},
+		));
 
 		Ok(())
 	}
@@ -363,7 +324,10 @@ impl NodeGraphExecutor {
 
 					if let Some(export_config) = execution_context.export_config {
 						// Special handling for exporting the artwork
-						self.export(node_graph_output, export_config, responses)?;
+						self.process_export(node_graph_output, export_config, responses)?;
+					} else if execution_context.render_config.for_eyedropper {
+						// Special handling for Eyedropper tool preview
+						self.process_eyedropper_preview(node_graph_output, responses)?;
 					} else {
 						self.process_node_graph_output(node_graph_output, responses)?;
 					}
@@ -464,6 +428,109 @@ impl NodeGraphExecutor {
 
 		Ok(())
 	}
+
+	fn process_eyedropper_preview(&self, node_graph_output: TaggedValue, responses: &mut VecDeque<Message>) -> Result<(), String> {
+		match node_graph_output {
+			#[cfg(feature = "gpu")]
+			TaggedValue::RenderOutput(RenderOutput {
+				data: RenderOutputType::Buffer { data, width, height },
+				..
+			}) => {
+				responses.add(EyedropperToolMessage::PreviewImage { data, width, height });
+			}
+			_ => {
+				// TODO: Support Eyedropper preview in SVG mode on desktop
+			}
+		};
+
+		Ok(())
+	}
+
+	fn process_export(&self, node_graph_output: TaggedValue, export_config: ExportConfig, responses: &mut VecDeque<Message>) -> Result<(), String> {
+		let ExportConfig {
+			file_type,
+			name,
+			size,
+			scale_factor,
+			#[cfg(feature = "gpu")]
+			transparent_background,
+			artboard_name,
+			artboard_count,
+			..
+		} = export_config;
+
+		let file_extension = match file_type {
+			FileType::Svg => "svg",
+			FileType::Png => "png",
+			FileType::Jpg => "jpg",
+		};
+		let base_name = match (artboard_name, artboard_count) {
+			(Some(artboard_name), count) if count > 1 => format!("{name} - {artboard_name}"),
+			_ => name,
+		};
+		let name = format!("{base_name}.{file_extension}");
+
+		match node_graph_output {
+			TaggedValue::RenderOutput(RenderOutput {
+				data: RenderOutputType::Svg { svg, .. },
+				..
+			}) => {
+				if file_type == FileType::Svg {
+					responses.add(FrontendMessage::TriggerSaveFile { name, content: svg.into_bytes() });
+				} else {
+					let mime = file_type.to_mime().to_string();
+					let size = (size * scale_factor).into();
+					responses.add(FrontendMessage::TriggerExportImage { svg, name, mime, size });
+				}
+			}
+			#[cfg(feature = "gpu")]
+			TaggedValue::RenderOutput(RenderOutput {
+				data: RenderOutputType::Buffer { data, width, height },
+				..
+			}) if file_type != FileType::Svg => {
+				use image::buffer::ConvertBuffer;
+				use image::{ImageFormat, RgbImage, RgbaImage};
+
+				let Some(image) = RgbaImage::from_raw(width, height, data) else {
+					return Err("Failed to create image buffer for export".to_string());
+				};
+
+				let mut encoded = Vec::new();
+				let mut cursor = std::io::Cursor::new(&mut encoded);
+
+				match file_type {
+					FileType::Png => {
+						let result = if transparent_background {
+							image.write_to(&mut cursor, ImageFormat::Png)
+						} else {
+							let image: RgbImage = image.convert();
+							image.write_to(&mut cursor, ImageFormat::Png)
+						};
+						if let Err(err) = result {
+							return Err(format!("Failed to encode PNG: {err}"));
+						}
+					}
+					FileType::Jpg => {
+						let image: RgbImage = image.convert();
+						let result = image.write_to(&mut cursor, ImageFormat::Jpeg);
+						if let Err(err) = result {
+							return Err(format!("Failed to encode JPG: {err}"));
+						}
+					}
+					FileType::Svg => {
+						return Err("SVG cannot be exported from an image buffer".to_string());
+					}
+				}
+
+				responses.add(FrontendMessage::TriggerSaveFile { name, content: encoded });
+			}
+			_ => {
+				return Err(format!("Incorrect render type for exporting to an SVG ({file_type:?}, {node_graph_output})"));
+			}
+		};
+
+		Ok(())
+	}
 }
 
 // Re-export for usage by tests in other modules

editor/src/node_graph_executor/runtime.rs

@@ -246,7 +246,7 @@ impl NodeRuntime {
 						Ok(TaggedValue::RenderOutput(RenderOutput {
 							data: RenderOutputType::Texture(image_texture),
 							metadata,
-						})) if render_config.for_export => {
+						})) if render_config.for_export || render_config.for_eyedropper => {
 							let executor = self
 								.editor_api
 								.application_io