Add input hints to interactions in the node graph (#2415)

* node_graph: add hints in status bar

* allow in-progress interactions

* Fix node graph hints not restoring after panning operation

* Code review

---------

Co-authored-by: Keavon Chambers <keavon@keavon.com>

editor/src/messages/portfolio/document/document_message_handler.rs

@@ -508,6 +508,7 @@ impl MessageHandler<DocumentMessage, DocumentMessageData<'_>> for DocumentMessag
 					responses.add(NodeGraphMessage::SetGridAlignedEdges);
 					responses.add(NodeGraphMessage::UpdateGraphBarRight);
 					responses.add(NodeGraphMessage::SendGraph);
+					responses.add(NodeGraphMessage::UpdateHints);
 				} else {
 					responses.add(ToolMessage::ActivateTool { tool_type: *current_tool });
 				}
@@ -2479,6 +2480,10 @@ impl DocumentMessageHandler {
 		let insert_index = if relative_index_offset < 0 { neighbor_index } else { neighbor_index + 1 };
 		responses.add(DocumentMessage::MoveSelectedLayersTo { parent, insert_index });
 	}
+
+	pub fn graph_view_overlay_open(&self) -> bool {
+		self.graph_view_overlay_open
+	}
 }
 
 /// Create a network interface with a single export

editor/src/messages/portfolio/document/node_graph/node_graph_message.rs

@@ -212,5 +212,6 @@ pub enum NodeGraphMessage {
 	UpdateActionButtons,
 	UpdateGraphBarRight,
 	UpdateInSelectedNetwork,
+	UpdateHints,
 	SendSelectedNodes,
 }

editor/src/messages/portfolio/document/node_graph/node_graph_message_handler.rs

@@ -15,6 +15,8 @@ use crate::messages::portfolio::document::utility_types::network_interface::{
 use crate::messages::portfolio::document::utility_types::nodes::{CollapsedLayers, LayerPanelEntry};
 use crate::messages::prelude::*;
 use crate::messages::tool::common_functionality::auto_panning::AutoPanning;
+use crate::messages::tool::tool_messages::tool_prelude::{Key, MouseMotion};
+use crate::messages::tool::utility_types::{HintData, HintGroup, HintInfo};
 use glam::{DAffine2, DVec2, IVec2};
 use graph_craft::document::{DocumentNodeImplementation, NodeId, NodeInput};
 use graph_craft::proto::GraphErrors;
@@ -43,14 +45,16 @@ pub struct NodeGraphMessageHandler {
 	pub node_graph_errors: GraphErrors,
 	has_selection: bool,
 	widgets: [LayoutGroup; 2],
-	/// The start position when dragging nodes
-	pub drag_start: Option<DragStart>,
 	/// Used to add a transaction for the first node move when dragging.
 	begin_dragging: bool,
 	/// Used to prevent entering a nested network if the node is dragged after double clicking
-	drag_occurred: bool,
+	node_has_moved_in_drag: bool,
-	/// Stored in node graph coordinates
+	/// If dragging the selected nodes, this stores the starting position both in viewport and node graph coordinates,
-	box_selection_start: Option<DVec2>,
+	/// plus a flag indicating if it has been dragged since the mousedown began.
+	pub drag_start: Option<(DragStart, bool)>,
+	/// If dragging the background to create a box selection, this stores its starting point in node graph coordinates,
+	/// plus a flag indicating if it has been dragged since the mousedown began.
+	box_selection_start: Option<(DVec2, bool)>,
 	/// Restore the selection before box selection if it is aborted
 	selection_before_pointer_down: Vec<NodeId>,
 	/// If the grip icon is held during a drag, then shift without pushing other nodes
@@ -294,7 +298,7 @@ impl<'a> MessageHandler<NodeGraphMessage, NodeGraphHandlerData<'a>> for NodeGrap
 			}
 			NodeGraphMessage::EnterNestedNetwork => {
 				// Do not enter the nested network if the node was dragged
-				if self.drag_occurred {
+				if self.node_has_moved_in_drag {
 					return;
 				}
 
@@ -377,7 +381,7 @@ impl<'a> MessageHandler<NodeGraphMessage, NodeGraphHandlerData<'a>> for NodeGrap
 				) else {
 					return;
 				};
-				//Ensure that nodes can be grouped by checking if there is an unselected node between selected nodes
+				// Ensure that nodes can be grouped by checking if there is an unselected node between selected nodes
 				for selected_node_id in &selected_node_ids {
 					for input_index in 0..network_interface.number_of_inputs(selected_node_id, breadcrumb_network_path) {
 						let input_connector = InputConnector::node(*selected_node_id, input_index);
@@ -580,8 +584,8 @@ impl<'a> MessageHandler<NodeGraphMessage, NodeGraphHandlerData<'a>> for NodeGrap
 				if right_click {
 					// Abort dragging a node
 					if self.drag_start.is_some() {
-						responses.add(DocumentMessage::AbortTransaction);
 						self.drag_start = None;
+						responses.add(DocumentMessage::AbortTransaction);
 						responses.add(NodeGraphMessage::SelectedNodesSet {
 							nodes: self.selection_before_pointer_down.clone(),
 						});
@@ -598,18 +602,16 @@ impl<'a> MessageHandler<NodeGraphMessage, NodeGraphHandlerData<'a>> for NodeGrap
 					}
 					// Abort dragging a wire
 					if self.wire_in_progress_from_connector.is_some() {
-						responses.add(DocumentMessage::AbortTransaction);
 						self.wire_in_progress_from_connector = None;
 						self.wire_in_progress_to_connector = None;
+						responses.add(DocumentMessage::AbortTransaction);
 						responses.add(FrontendMessage::UpdateWirePathInProgress { wire_path: None });
 						return;
 					}
 
 					let context_menu_data = if let Some(node_id) = clicked_id {
-						ContextMenuData::ToggleLayer {
+						let currently_is_node = !network_interface.is_layer(&node_id, selection_network_path);
-							node_id,
+						ContextMenuData::ToggleLayer { node_id, currently_is_node }
-							currently_is_node: !network_interface.is_layer(&node_id, selection_network_path),
-						}
 					} else {
 						ContextMenuData::CreateNode
 					};
@@ -721,6 +723,7 @@ impl<'a> MessageHandler<NodeGraphMessage, NodeGraphHandlerData<'a>> for NodeGrap
 					self.initial_disconnecting = false;
 
 					self.wire_in_progress_from_connector = network_interface.output_position(&clicked_output, selection_network_path);
+					self.update_node_graph_hints(responses);
 					return;
 				}
 
@@ -764,9 +767,10 @@ impl<'a> MessageHandler<NodeGraphMessage, NodeGraphHandlerData<'a>> for NodeGrap
 							round_y: 0,
 						};
 
-						self.drag_start = Some(drag_start);
+						self.drag_start = Some((drag_start, false));
 						self.begin_dragging = true;
-						self.drag_occurred = false;
+						self.node_has_moved_in_drag = false;
+						self.update_node_graph_hints(responses);
 					}
 
 					// Update the selection if it was modified
@@ -783,7 +787,8 @@ impl<'a> MessageHandler<NodeGraphMessage, NodeGraphHandlerData<'a>> for NodeGrap
 				if !shift_click {
 					responses.add(NodeGraphMessage::SelectedNodesSet { nodes: Vec::new() })
 				}
-				self.box_selection_start = Some(node_graph_point);
+				self.box_selection_start = Some((node_graph_point, false));
+				self.update_node_graph_hints(responses);
 			}
 			NodeGraphMessage::PointerMove { shift } => {
 				if selection_network_path != breadcrumb_network_path {
@@ -872,11 +877,15 @@ impl<'a> MessageHandler<NodeGraphMessage, NodeGraphHandlerData<'a>> for NodeGrap
 						};
 						responses.add(FrontendMessage::UpdateWirePathInProgress { wire_path: Some(wire_path) });
 					}
-				} else if let Some(drag_start) = &mut self.drag_start {
+				} else if let Some((drag_start, dragged)) = &mut self.drag_start {
-					self.drag_occurred = true;
+					if drag_start.start_x != point.x || drag_start.start_y != point.y {
+						*dragged = true;
+					}
+
+					self.node_has_moved_in_drag = true;
 					if self.begin_dragging {
 						self.begin_dragging = false;
-						if ipp.keyboard.get(crate::messages::tool::tool_messages::tool_prelude::Key::Alt as usize) {
+						if ipp.keyboard.get(Key::Alt as usize) {
 							responses.add(NodeGraphMessage::DuplicateSelectedNodes);
 							// Duplicating sets a 2x2 offset, so shift the nodes back to the original position
 							responses.add(NodeGraphMessage::ShiftSelectedNodesByAmount {
@@ -898,8 +907,12 @@ impl<'a> MessageHandler<NodeGraphMessage, NodeGraphHandlerData<'a>> for NodeGrap
 					graph_delta.y -= previous_round_y;
 
 					responses.add(NodeGraphMessage::ShiftSelectedNodesByAmount { graph_delta, rubber_band: true });
-				} else if self.box_selection_start.is_some() {
+
+					self.update_node_graph_hints(responses);
+				} else if let Some((_, box_selection_dragged)) = &mut self.box_selection_start {
+					*box_selection_dragged = true;
 					responses.add(NodeGraphMessage::UpdateBoxSelection);
+					self.update_node_graph_hints(responses);
 				} else if self.reordering_import.is_some() {
 					let Some(modify_import_export) = network_interface.modify_import_export(selection_network_path) else {
 						log::error!("Could not get modify import export in PointerUp");
@@ -1016,18 +1029,20 @@ impl<'a> MessageHandler<NodeGraphMessage, NodeGraphHandlerData<'a>> for NodeGrap
 					}
 				}
 				// End of dragging a node
-				else if let Some(drag_start) = &self.drag_start {
+				else if let Some((drag_start, _)) = &self.drag_start {
 					self.shift_without_push = false;
+
 					// Reset all offsets to end the rubber banding while dragging
 					network_interface.unload_stack_dependents_y_offset(selection_network_path);
 					let Some(selected_nodes) = network_interface.selected_nodes_in_nested_network(selection_network_path) else {
 						log::error!("Could not get selected nodes in PointerUp");
 						return;
 					};
+
 					// Only select clicked node if multiple are selected and they were not dragged
 					if let Some(select_if_not_dragged) = self.select_if_not_dragged {
-						if drag_start.start_x == point.x
+						let not_dragged = drag_start.start_x == point.x && drag_start.start_y == point.y;
-							&& drag_start.start_y == point.y
+						if not_dragged
 							&& (selected_nodes.selected_nodes_ref().len() != 1
 								|| selected_nodes
 									.selected_nodes_ref()
@@ -1218,6 +1233,7 @@ impl<'a> MessageHandler<NodeGraphMessage, NodeGraphHandlerData<'a>> for NodeGrap
 				responses.add(FrontendMessage::UpdateBox { box_selection: None });
 				responses.add(FrontendMessage::UpdateImportReorderIndex { index: None });
 				responses.add(FrontendMessage::UpdateExportReorderIndex { index: None });
+				self.update_node_graph_hints(responses);
 			}
 			NodeGraphMessage::PointerOutsideViewport { shift } => {
 				if self.drag_start.is_some() || self.box_selection_start.is_some() {
@@ -1306,6 +1322,7 @@ impl<'a> MessageHandler<NodeGraphMessage, NodeGraphHandlerData<'a>> for NodeGrap
 						has_left_input_wire,
 					});
 					responses.add(NodeGraphMessage::SendSelectedNodes);
+					self.update_node_graph_hints(responses);
 				}
 			}
 			NodeGraphMessage::SetGridAlignedEdges => {
@@ -1542,7 +1559,7 @@ impl<'a> MessageHandler<NodeGraphMessage, NodeGraphHandlerData<'a>> for NodeGrap
 				responses.add(PropertiesPanelMessage::Refresh);
 			}
 			NodeGraphMessage::UpdateBoxSelection => {
-				if let Some(box_selection_start) = self.box_selection_start {
+				if let Some((box_selection_start, _)) = self.box_selection_start {
 					// The mouse button was released but we missed the pointer up event
 					// if ((e.buttons & 1) === 0) {
 					// 	completeBoxSelection();
@@ -1572,7 +1589,7 @@ impl<'a> MessageHandler<NodeGraphMessage, NodeGraphHandlerData<'a>> for NodeGrap
 						.inverse()
 						.transform_point2(ipp.mouse.position);
 
-					let shift = ipp.keyboard.get(crate::messages::tool::tool_messages::tool_prelude::Key::Shift as usize);
+					let shift = ipp.keyboard.get(Key::Shift as usize);
 					let Some(selected_nodes) = network_interface.selected_nodes_in_nested_network(selection_network_path) else {
 						log::error!("Could not get selected nodes in PointerMove");
 						return;
@@ -1666,6 +1683,9 @@ impl<'a> MessageHandler<NodeGraphMessage, NodeGraphHandlerData<'a>> for NodeGrap
 			NodeGraphMessage::UpdateInSelectedNetwork => responses.add(FrontendMessage::UpdateInSelectedNetwork {
 				in_selected_network: selection_network_path == breadcrumb_network_path,
 			}),
+			NodeGraphMessage::UpdateHints => {
+				self.update_node_graph_hints(responses);
+			}
 			NodeGraphMessage::SendSelectedNodes => {
 				let Some(selected_nodes) = network_interface.selected_nodes_in_nested_network(breadcrumb_network_path) else {
 					log::error!("Could not get selected nodes in NodeGraphMessage::SendSelectedNodes");
@@ -1751,7 +1771,7 @@ impl NodeGraphMessageHandler {
 		let mut widgets = vec![
 			PopoverButton::new()
 				.icon(Some("Node".to_string()))
-				.tooltip("Add a new node")
+				.tooltip("New Node (Right Click)")
 				.popover_layout({
 					let node_chooser = NodeCatalog::new()
 						.on_update(move |node_type| {
@@ -2421,6 +2441,46 @@ impl NodeGraphMessageHandler {
 			DVec2::new(input_position.x, input_position.y),
 		]
 	}
+
+	pub fn update_node_graph_hints(&self, responses: &mut VecDeque<Message>) {
+		// A wire is in progress and its start and end connectors are set
+		let wiring = self.wire_in_progress_from_connector.is_some();
+
+		// Node gragging is in progress (having already moved at least one pixel from the mouse down position)
+		let dragging_nodes = self.drag_start.as_ref().is_some_and(|(_, dragged)| *dragged);
+
+		// A box selection is in progress
+		let dragging_box_selection = self.box_selection_start.is_some_and(|(_, box_selection_dragged)| box_selection_dragged);
+
+		// Cancel the ongoing action
+		if wiring || dragging_nodes || dragging_box_selection {
+			let hint_data = HintData(vec![HintGroup(vec![HintInfo::mouse(MouseMotion::Rmb, ""), HintInfo::keys([Key::Escape], "Cancel").prepend_slash()])]);
+			responses.add(FrontendMessage::UpdateInputHints { hint_data });
+			return;
+		}
+
+		// Default hints for all other states
+		let mut hint_data = HintData(vec![
+			HintGroup(vec![HintInfo::mouse(MouseMotion::Rmb, "Add Node")]),
+			HintGroup(vec![HintInfo::mouse(MouseMotion::Lmb, "Select Node"), HintInfo::keys([Key::Shift], "Extend").prepend_plus()]),
+			HintGroup(vec![HintInfo::mouse(MouseMotion::LmbDrag, "Select Area"), HintInfo::keys([Key::Shift], "Extend").prepend_plus()]),
+		]);
+		if self.has_selection {
+			hint_data.0.extend([
+				HintGroup(vec![HintInfo::mouse(MouseMotion::LmbDrag, "Drag Selected")]),
+				HintGroup(vec![HintInfo::keys([Key::Delete], "Delete Selected"), HintInfo::keys([Key::Control], "Keep Children").prepend_plus()]),
+				HintGroup(vec![
+					HintInfo::keys_and_mouse([Key::Alt], MouseMotion::LmbDrag, "Move Duplicate"),
+					HintInfo::keys([Key::Control, Key::KeyD], "Duplicate").add_mac_keys([Key::Command, Key::KeyD]),
+				]),
+			]);
+		}
+		hint_data.0.extend([
+			HintGroup(vec![HintInfo::mouse(MouseMotion::LmbDouble, "Enter Node Subgraph")]),
+			HintGroup(vec![HintInfo::keys_and_mouse([Key::Alt], MouseMotion::Lmb, "Preview Node Output")]),
+		]);
+		responses.add(FrontendMessage::UpdateInputHints { hint_data });
+	}
 }
 
 #[derive(Default)]
@@ -2499,7 +2559,7 @@ impl Default for NodeGraphMessageHandler {
 			widgets: [LayoutGroup::Row { widgets: Vec::new() }, LayoutGroup::Row { widgets: Vec::new() }],
 			drag_start: None,
 			begin_dragging: false,
-			drag_occurred: false,
+			node_has_moved_in_drag: false,
 			shift_without_push: false,
 			box_selection_start: None,
 			selection_before_pointer_down: Vec::new(),
@@ -2527,7 +2587,7 @@ impl PartialEq for NodeGraphMessageHandler {
 			&& self.widgets == other.widgets
 			&& self.drag_start == other.drag_start
 			&& self.begin_dragging == other.begin_dragging
-			&& self.drag_occurred == other.drag_occurred
+			&& self.node_has_moved_in_drag == other.node_has_moved_in_drag
 			&& self.box_selection_start == other.box_selection_start
 			&& self.initial_disconnecting == other.initial_disconnecting
 			&& self.select_if_not_dragged == other.select_if_not_dragged

editor/src/messages/tool/tool_message_handler.rs

@@ -263,6 +263,8 @@ impl MessageHandler<ToolMessage, ToolMessageData<'_>> for ToolMessageHandler {
 				let tool_data = &mut self.tool_state.tool_data;
 
 				if let Some(tool) = tool_data.tools.get_mut(&tool_type) {
+					let graph_view_overlay_open = document.graph_view_overlay_open();
+
 					if tool_type == tool_data.active_tool_type {
 						let mut data = ToolActionHandlerData {
 							document,
@@ -275,7 +277,10 @@ impl MessageHandler<ToolMessage, ToolMessageData<'_>> for ToolMessageHandler {
 							preferences,
 						};
 						if matches!(tool_message, ToolMessage::UpdateHints) {
-							if self.transform_layer_handler.is_transforming() {
+							if graph_view_overlay_open {
+								// When graph view is open, forward the hint update to the node graph handler
+								responses.add(NodeGraphMessage::UpdateHints);
+							} else if self.transform_layer_handler.is_transforming() {
 								self.transform_layer_handler.hints(responses);
 							} else {
 								tool.process_message(ToolMessage::UpdateHints, responses, &mut data)

editor/src/messages/tool/tool_messages/path_tool.rs

@@ -1442,7 +1442,7 @@ impl Fsm for PathToolFsmState {
 	fn update_hints(&self, responses: &mut VecDeque<Message>) {
 		let hint_data = match self {
 			PathToolFsmState::Ready => HintData(vec![
-				HintGroup(vec![HintInfo::mouse(MouseMotion::Lmb, "Select Point"), HintInfo::keys([Key::Shift], "Extend Selection").prepend_plus()]),
+				HintGroup(vec![HintInfo::mouse(MouseMotion::Lmb, "Select Point"), HintInfo::keys([Key::Shift], "Extend").prepend_plus()]),
 				HintGroup(vec![HintInfo::mouse(MouseMotion::LmbDrag, "Select Area"), HintInfo::keys([Key::Control], "Lasso").prepend_plus()]),
 				HintGroup(vec![HintInfo::mouse(MouseMotion::Lmb, "Insert Point on Segment")]),
 				// TODO: Only show if at least one anchor is selected, and dynamically show either "Smooth" or "Sharp" based on the current state

editor/src/messages/tool/tool_messages/select_tool.rs

@@ -1497,11 +1497,10 @@ impl Fsm for SelectToolFsmState {
 		match self {
 			SelectToolFsmState::Ready { selection } => {
 				let hint_data = HintData(vec![
-					HintGroup(vec![HintInfo::mouse(MouseMotion::LmbDrag, "Drag Selected")]),
 					HintGroup({
-						let mut hints = vec![HintInfo::mouse(MouseMotion::Lmb, "Select Object"), HintInfo::keys([Key::Shift], "Extend Selection").prepend_plus()];
+						let mut hints = vec![HintInfo::mouse(MouseMotion::Lmb, "Select Object"), HintInfo::keys([Key::Shift], "Extend").prepend_plus()];
 						if *selection == NestedSelectionBehavior::Shallowest {
-							hints.extend([HintInfo::keys([Key::Accel], "Deepest").prepend_plus(), HintInfo::mouse(MouseMotion::LmbDouble, "Deepen Selection")]);
+							hints.extend([HintInfo::keys([Key::Accel], "Deepest").prepend_plus(), HintInfo::mouse(MouseMotion::LmbDouble, "Deepen")]);
 						}
 						hints
 					}),
@@ -1511,6 +1510,8 @@ impl Fsm for SelectToolFsmState {
 						HintInfo::keys([Key::Alt], "Subtract").prepend_plus(),
 						HintInfo::keys([Key::Control], "Lasso").prepend_plus(),
 					]),
+					// TODO: Make all the following hints only appear if there is at least one selected layer
+					HintGroup(vec![HintInfo::mouse(MouseMotion::LmbDrag, "Drag Selected")]),
 					HintGroup(vec![HintInfo::multi_keys([[Key::KeyG], [Key::KeyR], [Key::KeyS]], "Grab/Rotate/Scale Selected")]),
 					HintGroup(vec![
 						HintInfo::arrow_keys("Nudge Selected"),
@@ -1546,7 +1547,7 @@ impl Fsm for SelectToolFsmState {
 					HintGroup(vec![HintInfo::keys([Key::Shift], "Extend"), HintInfo::keys([Key::Alt], "Subtract")]),
 					// TODO: Re-select deselected layers during drag when Shift is pressed, and re-deselect if Shift is released before drag ends.
 					// TODO: (See https://discord.com/channels/731730685944922173/1216976541947531264/1321360311298818048)
-					// HintGroup(vec![HintInfo::keys([Key::Shift], "Extend Selection")])
+					// HintGroup(vec![HintInfo::keys([Key::Shift], "Extend")])
 				]);
 				responses.add(FrontendMessage::UpdateInputHints { hint_data });
 			}