Desktop: Make input routing aware of floating menus (#4498)

* Desktop: Make input routing aware of floating menus

* Desktop: Route mouse input like pen input

* Desktop: Model pointer input routing as a state machine

desktop/src/app.rs

@@ -280,6 +280,9 @@ impl App {
 					render_state.set_viewport_scale([viewport_scale_x as f32, viewport_scale_y as f32]);
 				}
 			}
+			DesktopFrontendMessage::WindowUpdateDirectInput { enabled } => {
+				self.input_state.set_direct_input(enabled);
+			}
 			DesktopFrontendMessage::UpdateUIScale { scale } => {
 				self.ui_scale = scale;
 				self.resize();

desktop/src/input.rs

@@ -4,17 +4,16 @@ use winit::event::{ButtonSource, ElementState, MouseButton, MouseScrollDelta, Po
 use winit::keyboard::ModifiersState;
 
 use crate::ui::{InputEvent, MULTICLICK_ALLOWED_TRAVEL, MULTICLICK_TIMEOUT, PINCH_ZOOM_SPEED, SCROLL_LINE_HEIGHT, SCROLL_LINE_WIDTH, SCROLL_SPEED_X, SCROLL_SPEED_Y};
-use crate::wrapper::messages::{InputMessage, ModifierKeys, MouseKeys, PointerState, ScrollDelta};
+use crate::wrapper::messages::{EditorPointerState, InputMessage, ModifierKeys, MouseKeys, ScrollDelta};
 
 pub(crate) struct InputState {
 	start: Instant,
 	viewport_info: Option<ViewportInfo>,
-	pointer_lock_position: Option<PhysicalPosition<f64>>,
 	modifiers: ModifiersState,
 	pointer_position: PhysicalPosition<f64>,
-	pointer_keys: MouseKeys,
+	pointer_state: PointerState,
-	ui_capture: bool,
 	click_tracker: ClickTracker,
+	direct_input: bool,
 }
 
 impl InputState {
@@ -22,12 +21,11 @@ impl InputState {
 		Self {
 			start: Instant::now(),
 			viewport_info: None,
-			pointer_lock_position: None,
 			modifiers: ModifiersState::default(),
 			pointer_position: PhysicalPosition::default(),
-			pointer_keys: MouseKeys::empty(),
+			pointer_state: PointerState::Hover { route: Route::Ui },
-			ui_capture: true,
 			click_tracker: ClickTracker::default(),
+			direct_input: false,
 		}
 	}
 
@@ -35,20 +33,44 @@ impl InputState {
 		self.viewport_info = Some(ViewportInfo { x, y, width, height, scale });
 	}
 
+	pub(crate) fn set_direct_input(&mut self, enabled: bool) {
+		self.direct_input = enabled;
+	}
+
 	pub(crate) fn lock_pointer(&mut self) {
-		self.pointer_lock_position = Some(self.pointer_position);
+		self.pointer_state = match self.pointer_state {
+			PointerState::Hover { route } => PointerState::Locked {
+				route,
+				keys: MouseKeys::empty(),
+				position: self.pointer_position,
+			},
+			PointerState::Stroke { route, keys } | PointerState::Locked { route, keys, .. } => PointerState::Locked {
+				route,
+				keys,
+				position: self.pointer_position,
+			},
+		};
 	}
 
 	pub(crate) fn unlock_pointer(&mut self) -> Option<PhysicalPosition<f64>> {
-		let position = self.pointer_lock_position.take();
+		let PointerState::Locked {
-		if let Some(position) = position {
+			route: resume,
-			self.pointer_position = position;
+			keys,
-		}
+			position: restore,
-		position
+		} = self.pointer_state
+		else {
+			return None;
+		};
+		self.pointer_position = restore;
+		self.pointer_state = match keys.is_empty() {
+			true => PointerState::Hover { route: Route::Ui },
+			false => PointerState::Stroke { route: resume, keys },
+		};
+		Some(restore)
 	}
 
 	pub(crate) fn pointer_locked(&self) -> bool {
-		self.pointer_lock_position.is_some()
+		matches!(self.pointer_state, PointerState::Locked { .. })
 	}
 
 	pub(crate) fn modifiers(&self) -> ModifiersState {
@@ -60,24 +82,28 @@ impl InputState {
 			WindowEvent::PointerMoved { position, source, .. } => {
 				self.pointer_position = *position;
 
-				let PointerSource::TabletTool { kind, data } = source else {
+				let route = match self.pointer_state {
-					ui_callback(InputEvent::pointer().position(*position).moved().modifiers(self.modifiers).build());
+					PointerState::Hover { .. } => {
-					return;
+						let next = self.route(*position);
+						self.pointer_state = PointerState::Hover { route: next };
+						next
+					}
+					PointerState::Stroke { route, .. } => route,
+					PointerState::Locked { keys, route: resume, .. } => match keys.is_empty() {
+						true => Route::Ui,
+						false => resume,
+					},
 				};
-				let ui_capture = if self.pointer_keys.is_empty() {
+				match route {
-					self.pointer_locked() || !self.in_viewport(*position)
+					Route::Ui => ui_callback(InputEvent::pointer().position(*position).moved().modifiers(self.modifiers).build()),
-				} else {
+					Route::Editor => editor_callback(InputMessage::PointerMove {
-					self.ui_capture
+						editor_mouse_state: match source {
-				};
+							PointerSource::TabletTool { kind, data } => self.tablet_pointer_state(kind, data),
-				if ui_capture {
+							_ => self.pointer_state(),
-					ui_callback(InputEvent::pointer().position(*position).moved().modifiers(self.modifiers).build());
+						},
-					return;
+						modifier_keys: self.modifier_keys(),
+					}),
 				}
-
-				editor_callback(InputMessage::PointerMove {
-					editor_mouse_state: self.tablet_pointer_state(kind, data),
-					modifier_keys: self.modifier_keys(),
-				});
 			}
 			WindowEvent::PointerEntered { position, .. } => {
 				self.pointer_position = *position;
@@ -91,13 +117,6 @@ impl InputState {
 			WindowEvent::PointerButton { state, button, position, .. } => {
 				self.pointer_position = *position;
 
-				let tablet = matches!(button, ButtonSource::TabletTool { .. });
-
-				// Stroke keeps capture decided from first button press until all buttons are released.
-				if state.is_pressed() && self.pointer_keys.is_empty() {
-					self.ui_capture = self.pointer_locked() || !tablet || !self.in_viewport(*position);
-				}
-
 				let mouse_button = button.clone().mouse_button();
 				let keys = match mouse_button {
 					Some(MouseButton::Left) => MouseKeys::LEFT,
@@ -107,15 +126,41 @@ impl InputState {
 					Some(MouseButton::Forward) => MouseKeys::FORWARD,
 					_ => MouseKeys::NONE,
 				};
-				match state {
+
-					ElementState::Pressed => self.pointer_keys.insert(keys),
+				let (pointer, route) = match self.pointer_state {
-					ElementState::Released => self.pointer_keys.remove(keys),
+					PointerState::Hover { route } => match (state.is_pressed(), keys.is_empty()) {
-				}
+						(true, false) => {
+							let route = self.route(*position);
+							(PointerState::Stroke { route, keys }, route)
+						}
+						(true, true) => (PointerState::Hover { route }, self.route(*position)),
+						(false, _) => (PointerState::Hover { route }, route),
+					},
+					PointerState::Stroke { route, keys: mut held } => {
+						match state.is_pressed() {
+							true => held.insert(keys),
+							false => held.remove(keys),
+						}
+						match held.is_empty() {
+							true => (PointerState::Hover { route }, route),
+							false => (PointerState::Stroke { route, keys: held }, route),
+						}
+					}
+					PointerState::Locked { route, keys: mut held, position } => {
+						let resume = if state.is_pressed() && held.is_empty() { Route::Ui } else { route };
+						match state.is_pressed() {
+							true => held.insert(keys),
+							false => held.remove(keys),
+						}
+						(PointerState::Locked { route: resume, keys: held, position }, Route::Ui)
+					}
+				};
+				self.pointer_state = pointer;
 
 				let count = mouse_button.map_or(1, |button| self.click_tracker.input(*position, button, *state));
 
 				let back_or_forward = matches!(mouse_button, Some(MouseButton::Back | MouseButton::Forward));
-				if self.pointer_locked() || !(back_or_forward || (tablet && !self.ui_capture)) {
+				if self.pointer_locked() || keys.is_empty() || !(back_or_forward || route == Route::Editor) {
 					let pointer = InputEvent::pointer().position(*position);
 					let input = match state {
 						ElementState::Pressed => pointer.pressed(button.clone(), count),
@@ -135,7 +180,7 @@ impl InputState {
 					ElementState::Released if count % 2 == 0 => {
 						editor_callback(InputMessage::PointerUp { editor_mouse_state, modifier_keys });
 						editor_callback(InputMessage::DoubleClick {
-							editor_mouse_state: PointerState {
+							editor_mouse_state: EditorPointerState {
 								mouse_keys: keys,
 								..editor_mouse_state
 							},
@@ -146,7 +191,7 @@ impl InputState {
 				}
 			}
 			WindowEvent::MouseWheel { delta, .. } => {
-				if self.pointer_locked() || !self.in_viewport(self.pointer_position) {
+				if self.pointer_locked() || self.ui_captures(self.pointer_position) {
 					let input = match delta {
 						MouseScrollDelta::LineDelta(x, y) => InputEvent::pointer().scrolled_lines(f64::from(*x), f64::from(*y)),
 						MouseScrollDelta::PixelDelta(position) => InputEvent::pointer().scrolled_pixels(position.x, position.y),
@@ -163,12 +208,12 @@ impl InputState {
 				let scroll_delta = ScrollDelta::new(-x * SCROLL_SPEED_X, -y * SCROLL_SPEED_Y, 0.);
 
 				editor_callback(InputMessage::WheelScroll {
-					editor_mouse_state: PointerState { scroll_delta, ..self.pointer_state() },
+					editor_mouse_state: EditorPointerState { scroll_delta, ..self.pointer_state() },
 					modifier_keys: self.modifier_keys(),
 				});
 			}
 			WindowEvent::PinchGesture { delta, .. } => {
-				if self.pointer_locked() || !self.in_viewport(self.pointer_position) || !delta.is_normal() {
+				if self.pointer_locked() || self.ui_captures(self.pointer_position) || !delta.is_normal() {
 					ui_callback(InputEvent::pointer().zoomed(*delta).modifiers(self.modifiers).build());
 					return;
 				}
@@ -176,7 +221,7 @@ impl InputState {
 				// TODO: This is a temporary solution to handle pinch gestures, we should handle pinch gestures editor-side instead.
 				let scroll_delta = ScrollDelta::new(0., -delta * PINCH_ZOOM_SPEED, 0.);
 				editor_callback(InputMessage::WheelScroll {
-					editor_mouse_state: PointerState { scroll_delta, ..self.pointer_state() },
+					editor_mouse_state: EditorPointerState { scroll_delta, ..self.pointer_state() },
 					modifier_keys: self.modifier_keys() | ModifierKeys::CONTROL,
 				});
 			}
@@ -192,21 +237,32 @@ impl InputState {
 		self.viewport_info.as_ref().map_or(1., |info| info.scale)
 	}
 
-	fn in_viewport(&self, position: PhysicalPosition<f64>) -> bool {
+	fn ui_captures(&self, position: PhysicalPosition<f64>) -> bool {
-		self.viewport_info.as_ref().is_some_and(|info| info.contains(position))
+		!self.direct_input || !self.viewport_info.as_ref().is_some_and(|info| info.contains(position))
 	}
 
-	fn pointer_state(&self) -> PointerState {
+	fn route(&self, position: PhysicalPosition<f64>) -> Route {
-		PointerState {
+		if self.ui_captures(position) { Route::Ui } else { Route::Editor }
+	}
+
+	fn pointer_keys(&self) -> MouseKeys {
+		match self.pointer_state {
+			PointerState::Hover { .. } => MouseKeys::empty(),
+			PointerState::Stroke { keys, .. } | PointerState::Locked { keys, .. } => keys,
+		}
+	}
+
+	fn pointer_state(&self) -> EditorPointerState {
+		EditorPointerState {
 			editor_position: (self.pointer_position.x / self.scale(), self.pointer_position.y / self.scale()).into(),
-			mouse_keys: self.pointer_keys,
+			mouse_keys: self.pointer_keys(),
 			time: Some(self.start.elapsed().as_secs_f64() * 1000.),
 			..Default::default()
 		}
 	}
 
-	fn tablet_pointer_state(&self, kind: &TabletToolKind, data: &TabletToolData) -> PointerState {
+	fn tablet_pointer_state(&self, kind: &TabletToolKind, data: &TabletToolData) -> EditorPointerState {
-		PointerState {
+		EditorPointerState {
 			pressure: data.force.map(|force| force.normalized(None)),
 			tilt: data.clone().tilt().map(|tilt| (f64::from(tilt.x), f64::from(tilt.y)).into()),
 			twist: data.twist.map(f64::from),
@@ -226,6 +282,19 @@ impl InputState {
 	}
 }
 
+#[derive(Clone, Copy)]
+enum PointerState {
+	Hover { route: Route },
+	Stroke { route: Route, keys: MouseKeys },
+	Locked { route: Route, keys: MouseKeys, position: PhysicalPosition<f64> },
+}
+
+#[derive(Clone, Copy, PartialEq)]
+enum Route {
+	Ui,
+	Editor,
+}
+
 struct ViewportInfo {
 	x: f64,
 	y: f64,

desktop/wrapper/src/intercept_frontend_message.rs

@@ -122,6 +122,9 @@ pub(super) fn intercept_frontend_message(dispatcher: &mut DesktopWrapperMessageD
 		FrontendMessage::WindowPointerLock => {
 			dispatcher.respond(DesktopFrontendMessage::PointerLock);
 		}
+		FrontendMessage::WindowUpdateDirectInput { enabled } => {
+			dispatcher.respond(DesktopFrontendMessage::WindowUpdateDirectInput { enabled });
+		}
 		FrontendMessage::WindowClose => {
 			dispatcher.respond(DesktopFrontendMessage::WindowClose);
 		}

desktop/wrapper/src/messages.rs

@@ -5,7 +5,7 @@ pub(crate) use graphite_editor::messages::prelude::Message as EditorMessage;
 
 pub use graphite_editor::messages::frontend::utility_types::{DocumentInfo, FileFilter, PersistedState};
 pub use graphite_editor::messages::input_mapper::utility_types::keyboard::{Key, ModifierKeys};
-pub use graphite_editor::messages::input_mapper::utility_types::pointer::{EditorPointerState as PointerState, EditorPosition as Position, MouseKeys, ScrollDelta};
+pub use graphite_editor::messages::input_mapper::utility_types::pointer::{EditorPointerState, MouseKeys, ScrollDelta};
 pub use graphite_editor::messages::prelude::DocumentId;
 pub use graphite_editor::messages::prelude::InputPreprocessorMessage as InputMessage;
 pub use graphite_editor::messages::prelude::PreferencesMessageHandler as Preferences;
@@ -40,6 +40,9 @@ pub enum DesktopFrontendMessage {
 	UpdateUIScale {
 		scale: f64,
 	},
+	WindowUpdateDirectInput {
+		enabled: bool,
+	},
 	UpdateOverlays(vello::Scene),
 	PersistenceWriteDocument {
 		id: DocumentId,

editor/src/messages/app_window/app_window_message.rs

@@ -5,6 +5,7 @@ use crate::messages::prelude::*;
 pub enum AppWindowMessage {
 	PointerLock,
 	PointerLockMove { x: f64, y: f64 },
+	DirectInput { enabled: bool },
 	Restart,
 	Close,
 	Minimize,

editor/src/messages/app_window/app_window_message_handler.rs

@@ -17,6 +17,12 @@ impl MessageHandler<AppWindowMessage, ()> for AppWindowMessageHandler {
 			AppWindowMessage::PointerLockMove { x, y } => {
 				responses.add(FrontendMessage::WindowPointerLockMove { position: (x, y) });
 			}
+			AppWindowMessage::DirectInput { enabled } => {
+				#[cfg(not(target_family = "wasm"))]
+				responses.add(FrontendMessage::WindowUpdateDirectInput { enabled });
+				#[cfg(target_family = "wasm")]
+				let _ = enabled;
+			}
 			AppWindowMessage::Close => {
 				#[cfg(not(target_family = "wasm"))]
 				responses.add(FrontendMessage::WindowClose);

editor/src/messages/frontend/frontend_message.rs

@@ -350,6 +350,10 @@ pub enum FrontendMessage {
 		position: (f64, f64),
 	},
 	#[cfg(not(target_family = "wasm"))]
+	WindowUpdateDirectInput {
+		enabled: bool,
+	},
+	#[cfg(not(target_family = "wasm"))]
 	WindowClose,
 	#[cfg(not(target_family = "wasm"))]
 	WindowMinimize,

frontend/src/components/layout/FloatingMenu.svelte

@@ -17,10 +17,11 @@
 </script>
 
 <script lang="ts">
-	import { onMount, onDestroy, afterUpdate, createEventDispatcher, tick } from "svelte";
+	import { getContext, onMount, onDestroy, afterUpdate, createEventDispatcher, tick } from "svelte";
 	import LayoutCol from "/src/components/layout/LayoutCol.svelte";
+	import { onFloatingMenuOpenChange } from "/src/utility-functions/input";
 	import { browserVersion } from "/src/utility-functions/platform";
-	import type { MenuDirection } from "/wrapper/pkg/graphite_wasm_wrapper";
+	import type { EditorWrapper, MenuDirection } from "/wrapper/pkg/graphite_wasm_wrapper";
 
 	const BUTTON_LEFT = 0;
 	const POINTER_STRAY_DISTANCE = 100;
@@ -42,6 +43,9 @@
 	export let escapeCloses = true;
 	export let strayCloses = true;
 
+	const editor = getContext<EditorWrapper>("editor");
+	const menuId = String(Math.random()).substring(2);
+
 	let tail: HTMLDivElement | undefined;
 	let self: HTMLDivElement | undefined;
 	let floatingMenuContainer: HTMLDivElement | undefined;
@@ -65,6 +69,8 @@
 
 	$: watchOpenChange(open);
 
+	$: onFloatingMenuOpenChange(menuId, open && type !== "Tooltip" && type !== "Cursor", editor);
+
 	$: minWidthStyleValue = measuringOngoing ? "0" : `${Math.max(minWidth, minWidthParentWidth)}px`;
 	$: displayTail = open && type === "Popover";
 	$: displayContainer = open || measuringOngoing;
@@ -157,6 +163,7 @@
 	});
 
 	onDestroy(() => {
+		onFloatingMenuOpenChange(menuId, false, editor);
 		containerResizeObserver.disconnect();
 		dialogResizeObserver?.disconnect();
 		window.removeEventListener("pointermove", pointerMoveHandler);

frontend/src/components/panels/Document.svelte

@@ -15,6 +15,7 @@
 	import type { MessageBody } from "/src/subscriptions-router";
 	import { fillChoiceColor, createSRgba8 } from "/src/utility-functions/colors";
 	import { pasteFile } from "/src/utility-functions/files";
+	import { cleanupInputField } from "/src/utility-functions/input";
 	import { textInputCleanup } from "/src/utility-functions/keyboard-entry";
 	import { rasterizeSVGCanvas } from "/src/utility-functions/rasterization";
 	import { setupViewportResizeObserver, hasFirstArtworkBeenReceived, markFirstArtworkReceived } from "/src/utility-functions/viewports";
@@ -566,6 +567,7 @@
 		viewportResizeObserver?.disconnect();
 		removeUpdatePixelRatio?.();
 		addedFontFaces.forEach((face) => window.document.fonts.delete(face));
+		cleanupInputField(editor);
 
 		subscriptions.unsubscribeFrontendMessage("UpdateDocumentArtwork");
 		subscriptions.unsubscribeFrontendMessage("UpdateEyedropperSamplingState");

frontend/src/managers/input.ts

@@ -19,6 +19,7 @@ import {
 	onContextMenu,
 	onPaste,
 	onPointerLockChange,
+	updateDirectInput,
 } from "/src/utility-functions/input";
 import type { EditorWrapper } from "/wrapper/pkg/graphite_wasm_wrapper";
 
@@ -42,7 +43,7 @@ const listeners: Listener[] = [
 	{ target: window, eventName: "mousedown", action: (e: MouseEvent) => onMouseDown(e) },
 	{ target: window, eventName: "mouseup", action: (e: MouseEvent) => editorWrapper && onPotentialDoubleClick(e, editorWrapper) },
 	{ target: window, eventName: "wheel", action: (e: WheelEvent) => editorWrapper && onWheelScroll(e, editorWrapper), options: { passive: false } },
-	{ target: window, eventName: "modifyinputfield", action: (e: CustomEvent) => onModifyInputField(e) },
+	{ target: window, eventName: "modifyinputfield", action: (e: CustomEvent) => editorWrapper && onModifyInputField(e, editorWrapper) },
 	{ target: window, eventName: "focusout", action: () => onFocusOut() },
 	{ target: window.document, eventName: "contextmenu", action: (e: MouseEvent) => onContextMenu(e) },
 	{ target: window.document, eventName: "fullscreenchange", action: () => fullscreenModeChanged() },
@@ -79,6 +80,8 @@ export function createInputManager(subscriptions: SubscriptionsRouter, editor: E
 	// Add event bindings for the lifetime of the application
 	listeners.forEach(({ target, eventName, action, options }) => target.addEventListener(eventName, action, options));
 
+	updateDirectInput(editor);
+
 	// Focus the app container
 	const app = window.document.querySelector("[data-app-container]");
 	if (app instanceof HTMLElement) app.focus();

frontend/src/utility-functions/input.ts

@@ -20,6 +20,7 @@ let canvasFocused = true;
 let inPointerLock = false;
 let lastShakeTime = 0;
 const shakeSamples: { x: number; y: number; time: number }[] = [];
+const openFloatingMenus = new Set<string>();
 
 // Keyboard events
 
@@ -243,8 +244,14 @@ export function onWheelScroll(e: WheelEvent, editor: EditorWrapper) {
 
 // Receives a custom event dispatched when the user begins interactively editing with the text tool.
 // We keep a copy of the text input element to check against when it's active for text entry.
-export function onModifyInputField(e: CustomEvent) {
+export function onModifyInputField(e: CustomEvent, editor: EditorWrapper) {
 	textToolInteractiveInputElement = e.detail;
+	updateDirectInput(editor);
+}
+
+export function cleanupInputField(editor: EditorWrapper) {
+	textToolInteractiveInputElement = undefined;
+	updateDirectInput(editor);
 }
 
 // Window events
@@ -387,3 +394,20 @@ function potentiallyRestoreCanvasFocus(e: Event) {
 		app?.focus();
 	}
 }
+
+// Direct input (desktop only)
+
+// Desktop may route viewport pointer input directly to the editor if no floating menus are open
+export function updateDirectInput(editor: EditorWrapper) {
+	if (import.meta.env.MODE !== "native") return;
+
+	const uiCapturesViewport = openFloatingMenus.size > 0 || textToolInteractiveInputElement !== undefined;
+	editor.appWindowDirectInput(!uiCapturesViewport);
+}
+
+export function onFloatingMenuOpenChange(menuId: string, open: boolean, editor: EditorWrapper) {
+	if (open) openFloatingMenus.add(menuId);
+	else openFloatingMenus.delete(menuId);
+
+	updateDirectInput(editor);
+}

frontend/wrapper/src/editor_commands.rs

@@ -79,6 +79,10 @@ mod editor_commands {
 		}
 	}
 
+	fn app_window_direct_input(enabled: bool) -> Message {
+		AppWindowMessage::DirectInput { enabled }.into()
+	}
+
 	/// Start Pointer Lock
 	fn app_window_pointer_lock() -> Message {
 		AppWindowMessage::PointerLock.into()