Desktop: Mirror direct viewport input to the UI (#4502)

Desktop: Mirror direct-routed input to the UI

desktop/src/input.rs

@@ -6,6 +6,18 @@ use winit::keyboard::ModifiersState;
 use crate::ui::{InputEvent, MULTICLICK_ALLOWED_TRAVEL, MULTICLICK_TIMEOUT, PINCH_ZOOM_SPEED, SCROLL_LINE_HEIGHT, SCROLL_LINE_WIDTH, SCROLL_SPEED_X, SCROLL_SPEED_Y};
 use crate::wrapper::messages::{EditorPointerState, InputMessage, ModifierKeys, MouseKeys, ScrollDelta};
 
+// Marks input event as observe-only, meaning it should not result in any input messages being send by the frontend.
+// And should instead be ignored by frontend input processing and only be used to update the UI state (hover, cursor).
+// TODO(Timon): Remove and find a less hacky solution
+trait ObserveOnlyExt {
+	fn observe_only(self) -> Self;
+}
+impl ObserveOnlyExt for crate::ui::InputEventBuilder {
+	fn observe_only(self) -> Self {
+		self.num_lock(true)
+	}
+}
+
 pub(crate) struct InputState {
 	start: Instant,
 	viewport_info: Option<ViewportInfo>,
@@ -96,13 +108,18 @@ impl InputState {
 				};
 				match route {
 					Route::Ui => ui_callback(InputEvent::pointer().position(*position).moved().modifiers(self.modifiers).build()),
-					Route::Editor => editor_callback(InputMessage::PointerMove {
-						editor_mouse_state: match source {
-							PointerSource::TabletTool { kind, data } => self.tablet_pointer_state(kind, data),
-							_ => self.pointer_state(),
-						},
-						modifier_keys: self.modifier_keys(),
-					}),
+					Route::Editor => {
+						if !self.pointer_locked() {
+							ui_callback(InputEvent::pointer().position(*position).moved().modifiers(self.modifiers).observe_only().build());
+						}
+						editor_callback(InputMessage::PointerMove {
+							editor_mouse_state: match source {
+								PointerSource::TabletTool { kind, data } => self.tablet_pointer_state(kind, data),
+								_ => self.pointer_state(),
+							},
+							modifier_keys: self.modifier_keys(),
+						});
+					}
 				}
 			}
 			WindowEvent::PointerEntered { position, .. } => {
@@ -160,15 +177,19 @@ impl InputState {
 				let count = mouse_button.map_or(1, |button| self.click_tracker.input(*position, button, *state));
 
 				let back_or_forward = matches!(mouse_button, Some(MouseButton::Back | MouseButton::Forward));
+				let pointer = InputEvent::pointer().position(*position);
+				let input = match state {
+					ElementState::Pressed => pointer.pressed(button.clone(), count),
+					ElementState::Released => pointer.released(button.clone(), count),
+				}
+				.modifiers(self.modifiers);
 				if self.pointer_locked() || keys.is_empty() || !(back_or_forward || route == Route::Editor) {
-					let pointer = InputEvent::pointer().position(*position);
-					let input = match state {
-						ElementState::Pressed => pointer.pressed(button.clone(), count),
-						ElementState::Released => pointer.released(button.clone(), count),
-					};
-					ui_callback(input.modifiers(self.modifiers).build());
+					ui_callback(input.build());
 					return;
 				}
+				if route == Route::Editor {
+					ui_callback(input.observe_only().build());
+				}
 
 				let editor_mouse_state = match button {
 					ButtonSource::TabletTool { kind, data, .. } => self.tablet_pointer_state(kind, data),

frontend/src/utility-functions/input.ts

@@ -108,12 +108,19 @@ export async function onKeyUp(e: KeyboardEvent, editor: EditorWrapper, dialogSto
 
 // Pointer events
 
+// On desktop, num lock marks events as observe-only, do not redirect them to the editor.
+function isObserveOnly(e: MouseEvent): boolean {
+	return import.meta.env.MODE === "native" && e.getModifierState("NumLock");
+}
+
 // While any pointer button is already down, additional button down events are not reported, but they are sent as `pointermove` events and these are handled in the backend
 export function onPointerMove(e: PointerEvent, editor: EditorWrapper, documentStore: DocumentStore) {
 	potentiallyRestoreCanvasFocus(e);
 
 	if (!e.buttons) viewportPointerInteractionOngoing = false;
 
+	if (isObserveOnly(e)) return;
+
 	// Don't redirect pointer movement to the backend if there's no ongoing interaction and it's over a floating menu, or the graph overlay, on top of the canvas
 	// TODO: A better approach is to pass along a boolean to the backend's input preprocessor so it can know if it's being occluded by the GUI.
 	// TODO: This would allow it to properly decide to act on removing hover focus from something that was hovered in the canvas before moving over the GUI.
@@ -134,6 +141,8 @@ export function onPointerDown(e: PointerEvent, editor: EditorWrapper, dialogStor
 	potentiallyRestoreCanvasFocus(e);
 	potentiallyClearTextSelection(e);
 
+	if (isObserveOnly(e)) return;
+
 	const inFloatingMenu = e.target instanceof Element && e.target.closest("[data-floating-menu-content]");
 	const isTargetingCanvas = !inFloatingMenu && e.target instanceof Element && e.target.closest("[data-viewport], [data-viewport-container], [data-node-graph]");
 	const inDialog = e.target instanceof Element && e.target.closest("[data-dialog] [data-floating-menu-content]");
@@ -172,7 +181,7 @@ export function onPointerUp(e: PointerEvent, editor: EditorWrapper) {
 
 	if (!e.buttons) viewportPointerInteractionOngoing = false;
 
-	if (textToolInteractiveInputElement) return;
+	if (isObserveOnly(e) || textToolInteractiveInputElement) return;
 
 	const modifiers = makeKeyboardModifiersBitfield(e);
 	editor.onMouseUp(e.clientX, e.clientY, e.buttons, modifiers, ...pointerAttributes(e));
@@ -181,7 +190,7 @@ export function onPointerUp(e: PointerEvent, editor: EditorWrapper) {
 // Mouse events
 
 export function onPotentialDoubleClick(e: MouseEvent, editor: EditorWrapper) {
-	if (textToolInteractiveInputElement || inPointerLock) return;
+	if (isObserveOnly(e) || textToolInteractiveInputElement || inPointerLock) return;
 
 	// Allow only events within the viewport or node graph boundaries
 	const isTargetingCanvas = e.target instanceof Element && e.target.closest("[data-viewport], [data-viewport-container], [data-node-graph]");