Desktop: Make input routing aware of floating menus (#4498)

* Desktop: Make input routing aware of floating menus

* Desktop: Route mouse input like pen input

* Desktop: Model pointer input routing as a state machine

frontend/src/components/layout/FloatingMenu.svelte

@@ -17,10 +17,11 @@
 </script>
 
 <script lang="ts">
-	import { onMount, onDestroy, afterUpdate, createEventDispatcher, tick } from "svelte";
+	import { getContext, onMount, onDestroy, afterUpdate, createEventDispatcher, tick } from "svelte";
 	import LayoutCol from "/src/components/layout/LayoutCol.svelte";
+	import { onFloatingMenuOpenChange } from "/src/utility-functions/input";
 	import { browserVersion } from "/src/utility-functions/platform";
-	import type { MenuDirection } from "/wrapper/pkg/graphite_wasm_wrapper";
+	import type { EditorWrapper, MenuDirection } from "/wrapper/pkg/graphite_wasm_wrapper";
 
 	const BUTTON_LEFT = 0;
 	const POINTER_STRAY_DISTANCE = 100;
@@ -42,6 +43,9 @@
 	export let escapeCloses = true;
 	export let strayCloses = true;
 
+	const editor = getContext<EditorWrapper>("editor");
+	const menuId = String(Math.random()).substring(2);
+
 	let tail: HTMLDivElement | undefined;
 	let self: HTMLDivElement | undefined;
 	let floatingMenuContainer: HTMLDivElement | undefined;
@@ -65,6 +69,8 @@
 
 	$: watchOpenChange(open);
 
+	$: onFloatingMenuOpenChange(menuId, open && type !== "Tooltip" && type !== "Cursor", editor);
+
 	$: minWidthStyleValue = measuringOngoing ? "0" : `${Math.max(minWidth, minWidthParentWidth)}px`;
 	$: displayTail = open && type === "Popover";
 	$: displayContainer = open || measuringOngoing;
@@ -157,6 +163,7 @@
 	});
 
 	onDestroy(() => {
+		onFloatingMenuOpenChange(menuId, false, editor);
 		containerResizeObserver.disconnect();
 		dialogResizeObserver?.disconnect();
 		window.removeEventListener("pointermove", pointerMoveHandler);

frontend/src/components/panels/Document.svelte

@@ -15,6 +15,7 @@
 	import type { MessageBody } from "/src/subscriptions-router";
 	import { fillChoiceColor, createSRgba8 } from "/src/utility-functions/colors";
 	import { pasteFile } from "/src/utility-functions/files";
+	import { cleanupInputField } from "/src/utility-functions/input";
 	import { textInputCleanup } from "/src/utility-functions/keyboard-entry";
 	import { rasterizeSVGCanvas } from "/src/utility-functions/rasterization";
 	import { setupViewportResizeObserver, hasFirstArtworkBeenReceived, markFirstArtworkReceived } from "/src/utility-functions/viewports";
@@ -566,6 +567,7 @@
 		viewportResizeObserver?.disconnect();
 		removeUpdatePixelRatio?.();
 		addedFontFaces.forEach((face) => window.document.fonts.delete(face));
+		cleanupInputField(editor);
 
 		subscriptions.unsubscribeFrontendMessage("UpdateDocumentArtwork");
 		subscriptions.unsubscribeFrontendMessage("UpdateEyedropperSamplingState");

frontend/src/managers/input.ts

@@ -19,6 +19,7 @@ import {
 	onContextMenu,
 	onPaste,
 	onPointerLockChange,
+	updateDirectInput,
 } from "/src/utility-functions/input";
 import type { EditorWrapper } from "/wrapper/pkg/graphite_wasm_wrapper";
 
@@ -42,7 +43,7 @@ const listeners: Listener[] = [
 	{ target: window, eventName: "mousedown", action: (e: MouseEvent) => onMouseDown(e) },
 	{ target: window, eventName: "mouseup", action: (e: MouseEvent) => editorWrapper && onPotentialDoubleClick(e, editorWrapper) },
 	{ target: window, eventName: "wheel", action: (e: WheelEvent) => editorWrapper && onWheelScroll(e, editorWrapper), options: { passive: false } },
-	{ target: window, eventName: "modifyinputfield", action: (e: CustomEvent) => onModifyInputField(e) },
+	{ target: window, eventName: "modifyinputfield", action: (e: CustomEvent) => editorWrapper && onModifyInputField(e, editorWrapper) },
 	{ target: window, eventName: "focusout", action: () => onFocusOut() },
 	{ target: window.document, eventName: "contextmenu", action: (e: MouseEvent) => onContextMenu(e) },
 	{ target: window.document, eventName: "fullscreenchange", action: () => fullscreenModeChanged() },
@@ -79,6 +80,8 @@ export function createInputManager(subscriptions: SubscriptionsRouter, editor: E
 	// Add event bindings for the lifetime of the application
 	listeners.forEach(({ target, eventName, action, options }) => target.addEventListener(eventName, action, options));
 
+	updateDirectInput(editor);
+
 	// Focus the app container
 	const app = window.document.querySelector("[data-app-container]");
 	if (app instanceof HTMLElement) app.focus();

frontend/src/utility-functions/input.ts

@@ -20,6 +20,7 @@ let canvasFocused = true;
 let inPointerLock = false;
 let lastShakeTime = 0;
 const shakeSamples: { x: number; y: number; time: number }[] = [];
+const openFloatingMenus = new Set<string>();
 
 // Keyboard events
 
@@ -243,8 +244,14 @@ export function onWheelScroll(e: WheelEvent, editor: EditorWrapper) {
 
 // Receives a custom event dispatched when the user begins interactively editing with the text tool.
 // We keep a copy of the text input element to check against when it's active for text entry.
-export function onModifyInputField(e: CustomEvent) {
+export function onModifyInputField(e: CustomEvent, editor: EditorWrapper) {
 	textToolInteractiveInputElement = e.detail;
+	updateDirectInput(editor);
+}
+
+export function cleanupInputField(editor: EditorWrapper) {
+	textToolInteractiveInputElement = undefined;
+	updateDirectInput(editor);
 }
 
 // Window events
@@ -387,3 +394,20 @@ function potentiallyRestoreCanvasFocus(e: Event) {
 		app?.focus();
 	}
 }
+
+// Direct input (desktop only)
+
+// Desktop may route viewport pointer input directly to the editor if no floating menus are open
+export function updateDirectInput(editor: EditorWrapper) {
+	if (import.meta.env.MODE !== "native") return;
+
+	const uiCapturesViewport = openFloatingMenus.size > 0 || textToolInteractiveInputElement !== undefined;
+	editor.appWindowDirectInput(!uiCapturesViewport);
+}
+
+export function onFloatingMenuOpenChange(menuId: string, open: boolean, editor: EditorWrapper) {
+	if (open) openFloatingMenus.add(menuId);
+	else openFloatingMenus.delete(menuId);
+
+	updateDirectInput(editor);
+}