Support input from drawing tablets on web (#4445)

* Rename the editor mouse state types to Pointer* and the input_ modules

* Add optional pen attributes to the editor pointer state types

* Desktop: Send tablet pointer clicks to CEF as mouse clicks

* Desktop: End pointer lock and pending window drags on any left-equivalent release

* Desktop: Route pointer input carrying pen attributes directly to the editor

* Desktop: Route wheel and pinch input inside the viewport directly to the editor

* Desktop: Track double clicks on the direct editor route

* Desktop: Move the pointer lock position into InputState

* Restore the input state pointer position when pointer lock ends (review)

* Fix pen input being swallowed by Windows resize helper

* Update winit to 0.31.0-beta.2 from crates.io

* Add event time to the editor pointer states

The brush trace's hot core is position plus per-sample time, and the
frame clock is too coarse for high-rate tablets: several samples share
one timestamp and speed dynamics see dt = 0. The desktop stamps
editor-routed pointer input with a monotonic arrival time since winit
does not surface hardware timestamps; the web path can thread the
PointerEvent timeStamp through later.

* Capture pen input via browser api

* Replay coalesced pointer samples for smoother pen strokes

* Handle missing or empty coalesced pen events

frontend/src/utility-functions/input.ts

@@ -123,7 +123,10 @@ export function onPointerMove(e: PointerEvent, editor: EditorWrapper, documentSt
 
 	const modifiers = makeKeyboardModifiersBitfield(e);
 	if (detectShake(e)) editor.onMouseShake(e.clientX, e.clientY, e.buttons, modifiers);
-	editor.onMouseMove(e.clientX, e.clientY, e.buttons, modifiers);
+
+	const coalesced = e.pointerType === "pen" && typeof e.getCoalescedEvents === "function" ? e.getCoalescedEvents() : [];
+	const samples = coalesced.length > 0 ? coalesced : [e];
+	for (const sample of samples) editor.onMouseMove(sample.clientX, sample.clientY, sample.buttons, modifiers, ...pointerAttributes(sample));
 }
 
 export function onPointerDown(e: PointerEvent, editor: EditorWrapper, dialogStore: DialogStore) {
@@ -153,7 +156,7 @@ export function onPointerDown(e: PointerEvent, editor: EditorWrapper, dialogStor
 
 	if (viewportPointerInteractionOngoing && isTargetingCanvas instanceof Element) {
 		const modifiers = makeKeyboardModifiersBitfield(e);
-		editor.onMouseDown(e.clientX, e.clientY, e.buttons, modifiers);
+		editor.onMouseDown(e.clientX, e.clientY, e.buttons, modifiers, ...pointerAttributes(e));
 	}
 }
 
@@ -171,7 +174,7 @@ export function onPointerUp(e: PointerEvent, editor: EditorWrapper) {
 	if (textToolInteractiveInputElement) return;
 
 	const modifiers = makeKeyboardModifiersBitfield(e);
-	editor.onMouseUp(e.clientX, e.clientY, e.buttons, modifiers);
+	editor.onMouseUp(e.clientX, e.clientY, e.buttons, modifiers, ...pointerAttributes(e));
 }
 
 // Mouse events
@@ -278,6 +281,12 @@ export function onFocusOut() {
 	canvasFocused = false;
 }
 
+function pointerAttributes(e: PointerEvent): [number, number | undefined, number | undefined, number | undefined, number | undefined, number | undefined, boolean] {
+	const isPen = e.pointerType === "pen";
+	const eraser = isPen && (e.buttons & 0b10_0000) !== 0; // Pen eraser end is reported as the 32 button bit
+	return [e.timeStamp, isPen ? e.pressure : undefined, isPen ? e.tiltX : undefined, isPen ? e.tiltY : undefined, isPen ? e.twist : undefined, isPen ? e.tangentialPressure : undefined, eraser];
+}
+
 function detectShake(e: PointerEvent | MouseEvent): boolean {
 	const SENSITIVITY_DIRECTION_CHANGES = 3;
 	const SENSITIVITY_DISTANCE_TO_DISPLACEMENT_RATIO = 0.1;

frontend/wrapper/src/editor_commands.rs

@@ -299,8 +299,31 @@ mod editor_commands {
 	}
 
 	/// Mouse movement within the screenspace bounds of the viewport
-	fn on_mouse_move(x: f64, y: f64, mouse_keys: u8, modifiers: u8) -> Message {
+	fn on_mouse_move(
-		let editor_mouse_state = EditorPointerState::from_keys_and_editor_position(mouse_keys, (x, y).into());
+		x: f64,
+		y: f64,
+		mouse_keys: u8,
+		modifiers: u8,
+		time: Option<f64>,
+		pressure: Option<f64>,
+		tilt_x: Option<f64>,
+		tilt_y: Option<f64>,
+		twist: Option<f64>,
+		tangential: Option<f64>,
+		eraser: bool,
+	) -> Message {
+		let editor_mouse_state = EditorPointerState {
+			time,
+			pressure,
+			tilt: match (tilt_x, tilt_y) {
+				(Some(tilt_x), Some(tilt_y)) => Some((tilt_x, tilt_y).into()),
+				_ => None,
+			},
+			twist,
+			wheel: tangential,
+			eraser,
+			..EditorPointerState::from_keys_and_editor_position(mouse_keys, (x, y).into())
+		};
 		let modifier_keys = ModifierKeys::from_bits(modifiers).expect("Invalid modifier keys");
 		InputPreprocessorMessage::PointerMove { editor_mouse_state, modifier_keys }.into()
 	}
@@ -314,15 +337,61 @@ mod editor_commands {
 	}
 
 	/// A mouse button depressed within screenspace the bounds of the viewport
-	fn on_mouse_down(x: f64, y: f64, mouse_keys: u8, modifiers: u8) -> Message {
+	fn on_mouse_down(
-		let editor_mouse_state = EditorPointerState::from_keys_and_editor_position(mouse_keys, (x, y).into());
+		x: f64,
+		y: f64,
+		mouse_keys: u8,
+		modifiers: u8,
+		time: Option<f64>,
+		pressure: Option<f64>,
+		tilt_x: Option<f64>,
+		tilt_y: Option<f64>,
+		twist: Option<f64>,
+		tangential: Option<f64>,
+		eraser: bool,
+	) -> Message {
+		let editor_mouse_state = EditorPointerState {
+			time,
+			pressure,
+			tilt: match (tilt_x, tilt_y) {
+				(Some(tilt_x), Some(tilt_y)) => Some((tilt_x, tilt_y).into()),
+				_ => None,
+			},
+			twist,
+			wheel: tangential,
+			eraser,
+			..EditorPointerState::from_keys_and_editor_position(mouse_keys, (x, y).into())
+		};
 		let modifier_keys = ModifierKeys::from_bits(modifiers).expect("Invalid modifier keys");
 		InputPreprocessorMessage::PointerDown { editor_mouse_state, modifier_keys }.into()
 	}
 
 	/// A mouse button released
-	fn on_mouse_up(x: f64, y: f64, mouse_keys: u8, modifiers: u8) -> Message {
+	fn on_mouse_up(
-		let editor_mouse_state = EditorPointerState::from_keys_and_editor_position(mouse_keys, (x, y).into());
+		x: f64,
+		y: f64,
+		mouse_keys: u8,
+		modifiers: u8,
+		time: Option<f64>,
+		pressure: Option<f64>,
+		tilt_x: Option<f64>,
+		tilt_y: Option<f64>,
+		twist: Option<f64>,
+		tangential: Option<f64>,
+		eraser: bool,
+	) -> Message {
+		let editor_mouse_state = EditorPointerState {
+			time,
+			pressure,
+			tilt: match (tilt_x, tilt_y) {
+				(Some(tilt_x), Some(tilt_y)) => Some((tilt_x, tilt_y).into()),
+				_ => None,
+			},
+			twist,
+			wheel: tangential,
+			eraser,
+			..EditorPointerState::from_keys_and_editor_position(mouse_keys, (x, y).into())
+		};
 		let modifier_keys = ModifierKeys::from_bits(modifiers).expect("Invalid modifier keys");
 		InputPreprocessorMessage::PointerUp { editor_mouse_state, modifier_keys }.into()
 	}