Desktop: Support input from drawing tablets (#4354)

desktop/src/app.rs

@@ -9,19 +9,21 @@ use std::sync::mpsc::{Receiver, SyncSender};
 use std::thread;
 use std::time::{Duration, Instant};
 use winit::application::ApplicationHandler;
-use winit::dpi::{PhysicalPosition, PhysicalSize};
-use winit::event::{ButtonSource, ElementState, MouseButton, StartCause, WindowEvent};
+use winit::dpi::PhysicalSize;
+use winit::event::{ElementState, MouseButton, StartCause, WindowEvent};
+use winit::event_loop::run_on_demand::EventLoopExtRunOnDemand;
 use winit::event_loop::{ActiveEventLoop, ControlFlow, EventLoop};
 use winit::window::WindowId;
 
 use crate::dirs;
 use crate::event::{AppEvent, AppEventScheduler};
+use crate::input::{InputAction, InputState};
 use crate::persist;
 use crate::preferences;
 use crate::render::{RenderError, RenderState};
 use crate::ui::{UiCommand, UiInstance};
 use crate::window::Window;
-use crate::wrapper::messages::{DesktopFrontendMessage, DesktopWrapperMessage, InputMessage, MouseKeys, MouseState, Preferences};
+use crate::wrapper::messages::{DesktopFrontendMessage, DesktopWrapperMessage, Preferences};
 use crate::wrapper::{DesktopWrapper, MmapResourceStorage, NodeGraphExecutionResult, WgpuContext, serialize_frontend_messages};
 
 pub(crate) struct App {
@@ -33,8 +35,7 @@ pub(crate) struct App {
 	window_maximized: bool,
 	window_fullscreen: bool,
 	window_pending_drag: bool,
-	pointer_position: PhysicalPosition<f64>,
-	pointer_lock_position: Option<PhysicalPosition<f64>>,
+	input_state: InputState,
 	ui_scale: f64,
 	app_event_receiver: Receiver<AppEvent>,
 	app_event_scheduler: AppEventScheduler,
@@ -111,8 +112,7 @@ impl App {
 			window_maximized: false,
 			window_fullscreen: false,
 			window_pending_drag: false,
-			pointer_position: Default::default(),
-			pointer_lock_position: Default::default(),
+			input_state: InputState::new(),
 			ui_scale: 1.,
 			app_event_receiver,
 			app_event_scheduler,
@@ -130,8 +130,8 @@ impl App {
 		}
 	}
 
-	pub(crate) fn run(mut self, event_loop: EventLoop) -> ExitReason {
-		event_loop.run_app(&mut self).unwrap();
+	pub(crate) fn run(mut self, mut event_loop: EventLoop) -> ExitReason {
+		event_loop.run_app_on_demand(&mut self).unwrap();
 		self.exit_reason
 	}
 
@@ -269,6 +269,8 @@ impl App {
 				});
 			}
 			DesktopFrontendMessage::UpdateViewportPhysicalBounds { x, y, width, height } => {
+				self.input_state.set_viewport_info(x, y, width, height, self.window_scale);
+
 				if let Some(render_state) = &mut self.render_state
 					&& let Some(window) = &self.window
 				{
@@ -347,7 +349,7 @@ impl App {
 				}
 			}
 			DesktopFrontendMessage::PointerLock => {
-				self.pointer_lock_position = Some(self.pointer_position);
+				self.input_state.lock_pointer();
 				if let Some(window) = &self.window {
 					window.start_pointer_lock();
 				}
@@ -540,14 +542,13 @@ impl ApplicationHandler for App {
 
 	fn window_event(&mut self, _event_loop: &dyn ActiveEventLoop, _window_id: WindowId, event: WindowEvent) {
 		// Handle pointer lock release
-		if let Some(pointer_lock_position) = self.pointer_lock_position
-			&& let WindowEvent::PointerButton {
-				state: ElementState::Released,
-				button: ButtonSource::Mouse(MouseButton::Left),
-				..
-			} = event
+		if let WindowEvent::PointerButton {
+			state: ElementState::Released,
+			button,
+			..
+		} = &event && button.clone().mouse_button() == Some(MouseButton::Left)
+			&& let Some(pointer_lock_position) = self.input_state.unlock_pointer()
 		{
-			self.pointer_lock_position = None;
 			if let Some(window) = &self.window {
 				window.end_pointer_lock();
 			}
@@ -559,7 +560,12 @@ impl ApplicationHandler for App {
 			}));
 		}
 
-		self.ui.send(UiCommand::Input(event.clone()));
+		for action in self.input_state.process(&event) {
+			match action {
+				InputAction::Ui(event) => self.ui.send(UiCommand::Input(event)),
+				InputAction::Editor(message) => self.app_event_scheduler.schedule(AppEvent::DesktopWrapperMessage(message)),
+			}
+		}
 
 		match event {
 			WindowEvent::CloseRequested => {
@@ -616,50 +622,20 @@ impl ApplicationHandler for App {
 				}
 			}
 
-			// Forward and Back buttons are not supported by CEF and thus need to be directly forwarded the editor
-			WindowEvent::PointerButton {
-				button: ButtonSource::Mouse(button),
-				state: ElementState::Pressed,
-				..
-			} => {
-				let mouse_keys = match button {
-					MouseButton::Back => Some(MouseKeys::BACK),
-					MouseButton::Forward => Some(MouseKeys::FORWARD),
-					_ => None,
-				};
-				if let Some(mouse_keys) = mouse_keys {
-					let message = DesktopWrapperMessage::Input(InputMessage::PointerDown {
-						editor_mouse_state: MouseState { mouse_keys, ..Default::default() },
-						modifier_keys: Default::default(),
-					});
-					self.app_event_scheduler.schedule(AppEvent::DesktopWrapperMessage(message));
-
-					let message = DesktopWrapperMessage::Input(InputMessage::PointerUp {
-						editor_mouse_state: Default::default(),
-						modifier_keys: Default::default(),
-					});
-					self.app_event_scheduler.schedule(AppEvent::DesktopWrapperMessage(message));
-				}
-			}
-
-			WindowEvent::PointerMoved { position, .. } | WindowEvent::PointerLeft { position: Some(position), .. } | WindowEvent::PointerEntered { position, .. }
-				if self.pointer_lock_position.is_none() =>
+			WindowEvent::PointerMoved { .. } | WindowEvent::PointerLeft { position: Some(_), .. } | WindowEvent::PointerEntered { .. }
+				if !self.input_state.pointer_locked() && self.window_pending_drag =>
 			{
-				self.pointer_position = position;
-
-				if self.window_pending_drag {
-					self.window_pending_drag = false;
-					if let Some(window) = &self.window {
-						window.start_drag();
-					}
+				self.window_pending_drag = false;
+				if let Some(window) = &self.window {
+					window.start_drag();
 				}
 			}
 
 			WindowEvent::PointerButton {
-				button: ButtonSource::Mouse(MouseButton::Left),
+				button,
 				state: ElementState::Released,
 				..
-			} => {
+			} if button.clone().mouse_button() == Some(MouseButton::Left) => {
 				self.window_pending_drag = false;
 			}
 
@@ -668,7 +644,7 @@ impl ApplicationHandler for App {
 	}
 
 	fn device_event(&mut self, _event_loop: &dyn ActiveEventLoop, _device_id: Option<winit::event::DeviceId>, event: winit::event::DeviceEvent) {
-		if self.pointer_lock_position.is_some()
+		if self.input_state.pointer_locked()
 			&& let winit::event::DeviceEvent::PointerMotion { delta: (x, y) } = event
 		{
 			let message = DesktopWrapperMessage::PointerLockMove { x, y };

desktop/src/input.rs

@@ -0,0 +1,242 @@
+use std::time::Instant;
+use winit::dpi::PhysicalPosition;
+use winit::event::{ButtonSource, ElementState, MouseButton, MouseScrollDelta, PointerSource, TabletToolData, TabletToolKind, WindowEvent};
+
+use crate::ui::{MULTICLICK_ALLOWED_TRAVEL, MULTICLICK_TIMEOUT, PINCH_ZOOM_SPEED, SCROLL_LINE_HEIGHT, SCROLL_LINE_WIDTH, SCROLL_SPEED_X, SCROLL_SPEED_Y};
+use crate::wrapper::messages::{DesktopWrapperMessage, InputMessage, ModifierKeys, MouseKeys, PointerState, ScrollDelta};
+
+pub(crate) struct InputState {
+	viewport_info: Option<ViewportInfo>,
+	pointer_lock_position: Option<PhysicalPosition<f64>>,
+	modifier_keys: ModifierKeys,
+	pointer_position: PhysicalPosition<f64>,
+	pointer_keys: MouseKeys,
+	ui_capture: bool,
+	multiclick: Option<Multiclick>,
+}
+
+pub(crate) enum InputAction {
+	Ui(WindowEvent),
+	Editor(DesktopWrapperMessage),
+}
+
+impl InputAction {
+	fn editor(message: InputMessage) -> Self {
+		Self::Editor(DesktopWrapperMessage::Input(message))
+	}
+}
+
+impl InputState {
+	pub(crate) fn new() -> Self {
+		Self {
+			viewport_info: None,
+			pointer_lock_position: None,
+			modifier_keys: ModifierKeys::empty(),
+			pointer_position: PhysicalPosition::default(),
+			pointer_keys: MouseKeys::empty(),
+			ui_capture: true,
+			multiclick: None,
+		}
+	}
+
+	pub(crate) fn set_viewport_info(&mut self, x: f64, y: f64, width: f64, height: f64, scale: f64) {
+		self.viewport_info = Some(ViewportInfo { x, y, width, height, scale });
+	}
+
+	pub(crate) fn lock_pointer(&mut self) {
+		self.pointer_lock_position = Some(self.pointer_position);
+	}
+
+	pub(crate) fn unlock_pointer(&mut self) -> Option<PhysicalPosition<f64>> {
+		let position = self.pointer_lock_position.take();
+		if let Some(position) = position {
+			self.pointer_position = position;
+		}
+		position
+	}
+
+	pub(crate) fn pointer_locked(&self) -> bool {
+		self.pointer_lock_position.is_some()
+	}
+
+	pub(crate) fn process(&mut self, event: &WindowEvent) -> Vec<InputAction> {
+		match event {
+			WindowEvent::PointerMoved { position, source, .. } => {
+				self.pointer_position = *position;
+
+				let PointerSource::TabletTool { kind, data } = source else {
+					return vec![InputAction::Ui(event.clone())];
+				};
+				let ui_capture = if self.pointer_keys.is_empty() {
+					self.pointer_locked() || !self.in_viewport(*position)
+				} else {
+					self.ui_capture
+				};
+				if ui_capture {
+					return vec![InputAction::Ui(event.clone())];
+				}
+
+				vec![InputAction::editor(InputMessage::PointerMove {
+					editor_mouse_state: self.tablet_pointer_state(kind, data),
+					modifier_keys: self.modifier_keys,
+				})]
+			}
+			WindowEvent::PointerEntered { position, .. } | WindowEvent::PointerLeft { position: Some(position), .. } => {
+				self.pointer_position = *position;
+				vec![InputAction::Ui(event.clone())]
+			}
+			WindowEvent::PointerButton { state, button, position, .. } => {
+				self.pointer_position = *position;
+
+				let tablet = matches!(button, ButtonSource::TabletTool { .. });
+
+				// Stroke keeps capture decided from first button press until all buttons are released.
+				if state.is_pressed() && self.pointer_keys.is_empty() {
+					self.ui_capture = self.pointer_locked() || !tablet || !self.in_viewport(*position);
+				}
+
+				let mouse_button = button.clone().mouse_button();
+				let keys = match mouse_button {
+					Some(MouseButton::Left) => MouseKeys::LEFT,
+					Some(MouseButton::Right) => MouseKeys::RIGHT,
+					Some(MouseButton::Middle) => MouseKeys::MIDDLE,
+					Some(MouseButton::Back) => MouseKeys::BACK,
+					Some(MouseButton::Forward) => MouseKeys::FORWARD,
+					_ => MouseKeys::NONE,
+				};
+				match state {
+					ElementState::Pressed => self.pointer_keys.insert(keys),
+					ElementState::Released => self.pointer_keys.remove(keys),
+				}
+
+				let back_or_forward = matches!(mouse_button, Some(MouseButton::Back | MouseButton::Forward));
+				if self.pointer_locked() || !(back_or_forward || (tablet && !self.ui_capture)) {
+					return vec![InputAction::Ui(event.clone())];
+				}
+
+				let editor_mouse_state = match button {
+					ButtonSource::TabletTool { kind, data, .. } => self.tablet_pointer_state(kind, data),
+					_ => self.pointer_state(),
+				};
+				let modifier_keys = self.modifier_keys;
+				match state {
+					ElementState::Pressed => vec![InputAction::editor(InputMessage::PointerDown { editor_mouse_state, modifier_keys })],
+					ElementState::Released => {
+						let mut actions = vec![InputAction::editor(InputMessage::PointerUp { editor_mouse_state, modifier_keys })];
+						if let Some(mouse_button) = mouse_button
+							&& self.track_multiclick(mouse_button, *position)
+						{
+							actions.push(InputAction::editor(InputMessage::DoubleClick {
+								editor_mouse_state: PointerState {
+									mouse_keys: keys,
+									..editor_mouse_state
+								},
+								modifier_keys,
+							}));
+						}
+						actions
+					}
+				}
+			}
+			WindowEvent::MouseWheel { delta, .. } => {
+				if self.pointer_locked() || !self.in_viewport(self.pointer_position) {
+					return vec![InputAction::Ui(event.clone())];
+				}
+
+				let (x, y) = match delta {
+					MouseScrollDelta::LineDelta(x, y) => (f64::from(*x) * SCROLL_LINE_WIDTH, f64::from(*y) * SCROLL_LINE_HEIGHT),
+					MouseScrollDelta::PixelDelta(position) => (position.x, position.y),
+				};
+
+				let scroll_delta = ScrollDelta::new(-x * SCROLL_SPEED_X, -y * SCROLL_SPEED_Y, 0.);
+
+				vec![InputAction::editor(InputMessage::WheelScroll {
+					editor_mouse_state: PointerState { scroll_delta, ..self.pointer_state() },
+					modifier_keys: self.modifier_keys,
+				})]
+			}
+			WindowEvent::PinchGesture { delta, .. } => {
+				if self.pointer_locked() || !self.in_viewport(self.pointer_position) || !delta.is_normal() {
+					return vec![InputAction::Ui(event.clone())];
+				}
+
+				// TODO: This is a temporary solution to handle pinch gestures, we should handle pinch gestures editor-side instead.
+				let scroll_delta = ScrollDelta::new(0., -delta * PINCH_ZOOM_SPEED, 0.);
+				vec![InputAction::editor(InputMessage::WheelScroll {
+					editor_mouse_state: PointerState { scroll_delta, ..self.pointer_state() },
+					modifier_keys: self.modifier_keys | ModifierKeys::CONTROL,
+				})]
+			}
+			WindowEvent::ModifiersChanged(modifiers) => {
+				let modifiers = modifiers.state();
+				self.modifier_keys = ModifierKeys::empty();
+				self.modifier_keys.set(ModifierKeys::SHIFT, modifiers.shift_key());
+				self.modifier_keys.set(ModifierKeys::CONTROL, modifiers.control_key());
+				self.modifier_keys.set(ModifierKeys::ALT, modifiers.alt_key());
+				self.modifier_keys.set(ModifierKeys::META_OR_COMMAND, modifiers.meta_key());
+
+				vec![InputAction::Ui(event.clone())]
+			}
+			_ => vec![InputAction::Ui(event.clone())],
+		}
+	}
+
+	fn scale(&self) -> f64 {
+		self.viewport_info.as_ref().map_or(1., |info| info.scale)
+	}
+
+	fn in_viewport(&self, position: PhysicalPosition<f64>) -> bool {
+		self.viewport_info.as_ref().is_some_and(|info| info.contains(position))
+	}
+
+	fn pointer_state(&self) -> PointerState {
+		PointerState {
+			editor_position: (self.pointer_position.x / self.scale(), self.pointer_position.y / self.scale()).into(),
+			mouse_keys: self.pointer_keys,
+			..Default::default()
+		}
+	}
+
+	fn tablet_pointer_state(&self, kind: &TabletToolKind, data: &TabletToolData) -> PointerState {
+		PointerState {
+			pressure: data.force.map(|force| force.normalized(None)),
+			tilt: data.clone().tilt().map(|tilt| (f64::from(tilt.x), f64::from(tilt.y)).into()),
+			twist: data.twist.map(f64::from),
+			wheel: data.tangential_force.map(f64::from),
+			eraser: matches!(kind, TabletToolKind::Eraser),
+			..self.pointer_state()
+		}
+	}
+
+	fn track_multiclick(&mut self, button: MouseButton, position: PhysicalPosition<f64>) -> bool {
+		let now = Instant::now();
+		let travel = MULTICLICK_ALLOWED_TRAVEL as f64;
+		let double = self.multiclick.take().is_some_and(|click| {
+			click.button == button && now.duration_since(click.time) <= MULTICLICK_TIMEOUT && (position.x - click.position.x).abs() <= travel && (position.y - click.position.y).abs() <= travel
+		});
+		if !double {
+			self.multiclick = Some(Multiclick { button, time: now, position });
+		}
+		double
+	}
+}
+
+struct ViewportInfo {
+	x: f64,
+	y: f64,
+	width: f64,
+	height: f64,
+	scale: f64,
+}
+
+impl ViewportInfo {
+	fn contains(&self, position: PhysicalPosition<f64>) -> bool {
+		position.x >= self.x && position.y >= self.y && position.x <= self.x + self.width && position.y <= self.y + self.height
+	}
+}
+
+struct Multiclick {
+	button: MouseButton,
+	time: Instant,
+	position: PhysicalPosition<f64>,
+}

desktop/src/lib.rs

@@ -17,6 +17,7 @@ mod cli;
 mod dirs;
 mod event;
 mod gpu_context;
+mod input;
 mod persist;
 mod preferences;
 mod render;

desktop/src/window/win/native_handle.rs

@@ -344,6 +344,14 @@ unsafe fn position_helper(main: HWND, helper: HWND) {
 	let h = (r.bottom - r.top) + RESIZE_BAND_THICKNESS * 2;
 
 	let _ = unsafe { SetWindowPos(helper, Some(main), x, y, w, h, SWP_NOACTIVATE | SWP_NOSENDCHANGING) };
+
+	let outer = unsafe { CreateRectRgn(0, 0, w, h) };
+	let inner = unsafe { CreateRectRgn(RESIZE_BAND_THICKNESS, RESIZE_BAND_THICKNESS, w - RESIZE_BAND_THICKNESS, h - RESIZE_BAND_THICKNESS) };
+	let _ = unsafe { CombineRgn(Some(outer), Some(outer), Some(inner), RGN_DIFF) };
+	let _ = unsafe { DeleteObject(inner.into()) };
+	if unsafe { SetWindowRgn(helper, Some(outer), false) } == 0 {
+		let _ = unsafe { DeleteObject(outer.into()) };
+	}
 }
 
 unsafe fn calculate_hit(helper: HWND, lparam: LPARAM) -> u32 {