Desktop: Restructure UI input handling (#4500)

desktop/ui/src/context.rs

@@ -223,6 +223,7 @@ fn create_browser(delegate: BrowserDelegate, frames: FrameStreamer, view_info_se
 	.map(|browser| BrowserContext {
 		delegate,
 		browser,
+		input: input::InputState::default(),
 		view_info_sender,
 		_instance_dir: instance_dir,
 	})
@@ -252,12 +253,8 @@ pub(crate) enum InitError {
 pub(crate) struct CefContextHandle;
 
 impl CefContextHandle {
-	pub(crate) fn apply_input(&self, events: Vec<InputEvent>) {
-		with_context(move |context| {
-			for event in &events {
-				input::apply(&context.browser, event);
-			}
-		});
+	pub(crate) fn process_input(&self, event: InputEvent) {
+		with_context(move |context| input::process(&mut context.input, &context.browser, &event));
 	}
 
 	pub(crate) fn update_view_info(&self, update: ViewInfoUpdate) {
@@ -276,6 +273,7 @@ impl CefContextHandle {
 struct BrowserContext {
 	delegate: BrowserDelegate,
 	browser: Browser,
+	input: input::InputState,
 	view_info_sender: Sender<ViewInfoUpdate>,
 	_instance_dir: TempDir,
 }

desktop/ui/src/input.rs

@@ -1,253 +1,235 @@
-use cef::sys::{cef_key_event_type_t, cef_mouse_button_type_t};
-use cef::{Browser, ImplBrowser, ImplBrowserHost, KeyEvent, MouseEvent};
-use winit::event::{ButtonSource, ElementState, MouseButton, MouseScrollDelta, WindowEvent};
+pub(crate) mod event;
+pub(crate) use event::InputEvent;
 
 mod keymap;
 use keymap::{ToCharRepresentation, ToNativeKeycode, ToVKBits};
 
-mod state;
-pub(crate) use state::{CefModifiers, InputState};
+use cef::sys::{cef_event_flags_t, cef_key_event_type_t, cef_mouse_button_type_t};
+use cef::{Browser, ImplBrowser, ImplBrowserHost, KeyEvent, MouseEvent};
+use winit::dpi::PhysicalPosition;
+use winit::keyboard::{Key, KeyLocation, NamedKey};
 
 use super::consts::{PINCH_ZOOM_SPEED, SCROLL_LINE_HEIGHT, SCROLL_LINE_WIDTH, SCROLL_SPEED_X, SCROLL_SPEED_Y};
+use event::{InputEventKind, KeyAction, Modifiers, MouseButton, PointerAction};
 
-/// A window input translated into the plain data CEF consumes — no winit types, so it can
-/// be applied to a browser living in another process.
-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
-pub(crate) enum InputEvent {
-	MouseMove { data: MouseData, leave: bool },
-	MouseClick { data: MouseData, button: MouseButtonKind, up: bool, click_count: i32 },
-	MouseWheel { data: MouseData, delta_x: i32, delta_y: i32 },
-	Key(KeyData),
+#[derive(Default)]
+pub(crate) struct InputState {
+	position: PhysicalPosition<f64>,
+	buttons: ButtonStates,
 }
 
-#[derive(Clone, Copy, Debug, serde::Serialize, serde::Deserialize)]
-pub(crate) struct MouseData {
-	pub(crate) x: i32,
-	pub(crate) y: i32,
-	pub(crate) modifiers: u32,
+impl InputState {
+	fn pointer_move(&mut self, position: PhysicalPosition<f64>) -> bool {
+		let moved = (position.x as i32, position.y as i32) != (self.position.x as i32, self.position.y as i32);
+		self.position = position;
+		moved
+	}
 }
 
-#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
-pub(crate) enum MouseButtonKind {
-	Left,
-	Right,
-	Middle,
-}
+pub(crate) fn process(state: &mut InputState, browser: &Browser, event: &InputEvent) {
+	let Some(host) = browser.host() else { return };
+	match &event.kind {
+		InputEventKind::Pointer { position, action } => {
+			let position = position.unwrap_or(state.position);
+			match *action {
+				PointerAction::Move => {
+					if !state.pointer_move(position) {
+						return;
+					}
+					let flags = event_flags(&event.modifiers, &state.buttons);
+					host.send_mouse_move_event(Some(&mouse_event(position, flags)), 0);
+				}
+				PointerAction::Enter => {
+					state.pointer_move(position);
+					let flags = event_flags(&event.modifiers, &state.buttons);
+					host.send_mouse_move_event(Some(&mouse_event(position, flags)), 0);
+				}
+				PointerAction::Exit => {
+					state.pointer_move(position);
+					let flags = event_flags(&event.modifiers, &state.buttons);
+					host.send_mouse_move_event(Some(&mouse_event(position, flags)), 1);
+				}
+				PointerAction::Press { button, count } | PointerAction::Release { button, count } => {
+					state.pointer_move(position);
+					let cef_button = match button {
+						MouseButton::Left => cef_mouse_button_type_t::MBT_LEFT,
+						MouseButton::Right => cef_mouse_button_type_t::MBT_RIGHT,
+						MouseButton::Middle => cef_mouse_button_type_t::MBT_MIDDLE,
+						MouseButton::Unknown => return,
+					};
+					let up = matches!(action, PointerAction::Release { .. });
+					state.buttons.update(button, !up);
 
-#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
-pub(crate) enum KeyEventKind {
-	RawKeyDown,
-	KeyUp,
-	Char,
-}
+					// CEF only understands single, double and triple clicks; further clicks alternate between double and triple.
+					let count = match count {
+						0 | 1 => 1,
+						count if count % 2 == 0 => 2,
+						_ => 3,
+					};
 
-#[derive(Clone, Copy, Debug, serde::Serialize, serde::Deserialize)]
-pub(crate) struct KeyData {
-	pub(crate) kind: KeyEventKind,
-	pub(crate) modifiers: u32,
-	pub(crate) windows_key_code: i32,
-	pub(crate) native_key_code: i32,
-	pub(crate) character: u16,
-	pub(crate) unmodified_character: u16,
-}
-
-/// Turns a winit event into zero or more [`InputEvent`]s, updating the tracked input state
-/// (cursor position, click counting, modifiers) along the way.
-pub(crate) fn translate(input_state: &mut InputState, event: &WindowEvent) -> Vec<InputEvent> {
-	match event {
-		WindowEvent::PointerMoved { position, .. } => {
-			if !input_state.cursor_move(position) {
-				return Vec::new();
+					let flags = event_flags(&event.modifiers, &state.buttons);
+					host.send_mouse_click_event(Some(&mouse_event(position, flags)), cef::MouseButtonType::from(cef_button), up as i32, count);
+				}
+				PointerAction::ScrollLines { x, y } => {
+					let flags = event_flags(&event.modifiers, &state.buttons);
+					let delta_x = (x * SCROLL_LINE_WIDTH * SCROLL_SPEED_X) as i32;
+					let delta_y = (y * SCROLL_LINE_HEIGHT * SCROLL_SPEED_Y) as i32;
+					host.send_mouse_wheel_event(Some(&mouse_event(position, flags)), delta_x, delta_y);
+				}
+				PointerAction::ScrollPixels { x, y } => {
+					let flags = event_flags(&event.modifiers, &state.buttons) | PRECISION_SCROLLING_DELTA;
+					host.send_mouse_wheel_event(Some(&mouse_event(position, flags)), (x * SCROLL_SPEED_X) as i32, (y * SCROLL_SPEED_Y) as i32);
+				}
+				PointerAction::Zoom(delta) => {
+					if !delta.is_normal() {
+						return;
+					}
+					let flags = CONTROL_DOWN | PRECISION_SCROLLING_DELTA;
+					host.send_mouse_wheel_event(Some(&mouse_event(position, flags)), 0, (delta * PINCH_ZOOM_SPEED).round() as i32);
+				}
 			}
-			vec![InputEvent::MouseMove {
-				data: input_state.mouse_data(),
-				leave: false,
-			}]
 		}
-		WindowEvent::PointerEntered { position, .. } => {
-			let _ = input_state.cursor_move(position);
-			vec![InputEvent::MouseMove {
-				data: input_state.mouse_data(),
-				leave: false,
-			}]
-		}
-		WindowEvent::PointerLeft { position, .. } => {
-			if let Some(position) = position {
-				let _ = input_state.cursor_move(position);
-			}
-			vec![InputEvent::MouseMove {
-				data: input_state.mouse_data(),
-				leave: true,
-			}]
-		}
-		WindowEvent::PointerButton { state, button, position, .. } => {
-			let mouse_button = match button {
-				ButtonSource::Mouse(mouse_button) => Some(*mouse_button),
-				ButtonSource::TabletTool { button, .. } => (*button).into(),
-				_ => None, // TODO: Handle touch input
-			};
-			let Some(mouse_button) = mouse_button else {
-				return Vec::new();
-			};
+		InputEventKind::Key {
+			key,
+			key_without_modifiers,
+			physical_key,
+			location,
+			text,
+			action,
+		} => {
+			let mut flags = event_flags(&event.modifiers, &state.buttons);
 
-			let _ = input_state.cursor_move(position);
-			let click_count = input_state.mouse_input(&mouse_button, state).into();
-			let up = matches!(state, ElementState::Released);
-			let button = match mouse_button {
-				MouseButton::Left => MouseButtonKind::Left,
-				MouseButton::Right => MouseButtonKind::Right,
-				MouseButton::Middle => MouseButtonKind::Middle,
-				_ => return Vec::new(),
-			};
-
-			vec![InputEvent::MouseClick {
-				data: input_state.mouse_data(),
-				button,
-				up,
-				click_count,
-			}]
-		}
-		WindowEvent::MouseWheel { delta, phase: _, device_id: _, .. } => {
-			let (mut delta_x, mut delta_y) = match delta {
-				MouseScrollDelta::LineDelta(x, y) => (x * SCROLL_LINE_WIDTH as f32, y * SCROLL_LINE_HEIGHT as f32),
-				MouseScrollDelta::PixelDelta(physical_position) => (physical_position.x as f32, physical_position.y as f32),
-			};
-			delta_x *= SCROLL_SPEED_X as f32;
-			delta_y *= SCROLL_SPEED_Y as f32;
-
-			vec![InputEvent::MouseWheel {
-				data: input_state.mouse_data(),
-				delta_x: delta_x as i32,
-				delta_y: delta_y as i32,
-			}]
-		}
-		WindowEvent::ModifiersChanged(modifiers) => {
-			input_state.modifiers_changed(&modifiers.state());
-			Vec::new()
-		}
-		WindowEvent::KeyboardInput { device_id: _, event, is_synthetic: _ } => {
-			input_state.modifiers_apply_key_event(&event.logical_key, &event.state);
-
-			let mut kind = match (event.state, &event.logical_key) {
-				(ElementState::Pressed, winit::keyboard::Key::Character(_)) => KeyEventKind::Char,
-				(ElementState::Pressed, _) => KeyEventKind::RawKeyDown,
-				(ElementState::Released, _) => KeyEventKind::KeyUp,
-			};
-
-			let modifiers = input_state.cef_modifiers(&event.location, event.repeat).into();
-
-			let windows_key_code = match &event.logical_key {
-				winit::keyboard::Key::Named(named) => named.to_vk_bits(),
-				winit::keyboard::Key::Character(char) => char.chars().next().unwrap_or_default().to_vk_bits(),
+			let own_flag = match key {
+				Key::Named(NamedKey::Shift) => SHIFT_DOWN,
+				Key::Named(NamedKey::Control) => CONTROL_DOWN,
+				Key::Named(NamedKey::Alt) => ALT_DOWN,
+				Key::Named(NamedKey::AltGraph) => ALTGR_DOWN,
+				Key::Named(NamedKey::Meta) => COMMAND_DOWN,
 				_ => 0,
 			};
+			match action {
+				KeyAction::Press | KeyAction::Repeat => flags |= own_flag,
+				KeyAction::Release => flags &= !own_flag,
+			}
 
-			let native_key_code = event.physical_key.to_native_keycode();
+			flags |= match location {
+				KeyLocation::Left => IS_LEFT,
+				KeyLocation::Right => IS_RIGHT,
+				KeyLocation::Numpad => IS_KEY_PAD,
+				KeyLocation::Standard => 0,
+			};
+			if *action == KeyAction::Repeat {
+				flags |= IS_REPEAT;
+			}
 
-			let char_representation = event.logical_key.to_char_representation();
+			let windows_key_code = match key {
+				Key::Named(named) => named.to_vk_bits(),
+				Key::Character(char) => char.chars().next().unwrap_or_default().to_vk_bits(),
+				_ => 0,
+			};
+			let native_key_code = physical_key.to_native_keycode();
+
+			let char_representation = key.to_char_representation();
 			#[allow(unused_mut)]
 			let mut character = char_representation as u16;
-
-			if event.state == ElementState::Pressed && character != 0 {
-				kind = KeyEventKind::Char;
-			}
-
-			let unmodified_character = event.key_without_modifiers.to_char_representation() as u16;
+			let unmodified_character = key_without_modifiers.to_char_representation() as u16;
 
 			#[cfg(target_os = "macos")] // See https://www.magpcss.org/ceforum/viewtopic.php?start=10&t=11650
-			if character == 0 && unmodified_character == 0 && event.text_with_all_modifiers.is_some() {
+			if character == 0 && unmodified_character == 0 && text.is_some() {
 				character = 1;
 			}
+			#[cfg(not(target_os = "macos"))]
+			let _ = text;
 
-			let key = KeyData {
-				kind,
-				modifiers,
+			let key_event = |kind: cef_key_event_type_t, windows_key_code: i32| KeyEvent {
+				type_: kind.into(),
+				modifiers: flags,
 				windows_key_code,
 				native_key_code,
 				character,
 				unmodified_character,
-			};
-
-			if kind == KeyEventKind::Char {
-				// CEF expects a raw key-down before the character event it produces.
-				vec![
-					InputEvent::Key(KeyData {
-						kind: KeyEventKind::RawKeyDown,
-						..key
-					}),
-					InputEvent::Key(KeyData {
-						windows_key_code: char_representation as i32,
-						..key
-					}),
-				]
-			} else {
-				vec![InputEvent::Key(key)]
-			}
-		}
-		WindowEvent::PinchGesture { delta, .. } => {
-			if !delta.is_normal() {
-				return Vec::new();
-			}
-
-			let data = MouseData {
-				modifiers: CefModifiers::PINCH_MODIFIERS.into(),
-				..input_state.mouse_data()
-			};
-
-			vec![InputEvent::MouseWheel {
-				data,
-				delta_x: 0,
-				delta_y: (delta * PINCH_ZOOM_SPEED).round() as i32,
-			}]
-		}
-		_ => Vec::new(),
-	}
-}
-
-/// Sends a translated [`InputEvent`] to the browser. Must run on the thread owning the browser.
-pub(crate) fn apply(browser: &Browser, event: &InputEvent) {
-	let Some(host) = browser.host() else { return };
-	match event {
-		InputEvent::MouseMove { data, leave } => {
-			host.send_mouse_move_event(Some(&data.into()), *leave as i32);
-		}
-		InputEvent::MouseClick { data, button, up, click_count } => {
-			let cef_button = cef::MouseButtonType::from(match button {
-				MouseButtonKind::Left => cef_mouse_button_type_t::MBT_LEFT,
-				MouseButtonKind::Right => cef_mouse_button_type_t::MBT_RIGHT,
-				MouseButtonKind::Middle => cef_mouse_button_type_t::MBT_MIDDLE,
-			});
-			host.send_mouse_click_event(Some(&data.into()), cef_button, *up as i32, *click_count);
-		}
-		InputEvent::MouseWheel { data, delta_x, delta_y } => {
-			host.send_mouse_wheel_event(Some(&data.into()), *delta_x, *delta_y);
-		}
-		InputEvent::Key(key) => {
-			let key_event = KeyEvent {
-				type_: match key.kind {
-					KeyEventKind::RawKeyDown => cef_key_event_type_t::KEYEVENT_RAWKEYDOWN,
-					KeyEventKind::KeyUp => cef_key_event_type_t::KEYEVENT_KEYUP,
-					KeyEventKind::Char => cef_key_event_type_t::KEYEVENT_CHAR,
-				}
-				.into(),
-				modifiers: key.modifiers,
-				windows_key_code: key.windows_key_code,
-				native_key_code: key.native_key_code,
-				character: key.character,
-				unmodified_character: key.unmodified_character,
 				..Default::default()
 			};
-			host.send_key_event(Some(&key_event));
+
+			match action {
+				KeyAction::Press | KeyAction::Repeat if char_representation != '\0' => {
+					host.send_key_event(Some(&key_event(cef_key_event_type_t::KEYEVENT_RAWKEYDOWN, windows_key_code)));
+					host.send_key_event(Some(&key_event(cef_key_event_type_t::KEYEVENT_CHAR, char_representation as i32)));
+				}
+				KeyAction::Press | KeyAction::Repeat => {
+					host.send_key_event(Some(&key_event(cef_key_event_type_t::KEYEVENT_RAWKEYDOWN, windows_key_code)));
+				}
+				KeyAction::Release => {
+					host.send_key_event(Some(&key_event(cef_key_event_type_t::KEYEVENT_KEYUP, windows_key_code)));
+				}
+			}
 		}
 	}
 }
 
-impl From<&MouseData> for MouseEvent {
-	fn from(data: &MouseData) -> Self {
-		MouseEvent {
-			x: data.x,
-			y: data.y,
-			modifiers: data.modifiers,
+#[derive(Default)]
+struct ButtonStates {
+	left: bool,
+	right: bool,
+	middle: bool,
+}
+
+impl ButtonStates {
+	fn update(&mut self, button: MouseButton, held: bool) {
+		match button {
+			MouseButton::Left => self.left = held,
+			MouseButton::Right => self.right = held,
+			MouseButton::Middle => self.middle = held,
+			MouseButton::Unknown => {}
 		}
 	}
 }
+
+fn mouse_event(position: PhysicalPosition<f64>, flags: u32) -> MouseEvent {
+	MouseEvent {
+		x: position.x as i32,
+		y: position.y as i32,
+		modifiers: flags,
+	}
+}
+
+fn event_flags(modifiers: &Modifiers, buttons: &ButtonStates) -> u32 {
+	let bit = |condition: bool, flag: u32| if condition { flag } else { 0 };
+	bit(modifiers.shift, SHIFT_DOWN)
+		| bit(modifiers.control, CONTROL_DOWN)
+		| bit(modifiers.alt, ALT_DOWN)
+		| bit(modifiers.alt_graph, ALTGR_DOWN)
+		| bit(modifiers.meta, COMMAND_DOWN)
+		| bit(modifiers.caps_lock, CAPS_LOCK_ON)
+		| bit(modifiers.num_lock, NUM_LOCK_ON)
+		| bit(buttons.left, LEFT_MOUSE_BUTTON)
+		| bit(buttons.right, RIGHT_MOUSE_BUTTON)
+		| bit(buttons.middle, MIDDLE_MOUSE_BUTTON)
+}
+
+const fn flag(flag: cef_event_flags_t) -> u32 {
+	#[cfg(not(target_os = "windows"))]
+	{
+		flag.0
+	}
+	#[cfg(target_os = "windows")]
+	{
+		flag.0 as u32
+	}
+}
+
+const SHIFT_DOWN: u32 = flag(cef_event_flags_t::EVENTFLAG_SHIFT_DOWN);
+const CONTROL_DOWN: u32 = flag(cef_event_flags_t::EVENTFLAG_CONTROL_DOWN);
+const ALT_DOWN: u32 = flag(cef_event_flags_t::EVENTFLAG_ALT_DOWN);
+const ALTGR_DOWN: u32 = flag(cef_event_flags_t::EVENTFLAG_ALTGR_DOWN);
+const COMMAND_DOWN: u32 = flag(cef_event_flags_t::EVENTFLAG_COMMAND_DOWN);
+const CAPS_LOCK_ON: u32 = flag(cef_event_flags_t::EVENTFLAG_CAPS_LOCK_ON);
+const NUM_LOCK_ON: u32 = flag(cef_event_flags_t::EVENTFLAG_NUM_LOCK_ON);
+const LEFT_MOUSE_BUTTON: u32 = flag(cef_event_flags_t::EVENTFLAG_LEFT_MOUSE_BUTTON);
+const MIDDLE_MOUSE_BUTTON: u32 = flag(cef_event_flags_t::EVENTFLAG_MIDDLE_MOUSE_BUTTON);
+const RIGHT_MOUSE_BUTTON: u32 = flag(cef_event_flags_t::EVENTFLAG_RIGHT_MOUSE_BUTTON);
+const IS_LEFT: u32 = flag(cef_event_flags_t::EVENTFLAG_IS_LEFT);
+const IS_RIGHT: u32 = flag(cef_event_flags_t::EVENTFLAG_IS_RIGHT);
+const IS_KEY_PAD: u32 = flag(cef_event_flags_t::EVENTFLAG_IS_KEY_PAD);
+const IS_REPEAT: u32 = flag(cef_event_flags_t::EVENTFLAG_IS_REPEAT);
+const PRECISION_SCROLLING_DELTA: u32 = flag(cef_event_flags_t::EVENTFLAG_PRECISION_SCROLLING_DELTA);

desktop/ui/src/input/event.rs

@@ -0,0 +1,245 @@
+use winit::dpi::PhysicalPosition;
+use winit::event::{ElementState, KeyEvent};
+use winit::keyboard::{Key, KeyLocation, ModifiersState, PhysicalKey, SmolStr};
+
+#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
+pub struct InputEvent {
+	pub(crate) kind: InputEventKind,
+	pub(crate) modifiers: Modifiers,
+}
+
+impl InputEvent {
+	pub fn pointer() -> PointerInputEventBuilder {
+		PointerInputEventBuilder { position: UnknownPosition }
+	}
+
+	pub fn key(event: &KeyEvent) -> InputEventBuilder {
+		let action = match (event.state, event.repeat) {
+			(ElementState::Pressed, false) => KeyAction::Press,
+			(ElementState::Pressed, true) => KeyAction::Repeat,
+			(ElementState::Released, _) => KeyAction::Release,
+		};
+		InputEventBuilder::new(InputEventKind::Key {
+			key: event.logical_key.clone(),
+			key_without_modifiers: event.key_without_modifiers.clone(),
+			physical_key: event.physical_key,
+			location: event.location,
+			text: event.text_with_all_modifiers.clone(),
+			action,
+		})
+	}
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
+pub enum MouseButton {
+	Left,
+	Right,
+	Middle,
+	Unknown,
+}
+
+impl From<winit::event::MouseButton> for MouseButton {
+	fn from(button: winit::event::MouseButton) -> Self {
+		match button {
+			winit::event::MouseButton::Left => Self::Left,
+			winit::event::MouseButton::Right => Self::Right,
+			winit::event::MouseButton::Middle => Self::Middle,
+			_ => Self::Unknown,
+		}
+	}
+}
+
+impl From<winit::event::ButtonSource> for MouseButton {
+	fn from(button: winit::event::ButtonSource) -> Self {
+		button.mouse_button().map_or(Self::Unknown, Self::from)
+	}
+}
+
+#[must_use]
+#[derive(Clone, Debug)]
+pub struct PointerInputEventBuilder<P = UnknownPosition> {
+	position: P,
+}
+
+impl PointerInputEventBuilder<UnknownPosition> {
+	pub fn position(self, position: PhysicalPosition<f64>) -> PointerInputEventBuilder<PhysicalPosition<f64>> {
+		PointerInputEventBuilder { position }
+	}
+}
+
+impl PointerInputEventBuilder<PhysicalPosition<f64>> {
+	pub fn moved(self) -> InputEventBuilder {
+		self.finish(PointerAction::Move)
+	}
+
+	pub fn entered(self) -> InputEventBuilder {
+		self.finish(PointerAction::Enter)
+	}
+}
+
+impl<P: PointerPosition> PointerInputEventBuilder<P> {
+	pub fn exited(self) -> InputEventBuilder {
+		self.finish(PointerAction::Exit)
+	}
+
+	pub fn pressed(self, button: impl Into<MouseButton>, count: u32) -> InputEventBuilder {
+		self.finish(PointerAction::Press { button: button.into(), count })
+	}
+
+	pub fn released(self, button: impl Into<MouseButton>, count: u32) -> InputEventBuilder {
+		self.finish(PointerAction::Release { button: button.into(), count })
+	}
+
+	pub fn scrolled_lines(self, x: f64, y: f64) -> InputEventBuilder {
+		self.finish(PointerAction::ScrollLines { x, y })
+	}
+
+	pub fn scrolled_pixels(self, x: f64, y: f64) -> InputEventBuilder {
+		self.finish(PointerAction::ScrollPixels { x, y })
+	}
+
+	pub fn zoomed(self, delta: f64) -> InputEventBuilder {
+		self.finish(PointerAction::Zoom(delta))
+	}
+
+	fn finish(self, action: PointerAction) -> InputEventBuilder {
+		InputEventBuilder::new(InputEventKind::Pointer {
+			position: self.position.position(),
+			action,
+		})
+	}
+}
+
+#[must_use]
+#[derive(Clone, Debug)]
+pub struct InputEventBuilder {
+	kind: InputEventKind,
+	modifiers: Modifiers,
+}
+
+impl InputEventBuilder {
+	fn new(kind: InputEventKind) -> Self {
+		Self {
+			kind,
+			modifiers: Modifiers::default(),
+		}
+	}
+
+	pub fn shift(mut self, on: bool) -> Self {
+		self.modifiers.shift = on;
+		self
+	}
+
+	pub fn control(mut self, on: bool) -> Self {
+		self.modifiers.control = on;
+		self
+	}
+
+	pub fn alt(mut self, on: bool) -> Self {
+		self.modifiers.alt = on;
+		self
+	}
+
+	pub fn alt_graph(mut self, on: bool) -> Self {
+		self.modifiers.alt_graph = on;
+		self
+	}
+
+	pub fn meta(mut self, on: bool) -> Self {
+		self.modifiers.meta = on;
+		self
+	}
+
+	pub fn caps_lock(mut self, on: bool) -> Self {
+		self.modifiers.caps_lock = on;
+		self
+	}
+
+	pub fn num_lock(mut self, on: bool) -> Self {
+		self.modifiers.num_lock = on;
+		self
+	}
+
+	pub fn modifiers(mut self, modifiers: ModifiersState) -> Self {
+		self.modifiers.shift = modifiers.shift_key();
+		self.modifiers.control = modifiers.control_key();
+		self.modifiers.alt = modifiers.alt_key();
+		self.modifiers.meta = modifiers.meta_key();
+		self
+	}
+
+	pub fn build(self) -> InputEvent {
+		InputEvent {
+			kind: self.kind,
+			modifiers: self.modifiers,
+		}
+	}
+}
+
+#[expect(private_bounds)]
+pub trait PointerPosition: OptionalPosition {}
+impl PointerPosition for UnknownPosition {}
+impl PointerPosition for PhysicalPosition<f64> {}
+
+#[derive(Clone, Copy, Debug)]
+pub struct UnknownPosition;
+
+trait OptionalPosition {
+	fn position(self) -> Option<PhysicalPosition<f64>>;
+}
+impl OptionalPosition for UnknownPosition {
+	fn position(self) -> Option<PhysicalPosition<f64>> {
+		None
+	}
+}
+impl OptionalPosition for PhysicalPosition<f64> {
+	fn position(self) -> Option<PhysicalPosition<f64>> {
+		Some(self)
+	}
+}
+
+#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
+pub(crate) enum InputEventKind {
+	Pointer {
+		position: Option<PhysicalPosition<f64>>,
+		action: PointerAction,
+	},
+	Key {
+		key: Key,
+		key_without_modifiers: Key,
+		physical_key: PhysicalKey,
+		location: KeyLocation,
+		text: Option<SmolStr>,
+		action: KeyAction,
+	},
+}
+
+#[derive(Clone, Copy, Debug, serde::Serialize, serde::Deserialize)]
+pub(crate) enum PointerAction {
+	Move,
+	Enter,
+	Exit,
+	Press { button: MouseButton, count: u32 },
+	Release { button: MouseButton, count: u32 },
+	ScrollLines { x: f64, y: f64 },
+	ScrollPixels { x: f64, y: f64 },
+	Zoom(f64),
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
+pub(crate) enum KeyAction {
+	Press,
+	Repeat,
+	Release,
+}
+
+#[derive(Clone, Copy, Debug, Default, serde::Serialize, serde::Deserialize)]
+pub(crate) struct Modifiers {
+	pub(crate) shift: bool,
+	pub(crate) control: bool,
+	pub(crate) alt: bool,
+	pub(crate) alt_graph: bool,
+	pub(crate) meta: bool,
+	pub(crate) caps_lock: bool,
+	pub(crate) num_lock: bool,
+}

desktop/ui/src/input/state.rs

@@ -1,256 +0,0 @@
-use cef::sys::cef_event_flags_t;
-use std::time::Instant;
-use winit::dpi::PhysicalPosition;
-use winit::event::{ElementState, MouseButton};
-use winit::keyboard::{Key, KeyLocation, ModifiersState, NamedKey};
-
-use super::MouseData;
-use crate::consts::{MULTICLICK_ALLOWED_TRAVEL, MULTICLICK_TIMEOUT};
-
-#[derive(Default)]
-pub(crate) struct InputState {
-	modifiers: ModifiersState,
-	mouse_position: MousePosition,
-	mouse_state: MouseState,
-	mouse_click_tracker: ClickTracker,
-}
-impl InputState {
-	pub(crate) fn modifiers_changed(&mut self, modifiers: &ModifiersState) {
-		self.modifiers = *modifiers;
-	}
-
-	pub(crate) fn modifiers_apply_key_event(&mut self, key: &Key, state: &ElementState) {
-		let bits = match key {
-			Key::Named(NamedKey::Shift) => ModifiersState::SHIFT,
-			Key::Named(NamedKey::Control) => ModifiersState::CONTROL,
-			Key::Named(NamedKey::Alt) => ModifiersState::ALT,
-			Key::Named(NamedKey::Meta) => ModifiersState::META,
-			_ => return,
-		};
-		let is_pressed = matches!(state, ElementState::Pressed);
-		self.modifiers.set(bits, is_pressed);
-	}
-
-	pub(crate) fn cursor_move(&mut self, position: &PhysicalPosition<f64>) -> bool {
-		let new = position.into();
-		if self.mouse_position == new {
-			return false;
-		}
-		self.mouse_position = new;
-		true
-	}
-
-	pub(crate) fn mouse_input(&mut self, button: &MouseButton, state: &ElementState) -> ClickCount {
-		self.mouse_state.update(button, state);
-		self.mouse_click_tracker.input(button, state, self.mouse_position)
-	}
-
-	pub(crate) fn cef_modifiers(&self, location: &KeyLocation, is_repeat: bool) -> CefModifiers {
-		CefModifiers::new(self, location, is_repeat)
-	}
-
-	pub(crate) fn cef_mouse_modifiers(&self) -> CefModifiers {
-		self.cef_modifiers(&KeyLocation::Standard, false)
-	}
-
-	pub(crate) fn mouse_data(&self) -> MouseData {
-		MouseData {
-			x: self.mouse_position.x,
-			y: self.mouse_position.y,
-			modifiers: self.cef_mouse_modifiers().into(),
-		}
-	}
-}
-
-#[derive(Default, Clone, Copy, Eq, PartialEq)]
-pub(crate) struct MousePosition {
-	x: i32,
-	y: i32,
-}
-impl From<&PhysicalPosition<f64>> for MousePosition {
-	fn from(position: &PhysicalPosition<f64>) -> Self {
-		Self {
-			x: position.x as i32,
-			y: position.y as i32,
-		}
-	}
-}
-
-#[derive(Default, Clone)]
-pub(crate) struct MouseState {
-	left: bool,
-	right: bool,
-	middle: bool,
-}
-impl MouseState {
-	pub(crate) fn update(&mut self, button: &MouseButton, state: &ElementState) {
-		match state {
-			ElementState::Pressed => match button {
-				MouseButton::Left => self.left = true,
-				MouseButton::Right => self.right = true,
-				MouseButton::Middle => self.middle = true,
-				_ => {}
-			},
-			ElementState::Released => match button {
-				MouseButton::Left => self.left = false,
-				MouseButton::Right => self.right = false,
-				MouseButton::Middle => self.middle = false,
-				_ => {}
-			},
-		}
-	}
-}
-
-#[derive(Default)]
-struct ClickTracker {
-	left: Option<ClickRecord>,
-	middle: Option<ClickRecord>,
-	right: Option<ClickRecord>,
-}
-impl ClickTracker {
-	fn input(&mut self, button: &MouseButton, state: &ElementState, position: MousePosition) -> ClickCount {
-		let record = match button {
-			MouseButton::Left => &mut self.left,
-			MouseButton::Right => &mut self.right,
-			MouseButton::Middle => &mut self.middle,
-			_ => return ClickCount::Single,
-		};
-
-		let Some(record) = record else {
-			*record = Some(ClickRecord {
-				down_position: position,
-				up_position: position,
-				..Default::default()
-			});
-			return ClickCount::Single;
-		};
-
-		let now = Instant::now();
-		let within_time = now.saturating_duration_since(record.time) <= MULTICLICK_TIMEOUT;
-
-		let (prev_count, prev_position) = match state {
-			ElementState::Pressed => (record.down_count, record.down_position),
-			ElementState::Released => (record.up_count, record.up_position),
-		};
-
-		let dx = position.x.abs_diff(prev_position.x) as usize;
-		let dy = position.y.abs_diff(prev_position.y) as usize;
-		let within_dist = dx <= MULTICLICK_ALLOWED_TRAVEL && dy <= MULTICLICK_ALLOWED_TRAVEL;
-
-		let count = match (prev_count, within_time, within_dist) {
-			(ClickCount::Single, true, true) => ClickCount::Double,
-			(ClickCount::Double, true, true) => ClickCount::Triple,
-			(ClickCount::Triple, true, true) => ClickCount::Double,
-			_ => ClickCount::Single,
-		};
-
-		record.time = now;
-
-		match state {
-			ElementState::Pressed => {
-				record.down_position = position;
-				record.down_count = count;
-			}
-			ElementState::Released => {
-				record.up_position = position;
-				record.up_count = count;
-			}
-		}
-
-		count
-	}
-}
-
-#[derive(Clone, Copy, PartialEq, Default)]
-pub(crate) enum ClickCount {
-	#[default]
-	Single,
-	Double,
-	Triple,
-}
-impl From<ClickCount> for i32 {
-	fn from(count: ClickCount) -> i32 {
-		match count {
-			ClickCount::Single => 1,
-			ClickCount::Double => 2,
-			ClickCount::Triple => 3,
-		}
-	}
-}
-
-#[derive(Clone, Copy)]
-struct ClickRecord {
-	time: Instant,
-	down_position: MousePosition,
-	up_position: MousePosition,
-	down_count: ClickCount,
-	up_count: ClickCount,
-}
-
-impl Default for ClickRecord {
-	fn default() -> Self {
-		Self {
-			time: Instant::now(),
-			down_position: Default::default(),
-			up_position: Default::default(),
-			down_count: Default::default(),
-			up_count: Default::default(),
-		}
-	}
-}
-
-pub(crate) struct CefModifiers(cef_event_flags_t);
-impl CefModifiers {
-	fn new(input_state: &InputState, location: &KeyLocation, is_repeat: bool) -> Self {
-		let mut inner = cef_event_flags_t::EVENTFLAG_NONE;
-
-		if input_state.modifiers.shift_key() {
-			inner |= cef_event_flags_t::EVENTFLAG_SHIFT_DOWN;
-		}
-		if input_state.modifiers.control_key() {
-			inner |= cef_event_flags_t::EVENTFLAG_CONTROL_DOWN;
-		}
-		if input_state.modifiers.alt_key() {
-			inner |= cef_event_flags_t::EVENTFLAG_ALT_DOWN;
-		}
-		if input_state.modifiers.meta_key() {
-			inner |= cef_event_flags_t::EVENTFLAG_COMMAND_DOWN;
-		}
-
-		if input_state.mouse_state.left {
-			inner |= cef_event_flags_t::EVENTFLAG_LEFT_MOUSE_BUTTON;
-		}
-		if input_state.mouse_state.right {
-			inner |= cef_event_flags_t::EVENTFLAG_RIGHT_MOUSE_BUTTON;
-		}
-		if input_state.mouse_state.middle {
-			inner |= cef_event_flags_t::EVENTFLAG_MIDDLE_MOUSE_BUTTON;
-		}
-
-		if is_repeat {
-			inner |= cef_event_flags_t::EVENTFLAG_IS_REPEAT;
-		}
-
-		inner |= match location {
-			KeyLocation::Left => cef_event_flags_t::EVENTFLAG_IS_LEFT,
-			KeyLocation::Right => cef_event_flags_t::EVENTFLAG_IS_RIGHT,
-			KeyLocation::Numpad => cef_event_flags_t::EVENTFLAG_IS_KEY_PAD,
-			KeyLocation::Standard => cef_event_flags_t::EVENTFLAG_NONE,
-		};
-
-		Self(inner)
-	}
-
-	pub(super) const PINCH_MODIFIERS: Self = Self(cef_event_flags_t(
-		cef_event_flags_t::EVENTFLAG_CONTROL_DOWN.0 | cef_event_flags_t::EVENTFLAG_PRECISION_SCROLLING_DELTA.0,
-	));
-}
-
-impl From<CefModifiers> for u32 {
-	fn from(val: CefModifiers) -> Self {
-		#[cfg(not(target_os = "windows"))]
-		return val.0.0;
-		#[cfg(target_os = "windows")]
-		return val.0.0 as u32;
-	}
-}

desktop/ui/src/lib.rs

@@ -23,6 +23,7 @@ mod utility;
 mod view;
 
 pub use consts::{MULTICLICK_ALLOWED_TRAVEL, MULTICLICK_TIMEOUT, PINCH_ZOOM_SPEED, SCROLL_LINE_HEIGHT, SCROLL_LINE_WIDTH, SCROLL_SPEED_X, SCROLL_SPEED_Y};
+pub use input::event::{InputEvent, InputEventBuilder, MouseButton, PointerInputEventBuilder, PointerPosition, UnknownPosition};
 
 pub struct UiContext<S: Stage = Started> {
 	inner: S::ContextData,
@@ -69,7 +70,6 @@ impl UiContext<Started> {
 		Ok(UiInstance {
 			inner: Arc::new(UiInstanceInner {
 				host: self.inner.clone(),
-				input: Mutex::new(input::InputState::default()),
 				events: Mutex::new(events),
 				queue,
 				shutdown_complete: Mutex::new(shutdown_complete),
@@ -143,7 +143,6 @@ pub struct UiInstance {
 
 pub(crate) struct UiInstanceInner {
 	host: Arc<HostHandle>,
-	input: Mutex<input::InputState>,
 	events: Mutex<Receiver<UiEvent>>,
 	queue: events::EventQueue,
 	shutdown_complete: Mutex<Receiver<()>>,
@@ -154,18 +153,7 @@ impl UiInstance {
 	pub fn send(&self, command: UiCommand) {
 		let shared = &self.inner;
 		match command {
-			UiCommand::Input(event) => {
-				let events = {
-					let Ok(mut input) = shared.input.lock() else {
-						tracing::error!("Failed to lock the input state");
-						return;
-					};
-					input::translate(&mut input, &event)
-				};
-				if !events.is_empty() {
-					shared.host.send(HostControlMessage::Input(events));
-				}
-			}
+			UiCommand::Input(event) => shared.host.send(HostControlMessage::Input(event)),
 			UiCommand::Resized { width, height } => shared.host.send(HostControlMessage::UpdateViewInfo(view::ViewInfoUpdate::Size { width, height })),
 			UiCommand::ScaleChanged(scale) => shared.host.send(HostControlMessage::UpdateViewInfo(view::ViewInfoUpdate::Scale(scale))),
 			UiCommand::Refresh => shared.host.send(HostControlMessage::RefreshViewInfo),
@@ -222,7 +210,7 @@ impl Drop for UiInstanceInner {
 
 #[derive(Debug)]
 pub enum UiCommand {
-	Input(winit::event::WindowEvent),
+	Input(InputEvent),
 	Resized { width: u32, height: u32 },
 	ScaleChanged(f64),
 	Refresh,

desktop/ui/src/remote/host.rs

@@ -99,7 +99,7 @@ enum ControlOutcome {
 fn control_loop(receiver: &IpcReceiver<HostControlMessage>, context: &CefContextHandle, sequence: &SequenceState) -> ControlOutcome {
 	loop {
 		match receiver.recv() {
-			Ok(HostControlMessage::Input(events)) => context.apply_input(events),
+			Ok(HostControlMessage::Input(event)) => context.process_input(event),
 			Ok(HostControlMessage::UpdateViewInfo(update)) => context.update_view_info(update),
 			Ok(HostControlMessage::RefreshViewInfo) => context.refresh_view_info(),
 			Ok(HostControlMessage::SendWebMessage(message)) => context.send_web_message(message),

desktop/ui/src/remote/messages.rs

@@ -8,7 +8,7 @@ use crate::view::ViewInfoUpdate;
 
 #[derive(Serialize, Deserialize)]
 pub(crate) enum HostControlMessage {
-	Input(Vec<InputEvent>),
+	Input(InputEvent),
 	UpdateViewInfo(ViewInfoUpdate),
 	RefreshViewInfo,
 	SendWebMessage(Vec<u8>),