Improve Frontend -> Backend user input system (#348)

Includes refactor that sends coordinates of the document viewports to the backend so input is sent relative to the application window
Closes #124
Fixes #291

* Improve Frontend -> Backend user input system

* Code review changes

* More code review changes

* Fix TS error

frontend/src/components/panels/Document.vue

@@ -118,7 +118,7 @@
 						<CanvasRuler :origin="0" :majorMarkSpacing="100" :direction="RulerDirection.Vertical" />
 					</LayoutCol>
 					<LayoutCol :class="'canvas-area'">
-						<div class="canvas" @mousedown="canvasMouseDown" @mouseup="canvasMouseUp" @mousemove="canvasMouseMove" ref="canvas">
+						<div class="canvas" ref="canvas">
 							<svg v-html="viewportSvg" :style="{ width: canvasSvgWidth, height: canvasSvgHeight }"></svg>
 						</div>
 					</LayoutCol>
@@ -210,7 +210,6 @@
 <script lang="ts">
 import { defineComponent } from "vue";
 
-import { makeModifiersBitfield } from "@/utilities/input";
 import { ResponseType, registerResponseHandler, Response, UpdateCanvas, SetActiveTool, SetCanvasZoom, SetCanvasRotation } from "@/utilities/response-handler";
 import { SeparatorDirection, SeparatorType } from "@/components/widgets/widgets";
 import { comingSoon } from "@/utilities/errors";
@@ -259,25 +258,6 @@ export default defineComponent({
 
 			this.canvasSvgWidth = `${width}px`;
 			this.canvasSvgHeight = `${height}px`;
-
-			(await wasm).viewport_resize(width, height);
-		},
-		async canvasMouseDown(e: MouseEvent) {
-			const modifiers = makeModifiersBitfield(e.ctrlKey, e.shiftKey, e.altKey);
-			(await wasm).on_mouse_down(e.offsetX, e.offsetY, e.buttons, modifiers);
-		},
-		async canvasMouseUp(e: MouseEvent) {
-			const modifiers = makeModifiersBitfield(e.ctrlKey, e.shiftKey, e.altKey);
-			(await wasm).on_mouse_up(e.offsetX, e.offsetY, e.buttons, modifiers);
-		},
-		async canvasMouseMove(e: MouseEvent) {
-			const modifiers = makeModifiersBitfield(e.ctrlKey, e.shiftKey, e.altKey);
-			(await wasm).on_mouse_move(e.offsetX, e.offsetY, modifiers);
-		},
-		async canvasMouseScroll(e: WheelEvent) {
-			e.preventDefault();
-			const modifiers = makeModifiersBitfield(e.ctrlKey, e.shiftKey, e.altKey);
-			(await wasm).on_mouse_scroll(e.deltaX, e.deltaY, e.deltaZ, modifiers);
 		},
 		async setCanvasZoom(newZoom: number) {
 			(await wasm).set_canvas_zoom(newZoom / 100);
@@ -327,12 +307,8 @@ export default defineComponent({
 			}
 		});
 
-		// TODO: Move event listeners to `main.ts`
-		const canvas = this.$refs.canvas as HTMLElement;
-		canvas.addEventListener("wheel", this.canvasMouseScroll, { passive: false });
-
-		window.addEventListener("resize", () => this.viewportResize());
-		window.addEventListener("DOMContentLoaded", () => this.viewportResize());
+		window.addEventListener("resize", this.viewportResize);
+		window.addEventListener("DOMContentLoaded", this.viewportResize);
 	},
 	data() {
 		return {

frontend/src/main.ts

@@ -1,17 +1,26 @@
 import { createApp } from "vue";
 
 import { fullscreenModeChanged } from "@/utilities/fullscreen";
-import { handleKeyUp, handleKeyDown, handleMouseDown } from "@/utilities/input";
+import { onKeyUp, onKeyDown, onMouseMove, onMouseDown, onMouseUp, onMouseScroll, onWindowResize } from "@/utilities/input";
 import "@/utilities/errors";
 
 import App from "@/App.vue";
 
 // Bind global browser events
+window.addEventListener("resize", onWindowResize);
+window.addEventListener("DOMContentLoaded", onWindowResize);
+
 document.addEventListener("contextmenu", (e) => e.preventDefault());
 document.addEventListener("fullscreenchange", () => fullscreenModeChanged());
-window.addEventListener("keyup", (e: KeyboardEvent) => handleKeyUp(e));
-window.addEventListener("keydown", (e: KeyboardEvent) => handleKeyDown(e));
-window.addEventListener("mousedown", (e: MouseEvent) => handleMouseDown(e));
+
+window.addEventListener("keyup", onKeyUp);
+window.addEventListener("keydown", onKeyDown);
+
+window.addEventListener("mousemove", onMouseMove);
+window.addEventListener("mousedown", onMouseDown);
+window.addEventListener("mouseup", onMouseUp);
+
+window.addEventListener("wheel", onMouseScroll, { passive: false });
 
 // Initialize the Vue application
 createApp(App).mount("#app");

frontend/src/utilities/input.ts

@@ -3,7 +3,11 @@ import { dialogIsVisible, dismissDialog, submitDialog } from "@/utilities/dialog
 
 const wasm = import("@/../wasm/pkg");
 
-export function shouldRedirectKeyboardEventToBackend(e: KeyboardEvent): boolean {
+let viewportMouseInteractionOngoing = false;
+
+// Keyboard events
+
+function shouldRedirectKeyboardEventToBackend(e: KeyboardEvent): boolean {
 	// Don't redirect user input from text entry into HTML elements
 	const target = e.target as HTMLElement;
 	if (target.nodeName === "INPUT" || target.nodeName === "TEXTAREA" || target.isContentEditable) return false;
@@ -31,10 +35,10 @@ export function shouldRedirectKeyboardEventToBackend(e: KeyboardEvent): boolean
 	return true;
 }
 
-export async function handleKeyDown(e: KeyboardEvent) {
+export async function onKeyDown(e: KeyboardEvent) {
 	if (shouldRedirectKeyboardEventToBackend(e)) {
 		e.preventDefault();
-		const modifiers = makeModifiersBitfield(e.ctrlKey, e.shiftKey, e.altKey);
+		const modifiers = makeModifiersBitfield(e);
 		(await wasm).on_key_down(e.key, modifiers);
 		return;
 	}
@@ -48,27 +52,77 @@ export async function handleKeyDown(e: KeyboardEvent) {
 	}
 }
 
-export async function handleKeyUp(e: KeyboardEvent) {
+export async function onKeyUp(e: KeyboardEvent) {
 	if (shouldRedirectKeyboardEventToBackend(e)) {
 		e.preventDefault();
-		const modifiers = makeModifiersBitfield(e.ctrlKey, e.shiftKey, e.altKey);
+		const modifiers = makeModifiersBitfield(e);
 		(await wasm).on_key_up(e.key, modifiers);
 	}
 }
 
-export async function handleMouseDown(e: MouseEvent) {
+// Mouse events
+
+export async function onMouseMove(e: MouseEvent) {
+	if (!e.buttons) viewportMouseInteractionOngoing = false;
+
+	const modifiers = makeModifiersBitfield(e);
+	(await wasm).on_mouse_move(e.clientX, e.clientY, e.buttons, modifiers);
+}
+
+export async function onMouseDown(e: MouseEvent) {
 	const target = e.target && (e.target as HTMLElement);
-	const clickedInsideDialog = target && target.closest(".dialog-modal .floating-menu-content");
+	const inCanvas = target && target.closest(".canvas");
+	const inDialog = target && target.closest(".dialog-modal .floating-menu-content");
 
-	if (dialogIsVisible() && !clickedInsideDialog) {
+	// Block middle mouse button auto-scroll mode
+	if (e.button === 1) e.preventDefault();
+
+	if (dialogIsVisible() && !inDialog) {
 		dismissDialog();
-
 		e.preventDefault();
 		e.stopPropagation();
 	}
+
+	if (inCanvas) viewportMouseInteractionOngoing = true;
+
+	if (viewportMouseInteractionOngoing) {
+		const modifiers = makeModifiersBitfield(e);
+		(await wasm).on_mouse_down(e.clientX, e.clientY, e.buttons, modifiers);
+	}
 }
 
-export function makeModifiersBitfield(control: boolean, shift: boolean, alt: boolean): number {
-	// eslint-disable-next-line no-bitwise
-	return Number(control) | (Number(shift) << 1) | (Number(alt) << 2);
+export async function onMouseUp(e: MouseEvent) {
+	if (!e.buttons) viewportMouseInteractionOngoing = false;
+
+	const modifiers = makeModifiersBitfield(e);
+	(await wasm).on_mouse_up(e.clientX, e.clientY, e.buttons, modifiers);
+}
+
+export async function onMouseScroll(e: WheelEvent) {
+	const target = e.target && (e.target as HTMLElement);
+	const inCanvas = target && target.closest(".canvas");
+
+	if (inCanvas) {
+		e.preventDefault();
+		const modifiers = makeModifiersBitfield(e);
+		(await wasm).on_mouse_scroll(e.clientX, e.clientY, e.buttons, e.deltaX, e.deltaY, e.deltaZ, modifiers);
+	}
+}
+
+export async function onWindowResize() {
+	const viewports = Array.from(document.querySelectorAll(".canvas"));
+	const boundsOfViewports = viewports.map((canvas) => {
+		const bounds = canvas.getBoundingClientRect();
+		return [bounds.left, bounds.top, bounds.right, bounds.bottom];
+	});
+
+	const flattened = boundsOfViewports.flat();
+	const data = Float64Array.from(flattened);
+
+	if (boundsOfViewports.length > 0) (await wasm).bounds_of_viewports(data);
+}
+
+export function makeModifiersBitfield(e: MouseEvent | KeyboardEvent): number {
+	// eslint-disable-next-line no-bitwise
+	return Number(e.ctrlKey) | (Number(e.shiftKey) << 1) | (Number(e.altKey) << 2);
 }

frontend/wasm/src/document.rs

@@ -2,11 +2,11 @@ use crate::shims::Error;
 use crate::wrappers::{translate_key, translate_tool, Color};
 use crate::EDITOR_STATE;
 use editor::input::input_preprocessor::ModifierKeys;
-use editor::input::mouse::ScrollDelta;
+use editor::input::mouse::{EditorMouseState, ScrollDelta, ViewportBounds};
 use editor::message_prelude::*;
 use editor::misc::EditorError;
 use editor::tool::{tool_options::ToolOptions, tools, ToolType};
-use editor::{input::mouse::MouseState, LayerId};
+use editor::LayerId;
 use graphene::layers::BlendMode;
 use wasm_bindgen::prelude::*;
 
@@ -104,47 +104,57 @@ pub fn close_all_documents_with_confirmation() -> Result<(), JsValue> {
 	EDITOR_STATE.with(|editor| editor.borrow_mut().handle_message(DocumentsMessage::CloseAllDocumentsWithConfirmation).map_err(convert_error))
 }
 
-// TODO: Call event when the panels are resized
-/// Viewport resized
+/// Send new bounds when document panel viewports get resized or moved within the editor
+/// [left, top, right, bottom]...
 #[wasm_bindgen]
-pub fn viewport_resize(new_width: f64, new_height: f64) -> Result<(), JsValue> {
-	let ev = InputPreprocessorMessage::ViewportResize((new_width, new_height).into());
+pub fn bounds_of_viewports(bounds_of_viewports: &[f64]) -> Result<(), JsValue> {
+	let chunked: Vec<_> = bounds_of_viewports.chunks(4).map(ViewportBounds::from_slice).collect();
+	let ev = InputPreprocessorMessage::BoundsOfViewports((chunked).into());
 	EDITOR_STATE.with(|editor| editor.borrow_mut().handle_message(ev)).map_err(convert_error)
 }
 
-// TODO: When a mouse button is down that started in the viewport, this should trigger even when the mouse is outside the viewport (or even the browser window if the browser supports it)
 /// Mouse movement within the screenspace bounds of the viewport
 #[wasm_bindgen]
-pub fn on_mouse_move(x: f64, y: f64, modifiers: u8) -> Result<(), JsValue> {
-	let mods = ModifierKeys::from_bits(modifiers).expect("invalid modifier keys");
+pub fn on_mouse_move(x: f64, y: f64, mouse_keys: u8, modifiers: u8) -> Result<(), JsValue> {
+	let editor_mouse_state = EditorMouseState::from_keys_and_editor_position(mouse_keys, (x, y).into());
 
-	// TODO: Convert these screenspace viewport coordinates to canvas coordinates based on the current zoom and pan
-	let ev = InputPreprocessorMessage::MouseMove((x, y).into(), mods);
+	let modifier_keys = ModifierKeys::from_bits(modifiers).expect("invalid modifier keys");
+
+	let ev = InputPreprocessorMessage::MouseMove(editor_mouse_state, modifier_keys);
 	EDITOR_STATE.with(|editor| editor.borrow_mut().handle_message(ev)).map_err(convert_error)
 }
 
 /// Mouse scrolling within the screenspace bounds of the viewport
 #[wasm_bindgen]
-pub fn on_mouse_scroll(delta_x: i32, delta_y: i32, delta_z: i32, modifiers: u8) -> Result<(), JsValue> {
-	// TODO: Convert these screenspace viewport coordinates to canvas coordinates based on the current zoom and pan
-	let mods = ModifierKeys::from_bits(modifiers).expect("invalid modifier keys");
-	let ev = InputPreprocessorMessage::MouseScroll(ScrollDelta::new(delta_x, delta_y, delta_z), mods);
+pub fn on_mouse_scroll(x: f64, y: f64, mouse_keys: u8, wheel_delta_x: i32, wheel_delta_y: i32, wheel_delta_z: i32, modifiers: u8) -> Result<(), JsValue> {
+	let mut editor_mouse_state = EditorMouseState::from_keys_and_editor_position(mouse_keys, (x, y).into());
+	editor_mouse_state.scroll_delta = ScrollDelta::new(wheel_delta_x, wheel_delta_y, wheel_delta_z);
+
+	let modifier_keys = ModifierKeys::from_bits(modifiers).expect("invalid modifier keys");
+
+	let ev = InputPreprocessorMessage::MouseScroll(editor_mouse_state, modifier_keys);
 	EDITOR_STATE.with(|editor| editor.borrow_mut().handle_message(ev)).map_err(convert_error)
 }
 
 /// A mouse button depressed within screenspace the bounds of the viewport
 #[wasm_bindgen]
 pub fn on_mouse_down(x: f64, y: f64, mouse_keys: u8, modifiers: u8) -> Result<(), JsValue> {
-	let mods = ModifierKeys::from_bits(modifiers).expect("invalid modifier keys");
-	let ev = InputPreprocessorMessage::MouseDown(MouseState::from_u8_pos(mouse_keys, (x, y).into()), mods);
+	let editor_mouse_state = EditorMouseState::from_keys_and_editor_position(mouse_keys, (x, y).into());
+
+	let modifier_keys = ModifierKeys::from_bits(modifiers).expect("invalid modifier keys");
+
+	let ev = InputPreprocessorMessage::MouseDown(editor_mouse_state, modifier_keys);
 	EDITOR_STATE.with(|editor| editor.borrow_mut().handle_message(ev)).map_err(convert_error)
 }
 
 /// A mouse button released
 #[wasm_bindgen]
 pub fn on_mouse_up(x: f64, y: f64, mouse_keys: u8, modifiers: u8) -> Result<(), JsValue> {
-	let mods = ModifierKeys::from_bits(modifiers).expect("invalid modifier keys");
-	let ev = InputPreprocessorMessage::MouseUp(MouseState::from_u8_pos(mouse_keys, (x, y).into()), mods);
+	let editor_mouse_state = EditorMouseState::from_keys_and_editor_position(mouse_keys, (x, y).into());
+
+	let modifier_keys = ModifierKeys::from_bits(modifiers).expect("invalid modifier keys");
+
+	let ev = InputPreprocessorMessage::MouseUp(editor_mouse_state, modifier_keys);
 	EDITOR_STATE.with(|editor| editor.borrow_mut().handle_message(ev)).map_err(convert_error)
 }