Add draggable skew triangles to the transform cage (#2300)

* Add triangle handles to transform cage for skew transform

Fixes #2299

* Add skew triangles

* Fix conflicts which github didn't show

* cargo fmt

* Fix needed

* remove unreachable

* use the trap and rect logic

* fix quad checks

* cursor fix; no triangles if already dragging and not skewing

* cargo fmt

* Resolve Clippy lints

* Add min length for triangle visibility

* Code review

---------

Co-authored-by: Keavon Chambers <keavon@keavon.com>

editor/src/messages/tool/common_functionality/graph_modification_utils.rs

@@ -48,15 +48,12 @@ pub fn merge_layers(document: &DocumentMessageHandler, first_layer: LayerNodeIde
 
 	let mut current_and_other_layer_is_spline = false;
 
-	match (find_spline(document, first_layer), find_spline(document, second_layer)) {
-		(Some(current_layer_spline), Some(other_layer_spline)) => {
-			responses.add(NodeGraphMessage::DeleteNodes {
-				node_ids: [current_layer_spline, other_layer_spline].to_vec(),
-				delete_children: false,
-			});
-			current_and_other_layer_is_spline = true;
-		}
-		_ => {}
+	if let (Some(current_layer_spline), Some(other_layer_spline)) = (find_spline(document, first_layer), find_spline(document, second_layer)) {
+		responses.add(NodeGraphMessage::DeleteNodes {
+			node_ids: [current_layer_spline, other_layer_spline].to_vec(),
+			delete_children: false,
+		});
+		current_and_other_layer_is_spline = true;
 	}
 
 	// Move the `second_layer` below the `first_layer` for positioning purposes

editor/src/messages/tool/common_functionality/snapping.rs

@@ -324,10 +324,10 @@ impl SnapManager {
 		let layer_bounds = document.metadata().transform_to_document(layer) * Quad::from_box(bounds);
 		let screen_bounds = document.metadata().document_to_viewport.inverse() * Quad::from_box([DVec2::ZERO, snap_data.input.viewport_bounds.size()]);
 		if screen_bounds.intersects(layer_bounds) {
-			if !self.alignment_candidates.as_ref().is_some_and(|candidates| candidates.len() > 100) {
+			if self.alignment_candidates.as_ref().is_none_or(|candidates| candidates.len() <= 100) {
 				self.alignment_candidates.get_or_insert_with(Vec::new).push(layer);
 			}
-			if quad.intersects(layer_bounds) && !self.candidates.as_ref().is_some_and(|candidates| candidates.len() > 10) {
+			if quad.intersects(layer_bounds) && self.candidates.as_ref().is_none_or(|candidates| candidates.len() <= 10) {
 				self.candidates.get_or_insert_with(Vec::new).push(layer);
 			}
 		}

editor/src/messages/tool/common_functionality/transformation_cage.rs

@@ -1,6 +1,7 @@
 use crate::consts::{
 	BOUNDS_ROTATE_THRESHOLD, BOUNDS_SELECT_THRESHOLD, COLOR_OVERLAY_WHITE, MAXIMUM_ALT_SCALE_FACTOR, MIN_LENGTH_FOR_CORNERS_VISIBILITY, MIN_LENGTH_FOR_EDGE_RESIZE_PRIORITY_OVER_CORNERS,
-	MIN_LENGTH_FOR_MIDPOINT_VISIBILITY, MIN_LENGTH_FOR_RESIZE_TO_INCLUDE_INTERIOR, RESIZE_HANDLE_SIZE, SELECTION_DRAG_ANGLE,
+	MIN_LENGTH_FOR_MIDPOINT_VISIBILITY, MIN_LENGTH_FOR_RESIZE_TO_INCLUDE_INTERIOR, MIN_LENGTH_FOR_SKEW_TRIANGLE_VISIBILITY, RESIZE_HANDLE_SIZE, SELECTION_DRAG_ANGLE, SKEW_TRIANGLE_OFFSET,
+	SKEW_TRIANGLE_SIZE,
 };
 use crate::messages::frontend::utility_types::MouseCursorIcon;
 use crate::messages::portfolio::document::overlays::utility_types::OverlayContext;
@@ -15,6 +16,9 @@ use glam::{DAffine2, DMat2, DVec2};
 
 use super::snapping::{self, SnapCandidatePoint, SnapConstraint, SnapData, SnapManager, SnappedPoint};
 
+/// (top, bottom, left, right)
+pub type EdgeBool = (bool, bool, bool, bool);
+
 pub struct SizeSnapData<'a> {
 	pub manager: &'a mut SnapManager,
 	pub points: &'a mut Vec<SnapCandidatePoint>,
@@ -250,7 +254,8 @@ impl SelectedEdges {
 		(DAffine2::from_scale(enlargement_factor), pivot)
 	}
 
-	pub fn skew_transform(&self, mouse: DVec2, to_viewport_transform: DAffine2) -> DAffine2 {
+	// TODO: Add free movement when Ctrl is pressed to allow dragging the whole edge, not just sliding it
+	pub fn skew_transform(&self, mouse: DVec2, to_viewport_transform: DAffine2, _free_movement: bool) -> DAffine2 {
 		// Skip if the matrix is singular (as it isn't really possible to skew).
 		if !to_viewport_transform.matrix2.determinant().recip().is_finite() {
 			return DAffine2::IDENTITY;
@@ -377,6 +382,171 @@ impl BoundingBoxManager {
 		]
 	}
 
+	pub fn get_closest_edge(&self, edges: EdgeBool, cursor: DVec2) -> EdgeBool {
+		if !edges.0 && !edges.1 && !edges.2 && !edges.3 {
+			return (false, false, false, false);
+		}
+
+		let cursor = self.transform.inverse().transform_point2(cursor);
+		let min = self.bounds[0].min(self.bounds[1]);
+		let max = self.bounds[0].max(self.bounds[1]);
+
+		let distances = [
+			edges.0.then(|| (cursor - DVec2::new(cursor.x, min.y)).length_squared()),
+			edges.1.then(|| (cursor - DVec2::new(cursor.x, max.y)).length_squared()),
+			edges.2.then(|| (cursor - DVec2::new(min.x, cursor.y)).length_squared()),
+			edges.3.then(|| (cursor - DVec2::new(max.x, cursor.y)).length_squared()),
+		];
+
+		let min_distance = distances.iter().filter_map(|&x| x).min_by(|a, b| a.partial_cmp(b).unwrap());
+
+		match min_distance {
+			Some(min) => (
+				edges.0 && distances[0].is_some_and(|d| (d - min).abs() < f64::EPSILON),
+				edges.1 && distances[1].is_some_and(|d| (d - min).abs() < f64::EPSILON),
+				edges.2 && distances[2].is_some_and(|d| (d - min).abs() < f64::EPSILON),
+				edges.3 && distances[3].is_some_and(|d| (d - min).abs() < f64::EPSILON),
+			),
+			None => (false, false, false, false),
+		}
+	}
+
+	pub fn check_skew_handle(&self, cursor: DVec2, edge: EdgeBool) -> bool {
+		if let Some([start, end]) = self.edge_endpoints_vector_from_edge_bool(edge) {
+			if (end - start).length() < MIN_LENGTH_FOR_SKEW_TRIANGLE_VISIBILITY {
+				return false;
+			}
+
+			let touches_triangle = |base: DVec2, direction: DVec2, cursor: DVec2| -> bool {
+				let normal = direction.perp();
+				let top = base + direction * SKEW_TRIANGLE_SIZE;
+				let edge1 = base + normal * SKEW_TRIANGLE_SIZE / 2.;
+				let edge2 = base - normal * SKEW_TRIANGLE_SIZE / 2.;
+
+				let v0 = edge1 - top;
+				let v1 = edge2 - top;
+				let v2 = cursor - top;
+
+				let d00 = v0.dot(v0);
+				let d01 = v0.dot(v1);
+				let d11 = v1.dot(v1);
+				let d20 = v2.dot(v0);
+				let d21 = v2.dot(v1);
+
+				let denom = d00 * d11 - d01 * d01;
+				let v = (d11 * d20 - d01 * d21) / denom;
+				let w = (d00 * d21 - d01 * d20) / denom;
+				let u = 1. - v - w;
+
+				u >= 0. && v >= 0. && w >= 0.
+			};
+
+			let edge_dir = (end - start).normalize();
+			let mid = end.midpoint(start);
+
+			for direction in [edge_dir, -edge_dir] {
+				let base = mid + direction * (3. + SKEW_TRIANGLE_OFFSET);
+				if touches_triangle(base, direction, cursor) {
+					return true;
+				}
+			}
+		}
+
+		false
+	}
+
+	pub fn edge_endpoints_vector_from_edge_bool(&self, edges: EdgeBool) -> Option<[DVec2; 2]> {
+		let quad = self.transform * Quad::from_box(self.bounds);
+		let category = self.overlay_display_category();
+
+		if matches!(
+			category,
+			TransformCageSizeCategory::Full | TransformCageSizeCategory::Narrow | TransformCageSizeCategory::ReducedLandscape
+		) {
+			if edges.0 {
+				return Some([quad.top_left(), quad.top_right()]);
+			}
+			if edges.1 {
+				return Some([quad.bottom_left(), quad.bottom_right()]);
+			}
+		}
+
+		if matches!(
+			category,
+			TransformCageSizeCategory::Full | TransformCageSizeCategory::Narrow | TransformCageSizeCategory::ReducedPortrait
+		) {
+			if edges.2 {
+				return Some([quad.top_left(), quad.bottom_left()]);
+			}
+			if edges.3 {
+				return Some([quad.top_right(), quad.bottom_right()]);
+			}
+		}
+		None
+	}
+
+	pub fn render_skew_gizmos(&mut self, overlay_context: &mut OverlayContext, hover_edge: EdgeBool) {
+		let mut draw_edge_triangles = |start: DVec2, end: DVec2| {
+			if (end - start).length() < MIN_LENGTH_FOR_SKEW_TRIANGLE_VISIBILITY {
+				return;
+			}
+
+			let edge_dir = (end - start).normalize();
+			let mid = end.midpoint(start);
+
+			for edge in [edge_dir, -edge_dir] {
+				overlay_context.draw_triangle(mid + edge * (3. + SKEW_TRIANGLE_OFFSET), edge, SKEW_TRIANGLE_SIZE, None, None);
+			}
+		};
+
+		if let Some([start, end]) = self.edge_endpoints_vector_from_edge_bool(hover_edge) {
+			draw_edge_triangles(start, end);
+		}
+	}
+
+	pub fn over_extended_edge_midpoint(&self, mouse: DVec2, hover_edge: EdgeBool) -> bool {
+		const HALF_WIDTH_OUTER_RECT: f64 = RESIZE_HANDLE_SIZE / 2. + SKEW_TRIANGLE_OFFSET + SKEW_TRIANGLE_SIZE;
+		const HALF_WIDTH_INNER_RECT: f64 = SKEW_TRIANGLE_OFFSET + RESIZE_HANDLE_SIZE / 2.;
+
+		const INNER_QUAD_CORNER: DVec2 = DVec2::new(HALF_WIDTH_INNER_RECT, RESIZE_HANDLE_SIZE / 2.);
+		const FULL_QUAD_CORNER: DVec2 = DVec2::new(HALF_WIDTH_OUTER_RECT, BOUNDS_SELECT_THRESHOLD);
+
+		let quad = self.transform * Quad::from_box(self.bounds);
+
+		let Some([start, end]) = self.edge_endpoints_vector_from_edge_bool(hover_edge) else {
+			return false;
+		};
+		if (end - start).length() < MIN_LENGTH_FOR_SKEW_TRIANGLE_VISIBILITY {
+			return false;
+		}
+
+		let angle;
+		let is_compact;
+		if hover_edge.0 || hover_edge.1 {
+			angle = (quad.top_left() - quad.top_right()).to_angle();
+			is_compact = (quad.top_left() - quad.bottom_left()).length_squared() < MIN_LENGTH_FOR_RESIZE_TO_INCLUDE_INTERIOR.powi(2);
+		} else if hover_edge.2 || hover_edge.3 {
+			angle = (quad.top_left() - quad.bottom_left()).to_angle();
+			is_compact = (quad.top_left() - quad.top_right()).length_squared() < MIN_LENGTH_FOR_RESIZE_TO_INCLUDE_INTERIOR.powi(2);
+		} else {
+			return false;
+		};
+
+		let has_triangle_hover = self.check_skew_handle(mouse, hover_edge);
+		let point = start.midpoint(end);
+
+		if is_compact {
+			let upper_rect = DAffine2::from_angle_translation(angle, point) * Quad::from_box([-FULL_QUAD_CORNER.with_y(0.), FULL_QUAD_CORNER]);
+			let inter_triangle_quad = DAffine2::from_angle_translation(angle, point) * Quad::from_box([-INNER_QUAD_CORNER, INNER_QUAD_CORNER]);
+
+			upper_rect.contains(mouse) || has_triangle_hover || inter_triangle_quad.contains(mouse)
+		} else {
+			let rect = DAffine2::from_angle_translation(angle, point) * Quad::from_box([-FULL_QUAD_CORNER, FULL_QUAD_CORNER]);
+
+			rect.contains(mouse) || has_triangle_hover
+		}
+	}
+
 	/// Update the position of the bounding box and transform handles
 	pub fn render_overlays(&mut self, overlay_context: &mut OverlayContext) {
 		let quad = self.transform * Quad::from_box(self.bounds);
@@ -388,18 +558,20 @@ impl BoundingBoxManager {
 		// Draw the bounding box rectangle
 		overlay_context.quad(quad, None);
 
-		let mut draw_handle = |point: DVec2| {
-			let quad = DAffine2::from_angle_translation((quad.top_left() - quad.top_right()).to_angle(), point)
-				* Quad::from_box([DVec2::splat(-RESIZE_HANDLE_SIZE / 2.), DVec2::splat(RESIZE_HANDLE_SIZE / 2.)]);
+		let mut draw_handle = |point: DVec2, angle: f64| {
+			let quad = DAffine2::from_angle_translation(angle, point) * Quad::from_box([DVec2::splat(-RESIZE_HANDLE_SIZE / 2.), DVec2::splat(RESIZE_HANDLE_SIZE / 2.)]);
 			overlay_context.quad(quad, Some(COLOR_OVERLAY_WHITE));
 		};
 
+		let horizontal_angle = (quad.top_left() - quad.bottom_left()).to_angle();
+		let vertical_angle = (quad.top_left() - quad.top_right()).to_angle();
+
 		// Draw the horizontal midpoint drag handles
 		if matches!(
 			category,
 			TransformCageSizeCategory::Full | TransformCageSizeCategory::Narrow | TransformCageSizeCategory::ReducedLandscape
 		) {
-			horizontal_edges.map(&mut draw_handle);
+			horizontal_edges.map(|point| draw_handle(point, horizontal_angle));
 		}
 
 		// Draw the vertical midpoint drag handles
@@ -407,21 +579,28 @@ impl BoundingBoxManager {
 			category,
 			TransformCageSizeCategory::Full | TransformCageSizeCategory::Narrow | TransformCageSizeCategory::ReducedPortrait
 		) {
-			vertical_edges.map(&mut draw_handle);
+			vertical_edges.map(|point| draw_handle(point, vertical_angle));
 		}
 
+		let angle = quad
+			.edges()
+			.map(|[x, y]| x.distance_squared(y))
+			.into_iter()
+			.reduce(|horizontal_distance, vertical_distance| if horizontal_distance > vertical_distance { horizontal_angle } else { vertical_angle })
+			.unwrap_or_default();
+
 		// Draw the corner drag handles
 		if matches!(
 			category,
 			TransformCageSizeCategory::Full | TransformCageSizeCategory::ReducedBoth | TransformCageSizeCategory::ReducedLandscape | TransformCageSizeCategory::ReducedPortrait
 		) {
-			quad.0.map(&mut draw_handle);
+			quad.0.map(|point| draw_handle(point, angle));
 		}
 
 		// Draw the flat line endpoint drag handles
 		if category == TransformCageSizeCategory::Flat {
-			draw_handle(self.transform.transform_point2(self.bounds[0]));
-			draw_handle(self.transform.transform_point2(self.bounds[1]));
+			draw_handle(self.transform.transform_point2(self.bounds[0]), angle);
+			draw_handle(self.transform.transform_point2(self.bounds[1]), angle);
 		}
 	}
 
@@ -482,7 +661,7 @@ impl BoundingBoxManager {
 	/// Returns which edge in the order:
 	///
 	/// `top, bottom, left, right`
-	pub fn check_selected_edges(&self, cursor: DVec2) -> Option<(bool, bool, bool, bool)> {
+	pub fn check_selected_edges(&self, cursor: DVec2) -> Option<EdgeBool> {
 		let cursor = self.transform.inverse().transform_point2(cursor);
 
 		let min = self.bounds[0].min(self.bounds[1]);
@@ -571,9 +750,22 @@ impl BoundingBoxManager {
 	}
 
 	/// Gets the required mouse cursor to show resizing bounds or optionally rotation
-	pub fn get_cursor(&self, input: &InputPreprocessorMessageHandler, rotate: bool) -> MouseCursorIcon {
+	pub fn get_cursor(&self, input: &InputPreprocessorMessageHandler, rotate: bool, dragging_bounds: bool, skew_edge: Option<EdgeBool>) -> MouseCursorIcon {
 		let edges = self.check_selected_edges(input.mouse.position);
 
+		let is_near_square = edges.is_some_and(|hover_edge| self.over_extended_edge_midpoint(input.mouse.position, hover_edge));
+		if dragging_bounds && is_near_square {
+			if let Some(skew_edge) = skew_edge {
+				if self.check_skew_handle(input.mouse.position, skew_edge) {
+					if skew_edge.0 || skew_edge.1 {
+						return MouseCursorIcon::EWResize;
+					} else if skew_edge.2 || skew_edge.3 {
+						return MouseCursorIcon::NSResize;
+					}
+				}
+			};
+		}
+
 		match edges {
 			Some((top, bottom, left, right)) if !self.is_bounds_flat() => match (top, bottom, left, right) {
 				(true, _, false, false) | (_, true, false, false) => MouseCursorIcon::NSResize,
@@ -599,7 +791,7 @@ fn skew_transform_singular() {
 		// The determinant is 0.
 		let transform = DAffine2::from_cols_array(&[2.; 6]);
 		// This shouldn't panic. We don't really care about the behavior in this test.
-		let _ = edge.skew_transform(DVec2::new(1.5, 1.5), transform);
+		let _ = edge.skew_transform(DVec2::new(1.5, 1.5), transform, false);
 	}
 }
 
@@ -615,7 +807,7 @@ fn skew_transform_correct() {
 		let to_viewport_transform = DAffine2::from_cols_array(&[2., 1., 0., 1., 2., 3.]);
 		// Random mouse position.
 		let mouse = DVec2::new(1.5, 1.5);
-		let final_transform = edge.skew_transform(mouse, to_viewport_transform);
+		let final_transform = edge.skew_transform(mouse, to_viewport_transform, false);
 
 		// This is the current handle that goes under the mouse.
 		let dragging_point = edge.pivot_from_bounds(edge.bounds[1], edge.bounds[0]);