Fix self-chaining of transforms; fix compass rose getting offset when rotating a layer (#2296)

* Fix self-chaining of transforms and compass rose under single layer

https://discord.com/channels/731730685944922173/931942323644928040/1340632846863175702
https://discord.com/channels/731730685944922173/931942323644928040/1340608972071243906

* When not invertible transformation, do nothing

* Fix overlays and compass control when can't be visible

* Simplify selection logic in compass states

* Show compass only if it was possible that it could be seen before dragging

* Prevent resizing line objects

* Code review

---------

Co-authored-by: Keavon Chambers <keavon@keavon.com>

editor/src/messages/tool/common_functionality/compass_rose.rs

@@ -1,4 +1,5 @@
 use crate::consts::{COMPASS_ROSE_ARROW_CLICK_TARGET_ANGLE, COMPASS_ROSE_HOVER_RING_DIAMETER, COMPASS_ROSE_RING_INNER_DIAMETER};
+use crate::messages::portfolio::document::utility_types::document_metadata::LayerNodeIdentifier;
 use crate::messages::prelude::DocumentMessageHandler;
 
 use glam::{DAffine2, DVec2};
@@ -10,9 +11,27 @@ pub struct CompassRose {
 }
 
 impl CompassRose {
-	pub fn refresh_transform(&mut self, document: &DocumentMessageHandler) {
-		let [min, max] = document.selected_visible_and_unlock_layers_bounding_box_viewport().unwrap_or([DVec2::ZERO, DVec2::ONE]);
-		self.compass_center = (DAffine2::from_translation(min) * DAffine2::from_scale(max - min)).transform_point2(DVec2::splat(0.5));
+	fn get_layer_pivot_transform(layer: LayerNodeIdentifier, document: &DocumentMessageHandler) -> DAffine2 {
+		let [min, max] = document.metadata().nonzero_bounding_box(layer);
+
+		let bounds_transform = DAffine2::from_translation(min) * DAffine2::from_scale(max - min);
+		let layer_transform = document.metadata().transform_to_viewport(layer);
+		layer_transform * bounds_transform
+	}
+	pub fn refresh_position(&mut self, document: &DocumentMessageHandler) {
+		let selected_nodes = document.network_interface.selected_nodes(&[]).unwrap();
+		let mut layers = selected_nodes.selected_visible_and_unlocked_layers(&document.network_interface);
+
+		let Some(first) = layers.next() else { return };
+		let count = layers.count() + 1;
+		let transform = if count == 1 {
+			Self::get_layer_pivot_transform(first, document)
+		} else {
+			let [min, max] = document.selected_visible_and_unlock_layers_bounding_box_viewport().unwrap_or([DVec2::ZERO, DVec2::ONE]);
+			DAffine2::from_translation(min) * DAffine2::from_scale(max - min)
+		};
+
+		self.compass_center = transform.transform_point2(DVec2::splat(0.5));
 	}
 
 	pub fn compass_rose_position(&self) -> DVec2 {

editor/src/messages/tool/common_functionality/pivot.rs

@@ -111,11 +111,12 @@ impl Pivot {
 			.selected_visible_and_unlocked_layers(&document.network_interface)
 		{
 			let transform = Self::get_layer_pivot_transform(layer, document);
+			// Only update the pivot when computed position is finite.
+			if transform.matrix2.determinant().abs() <= f64::EPSILON {
+				return;
+			};
 			let pivot = transform.inverse().transform_point2(position);
-			// Only update the pivot when computed position is finite. Infinite can happen when scale is 0.
-			if pivot.is_finite() {
-				responses.add(GraphOperationMessage::TransformSetPivot { layer, pivot });
-			}
+			responses.add(GraphOperationMessage::TransformSetPivot { layer, pivot });
 		}
 	}
 

editor/src/messages/tool/common_functionality/transformation_cage.rs

@@ -411,7 +411,7 @@ impl BoundingBoxManager {
 
 	fn overlay_display_category(&self, quad: Quad) -> HandleDisplayCategory {
 		// Check if the area is essentially zero because either the width or height is smaller than an epsilon
-		if (self.bounds[0] - self.bounds[1]).abs().cmple(DVec2::splat(1e-4)).any() {
+		if self.is_bounds_flat() {
 			return HandleDisplayCategory::Flat;
 		}
 
@@ -434,6 +434,10 @@ impl BoundingBoxManager {
 		HandleDisplayCategory::Narrow
 	}
 
+	fn is_bounds_flat(&self) -> bool {
+		(self.bounds[0] - self.bounds[1]).abs().cmple(DVec2::splat(1e-4)).any()
+	}
+
 	/// Compute the threshold in viewport space. This only works with affine transforms as it assumes lines remain parallel.
 	fn compute_viewport_threshold(&self, scalar: f64) -> [f64; 2] {
 		let inverse = self.transform.inverse();
@@ -521,18 +525,18 @@ impl BoundingBoxManager {
 
 	/// Gets the required mouse cursor to show resizing bounds or optionally rotation
 	pub fn get_cursor(&self, input: &InputPreprocessorMessageHandler, rotate: bool) -> MouseCursorIcon {
-		if let Some((top, bottom, left, right)) = self.check_selected_edges(input.mouse.position) {
-			match (top, bottom, left, right) {
+		let edges = self.check_selected_edges(input.mouse.position);
+
+		match edges {
+			Some((top, bottom, left, right)) if self.is_bounds_flat() => match (top, bottom, left, right) {
 				(true, _, false, false) | (_, true, false, false) => MouseCursorIcon::NSResize,
 				(false, false, true, _) | (false, false, _, true) => MouseCursorIcon::EWResize,
 				(true, _, true, _) | (_, true, _, true) => MouseCursorIcon::NWSEResize,
 				(true, _, _, true) | (_, true, true, _) => MouseCursorIcon::NESWResize,
 				_ => MouseCursorIcon::Default,
-			}
-		} else if rotate && self.check_rotate(input.mouse.position) {
-			MouseCursorIcon::Rotate
-		} else {
-			MouseCursorIcon::Default
+			},
+			_ if rotate && self.check_rotate(input.mouse.position) => MouseCursorIcon::Rotate,
+			_ => MouseCursorIcon::Default,
 		}
 	}
 }