Make Select tool hover overlay in 'shallow select' mode visualize the next deeper object (#2639)

* Fix overlay behaviour on hovering over a layer

* Add the deepen overlay code

* cleanup

editor/src/messages/portfolio/document/overlays/utility_types.rs

@@ -140,8 +140,8 @@ impl core::hash::Hash for OverlayContext {
 }
 
 impl OverlayContext {
-	pub fn quad(&mut self, quad: Quad, color_fill: Option<&str>) {
+	pub fn quad(&mut self, quad: Quad, stroke_color: Option<&str>, color_fill: Option<&str>) {
-		self.dashed_polygon(&quad.0, color_fill, None, None, None);
+		self.dashed_polygon(&quad.0, stroke_color, color_fill, None, None, None);
 	}
 
 	pub fn draw_triangle(&mut self, base: DVec2, direction: DVec2, size: f64, color_fill: Option<&str>, color_stroke: Option<&str>) {
@@ -168,15 +168,15 @@ impl OverlayContext {
 		self.end_dpi_aware_transform();
 	}
 
-	pub fn dashed_quad(&mut self, quad: Quad, color_fill: Option<&str>, dash_width: Option<f64>, dash_gap_width: Option<f64>, dash_offset: Option<f64>) {
+	pub fn dashed_quad(&mut self, quad: Quad, stroke_color: Option<&str>, color_fill: Option<&str>, dash_width: Option<f64>, dash_gap_width: Option<f64>, dash_offset: Option<f64>) {
-		self.dashed_polygon(&quad.0, color_fill, dash_width, dash_gap_width, dash_offset);
+		self.dashed_polygon(&quad.0, stroke_color, color_fill, dash_width, dash_gap_width, dash_offset);
 	}
 
-	pub fn polygon(&mut self, polygon: &[DVec2], color_fill: Option<&str>) {
+	pub fn polygon(&mut self, polygon: &[DVec2], stroke_color: Option<&str>, color_fill: Option<&str>) {
-		self.dashed_polygon(polygon, color_fill, None, None, None);
+		self.dashed_polygon(polygon, stroke_color, color_fill, None, None, None);
 	}
 
-	pub fn dashed_polygon(&mut self, polygon: &[DVec2], color_fill: Option<&str>, dash_width: Option<f64>, dash_gap_width: Option<f64>, dash_offset: Option<f64>) {
+	pub fn dashed_polygon(&mut self, polygon: &[DVec2], stroke_color: Option<&str>, color_fill: Option<&str>, dash_width: Option<f64>, dash_gap_width: Option<f64>, dash_offset: Option<f64>) {
 		if polygon.len() < 2 {
 			return;
 		}
@@ -214,7 +214,8 @@ impl OverlayContext {
 			self.render_context.fill();
 		}
 
-		self.render_context.set_stroke_style_str(COLOR_OVERLAY_BLUE);
+		let stroke_color = stroke_color.unwrap_or(COLOR_OVERLAY_BLUE);
+		self.render_context.set_stroke_style_str(stroke_color);
 		self.render_context.stroke();
 
 		// Reset the dash pattern back to solid
@@ -647,10 +648,11 @@ impl OverlayContext {
 	}
 
 	/// Used by the Select tool to outline a path selected or hovered.
-	pub fn outline(&mut self, subpaths: impl Iterator<Item = impl Borrow<Subpath<PointId>>>, transform: DAffine2) {
+	pub fn outline(&mut self, subpaths: impl Iterator<Item = impl Borrow<Subpath<PointId>>>, transform: DAffine2, color: Option<&str>) {
 		self.push_path(subpaths, transform);
 
-		self.render_context.set_stroke_style_str(COLOR_OVERLAY_BLUE);
+		let color = color.unwrap_or(COLOR_OVERLAY_BLUE);
+		self.render_context.set_stroke_style_str(color);
 		self.render_context.stroke();
 	}
 

editor/src/messages/tool/common_functionality/snapping.rs

@@ -449,10 +449,10 @@ impl SnapManager {
 		if let Some(ind) = &self.indicator {
 			for layer in &ind.outline_layers {
 				let &Some(layer) = layer else { continue };
-				overlay_context.outline(snap_data.document.metadata().layer_outline(layer), snap_data.document.metadata().transform_to_viewport(layer));
+				overlay_context.outline(snap_data.document.metadata().layer_outline(layer), snap_data.document.metadata().transform_to_viewport(layer), None);
 			}
 			if let Some(quad) = ind.target_bounds {
-				overlay_context.quad(to_viewport * quad, None);
+				overlay_context.quad(to_viewport * quad, None, None);
 			}
 			let viewport = to_viewport.transform_point2(ind.snapped_point_document);
 

editor/src/messages/tool/common_functionality/transformation_cage.rs

@@ -570,7 +570,7 @@ impl BoundingBoxManager {
 		let quad = self.transform * Quad::from_box(self.bounds);
 
 		// Draw the bounding box rectangle
-		overlay_context.quad(quad, None);
+		overlay_context.quad(quad, None, None);
 	}
 
 	/// Update the position of the bounding box and transform handles
@@ -587,7 +587,7 @@ impl BoundingBoxManager {
 
 		let mut draw_handle = |point: DVec2, angle: f64| {
 			let quad = DAffine2::from_angle_translation(angle, point) * Quad::from_box([DVec2::splat(-RESIZE_HANDLE_SIZE / 2.), DVec2::splat(RESIZE_HANDLE_SIZE / 2.)]);
-			overlay_context.quad(quad, Some(COLOR_OVERLAY_WHITE));
+			overlay_context.quad(quad, None, Some(COLOR_OVERLAY_WHITE));
 		};
 
 		let horizontal_angle = (quad.top_left() - quad.bottom_left()).to_angle();

editor/src/messages/tool/tool_messages/path_tool.rs

@@ -1064,10 +1064,10 @@ impl Fsm for PathToolFsmState {
 						let polygon = &tool_data.lasso_polygon;
 
 						match (selection_shape, selection_mode) {
-							(SelectionShapeType::Box, SelectionMode::Enclosed) => overlay_context.dashed_quad(quad, fill_color, Some(4.), Some(4.), Some(0.5)),
+							(SelectionShapeType::Box, SelectionMode::Enclosed) => overlay_context.dashed_quad(quad, None, fill_color, Some(4.), Some(4.), Some(0.5)),
-							(SelectionShapeType::Lasso, SelectionMode::Enclosed) => overlay_context.dashed_polygon(polygon, fill_color, Some(4.), Some(4.), Some(0.5)),
+							(SelectionShapeType::Lasso, SelectionMode::Enclosed) => overlay_context.dashed_polygon(polygon, None, fill_color, Some(4.), Some(4.), Some(0.5)),
-							(SelectionShapeType::Box, _) => overlay_context.quad(quad, fill_color),
+							(SelectionShapeType::Box, _) => overlay_context.quad(quad, None, fill_color),
-							(SelectionShapeType::Lasso, _) => overlay_context.polygon(polygon, fill_color),
+							(SelectionShapeType::Lasso, _) => overlay_context.polygon(polygon, None, fill_color),
 						}
 					}
 					Self::Dragging(_) => {

editor/src/messages/tool/tool_messages/select_tool.rs

@@ -527,11 +527,11 @@ impl Fsm for SelectToolFsmState {
 						.selected_visible_and_unlocked_layers(&document.network_interface)
 						.filter(|layer| !document.network_interface.is_artboard(&layer.to_node(), &[]))
 					{
-						overlay_context.outline(document.metadata().layer_outline(layer), document.metadata().transform_to_viewport(layer));
+						overlay_context.outline(document.metadata().layer_outline(layer), document.metadata().transform_to_viewport(layer), None);
 
 						if is_layer_fed_by_node_of_name(layer, &document.network_interface, "Text") {
 							let transformed_quad = document.metadata().transform_to_viewport(layer) * text_bounding_box(layer, document, font_cache);
-							overlay_context.dashed_quad(transformed_quad, None, Some(7.), Some(5.), None);
+							overlay_context.dashed_quad(transformed_quad, None, None, Some(7.), Some(5.), None);
 						}
 					}
 				}
@@ -573,7 +573,38 @@ impl Fsm for SelectToolFsmState {
 					let not_selected_click = click.filter(|&hovered_layer| !document.network_interface.selected_nodes().selected_layers_contains(hovered_layer, document.metadata()));
 					if let Some(layer) = not_selected_click {
 						if overlay_context.visibility_settings.hover_outline() {
-							overlay_context.outline(document.metadata().layer_outline(layer), document.metadata().transform_to_viewport(layer));
+							let mut hover_overlay_draw = |layer: LayerNodeIdentifier, color: Option<&str>| {
+								if layer.has_children(document.metadata()) {
+									if let Some(bounds) = document.metadata().bounding_box_viewport(layer) {
+										overlay_context.quad(Quad::from_box(bounds), color, None);
+									}
+								} else {
+									overlay_context.outline(document.metadata().layer_outline(layer), document.metadata().transform_to_viewport(layer), color);
+								}
+							};
+							let layer = match tool_data.nested_selection_behavior {
+								NestedSelectionBehavior::Deepest => document.find_deepest(&[layer]),
+								NestedSelectionBehavior::Shallowest => layer_selected_shallowest(layer, document),
+							}
+							.unwrap_or(layer);
+							hover_overlay_draw(layer, None);
+							if matches!(tool_data.nested_selection_behavior, NestedSelectionBehavior::Shallowest) {
+								let mut selected = document.network_interface.selected_nodes();
+								selected.add_selected_nodes(vec![layer.to_node()]);
+								if let Some(new_selected) = click.unwrap().ancestors(document.metadata()).filter(not_artboard(document)).find(|ancestor| {
+									ancestor
+										.parent(document.metadata())
+										.is_some_and(|parent| selected.selected_layers_contains(parent, document.metadata()))
+								}) {
+									let mut fill_color = graphene_std::Color::from_rgb_str(COLOR_OVERLAY_BLUE.strip_prefix('#').unwrap())
+										.unwrap()
+										.with_alpha(0.5)
+										.to_rgba_hex_srgb();
+									fill_color.insert(0, '#');
+									let fill_color = Some(fill_color.as_str());
+									hover_overlay_draw(new_selected, fill_color);
+								}
+							}
 						}
 
 						// Measure with Alt held down
@@ -786,7 +817,7 @@ impl Fsm for SelectToolFsmState {
 					if overlay_context.visibility_settings.selection_outline() {
 						// Draws a temporary outline on the layers that will be selected by the current box/lasso area
 						for layer in layers_to_outline {
-							overlay_context.outline(document.metadata().layer_outline(layer), document.metadata().transform_to_viewport(layer));
+							overlay_context.outline(document.metadata().layer_outline(layer), document.metadata().transform_to_viewport(layer), None);
 						}
 					}
 
@@ -801,10 +832,10 @@ impl Fsm for SelectToolFsmState {
 					let polygon = &tool_data.lasso_polygon;
 
 					match (selection_shape, current_selection_mode) {
-						(SelectionShapeType::Box, SelectionMode::Enclosed) => overlay_context.dashed_quad(quad, fill_color, Some(4.), Some(4.), Some(0.5)),
+						(SelectionShapeType::Box, SelectionMode::Enclosed) => overlay_context.dashed_quad(quad, None, fill_color, Some(4.), Some(4.), Some(0.5)),
-						(SelectionShapeType::Lasso, SelectionMode::Enclosed) => overlay_context.dashed_polygon(polygon, fill_color, Some(4.), Some(4.), Some(0.5)),
+						(SelectionShapeType::Lasso, SelectionMode::Enclosed) => overlay_context.dashed_polygon(polygon, None, fill_color, Some(4.), Some(4.), Some(0.5)),
-						(SelectionShapeType::Box, _) => overlay_context.quad(quad, fill_color),
+						(SelectionShapeType::Box, _) => overlay_context.quad(quad, None, fill_color),
-						(SelectionShapeType::Lasso, _) => overlay_context.polygon(polygon, fill_color),
+						(SelectionShapeType::Lasso, _) => overlay_context.polygon(polygon, None, fill_color),
 					}
 				}
 				self
@@ -1733,6 +1764,34 @@ fn drag_shallowest_manipulation(responses: &mut VecDeque<Message>, selected: Vec
 	});
 }
 
+fn layer_selected_shallowest(clicked_layer: LayerNodeIdentifier, document: &DocumentMessageHandler) -> Option<LayerNodeIdentifier> {
+	let metadata = document.metadata();
+	let selected_layers = document.network_interface.selected_nodes().selected_layers(document.metadata()).collect::<Vec<_>>();
+	let final_selection: Option<LayerNodeIdentifier> = (!selected_layers.is_empty() && selected_layers != vec![LayerNodeIdentifier::ROOT_PARENT]).then_some(()).and_then(|_| {
+		let mut relevant_layers = document.network_interface.selected_nodes().selected_layers(document.metadata()).collect::<Vec<_>>();
+		if !relevant_layers.contains(&clicked_layer) {
+			relevant_layers.push(clicked_layer);
+		}
+		clicked_layer
+			.ancestors(metadata)
+			.filter(not_artboard(document))
+			.find(|&ancestor| relevant_layers.iter().all(|layer| *layer == ancestor || ancestor.is_ancestor_of(metadata, layer)))
+			.and_then(|least_common_ancestor| {
+				let common_siblings: Vec<_> = least_common_ancestor.children(metadata).collect();
+				(clicked_layer == least_common_ancestor)
+					.then_some(least_common_ancestor)
+					.or_else(|| common_siblings.iter().find(|&&child| clicked_layer == child || child.is_ancestor_of(metadata, &clicked_layer)).copied())
+			})
+	});
+
+	if final_selection.is_some_and(|layer| selected_layers.iter().any(|selected| layer.is_child_of(metadata, selected))) {
+		return None;
+	}
+
+	let new_selected = final_selection.unwrap_or_else(|| clicked_layer.ancestors(document.metadata()).filter(not_artboard(document)).last().unwrap_or(clicked_layer));
+	Some(new_selected)
+}
+
 fn drag_deepest_manipulation(responses: &mut VecDeque<Message>, selected: Vec<LayerNodeIdentifier>, tool_data: &mut SelectToolData, document: &DocumentMessageHandler, remove: bool) {
 	let layer = document.find_deepest(&selected).unwrap_or(
 		LayerNodeIdentifier::ROOT_PARENT

editor/src/messages/tool/tool_messages/text_tool.rs

@@ -473,7 +473,7 @@ impl Fsm for TextToolFsmState {
 					if far.x != 0. && far.y != 0. {
 						let quad = Quad::from_box([DVec2::ZERO, far]);
 						let transformed_quad = document.metadata().transform_to_viewport(tool_data.layer) * quad;
-						overlay_context.quad(transformed_quad, Some(&("#".to_string() + &fill_color)));
+						overlay_context.quad(transformed_quad, None, Some(&("#".to_string() + &fill_color)));
 					}
 				}
 
@@ -488,11 +488,12 @@ impl Fsm for TextToolFsmState {
 					for layer in document.intersect_quad_no_artboards(quad, input) {
 						overlay_context.quad(
 							Quad::from_box(document.metadata().bounding_box_viewport(layer).unwrap_or([DVec2::ZERO; 2])),
+							None,
 							Some(&("#".to_string() + &fill_color)),
 						);
 					}
 
-					overlay_context.quad(quad, Some(&("#".to_string() + &fill_color)));
+					overlay_context.quad(quad, None, Some(&("#".to_string() + &fill_color)));
 				}
 
 				// TODO: implement bounding box for multiple layers