From d452ac0de1f4e1e5101d22bda06ddea645a8d7c5 Mon Sep 17 00:00:00 2001 From: Dennis Kobert Date: Tue, 25 Aug 2026 20:09:12 +0000 Subject: [PATCH] Hoist a batch input only where the compiler proves it cannot vary per lane --- node-graph/graph-craft/src/document.rs | 1 + node-graph/graph-craft/src/proto.rs | 20 ++- node-graph/libraries/core-types/src/record.rs | 11 +- .../libraries/core-types/src/runtime.rs | 1 + node-graph/node-macro/src/codegen.rs | 133 +++++++++++++----- node-graph/nodes/gcore/src/record.rs | 9 +- node-graph/nodes/graphic/src/record.rs | 9 +- node-graph/nodes/gstd/src/render_node.rs | 1 + 8 files changed, 145 insertions(+), 40 deletions(-) diff --git a/node-graph/graph-craft/src/document.rs b/node-graph/graph-craft/src/document.rs index 5995f4f839..b607b556d7 100644 --- a/node-graph/graph-craft/src/document.rs +++ b/node-graph/graph-craft/src/document.rs @@ -167,6 +167,7 @@ impl DocumentNode { original_location: self.original_location, skip_deduplication: self.skip_deduplication, context_features: self.context_features, + lane_invariant_inputs: 0, resolved: Default::default(), } } diff --git a/node-graph/graph-craft/src/proto.rs b/node-graph/graph-craft/src/proto.rs index 803e6c8fdc..f1763c55ee 100644 --- a/node-graph/graph-craft/src/proto.rs +++ b/node-graph/graph-craft/src/proto.rs @@ -151,6 +151,10 @@ pub struct ProtoNode { pub original_location: OriginalLocation, pub skip_deduplication: bool, pub(crate) context_features: ContextDependencies, + /// Bit `i` marks input `i` as invariant under the innermost index. Inputs at + /// position 32 and above never set a bit, so they read as varying. + #[serde(skip)] + pub(crate) lane_invariant_inputs: u32, #[serde(skip)] pub(crate) resolved: Resolved, } @@ -164,6 +168,7 @@ impl Default for ProtoNode { original_location: OriginalLocation::default(), skip_deduplication: false, context_features: Default::default(), + lane_invariant_inputs: 0, resolved: Default::default(), } } @@ -205,6 +210,7 @@ impl ProtoNode { }, skip_deduplication: false, context_features: Default::default(), + lane_invariant_inputs: 0, resolved: Default::default(), } } @@ -373,12 +379,14 @@ impl ProtoNetwork { pub fn compute_layouts(&mut self) { for index in 0..self.nodes.len() { + let lane_invariant = self.nodes[index].1.lane_invariant_inputs; let layout = { let node = &self.nodes[index].1; match &node.construction_args { ConstructionArgs::Value(value) => value.value_layout().map(|layout| core_types::record::RecordLayout { frame_bytes: layout.frame_bytes(), plan: Vec::new(), + lane_invariant, layout, }), ConstructionArgs::Nodes(inputs) => node.resolved.layout_meta.as_ref().and_then(|meta| { @@ -386,7 +394,10 @@ impl ProtoNetwork { .iter() .map(|input| self.nodes[input.0 as usize].1.resolved.layout.as_ref().map(|resolved| &resolved.layout)) .collect(); - meta.sources.iter().all(|&source| input_layouts[source as usize].is_some()).then(|| meta.resolve(&input_layouts)) + meta.sources.iter().all(|&source| input_layouts[source as usize].is_some()).then(|| core_types::record::RecordLayout { + lane_invariant, + ..meta.resolve(&input_layouts) + }) }), ConstructionArgs::Inline(_) => None, } @@ -557,14 +568,21 @@ impl ProtoNetwork { }; // Compute the dependencies for each branch and combine all of them + let mut lane_invariant_inputs = 0u32; for (input, &node) in inputs.iter().enumerate() { let branch = self.find_context_dependencies(node); + let reads_innermost = branch.0.features.contains(core_types::context::ContextFeatures::INDEX) && branch.0.index_levels.contains_level(0); + if !reads_innermost && input < 32 { + lane_invariant_inputs |= 1 << input; + } + let mut lifted = branch.0.clone(); lifted.index_levels = lifted.index_levels.lifted(0, pushed_levels.get(input).copied().unwrap_or(0)); combined_deps |= &lifted; branch_dependencies.push(branch); } + self.nodes[node_index].1.lane_invariant_inputs = lane_invariant_inputs; let mut new_deps = combined_deps.clone(); // Remove requirements which this node provides diff --git a/node-graph/libraries/core-types/src/record.rs b/node-graph/libraries/core-types/src/record.rs index 00d7ab8306..067a2558fd 100644 --- a/node-graph/libraries/core-types/src/record.rs +++ b/node-graph/libraries/core-types/src/record.rs @@ -300,6 +300,10 @@ pub struct RecordLayout { pub layout: Layout, pub frame_bytes: usize, pub plan: Vec<(usize, usize, usize)>, + /// Inputs whose value cannot change with the innermost index, as a bitmask + /// over input positions. Empty is the safe default: an uninstalled layout + /// rebinds every input per lane. + pub lane_invariant: u32, } /// its registry entry so the compiler can fold each wire's layout without @@ -400,7 +404,12 @@ impl LayoutMeta { } _ => Vec::new(), }; - RecordLayout { layout, frame_bytes, plan } + RecordLayout { + layout, + frame_bytes, + plan, + lane_invariant: 0, + } } } diff --git a/node-graph/libraries/core-types/src/runtime.rs b/node-graph/libraries/core-types/src/runtime.rs index be2e052012..b852846bc0 100644 --- a/node-graph/libraries/core-types/src/runtime.rs +++ b/node-graph/libraries/core-types/src/runtime.rs @@ -273,6 +273,7 @@ mod tests { frame_bytes: layout.frame_bytes(), plan: Vec::new(), layout: layout.clone(), + lane_invariant: u32::MAX, }); RecordExtract::new(graph, &layout) } diff --git a/node-graph/node-macro/src/codegen.rs b/node-graph/node-macro/src/codegen.rs index e3ff4d1bc8..48c1068e15 100644 --- a/node-graph/node-macro/src/codegen.rs +++ b/node-graph/node-macro/src/codegen.rs @@ -212,6 +212,7 @@ pub(crate) fn generate_node_code(crate_ident: &CrateIdent, parsed: &ParsedNodeFn state.push(quote!(pub(super) __plan: ::std::vec::Vec<(usize, usize, usize)>)); } state.push(quote!(pub(super) __frame_bytes: usize)); + state.push(quote!(pub(super) __lane_invariant: u32)); state.extend(reading_secondary_indices(&struct_regular_fields, record_skips_carrier).into_iter().map(|index| { let slot = format_ident!("__in_{index}"); quote!(pub(super) #slot: gcore::record::Layout) @@ -231,7 +232,7 @@ pub(crate) fn generate_node_code(crate_ident: &CrateIdent, parsed: &ParsedNodeFn })); state } else if routing_generic.is_some() { - let mut state = vec![quote!(pub(super) __layout: gcore::record::Layout)]; + let mut state = vec![quote!(pub(super) __layout: gcore::record::Layout), quote!(pub(super) __lane_invariant: u32)]; state.extend( routing_value_indices(&struct_regular_fields, routing_generic.as_ref().expect("guarded by the arm")) .into_iter() @@ -420,6 +421,9 @@ pub(crate) fn generate_node_code(crate_ident: &CrateIdent, parsed: &ParsedNodeFn // offsets from the carrier layout; `new` cannot fill that state. let routing_layout_param = (routing_generic.is_some() || opaque).then(|| quote!(__layout: &gcore::record::Layout,)).into_iter(); let routing_layout_init = (routing_generic.is_some() || opaque).then(|| quote!(__layout: __layout.clone(),)).into_iter(); + // The lane-invariance mask arrives with the resolved layout, so `new` starts + // from the safe empty mask. + let routing_invariant_init = routing_generic.is_some().then(|| quote!(__lane_invariant: 0,)).into_iter(); let routing_value_layouts: Vec = routing_generic.as_ref().map(|generic| routing_value_indices(&struct_regular_fields, generic)).unwrap_or_default(); let routing_in_params = routing_value_layouts.iter().map(|index| { let slot = format_ident!("__in_{index}"); @@ -503,6 +507,7 @@ pub(crate) fn generate_node_code(crate_ident: &CrateIdent, parsed: &ParsedNodeFn Self { #(#all_field_inits,)* #(#routing_layout_init)* + #(#routing_invariant_init)* #(#routing_in_inits)* #(#flip_layout_inits)* #(#flip_read_inits)* @@ -2052,7 +2057,7 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn // in the loop (or in the tail, for a carrier); everything else is // batch-invariant and hoists. let hoists = |index: usize| matches!(ir::value_binding(&node, index), ValueBinding::Materialized) || !node.inputs[index].subject; - let hoisted_binds = regular_fields + let hoisted_binds: Vec = regular_fields .iter() .enumerate() .filter(|(index, field)| matches!(field.ty, ParsedFieldType::Regular(_)) && hoists(*index)) @@ -2069,13 +2074,15 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn } } } - }); - let hoisted_clamps = regular_fields + }) + .collect(); + let hoisted_clamps: Vec = regular_fields .iter() .enumerate() .filter(|(index, field)| matches!(field.ty, ParsedFieldType::Regular(_)) && hoists(*index)) - .filter_map(|(_, field)| clamp_tokens(field)); - let lane_binds = regular_fields + .filter_map(|(_, field)| clamp_tokens(field)) + .collect(); + let lane_binds: Vec = regular_fields .iter() .enumerate() .filter(|(index, field)| match field.ty { @@ -2086,10 +2093,26 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn let body = bind_body(index, field, true); let clamp = clamp_tokens(field); quote!(#body #clamp) - }); + }) + .collect(); + // The rebind path with nothing hoisted: every non-carrier input binds + // fresh per lane, so an index-dependent edge reaches its own lane. + let rebound_lane_binds: Vec = regular_fields + .iter() + .enumerate() + .filter(|(index, field)| match field.ty { + ParsedFieldType::Node(_) => true, + ParsedFieldType::Regular(_) => !matches!(ir::value_binding(&node, *index), ValueBinding::Carrier), + }) + .map(|(index, field)| { + let body = bind_body(index, field, true); + let clamp = clamp_tokens(field); + quote!(#body #clamp) + }) + .collect(); // A hoisted value is moved into every lane's kernel call, so each // lane consumes a clone; view and borrow binds copy freely. - let lane_rebinds = regular_fields + let lane_rebinds: Vec = regular_fields .iter() .enumerate() .filter(|(index, field)| { @@ -2099,32 +2122,23 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn .map(|(_, field)| { let name = &field.pat_ident.ident; quote!(let #name = ::core::clone::Clone::clone(&#name);) - }); - quote! { - #batch_signature - { - let ::core::option::Option::Some(__scratch) = __scratch else { - return #core_types::node::BatchStatus::NeedBuffer; - }; - let ::core::option::Option::Some(__len) = __range.end.checked_sub(__range.start).and_then(|__len| usize::try_from(__len).ok()) else { - return #core_types::node::BatchStatus::InvalidRange; - }; - let __node_layout = >::layout(self); - let __stride = __node_layout.lane_stride(); - if __scratch.len() * 8 < __len * __stride { - return #core_types::node::BatchStatus::InvalidRange; - } - let __entry_mark = #core_types::record::stack::sp(); - let _entry_sp = __entry_mark; - let __cell = #cell_constructor; - let __base_ctx = { - let mut __ctx = *__input; - #core_types::context::InjectIndex::set_index(&mut __ctx, __range.start); - __ctx - }; - let __input = &__base_ctx; - #(#hoisted_binds)* - #(#hoisted_clamps)* + }) + .collect(); + // A hoisted input past bit 31 has no bit to check, so it never reads + // back as invariant and the node keeps rebinding it. + let hoistable_mask: u32 = regular_fields + .iter() + .enumerate() + .filter(|(index, field)| matches!(field.ty, ParsedFieldType::Regular(_)) && hoists(*index) && *index < 32) + .fold(0, |mask, (index, _)| mask | (1u32 << index)); + let fill_loop = |hoisted: Vec, clamps: Vec, rebinds: Vec, binds: Vec| { + let hoisted = hoisted.into_iter(); + let clamps = clamps.into_iter(); + let rebinds = rebinds.into_iter(); + let binds = binds.into_iter(); + quote! { + #(#hoisted)* + #(#clamps)* let __frames = __scratch.as_mut_ptr().cast::(); let mut __finality = #core_types::gpoll::Finality::AllFinal; let mut __filled = __len; @@ -2136,8 +2150,8 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn let __lane_mark = #core_types::record::stack::sp(); let _entry_sp = __lane_mark; let __cell = __cell.snapshot(); - #(#lane_rebinds)* - #(#lane_binds)* + #(#rebinds)* + #(#binds)* #lane_poll let __value = match __poll { #core_types::gpoll::GPoll::Final(__value) => __value, @@ -2171,6 +2185,50 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn // records of the node's layout. #core_types::node::BatchStatus::Filled(unsafe { #core_types::node::RecordBatchMut::new(__scratch, __filled, __node_layout) }, __finality, __hint) } + }; + let hoisted_fill = fill_loop(hoisted_binds, hoisted_clamps, lane_rebinds, lane_binds); + let rebound_fill = fill_loop(Vec::new(), Vec::new(), Vec::new(), rebound_lane_binds); + // With nothing hoisted the two fills are the same code, and the mask + // test would read as an empty bit mask. + let selected_fill = match hoistable_mask { + 0 => hoisted_fill, + mask => quote! { + // Binding once at the base lane is sound only where the + // installed layout marks every hoisted input invariant under + // the innermost index. + const __HOISTABLE: u32 = #mask; + if (self.__lane_invariant & __HOISTABLE) == __HOISTABLE { + #hoisted_fill + } else { + #rebound_fill + } + }, + }; + quote! { + #batch_signature + { + let ::core::option::Option::Some(__scratch) = __scratch else { + return #core_types::node::BatchStatus::NeedBuffer; + }; + let ::core::option::Option::Some(__len) = __range.end.checked_sub(__range.start).and_then(|__len| usize::try_from(__len).ok()) else { + return #core_types::node::BatchStatus::InvalidRange; + }; + let __node_layout = >::layout(self); + let __stride = __node_layout.lane_stride(); + if __scratch.len() * 8 < __len * __stride { + return #core_types::node::BatchStatus::InvalidRange; + } + let __entry_mark = #core_types::record::stack::sp(); + let _entry_sp = __entry_mark; + let __cell = #cell_constructor; + let __base_ctx = { + let mut __ctx = *__input; + #core_types::context::InjectIndex::set_index(&mut __ctx, __range.start); + __ctx + }; + let __input = &__base_ctx; + #selected_fill + } } } // The eager forward runs the shared copy-out loop with statically @@ -2273,11 +2331,13 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn Some(quote! { #(#write_installs)* self.__frame_bytes = __resolved.frame_bytes; + self.__lane_invariant = __resolved.lane_invariant; #plan self.__layout = __resolved.layout; }) } else if routing_generic.is_some() { Some(quote! { + self.__lane_invariant = __resolved.lane_invariant; self.__layout = __resolved.layout; }) } else { @@ -2442,6 +2502,7 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn #(#plan_default)* #(#marker_init)* __frame_bytes: 0, + __lane_invariant: 0, #(#read_names)* #(#write_defaults)* #(#mat_cache_defaults)* diff --git a/node-graph/nodes/gcore/src/record.rs b/node-graph/nodes/gcore/src/record.rs index d197804ca0..7b08df5605 100644 --- a/node-graph/nodes/gcore/src/record.rs +++ b/node-graph/nodes/gcore/src/record.rs @@ -601,7 +601,13 @@ mod tests { } fn install>>(mut node: N, meta: core_types::record::LayoutMeta, inputs: &[Option<&Layout>]) -> N { - >>::set_layout(&mut node, meta.resolve(inputs)); + // The fixtures wire constants into every eager input, which the compiler + // pass records as lane-invariant. + let resolved = core_types::record::RecordLayout { + lane_invariant: u32::MAX, + ..meta.resolve(inputs) + }; + >>::set_layout(&mut node, resolved); node } @@ -610,6 +616,7 @@ mod tests { frame_bytes: layout.frame_bytes(), plan: Vec::new(), layout: layout.clone(), + lane_invariant: u32::MAX, }; >>::set_layout(&mut node, bundle); node diff --git a/node-graph/nodes/graphic/src/record.rs b/node-graph/nodes/graphic/src/record.rs index f56d748285..0806ac7d4d 100644 --- a/node-graph/nodes/graphic/src/record.rs +++ b/node-graph/nodes/graphic/src/record.rs @@ -288,7 +288,13 @@ mod tests { } fn install>>(mut node: N, meta: record::LayoutMeta, inputs: &[Option<&Layout>]) -> N { - >>::set_layout(&mut node, meta.resolve(inputs)); + // The fixtures wire constants into every eager input, which the compiler + // pass records as lane-invariant. + let resolved = record::RecordLayout { + lane_invariant: u32::MAX, + ..meta.resolve(inputs) + }; + >>::set_layout(&mut node, resolved); node } @@ -297,6 +303,7 @@ mod tests { frame_bytes: layout.frame_bytes(), plan: Vec::new(), layout: layout.clone(), + lane_invariant: u32::MAX, }; >>::set_layout(&mut node, bundle); node diff --git a/node-graph/nodes/gstd/src/render_node.rs b/node-graph/nodes/gstd/src/render_node.rs index de4156ac00..6e917da751 100644 --- a/node-graph/nodes/gstd/src/render_node.rs +++ b/node-graph/nodes/gstd/src/render_node.rs @@ -278,6 +278,7 @@ mod tests { frame_bytes: layout.frame_bytes(), plan: Vec::new(), layout: layout.clone(), + lane_invariant: u32::MAX, }, ); let GPoll::Final(result) = Node::::eval(&graph, &ctx) else {