diff --git a/node-graph/graph-craft/src/document/value.rs b/node-graph/graph-craft/src/document/value.rs index 1b32f67b92..58beb70fbe 100644 --- a/node-graph/graph-craft/src/document/value.rs +++ b/node-graph/graph-craft/src/document/value.rs @@ -359,27 +359,27 @@ macro_rules! tagged_value { } /// Evaluates a typed edge and converts the landed value into a tagged value, with the coverage of [`Self::try_from_any`]. - pub fn from_edge(handle: EdgeHandle, ctx: &Context) -> Result, String> { + pub fn from_edge<'f>(handle: EdgeHandle, ctx: &Context<'f>, frames: &core_types::record::Frames<'f>) -> Result, String> { let ty = handle.ty().clone(); // ======================= // RECORD WIRES, WHICH LAND AS THEIR ELEMENT // ======================= if ty == core_types::registry::record_edge_type::<()>() { let edge = handle.downcast_record::<()>().map_err(|e| format!("{e:?}"))?; - return Ok(core_types::record::serve_edge(&edge, ctx).map(|_| TaggedValue::None)); + return Ok(core_types::record::serve_edge(&edge, ctx, frames).map(|_| TaggedValue::None)); } $( if ty == core_types::registry::record_edge_type::<$ty>() { let layout = handle.layout().clone(); let edge = handle.downcast_record::<$ty>().map_err(|e| format!("{e:?}"))?; - return Ok(core_types::record::serve_edge(&edge, ctx) + return Ok(core_types::record::serve_edge(&edge, ctx, frames) .map(|value| TaggedValue::$identifier(unsafe { core_types::record::read_element::<$ty>(layout.rec(&value)) }))); } )* if ty == core_types::registry::record_edge_type::() { let layout = handle.layout().clone(); let edge = handle.downcast_record::().map_err(|e| format!("{e:?}"))?; - return Ok(core_types::record::serve_edge(&edge, ctx) + return Ok(core_types::record::serve_edge(&edge, ctx, frames) .map(|value| TaggedValue::RenderOutput(unsafe { core_types::record::read_element::(layout.rec(&value)) }))); } Err(format!("Cannot convert edge of type {ty} to TaggedValue")) diff --git a/node-graph/interpreted-executor/src/dynamic_executor.rs b/node-graph/interpreted-executor/src/dynamic_executor.rs index 7cdcf3af3f..c4b58327c3 100644 --- a/node-graph/interpreted-executor/src/dynamic_executor.rs +++ b/node-graph/interpreted-executor/src/dynamic_executor.rs @@ -33,6 +33,9 @@ pub struct DynamicExecutor { // This allows us to keep the nodes around for one more frame which is used for introspection orphaned_nodes: HashSet, arena: Mutex, + /// The record frame space, grow-only across evaluations and lent to the + /// root by `&mut`. + frames: Mutex, runtime: Arc, live_sources: Vec, } @@ -49,6 +52,7 @@ impl Default for DynamicExecutor { typing_context: TypingContext::new(&node_registry::NODE_REGISTRY), orphaned_nodes: HashSet::new(), arena: Mutex::new(new_arena()), + frames: Mutex::new(core_types::record::FrameArena::new()), runtime: noop_runtime(), live_sources: Vec::new(), } @@ -85,6 +89,7 @@ impl DynamicExecutor { typing_context, orphaned_nodes: HashSet::new(), arena: Mutex::new(new_arena()), + frames: Mutex::new(core_types::record::FrameArena::new()), runtime, live_sources: sources, }) @@ -177,26 +182,25 @@ impl DynamicExecutor { .and_then(EdgeHandle::record_edge) .ok_or_else(|| IntrospectError::PathNotFound(node_path.to_vec()))?; let arena = self.arena.lock().unwrap_or_else(PoisonError::into_inner); - // SAFETY: between evaluations, nothing served on the stack is live. - unsafe { - core_types::record::stack::reserve(self.tree.stack_need()); - } + let mut buffer = self.frames.lock().unwrap_or_else(PoisonError::into_inner); + buffer.reserve(self.tree.stack_need()); + let frames = buffer.frames(); let generations = self.runtime.snapshot(); let scope = EvalScope::new(snapshot.try_real_time(), snapshot.try_animation_time(), snapshot.try_pointer_position(), &generations, &arena); let Some(ctx) = snapshot.rehydrate(&scope) else { return Err(IntrospectError::NoData); }; let layout = core_types::node::Node::::layout(&edge); - // SAFETY: the read closure finishes inside the scope, so no record - // above the entry survives it. - let _scope = unsafe { core_types::record::stack::ScopeGuard::enter() }; + // The batch borrows the frames the read closure is handed, so the read + // cannot outlive the scope that owns them. + let frames = frames.scope(); let result = if layout.depth > 0 { - match core_types::record::materialize_level(&edge, &ctx, &arena) { + match core_types::record::materialize_level(&edge, &ctx, &arena, &frames) { core_types::record::LevelStatus::Batch(batch, _) => read(layout, batch, &arena), _ => None, } } else { - match core_types::record::serve_edge(&edge, &ctx) { + match core_types::record::serve_edge(&edge, &ctx, &frames) { GPoll::Final(value) | GPoll::Partial(value) => { let rec = layout.rec(&value); // SAFETY: the serve produced one live record of the edge's layout. @@ -248,10 +252,13 @@ where return Err("Output node not found in executor".into()); }; let mut arena = self.arena.lock().unwrap_or_else(PoisonError::into_inner); - // SAFETY: between evaluations, nothing served on the stack is live. - unsafe { core_types::record::stack::reserve(self.tree.stack_need()); } let result = eval_root(&mut arena, &self.runtime, &input, |ctx| match TaggedValue::from_edge(handle.duplicate(), ctx) { - Ok(poll) => poll.map(Ok), - Err(error) => GPoll::Final(Err(error)), + let mut buffer = self.frames.lock().unwrap_or_else(PoisonError::into_inner); + buffer.reserve(self.tree.stack_need()); + let result = eval_root(&mut arena, &mut buffer, &self.runtime, &input, |ctx, frames| { + match TaggedValue::from_edge(handle.duplicate(), ctx, frames) { + Ok(poll) => poll.map(Ok), + Err(error) => GPoll::Final(Err(error)), + } }); match result { GPoll::Final(value) => Ok(GPoll::Final(value?)), @@ -265,7 +272,17 @@ where } } } -pub fn eval_root(arena: &mut Arena, runtime: &GraphRuntime, call_argument: DynSlot, eval: impl FnOnce(&ContextImpl) -> GPoll) -> GPoll { +/// One evaluation over the arena and the frame buffer, which the caller sized +/// to the graph's frame need: both are the evaluation's lifetime, so a record +/// served anywhere in the cone lives exactly as long as the arena it may +/// reference. +pub fn eval_root( + arena: &mut Arena, + buffer: &mut core_types::record::FrameArena, + runtime: &GraphRuntime, + call_argument: DynSlot, + eval: impl for<'e> FnOnce(&ContextImpl<'e>, &core_types::record::Frames<'e>) -> GPoll, +) -> GPoll { arena.reset(); let generations = runtime.snapshot(); let scope = EvalScope::new(None, None, None, &generations, arena); @@ -275,7 +292,8 @@ pub fn eval_root(arena: &mut Arena, runtime: &GraphRuntime, call_argume outer: None, }; let ctx = root.with_varargs(&link); - match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| eval(&ctx))) { + let frames = buffer.frames(); + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| eval(&ctx, &frames))) { Ok(result) => result, Err(_) => { arena.reset(); @@ -568,9 +586,10 @@ mod test { #[test] fn eval_root_builds_the_bare_root_with_the_call_argument_as_vararg_0() { let mut arena = Arena::new(64).unwrap(); + let mut buffer = core_types::record::FrameArena::new(); let runtime = GraphRuntime::new(InertSpawner); let argument = 21.5f64; - let result = eval_root(&mut arena, &runtime, &argument, |ctx| { + let result = eval_root(&mut arena, &mut buffer, &runtime, &argument, |ctx, _frames| { assert!(ctx.try_footprint().is_none(), "the bare root carries no axes"); GPoll::Final(ctx.vararg(0).ok().and_then(|slot| slot.downcast_ref::()).copied().unwrap_or(0.)) }); @@ -580,15 +599,16 @@ mod test { #[test] fn eval_root_resets_the_arena_at_eval_start() { let mut arena = Arena::new(64).unwrap(); + let mut buffer = core_types::record::FrameArena::new(); let runtime = GraphRuntime::new(InertSpawner); let cell = ArenaCell::new(); - eval_root(&mut arena, &runtime, &(), |ctx| { + eval_root(&mut arena, &mut buffer, &runtime, &(), |ctx, _frames| { let (_, weak) = ctx.scope().arena().alloc(5u32).unwrap(); cell.store(weak); GPoll::Final(()) }); assert!(cell.load(&arena).is_some(), "the introspection window spans until the next eval"); - eval_root(&mut arena, &runtime, &(), |ctx| { + eval_root(&mut arena, &mut buffer, &runtime, &(), |ctx, _frames| { assert!(cell.load(ctx.scope().arena()).is_none(), "the reset at eval start reclaims the previous frame"); GPoll::Final(()) }); @@ -597,16 +617,17 @@ mod test { #[test] fn a_panicking_eval_reports_the_error_and_resets_the_arena() { let mut arena = Arena::new(64).unwrap(); + let mut buffer = core_types::record::FrameArena::new(); let runtime = GraphRuntime::new(InertSpawner); let cell = ArenaCell::new(); - let result: GPoll<()> = eval_root(&mut arena, &runtime, &(), |ctx| { + let result: GPoll<()> = eval_root(&mut arena, &mut buffer, &runtime, &(), |ctx, _frames| { let (_, weak) = ctx.scope().arena().alloc(5u32).unwrap(); cell.store(weak); panic!("mid-eval"); }); assert_eq!(result, GPoll::panicked()); assert!(cell.load(&arena).is_none(), "reset-on-panic leaves no stale records"); - assert_eq!(eval_root(&mut arena, &runtime, &(), |_| GPoll::Final(7u32)), GPoll::Final(7)); + assert_eq!(eval_root(&mut arena, &mut buffer, &runtime, &(), |_, _| GPoll::Final(7u32)), GPoll::Final(7)); } #[test] @@ -623,11 +644,8 @@ mod test { let generations = []; let scope = EvalScope::new(None, None, None, &generations, &arena); let ctx = ContextImpl::root(&scope); - // SAFETY: between evaluations, nothing served on the stack is live. - unsafe { - core_types::record::stack::reserve(layout.frame_bytes()); - } - let GPoll::Final(value) = core_types::record::serve_edge(&edge, &ctx) else { + let frames = core_types::record::test_frames(layout.frame_bytes()); + let GPoll::Final(value) = core_types::record::serve_edge(&edge, &ctx, &frames) else { panic!("expected a final record"); }; assert_eq!(unsafe { core_types::record::read_element::(layout.rec(&value)) }, 2); @@ -671,11 +689,8 @@ mod test { let handle = executor.tree().get(NodeId(1)).unwrap(); let layout = handle.layout().clone(); let edge = handle.duplicate().downcast_record::().unwrap(); - // SAFETY: between evaluations, nothing served on the stack is live. - unsafe { - core_types::record::stack::reserve(executor.tree().stack_need()); - } - let GPoll::Final(value) = core_types::record::serve_edge(&edge, &ctx) else { + let frames = core_types::record::test_frames(executor.tree().stack_need()); + let GPoll::Final(value) = core_types::record::serve_edge(&edge, &ctx, &frames) else { panic!("the flipped clone must evaluate over record wires, got a non-final poll"); }; assert_eq!(unsafe { core_types::record::read_element::(layout.rec(&value)) }, 7.); @@ -701,11 +716,8 @@ mod test { let scope = EvalScope::new(None, None, None, &generations, &arena); let ctx = ContextImpl::root(&scope); let edge = executor.tree().get(NodeId(2)).unwrap().downcast_record::().unwrap(); - // SAFETY: between evaluations, nothing served on the stack is live. - unsafe { - core_types::record::stack::reserve(executor.tree().stack_need()); - } - let result = core_types::record::serve_edge(&edge, &ctx); + let frames = core_types::record::test_frames(executor.tree().stack_need()); + let result = core_types::record::serve_edge(&edge, &ctx, &frames); // The empty raster level folds to an empty palette: past-end at lane 0. assert!( matches!(&result, GPoll::Error(error) if error.kind == core_types::gpoll::ErrorKind::PastEnd), diff --git a/node-graph/libraries/core-types/src/arena.rs b/node-graph/libraries/core-types/src/arena.rs index 061d87336d..f6be9c7616 100644 --- a/node-graph/libraries/core-types/src/arena.rs +++ b/node-graph/libraries/core-types/src/arena.rs @@ -175,6 +175,15 @@ impl Arena { Some(unsafe { std::slice::from_raw_parts_mut(ptr, len) }) } + /// The generation-checked handle for a region this arena holds, `None` for + /// a pointer from anywhere else. The handle keeps the region's provenance, + /// so a cache stores one where it would otherwise launder an address. + pub fn handle_at(&self, ptr: *const u8) -> Option> { + let offset = (ptr as usize).checked_sub(self.base() as usize)?; + (offset < self.buf.len()).then_some(())?; + ArenaWeak::new(self.generation(), offset) + } + /// `false` once generations are exhausted, parking the arena on [`PARKED_GENERATION`] /// where every handle misses and further allocation is refused. pub fn reset(&mut self) -> bool { diff --git a/node-graph/libraries/core-types/src/node.rs b/node-graph/libraries/core-types/src/node.rs index 8643af7f6e..ab3925f4cb 100644 --- a/node-graph/libraries/core-types/src/node.rs +++ b/node-graph/libraries/core-types/src/node.rs @@ -105,10 +105,9 @@ pub struct RecordBatchMut<'a> { } impl<'a> RecordBatchMut<'a> { - /// # Safety - /// `scratch` must start with `len` initialized records of `layout`, packed - /// at `layout.lane_stride()` stride. - pub unsafe fn new(scratch: &'a mut [MaybeUninit], len: usize, layout: &'a crate::record::Layout) -> Self { + /// Minted only by a [`crate::record::SlotRun`] finishing its served lanes, + /// which is what makes the initialized prefix a fact rather than a contract. + pub(crate) fn new(scratch: &'a mut [MaybeUninit], len: usize, layout: &'a crate::record::Layout) -> Self { debug_assert!(len * layout.lane_stride() <= scratch.len() * 8); Self { scratch, len, layout } } @@ -318,9 +317,10 @@ pub trait Node { /// Serves the node's record through the caller's claim: the writes land /// in the claim and the returned proof is mintable only by its closing /// methods, so the served record is of the claimed layout by - /// construction. The caller claims the frame at [`Node::layout`] before - /// the call, so the node advances the record stack by exactly that frame. - fn serve<'e, 'l>(&self, input: &Input, slot: crate::record::FrameClaim<'l>) -> GPoll> + /// construction. The caller claims the frame at [`Node::layout`] out of + /// its own frame space, and the claim carries what is left, so the node + /// takes exactly its own frame out of the caller's free space. + fn serve<'e, 'l>(&self, input: &Input, slot: crate::record::FrameClaim<'e, 'l>) -> GPoll> where Input: crate::context::ExtractArena; @@ -328,7 +328,7 @@ pub trait Node { /// leveled primitive a structure node overrides to report a pushed level's /// size; the scalar base is one item at every level. Uncertainty rides the /// `GPoll` status axis. - fn extent_at<'e>(&self, _input: &Input, _level: u8) -> GPoll + fn extent_at<'e>(&self, _input: &Input, _level: u8, _frames: &crate::record::Frames<'e>) -> GPoll where Input: crate::context::ExtractArena, { @@ -338,16 +338,17 @@ pub trait Node { /// The composite domain query derived from [`extent_at`](Node::extent_at): /// one level, the product of the levels below or above it, or the whole /// domain's flat count. Consumers query this; nodes only write `extent_at`. - fn extent<'e>(&self, input: &Input, at: Level) -> GPoll + fn extent<'e>(&self, input: &Input, at: Level, frames: &crate::record::Frames<'e>) -> GPoll where Input: crate::context::ExtractArena, { - let product = |range: core::ops::Range| range.fold(GPoll::Final(Extent::Exactly(1)), |acc, level| Extent::mul(acc, self.extent_at(input, level))); + let product = + |range: core::ops::Range, frames: &crate::record::Frames<'e>| range.fold(GPoll::Final(Extent::Exactly(1)), |acc, level| Extent::mul(acc, self.extent_at(input, level, frames))); match at { - Level::At(level) => self.extent_at(input, level), - Level::Below(level) => product(0..level), - Level::Above(level) => product((level + 1)..self.depth()), - Level::Total => product(0..self.depth()), + Level::At(level) => self.extent_at(input, level, frames), + Level::Below(level) => product(0..level, frames), + Level::Above(level) => product((level + 1)..self.depth(), frames), + Level::Total => product(0..self.depth(), frames), } } @@ -364,8 +365,8 @@ pub trait Node { /// The record layout of this node's output; the shared empty layout for /// element-only producers. Consumers read their carrier's layout through - /// this at wiring, and the wiring layer derives stack sizing from the same - /// layouts, in the dynamic executor and exported source alike. + /// this at wiring, and the wiring layer derives the root buffer's sizing from + /// the same layouts, in the dynamic executor and exported source alike. fn layout(&self) -> &crate::record::Layout { crate::record::empty_layout() } @@ -379,11 +380,11 @@ pub trait Node { /// with copy-out ([`crate::record::fill_frames`]); overrides exist to beat /// that loop (resident lanes, direct fills, fewer erased calls), never for /// correctness. - fn eval_batch<'a, 'e>(&'a self, input: &'a Input, range: Range, scratch: Option<&'a mut [MaybeUninit]>) -> BatchStatus<'a> + fn eval_batch<'a, 'e>(&'a self, input: &'a Input, range: Range, scratch: Option<&'a mut [MaybeUninit]>, frames: &crate::record::Frames<'e>) -> BatchStatus<'a> where Input: InjectIndex + Copy + crate::context::ExtractArena, { - let _ = (input, range, scratch); + let _ = (input, range, scratch, frames); BatchStatus::Unbatched } } @@ -392,18 +393,18 @@ impl Node for &N where N: Node + ?Sized, { - fn serve<'e, 'l>(&self, input: &Input, slot: crate::record::FrameClaim<'l>) -> GPoll> + fn serve<'e, 'l>(&self, input: &Input, slot: crate::record::FrameClaim<'e, 'l>) -> GPoll> where Input: crate::context::ExtractArena, { (**self).serve(input, slot) } - fn extent_at<'e>(&self, input: &Input, level: u8) -> GPoll + fn extent_at<'e>(&self, input: &Input, level: u8, frames: &crate::record::Frames<'e>) -> GPoll where Input: crate::context::ExtractArena, { - (**self).extent_at(input, level) + (**self).extent_at(input, level, frames) } fn serialize(&self) -> Option> { @@ -414,11 +415,11 @@ where (**self).layout() } - fn eval_batch<'a, 'e>(&'a self, input: &'a Input, range: Range, scratch: Option<&'a mut [MaybeUninit]>) -> BatchStatus<'a> + fn eval_batch<'a, 'e>(&'a self, input: &'a Input, range: Range, scratch: Option<&'a mut [MaybeUninit]>, frames: &crate::record::Frames<'e>) -> BatchStatus<'a> where Input: InjectIndex + Copy + crate::context::ExtractArena, { - (**self).eval_batch(input, range, scratch) + (**self).eval_batch(input, range, scratch, frames) } } @@ -426,18 +427,18 @@ impl Node for Box where N: Node + ?Sized, { - fn serve<'e, 'l>(&self, input: &Input, slot: crate::record::FrameClaim<'l>) -> GPoll> + fn serve<'e, 'l>(&self, input: &Input, slot: crate::record::FrameClaim<'e, 'l>) -> GPoll> where Input: crate::context::ExtractArena, { (**self).serve(input, slot) } - fn extent_at<'e>(&self, input: &Input, level: u8) -> GPoll + fn extent_at<'e>(&self, input: &Input, level: u8, frames: &crate::record::Frames<'e>) -> GPoll where Input: crate::context::ExtractArena, { - (**self).extent_at(input, level) + (**self).extent_at(input, level, frames) } fn serialize(&self) -> Option> { @@ -448,11 +449,11 @@ where (**self).layout() } - fn eval_batch<'a, 'e>(&'a self, input: &'a Input, range: Range, scratch: Option<&'a mut [MaybeUninit]>) -> BatchStatus<'a> + fn eval_batch<'a, 'e>(&'a self, input: &'a Input, range: Range, scratch: Option<&'a mut [MaybeUninit]>, frames: &crate::record::Frames<'e>) -> BatchStatus<'a> where Input: InjectIndex + Copy + crate::context::ExtractArena, { - (**self).eval_batch(input, range, scratch) + (**self).eval_batch(input, range, scratch, frames) } } @@ -460,18 +461,18 @@ impl Node for std::sync::Arc where N: Node + ?Sized, { - fn serve<'e, 'l>(&self, input: &Input, slot: crate::record::FrameClaim<'l>) -> GPoll> + fn serve<'e, 'l>(&self, input: &Input, slot: crate::record::FrameClaim<'e, 'l>) -> GPoll> where Input: crate::context::ExtractArena, { (**self).serve(input, slot) } - fn extent_at<'e>(&self, input: &Input, level: u8) -> GPoll + fn extent_at<'e>(&self, input: &Input, level: u8, frames: &crate::record::Frames<'e>) -> GPoll where Input: crate::context::ExtractArena, { - (**self).extent_at(input, level) + (**self).extent_at(input, level, frames) } fn serialize(&self) -> Option> { @@ -482,11 +483,11 @@ where (**self).layout() } - fn eval_batch<'a, 'e>(&'a self, input: &'a Input, range: Range, scratch: Option<&'a mut [MaybeUninit]>) -> BatchStatus<'a> + fn eval_batch<'a, 'e>(&'a self, input: &'a Input, range: Range, scratch: Option<&'a mut [MaybeUninit]>, frames: &crate::record::Frames<'e>) -> BatchStatus<'a> where Input: InjectIndex + Copy + crate::context::ExtractArena, { - (**self).eval_batch(input, range, scratch) + (**self).eval_batch(input, range, scratch, frames) } } @@ -516,15 +517,15 @@ impl StatusCell { Self { no_partial: true, ..Self::new() } } - /// Claims the edge's own frame, serves through it, and folds the poll's - /// status into the cell. The claim is the caller's, so the edge's frame is - /// entered exactly once per evaluation. + /// Claims the edge's own frame out of `frames`, serves through it, and + /// folds the poll's status into the cell. The claim is the caller's, so + /// the edge's frame is claimed exactly once per evaluation. #[inline(always)] - pub fn eval_input<'e, Input, N: Node + ?Sized>(&self, input_index: usize, node: &N, input: &Input) -> Result, Interrupt> + pub fn eval_input<'e, Input, N: Node + ?Sized>(&self, input_index: usize, node: &N, input: &Input, frames: &crate::record::Frames<'e>) -> Result, Interrupt> where Input: crate::context::ExtractArena, { - let slot = crate::record::FrameClaim::enter(node.layout()); + let slot = frames.claim(node.layout()); match node.serve(input, slot) { GPoll::Final(served) => Ok(served.value()), GPoll::Partial(_) if self.no_partial => Err(Interrupt::Pending), @@ -588,23 +589,25 @@ impl StatusCell { } #[derive(Clone, Copy)] -pub struct LazyInput<'a, N> { +pub struct LazyInput<'a, 'f, N> { node: &'a N, cell: &'a StatusCell, input_index: usize, + frames: &'a crate::record::Frames<'f>, } -impl<'a, N> LazyInput<'a, N> { - pub fn new(node: &'a N, cell: &'a StatusCell, input_index: usize) -> Self { - Self { node, cell, input_index } +impl<'a, 'f, N> LazyInput<'a, 'f, N> { + pub fn new(node: &'a N, cell: &'a StatusCell, input_index: usize, frames: &'a crate::record::Frames<'f>) -> Self { + Self { node, cell, input_index, frames } } #[inline(always)] pub fn eval<'e, Input>(&self, ctx: &Input) -> Result, Interrupt> where N: crate::record::DerivedRecordEdge<'e, Input>, + 'f: 'e, { - self.node.eval_derived(self.cell, self.input_index, ctx) + self.node.eval_derived(self.cell, self.input_index, ctx, self.frames) } /// The edge's composite extent, for kernels that split or shift indices @@ -614,8 +617,9 @@ impl<'a, N> LazyInput<'a, N> { where N: Node, Input: crate::context::ExtractArena, + 'f: 'e, { - self.node.extent(ctx, at) + self.node.extent(ctx, at, self.frames) } } @@ -653,12 +657,13 @@ mod tests { #[test] fn the_default_advertises_no_batch_support() { + let frames = crate::record::test_frames(1 << 16); let arena = Arena::new(1024).unwrap(); let input = TestInput { index: 0, arena: &arena }; let mut scratch = [const { MaybeUninit::uninit() }; 4]; let node = double(); - assert!(matches!(node.eval_batch(&input, 2..6, Some(&mut scratch)), BatchStatus::Unbatched)); - assert!(matches!(node.eval_batch(&input, 2..6, None), BatchStatus::Unbatched)); + assert!(matches!(node.eval_batch(&input, 2..6, Some(&mut scratch), &frames), BatchStatus::Unbatched)); + assert!(matches!(node.eval_batch(&input, 2..6, None, &frames), BatchStatus::Unbatched)); } #[test] @@ -668,15 +673,12 @@ mod tests { let node = double(); let layout = Node::::layout(&node).clone(); let erased: Box> = Box::new(node); - // SAFETY: between evaluations, nothing served on the stack is live. - unsafe { - crate::record::stack::reserve(1 << 12); - } - let GPoll::Final(value) = serve_edge(&*erased, &input) else { + let frames = crate::record::test_frames(1 << 12); + let GPoll::Final(value) = serve_edge(&*erased, &input, &frames) else { panic!("the erased edge must serve a final record"); }; // SAFETY: the record was served at `layout`, whose element is the output. assert_eq!(unsafe { crate::record::read_element::(layout.rec(&value)) }, 42); - assert!(matches!(erased.eval_batch(&input, 0..2, None), BatchStatus::Unbatched)); + assert!(matches!(erased.eval_batch(&input, 0..2, None, &frames), BatchStatus::Unbatched)); } } diff --git a/node-graph/libraries/core-types/src/record.rs b/node-graph/libraries/core-types/src/record.rs index dbca779f2b..a46bc31d53 100644 --- a/node-graph/libraries/core-types/src/record.rs +++ b/node-graph/libraries/core-types/src/record.rs @@ -2,8 +2,9 @@ //! plus one field per written attribute; its [`Layout`] is computed at //! wiring from the upstream write set and never serialized. Records of //! inline layouts live in the [`RecordValue`] itself; larger ones live as -//! per-lane views on the per-thread record [`stack`], claimed per -//! evaluation. Kernels route them as opaque [`RecordValue`]s that carry +//! per-lane views on the evaluation's [`Frames`], which the root owns and +//! every node claims its own frame out of. Kernels route them as opaque +//! [`RecordValue`]s that carry //! their provenance. Only generated or wiring code touches offsets, so a //! safe kernel cannot misalign a field. @@ -428,7 +429,7 @@ impl Rec { } } -/// An opaque record value: every non-empty record spills to the record stack +/// An opaque record value: every non-empty record spills to a claimed frame /// and the value carries its pointer, while an empty record carries nothing. /// Only [`Layout::rec`] reads it, against the wiring-proven layout. #[derive(Clone, Copy)] @@ -473,8 +474,12 @@ impl<'e> RecordValue<'e> { } } - /// Rebinds the eval lifetime; validity stays stack and arena discipline, - /// which derived scopes share with their parent evaluation. + /// Rebinds the eval lifetime, lengthening a derived scope's record back + /// onto the outer evaluation's. Sound only because the record's bytes live + /// in the outer caller's slot region: a derived context shortens the + /// caller's frame space rather than owning any, so the frame the record + /// sits in outlives the derivation, exactly as it outlives the arena + /// borrow the derived context carries. fn rebind<'a>(self) -> RecordValue<'a> { RecordValue { ptr: self.ptr, @@ -489,11 +494,8 @@ impl<'e> RecordValue<'e> { /// is at `'d`: the equality binding `ExtractArena` is an /// unconstrained position under a higher rank. pub trait DerivedRecordEdge<'derived, C> { - fn eval_derived(&self, cell: &crate::node::StatusCell, input_index: usize, ctx: &C) -> Result, crate::gpoll::Interrupt>; - /// [`serve_edge`] at the derived context, for poll kernels that carry the - /// status themselves. - fn serve_derived(&self, ctx: &C) -> GPoll>; - fn extent_at_derived(&self, ctx: &C, level: u8) -> GPoll; + fn eval_derived(&self, cell: &crate::node::StatusCell, input_index: usize, ctx: &C, frames: &Frames<'derived>) -> Result, crate::gpoll::Interrupt>; + fn extent_at_derived(&self, ctx: &C, level: u8, frames: &Frames<'derived>) -> GPoll; } impl<'derived, C, N> DerivedRecordEdge<'derived, C> for N @@ -501,24 +503,20 @@ where N: Node, C: crate::context::ExtractArena, { - fn eval_derived(&self, cell: &crate::node::StatusCell, input_index: usize, ctx: &C) -> Result, crate::gpoll::Interrupt> { - cell.eval_input(input_index, self, ctx) + fn eval_derived(&self, cell: &crate::node::StatusCell, input_index: usize, ctx: &C, frames: &Frames<'derived>) -> Result, crate::gpoll::Interrupt> { + cell.eval_input(input_index, self, ctx, frames) } - fn serve_derived(&self, ctx: &C) -> GPoll> { - serve_edge(self, ctx) - } - - fn extent_at_derived(&self, ctx: &C, level: u8) -> GPoll { - self.extent_at(ctx, level) + fn extent_at_derived(&self, ctx: &C, level: u8, frames: &Frames<'derived>) -> GPoll { + self.extent_at(ctx, level, frames) } } -/// Fills caller scratch with one frame per lane of `range`: the edge -/// evaluates at each index, the record's frame copies out, and the stack -/// rewinds, so the stack peak stays at one lane's need and every lane's bytes -/// are distinct. Frame bytes carry no drop glue, so the copy is a move. -pub fn fill_frames<'a, 'e, C, N>(node: &'a N, input: &C, range: std::ops::Range, scratch: Option<&'a mut [std::mem::MaybeUninit]>) -> crate::node::BatchStatus<'a> +/// Fills caller scratch with one frame per lane of `range`: the edge serves +/// into the lane's own region of the slab, and the lane's own frame space is +/// free again at the next lane, so the frame peak stays at one lane's need and +/// every lane's bytes are distinct. +pub fn fill_frames<'a, 'e, C, N>(node: &'a N, input: &C, range: std::ops::Range, scratch: Option<&'a mut [std::mem::MaybeUninit]>, frames: &Frames<'e>) -> crate::node::BatchStatus<'a> where C: crate::context::InjectIndex + Copy + crate::context::ExtractArena, N: Node, @@ -530,50 +528,41 @@ where let Some(len) = range.end.checked_sub(range.start).and_then(|len| usize::try_from(len).ok()) else { return BatchStatus::InvalidRange; }; - let layout = node.layout(); - let stride = layout.lane_stride(); - if scratch.len() * 8 < len * stride { + let Some(mut run) = frames.run(scratch, len, node.layout()) else { return BatchStatus::InvalidRange; - } - let base = scratch.as_mut_ptr().cast::(); + }; let mut local = *input; let mut finality = crate::gpoll::Finality::AllFinal; - let mut filled = len; let mut hint = crate::gpoll::Extent::AtLeast(range.end as usize); for lane in 0..len { local.set_index(range.start + lane as u64); - // SAFETY: the lane's record is copied out before the scope releases it, - // and valueless exits serve nothing above the entry. - let _lane_scope = unsafe { stack::ScopeGuard::enter() }; - let value = match serve_edge(node, &local) { - GPoll::Final(value) => value, - GPoll::Partial(value) => { + let lane_frames = frames.scope(); + let slot = run.slot(lane, &lane_frames); + let served = match node.serve(&local, slot) { + GPoll::Final(served) => served, + GPoll::Partial(served) => { finality = crate::gpoll::Finality::Partial; - value + served } GPoll::Pending => return BatchStatus::Pending, GPoll::Fallback(boxed) => return BatchStatus::Error(boxed.1), // A lane past a lower-bound level ends the data: the fill comes // back short and the hint turns exact. GPoll::Error(error) if error.kind == crate::gpoll::ErrorKind::PastEnd => { - filled = lane; hint = crate::gpoll::Extent::Exactly(range.start as usize + lane); break; } GPoll::Error(error) => return BatchStatus::Error(*error), }; - // SAFETY: the lane region is in-bounds by the scratch check, and the - // frame is fully copied out before the lane scope releases it. - unsafe { std::ptr::copy_nonoverlapping(layout.rec(&value).ptr(), base.add(lane * stride), stride) }; + run.served(lane, &served); } - // SAFETY: the first `filled` lanes were filled above with records of `layout`. - BatchStatus::Filled(unsafe { crate::node::RecordBatchMut::new(scratch, filled, layout) }, finality, hint) + BatchStatus::Filled(run.finish(), finality, hint) } /// The driver a consumer runs on a record edge: a resident batch returns with /// no allocation, a node's own batch impl gets `n * frame_bytes` of arena /// scratch, and an unbatched edge falls back to the [`fill_frames`] loop. -pub fn materialize_batch<'a, 'e, C, N>(node: &'a N, input: &'a C, range: std::ops::Range, arena: &'a crate::arena::Arena) -> crate::node::BatchStatus<'a> +pub fn materialize_batch<'a, 'e, C, N>(node: &'a N, input: &'a C, range: std::ops::Range, arena: &'a crate::arena::Arena, frames: &Frames<'e>) -> crate::node::BatchStatus<'a> where C: crate::context::InjectIndex + Copy + crate::context::ExtractArena, N: Node, @@ -589,13 +578,13 @@ where trace: Vec::new(), }) }; - match node.eval_batch(input, range.clone(), None) { + match node.eval_batch(input, range.clone(), None, frames) { BatchStatus::Unbatched => match arena.alloc_scratch::(words) { - Some(scratch) => fill_frames(node, input, range, Some(scratch)), + Some(scratch) => fill_frames(node, input, range, Some(scratch), frames), None => exhausted(), }, BatchStatus::NeedBuffer => match arena.alloc_scratch::(words) { - Some(scratch) => node.eval_batch(input, range, Some(scratch)), + Some(scratch) => node.eval_batch(input, range, Some(scratch), frames), None => exhausted(), }, status => status, @@ -613,21 +602,21 @@ pub enum LevelStatus<'a> { /// fills once, a lower bound drains by guess-and-double until a short fill, /// each reply's hint seeding the next guess. The boundary consumers' driver; /// reducers inline the same protocol with their span offsets. -pub fn materialize_level<'a, 'e, C, N>(node: &'a N, input: &'a C, arena: &'a crate::arena::Arena) -> LevelStatus<'a> +pub fn materialize_level<'a, 'e, C, N>(node: &'a N, input: &'a C, arena: &'a crate::arena::Arena, frames: &Frames<'e>) -> LevelStatus<'a> where C: crate::context::InjectIndex + Copy + crate::context::ExtractArena, N: Node, { use crate::gpoll::{Extent, GraphError, Level}; use crate::node::BatchStatus; - let sized = match node.extent(input, Level::Total) { + let sized = match node.extent(input, Level::Total, frames) { GPoll::Final(Extent::Exactly(count)) => Ok(count), GPoll::Final(Extent::AtLeast(bound)) => Err(bound), GPoll::Pending => return LevelStatus::Pending, _ => return LevelStatus::Error(GraphError::new("materialize over a non-exact extent")), }; match sized { - Ok(count) => match materialize_batch(node, input, 0..count as u64, arena) { + Ok(count) => match materialize_batch(node, input, 0..count as u64, arena, frames) { BatchStatus::Lent(batch, finality, _) => LevelStatus::Batch(batch, finality), BatchStatus::Filled(batch, finality, _) => LevelStatus::Batch(batch.into_shared(), finality), BatchStatus::Pending => LevelStatus::Pending, @@ -637,7 +626,7 @@ where Err(bound) => { let mut guess = bound.max(16); loop { - let (batch, finality, hint) = match materialize_batch(node, input, 0..guess as u64, arena) { + let (batch, finality, hint) = match materialize_batch(node, input, 0..guess as u64, arena, frames) { BatchStatus::Lent(batch, finality, hint) => (batch, finality, hint), BatchStatus::Filled(batch, finality, hint) => (batch.into_shared(), finality, hint), BatchStatus::Pending => return LevelStatus::Pending, @@ -663,14 +652,15 @@ where /// its wiring-proven layout, the pairing the kernel's unsafe record /// operations rely on. The kernel must only pair the layout with values this /// edge produced. -pub struct RecordEdgeInput<'a, N> { +pub struct RecordEdgeInput<'a, 'e, N> { node: &'a N, layout: &'a Layout, + frames: &'a Frames<'e>, } -impl<'a, N> RecordEdgeInput<'a, N> { - pub fn new(node: &'a N, layout: &'a Layout) -> Self { - Self { node, layout } +impl<'a, 'e, N> RecordEdgeInput<'a, 'e, N> { + pub fn new(node: &'a N, layout: &'a Layout, frames: &'a Frames<'e>) -> Self { + Self { node, layout, frames } } pub fn layout(&self) -> &Layout { @@ -679,7 +669,7 @@ impl<'a, N> RecordEdgeInput<'a, N> { /// Serves the edge through the kernel's own claim: the kernel's output /// layout is the edge's, so the claim it was handed is the edge's frame. - pub fn serve<'e, 'l, C>(&self, ctx: &C, slot: FrameClaim<'l>) -> GPoll> + pub fn serve<'l, C>(&self, ctx: &C, slot: FrameClaim<'e, 'l>) -> GPoll> where N: Node, C: crate::context::ExtractArena, @@ -689,12 +679,12 @@ impl<'a, N> RecordEdgeInput<'a, N> { /// [`materialize_level`] over the edge: the wire's whole flat span as one /// batch. - pub fn materialize_level<'e, 'b, C>(&'b self, ctx: &'b C, arena: &'b crate::arena::Arena) -> LevelStatus<'b> + pub fn materialize_level<'b, C>(&'b self, ctx: &'b C, arena: &'b crate::arena::Arena) -> LevelStatus<'b> where N: Node, C: crate::context::InjectIndex + Copy + crate::context::ExtractArena, { - materialize_level(self.node, ctx, arena) + materialize_level(self.node, ctx, arena, self.frames) } } @@ -707,39 +697,48 @@ unsafe fn element_only(rec: Rec, _reads: &[Option]) -> El { unsafe { read_element::(rec) } } -pub struct ElementEdge<'a, Out, N> { +pub struct ElementEdge<'a, 'e, Out, N> { node: &'a N, layout: &'a Layout, reads: &'a [Option], read: unsafe fn(Rec, &[Option]) -> Out, + frames: &'a Frames<'e>, } -impl<'a, El: Clone, N> ElementEdge<'a, El, N> { - pub fn new(node: &'a N, layout: &'a Layout) -> Self { +impl<'a, 'e, El: Clone, N> ElementEdge<'a, 'e, El, N> { + pub fn new(node: &'a N, layout: &'a Layout, frames: &'a Frames<'e>) -> Self { Self { node, layout, reads: &[], read: element_only::, + frames, } } } -impl<'a, Out, N> ElementEdge<'a, Out, N> { +impl<'a, 'e, Out, N> ElementEdge<'a, 'e, Out, N> { /// `read` must be sound against the layout the offsets in `reads` were /// resolved from; the macro proves both at wiring. - pub fn with_reads(node: &'a N, layout: &'a Layout, reads: &'a [Option], read: unsafe fn(Rec, &[Option]) -> Out) -> Self { - Self { node, layout, reads, read } + pub fn with_reads(node: &'a N, layout: &'a Layout, reads: &'a [Option], read: unsafe fn(Rec, &[Option]) -> Out, frames: &'a Frames<'e>) -> Self { + Self { node, layout, reads, read, frames } } + /// The edge's element at `ctx`, read out of a record claimed beyond the + /// kernel's own frame; the claim dies with the call, so the record is free + /// again at the next one. pub fn eval<'d, C>(&self, ctx: &C) -> GPoll where N: DerivedRecordEdge<'d, C>, + 'e: 'd, { - // SAFETY: the read copies out by value, so no record above the entry - // (the edge's own frame) is live past the scope. - let _scope = unsafe { stack::ScopeGuard::enter() }; - self.node.serve_derived(ctx).map(|value| unsafe { (self.read)(self.layout.rec(&value), self.reads) }) + let cell = crate::node::StatusCell::new(); + let scope = self.frames.scope(); + match self.node.eval_derived(&cell, 0, ctx, &scope) { + // SAFETY: the read copies out by value against the edge's own layout. + Ok(value) => cell.finish(unsafe { (self.read)(self.layout.rec(&value), self.reads) }), + Err(interrupt) => interrupt.into(), + } } } @@ -747,17 +746,18 @@ impl<'a, Out, N> ElementEdge<'a, Out, N> { /// the kernel consumes the plain element, or the element beside its declared /// attribute reads. #[derive(Clone, Copy)] -pub struct ElementLazyInput<'a, Out, N> { +pub struct ElementLazyInput<'a, 'e, Out, N> { node: &'a N, cell: &'a crate::node::StatusCell, input_index: usize, layout: &'a Layout, reads: &'a [Option], read: unsafe fn(Rec, &[Option]) -> Out, + frames: &'a Frames<'e>, } -impl<'a, El: Clone, N> ElementLazyInput<'a, El, N> { - pub fn new(node: &'a N, cell: &'a crate::node::StatusCell, input_index: usize, layout: &'a Layout) -> Self { +impl<'a, 'e, El: Clone, N> ElementLazyInput<'a, 'e, El, N> { + pub fn new(node: &'a N, cell: &'a crate::node::StatusCell, input_index: usize, layout: &'a Layout, frames: &'a Frames<'e>) -> Self { Self { node, cell, @@ -765,14 +765,23 @@ impl<'a, El: Clone, N> ElementLazyInput<'a, El, N> { layout, reads: &[], read: element_only::, + frames, } } } -impl<'a, Out, N> ElementLazyInput<'a, Out, N> { +impl<'a, 'e, Out, N> ElementLazyInput<'a, 'e, Out, N> { /// `read` must be sound against the layout the offsets in `reads` were /// resolved from; the macro proves both at wiring. - pub fn with_reads(node: &'a N, cell: &'a crate::node::StatusCell, input_index: usize, layout: &'a Layout, reads: &'a [Option], read: unsafe fn(Rec, &[Option]) -> Out) -> Self { + pub fn with_reads( + node: &'a N, + cell: &'a crate::node::StatusCell, + input_index: usize, + layout: &'a Layout, + reads: &'a [Option], + read: unsafe fn(Rec, &[Option]) -> Out, + frames: &'a Frames<'e>, + ) -> Self { Self { node, cell, @@ -780,18 +789,20 @@ impl<'a, Out, N> ElementLazyInput<'a, Out, N> { layout, reads, read, + frames, } } + /// The read copies the element and declared attributes out by value, so + /// the record's claim dies with the call. pub fn eval<'d, C>(&self, ctx: &C) -> Result where N: DerivedRecordEdge<'d, C>, + 'e: 'd, { - // SAFETY: the read copies the element and declared attributes out by - // value, so no record above the entry (the edge's own frame) is live - // past the scope. - let _scope = unsafe { stack::ScopeGuard::enter() }; - let value = self.node.eval_derived(self.cell, self.input_index, ctx)?; + let scope = self.frames.scope(); + let value = self.node.eval_derived(self.cell, self.input_index, ctx, &scope)?; + // SAFETY: the reads are the edge's own layout's, resolved at wiring. Ok(unsafe { (self.read)(self.layout.rec(&value), self.reads) }) } } @@ -805,25 +816,26 @@ pub struct RecordLazyInput<'a, 'e, N> { cell: &'a crate::node::StatusCell, input_index: usize, inner_levels: u8, - _lifetime: std::marker::PhantomData RecordValue<'e>>, + frames: &'a Frames<'e>, } impl<'a, 'e, N> RecordLazyInput<'a, 'e, N> { - pub fn new(node: &'a N, cell: &'a crate::node::StatusCell, input_index: usize, inner_levels: u8) -> Self { + pub fn new(node: &'a N, cell: &'a crate::node::StatusCell, input_index: usize, inner_levels: u8, frames: &'a Frames<'e>) -> Self { Self { node, cell, input_index, inner_levels, - _lifetime: std::marker::PhantomData, + frames, } } pub fn eval<'d, C>(&self, ctx: &C) -> Result, crate::gpoll::Interrupt> where N: DerivedRecordEdge<'d, C>, + 'e: 'd, { - Ok(self.node.eval_derived(self.cell, self.input_index, ctx)?.rebind()) + Ok(self.node.eval_derived(self.cell, self.input_index, ctx, self.frames)?.rebind()) } /// The flat lane count of one copy: the product of the edge's inner-level @@ -834,7 +846,7 @@ impl<'a, 'e, N> RecordLazyInput<'a, 'e, N> { B: crate::context::DeriveCtx, N: for<'d> DerivedRecordEdge<'d, crate::context::Derived<'d, B>>, { - inner_extent_of(self.node, ctx, 0, self.inner_levels, self.input_index) + inner_extent_of(self.node, ctx, 0, self.inner_levels, self.input_index, self.frames) } /// The flat lane count of the copy at `copy`, for edges whose inner @@ -844,12 +856,12 @@ impl<'a, 'e, N> RecordLazyInput<'a, 'e, N> { B: crate::context::DeriveCtx, N: for<'d> DerivedRecordEdge<'d, crate::context::Derived<'d, B>>, { - inner_extent_of(self.node, ctx, copy, self.inner_levels, self.input_index) + inner_extent_of(self.node, ctx, copy, self.inner_levels, self.input_index, self.frames) } } /// See [`RecordLazyInput::inner_extent`]. -fn inner_extent_of(node: &N, ctx: &B, copy: u64, levels: u8, input_index: usize) -> Result +fn inner_extent_of(node: &N, ctx: &B, copy: u64, levels: u8, input_index: usize, frames: &Frames<'_>) -> Result where B: crate::context::DeriveCtx, N: for<'d> DerivedRecordEdge<'d, crate::context::Derived<'d, B>>, @@ -858,9 +870,9 @@ where let derived = ctx.push_level(&mut frame, copy, 0); let mut inner: u64 = 1; for level in 0..levels { - match node.extent_at_derived(&derived, level) { + match node.extent_at_derived(&derived, level, frames) { GPoll::Final(crate::gpoll::Extent::Exactly(count)) => inner *= count as u64, - GPoll::Final(crate::gpoll::Extent::AtLeast(_)) => return probed_inner(node, ctx, copy, input_index), + GPoll::Final(crate::gpoll::Extent::AtLeast(_)) => return probed_inner(node, ctx, copy, input_index, frames), GPoll::Pending => return Err(crate::gpoll::Interrupt::Pending), _ => return Err(crate::gpoll::GraphError::new("structure decomposition over a non-exact extent").into()), } @@ -871,7 +883,7 @@ where /// The flat lane count of one copy of a lower-bound edge, probed by /// evaluating lanes to the past-end signal. The probed records are /// discarded, and their statuses land in a scratch cell. -fn probed_inner(node: &N, ctx: &B, copy: u64, input_index: usize) -> Result +fn probed_inner(node: &N, ctx: &B, copy: u64, input_index: usize, frames: &Frames<'_>) -> Result where B: crate::context::DeriveCtx, N: for<'d> DerivedRecordEdge<'d, crate::context::Derived<'d, B>>, @@ -879,12 +891,12 @@ where let cell = crate::node::StatusCell::new(); let mut count: u64 = 0; loop { - // SAFETY: the probed record is discarded, so nothing above the entry - // is live past the scope. - let _scope = unsafe { stack::ScopeGuard::enter() }; + // The probed record is discarded, so the probe's claim is free again + // at the next iteration. + let probe_frames = frames.scope(); let mut frame = crate::context::IndexLink { index: 0, outer: None }; let probe = ctx.push_level(&mut frame, copy, count); - let result = node.eval_derived(&cell, input_index, &probe); + let result = node.eval_derived(&cell, input_index, &probe, &probe_frames); match result { Ok(_) => count += 1, Err(crate::gpoll::Interrupt::Error(error)) if error.kind == crate::gpoll::ErrorKind::PastEnd => return Ok(count), @@ -905,13 +917,21 @@ pub struct DerivedLazyInput<'a, 'e, Out, N> { inner_levels: u8, reads: &'a [Option], read: unsafe fn(Rec, &[Option]) -> Out, - _lifetime: std::marker::PhantomData RecordValue<'e>>, + frames: &'a Frames<'e>, } impl<'a, 'e, Out, N> DerivedLazyInput<'a, 'e, Out, N> { /// `read` must be sound against the layout the offsets in `reads` were /// resolved from; the macro proves both at wiring. - pub fn new(node: &'a N, cell: &'a crate::node::StatusCell, input_index: usize, inner_levels: u8, reads: &'a [Option], read: unsafe fn(Rec, &[Option]) -> Out) -> Self { + pub fn new( + node: &'a N, + cell: &'a crate::node::StatusCell, + input_index: usize, + inner_levels: u8, + reads: &'a [Option], + read: unsafe fn(Rec, &[Option]) -> Out, + frames: &'a Frames<'e>, + ) -> Self { Self { node, cell, @@ -919,7 +939,7 @@ impl<'a, 'e, Out, N> DerivedLazyInput<'a, 'e, Out, N> { inner_levels, reads, read, - _lifetime: std::marker::PhantomData, + frames, } } @@ -929,14 +949,15 @@ impl<'a, 'e, Out, N> DerivedLazyInput<'a, 'e, Out, N> { B: crate::context::DeriveCtx, N: for<'d> DerivedRecordEdge<'d, crate::context::Derived<'d, B>>, { - inner_extent_of(self.node, ctx, 0, self.inner_levels, self.input_index) + inner_extent_of(self.node, ctx, 0, self.inner_levels, self.input_index, self.frames) } pub fn eval<'d, C>(&self, ctx: &C) -> Result where N: DerivedRecordEdge<'d, C>, + 'e: 'd, { - let value: RecordValue<'e> = self.node.eval_derived(self.cell, self.input_index, ctx)?.rebind(); + let value: RecordValue<'e> = self.node.eval_derived(self.cell, self.input_index, ctx, self.frames)?.rebind(); // SAFETY: declared reads imply a non-empty layout, so the record is // spilled and its pointer is the frame the offsets index into. Ok(unsafe { (self.read)(Rec::new(value.ptr), self.reads) }) @@ -951,279 +972,212 @@ pub unsafe fn token_only<'e>(rec: Rec, _reads: &[Option]) -> RecordValue< RecordValue::spilled(rec) } -/// The per-thread record stack: every record evaluation claims its activation -/// frame at the stack pointer and evaluates its carrier beyond it, so slot -/// addresses are a property of the evaluating thread and no global assignment -/// exists. Thread-local by construction, so access is single-threaded without -/// claims or gates. Records are overwritten per lane and never touch the -/// arena. -pub mod stack { - use std::cell::Cell; +/// The evaluation's record frame space: a grow-only buffer the executor owns +/// and lends by `&mut`, sized by the wiring-derived frame need, so exhaustion +/// is an accounting failure the debug assertion catches rather than a hot-path +/// branch. Frame bytes carry no drop glue, so growth and reuse are plain +/// buffer operations. +#[derive(Debug, Default)] +pub struct FrameArena { + buf: Vec, +} - struct Stack { - base: Cell<*mut u8>, - capacity: Cell, - sp: Cell, +impl FrameArena { + pub fn new() -> Self { + Self { buf: Vec::new() } } - impl Stack { - fn free(&self) { - let base = self.base.get(); - if !base.is_null() { - drop(unsafe { Vec::from_raw_parts(base.cast::(), 0, self.capacity.get() / 8) }); - } + /// Grows the buffer to hold `bytes`, the root's wiring-derived frame need. + /// Grow-only, so repeated evaluations reuse one allocation. + pub fn reserve(&mut self, bytes: usize) { + let words = bytes.div_ceil(8).max(1); + if self.buf.len() < words { + self.buf.resize(words, 0); } } - impl Drop for Stack { - fn drop(&mut self) { - self.free(); + /// The whole buffer as free space, for one evaluation. + pub fn frames(&mut self) -> Frames<'_> { + Frames { + base: std::cell::Cell::new(self.buf.as_mut_ptr().cast::()), + words: std::cell::Cell::new(self.buf.len()), + _lifetime: std::marker::PhantomData, } } - - thread_local! { - static STACK: Stack = const { - Stack { - base: Cell::new(std::ptr::null_mut()), - capacity: Cell::new(0), - sp: Cell::new(0), - } - }; - } - - /// Ensures the calling thread's stack holds `bytes`, the root's wiring- - /// derived stack need, and resets the stack pointer. Called only between - /// evaluations, like the arena reset: nothing survives it, so frames - /// leaked by an interrupted evaluation are reclaimed here. - /// - /// # Safety - /// No record served on this thread's stack may be live: growth frees the - /// buffer and the reset releases every claimed frame. - pub unsafe fn reserve(bytes: usize) { - STACK.with(|stack| { - stack.sp.set(0); - if stack.capacity.get() >= bytes { - return; - } - let words = bytes.div_ceil(8).max(1); - let mut memory = vec![0u64; words]; - let base = memory.as_mut_ptr().cast::(); - std::mem::forget(memory); - stack.free(); - stack.base.set(base); - stack.capacity.set(words * 8); - }); - } - - /// Claims `bytes` (rounded to word alignment) at the stack pointer and - /// advances past them. The region stays claimed until [`pop`], and stays - /// readable until the next `push`. `reserve` derived from the root's - /// stack need makes the capacity bound exact, so overflow is a debug - /// assertion, not a hot-path branch. - pub fn push(bytes: usize) -> *mut u8 { - STACK.with(|stack| { - let sp = stack.sp.get(); - let next = sp + bytes.next_multiple_of(8); - debug_assert!(next <= stack.capacity.get(), "record stack overflow: reserve() must cover the root's stack need"); - stack.sp.set(next); - unsafe { stack.base.get().add(sp) } - }) - } - - /// Returns the stack pointer to `frame`, a pointer earlier returned by - /// [`push`] on this thread, releasing it and everything above it. Resets to - /// a checkpoint between repeated evaluations. - pub fn pop(frame: *mut u8) { - STACK.with(|stack| { - let offset = frame as usize - stack.base.get() as usize; - debug_assert!(offset <= stack.sp.get(), "pop target must lie within the claimed stack"); - stack.sp.set(offset); - }); - } - - /// Releases everything above `frame`'s `bytes`-sized region, keeping the - /// region itself. A node reclaims its inputs' frames on return but leaves - /// its own output readable for its consumer. - pub(crate) fn truncate_above(frame: *mut u8, bytes: usize) { - STACK.with(|stack| { - let top = frame as usize - stack.base.get() as usize + bytes.next_multiple_of(8); - debug_assert!(top <= stack.sp.get(), "truncate target must lie within the claimed stack"); - stack.sp.set(top); - }); - } - - /// The current stack pointer, a checkpoint to [`rewind`] to. - pub fn sp() -> usize { - STACK.with(|stack| stack.sp.get()) - } - - /// Resets the stack pointer to an earlier [`sp`] checkpoint, so a loop that - /// evaluates a subtree per iteration reuses the same slots each time. - /// - /// # Safety - /// No `Rec` or `RecordValue` into the region above `mark` may be used after - /// this call. The caller must have copied out everything it still needs. - pub unsafe fn rewind(mark: usize) { - STACK.with(|stack| { - debug_assert!(mark <= stack.sp.get(), "rewind target above the stack pointer"); - stack.sp.set(mark); - }); - } - - /// Rewinds to the entry stack pointer on drop, unwind included: the - /// structured form of a mark/rewind pair. A forgotten guard leaks its - /// region until the next reserve rather than releasing it. - pub struct ScopeGuard { - mark: usize, - } - - impl ScopeGuard { - /// # Safety - /// No `Rec` or `RecordValue` served above the entry point may be used - /// after the guard drops; copy out everything that survives the scope. - pub unsafe fn enter() -> Self { - Self { mark: sp() } - } - } - - /// Releases everything claimed above `mark`; a pointer already at or - /// below it stays. - pub(crate) fn release_above(mark: usize) { - STACK.with(|stack| { - if mark < stack.sp.get() { - stack.sp.set(mark); - } - }); - } - - impl Drop for ScopeGuard { - fn drop(&mut self) { - release_above(self.mark); - } - } - - /// Runs `body` under a [`ScopeGuard`]: every frame it claims releases on - /// return. - /// - /// # Safety - /// As [`ScopeGuard::enter`]: nothing served inside the scope may escape - /// it, through the return value or a captured location. - pub unsafe fn scoped(body: impl FnOnce() -> R) -> R { - let _scope = unsafe { ScopeGuard::enter() }; - body() - } } -/// Serves one record of a layout: claims the frame, takes type-checked -/// element and attribute writes, and closes into the served [`RecordValue`]. -/// Unwritten fields serve their census defaults. The single-record sibling -/// of [`RunBuilder`], for sources that spell their own serving. -pub struct FrameBuilder<'l, 'e> { - layout: &'l Layout, - arena: &'e crate::arena::Arena, - value: RecordValue<'e>, - frame: Option<*mut u8>, - wrote_element: bool, - exhausted: bool, +/// The free frame space at one point of an evaluation. [`Self::claim`] splits +/// a node's own frame off the front and the claim carries the remainder, so a +/// node's inputs claim beyond its frame, one after another, and the space they +/// used is free again once the claim dies: the release is the claim's +/// lifetime, not a rewind contract. The cursor is shared through `&self` so +/// the lazy edges a kernel holds claim beyond each other rather than over each +/// other. Covariant in `'e`, so a claim minted at the evaluation shortens onto +/// a derived context's arena lifetime. +pub struct Frames<'e> { + base: std::cell::Cell<*mut u8>, + words: std::cell::Cell, + _lifetime: std::marker::PhantomData<&'e ()>, } -impl<'l, 'e> FrameBuilder<'l, 'e> { - /// Claims the layout's frame and default-fills its fields. - pub fn new(layout: &'l Layout, arena: &'e crate::arena::Arena) -> Self { +impl std::fmt::Debug for Frames<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Frames").field("free_words", &self.words.get()).finish() + } +} + +impl<'e> Frames<'e> { + /// An independent cursor over the same free space: claims made through it + /// are free again when it dies, so a repeated evaluation reuses one region. + /// Two live cursors hand out the same region, so only one may be claimed + /// from at a time. + pub fn reborrow(&self) -> Frames<'e> { + Frames { + base: std::cell::Cell::new(self.base.get()), + words: std::cell::Cell::new(self.words.get()), + _lifetime: std::marker::PhantomData, + } + } + + /// Runs claims against this space and gives it back on the guard's drop, so + /// a loop that evaluates a subtree per iteration reuses the same region. + pub fn scope(&self) -> FrameScope<'_, 'e> { + FrameScope { + frames: self, + base: self.base.get(), + words: self.words.get(), + } + } + + /// The words still free, the observable the frame accounting asserts on. + pub fn free_words(&self) -> usize { + self.words.get() + } + + /// Claims `layout`'s frame at the front of the free space; the claim + /// carries the remainder, and an inline layout's record builds in the + /// claim itself. + pub fn claim<'l>(&self, layout: &'l Layout) -> FrameClaim<'e, 'l> { let frame = match layout.frame_bytes() { 0 => None, - bytes => Some(stack::push(bytes)), + bytes => Some(self.split(bytes)), }; - if let Some(frame) = frame { - // SAFETY: the frame was just claimed for this layout, and each - // field's region lies within it. - let bytes = unsafe { std::slice::from_raw_parts_mut(frame, layout.frame_bytes()) }; - bytes.fill(0); - for field in &layout.fields { - if let Some(info) = crate::attribute::info(field.name) - && info.size == field.size - { - (info.write_default_bytes)(&mut bytes[field.offset..field.offset + field.size]); - } - } - } - Self { + FrameClaim { layout, - arena, - value: RecordValue::zeroed(), + inline: RecordValue::zeroed(), frame, - wrote_element: false, - exhausted: false, + free: self.reborrow(), } } - fn dst(&mut self) -> *mut u8 { - match self.frame { - Some(frame) => frame, - None => self.value.as_mut_ptr(), - } + /// Splits `bytes` (rounded to word alignment) off the front. + fn split(&self, bytes: usize) -> *mut u8 { + let words = bytes.div_ceil(8); + debug_assert!(words <= self.words.get(), "record frame space exhausted: the root buffer must cover the graph's frame need"); + let frame = self.base.get(); + // SAFETY: the buffer covers the wiring-derived need, so the advanced + // cursor stays within it. + self.base.set(unsafe { frame.add(words * 8) }); + self.words.set(self.words.get() - words); + frame } - /// Moves the element in, parking a droppable payload. Arena exhaustion - /// surfaces at [`Self::finish`]. - pub fn element(&mut self, element: T) { - assert_eq!(std::any::TypeId::of::(), self.layout.element.type_id, "the served element must match the layout's element type"); - assert!(!self.wrote_element, "the element serves once"); - self.wrote_element = true; - let dst = self.dst(); - // SAFETY: dst is this record's own element slot and the type matches - // the layout's element. - if unsafe { write_element(dst, element, self.arena) }.is_none() { - self.exhausted = true; - } - } - - /// Writes the marker's value. The layout must carry the marker among its - /// level-0 fields, declared at the marker's value type. - pub fn attr(&mut self, value: A::Value<'e>) - where - A::Value<'static>: 'static, - { - let layout = self.layout; - let field = layout.fields.iter().find(|field| field.name == A::NAME && field.level == 0).expect("the layout carries the written marker"); - assert_eq!(field.type_id, std::any::TypeId::of::>(), "the field was declared at the marker's value type"); - let dst = self.dst(); - // SAFETY: the offset is the builder's own layout's and the value type - // matches the field's declared type. - unsafe { write_field(dst, field.offset, value) }; - } - - /// Writes a field by name and level, for layouts whose fields are not - /// census markers. The field must be declared at this value type. - pub fn field(&mut self, name: &str, level: u8, value: T) { - let layout = self.layout; - let field = layout.fields.iter().find(|field| field.name == name && field.level == level).expect("the layout carries the written field"); - assert_eq!(field.type_id, std::any::TypeId::of::(), "the field was declared at this value type"); - let dst = self.dst(); - // SAFETY: as for [`Self::attr`]. - unsafe { write_field(dst, field.offset, value) }; - } - - /// The served record. Panics unless the element was written, since an - /// unwritten parked slot must never become readable; `None` reports arena - /// exhaustion, with the frame released either way. - pub fn finish(mut self) -> Option> { - assert!(self.wrote_element || self.layout.element.size == 0, "the element serves before the frame closes"); - let value = match self.frame.take() { - // SAFETY: the frame was claimed for this layout and stays claimed, - // keeping the frame contract for the consumer's release. - Some(frame) => RecordValue::spilled(unsafe { Rec::new(frame.cast_const()) }), - None => std::mem::replace(&mut self.value, RecordValue::zeroed()), - }; - match self.exhausted { - true => None, - false => Some(value), - } + /// A run of same-layout slots over caller scratch: lanes serve in place, + /// each backed by its own region of the slab, and the collected proofs + /// certify the filled prefix. `None` where the scratch cannot hold `len` + /// lanes. + pub fn run<'a>(&self, scratch: &'a mut [std::mem::MaybeUninit], len: usize, layout: &'a Layout) -> Option> { + SlotRun::new(scratch, len, layout) } } -impl Drop for FrameBuilder<'_, '_> { - fn drop(&mut self) {} +/// Law-test scaffolding: a frame space of `bytes`, leaked so a fixture holds +/// it for the whole test without threading the buffer's own borrow. Production +/// roots own their buffer and lend it by `&mut`. +#[doc(hidden)] +pub fn test_frames(bytes: usize) -> Frames<'static> { + let arena: &'static mut FrameArena = Box::leak(Box::new(FrameArena::new())); + arena.reserve(bytes); + arena.frames() +} + +/// See [`Frames::scope`]. A forgotten guard leaks its region until the frame +/// space itself dies, rather than releasing it. +pub struct FrameScope<'s, 'e> { + frames: &'s Frames<'e>, + base: *mut u8, + words: usize, +} + +impl<'e> std::ops::Deref for FrameScope<'_, 'e> { + type Target = Frames<'e>; + + fn deref(&self) -> &Frames<'e> { + self.frames + } +} + +impl Drop for FrameScope<'_, '_> { + fn drop(&mut self) { + self.frames.base.set(self.base); + self.frames.words.set(self.words); + } +} + +/// A claimed run of same-layout slots over the caller's scratch: [`Self::slot`] +/// backs an ordinary claim's own region by the lane's region of the slab, so a +/// lane serves in place with no staging copy, and [`Self::served`] records the +/// proof. The filled prefix is the only readable part, which is what makes +/// [`Self::finish`] safe. +pub struct SlotRun<'a> { + scratch: &'a mut [std::mem::MaybeUninit], + layout: &'a Layout, + len: usize, + filled: usize, +} + +impl<'a> SlotRun<'a> { + fn new(scratch: &'a mut [std::mem::MaybeUninit], len: usize, layout: &'a Layout) -> Option> { + (scratch.len() * 8 >= len * layout.lane_stride()).then_some(SlotRun { scratch, layout, len, filled: 0 }) + } + + pub fn layout(&self) -> &'a Layout { + self.layout + } + + /// Lane `lane`'s claim: its own frame is the lane's region of the slab and + /// its free space is `frames`, so the lane's inputs claim beyond it. + pub fn slot<'e>(&mut self, lane: usize, frames: &Frames<'e>) -> FrameClaim<'e, 'a> { + assert!(lane < self.len, "lane {lane} out of bounds for a run of {}", self.len); + let stride = self.layout.lane_stride(); + // SAFETY: in-bounds by the assert against the capacity check `new` made. + let frame = unsafe { self.scratch.as_mut_ptr().cast::().add(lane * stride) }; + + FrameClaim { + layout: self.layout, + inline: RecordValue::zeroed(), + frame: (self.layout.frame_bytes() != 0).then_some(frame), + free: frames.reborrow(), + } + } + + /// Records lane `lane` as served; the proof came from that lane's slot. An + /// inline record rides the value's own storage, so it takes the one copy + /// the run makes. + pub fn served(&mut self, lane: usize, proof: &Served<'_>) { + if self.layout.size == 0 { + let stride = self.layout.lane_stride(); + // SAFETY: in-bounds by the capacity check `new` made, and the lane + // takes the whole inline record. + unsafe { std::ptr::copy_nonoverlapping(self.layout.rec(proof.record()).ptr(), self.scratch.as_mut_ptr().cast::().add(lane * stride), stride) }; + } + self.filled = self.filled.max(lane + 1); + } + + /// The served lanes as the caller's exclusive batch. + pub fn finish(self) -> crate::node::RecordBatchMut<'a> { + crate::node::RecordBatchMut::new(self.scratch, self.filled, self.layout) + } } /// Field-by-field carry from `from`'s layout into `to`'s, computed at @@ -1257,23 +1211,16 @@ pub unsafe fn write_field(dst: *mut u8, offset: usize, value: T) { } /// Finishes a carried record frame: the element lands beside the fields -/// already carried into `dst`, inline frames copy out of the scratch bytes, -/// and the frame releases in every branch, so the frame lifecycle closes -/// here. Arena exhaustion of a parked element reports as an error poll. +/// already carried into `dst`, and inline frames copy out of the scratch +/// bytes. Arena exhaustion of a parked element reports as an error poll. /// /// # Safety /// `dst` must be the claimed frame (or inline scratch when `frame_bytes` is /// 0) of a record whose element is `T` and whose frame size is `frame_bytes`, /// with every carried field already written. pub(crate) unsafe fn lift_poll_into<'e, T: Send + Sync>(poll: GPoll, dst: *mut u8, frame_bytes: usize, arena: &'e crate::arena::Arena) -> GPoll> { - let release = || { - if frame_bytes != 0 { - stack::truncate_above(dst, frame_bytes); - } - }; let build = |element: T| { let written = unsafe { write_element(dst, element, arena) }; - release(); written.map(|()| match frame_bytes { 0 => unsafe { dst.cast::().read() }, _ => RecordValue::spilled(unsafe { Rec::new(dst.cast_const()) }), @@ -1292,14 +1239,8 @@ pub(crate) unsafe fn lift_poll_into<'e, T: Send + Sync>(poll: GPoll, dst: *mu let (element, error) = *boxed; build(element).map_or_else(exhausted, |value| GPoll::Fallback(Box::new((value, error)))) } - GPoll::Pending => { - release(); - GPoll::Pending - } - GPoll::Error(error) => { - release(); - GPoll::Error(error) - } + GPoll::Pending => GPoll::Pending, + GPoll::Error(error) => GPoll::Error(error), } } @@ -1591,10 +1532,9 @@ impl SourcePlan { /// Evaluating it yields the source's record translated to the union layout /// (or forwarded untouched when the layouts already agree), so the kernel /// holds and returns record values without ever seeing the representation. -/// A translation claims its landing region from the record stack without -/// popping, so the value survives sibling evaluations; the region is -/// released with the enclosing frame, which bounds claims at one per source -/// evaluation the kernel performs. +/// A translation lands in the claim the caller minted, so the value survives +/// sibling evaluations and its region is free again with that claim, which +/// bounds claims at one per source evaluation the kernel performs. pub struct RecordSource { edge: N, plan: Option, @@ -1617,32 +1557,24 @@ pub unsafe fn copy_record_bytes(layout: &Layout, rec: Rec) -> Box<[u8]> { unsafe { std::slice::from_raw_parts(rec.ptr(), layout.size) }.into() } -/// A node's own output frame, claimed at eval entry: the one closing surface -/// for every exit. Writes land through it, [`Self::lift`] and [`Self::finish`] -/// serve the record, and its drop releases everything claimed above the frame -/// while keeping the frame itself, so the frame contract holds on value, -/// error, and pending exits alike with no per-exit ritual. -pub struct FrameClaim<'l> { +/// A node's own output frame, minted by its caller from the caller's frame +/// space: the one closing surface for every exit. Writes land through it, +/// [`Self::lift`] and [`Self::finish`] serve the record, and it carries the +/// free space beyond the frame, so a node's inputs claim past it and their +/// space is free again when the claim dies, on value, error, and pending exits +/// alike with no per-exit ritual. +pub struct FrameClaim<'e, 'l> { layout: &'l Layout, inline: RecordValue<'static>, frame: Option<*mut u8>, - own_end: usize, + free: Frames<'e>, } -impl<'l> FrameClaim<'l> { - /// Claims the layout's frame at the stack pointer; an inline layout's - /// record builds in the value itself. - pub fn enter(layout: &'l Layout) -> Self { - let frame = match layout.frame_bytes() { - 0 => None, - bytes => Some(stack::push(bytes)), - }; - Self { - layout, - inline: RecordValue::zeroed(), - frame, - own_end: stack::sp(), - } +impl<'e, 'l> FrameClaim<'e, 'l> { + /// The free space beyond this claim's frame, which the node's own inputs + /// claim from. + pub fn frames(&mut self) -> &Frames<'e> { + &mut self.free } fn dst(&mut self) -> *mut u8 { @@ -1695,7 +1627,7 @@ impl<'l> FrameClaim<'l> { /// writes on value polls, every poll keeps the frame claimed, and arena /// exhaustion of a parked element reports as an error poll. Panics where /// the element does not match the wired layout. - pub fn lift<'e, T: Send + Sync + dyn_any::StaticTypeSized>(mut self, poll: GPoll, arena: &'e crate::arena::Arena) -> GPoll> { + pub fn lift(mut self, poll: GPoll, arena: &'e crate::arena::Arena) -> GPoll> { self.check_element::(); let frame_bytes = self.layout.frame_bytes(); let dst = self.dst(); @@ -1720,7 +1652,7 @@ impl<'l> FrameClaim<'l> { /// # Safety /// The frame must hold a complete record of the layout, written through /// the carry, element, and field writes. - pub unsafe fn finish<'e>(mut self) -> RecordValue<'e> { + pub unsafe fn finish(mut self) -> RecordValue<'e> { match self.frame { Some(frame) => RecordValue::spilled(unsafe { Rec::new(frame.cast_const()) }), // SAFETY: the inline record is the value's own bytes. @@ -1729,7 +1661,7 @@ impl<'l> FrameClaim<'l> { } /// [`Self::lift`] with the proof-bearing return for [`Node::serve`]. - pub fn lift_served<'e, T: Send + Sync + dyn_any::StaticTypeSized>(self, poll: GPoll, arena: &'e crate::arena::Arena) -> GPoll> { + pub fn lift_served(self, poll: GPoll, arena: &'e crate::arena::Arena) -> GPoll> { self.lift(poll, arena).map(|value| Served { value }) } @@ -1737,7 +1669,7 @@ impl<'l> FrameClaim<'l> { /// /// # Safety /// As [`Self::finish`]. - pub unsafe fn finish_served<'e>(self) -> Served<'e> { + pub unsafe fn finish_served(self) -> Served<'e> { Served { value: unsafe { self.finish() } } } @@ -1747,7 +1679,7 @@ impl<'l> FrameClaim<'l> { /// /// # Safety /// `value` must be a live record of this frame's layout. - pub unsafe fn forward<'e>(mut self, value: &RecordValue<'_>) -> Served<'e> { + pub unsafe fn forward(mut self, value: &RecordValue<'_>) -> Served<'e> { let src = self.layout.rec(value).ptr(); unsafe { self.fill_copy(src); @@ -1765,6 +1697,61 @@ impl<'l> FrameClaim<'l> { } } +/// A materialized run of lanes as the arena region its frames live in: the +/// handle keeps the provenance the region was allocated with and carries the +/// generation, so resolving it re-checks liveness where an address would have +/// been trusted. The layout stays with the holder, which proved it at wiring. +#[derive(Clone, Copy)] +pub struct MaterializedSpan { + base: crate::arena::ArenaWeak, + len: usize, +} + +impl std::fmt::Debug for MaterializedSpan { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MaterializedSpan").field("len", &self.len).finish() + } +} + +impl MaterializedSpan { + /// `None` where the batch's frames are not this arena's, which is the + /// caller's cue to re-materialize rather than cache. + pub fn of(batch: &crate::node::RecordBatch<'_>, arena: &crate::arena::Arena) -> Option { + match batch.len() { + 0 => Some(MaterializedSpan { + base: crate::arena::ArenaWeak::NULL, + len: 0, + }), + len => Some(MaterializedSpan { + base: arena.handle_at(batch.get(0).rec().ptr())?, + len, + }), + } + } + + /// The span's lanes at `layout`, or `None` once the generation moved on. + pub fn batch<'a>(&self, arena: &'a crate::arena::Arena, layout: &'a Layout) -> Option> { + let base: *const u8 = match self.len { + 0 => std::ptr::NonNull::::dangling().as_ptr(), + _ => self.base.upgrade(arena)?, + }; + // SAFETY: the handle resolved in generation, so the region still holds + // the lanes it was published with, packed at the layout's stride; an + // empty span reads no lane. + Some(unsafe { crate::node::RecordBatch::new(base, self.len, layout) }) + } + + /// Lane `lane`'s record, or `None` past the span or once the generation + /// moved on. + pub fn lane(&self, arena: &crate::arena::Arena, lane: usize, layout: &Layout) -> Option<*const u8> { + (lane < self.len).then_some(())?; + let base: *const u8 = self.base.upgrade(arena)?; + // SAFETY: in-bounds by the length check, at the layout the span was + // published under. + Some(unsafe { base.add(lane * layout.lane_stride()) }) + } +} + /// The proof a record was served through a frame claim: mintable only by the /// claim's closing methods, so holding one means the record is of the /// claimed layout. @@ -1784,21 +1771,22 @@ impl<'e> Served<'e> { } } -/// Claims `node`'s own frame and serves through it: the caller-side half of -/// [`Node::serve`], for drivers that want the record rather than the proof. -pub fn serve_edge<'e, C, N>(node: &N, input: &C) -> GPoll> +/// Claims `node`'s own frame from `frames` and serves through it: the +/// caller-side half of [`Node::serve`], for drivers that want the record +/// rather than the proof. +pub fn serve_edge<'e, C, N>(node: &N, input: &C, frames: &Frames<'e>) -> GPoll> where N: Node + ?Sized, C: crate::context::ExtractArena, { - let slot = FrameClaim::enter(node.layout()); + let slot = frames.claim(node.layout()); node.serve(input, slot).map(Served::value) } -impl Drop for FrameClaim<'_> { - fn drop(&mut self) { - stack::release_above(self.own_end); - } +/// A claim shortens onto a derived context's arena lifetime. +#[cfg(test)] +fn claim_shortens<'long: 'short, 'short, 'l>(claim: FrameClaim<'long, 'l>) -> FrameClaim<'short, 'l> { + claim } /// A record deep-copied out of its evaluation: the packed bytes plus owned @@ -1833,26 +1821,10 @@ impl OwnedRecord { OwnedRecord { bytes, element, fields } } - /// Replays the copy into fresh storage of `layout`, the layout it was - /// copied out under, re-parking droppable payloads against `arena`; - /// `None` reports arena exhaustion. - pub fn replay<'e>(&self, layout: &Layout, arena: &'e crate::arena::Arena) -> Option> { - let mut value = RecordValue::zeroed(); - let dst = match layout.frame_bytes() { - 0 => value.as_mut_ptr(), - bytes => stack::push(bytes), - }; - let written = self.write_into(layout, dst, arena); - if layout.frame_bytes() != 0 { - stack::truncate_above(dst, layout.frame_bytes()); - value = RecordValue::spilled(unsafe { Rec::new(dst.cast_const()) }); - } - written.map(|()| value) - } - - /// [`Self::replay`] into a caller's claim rather than a fresh frame; the - /// claim's layout is the one the copy was taken at. - pub fn replay_into(&self, slot: &mut FrameClaim<'_>, arena: &crate::arena::Arena) -> Option<()> { + /// Replays the copy into a caller's claim, re-parking droppable payloads + /// against `arena`; the claim's layout is the one the copy was taken at, + /// and `None` reports arena exhaustion. + pub fn replay_into(&self, slot: &mut FrameClaim<'_, '_>, arena: &crate::arena::Arena) -> Option<()> { let layout = slot.layout; self.write_into(layout, slot.dst(), arena) } @@ -1925,20 +1897,18 @@ impl ServedRecord { } /// Polls `node` once and captures any served record: the record is -/// deep-copied at the node's own declared layout, then the record stack -/// rewinds to its entry state, so the result is owned and no stack slot -/// stays claimed. Assertion scaffolding for law tests; production consumers -/// read served records in place. -pub fn capture<'e, C, N>(node: &N, ctx: &C) -> GPoll +/// deep-copied at the node's own declared layout inside a frame scope, so the +/// result is owned and every claimed frame is free again. Assertion +/// scaffolding for law tests; production consumers read served records in +/// place. +pub fn capture<'e, C, N>(node: &N, ctx: &C, frames: &Frames<'e>) -> GPoll where N: Node + ?Sized, C: crate::context::ExtractArena, { - // SAFETY: any served record is deep-copied out inside the scope, so - // nothing served above the entry escapes it. - let _scope = unsafe { stack::ScopeGuard::enter() }; + let scope = frames.scope(); let layout = node.layout().clone(); - serve_edge(node, ctx).map(|value| ServedRecord { + serve_edge(node, ctx, &scope).map(|value| ServedRecord { // SAFETY: the poll served `value` at the node's declared layout and // nothing has claimed frames since. record: unsafe { OwnedRecord::copy_out(&layout, layout.rec(&value)) }, @@ -1974,7 +1944,7 @@ where El: Send + Sync + dyn_any::StaticTypeSized, F: Fn(&C) -> GPoll, { - fn serve<'e, 'l>(&self, input: &C, slot: FrameClaim<'l>) -> GPoll> + fn serve<'e, 'l>(&self, input: &C, slot: FrameClaim<'e, 'l>) -> GPoll> where C: crate::context::ExtractArena, { @@ -2008,15 +1978,15 @@ impl RecordExtract { impl RecordExtract { /// The edge's element, copied out of its record. - pub fn eval<'e, C>(&self, input: &C) -> GPoll + pub fn eval<'e, C>(&self, input: &C, frames: &Frames<'e>) -> GPoll where N: Node, C: crate::context::ExtractArena, { - // SAFETY: the element copies out by value, so no record above the - // entry (the edge's frame) is live past the scope. - let _scope = unsafe { stack::ScopeGuard::enter() }; - serve_edge(&self.edge, input).map(|value| unsafe { read_element::(self.layout.rec(&value)) }) + // The element copies out by value, so the edge's claim dies with + // the scope. + let scope = frames.scope(); + serve_edge(&self.edge, input, &scope).map(|value| unsafe { read_element::(self.layout.rec(&value)) }) } } @@ -2024,16 +1994,16 @@ impl Node for RecordSource where N: Node, { - fn serve<'e, 'l>(&self, input: &C, mut slot: FrameClaim<'l>) -> GPoll> + fn serve<'e, 'l>(&self, input: &C, mut slot: FrameClaim<'e, 'l>) -> GPoll> where C: crate::context::ExtractArena, { - // The source's frame is claimed above this one and dies with the - // claim's drop; the translated union record stays. + // The source's frame is claimed beyond this one and dies with the + // claim; the translated union record stays. let Some(plan) = &self.plan else { return self.edge.serve(input, slot); }; - match serve_edge(&self.edge, input) { + match serve_edge(&self.edge, input, &mut slot.frames().reborrow()) { GPoll::Final(value) => { // SAFETY: the value came from this edge, so it carries the // plan's source layout. @@ -2059,11 +2029,11 @@ where } } - fn extent_at<'x>(&self, input: &C, level: u8) -> GPoll + fn extent_at<'x>(&self, input: &C, level: u8, frames: &Frames<'x>) -> GPoll where C: crate::context::ExtractArena, { - self.edge.extent_at(input, level) + self.edge.extent_at(input, level, frames) } fn layout(&self) -> &Layout { @@ -2844,8 +2814,13 @@ mod tests { buffer.fill(u64::MAX); let replay_arena = crate::arena::Arena::new(1024).unwrap(); - // SAFETY: between evaluations, nothing served on the stack is live. - unsafe { stack::reserve(layout.frame_bytes()); } let value = copy.replay(&layout, &replay_arena).unwrap(); + let mut frame_arena = FrameArena::new(); + frame_arena.reserve(layout.frame_bytes()); + let frames = frame_arena.frames(); + let mut slot = frames.claim(&layout); + copy.replay_into(&mut slot, &replay_arena).unwrap(); + // SAFETY: the replay completes the record in the claimed frame. + let value = unsafe { slot.finish() }; let rec = layout.rec(&value); assert_eq!(unsafe { read_element::(rec) }, "element"); assert_eq!(unsafe { rec.read::<&str>(layout.offset_of("name", 0).unwrap()) }, "field"); @@ -2981,47 +2956,8 @@ mod tests { } #[test] - fn stack_frames_nest_and_release() { - // SAFETY: between evaluations, nothing served on the stack is live. - unsafe { stack::reserve(64); } let outer = stack::push(24); - let inner = stack::push(8); - assert_eq!(inner as usize - outer as usize, 24); - stack::pop(outer); - assert_eq!(stack::push(8), outer); - stack::pop(outer); - } - - #[test] - fn stack_rounds_frames_to_word_alignment() { - // SAFETY: between evaluations, nothing served on the stack is live. - unsafe { stack::reserve(64); } let first = stack::push(21); - let second = stack::push(8); - assert_eq!(second as usize - first as usize, 24); - stack::pop(first); - } - - #[test] - fn each_thread_gets_its_own_stack() { - // SAFETY: between evaluations, nothing served on the stack is live. - unsafe { stack::reserve(64); } let here = stack::push(8); - let here_address = here as usize; - std::thread::scope(|scope| { - scope - .spawn(move || { - // SAFETY: between evaluations, nothing served on the stack is live. - unsafe { stack::reserve(64); } let there = stack::push(8); - assert_ne!(here_address, there as usize, "stacks are per thread"); - stack::pop(there); - }) - .join() - .unwrap(); - }); - stack::pop(here); - } - - #[test] - fn a_captured_frame_serves_its_writes_and_defaults() { - use crate::attribute::{Opacity, Transform}; + fn a_captured_frame_serves_its_writes() { + use crate::attribute::{Attribute, Transform}; use glam::{DAffine2, DVec2}; struct Fixture { @@ -3029,16 +2965,18 @@ mod tests { } impl Node for Fixture { - fn serve<'e, 'l>(&self, input: &C, slot: FrameClaim<'l>) -> GPoll> + fn serve<'e, 'l>(&self, input: &C, mut slot: FrameClaim<'e, 'l>) -> GPoll> where C: crate::context::ExtractArena, { - let mut frame = FrameBuilder::new(&self.layout, crate::context::ExtractArena::arena(input)); - frame.element(String::from("parked")); - frame.attr::(DAffine2::from_translation(DVec2::new(3., 4.))); - let Some(value) = frame.finish() else { return GPoll::error("arena exhausted") }; - // SAFETY: the builder served a record of this node's layout. - GPoll::Final(unsafe { slot.forward(&value) }) + let offset = self.layout.offset_of(Transform::NAME, 0).expect("the fixture's layout carries the transform"); + if slot.element(String::from("parked"), crate::context::ExtractArena::arena(input)).is_none() { + return GPoll::error("arena exhausted"); + } + // SAFETY: the offset is this layout's own, at the marker's value type. + unsafe { slot.attr_at(offset, DAffine2::from_translation(DVec2::new(3., 4.))) }; + // SAFETY: the writes above complete the record. + GPoll::Final(unsafe { slot.finish_served() }) } fn layout(&self) -> &Layout { @@ -3046,31 +2984,63 @@ mod tests { } } - let layout = Layout::default().with_writes(0, element_write::(), &[FieldWrite::of::(0), FieldWrite::of::(0)]); - // SAFETY: between evaluations, nothing served on the stack is live. - unsafe { - stack::reserve(1 << 10); - } + let layout = Layout::default().with_writes(0, element_write::(), &[FieldWrite::of::(0)]); + let mut frame_arena = FrameArena::new(); + frame_arena.reserve(1 << 10); + let frames = frame_arena.frames(); let arena = crate::arena::Arena::new(1024).unwrap(); let generations = []; let scope = crate::context::EvalScope::new(None, None, None, &generations, &arena); let ctx = crate::context::ContextImpl::root(&scope); - let mark = stack::sp(); - let GPoll::Final(served) = capture(&Fixture { layout }, &ctx) else { + let free = frames.free_words(); + let GPoll::Final(served) = capture(&Fixture { layout }, &ctx, &frames) else { panic!("the fixture serves finally"); }; - assert_eq!(stack::sp(), mark, "capture returns every claimed slot"); + assert_eq!(frames.free_words(), free, "capture returns every claimed slot"); assert_eq!(served.element::(), "parked"); assert_eq!(served.attr::(), DAffine2::from_translation(DVec2::new(3., 4.))); - assert_eq!(served.attr::(), 1., "an unwritten field serves its census default"); } #[test] - #[should_panic(expected = "the served element must match the layout's element type")] + #[should_panic(expected = "must match the wired layout")] fn a_mistyped_element_is_rejected_at_the_write() { - // SAFETY: between evaluations, nothing served on the stack is live. - unsafe { stack::reserve(1 << 10); } let arena = crate::arena::Arena::new(256).unwrap(); + let arena = crate::arena::Arena::new(256).unwrap(); let layout = Layout::default().with_writes(0, element_write::(), &[]); - FrameBuilder::new(&layout, &arena).element(1u32); + let mut frame_arena = FrameArena::new(); + frame_arena.reserve(1 << 10); + let frames = frame_arena.frames(); + frames.claim(&layout).element(1u32, &arena); + } + + #[test] + fn a_scope_releases_and_reuses_its_frames() { + let layout = Layout::default().with_writes(0, element_write::(), &[f64_field("opacity")]); + let mut frame_arena = FrameArena::new(); + frame_arena.reserve(1 << 10); + let frames = frame_arena.frames(); + let free = frames.free_words(); + let mut addresses = Vec::new(); + for _ in 0..3 { + let scope = frames.scope(); + let mut claim = scope.claim(&layout); + addresses.push(claim.dst() as usize); + drop(claim); + drop(scope); + assert_eq!(frames.free_words(), free, "the scope returns its claims"); + } + assert!(addresses.windows(2).all(|pair| pair[0] == pair[1]), "each claim reuses the same region"); + } + + #[test] + fn a_claim_shortens_onto_a_derived_lifetime() { + fn shorten<'long: 'short, 'short, 'l>(claim: FrameClaim<'long, 'l>) -> FrameClaim<'short, 'l> { + claim_shortens(claim) + } + let layout = Layout::default().with_writes(0, element_write::(), &[]); + let mut frame_arena = FrameArena::new(); + frame_arena.reserve(64); + let frames = frame_arena.frames(); + let claim = shorten(frames.claim(&layout)); + assert_eq!(claim.layout, &layout); } } diff --git a/node-graph/libraries/core-types/src/registry.rs b/node-graph/libraries/core-types/src/registry.rs index 96f7c6f9f0..5a28135e27 100644 --- a/node-graph/libraries/core-types/src/registry.rs +++ b/node-graph/libraries/core-types/src/registry.rs @@ -136,62 +136,24 @@ impl Node for SharedEdge where N: Node + ?Sized, { - fn serve<'e, 'l>(&self, input: &Input, slot: crate::record::FrameClaim<'l>) -> crate::gpoll::GPoll> + /// A node takes exactly its own frame out of its caller's free space: the + /// caller minted the claim and kept the cursor, so the frame accounting is + /// structural here and asserted where a claim is split. + fn serve<'e, 'l>(&self, input: &Input, slot: crate::record::FrameClaim<'e, 'l>) -> crate::gpoll::GPoll> where Input: crate::context::ExtractArena, { - // Every node advances the record stack by exactly its own frame: the - // caller claimed it before the call, so serving must leave the stack - // where it found it. A mismatch is a leaked or over-released frame. - #[cfg(debug_assertions)] - let sp_before = crate::record::stack::sp(); - #[cfg(debug_assertions)] - let trace = { - static TRACE: std::sync::OnceLock = std::sync::OnceLock::new(); - *TRACE.get_or_init(|| std::env::var_os("GRAPHENE_SP_DEBUG").is_some()) - }; - #[cfg(debug_assertions)] - if trace { - eprintln!( - "sp> enter frame_bytes {} fields [{}] sp {}", - self.layout().frame_bytes(), - self.layout().fields.iter().map(|field| field.name.to_string()).collect::>().join(", "), - sp_before, - ); - } // SAFETY: `own` keeps the payload alive for `self`'s lifetime and Arc // payloads are address stable. - let result = unsafe { self.ptr.as_ref() }.serve(input, slot); - #[cfg(debug_assertions)] - if trace { - eprintln!("sp> exit frame_bytes {} sp {} -> {}", self.layout().frame_bytes(), sp_before, crate::record::stack::sp()); - } - #[cfg(debug_assertions)] - debug_assert_eq!( - crate::record::stack::sp(), - sp_before, - "{} left the record stack misaligned (frame_bytes {}, depth {}, fields [{}], poll {})", - std::any::type_name::(), - self.layout().frame_bytes(), - self.layout().depth, - self.layout().fields.iter().map(|field| field.name.to_string()).collect::>().join(", "), - match &result { - crate::gpoll::GPoll::Final(_) => "Final", - crate::gpoll::GPoll::Partial(_) => "Partial", - crate::gpoll::GPoll::Fallback(_) => "Fallback", - crate::gpoll::GPoll::Pending => "Pending", - crate::gpoll::GPoll::Error(_) => "Error", - }, - ); - result + unsafe { self.ptr.as_ref() }.serve(input, slot) } - fn extent_at<'x>(&self, input: &Input, level: u8) -> crate::gpoll::GPoll + fn extent_at<'x>(&self, input: &Input, level: u8, frames: &crate::record::Frames<'x>) -> crate::gpoll::GPoll where Input: crate::context::ExtractArena, { // SAFETY: as in serve. - unsafe { self.ptr.as_ref() }.extent_at(input, level) + unsafe { self.ptr.as_ref() }.extent_at(input, level, frames) } fn serialize(&self) -> Option> { @@ -204,12 +166,18 @@ where unsafe { self.ptr.as_ref() }.layout() } - fn eval_batch<'a, 'x>(&'a self, input: &'a Input, range: std::ops::Range, scratch: Option<&'a mut [std::mem::MaybeUninit]>) -> crate::node::BatchStatus<'a> + fn eval_batch<'a, 'x>( + &'a self, + input: &'a Input, + range: std::ops::Range, + scratch: Option<&'a mut [std::mem::MaybeUninit]>, + frames: &crate::record::Frames<'x>, + ) -> crate::node::BatchStatus<'a> where Input: crate::context::InjectIndex + Copy + crate::context::ExtractArena, { // SAFETY: as in serve. - unsafe { self.ptr.as_ref() }.eval_batch(input, range, scratch) + unsafe { self.ptr.as_ref() }.eval_batch(input, range, scratch, frames) } } @@ -350,7 +318,7 @@ mod tests { use std::sync::Arc; use std::sync::atomic::{AtomicU32, Ordering}; - use crate::record::{FrameClaim, Layout, LiftedSource, Served, element_write, read_element, serve_edge, stack}; + use crate::record::{FrameClaim, Layout, LiftedSource, Served, element_write, read_element, serve_edge}; fn counting() -> LiftedSource Fn(&ContextImpl<'c>) -> GPoll> { let count = AtomicU32::new(0); @@ -378,7 +346,7 @@ mod tests { } impl Node for LendNode { - fn serve<'e, 'l>(&self, input: &Input, slot: FrameClaim<'l>) -> GPoll> + fn serve<'e, 'l>(&self, input: &Input, slot: FrameClaim<'e, 'l>) -> GPoll> where Input: ExtractArena, { @@ -399,10 +367,7 @@ mod tests { let generations = []; let scope = scope_fixture(&generations, &arena); let ctx = ContextImpl::root(&scope); - // SAFETY: between evaluations, nothing served on the stack is live. - unsafe { - stack::reserve(1 << 12); - } + let frames = crate::record::test_frames(1 << 12); let node = LendNode::new("held"); let layout = Node::::layout(&node).clone(); @@ -410,7 +375,7 @@ mod tests { assert_eq!(*handle.ty(), concrete!(String)); let wired = handle.downcast_erased::(concrete!(String)).unwrap(); - let GPoll::Final(value) = serve_edge(&wired, &ctx) else { + let GPoll::Final(value) = serve_edge(&wired, &ctx, &frames) else { panic!("borrow-carrying output must serve through the erased edge"); }; // SAFETY: the record was served at `layout`, whose element is the borrow. @@ -450,7 +415,7 @@ mod tests { Vec: Send + Sync + dyn_any::StaticTypeSized, Node0: for<'x> crate::record::DerivedRecordEdge<'x, crate::context::Derived<'x, C>>, { - fn serve<'e, 'l>(&self, input: &C, slot: FrameClaim<'l>) -> GPoll> + fn serve<'e, 'l>(&self, input: &C, mut slot: FrameClaim<'e, 'l>) -> GPoll> where C: ExtractArena, { @@ -458,11 +423,11 @@ mod tests { let spilled = input.index_head(); let mut result = Vec::new(); for index in 0..3 { - // SAFETY: the element copies out by value, so the content's - // frame is dead by the time the scope releases it. - let _scope = unsafe { stack::ScopeGuard::enter() }; + // The element copies out by value, so the content's frame is + // dead when the scope ends. + let scope = slot.frames().scope(); let derived = input.promoted(&spilled, index); - match self.content.eval_derived(&cell, 0, &derived) { + match self.content.eval_derived(&cell, 0, &derived, &scope) { // SAFETY: the content served at its own layout, whose // element is `T`. Ok(value) => result.push(unsafe { read_element::(self.inner.rec(&value)) }), @@ -491,12 +456,9 @@ mod tests { let nested = RepeatNode::<_, Vec>>::new(inner, inner_layout); let layout = Node::::layout(&nested).clone(); let erased: Box = Box::new(nested); - // SAFETY: between evaluations, nothing served on the stack is live. - unsafe { - stack::reserve(1 << 12); - } + let frames = crate::record::test_frames(1 << 12); - let GPoll::Final(value) = serve_edge(&*erased, &ctx) else { + let GPoll::Final(value) = serve_edge(&*erased, &ctx, &frames) else { panic!("nested repeat must evaluate"); }; // SAFETY: the record was served at `layout`, whose element is the output. @@ -529,7 +491,7 @@ mod tests { C: Ctx + crate::context::DeriveCtx + crate::context::ExtractFootprint, Node0: for<'x> crate::record::DerivedRecordEdge<'x, crate::context::Derived<'x, C>>, { - fn serve<'e, 'l>(&self, input: &C, slot: FrameClaim<'l>) -> GPoll> + fn serve<'e, 'l>(&self, input: &C, mut slot: FrameClaim<'e, 'l>) -> GPoll> where C: ExtractArena, { @@ -537,12 +499,12 @@ mod tests { let cell = crate::node::StatusCell::new(); let mut footprint = input.try_footprint().copied().unwrap_or(Footprint::DEFAULT); footprint.resolution.x += 7; - // SAFETY: the element copies out by value, so the content's frame is - // dead by the time the scope releases it. + // The element copies out by value, so the content's frame is dead + // when the scope ends. let value = { - let _scope = unsafe { stack::ScopeGuard::enter() }; + let scope = slot.frames().scope(); let derived = input.with_footprint(&footprint); - match self.content.eval_derived(&cell, 0, &derived) { + match self.content.eval_derived(&cell, 0, &derived, &scope) { // SAFETY: the content served at its own layout, whose // element is the resolution. Ok(value) => unsafe { read_element::(self.inner.rec(&value)) }, @@ -566,10 +528,7 @@ mod tests { let generations = []; let scope = scope_fixture(&generations, &arena); let ctx = ContextImpl::root(&scope); - // SAFETY: between evaluations, nothing served on the stack is live. - unsafe { - stack::reserve(1 << 12); - } + let frames = crate::record::test_frames(1 << 12); let resolution = LiftedSource::::new(|input: &ContextImpl| GPoll::Final(input.try_footprint().map(|footprint| footprint.resolution.x).unwrap_or(0))); let resolution_layout = Node::::layout(&resolution).clone(); @@ -578,7 +537,7 @@ mod tests { let graph = ShiftFootprintNode::new(shifted, shifted_layout); let layout = Node::::layout(&graph).clone(); - let GPoll::Final(value) = serve_edge(&graph, &ctx) else { + let GPoll::Final(value) = serve_edge(&graph, &ctx, &frames) else { panic!("the footprint shift must reach the content"); }; // SAFETY: the record was served at `layout`, whose element is the resolution. @@ -626,19 +585,16 @@ mod tests { let handle = EdgeHandle::new_record::(Arc::new(counting) as Arc); let duplicate = handle.duplicate(); assert_eq!(*duplicate.ty(), record_edge_type::()); - // SAFETY: between evaluations, nothing served on the stack is live. - unsafe { - stack::reserve(1 << 12); - } + let frames = crate::record::test_frames(1 << 12); let first = handle.downcast_record::().unwrap(); let second = duplicate.downcast_record::().unwrap(); // SAFETY: each record was served at `layout`, whose element is the count. let count = |value| unsafe { layout.rec(&value).element::() }; - assert_eq!(serve_edge(&first, &ctx).map(count), GPoll::Final(1)); - assert_eq!(serve_edge(&second, &ctx).map(count), GPoll::Final(2)); + assert_eq!(serve_edge(&first, &ctx, &frames).map(count), GPoll::Final(1)); + assert_eq!(serve_edge(&second, &ctx, &frames).map(count), GPoll::Final(2)); drop(first); - assert_eq!(serve_edge(&second, &ctx).map(count), GPoll::Final(3)); + assert_eq!(serve_edge(&second, &ctx, &frames).map(count), GPoll::Final(3)); } } diff --git a/node-graph/libraries/core-types/src/runtime.rs b/node-graph/libraries/core-types/src/runtime.rs index c0ca0ed509..77e0d65f92 100644 --- a/node-graph/libraries/core-types/src/runtime.rs +++ b/node-graph/libraries/core-types/src/runtime.rs @@ -171,7 +171,7 @@ mod tests { use crate::context::{ContextImpl, Ctx, EvalScope, ExtractFootprint, ExtractVarArgs, VarArgLink, VarArgSlots}; use crate::gpoll::GPoll; use crate::node::Node; - use crate::record::{Layout, LiftedSource, RecordExtract, element_write, stack}; + use crate::record::{Layout, LiftedSource, RecordExtract, element_write}; use crate::transform::Footprint; use std::sync::Mutex; use std::sync::atomic::{AtomicU32, Ordering}; @@ -266,10 +266,6 @@ mod tests { where El::Static: Clone + Send + Sync, { - // SAFETY: between evaluations, nothing served on the stack is live. - unsafe { - stack::reserve(1 << 12); - } let layout = element_layout::(); graph.set_layout(crate::record::RecordLayout { frame_bytes: layout.frame_bytes(), @@ -339,6 +335,7 @@ mod tests { #[test] fn async_source_spawns_once_and_lands_via_the_slot() { + let frames = crate::record::test_frames(1 << 16); let arena = Arena::new(64).unwrap(); let generations = []; let scope = scope_fixture(&generations, &arena); @@ -354,18 +351,19 @@ mod tests { &element_layout::(), )); - assert_eq!(graph.eval(&ctx), GPoll::Pending); - assert_eq!(graph.eval(&ctx), GPoll::Pending); + assert_eq!(graph.eval(&ctx, &frames), GPoll::Pending); + assert_eq!(graph.eval(&ctx, &frames), GPoll::Pending); assert_eq!(SLOW_DOUBLE_RUNS.load(Ordering::Relaxed), 0); assert_eq!(runtime.drain(), vec![7]); assert_eq!(SLOW_DOUBLE_RUNS.load(Ordering::Relaxed), 1); - assert_eq!(graph.eval(&ctx), GPoll::Final(42.0)); - assert_eq!(graph.eval(&ctx), GPoll::Final(42.0)); + assert_eq!(graph.eval(&ctx, &frames), GPoll::Final(42.0)); + assert_eq!(graph.eval(&ctx, &frames), GPoll::Final(42.0)); assert_eq!(SLOW_DOUBLE_RUNS.load(Ordering::Relaxed), 1); } #[test] fn async_source_reports_the_placeholder_while_in_flight() { + let frames = crate::record::test_frames(1 << 16); let arena = Arena::new(64).unwrap(); let generations = []; let scope = scope_fixture(&generations, &arena); @@ -381,13 +379,14 @@ mod tests { &element_layout::(), )); - assert_eq!(graph.eval(&ctx), GPoll::Partial(-1.0)); + assert_eq!(graph.eval(&ctx, &frames), GPoll::Partial(-1.0)); runtime.drain(); - assert_eq!(graph.eval(&ctx), GPoll::Final(42.0)); + assert_eq!(graph.eval(&ctx, &frames), GPoll::Final(42.0)); } #[test] fn no_partial_maps_the_placeholder_frame_to_pending() { + let frames = crate::record::test_frames(1 << 16); let arena = Arena::new(64).unwrap(); let generations = []; let scope = scope_fixture(&generations, &arena); @@ -403,13 +402,14 @@ mod tests { &element_layout::(), )); - assert_eq!(graph.eval(&ctx), GPoll::Pending); + assert_eq!(graph.eval(&ctx, &frames), GPoll::Pending); runtime.drain(); - assert_eq!(graph.eval(&ctx), GPoll::Final(42.0)); + assert_eq!(graph.eval(&ctx, &frames), GPoll::Final(42.0)); } #[test] fn prologue_runs_sync_and_spawns_once() { + let frames = crate::record::test_frames(1 << 16); let arena = Arena::new(64).unwrap(); let generations = []; let scope = scope_fixture(&generations, &arena); @@ -425,17 +425,18 @@ mod tests { &element_layout::(), )); - assert_eq!(graph.eval(&ctx), GPoll::Pending); + assert_eq!(graph.eval(&ctx, &frames), GPoll::Pending); assert_eq!(STAGED_RUNS.load(Ordering::Relaxed), 1, "the prologue runs synchronously on the miss"); - assert_eq!(graph.eval(&ctx), GPoll::Pending); + assert_eq!(graph.eval(&ctx, &frames), GPoll::Pending); assert_eq!(STAGED_RUNS.load(Ordering::Relaxed), 1, "in flight must not rerun the prologue"); assert_eq!(runtime.drain(), vec![8]); - assert_eq!(graph.eval(&ctx), GPoll::Final(42.0)); + assert_eq!(graph.eval(&ctx, &frames), GPoll::Final(42.0)); assert_eq!(STAGED_RUNS.load(Ordering::Relaxed), 1); } #[test] fn prologue_interrupt_defers_the_spawn() { + let frames = crate::record::test_frames(1 << 16); let arena = Arena::new(64).unwrap(); let generations = []; let scope = scope_fixture(&generations, &arena); @@ -454,16 +455,17 @@ mod tests { &element_layout::(), )); - assert_eq!(graph.eval(&ctx), GPoll::Pending); + assert_eq!(graph.eval(&ctx, &frames), GPoll::Pending); assert_eq!(runtime.drain(), Vec::::new(), "an interrupted prologue must not spawn or claim the slot"); gate.store(true, Ordering::Relaxed); - assert_eq!(graph.eval(&ctx), GPoll::Pending); + assert_eq!(graph.eval(&ctx, &frames), GPoll::Pending); assert_eq!(runtime.drain(), vec![9]); - assert_eq!(graph.eval(&ctx), GPoll::Final(42.0)); + assert_eq!(graph.eval(&ctx, &frames), GPoll::Final(42.0)); } #[test] fn async_kernels_read_captured_varargs() { + let frames = crate::record::test_frames(1 << 16); let arena = Arena::new(64).unwrap(); let generations = []; let scope = scope_fixture(&generations, &arena); @@ -485,13 +487,14 @@ mod tests { &element_layout::(), )); - assert_eq!(graph.eval(&ctx), GPoll::Pending); + assert_eq!(graph.eval(&ctx, &frames), GPoll::Pending); runtime.drain(); - assert_eq!(graph.eval(&ctx), GPoll::Final(21.5)); + assert_eq!(graph.eval(&ctx, &frames), GPoll::Final(21.5)); } #[test] fn async_kernels_read_the_captured_context_snapshot() { + let frames = crate::record::test_frames(1 << 16); let arena = Arena::new(64).unwrap(); let generations = []; let scope = scope_fixture(&generations, &arena); @@ -509,9 +512,9 @@ mod tests { &element_layout::(), )); - assert_eq!(graph.eval(&ctx), GPoll::Pending); + assert_eq!(graph.eval(&ctx, &frames), GPoll::Pending); runtime.drain(); - assert_eq!(graph.eval(&ctx), GPoll::Final(Footprint::DEFAULT.resolution.x)); + assert_eq!(graph.eval(&ctx, &frames), GPoll::Final(Footprint::DEFAULT.resolution.x)); } #[test] @@ -610,6 +613,7 @@ mod tests { #[test] fn an_immediately_ready_kernel_returns_final_on_the_first_eval() { + let frames = crate::record::test_frames(1 << 16); let arena = Arena::new(64).unwrap(); let runtime = Arc::new(GraphRuntime::new(CollectSpawner::default())); runtime.retain_sources(&[13]); @@ -625,7 +629,7 @@ mod tests { let snapshot = runtime.snapshot(); let scope = EvalScope::new(None, None, None, &snapshot, &arena); let ctx = ContextImpl::root(&scope); - assert_eq!(graph.eval(&ctx), GPoll::Final(42.0)); + assert_eq!(graph.eval(&ctx, &frames), GPoll::Final(42.0)); assert!(!runtime.take_dirty()); assert_eq!(runtime.snapshot(), vec![(13, 0)]); assert_eq!(runtime.spawner().drain(), 0); @@ -633,6 +637,7 @@ mod tests { #[test] fn a_source_slot_lands_through_the_runtime_while_downstream_keys_invalidate() { + let frames = crate::record::test_frames(1 << 16); let arena = Arena::new(64).unwrap(); let runtime = Arc::new(GraphRuntime::new(CollectSpawner::default())); runtime.retain_sources(&[11]); @@ -648,7 +653,7 @@ mod tests { let snapshot = runtime.snapshot(); let scope = EvalScope::new(None, None, None, &snapshot, &arena); let ctx = ContextImpl::root(&scope); - assert_eq!(graph.eval(&ctx), GPoll::Pending); + assert_eq!(graph.eval(&ctx, &frames), GPoll::Pending); assert!(!runtime.take_dirty()); assert_eq!(runtime.spawner().drain(), 1); @@ -658,7 +663,7 @@ mod tests { let bumped_scope = EvalScope::new(None, None, None, &bumped, &arena); let bumped_ctx = ContextImpl::root(&bumped_scope); - assert_eq!(graph.eval(&bumped_ctx), GPoll::Final(42.0), "the own-generation-excluded key replays the landed slot"); + assert_eq!(graph.eval(&bumped_ctx, &frames), GPoll::Final(42.0), "the own-generation-excluded key replays the landed slot"); assert_eq!(runtime.spawner().drain(), 0, "a slot hit must not respawn"); let downstream_key = crate::registry::cache_key(&ContextImpl::root(&scope)); diff --git a/node-graph/libraries/core-types/src/value.rs b/node-graph/libraries/core-types/src/value.rs index da652d2629..4ce62f6369 100644 --- a/node-graph/libraries/core-types/src/value.rs +++ b/node-graph/libraries/core-types/src/value.rs @@ -21,7 +21,7 @@ impl crate::node::Node for ValueSource where T: Clone + Send + Sync + dyn_any::StaticTypeSized, { - fn serve<'e, 'l>(&self, input: &C, slot: crate::record::FrameClaim<'l>) -> crate::gpoll::GPoll> + fn serve<'e, 'l>(&self, input: &C, slot: crate::record::FrameClaim<'e, 'l>) -> crate::gpoll::GPoll> where C: crate::context::ExtractArena, { @@ -65,7 +65,7 @@ where C: crate::context::ExtractIndex, T: Clone + Send + Sync + dyn_any::StaticTypeSized, { - fn serve<'e, 'l>(&self, input: &C, slot: crate::record::FrameClaim<'l>) -> crate::gpoll::GPoll> + fn serve<'e, 'l>(&self, input: &C, slot: crate::record::FrameClaim<'e, 'l>) -> crate::gpoll::GPoll> where C: crate::context::ExtractArena, { @@ -75,7 +75,7 @@ where slot.lift_served(crate::gpoll::GPoll::Final(value.clone()), input.arena()) } - fn extent_at<'x>(&self, _input: &C, level: u8) -> crate::gpoll::GPoll + fn extent_at<'x>(&self, _input: &C, level: u8, _frames: &crate::record::Frames<'x>) -> crate::gpoll::GPoll where C: crate::context::ExtractArena, { diff --git a/node-graph/libraries/graphic-types/src/boundary.rs b/node-graph/libraries/graphic-types/src/boundary.rs index 5d618dcaba..fc58abed1f 100644 --- a/node-graph/libraries/graphic-types/src/boundary.rs +++ b/node-graph/libraries/graphic-types/src/boundary.rs @@ -24,12 +24,12 @@ pub enum LevelGroup<'e> { /// The renderer's flip form: the wire's whole extent materialized into a /// group over the level's records, ready for the group render bridge. -pub fn materialize_group<'a, 'e, C, N>(node: &'a N, input: &'a C, arena: &'a Arena) -> LevelGroup<'a> +pub fn materialize_group<'a, 'e, C, N>(node: &'a N, input: &'a C, arena: &'a Arena, frames: &core_types::record::Frames<'e>) -> LevelGroup<'a> where C: InjectIndex + Copy + core_types::context::ExtractArena, N: Node, { - match materialize_level(node, input, arena) { + match materialize_level(node, input, arena, frames) { LevelStatus::Batch(batch, finality) => { // SAFETY: a materialized batch's frames are arena-resident. let item = unsafe { GroupItem::from_resident(batch) }; diff --git a/node-graph/libraries/graphic-types/src/graphic.rs b/node-graph/libraries/graphic-types/src/graphic.rs index 0559bc9a0d..5e737ec920 100644 --- a/node-graph/libraries/graphic-types/src/graphic.rs +++ b/node-graph/libraries/graphic-types/src/graphic.rs @@ -1647,8 +1647,11 @@ mod run_tests { drop(source); let arena = core_types::arena::Arena::new(1 << 16).unwrap(); - // SAFETY: between evaluations, nothing served on the stack is live. - unsafe { core_types::record::stack::reserve(layout.frame_bytes()); } let value = owned.replay(&layout, &arena).expect("the arena holds the replay"); + let frames = core_types::record::test_frames(layout.frame_bytes()); + let mut slot = frames.claim(&layout); + owned.replay_into(&mut slot, &arena).expect("the arena holds the replay"); + // SAFETY: the replay completes the record in the claimed frame. + let value = unsafe { slot.finish() }; // SAFETY: the replay wrote a record of `layout`. let served = unsafe { layout.rec(&value).read::>>(offset) }.expect("the fill replays present"); assert_eq!(map_groups_to_legacy(served.element(0).unwrap()), expected); diff --git a/node-graph/node-macro/src/codegen.rs b/node-graph/node-macro/src/codegen.rs index 76332e241d..88e069cddf 100644 --- a/node-graph/node-macro/src/codegen.rs +++ b/node-graph/node-macro/src/codegen.rs @@ -21,6 +21,22 @@ use metadata::generate_node_input_references; static NODE_ID: AtomicU64 = AtomicU64::new(0); +/// Binds in evaluation order with the lazy wrappers last: a wrapper takes the +/// free space as it stands when it is built, so every input whose record the +/// kernel still holds must have claimed its frame by then. +fn lazy_last<'a>(binds: impl Iterator, lazy_entry: &TokenStream2) -> Vec { + let mut ordered: Vec<(bool, TokenStream2)> = binds.map(|(field, body)| (matches!(field.ty, ParsedFieldType::Node(_)), body)).collect(); + ordered.sort_by_key(|(lazy, _)| *lazy); + let mut declared = false; + ordered + .into_iter() + .map(|(lazy, body)| match lazy && !std::mem::replace(&mut declared, true) { + true => quote!(#lazy_entry #body), + false => body, + }) + .collect() +} + /// The regular inputs that materialize whole in the eval prologue, which is /// where the per-node batch cache slots attach. fn materialized_indices(regular_fields: &[&ParsedField], node: &ir::Node) -> Vec { @@ -271,7 +287,7 @@ pub(crate) fn generate_node_code(crate_ident: &CrateIdent, parsed: &ParsedNodeFn // per-lane evals share one materialization. record_state_fields.extend(materialized_indices(&struct_regular_fields, &node).into_iter().map(|index| { let slot = format_ident!("__mat_cache_{index}"); - quote!(pub(super) #slot: ::std::sync::Arc<::std::sync::Mutex<::core::option::Option<(u64, u64, usize, usize)>>>) + quote!(pub(super) #slot: ::std::sync::Arc<::std::sync::Mutex<::core::option::Option<(u64, #core_types::record::MaterializedSpan)>>>) })); let async_source = parsed.injects_async_source_fields(); @@ -866,6 +882,25 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn let wants_record_lifetime = routing_generic.is_some() || ((record_io || flip) && kernel_lazy); let ctx_declares_arena = ctx_param.is_some_and(|ctx_param| ctx_param.bounds.iter().any(extracts_arena)); let bind_record_arena = wants_record_lifetime && !ctx_declares_arena; + // The lifetime a lazy wrapper's frame space is named at: the kernel's own + // arena lifetime, since the records it hands back live in that space. + let declared_arena_lifetime = ctx_param.and_then(|ctx_param| { + ctx_param.bounds.iter().find_map(|bound| match bound { + TypeParamBound::Trait(trait_bound) if extracts_arena(bound) => match &trait_bound.path.segments.last().expect("checked by the predicate").arguments { + PathArguments::AngleBracketed(args) => args.args.iter().find_map(|arg| match arg { + GenericArgument::Lifetime(lifetime) => Some(lifetime.clone()), + _ => None, + }), + _ => None, + }, + _ => None, + }) + }); + let frames_lifetime = match (&declared_arena_lifetime, wants_record_lifetime) { + (Some(lifetime), _) => quote!(#lifetime), + (None, true) => quote!('__record), + (None, false) => quote!('_), + }; if bind_record_arena { ctx_bounds.push(quote!(#core_types::context::ExtractArena)); } @@ -1089,19 +1124,19 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn ParsedFieldType::Node(NodeParsedField { output_type, .. }) => { let source_generic = format_ident!("__Source{index}"); match (ir::lazy_binding(&node, index), raw_lazy) { - (LazyBinding::DeriveRouting, _) => quote!(#pat: #core_types::record::RecordLazyInput<'_, '__record, #source_generic>), + (LazyBinding::DeriveRouting, _) => quote!(#pat: #core_types::record::RecordLazyInput<'_, #frames_lifetime, #source_generic>), (LazyBinding::DeriveCarrier, _) => { let out = lazy_read_out(field, output_type); - quote!(#pat: #core_types::record::DerivedLazyInput<'_, '__record, #out, #source_generic>) + quote!(#pat: #core_types::record::DerivedLazyInput<'_, #frames_lifetime, #out, #source_generic>) } - (LazyBinding::OpaqueRecord, _) => quote!(#pat: &#core_types::record::RecordEdgeInput<'_, #source_generic>), + (LazyBinding::OpaqueRecord, _) => quote!(#pat: &#core_types::record::RecordEdgeInput<'_, #frames_lifetime, #source_generic>), (LazyBinding::Element, true) => { let out = lazy_read_out(field, output_type); - quote!(#pat: &#core_types::record::ElementEdge<'_, #out, #source_generic>) + quote!(#pat: &#core_types::record::ElementEdge<'_, #frames_lifetime, #out, #source_generic>) } (LazyBinding::Element, false) => { let out = lazy_read_out(field, output_type); - quote!(#pat: #core_types::record::ElementLazyInput<'_, #out, #source_generic>) + quote!(#pat: #core_types::record::ElementLazyInput<'_, #frames_lifetime, #out, #source_generic>) } (LazyBinding::Generic, true) => { let bound = lazy_bound(); @@ -1109,7 +1144,7 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn } (LazyBinding::Generic, false) => { let bound = lazy_bound(); - quote!(#pat: #core_types::node::LazyInput<'_, impl #bound>) + quote!(#pat: #core_types::node::LazyInput<'_, #frames_lifetime, impl #bound>) } } } @@ -1218,33 +1253,35 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn .collect::>() }; - // Every exit closes through the frame claim's drop, so no exit path - // reclaims by hand. - let interrupt_close = quote!(); let frame_entry = quote! { #[allow(unused_mut, unused_variables)] let mut __frame = __slot; }; - // The batch loop is not a serve, so each lane claims its own frame. + // The batch loop is not a serve: the lane's own frame is its region of the + // run's slab, so it serves in place. let lane_frame_entry = quote! { #[allow(unused_mut, unused_variables)] - let mut __frame = #core_types::record::FrameClaim::enter(__node_layout); + let mut __frame = __run.slot(__lane, &__lane_frames); }; - let bind_body = |index: usize, field: &ParsedField, batch_mode: bool| { + // A lazy edge claims beyond every input frame this node holds, and its + // cursor is shared, so the edges a kernel drives claim past each other. + let lazy_frames_entry = quote! { + let __lazy_frames = __frame.frames().reborrow(); + }; + let bind_body = |index: usize, field: &ParsedField, batch_mode: bool, frames: &TokenStream2| { let name = &field.pat_ident.ident; // The bind's failure exits return through the enclosing fn: `GPoll` in // `eval`, `BatchStatus` in the generated `eval_batch`. - let close = interrupt_close.clone(); let pending = match batch_mode { - false => quote!({ #close return #core_types::gpoll::GPoll::Pending; }), + false => quote!(return #core_types::gpoll::GPoll::Pending), true => quote!(return #core_types::node::BatchStatus::Pending), }; let fail = |error: TokenStream2| match batch_mode { - false => quote!({ #close return #core_types::gpoll::GPoll::Error(::std::boxed::Box::new(#error)); }), + false => quote!(return #core_types::gpoll::GPoll::Error(::std::boxed::Box::new(#error))), true => quote!(return #core_types::node::BatchStatus::Error(#error)), }; let interrupt_return = match batch_mode { - false => quote!({ #close return interrupt.into(); }), + false => quote!(return interrupt.into()), true => quote!(return interrupt.into()), }; match &field.ty { @@ -1273,19 +1310,14 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn #core_types::context::InjectIndex::set_index(&mut __keyed, 0); #core_types::registry::cache_key(&__keyed) }; - let __mat_generation = __arena.generation(); let __mat_hit = match *self.#cache_slot.lock().unwrap() { - ::core::option::Option::Some((__key, __generation, __base, __len)) if __key == __mat_key && __generation == __mat_generation => { - ::core::option::Option::Some((__base, __len)) - } + ::core::option::Option::Some((__key, __span)) if __key == __mat_key => __span.batch(__arena, #core_types::node::Node::<#ctx_ident>::layout(&self.#name)), _ => ::core::option::Option::None, }; let __batch = match __mat_hit { - // SAFETY: within the generation the cached batch stays - // live, immutable, and of this edge's layout. - ::core::option::Option::Some((__base, __len)) => unsafe { #core_types::node::RecordBatch::new(__base as *const u8, __len, #core_types::node::Node::<#ctx_ident>::layout(&self.#name)) }, + ::core::option::Option::Some(__batch) => __batch, ::core::option::Option::None => { - let __sized = match #core_types::node::Node::extent(&self.#name, __input, #core_types::gpoll::Level::Total) { + let __sized = match #core_types::node::Node::extent(&self.#name, __input, #core_types::gpoll::Level::Total, #frames) { #core_types::gpoll::GPoll::Final(#core_types::gpoll::Extent::Exactly(__count)) => ::core::result::Result::Ok(__count), #core_types::gpoll::GPoll::Final(#core_types::gpoll::Extent::AtLeast(__bound)) => ::core::result::Result::Err(__bound), #core_types::gpoll::GPoll::Pending => #pending, @@ -1294,7 +1326,7 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn let __fresh = match __sized { ::core::result::Result::Ok(__count) => { let __start: u64 = 0; - match #core_types::record::materialize_batch(&self.#name, __input, __start..__start + __count as u64, __arena) { + match #core_types::record::materialize_batch(&self.#name, __input, __start..__start + __count as u64, __arena, #frames) { #core_types::node::BatchStatus::Lent(__batch, ..) => __batch, #core_types::node::BatchStatus::Filled(__batch, ..) => __batch.into_shared(), #core_types::node::BatchStatus::Pending => #pending, @@ -1308,7 +1340,7 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn ::core::result::Result::Err(__bound) => { let mut __guess = __bound.max(16); loop { - let (__batch, __hint) = match #core_types::record::materialize_batch(&self.#name, __input, 0..__guess as u64, __arena) { + let (__batch, __hint) = match #core_types::record::materialize_batch(&self.#name, __input, 0..__guess as u64, __arena, #frames) { #core_types::node::BatchStatus::Lent(__batch, _, __hint) => (__batch, __hint), #core_types::node::BatchStatus::Filled(__batch, _, __hint) => (__batch.into_shared(), __hint), #core_types::node::BatchStatus::Pending => #pending, @@ -1328,11 +1360,9 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn } } }; - let __base = match __fresh.len() { - 0 => 0usize, - _ => __fresh.get(0).rec().ptr() as usize, - }; - *self.#cache_slot.lock().unwrap() = ::core::option::Option::Some((__mat_key, __mat_generation, __base, __fresh.len())); + if let ::core::option::Option::Some(__span) = #core_types::record::MaterializedSpan::of(&__fresh, __arena) { + *self.#cache_slot.lock().unwrap() = ::core::option::Option::Some((__mat_key, __span)); + } __fresh } }; @@ -1340,14 +1370,13 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn } } // A reading secondary input claims a record edge: the element and - // the declared reads copy out right after its eval, before any - // later sibling eval can reuse the record stack. + // the declared reads copy out right after its eval. ValueBinding::ReadingSecondary => { let slot = format_ident!("__in_{index}"); let rec_local = format_ident!("__rec_{index}"); let bindings: Vec = reads_of(index).into_iter().map(|(slot, read)| read_binding(slot, read, quote!(#rec_local))).collect(); quote! { - let #name = match __cell.eval_input(#index, &self.#name, __input) { + let #name = match __cell.eval_input(#index, &self.#name, __input, #frames) { Ok(value) => value, Err(interrupt) => #interrupt_return, }; @@ -1356,13 +1385,13 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn let #name: #ty = unsafe { #core_types::record::read_element(#rec_local) }; } } - // The lend input's frame survives on the record stack until this - // node's frame is reclaimed, so the borrow stays valid in place. + // The lend input's frame is claimed out of this node's own claim and + // lives as long as it does, so the borrow stays valid in place. ValueBinding::Lend => { let slot = format_ident!("__in_{index}"); let record_local = format_ident!("__record_{index}"); quote! { - let #record_local = match __cell.eval_input(#index, &self.#name, __input) { + let #record_local = match __cell.eval_input(#index, &self.#name, __input, #frames) { Ok(value) => value, Err(interrupt) => #interrupt_return, }; @@ -1375,7 +1404,7 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn ValueBinding::RecordElement => { let slot = format_ident!("__in_{index}"); quote! { - let #name = match __cell.eval_input(#index, &self.#name, __input) { + let #name = match __cell.eval_input(#index, &self.#name, __input, #frames) { Ok(value) => value, Err(interrupt) => #interrupt_return, }; @@ -1394,7 +1423,7 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn } }); quote! { - let #name = match __cell.eval_input(#index, &self.#name, __input) { + let #name = match __cell.eval_input(#index, &self.#name, __input, #frames) { Ok(value) => value, Err(interrupt) => #interrupt_return, }; @@ -1406,20 +1435,20 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn // A raw poll edge is threaded straight through, so it does not bind here. (LazyBinding::Generic, true) => quote!(), (LazyBinding::DeriveRouting, _) => quote! { - let #name = #core_types::record::RecordLazyInput::new(&self.#name, &__cell, #index, self.__layout.depth.saturating_sub(#pushed_levels)); + let #name = #core_types::record::RecordLazyInput::new(&self.#name, &__cell, #index, self.__layout.depth.saturating_sub(#pushed_levels), &__lazy_frames); }, (LazyBinding::DeriveCarrier, _) => { let reads = reads_of(index); let read_fn = format_ident!("__{}_read_{}", fn_name, index); match reads.is_empty() { true => quote! { - let #name = #core_types::record::DerivedLazyInput::new(&self.#name, &__cell, #index, self.__layout.depth.saturating_sub(#pushed_levels), &[], #core_types::record::token_only); + let #name = #core_types::record::DerivedLazyInput::new(&self.#name, &__cell, #index, self.__layout.depth.saturating_sub(#pushed_levels), &[], #core_types::record::token_only, &__lazy_frames); }, false => { let slot_idents: Vec = reads.iter().map(|(slot, _)| format_ident!("__read_{slot}")).collect(); quote! { let __carrier_reads = [#(self.#slot_idents),*]; - let #name = #core_types::record::DerivedLazyInput::new(&self.#name, &__cell, #index, self.__layout.depth.saturating_sub(#pushed_levels), &__carrier_reads, self::#read_fn); + let #name = #core_types::record::DerivedLazyInput::new(&self.#name, &__cell, #index, self.__layout.depth.saturating_sub(#pushed_levels), &__carrier_reads, self::#read_fn, &__lazy_frames); } } } @@ -1428,13 +1457,13 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn let slot = format_ident!("__in_{index}"); match field.attribute_reads.is_empty() { true => quote! { - let #name = #core_types::record::ElementEdge::<#output_type, _>::new(&self.#name, &self.#slot); + let #name = #core_types::record::ElementEdge::<#output_type, _>::new(&self.#name, &self.#slot, &__lazy_frames); }, false => { let arr = format_ident!("__reads_{index}"); let read_fn = format_ident!("__{}_read_{}", fn_name, index); quote! { - let #name = #core_types::record::ElementEdge::with_reads(&self.#name, &self.#slot, &self.#arr, self::#read_fn); + let #name = #core_types::record::ElementEdge::with_reads(&self.#name, &self.#slot, &self.#arr, self::#read_fn, &__lazy_frames); } } } @@ -1443,22 +1472,22 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn let slot = format_ident!("__in_{index}"); match field.attribute_reads.is_empty() { true => quote! { - let #name = #core_types::record::ElementLazyInput::<#output_type, _>::new(&self.#name, &__cell, #index, &self.#slot); + let #name = #core_types::record::ElementLazyInput::<#output_type, _>::new(&self.#name, &__cell, #index, &self.#slot, &__lazy_frames); }, false => { let arr = format_ident!("__reads_{index}"); let read_fn = format_ident!("__{}_read_{}", fn_name, index); quote! { - let #name = #core_types::record::ElementLazyInput::with_reads(&self.#name, &__cell, #index, &self.#slot, &self.#arr, self::#read_fn); + let #name = #core_types::record::ElementLazyInput::with_reads(&self.#name, &__cell, #index, &self.#slot, &self.#arr, self::#read_fn, &__lazy_frames); } } } } (LazyBinding::OpaqueRecord, _) => quote! { - let #name = #core_types::record::RecordEdgeInput::new(&self.#name, &self.__layout); + let #name = #core_types::record::RecordEdgeInput::new(&self.#name, &self.__layout, &__lazy_frames); }, (LazyBinding::Generic, false) => quote! { - let #name = #core_types::node::LazyInput::new(&self.#name, &__cell, #index); + let #name = #core_types::node::LazyInput::new(&self.#name, &__cell, #index, &__lazy_frames); }, }, } @@ -1466,7 +1495,7 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn // A bind whose element copies out reclaims the edge's frame; a forwarded // record must outlive the bind, so its frame stays. - let reads_out_at = |index: usize| match ®ular_fields[index].ty { + let _reads_out_at = |index: usize| match ®ular_fields[index].ty { ParsedFieldType::Regular(RegularParsedField { ty, .. }) => !routing_source(ty) && ir::value_binding(&node, index).reads_out(), ParsedFieldType::Node(_) => false, }; @@ -1519,7 +1548,7 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn let query = format_ident!("__extent_query_{index}"); let extent_edge = |query: &Ident, arg: &Ident| { quote! { - let #query = |_: u64, __lvl: u8| #core_types::node::Node::extent_at(&self.#name, __input, __lvl); + let #query = |_: u64, __lvl: u8| #core_types::node::Node::extent_at(&self.#name, __input, __lvl, &__frames.scope()); let #arg = #core_types::extent::ExtentIn::new(&#query); } }; @@ -1529,7 +1558,7 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn let #query = |__copy: u64, __lvl: u8| { let mut __frame = #core_types::context::IndexLink { index: 0, outer: None }; let __derived = #core_types::context::DeriveCtx::push_level(__input, &mut __frame, __copy, 0); - #core_types::record::DerivedRecordEdge::extent_at_derived(&self.#name, &__derived, __lvl) + #core_types::record::DerivedRecordEdge::extent_at_derived(&self.#name, &__derived, __lvl, &__frames.scope()) }; let #arg = #core_types::extent::ExtentIn::new(&#query); }, @@ -1544,12 +1573,12 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn quote! { let #query = || { let __arena = #core_types::context::ExtractArena::arena(__input); - let __count = match #core_types::node::Node::extent(&self.#name, __input, #core_types::gpoll::Level::Total) { + let __count = match #core_types::node::Node::extent(&self.#name, __input, #core_types::gpoll::Level::Total, &__frames.scope()) { #core_types::gpoll::GPoll::Final(#core_types::gpoll::Extent::Exactly(__count)) => __count, #core_types::gpoll::GPoll::Pending => return #core_types::gpoll::GPoll::Pending, _ => return #core_types::gpoll::GPoll::Error(::std::boxed::Box::new(#core_types::gpoll::GraphError::new("extent over a non-exact ranked input"))), }; - match #core_types::record::materialize_batch(&self.#name, __input, 0..__count as u64, __arena) { + match #core_types::record::materialize_batch(&self.#name, __input, 0..__count as u64, __arena, __frames) { #core_types::node::BatchStatus::Lent(__batch, ..) => #core_types::gpoll::GPoll::Final(unsafe { #core_types::node::List::<#ty>::new(__batch) }), #core_types::node::BatchStatus::Filled(__batch, ..) => #core_types::gpoll::GPoll::Final(unsafe { #core_types::node::List::<#ty>::new(__batch.into_shared()) }), #core_types::node::BatchStatus::Pending => #core_types::gpoll::GPoll::Pending, @@ -1557,7 +1586,7 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn _ => #core_types::gpoll::GPoll::Error(::std::boxed::Box::new(#core_types::gpoll::GraphError::new("extent could not materialize a ranked input"))), } }; - let __total = || #core_types::node::Node::extent(&self.#name, __input, #core_types::gpoll::Level::Total); + let __total = || #core_types::node::Node::extent(&self.#name, __input, #core_types::gpoll::Level::Total, &__frames.scope()); let #arg = #core_types::extent::ListIn::new(&#query, &__total); } } @@ -1574,10 +1603,10 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn }; quote! { let #query = || { - // SAFETY: the element copies out by value; extent - // queries leave the record stack untouched. - let __scope = unsafe { #core_types::record::stack::ScopeGuard::enter() }; - #core_types::record::serve_edge(&self.#name, __input) + // The element copies out by value, so the edge's + // claim dies with the query. + let __scope = __frames.scope(); + #core_types::record::serve_edge(&self.#name, __input, &__scope) .map(|__value| unsafe { #core_types::record::read_element::<#ty>(#layout.rec(&__value)) }) }; let #arg = #core_types::extent::ValueIn::new(&#query); @@ -1592,7 +1621,7 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn arg_names.push(arg); } quote! { - fn extent_at<'__serve>(&self, __input: &#ctx_ident, __level: u8) -> #core_types::gpoll::GPoll<#core_types::gpoll::Extent> + fn extent_at<'__serve>(&self, __input: &#ctx_ident, __level: u8, __frames: &#core_types::record::Frames<'__serve>) -> #core_types::gpoll::GPoll<#core_types::gpoll::Extent> where #ctx_ident: #core_types::context::ExtractArena, { @@ -1603,7 +1632,7 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn } } else if let Some(path) = &parsed.attributes.extent_raw { quote! { - fn extent_at<'__serve>(&self, __input: &#ctx_ident, __level: u8) -> #core_types::gpoll::GPoll<#core_types::gpoll::Extent> + fn extent_at<'__serve>(&self, __input: &#ctx_ident, __level: u8, _: &#core_types::record::Frames<'__serve>) -> #core_types::gpoll::GPoll<#core_types::gpoll::Extent> where #ctx_ident: #core_types::context::ExtractArena, { @@ -1621,19 +1650,19 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn let __query = |_: u64, __lvl: u8| { let __head = #core_types::context::DeriveCtx::index_head(__input); let __derived = #core_types::context::DeriveCtx::replaced(__input, __head.index); - #core_types::record::DerivedRecordEdge::extent_at_derived(&self.#name, &__derived, __lvl) + #core_types::record::DerivedRecordEdge::extent_at_derived(&self.#name, &__derived, __lvl, &__frames.scope()) }; }, _ => quote! { - let __query = |_: u64, __lvl: u8| #core_types::node::Node::extent_at(&self.#name, __input, __lvl); + let __query = |_: u64, __lvl: u8| #core_types::node::Node::extent_at(&self.#name, __input, __lvl, &__frames.scope()); }, }, ParsedFieldType::Regular(_) => quote! { - let __query = |_: u64, __lvl: u8| #core_types::node::Node::extent_at(&self.#name, __input, __lvl); + let __query = |_: u64, __lvl: u8| #core_types::node::Node::extent_at(&self.#name, __input, __lvl, &__frames.scope()); }, }; quote! { - fn extent_at<'__serve>(&self, __input: &#ctx_ident, __level: u8) -> #core_types::gpoll::GPoll<#core_types::gpoll::Extent> + fn extent_at<'__serve>(&self, __input: &#ctx_ident, __level: u8, __frames: &#core_types::record::Frames<'__serve>) -> #core_types::gpoll::GPoll<#core_types::gpoll::Extent> where #ctx_ident: #core_types::context::ExtractArena, { @@ -1647,7 +1676,7 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn // A leveled output without an extent fn reports a lower bound; // consumers size it by draining to the past-end signal. quote! { - fn extent_at<'__serve>(&self, _: &#ctx_ident, _: u8) -> #core_types::gpoll::GPoll<#core_types::gpoll::Extent> + fn extent_at<'__serve>(&self, _: &#ctx_ident, _: u8, _: &#core_types::record::Frames<'__serve>) -> #core_types::gpoll::GPoll<#core_types::gpoll::Extent> where #ctx_ident: #core_types::context::ExtractArena, { @@ -1676,6 +1705,7 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn __input: &'__batch #ctx_ident, __range: ::std::ops::Range, __scratch: Option<&'__batch mut [::std::mem::MaybeUninit]>, + __frames: &#core_types::record::Frames<'__serve>, ) -> #core_types::node::BatchStatus<'__batch> where #ctx_ident: #core_types::context::InjectIndex + Copy + #core_types::context::ExtractArena, @@ -1758,7 +1788,7 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn quote! { match #kernel_call { Ok(value) => __cell.finish(#served), - Err(interrupt) => { #interrupt_close interrupt.into() } + Err(interrupt) => interrupt.into() } } } @@ -1793,7 +1823,7 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn }; let clamp = clamp_tokens(field); quote! { - let __src = match __cell.eval_input(0, &self.#name, __input) { + let __src = match __cell.eval_input(0, &self.#name, __input, __frame.frames()) { Ok(value) => value, Err(interrupt) => return interrupt.into(), }; @@ -1889,9 +1919,9 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn let carrier_eval = (!skips_carrier && !lazy_carrier).then(|| { let name = ®ular_fields[0].pat_ident.ident; quote! { - let __src = match __cell.eval_input(0, &self.#name, __input) { + let __src = match __cell.eval_input(0, &self.#name, __input, __frame.frames()) { Ok(value) => value, - Err(interrupt) => { #interrupt_close return interrupt.into(); } + Err(interrupt) => return interrupt.into() }; let __src_rec = self.__carrier.rec(&__src); } @@ -1922,7 +1952,7 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn Dialect::Interrupt => quote! { match #record_kernel_call { Ok(__value) => __value, - Err(__interrupt) => { #interrupt_close return __interrupt.into(); } + Err(__interrupt) => return __interrupt.into() } }, _ => quote!(#record_kernel_call), @@ -2062,7 +2092,7 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn Dialect::FutureInterrupt => quote! { let __future = match #kernel_call { Ok(future) => future, - Err(interrupt) => { #interrupt_close return interrupt.into(); } + Err(interrupt) => return interrupt.into() }; }, _ => quote!(let __future = #kernel_call;), @@ -2088,16 +2118,16 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn // base lane, so the per-lane loop runs only the kernel and the carrier; // eager inputs are batch-invariant by contract (per-lane variance rides // lazy carriers). - // The fill loop copies each lane's frame out, so it needs the record, not - // the serving proof. + // Each lane serves in place into its own region of the run, so the loop + // collects the serving proofs. let hoisted_lane_poll = match tail_form { Tail::Record => record_tail_core.clone().map(|core| { quote! { #core - let __poll = __cell.finish(__value).map(#core_types::record::Served::value); + let __poll = __cell.finish(__value); } }), - Tail::Forward if routing_generic.is_some() => Some(quote!(let __poll = #lift.map(#core_types::record::Served::value);)), + Tail::Forward if routing_generic.is_some() => Some(quote!(let __poll = #lift;)), _ => None, }; // A serving-lifetime element rides the per-lane fill loop: the hoisted @@ -2112,7 +2142,7 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn (Some(path), ..) => quote! { #batch_signature { - #path(self, __input, __range, __scratch) + #path(self, __input, __range, __scratch, __frames) } }, (None, true, Some(lane_poll)) => { @@ -2124,21 +2154,7 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn .iter() .enumerate() .filter(|(index, field)| matches!(field.ty, ParsedFieldType::Regular(_)) && hoists(*index)) - .map(|(index, field)| { - let body = bind_body(index, field, true); - match reads_out_at(index) { - false => body, - true => { - let mark = format_ident!("__scope_{index}"); - quote! { - // SAFETY: the bind's reads copy out by value. - let #mark = unsafe { #core_types::record::stack::ScopeGuard::enter() }; - #body - drop(#mark); - } - } - } - }) + .map(|(index, field)| bind_body(index, field, true, "e!((&*__frames)))) .collect(); let hoisted_clamps: Vec = regular_fields .iter() @@ -2146,34 +2162,38 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn .filter(|(index, field)| matches!(field.ty, ParsedFieldType::Regular(_)) && hoists(*index)) .filter_map(|(_, field)| clamp_tokens(field)) .collect(); - let lane_binds: Vec = regular_fields - .iter() - .enumerate() - .filter(|(index, field)| match field.ty { - ParsedFieldType::Node(_) => true, - ParsedFieldType::Regular(_) => !hoists(*index) && !matches!(ir::value_binding(&node, *index), ValueBinding::Carrier), - }) - .map(|(index, field)| { - let body = bind_body(index, field, true); - let clamp = clamp_tokens(field); - quote!(#body #clamp) - }) - .collect(); + let lane_binds: Vec = lazy_last( + regular_fields + .iter() + .enumerate() + .filter(|(index, field)| match field.ty { + ParsedFieldType::Node(_) => true, + ParsedFieldType::Regular(_) => !hoists(*index) && !matches!(ir::value_binding(&node, *index), ValueBinding::Carrier), + }) + .map(|(index, field)| { + let body = bind_body(index, field, true, "e!(__frame.frames())); + let clamp = clamp_tokens(field); + (field, quote!(#body #clamp)) + }), + &lazy_frames_entry, + ); // The rebind path with nothing hoisted: every non-carrier input binds // fresh per lane, so an index-dependent edge reaches its own lane. - let rebound_lane_binds: Vec = regular_fields - .iter() - .enumerate() - .filter(|(index, field)| match field.ty { - ParsedFieldType::Node(_) => true, - ParsedFieldType::Regular(_) => !matches!(ir::value_binding(&node, *index), ValueBinding::Carrier), - }) - .map(|(index, field)| { - let body = bind_body(index, field, true); - let clamp = clamp_tokens(field); - quote!(#body #clamp) - }) - .collect(); + let rebound_lane_binds: Vec = lazy_last( + regular_fields + .iter() + .enumerate() + .filter(|(index, field)| match field.ty { + ParsedFieldType::Node(_) => true, + ParsedFieldType::Regular(_) => !matches!(ir::value_binding(&node, *index), ValueBinding::Carrier), + }) + .map(|(index, field)| { + let body = bind_body(index, field, true, "e!(__frame.frames())); + let clamp = clamp_tokens(field); + (field, quote!(#body #clamp)) + }), + &lazy_frames_entry, + ); // A hoisted value is moved into every lane's kernel call, so each // lane consumes a clone; view and borrow binds copy freely. let lane_rebinds: Vec = regular_fields @@ -2203,23 +2223,24 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn quote! { #(#hoisted)* #(#clamps)* - let __frames = __scratch.as_mut_ptr().cast::(); + let ::core::option::Option::Some(mut __run) = __frames.run(__scratch, __len, __node_layout) else { + return #core_types::node::BatchStatus::InvalidRange; + }; let mut __finality = #core_types::gpoll::Finality::AllFinal; - let mut __filled = __len; let mut __hint = #core_types::gpoll::Extent::AtLeast(__range.end as usize); let mut __lane_ctx = __base_ctx; for __lane in 0..__len { #core_types::context::InjectIndex::set_index(&mut __lane_ctx, __range.start + __lane as u64); let __input = &__lane_ctx; - // SAFETY: the lane's record copies out before the scope - // releases it. - let __lane_scope = unsafe { #core_types::record::stack::ScopeGuard::enter() }; + // The lane's inputs claim beyond its slab region, and their + // space is free again at the next lane. + let __lane_frames = __frames.scope(); #lane_frame_entry let __cell = __cell.snapshot(); #(#rebinds)* #(#binds)* #lane_poll - let __value = match __poll { + let __served = match __poll { #core_types::gpoll::GPoll::Final(__value) => __value, #core_types::gpoll::GPoll::Partial(__value) => { __finality = #core_types::gpoll::Finality::Partial; @@ -2230,21 +2251,14 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn // A lane past a lower-bound level ends the data: the fill // comes back short and the hint turns exact. #core_types::gpoll::GPoll::Error(__error) if __error.kind == #core_types::gpoll::ErrorKind::PastEnd => { - __filled = __lane; __hint = #core_types::gpoll::Extent::Exactly(__range.start as usize + __lane); break; } #core_types::gpoll::GPoll::Error(__error) => return #core_types::node::BatchStatus::Error(*__error), }; - // SAFETY: the lane region is in-bounds by the scratch check, - // and the frame is fully copied out before the lane scope - // releases it. - unsafe { ::core::ptr::copy_nonoverlapping(__node_layout.rec(&__value).ptr(), __frames.add(__lane * __stride), __stride) }; + __run.served(__lane, &__served); } - drop(__entry_scope); - // SAFETY: the first `__filled` lanes were filled above with - // records of the node's layout. - #core_types::node::BatchStatus::Filled(unsafe { #core_types::node::RecordBatchMut::new(__scratch, __filled, __node_layout) }, __finality, __hint) + #core_types::node::BatchStatus::Filled(__run.finish(), __finality, __hint) } }; let hoisted_fill = fill_loop(hoisted_binds, hoisted_clamps, lane_rebinds, lane_binds); @@ -2275,12 +2289,9 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn return #core_types::node::BatchStatus::InvalidRange; }; let __node_layout = >::layout(self); - let __stride = __node_layout.lane_stride(); - if __scratch.len() * 8 < __len * __stride { - return #core_types::node::BatchStatus::InvalidRange; - } - // SAFETY: every lane copies into the caller's scratch. - let __entry_scope = unsafe { #core_types::record::stack::ScopeGuard::enter() }; + // The batch's own claims are free again when it returns, so + // the caller's free space comes back as it was lent. + let __frames = __frames.scope(); let __cell = #cell_constructor; let __base_ctx = { let mut __ctx = *__input; @@ -2297,7 +2308,7 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn (None, true, None) => quote! { #batch_signature { - #core_types::record::fill_frames(self, __input, __range, __scratch) + #core_types::record::fill_frames(self, __input, __range, __scratch, __frames) } }, (None, false, _) => quote!(), @@ -2580,12 +2591,17 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn }); // The eval body as an ordered step sequence: bind each input, clamp, then the - // tail. The record-stack mark/rewind of a read-out bind is applied here from - // the role, so the discipline is structural rather than per-arm. - let eval_steps: Vec = regular_fields + // tail. Every input's frame is claimed out of this node's own claim and + // stays claimed until it dies, which is the sizing the wiring layer derives. + let mut bind_order: Vec<(bool, usize, &&ParsedField)> = regular_fields .iter() .enumerate() - .map(|(index, field)| EvalStep::Bind(index, field)) + .map(|(index, field)| (matches!(field.ty, ParsedFieldType::Node(_)), index, field)) + .collect(); + bind_order.sort_by_key(|(lazy, ..)| *lazy); + let eval_steps: Vec = bind_order + .iter() + .map(|(_, index, field)| EvalStep::Bind(*index, field)) .chain( regular_fields .iter() @@ -2595,26 +2611,21 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn ) .chain(std::iter::once(EvalStep::Tail(tail_form))) .collect(); - let eval_body = eval_steps.iter().map(|step| match step { - EvalStep::Bind(index, field) => { - let body = bind_body(*index, field, false); - let reads_out = reads_out_at(*index); - match reads_out { - false => body, - true => { - let mark = format_ident!("__scope_{index}"); - quote! { - // SAFETY: the bind's reads copy out by value. - let #mark = unsafe { #core_types::record::stack::ScopeGuard::enter() }; - #body - drop(#mark); - } + let mut lazy_declared = false; + let eval_body: Vec = eval_steps + .iter() + .map(|step| match step { + EvalStep::Bind(index, field) => { + let body = bind_body(*index, field, false, "e!(__frame.frames())); + match matches!(field.ty, ParsedFieldType::Node(_)) && !std::mem::replace(&mut lazy_declared, true) { + true => quote!(#lazy_frames_entry #body), + false => body, } } - } - EvalStep::Clamp(field) => clamp_tokens(field).unwrap_or_default(), - EvalStep::Tail(form) => lower_tail(*form), - }); + EvalStep::Clamp(field) => clamp_tokens(field).unwrap_or_default(), + EvalStep::Tail(form) => lower_tail(*form), + }) + .collect(); let top_level = quote! { #cfg @@ -2629,29 +2640,10 @@ pub(crate) fn generate_node_impl(crate_ident: &CrateIdent, parsed: &ParsedNodeFn #(#flip_bounds,)* #(#where_predicates,)* { - fn serve<'__serve, '__slot>(&self, __input: &#ctx_ident, __slot: #core_types::record::FrameClaim<'__slot>) -> #core_types::gpoll::GPoll<#core_types::record::Served<'__serve>> + fn serve<'__serve, '__slot>(&self, __input: &#ctx_ident, __slot: #core_types::record::FrameClaim<'__serve, '__slot>) -> #core_types::gpoll::GPoll<#core_types::record::Served<'__serve>> where #ctx_ident: #core_types::context::ExtractArena, { - // The exit trace rides a guard so early returns report too, which - // is what pins a frame leak to its node. - #[cfg(debug_assertions)] - let __sp_trace = { - static __SP_TRACE: ::std::sync::OnceLock = ::std::sync::OnceLock::new(); - struct __SpTrace(&'static str, usize); - impl ::core::ops::Drop for __SpTrace { - fn drop(&mut self) { - ::std::eprintln!("node> {} exit sp {} -> {}", self.0, self.1, #core_types::record::stack::sp()); - } - } - match *__SP_TRACE.get_or_init(|| ::std::env::var_os("GRAPHENE_SP_DEBUG").is_some()) { - true => { - ::std::eprintln!("node> {} enter sp {}", ::std::stringify!(#fn_name), #core_types::record::stack::sp()); - Some(__SpTrace(::std::stringify!(#fn_name), #core_types::record::stack::sp())) - } - false => None, - } - }; #frame_entry let __cell = #cell_constructor; #(#eval_body)* diff --git a/node-graph/node-macro/src/codegen/ir.rs b/node-graph/node-macro/src/codegen/ir.rs index 08c491506c..0d784dccf5 100644 --- a/node-graph/node-macro/src/codegen/ir.rs +++ b/node-graph/node-macro/src/codegen/ir.rs @@ -771,7 +771,7 @@ mod tests { assert_bridge( quote!(category("")), quote! { - fn memo<'e, 'l>(_: impl Ctx, #[data] cache: Store, content: impl Node>, slot: FrameClaim<'l>) -> GPoll> { content.serve(&(), slot) } + fn memo<'e, 'l>(_: impl Ctx, #[data] cache: Store, content: impl Node>, slot: FrameClaim<'e, 'l>) -> GPoll> { content.serve(&(), slot) } }, ); } @@ -1013,7 +1013,7 @@ mod tests { assert_bindings( quote!(category("")), quote!( - fn memo<'e, 'l>(_: impl Ctx, #[data] cache: Store, content: impl Node>, slot: FrameClaim<'l>) -> GPoll> { + fn memo<'e, 'l>(_: impl Ctx, #[data] cache: Store, content: impl Node>, slot: FrameClaim<'e, 'l>) -> GPoll> { content.serve(&(), slot) } ), diff --git a/node-graph/nodes/gcore/src/memo.rs b/node-graph/nodes/gcore/src/memo.rs index 2d195de7cd..961e1cbcee 100644 --- a/node-graph/nodes/gcore/src/memo.rs +++ b/node-graph/nodes/gcore/src/memo.rs @@ -3,7 +3,7 @@ use core_types::context::{Ctx, CtxSnapshot, DeriveCtx, ExtractAll, ModifyIndex}; use core_types::frame_table::{FrameTable, Lookup}; use core_types::gpoll::{Finality, GPoll}; use core_types::graphene_hash::CacheHash; -use core_types::record::{FrameClaim, LevelStatus, OwnedRecord, Served, copy_record_bytes}; +use core_types::record::{FrameClaim, LevelStatus, MaterializedSpan, OwnedRecord, Served, copy_record_bytes}; use core_types::registry::cache_key; use std::sync::Arc; use std::sync::Mutex; @@ -14,9 +14,9 @@ use std::sync::Mutex; #[derive(Debug)] pub struct MemoLevel { key: u64, - generation: u64, - frames: usize, - stride: usize, + /// The arena region the level materialized into, resolvable only while its + /// generation is live. + span: Option, lanes: Vec, finality: Finality, } @@ -33,7 +33,7 @@ fn memoize<'e, 'l>( ctx: impl Ctx + CacheHash + DeriveCtx + ExtractArena<'e> + ModifyIndex + Copy, #[data] cache: Arc>>, content: impl Node>, - slot: FrameClaim<'l>, + slot: FrameClaim<'e, 'l>, ) -> GPoll> { // A scalar wire's value may depend on the consuming lane (index readers), // so only a leveled wire, whose level covers every lane by construction, @@ -57,15 +57,17 @@ fn memoize<'e, 'l>( }; // The claim is this node's output frame: a hit fills it from the cached // bytes, and every valueless exit drops it with the frame still claimed. - let serve = |entry: &MemoLevel, mut slot: FrameClaim<'l>| { + let serve = |entry: &MemoLevel, mut slot: FrameClaim<'e, 'l>| { if lane >= entry.lanes.len() { // The cached level ends here; the past-end signal serves drains. return GPoll::Error(Box::new(core_types::gpoll::GraphError::past_end())); } - if entry.generation == ctx.arena().generation() { - // SAFETY: within the generation the materialized batch stays live, - // immutable, and laid out at the recorded stride. - unsafe { slot.fill_copy((entry.frames + lane * entry.stride) as *const u8) }; + if let Some(span) = entry.span + && let Some(src) = span.lane(ctx.arena(), lane, content.layout()) + { + // SAFETY: the span resolved in generation, so the lane is live and + // immutable at the layout it was materialized under. + unsafe { slot.fill_copy(src) }; // SAFETY: the copy images a complete record of this layout. return finalized(unsafe { slot.finish_served() }, &entry.finality); } @@ -88,12 +90,7 @@ fn memoize<'e, 'l>( let lanes: Vec = (0..batch.len()).map(|index| unsafe { OwnedRecord::copy_out(layout, batch.get(index).rec()) }).collect(); let entry = MemoLevel { key, - generation: ctx.arena().generation(), - frames: match batch.len() { - 0 => 0, - _ => batch.get(0).rec().ptr() as usize, - }, - stride: layout.lane_stride(), + span: MaterializedSpan::of(&batch, ctx.arena()), lanes, finality, }; @@ -119,9 +116,7 @@ fn memoize<'e, 'l>( key, // A scalar record replays from the deep copy; the value the serve // returned already lives in this frame. - generation: u64::MAX, - frames: 0, - stride: 0, + span: None, lanes: vec![copy], finality, }); @@ -134,7 +129,7 @@ fn frame_memo<'e, 'l>( ctx: impl Ctx + CacheHash + ExtractArena<'e>, #[data] cell: ArenaCell, 32>>, content: impl Node>, - frame: FrameClaim<'l>, + frame: FrameClaim<'e, 'l>, ) -> GPoll> { let arena = ctx.arena(); let table = match cell.load(arena) { @@ -150,7 +145,7 @@ fn frame_memo<'e, 'l>( // SAFETY: published bytes are same-frame copies of this edge's records, // so they carry the edge's layout with live parked references, and the // claim is that layout's frame. - let revive = |mut frame: FrameClaim<'l>, bytes: &Box<[u8]>| unsafe { + let revive = |mut frame: FrameClaim<'e, 'l>, bytes: &Box<[u8]>| unsafe { frame.fill_copy(bytes.as_ptr()); frame.finish_served() }; @@ -191,7 +186,7 @@ fn monitor<'e, 'l>( ctx: impl Ctx + DeriveCtx + ExtractAll + ExtractArena<'e> + ModifyIndex + Copy, #[data] io: MonitorValue, content: impl Node>, - slot: FrameClaim<'l>, + slot: FrameClaim<'e, 'l>, ) -> GPoll> { if ctx.index() == 0 { *io.lock().unwrap() = Some(CtxSnapshot::capture(ctx)); @@ -233,10 +228,6 @@ mod tests { } fn scope_fixture<'a>(generations: &'a [(SourceId, u64)], arena: &'a Arena) -> EvalScope<'a> { - // SAFETY: between evaluations, nothing served on the stack is live. - unsafe { - core_types::record::stack::reserve(1 << 16); - } EvalScope::new(Some(0.5), None, None, generations, arena) } @@ -249,6 +240,7 @@ mod tests { #[test] fn monitor_serialize_recreates_the_value_from_its_snapshot() { + let frames = core_types::record::test_frames(1 << 16); let arena = Arena::new(1024).unwrap(); let generations = []; let scope = scope_fixture(&generations, &arena); @@ -260,14 +252,14 @@ mod tests { assert!(handle.serialize().is_none(), "no snapshot before the first eval"); let edge = handle.duplicate().downcast_record::().unwrap(); - let GPoll::Final(_) = core_types::record::serve_edge(&edge, &ctx) else { + let GPoll::Final(_) = core_types::record::serve_edge(&edge, &ctx, &frames) else { panic!("expected a final record"); }; let io = handle.serialize().expect("the eval landed a snapshot"); let snapshot = io.downcast_ref::().expect("the monitor serializes its context snapshot"); let ctx = snapshot.rehydrate(&scope).expect("the arena holds the chains"); - let GPoll::Final(served) = core_types::record::capture(&edge, &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&edge, &ctx, &frames) else { panic!("expected a final record"); }; assert_eq!(served.element::(), 11); @@ -275,6 +267,7 @@ mod tests { #[test] fn a_leveled_monitor_recreates_the_whole_extent() { + let frames = core_types::record::test_frames(1 << 16); let arena = Arena::new(1 << 12).unwrap(); let generations = []; let scope = scope_fixture(&generations, &arena); @@ -286,14 +279,14 @@ mod tests { let handle = EdgeHandle::new_record::(Arc::new(monitor) as Arc); let edge = handle.duplicate().downcast_record::().unwrap(); - let GPoll::Final(_) = core_types::record::serve_edge(&edge, &ctx) else { + let GPoll::Final(_) = core_types::record::serve_edge(&edge, &ctx, &frames) else { panic!("expected a final record"); }; let io = handle.serialize().expect("the eval landed a snapshot"); let snapshot = io.downcast_ref::().expect("the monitor serializes its context snapshot"); let ctx = snapshot.rehydrate(&scope).expect("the arena holds the chains"); - let LevelStatus::Batch(batch, _) = core_types::record::materialize_level(&edge, &ctx, &arena) else { + let LevelStatus::Batch(batch, _) = core_types::record::materialize_level(&edge, &ctx, &arena, &frames) else { panic!("expected a materialized level"); }; assert_eq!(batch.len(), 3, "the recreation holds the whole extent, not the addressed lane"); @@ -304,8 +297,8 @@ mod tests { #[test] fn memo_copy_out_consults_the_deep_element_clone() { - #[derive(Clone, Debug, PartialEq)] - #[derive(dyn_any::DynAny)] + let frames = core_types::record::test_frames(1 << 16); + #[derive(Clone, Debug, PartialEq, dyn_any::DynAny)] struct Payload(String, u32); unsafe fn deep(ptr: *const u8) -> Box { let value = unsafe { core_types::record::borrow_element::(core_types::record::Rec::new(ptr)) }; @@ -326,12 +319,17 @@ mod tests { let memoized = MemoizeNode::new(lifted::(Payload("deep".to_string(), 0)), &layout); let memoized = core_types::record::RecordExtract::::new(memoized, &layout); - assert_eq!(memoized.eval(&ctx), GPoll::Final(Payload("deep".to_string(), 0)), "the miss serves the live value"); - assert_eq!(memoized.eval(&ctx), GPoll::Final(Payload("deep".to_string(), 2)), "the hit replays through both halves of the deep glue"); + assert_eq!(memoized.eval(&ctx, &frames), GPoll::Final(Payload("deep".to_string(), 0)), "the miss serves the live value"); + assert_eq!( + memoized.eval(&ctx, &frames), + GPoll::Final(Payload("deep".to_string(), 2)), + "the hit replays through both halves of the deep glue" + ); } #[test] fn memoize_caches_across_evals() { + let frames = core_types::record::test_frames(1 << 16); let arena = Arena::new(1024).unwrap(); let generations = []; let scope = scope_fixture(&generations, &arena); @@ -341,12 +339,13 @@ mod tests { let memoized = MemoizeNode::new(counting(), &layout); let memoized = core_types::record::RecordExtract::::new(memoized, &layout); - assert_eq!(memoized.eval(&ctx), GPoll::Final(1)); - assert_eq!(memoized.eval(&ctx), GPoll::Final(1)); + assert_eq!(memoized.eval(&ctx, &frames), GPoll::Final(1)); + assert_eq!(memoized.eval(&ctx, &frames), GPoll::Final(1)); } #[test] fn memo_invalidates_on_generation_bump() { + let frames = core_types::record::test_frames(1 << 16); let arena = Arena::new(1024).unwrap(); let source: SourceId = 7; let before = [(source, 1)]; @@ -358,13 +357,14 @@ mod tests { let memoized = MemoizeNode::new(counting(), &layout); let memoized = core_types::record::RecordExtract::::new(memoized, &layout); - assert_eq!(memoized.eval(&ContextImpl::root(&scope_before)), GPoll::Final(1)); - assert_eq!(memoized.eval(&ContextImpl::root(&scope_before)), GPoll::Final(1)); - assert_eq!(memoized.eval(&ContextImpl::root(&scope_after)), GPoll::Final(2)); + assert_eq!(memoized.eval(&ContextImpl::root(&scope_before), &frames), GPoll::Final(1)); + assert_eq!(memoized.eval(&ContextImpl::root(&scope_before), &frames), GPoll::Final(1)); + assert_eq!(memoized.eval(&ContextImpl::root(&scope_after), &frames), GPoll::Final(2)); } #[test] fn memo_replays_partiality_on_hit() { + let frames = core_types::record::test_frames(1 << 16); let arena = Arena::new(1024).unwrap(); let generations = []; let scope = scope_fixture(&generations, &arena); @@ -374,12 +374,13 @@ mod tests { let memoized = MemoizeNode::new(partial_counting(), &layout); let memoized = core_types::record::RecordExtract::::new(memoized, &layout); - assert_eq!(memoized.eval(&ctx), GPoll::Partial(1)); - assert_eq!(memoized.eval(&ctx), GPoll::Partial(1)); + assert_eq!(memoized.eval(&ctx, &frames), GPoll::Partial(1)); + assert_eq!(memoized.eval(&ctx, &frames), GPoll::Partial(1)); } #[test] fn memoized_edges_stack_and_rewire() { + let frames = core_types::record::test_frames(1 << 16); let arena = Arena::new(1024).unwrap(); let generations = []; let scope = scope_fixture(&generations, &arena); @@ -391,12 +392,13 @@ mod tests { let stacked = MemoizeNode::new(memoized.downcast_record::().unwrap(), &layout); let stacked = core_types::record::RecordExtract::::new(stacked, &layout); - assert_eq!(stacked.eval(&ctx), GPoll::Final(1)); - assert_eq!(stacked.eval(&ctx), GPoll::Final(1)); + assert_eq!(stacked.eval(&ctx, &frames), GPoll::Final(1)); + assert_eq!(stacked.eval(&ctx, &frames), GPoll::Final(1)); } #[test] fn frame_memo_shares_one_record_copy_per_frame() { + let frames = core_types::record::test_frames(1 << 16); let arena = Arena::new(4096).unwrap(); let generations = []; let scope = scope_fixture(&generations, &arena); @@ -405,10 +407,10 @@ mod tests { let layout = element_layout::(); let memo = FrameMemoNode::new(lifted::("lent out".to_string()), &layout); - let GPoll::Final(first) = core_types::record::serve_edge(&memo, &ctx) else { + let GPoll::Final(first) = core_types::record::serve_edge(&memo, &ctx, &frames) else { panic!("the miss must fill the frame table"); }; - let GPoll::Final(second) = core_types::record::serve_edge(&memo, &ctx) else { + let GPoll::Final(second) = core_types::record::serve_edge(&memo, &ctx, &frames) else { panic!("the hit must revive the published record"); }; let first: &String = unsafe { core_types::record::borrow_element(layout.rec(&first)) }; diff --git a/node-graph/nodes/gcore/src/record.rs b/node-graph/nodes/gcore/src/record.rs index beea7562d5..165b71911e 100644 --- a/node-graph/nodes/gcore/src/record.rs +++ b/node-graph/nodes/gcore/src/record.rs @@ -398,7 +398,7 @@ mod tests { use core_types::context::{ContextImpl, EvalScope, ExtractArena}; use core_types::gpoll::GPoll; use core_types::node::Node; - use core_types::record::{FrameClaim, Layout, LiftedSource, Rec, RecordSource, Served, stack}; + use core_types::record::{FrameClaim, Layout, LiftedSource, Rec, RecordSource, Served}; use core_types::value::ValueSource; struct RecordSourceNode { @@ -408,19 +408,21 @@ mod tests { partial: bool, } - impl Node for RecordSourceNode { - fn serve<'e, 'l>(&self, input: &C, slot: FrameClaim<'l>) -> GPoll> + impl Node for RecordSourceNode { + fn serve<'e, 'l>(&self, input: &C, slot: FrameClaim<'e, 'l>) -> GPoll> where C: ExtractArena, { - let mut frame = core_types::record::FrameBuilder::new(&self.layout, ExtractArena::arena(input)); - frame.element(self.element); - for (name, field) in &self.fields { - frame.field::(name, 0, *field); + let mut frame = slot; + let arena = ExtractArena::arena(input); + if frame.element(self.element, arena).is_none() { + return GPoll::arena_exhausted(); } - let Some(value) = frame.finish() else { return GPoll::arena_exhausted() }; - // SAFETY: the builder served a record of this node's layout. - let served = unsafe { slot.forward(&value) }; + for (name, field) in &self.fields { + write_field_at(&mut frame, &self.layout, name, 0, *field); + } + // SAFETY: the writes above complete the record of this layout. + let served = unsafe { frame.finish_served() }; match self.partial { true => GPoll::Partial(served), false => GPoll::Final(served), @@ -439,22 +441,24 @@ mod tests { } impl Node for LeveledSourceNode { - fn serve<'e, 'l>(&self, input: &C, slot: FrameClaim<'l>) -> GPoll> + fn serve<'e, 'l>(&self, input: &C, slot: FrameClaim<'e, 'l>) -> GPoll> where C: ExtractArena, { let element = self.elements[input.innermost_index() as usize % self.elements.len()]; - let mut frame = core_types::record::FrameBuilder::new(&self.layout, ExtractArena::arena(input)); - frame.element(element); - if let Some((name, value)) = self.field { - frame.field::(name, 0, value); + let mut frame = slot; + let arena = ExtractArena::arena(input); + if frame.element(element, arena).is_none() { + return GPoll::arena_exhausted(); } - let Some(value) = frame.finish() else { return GPoll::arena_exhausted() }; - // SAFETY: the builder served a record of this node's layout. - GPoll::Final(unsafe { slot.forward(&value) }) + if let Some((name, value)) = self.field { + write_field_at(&mut frame, &self.layout, name, 0, value); + } + // SAFETY: the writes above complete the record of this layout. + GPoll::Final(unsafe { frame.finish_served() }) } - fn extent_at<'x>(&self, _input: &C, _level: u8) -> GPoll + fn extent_at<'x>(&self, _input: &C, _level: u8, _frames: &core_types::record::Frames<'x>) -> GPoll where C: ExtractArena, { @@ -472,20 +476,22 @@ mod tests { } impl Node for LeveledTransformSource { - fn serve<'e, 'l>(&self, input: &C, slot: FrameClaim<'l>) -> GPoll> + fn serve<'e, 'l>(&self, input: &C, slot: FrameClaim<'e, 'l>) -> GPoll> where C: ExtractArena, { let (element, transform) = self.rows[input.innermost_index() as usize % self.rows.len()]; - let mut frame = core_types::record::FrameBuilder::new(&self.layout, ExtractArena::arena(input)); - frame.element(element); - frame.attr::(transform); - let Some(value) = frame.finish() else { return GPoll::arena_exhausted() }; - // SAFETY: the builder served a record of this node's layout. - GPoll::Final(unsafe { slot.forward(&value) }) + let mut frame = slot; + let arena = ExtractArena::arena(input); + if frame.element(element, arena).is_none() { + return GPoll::arena_exhausted(); + } + write_attr_at::(&mut frame, &self.layout, transform); + // SAFETY: the writes above complete the record of this layout. + GPoll::Final(unsafe { frame.finish_served() }) } - fn extent_at<'x>(&self, _input: &C, _level: u8) -> GPoll + fn extent_at<'x>(&self, _input: &C, _level: u8, _frames: &core_types::record::Frames<'x>) -> GPoll where C: ExtractArena, { @@ -505,7 +511,7 @@ mod tests { } impl Node for DrainSourceNode { - fn serve<'e, 'l>(&self, input: &C, slot: FrameClaim<'l>) -> GPoll> + fn serve<'e, 'l>(&self, input: &C, slot: FrameClaim<'e, 'l>) -> GPoll> where C: ExtractArena, { @@ -513,14 +519,16 @@ mod tests { if lane >= self.count as u64 { return GPoll::past_end(); } - let mut frame = core_types::record::FrameBuilder::new(&self.layout, ExtractArena::arena(input)); - frame.element(lane as f64); - let Some(value) = frame.finish() else { return GPoll::arena_exhausted() }; - // SAFETY: the builder served a record of this node's layout. - GPoll::Final(unsafe { slot.forward(&value) }) + let mut frame = slot; + let arena = ExtractArena::arena(input); + if frame.element(lane as f64, arena).is_none() { + return GPoll::arena_exhausted(); + } + // SAFETY: the writes above complete the record of this layout. + GPoll::Final(unsafe { frame.finish_served() }) } - fn extent_at<'x>(&self, _input: &C, _level: u8) -> GPoll + fn extent_at<'x>(&self, _input: &C, _level: u8, _frames: &core_types::record::Frames<'x>) -> GPoll where C: ExtractArena, { @@ -537,18 +545,20 @@ mod tests { } impl Node for IndexSourceNode { - fn serve<'e, 'l>(&self, input: &C, slot: FrameClaim<'l>) -> GPoll> + fn serve<'e, 'l>(&self, input: &C, slot: FrameClaim<'e, 'l>) -> GPoll> where C: ExtractArena, { // Depth-0 content varying per copy: the enclosing (pushed) level's // index sits one link above the content's own innermost lane. let element = input.try_index().and_then(|mut indices| indices.nth(1)).unwrap_or(0) as f64; - let mut frame = core_types::record::FrameBuilder::new(&self.layout, ExtractArena::arena(input)); - frame.element(element); - let Some(value) = frame.finish() else { return GPoll::arena_exhausted() }; - // SAFETY: the builder served a record of this node's layout. - GPoll::Final(unsafe { slot.forward(&value) }) + let mut frame = slot; + let arena = ExtractArena::arena(input); + if frame.element(element, arena).is_none() { + return GPoll::arena_exhausted(); + } + // SAFETY: the writes above complete the record of this layout. + GPoll::Final(unsafe { frame.finish_served() }) } fn layout(&self) -> &Layout { @@ -556,11 +566,27 @@ mod tests { } } + /// Writes a field at the layout's resolved offset, the wiring-proven pairing + /// a generated node performs. + fn write_field_at(frame: &mut FrameClaim<'_, '_>, layout: &Layout, name: &str, level: u8, value: T) { + let field = layout + .fields + .iter() + .find(|field| field.name == name && field.level == level) + .expect("the layout carries the written field"); + assert_eq!(field.type_id, std::any::TypeId::of::(), "the field was declared at this value type"); + // SAFETY: the offset is this layout's own, at the field's declared type. + unsafe { frame.attr_at(field.offset, value) }; + } + + /// [`write_field_at`] for a census marker at level 0. + fn write_attr_at(frame: &mut FrameClaim<'_, '_>, layout: &Layout, value: A::Value<'static>) + where + A::Value<'static>: Copy + 'static, + { + write_field_at(frame, layout, A::NAME, 0, value); + } fn scope_fixture<'a>(generations: &'a [(SourceId, u64)], arena: &'a Arena) -> EvalScope<'a> { - // SAFETY: between evaluations, nothing served on the stack is live. - unsafe { - stack::reserve(1 << 16); - } EvalScope::new(Some(0.5), None, None, generations, arena) } @@ -600,9 +626,9 @@ mod tests { Layout::default().with_writes(1, core_types::record::element_write::(), &writes) } - fn reserve_for(layouts: &[&Layout]) { - // SAFETY: between evaluations, nothing served on the stack is live. - unsafe { stack::reserve(layouts.iter().map(|layout| layout.frame_bytes()).sum::().max(1 << 12)); } } + fn frames_for(layouts: &[&Layout]) -> core_types::record::Frames<'static> { + core_types::record::test_frames(layouts.iter().map(|layout| layout.frame_bytes()).sum::().max(1 << 12)) + } fn install>>(mut node: N, meta: core_types::record::LayoutMeta, inputs: &[Option<&Layout>]) -> N { // The fixtures wire constants into every eager input, which the compiler @@ -663,11 +689,11 @@ mod tests { let base = f64_layout(&[]); let leveled = repeat_opacity_layout(&base); - reserve_for(&[&base, &leveled]); + let frames = frames_for(&[&base, &leveled]); let node = install(RepeatOpacityNode::new(bare_source(&base, 7.), ValueSource::new(8u32), &base), repeat_opacity_layout_meta(), &[Some(&base)]); assert_eq!(Node::::layout(&node), &leveled); - let GPoll::Final(served) = core_types::record::capture(&node, &indexed) else { + let GPoll::Final(served) = core_types::record::capture(&node, &indexed, &frames) else { panic!("expected a final record"); }; assert_eq!(served.element::(), 7.); @@ -682,11 +708,11 @@ mod tests { let scope = scope_fixture(&generations, &arena); let ctx = ContextImpl::root(&scope); let base = f64_layout(&[]); - reserve_for(&[&base]); + let frames = frames_for(&[&base]); let node = install(RepeatOpacityNode::new(bare_source(&base, 7.), ValueSource::new(3u32), &base), repeat_opacity_layout_meta(), &[Some(&base)]); // The pushed level (0, the only level) reports the copy count. - assert_eq!(node.extent_at(&ctx, 0), core_types::gpoll::GPoll::Final(core_types::gpoll::Extent::Exactly(3))); + assert_eq!(node.extent_at(&ctx, 0, &frames), core_types::gpoll::GPoll::Final(core_types::gpoll::Extent::Exactly(3))); } #[test] @@ -699,7 +725,7 @@ mod tests { let base = f64_layout(&[]); let (count_edge, count_layout) = lifted_value(3u32); let (reverse_edge, reverse_layout) = lifted_value(false); - reserve_for(&[&base, &count_layout]); + let frames = frames_for(&[&base, &count_layout]); let meta = core_types::record::LayoutMeta { sources: vec![0], @@ -717,9 +743,9 @@ mod tests { ); let leveled = Node::::layout(&node).clone(); assert_eq!(leveled.depth, 1, "the IList return pushed one rank level above the depth-0 content"); - assert_eq!(node.extent_at(&ctx, 0), GPoll::Final(core_types::gpoll::Extent::Exactly(3))); + assert_eq!(node.extent_at(&ctx, 0, &frames), GPoll::Final(core_types::gpoll::Extent::Exactly(3))); - let GPoll::Final(served) = core_types::record::capture(&node, &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&node, &ctx, &frames) else { panic!("expected a final record"); }; assert_eq!(served.element::(), 7., "the opaque generic element forwarded unchanged"); @@ -735,7 +761,7 @@ mod tests { let base = f64_layout(&[]); let (count_edge, count_layout) = lifted_value(4u32); let (reverse_edge, reverse_layout) = lifted_value(false); - reserve_for(&[&base, &count_layout]); + let frames = frames_for(&[&base, &count_layout]); let meta = core_types::record::LayoutMeta { sources: vec![0], @@ -761,7 +787,7 @@ mod tests { let head = ctx.index_head(); for copy in 0..4 { let lane = ctx.promoted(&head, copy); - let GPoll::Final(served) = core_types::record::capture(&repeat, &lane) else { + let GPoll::Final(served) = core_types::record::capture(&repeat, &lane, &frames) else { panic!("expected a final record"); }; // The copy evaluated its content at its own pushed index. @@ -779,7 +805,7 @@ mod tests { let base = f64_layout(&[]); let (count_edge, count_layout) = lifted_value(4u32); let (reverse_edge, reverse_layout) = lifted_value(true); - reserve_for(&[&base, &count_layout, &reverse_layout]); + let frames = frames_for(&[&base, &count_layout, &reverse_layout]); let meta = core_types::record::LayoutMeta { sources: vec![0], @@ -805,7 +831,7 @@ mod tests { let head = ctx.index_head(); for copy in 0..4u64 { let lane = ctx.promoted(&head, copy); - let GPoll::Final(served) = core_types::record::capture(&repeat, &lane) else { + let GPoll::Final(served) = core_types::record::capture(&repeat, &lane, &frames) else { panic!("expected a final record"); }; // Reversed: copy `j` evaluates its content at index `count - 1 - j`. @@ -824,7 +850,7 @@ mod tests { let leveled_content = repeat_opacity_layout(&base); let (count_edge, count_layout) = lifted_value(2u32); let (reverse_edge, reverse_layout) = lifted_value(false); - reserve_for(&[&base, &leveled_content, &count_layout, &reverse_layout]); + let frames = frames_for(&[&base, &leveled_content, &count_layout, &reverse_layout]); let content = install(RepeatOpacityNode::new(bare_source(&base, 7.), ValueSource::new(3u32), &base), repeat_opacity_layout_meta(), &[Some(&base)]); let meta = core_types::record::LayoutMeta { @@ -850,13 +876,13 @@ mod tests { ); let two_level = Node::::layout(&repeat).clone(); assert_eq!(two_level.depth, 2, "the pushed level sits above the content's own level"); - assert_eq!(repeat.extent_at(&ctx, 1), GPoll::Final(Extent::Exactly(2)), "the pushed level's extent is the copy count"); - assert_eq!(repeat.extent_at(&ctx, 0), GPoll::Final(Extent::Exactly(3)), "the content's level forwards"); + assert_eq!(repeat.extent_at(&ctx, 1, &frames), GPoll::Final(Extent::Exactly(2)), "the pushed level's extent is the copy count"); + assert_eq!(repeat.extent_at(&ctx, 0, &frames), GPoll::Final(Extent::Exactly(3)), "the content's level forwards"); let head = ctx.index_head(); for flat in 0..6u64 { let lane = ctx.promoted(&head, flat); - let GPoll::Final(served) = core_types::record::capture(&repeat, &lane) else { + let GPoll::Final(served) = core_types::record::capture(&repeat, &lane, &frames) else { panic!("expected a final record"); }; assert_eq!(served.element::(), 7.); @@ -880,7 +906,7 @@ mod tests { fields: vec![(Opacity::NAME, 0.5)], partial: false, }; - reserve_for(&[&base]); + let frames = frames_for(&[&base]); let node = install( RepeatFadedNode::new(RecordSource::new(content, &base, &base), ValueSource::new(4u32), &base), @@ -889,12 +915,12 @@ mod tests { ); let leveled = Node::::layout(&node).clone(); assert_eq!(leveled.depth, 1, "the IList return pushed one rank level above the content"); - assert_eq!(node.extent_at(&ctx, 0), GPoll::Final(Extent::Exactly(4))); + assert_eq!(node.extent_at(&ctx, 0, &frames), GPoll::Final(Extent::Exactly(4))); let head = ctx.index_head(); for copy in 0..4u64 { let lane = ctx.promoted(&head, copy); - let GPoll::Final(served) = core_types::record::capture(&node, &lane) else { + let GPoll::Final(served) = core_types::record::capture(&node, &lane, &frames) else { panic!("expected a final record"); }; // The content row's element forwards; its opacity re-scales per copy. @@ -913,7 +939,7 @@ mod tests { let base_layout = leveled_f64_layout(&[Opacity::NAME]); let new_layout = leveled_f64_layout(&[Length::NAME]); let union = Layout::union(&[&base_layout, &new_layout]); - reserve_for(&[&base_layout, &new_layout, &union]); + let frames = frames_for(&[&base_layout, &new_layout, &union]); let base = LeveledSourceNode { layout: base_layout.clone(), @@ -941,13 +967,13 @@ mod tests { ); let out = Node::::layout(&node).clone(); assert_eq!(out.depth, 1); - assert_eq!(node.extent_at(&ctx, 0), GPoll::Final(Extent::Exactly(5)), "the top level sums both sides"); + assert_eq!(node.extent_at(&ctx, 0, &frames), GPoll::Final(Extent::Exactly(5)), "the top level sums both sides"); let head = ctx.index_head(); let expected = [10., 11., 100., 101., 102.]; for (lane, &element) in expected.iter().enumerate() { let scoped = ctx.promoted(&head, lane as u64); - let GPoll::Final(served) = core_types::record::capture(&node, &scoped) else { + let GPoll::Final(served) = core_types::record::capture(&node, &scoped, &frames) else { panic!("expected a final record"); }; assert_eq!(served.element::(), element); @@ -976,7 +1002,7 @@ mod tests { let bc = Layout::union(&[&b_layout, &c_layout]); let left_union = Layout::union(&[&ab, &c_layout]); let right_union = Layout::union(&[&a_layout, &bc]); - reserve_for(&[&a_layout, &b_layout, &c_layout, &ab, &bc, &left_union, &right_union]); + let frames = frames_for(&[&a_layout, &b_layout, &c_layout, &ab, &bc, &left_union, &right_union]); let a = || LeveledSourceNode { layout: a_layout.clone(), @@ -1024,8 +1050,8 @@ mod tests { &[Some(&a_layout), Some(&bc)], ); - assert_eq!(left.extent_at(&ctx, 0), GPoll::Final(Extent::Exactly(6))); - assert_eq!(right.extent_at(&ctx, 0), GPoll::Final(Extent::Exactly(6))); + assert_eq!(left.extent_at(&ctx, 0, &frames), GPoll::Final(Extent::Exactly(6))); + assert_eq!(right.extent_at(&ctx, 0, &frames), GPoll::Final(Extent::Exactly(6))); let head = ctx.index_head(); let check = |poll: GPoll, lane: usize, (element, opacity, length): (f64, f64, f64)| { @@ -1039,8 +1065,8 @@ mod tests { let expected = [(10., 0.5, 0.), (11., 0.5, 0.), (100., 1., 7.), (101., 1., 7.), (102., 1., 7.), (1000., 0.25, 0.)]; for (lane, &row) in expected.iter().enumerate() { let scoped = ctx.promoted(&head, lane as u64); - check(core_types::record::capture(&left, &scoped), lane, row); - check(core_types::record::capture(&right, &scoped), lane, row); + check(core_types::record::capture(&left, &scoped, &frames), lane, row); + check(core_types::record::capture(&right, &scoped, &frames), lane, row); } } @@ -1055,7 +1081,7 @@ mod tests { let new_layout = leveled_f64_layout(&[Length::NAME]); let union = Layout::union(&[&base_layout, &new_layout]); let out = f64_layout(&[]); - reserve_for(&[&base_layout, &new_layout, &union, &out]); + let frames = frames_for(&[&base_layout, &new_layout, &union, &out]); let base = LeveledSourceNode { layout: base_layout.clone(), @@ -1086,7 +1112,7 @@ mod tests { let head = ctx.index_head(); let per_lane: f64 = (0..5u64) .map(|lane| { - let GPoll::Final(served) = core_types::record::capture(&extend, &ctx.promoted(&head, lane)) else { + let GPoll::Final(served) = core_types::record::capture(&extend, &ctx.promoted(&head, lane), &frames) else { panic!("expected a final record at lane {lane}"); }; served.element::() @@ -1094,7 +1120,7 @@ mod tests { .sum(); let node = install_flip(SumNode::new(extend, &wire), &out); - let GPoll::Final(served) = core_types::record::capture(&node, &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&node, &ctx, &frames) else { panic!("expected a final record"); }; // The fold's batch walks both sides of the seam in one range. @@ -1133,7 +1159,7 @@ mod tests { let (base_reverse, base_reverse_layout) = lifted_value(false); let (new_count, new_count_layout) = lifted_value(4u32); let (new_reverse, new_reverse_layout) = lifted_value(false); - reserve_for(&[&content_layout, &base_count_layout, &new_count_layout]); + let frames = frames_for(&[&content_layout, &base_count_layout, &new_count_layout]); let repeat = |elements: Vec, count_edge, reverse_edge, count_layout: &Layout, reverse_layout: &Layout| { let content = LeveledSourceNode { @@ -1163,12 +1189,12 @@ mod tests { let node = install(ExtendNode::new(base, new, &two_level), extend_meta(), &[Some(&two_level), Some(&two_level)]); let out = Node::::layout(&node).clone(); assert_eq!(out.depth, 2); - assert_eq!(node.extent_at(&ctx, 1), GPoll::Final(Extent::Exactly(6)), "the top level sums both sides' copies"); - assert_eq!(node.extent_at(&ctx, 0), GPoll::Final(Extent::Exactly(3)), "the inner level forwards the shared extent"); + assert_eq!(node.extent_at(&ctx, 1, &frames), GPoll::Final(Extent::Exactly(6)), "the top level sums both sides' copies"); + assert_eq!(node.extent_at(&ctx, 0, &frames), GPoll::Final(Extent::Exactly(3)), "the inner level forwards the shared extent"); let head = ctx.index_head(); for (lane, element) in [(0u64, 1.), (5, 3.), (6, 10.), (17, 30.)] { - let GPoll::Final(served) = core_types::record::capture(&node, &ctx.promoted(&head, lane)) else { + let GPoll::Final(served) = core_types::record::capture(&node, &ctx.promoted(&head, lane), &frames) else { panic!("expected a final record at lane {lane}"); }; assert_eq!(served.element::(), element, "flat lane {lane}"); @@ -1182,8 +1208,8 @@ mod tests { let base = repeat(vec![1., 2., 3.], base_count, base_reverse, &base_count_layout, &base_reverse_layout); let new = repeat(vec![10., 20., 30., 40.], new_count, new_reverse, &new_count_layout, &new_reverse_layout); let ragged = install(ExtendNode::new(base, new, &two_level), extend_meta(), &[Some(&two_level), Some(&two_level)]); - assert_eq!(ragged.extent_at(&ctx, 1), GPoll::Final(Extent::Exactly(6))); - assert!(matches!(ragged.extent_at(&ctx, 0), GPoll::Error(_)), "ragged inner extents must not report a flat total"); + assert_eq!(ragged.extent_at(&ctx, 1, &frames), GPoll::Final(Extent::Exactly(6))); + assert!(matches!(ragged.extent_at(&ctx, 0, &frames), GPoll::Error(_)), "ragged inner extents must not report a flat total"); } #[test] @@ -1194,7 +1220,7 @@ mod tests { let ctx = ContextImpl::root(&scope); let layout = leveled_f64_layout(&[]); - reserve_for(&[&layout]); + let frames = frames_for(&[&layout]); let meta = || core_types::record::LayoutMeta { sources: vec![0], reads: vec![], @@ -1218,13 +1244,13 @@ mod tests { ) }; + let head = ctx.index_head(); for (index, expected) in [(1., vec![10., 12.]), (-1., vec![10., 11.]), (5., vec![10., 11., 12.])] { let node = build(index); - assert_eq!(node.extent_at(&ctx, 0), GPoll::Final(Extent::Exactly(expected.len())), "omit at {index}"); - let head = ctx.index_head(); + assert_eq!(node.extent_at(&ctx, 0, &frames), GPoll::Final(Extent::Exactly(expected.len())), "omit at {index}"); for (lane, &element) in expected.iter().enumerate() { let scoped = ctx.promoted(&head, lane as u64); - let GPoll::Final(served) = core_types::record::capture(&node, &scoped) else { + let GPoll::Final(served) = core_types::record::capture(&node, &scoped, &frames) else { panic!("expected a final record"); }; assert_eq!(served.element::(), element, "omit at {index}, lane {lane}"); @@ -1240,7 +1266,7 @@ mod tests { let ctx = ContextImpl::root(&scope); let layout = leveled_f64_layout(&[Opacity::NAME]); - reserve_for(&[&layout]); + let frames = frames_for(&[&layout]); let build = |index: f64| { let content = LeveledSourceNode { layout: layout.clone(), @@ -1264,13 +1290,13 @@ mod tests { ) }; + let head = ctx.index_head(); for (index, expected) in [(1., Some(11.)), (-1., Some(12.)), (9., None)] { let node = build(index); - assert_eq!(node.extent_at(&ctx, 0), GPoll::Final(Extent::Exactly(expected.is_some() as usize)), "index {index}"); + assert_eq!(node.extent_at(&ctx, 0, &frames), GPoll::Final(Extent::Exactly(expected.is_some() as usize)), "index {index}"); let Some(element) = expected else { continue }; - let head = ctx.index_head(); let scoped = ctx.promoted(&head, 0); - let GPoll::Final(served) = core_types::record::capture(&node, &scoped) else { + let GPoll::Final(served) = core_types::record::capture(&node, &scoped, &frames) else { panic!("expected a final record"); }; assert_eq!(served.element::(), element); @@ -1288,7 +1314,7 @@ mod tests { let layout = leveled_f64_layout(&[]); let out = f64_layout(&[]); - reserve_for(&[&layout, &out]); + let frames = frames_for(&[&layout, &out]); for (index, expected) in [(1., 11.), (-1., 12.), (9., 0.)] { let content = LeveledSourceNode { @@ -1298,7 +1324,7 @@ mod tests { }; let (index_edge, index_layout) = lifted_value(index); let node = install_flip(ExtractElementNode::new(RecordSource::new(content, &layout, &layout), index_edge, &layout, &index_layout), &out); - let GPoll::Final(served) = core_types::record::capture(&node, &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&node, &ctx, &frames) else { panic!("expected a final record"); }; assert_eq!(served.element::(), expected, "extract at {index}"); @@ -1313,7 +1339,7 @@ mod tests { let ctx = ContextImpl::root(&scope); let layout = Layout::default().with_writes(1, core_types::record::element_write::(), &[core_types::record::FieldWrite::of::(0)]); - reserve_for(&[&layout]); + let frames = frames_for(&[&layout]); let content = |rows: &[(f64, f64)]| LeveledTransformSource { layout: layout.clone(), rows: rows.iter().map(|&(element, x)| (element, DAffine2::from_translation(glam::DVec2::new(x, 0.)))).collect(), @@ -1331,12 +1357,12 @@ mod tests { let kept = build(true); let out = Node::::layout(&kept).clone(); assert_eq!(out.depth, 1); - assert_eq!(kept.extent_at(&ctx, 0), GPoll::Final(Extent::Exactly(6))); + assert_eq!(kept.extent_at(&ctx, 0, &frames), GPoll::Final(Extent::Exactly(6))); let expected = [(1., 10.), (2., 30.), (3., 20.), (1., 30.), (2., 10.), (3., 20.)]; let head = ctx.index_head(); for (lane, &(element, x)) in expected.iter().enumerate() { let scoped = ctx.promoted(&head, lane as u64); - let GPoll::Final(served) = core_types::record::capture(&kept, &scoped) else { + let GPoll::Final(served) = core_types::record::capture(&kept, &scoped, &frames) else { panic!("expected a final record"); }; assert_eq!(served.element::(), element, "lane {lane}"); @@ -1345,8 +1371,8 @@ mod tests { } let replaced = build(false); - assert_eq!(replaced.extent_at(&ctx, 0), GPoll::Final(Extent::Exactly(3))); - let GPoll::Final(served) = core_types::record::capture(&replaced, &ctx.promoted(&head, 0)) else { + assert_eq!(replaced.extent_at(&ctx, 0, &frames), GPoll::Final(Extent::Exactly(3))); + let GPoll::Final(served) = core_types::record::capture(&replaced, &ctx.promoted(&head, 0), &frames) else { panic!("expected a final record"); }; let transform: DAffine2 = served.attr::(); @@ -1362,7 +1388,7 @@ mod tests { // The subject carries Transform, which the kernel never declares. let layout = Layout::default().with_writes(1, core_types::record::element_write::(), &[core_types::record::FieldWrite::of::(0)]); - reserve_for(&[&layout]); + let frames = frames_for(&[&layout]); let rows = [(1., 10.), (2., 30.), (3., 20.)]; let content = LeveledTransformSource { layout: layout.clone(), @@ -1377,11 +1403,11 @@ mod tests { let out = Node::::layout(&node).clone(); assert_eq!(out.depth, 1, "gathering preserves the subject's depth"); assert!(out.offset_of(::NAME, 0).is_some(), "the undeclared attribute survives"); - assert_eq!(node.extent_at(&ctx, 0), GPoll::Final(Extent::Exactly(3))); + assert_eq!(node.extent_at(&ctx, 0, &frames), GPoll::Final(Extent::Exactly(3))); let head = ctx.index_head(); for (lane, &(element, x)) in rows.iter().rev().enumerate() { - let GPoll::Final(served) = core_types::record::capture(&node, &ctx.promoted(&head, lane as u64)) else { + let GPoll::Final(served) = core_types::record::capture(&node, &ctx.promoted(&head, lane as u64), &frames) else { panic!("expected a final record"); }; assert_eq!(served.element::(), element, "lane {lane} takes the gathered element"); @@ -1400,7 +1426,7 @@ mod tests { let ctx = ContextImpl::root(&scope); let layout = Layout::default().with_writes(1, core_types::record::element_write::(), &[core_types::record::FieldWrite::of::(0)]); - reserve_for(&[&layout]); + let frames = frames_for(&[&layout]); let content = LeveledTransformSource { layout: layout.clone(), rows: [(1., 10.), (2., 30.), (3., 20.)] @@ -1413,16 +1439,16 @@ mod tests { let head = ctx.index_head(); let scoped = ctx.promoted(&head, 0); - assert!(matches!(node.eval_batch(&scoped, 0..6, None), core_types::node::BatchStatus::NeedBuffer)); + assert!(matches!(node.eval_batch(&scoped, 0..6, None, &frames), core_types::node::BatchStatus::NeedBuffer)); let mut scratch = vec![std::mem::MaybeUninit::::uninit(); 6 * out.lane_stride() / 8]; - let core_types::node::BatchStatus::Filled(batch, finality, _) = node.eval_batch(&scoped, 0..6, Some(&mut scratch)) else { + let core_types::node::BatchStatus::Filled(batch, finality, _) = node.eval_batch(&scoped, 0..6, Some(&mut scratch), &frames) else { panic!("expected a filled batch"); }; assert_eq!(finality, core_types::gpoll::Finality::AllFinal); let batch = batch.into_shared(); assert_eq!(batch.len(), 6); for lane in 0..6 { - let GPoll::Final(served) = core_types::record::capture(&node, &ctx.promoted(&head, lane as u64)) else { + let GPoll::Final(served) = core_types::record::capture(&node, &ctx.promoted(&head, lane as u64), &frames) else { panic!("expected a final record"); }; assert_eq!(unsafe { batch.get(lane).element::() }, served.element::(), "lane {lane}"); @@ -1450,7 +1476,7 @@ mod tests { let ctx = ContextImpl::root(&scope); let layout = Layout::default().with_writes(1, core_types::record::element_write::(), &[core_types::record::FieldWrite::of::(0)]); - reserve_for(&[&layout]); + let frames = frames_for(&[&layout]); let content = LeveledTransformSource { layout: layout.clone(), rows: [(1., 10.), (2., 30.), (3., 20.)] @@ -1471,7 +1497,7 @@ mod tests { let scoped = ctx.promoted(&head, 0); let mut scratch = vec![std::mem::MaybeUninit::::uninit(); 6 * out.lane_stride() / 8]; - let core_types::node::BatchStatus::Filled(batch, ..) = node.eval_batch(&scoped, 0..6, Some(&mut scratch)) else { + let core_types::node::BatchStatus::Filled(batch, ..) = node.eval_batch(&scoped, 0..6, Some(&mut scratch), &frames) else { panic!("expected a filled batch"); }; assert_eq!(batch.len(), 6); @@ -1493,7 +1519,7 @@ mod tests { let ctx = ContextImpl::root(&scope); let layout = Layout::default().with_writes(1, core_types::record::element_write::(), &[core_types::record::FieldWrite::of::(0)]); - reserve_for(&[&layout]); + let frames = frames_for(&[&layout]); let content = LeveledTransformSource { layout: layout.clone(), rows: [(1., 10.), (2., 30.), (3., 20.)] @@ -1512,7 +1538,7 @@ mod tests { let scoped = ctx.promoted(&head, 0); let mut scratch = vec![std::mem::MaybeUninit::::uninit(); 6 * out.lane_stride() / 8]; - let core_types::node::BatchStatus::Filled(batch, ..) = node.eval_batch(&scoped, 0..6, Some(&mut scratch)) else { + let core_types::node::BatchStatus::Filled(batch, ..) = node.eval_batch(&scoped, 0..6, Some(&mut scratch), &frames) else { panic!("expected a filled batch"); }; assert_eq!(batch.len(), 6); @@ -1530,7 +1556,7 @@ mod tests { let leveled_content = repeat_opacity_layout(&base); let (count_edge, count_layout) = lifted_value(2u32); let (reverse_edge, reverse_layout) = lifted_value(false); - reserve_for(&[&base, &leveled_content, &count_layout, &reverse_layout]); + let frames = frames_for(&[&base, &leveled_content, &count_layout, &reverse_layout]); // Element = the outer copy, so the total fold sums across both copies. let content = install( @@ -1565,7 +1591,7 @@ mod tests { let node = install(SumNode::new(nested, &two_level), sum_layout_meta(), &[Some(&two_level)]); let out = Node::::layout(&node).clone(); assert_eq!(out.depth, 0, "the fold consumes the whole wire"); - let GPoll::Final(served) = core_types::record::capture(&node, &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&node, &ctx, &frames) else { panic!("expected a final record"); }; // Two copies of three lanes, each lane the copy index: 0 * 3 + 1 * 3. @@ -1584,7 +1610,7 @@ mod tests { let (count_edge, count_layout) = lifted_value(2u32); let (reverse_edge, reverse_layout) = lifted_value(false); let out = f64_layout(&[]); - reserve_for(&[&base, &leveled_content, &count_layout, &reverse_layout, &out]); + let frames = frames_for(&[&base, &leveled_content, &count_layout, &reverse_layout, &out]); let content = install(RepeatOpacityNode::new(bare_source(&base, 7.), ValueSource::new(3u32), &base), repeat_opacity_layout_meta(), &[Some(&base)]); let meta = core_types::record::LayoutMeta { @@ -1612,7 +1638,7 @@ mod tests { assert_eq!(two_level.depth, 2); let node = install_flip(SumNestedNode::new(nested, &two_level), &out); - let GPoll::Final(served) = core_types::record::capture(&node, &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&node, &ctx, &frames) else { panic!("expected a final record"); }; // Two copies of three lanes of 7: the nested fold flattens 2 x 3. @@ -1621,6 +1647,7 @@ mod tests { #[test] fn read_row_exposes_the_vararg_items_as_lanes() { + let frames = core_types::record::test_frames(1 << 16); use core_types::Color; let arena = Arena::new(1024).unwrap(); @@ -1632,17 +1659,17 @@ mod tests { let out = Node::::layout(&node).clone(); assert_eq!(out.depth, 1); // No row pushed: an empty level, matching the legacy empty-list return. - assert_eq!(node.extent_at(&ctx, 0), GPoll::Final(Extent::Exactly(0))); + assert_eq!(node.extent_at(&ctx, 0, &frames), GPoll::Final(Extent::Exactly(0))); let mut item = core_types::list::List::new_from_element(Color::BLACK); item.push(core_types::list::Item::new_from_element(Color::WHITE)); let scoped = ctx.push_vararg(&item); let base = scoped.ctx(); - assert_eq!(node.extent_at(&base, 0), GPoll::Final(Extent::Exactly(2))); + assert_eq!(node.extent_at(&base, 0, &frames), GPoll::Final(Extent::Exactly(2))); let head = base.index_head(); for (lane, expected) in [(0u64, Color::BLACK), (1, Color::WHITE)] { - let GPoll::Final(served) = core_types::record::capture(&node, &base.promoted(&head, lane)) else { + let GPoll::Final(served) = core_types::record::capture(&node, &base.promoted(&head, lane), &frames) else { panic!("expected a final record at lane {lane}"); }; assert_eq!(served.element::(), expected, "lane {lane}"); @@ -1657,7 +1684,7 @@ mod tests { let ctx = ContextImpl::root(&scope); let source_layout = leveled_f64_layout(&[]); - reserve_for(&[&source_layout]); + let frames = frames_for(&[&source_layout]); let source = LeveledSourceNode { layout: source_layout.clone(), elements: vec![10., 11.], @@ -1674,8 +1701,8 @@ mod tests { let head = ctx.index_head(); for (lane, element) in [(0u64, 10.), (1, 11.)] { - let _lane_scope = unsafe { stack::ScopeGuard::enter() }; - let GPoll::Final(value) = core_types::record::serve_edge(&node, &ctx.promoted(&head, lane)) else { + let lane_frames = frames.scope(); + let GPoll::Final(value) = core_types::record::serve_edge(&node, &ctx.promoted(&head, lane), &lane_frames) else { panic!("expected a final record at lane {lane}"); }; let rec = out.rec(&value); @@ -1693,7 +1720,7 @@ mod tests { let leveled = leveled_f64_layout(&[]); let out = f64_layout(&[]); - reserve_for(&[&leveled, &out]); + let frames = frames_for(&[&leveled, &out]); let (count_edge, count_layout) = lifted_value(2u32); let (reverse_edge, reverse_layout) = lifted_value(false); let meta = core_types::record::LayoutMeta { @@ -1724,7 +1751,7 @@ mod tests { // flat index decomposes across both copies. let head = ctx.index_head(); for (lane, expected) in [(0u64, 0.), (7, 2.), (9, 4.)] { - let GPoll::Final(served) = core_types::record::capture(&repeat, &ctx.promoted(&head, lane)) else { + let GPoll::Final(served) = core_types::record::capture(&repeat, &ctx.promoted(&head, lane), &frames) else { panic!("expected a final record at lane {lane}"); }; assert_eq!(served.element::(), expected, "lane {lane}"); @@ -1733,7 +1760,7 @@ mod tests { // A full fold over the composite drains through the structure node; // the partial fold stays excluded with M3. let node = install_flip(SumNestedNode::new(repeat, &two_level), &out); - let GPoll::Final(served) = core_types::record::capture(&node, &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&node, &ctx, &frames) else { panic!("expected a final record"); }; assert_eq!(served.element::(), 20.); @@ -1749,10 +1776,10 @@ mod tests { let source = core_types::value::LeveledValueSource::new(vec![1.5, 2.25, 3.75]); let leveled = Node::::layout(&source).clone(); let out = f64_layout(&[]); - reserve_for(&[&leveled, &out]); + let frames = frames_for(&[&leveled, &out]); let node = install_flip(SumNode::new(source, &leveled), &out); - let GPoll::Final(served) = core_types::record::capture(&node, &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&node, &ctx, &frames) else { panic!("expected a final record"); }; assert_eq!(served.element::(), 7.5); @@ -1767,14 +1794,14 @@ mod tests { let leveled = leveled_f64_layout(&[]); let out = f64_layout(&[]); - reserve_for(&[&leveled, &out]); + let frames = frames_for(&[&leveled, &out]); // 5 lanes end inside the first guess; 20 force a full first fill, a // hint-seeded regrow, and a short second fill. for count in [5usize, 20] { let source = DrainSourceNode { layout: leveled.clone(), count }; let node = install_flip(SumNode::new(source, &leveled), &out); - let GPoll::Final(served) = core_types::record::capture(&node, &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&node, &ctx, &frames) else { panic!("expected a final record at count {count}"); }; let expected = (count * (count - 1) / 2) as f64; @@ -1792,13 +1819,13 @@ mod tests { let base = f64_layout(&[]); let leveled = repeat_opacity_layout(&base); let out = f64_layout(&[]); - reserve_for(&[&base, &leveled, &out]); + let frames = frames_for(&[&base, &leveled, &out]); let repeat = install(RepeatOpacityNode::new(bare_source(&base, 7.), ValueSource::new(3u32), &base), repeat_opacity_layout_meta(), &[Some(&base)]); let node = install_flip(SumNode::new(repeat, &leveled), &out); assert_eq!(Node::::layout(&node).depth, 0, "the reducer collapsed the rank level"); - let GPoll::Final(served) = core_types::record::capture(&node, &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&node, &ctx, &frames) else { panic!("expected a final record"); }; // sum(repeat(3, 7)) folds three copies of the element back to a scalar. @@ -1816,7 +1843,7 @@ mod tests { let (count_edge, count_layout) = lifted_value(4u32); let (reverse_edge, reverse_layout) = lifted_value(false); let out = f64_layout(&[]); - reserve_for(&[&base, &count_layout, &out]); + let frames = frames_for(&[&base, &count_layout, &out]); let meta = core_types::record::LayoutMeta { sources: vec![0], @@ -1842,7 +1869,7 @@ mod tests { let leveled = Node::::layout(&repeat).clone(); let node = install_flip(SumNode::new(repeat, &leveled), &out); - let GPoll::Final(served) = core_types::record::capture(&node, &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&node, &ctx, &frames) else { panic!("expected a final record"); }; // Every copy evaluates at its own index, so the lanes must be distinct @@ -1874,7 +1901,7 @@ mod tests { let source_layout = f64_layout(&[]); let modified = multiply_opacity_layout(&source_layout); let stacked = multiply_opacity_layout(&modified); - reserve_for(&[&source_layout, &modified, &stacked]); + let frames = frames_for(&[&source_layout, &modified, &stacked]); let chain = install( MultiplyOpacityNode::new( @@ -1890,7 +1917,7 @@ mod tests { &[Some(&modified)], ); assert_eq!(Node::::layout(&chain), &stacked); - let GPoll::Final(served) = core_types::record::capture(&chain, &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&chain, &ctx, &frames) else { panic!("expected a final record"); }; assert_eq!(served.element::(), 2.); @@ -1906,10 +1933,10 @@ mod tests { let source_layout = f64_layout(&[]); let measured = measure_layout(&source_layout); - reserve_for(&[&source_layout, &measured]); + let frames = frames_for(&[&source_layout, &measured]); let chain = install(MeasureNode::new(bare_source(&source_layout, -2.), &source_layout), measure_layout_meta(), &[Some(&source_layout)]); - let GPoll::Final(served) = core_types::record::capture(&chain, &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&chain, &ctx, &frames) else { panic!("expected a final record"); }; assert_eq!(served.element::(), -2.); @@ -1926,7 +1953,7 @@ mod tests { let source_layout = f64_layout(&[]); let modified = multiply_opacity_layout(&source_layout); let measured = measure_layout(&modified); - reserve_for(&[&source_layout, &modified, &measured]); + let frames = frames_for(&[&source_layout, &modified, &measured]); let chain = install( MeasureNode::new( @@ -1940,7 +1967,7 @@ mod tests { measure_layout_meta(), &[Some(&modified)], ); - let GPoll::Final(served) = core_types::record::capture(&chain, &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&chain, &ctx, &frames) else { panic!("expected a final record"); }; assert_eq!(served.attr::(), 0.5); @@ -1957,10 +1984,10 @@ mod tests { let source_layout = f64_layout(&[]); let modified = multiply_opacity_layout(&source_layout); let shaded = shade_layout(&modified); - reserve_for(&[&source_layout, &modified, &shaded]); + let frames = frames_for(&[&source_layout, &modified, &shaded]); let bare = install(ShadeNode::new(bare_source(&source_layout, 4.), &source_layout), shade_layout_meta(), &[Some(&source_layout)]); - let GPoll::Final(served) = core_types::record::capture(&bare, &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&bare, &ctx, &frames) else { panic!("expected a final record"); }; assert_eq!(served.element::(), 4.); @@ -1977,7 +2004,7 @@ mod tests { shade_layout_meta(), &[Some(&modified)], ); - let GPoll::Final(served) = core_types::record::capture(&chain, &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&chain, &ctx, &frames) else { panic!("expected a final record"); }; assert_eq!(served.element::(), 2.); @@ -1995,10 +2022,14 @@ mod tests { let f64_faded = fade_layout(&f64_source); let u32_source = Layout::default().with_writes(0, core_types::record::element_write::(), &[]); let u32_faded = fade_layout(&u32_source); - reserve_for(&[&f64_source, &f64_faded, &u32_source, &u32_faded]); + let frames = frames_for(&[&f64_source, &f64_faded, &u32_source, &u32_faded]); - let wide = install(FadeNode::new(bare_source(&f64_source, 8.), ValueSource::new(0.5), &f64_source), fade_layout_meta(), &[Some(&f64_source)]); - let GPoll::Final(served) = core_types::record::capture(&wide, &ctx) else { + let wide = install( + FadeNode::new(bare_source(&f64_source, 8.), ValueSource::new(0.5), &f64_source), + fade_layout_meta(), + &[Some(&f64_source)], + ); + let GPoll::Final(served) = core_types::record::capture(&wide, &ctx, &frames) else { panic!("expected a final record"); }; assert_eq!(served.element::(), 8.); @@ -2018,7 +2049,7 @@ mod tests { fade_layout_meta(), &[Some(&u32_source)], ); - let GPoll::Final(served) = core_types::record::capture(&narrow, &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&narrow, &ctx, &frames) else { panic!("expected a final record"); }; assert_eq!(served.element::(), 7); @@ -2033,11 +2064,11 @@ mod tests { let ctx = ContextImpl::root(&scope); let layout = source_opacity_layout(); - reserve_for(&[&layout]); + let frames = frames_for(&[&layout]); let node = install(SourceOpacityNode::new(ValueSource::new(()), ValueSource::new(3.), ValueSource::new(0.25)), source_opacity_layout_meta(), &[]); assert_eq!(Node::::layout(&node), &layout); - let GPoll::Final(served) = core_types::record::capture(&node, &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&node, &ctx, &frames) else { panic!("expected a final record"); }; assert_eq!(served.element::(), 3.); @@ -2053,7 +2084,7 @@ mod tests { let source_layout = f64_layout(&[]); let modified = multiply_opacity_layout(&source_layout); - reserve_for(&[&source_layout, &modified]); + let frames = frames_for(&[&source_layout, &modified]); let chain = install( MultiplyOpacityNode::new( @@ -2069,7 +2100,7 @@ mod tests { multiply_opacity_layout_meta(), &[Some(&source_layout)], ); - let GPoll::Partial(served) = core_types::record::capture(&chain, &ctx) else { + let GPoll::Partial(served) = core_types::record::capture(&chain, &ctx, &frames) else { panic!("expected a partial record"); }; assert_eq!(served.attr::(), 0.5); @@ -2084,14 +2115,14 @@ mod tests { let source_layout = f64_layout(&[]); let modified = checked_multiply_opacity_layout(&source_layout); - reserve_for(&[&source_layout, &modified]); + let frames = frames_for(&[&source_layout, &modified]); let ok = install( CheckedMultiplyOpacityNode::new(bare_source(&source_layout, 1.), ValueSource::new(0.5), &source_layout), checked_multiply_opacity_layout_meta(), &[Some(&source_layout)], ); - let GPoll::Final(served) = core_types::record::capture(&ok, &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&ok, &ctx, &frames) else { panic!("expected a final record"); }; assert_eq!(served.attr::(), 0.5); @@ -2101,7 +2132,7 @@ mod tests { checked_multiply_opacity_layout_meta(), &[Some(&source_layout)], ); - let GPoll::Error(error) = core_types::record::serve_edge(&failing, &ctx) else { + let GPoll::Error(error) = core_types::record::serve_edge(&failing, &ctx, &frames) else { panic!("expected an error"); }; assert!(error.kind == "negative factor"); @@ -2117,7 +2148,7 @@ mod tests { let source_layout = f64_layout(&[]); let modified = multiply_opacity_layout(&source_layout); let scaled = scale_layout(&modified); - reserve_for(&[&source_layout, &modified, &scaled]); + let frames = frames_for(&[&source_layout, &modified, &scaled]); let chain = install( ScaleNode::new( @@ -2132,7 +2163,7 @@ mod tests { scale_layout_meta(), &[Some(&modified)], ); - let GPoll::Final(served) = core_types::record::capture(&chain, &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&chain, &ctx, &frames) else { panic!("expected a final record"); }; assert_eq!(served.element::(), 6.); @@ -2149,7 +2180,7 @@ mod tests { let carrier_layout = f64_layout(&["opacity"]); let secondary_layout = f64_layout(&["opacity"]); let transferred = transfer_opacity_layout(&carrier_layout); - reserve_for(&[&carrier_layout, &secondary_layout, &transferred]); + let frames = frames_for(&[&carrier_layout, &secondary_layout, &transferred]); let chain = install( TransferOpacityNode::new( @@ -2161,7 +2192,7 @@ mod tests { transfer_opacity_layout_meta(), &[Some(&carrier_layout), None], ); - let GPoll::Final(served) = core_types::record::capture(&chain, &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&chain, &ctx, &frames) else { panic!("expected a final record"); }; assert_eq!(served.element::(), 5.); @@ -2178,7 +2209,7 @@ mod tests { transfer_opacity_layout_meta(), &[Some(&carrier_layout), None], ); - let GPoll::Final(served) = core_types::record::capture(&defaulted, &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&defaulted, &ctx, &frames) else { panic!("expected a final record"); }; assert_eq!(served.attr::(), 0.5, "an absent secondary attribute reads its default"); @@ -2194,7 +2225,7 @@ mod tests { let source_layout = f64_layout(&["opacity"]); let factor = ValueSource::new(3.); let factor_layout = Node::::layout(&factor).clone(); - reserve_for(&[&source_layout]); + let frames = frames_for(&[&source_layout]); let node = install( BoostNode::new(f64_record_source(&source_layout, 2., vec![("opacity", 0.25)]), factor, &source_layout, &factor_layout), @@ -2203,7 +2234,7 @@ mod tests { ); let out_layout = Node::::layout(&node).clone(); out_layout.offset_of(Opacity::NAME, 0).expect("the primary input's fields pass through to the output"); - let GPoll::Final(served) = core_types::record::capture(&node, &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&node, &ctx, &frames) else { panic!("expected a final record"); }; assert_eq!(served.element::(), 6.); @@ -2219,7 +2250,7 @@ mod tests { let source_layout = f64_layout(&["opacity"]); let (factor, factor_layout) = lifted_value(3.); - reserve_for(&[&source_layout]); + let frames = frames_for(&[&source_layout]); let node = install( BoostPollNode::new(f64_record_source(&source_layout, 2., vec![("opacity", 0.25)]), factor, &source_layout, &factor_layout), @@ -2228,7 +2259,7 @@ mod tests { ); let out_layout = Node::::layout(&node).clone(); out_layout.offset_of(Opacity::NAME, 0).expect("the primary input's fields pass through the poll kernel"); - let GPoll::Final(served) = core_types::record::capture(&node, &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&node, &ctx, &frames) else { panic!("expected a final record"); }; assert_eq!(served.element::(), 6.); @@ -2245,7 +2276,7 @@ mod tests { let carrier_layout = f64_layout(&["opacity"]); let by_layout = f64_layout(&["opacity", "length"]); assert!(by_layout.frame_bytes() != 0, "the borrow must point into a spilled frame to exercise the park"); - reserve_for(&[&carrier_layout, &by_layout]); + let frames = frames_for(&[&carrier_layout, &by_layout]); let node = install( OffsetNode::new( @@ -2257,7 +2288,7 @@ mod tests { offset_layout_meta(), &[Some(&carrier_layout)], ); - let GPoll::Final(served) = core_types::record::capture(&node, &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&node, &ctx, &frames) else { panic!("expected a final record"); }; assert_eq!(served.element::(), 42., "the parked borrow survives the carrier evaluation reusing its frame"); @@ -2284,7 +2315,7 @@ mod tests { let source_layout = f64_layout(&["opacity"]); let (runtime, runtime_layout) = lifted_value(core_types::runtime::RuntimeHandle(std::sync::Arc::new(InlineRuntime))); let (source_id, source_id_layout) = lifted_value(7 as SourceId); - reserve_for(&[&source_layout]); + let frames = frames_for(&[&source_layout]); let node = install( DoubleAsyncNode::new( @@ -2301,13 +2332,13 @@ mod tests { let out_layout = Node::::layout(&node).clone(); out_layout.offset_of(Opacity::NAME, 0).expect("the carrier's fields pass through the async source"); - let GPoll::Final(served) = core_types::record::capture(&node, &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&node, &ctx, &frames) else { panic!("an inline completion is final on the spawning eval"); }; assert_eq!(served.element::(), 6.); assert_eq!(served.attr::(), 0.25); - let GPoll::Final(served) = core_types::record::capture(&node, &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&node, &ctx, &frames) else { panic!("a slot hit is final"); }; assert_eq!(served.element::(), 6., "the slot hit replays the element"); @@ -2324,7 +2355,7 @@ mod tests { let unit = ValueSource::new(()); let unit_layout = Node::::layout(&unit).clone(); let content_layout = f64_layout(&["opacity"]); - reserve_for(&[&content_layout]); + let frames = frames_for(&[&content_layout]); let run = |opacity: Option| { let evals = std::sync::Arc::new(std::sync::atomic::AtomicU32::new(0)); @@ -2345,7 +2376,7 @@ mod tests { ), &f64_layout(&[]), ); - let GPoll::Final(value) = core_types::record::serve_edge(&node, &ctx) else { + let GPoll::Final(value) = core_types::record::serve_edge(&node, &ctx, &frames) else { panic!("expected a final record"); }; let element = unsafe { Node::::layout(&node).rec(&value).element::() }; @@ -2369,7 +2400,7 @@ mod tests { let stripped = strip_opacity_layout(&modified); assert!(stripped.offset_of(Opacity::NAME, 0).is_none(), "the removed name leaves the output layout"); let shaded = shade_layout(&stripped); - reserve_for(&[&source_layout, &modified, &stripped, &shaded]); + let frames = frames_for(&[&source_layout, &modified, &stripped, &shaded]); let chain = install( ShadeNode::new( @@ -2390,7 +2421,7 @@ mod tests { shade_layout_meta(), &[Some(&stripped)], ); - let GPoll::Final(served) = core_types::record::capture(&chain, &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&chain, &ctx, &frames) else { panic!("expected a final record"); }; assert_eq!(served.element::(), 4., "a read after the removal yields the declared default"); @@ -2406,14 +2437,14 @@ mod tests { let source_layout = f64_layout(&["opacity", "length"]); let relengthed = relength_layout(&source_layout); assert!(relengthed.offset_of(Opacity::NAME, 0).is_none()); - reserve_for(&[&source_layout, &relengthed]); + let frames = frames_for(&[&source_layout, &relengthed]); let chain = install( RelengthNode::new(f64_record_source(&source_layout, 3., vec![("opacity", 0.25), ("length", 9.)]), &source_layout), relength_layout_meta(), &[Some(&source_layout)], ); - let GPoll::Final(served) = core_types::record::capture(&chain, &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&chain, &ctx, &frames) else { panic!("expected a final record"); }; assert_eq!(served.element::(), 3.); @@ -2430,7 +2461,7 @@ mod tests { let source_layout = f64_layout(&[]); let labeled = label_layout(&source_layout); let relabeled = label_layout(&labeled); - reserve_for(&[&source_layout, &labeled, &relabeled]); + let frames = frames_for(&[&source_layout, &labeled, &relabeled]); let chain = install( LabelNode::new( @@ -2445,7 +2476,7 @@ mod tests { label_layout_meta(), &[Some(&labeled)], ); - let GPoll::Final(value) = core_types::record::serve_edge(&chain, &ctx) else { + let GPoll::Final(value) = core_types::record::serve_edge(&chain, &ctx, &frames) else { panic!("expected a final record"); }; let rec = relabeled.rec(&value); @@ -2481,13 +2512,13 @@ mod tests { let generations = []; let layout = f64_layout(&["opacity"]); - reserve_for(&[&layout, &layout]); + let frames = frames_for(&[&layout, &layout]); let monitor = crate::memo::MonitorNode::new(f64_record_source(&layout, 4., vec![("opacity", 0.25)]), &layout); let scope = scope_fixture(&generations, &arena); { let ctx = ContextImpl::root(&scope); - let GPoll::Final(served) = core_types::record::capture(&monitor, &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&monitor, &ctx, &frames) else { panic!("expected a final record"); }; assert_eq!(served.element::(), 4.); @@ -2496,7 +2527,7 @@ mod tests { let io = Node::::serialize(&monitor).unwrap(); let snapshot = io.downcast_ref::().expect("the monitor serializes its context snapshot"); let ctx = snapshot.rehydrate(&scope).expect("the arena holds the chains"); - let GPoll::Final(served) = core_types::record::capture(&monitor, &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&monitor, &ctx, &frames) else { panic!("expected a final record"); }; assert_eq!(served.element::(), 4.); @@ -2513,7 +2544,7 @@ mod tests { let layout_a = f64_layout(&["opacity"]); let layout_b = f64_layout(&["length"]); let union = Layout::union(&[&layout_a, &layout_b]); - reserve_for(&[&layout_a, &layout_b, &union, &union]); + let frames = frames_for(&[&layout_a, &layout_b, &union, &union]); let taken = |second: bool| { let (condition, condition_layout) = lifted_value(second); @@ -2526,14 +2557,14 @@ mod tests { ) }; - let GPoll::Final(served) = core_types::record::capture(&taken(false), &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&taken(false), &ctx, &frames) else { panic!("expected a final record"); }; assert_eq!(served.element::(), 1.); assert_eq!(served.field::("opacity", 0), 0.5); assert_eq!(served.field::("length", 0), 0.); - let GPoll::Final(served) = core_types::record::capture(&taken(true), &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&taken(true), &ctx, &frames) else { panic!("expected a final record"); }; assert_eq!(served.element::(), 3.); @@ -2551,7 +2582,7 @@ mod tests { let layout_a = f64_layout(&["opacity"]); let layout_b = f64_layout(&["length"]); let union = Layout::union(&[&layout_a, &layout_b]); - reserve_for(&[&layout_a, &layout_b, &union, &union]); + let frames = frames_for(&[&layout_a, &layout_b, &union, &union]); let (condition, condition_layout) = lifted_value(false); let chain = HoldFirstNode::new( @@ -2562,7 +2593,7 @@ mod tests { &condition_layout, ); - let GPoll::Final(served) = core_types::record::capture(&chain, &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&chain, &ctx, &frames) else { panic!("expected a final record"); }; assert_eq!(served.element::(), 1.); @@ -2575,7 +2606,7 @@ mod tests { } impl Node for RealTimeProbe { - fn serve<'e, 'l>(&self, input: &C, slot: FrameClaim<'l>) -> GPoll> + fn serve<'e, 'l>(&self, input: &C, slot: FrameClaim<'e, 'l>) -> GPoll> where C: ExtractArena, { @@ -2583,11 +2614,13 @@ mod tests { Some(_) => 1., None => 0., }; - let mut frame = core_types::record::FrameBuilder::new(&self.layout, ExtractArena::arena(input)); - frame.element(element); - let Some(value) = frame.finish() else { return GPoll::arena_exhausted() }; - // SAFETY: the builder served a record of this node's layout. - GPoll::Final(unsafe { slot.forward(&value) }) + let mut frame = slot; + let arena = ExtractArena::arena(input); + if frame.element(element, arena).is_none() { + return GPoll::arena_exhausted(); + } + // SAFETY: the writes above complete the record of this layout. + GPoll::Final(unsafe { frame.finish_served() }) } fn layout(&self) -> &Layout { @@ -2605,13 +2638,13 @@ mod tests { let ctx = ContextImpl::root(&scope); let layout = f64_layout(&[]); - reserve_for(&[&layout]); + let frames = frames_for(&[&layout]); let probed = |features: ContextFeatures| { let (modification, modification_layout) = lifted_value(ContextModification::from_sources(features, &[])); let node = crate::context_modification::ContextModificationNode::new(RealTimeProbe { layout: layout.clone() }, modification, &layout, &modification_layout); assert_eq!(Node::::layout(&node), &layout); - let GPoll::Final(served) = core_types::record::capture(&node, &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&node, &ctx, &frames) else { panic!("expected a final record"); }; served.element::() @@ -2631,7 +2664,7 @@ mod tests { let ctx = ContextImpl::root(&scope); let layout = f64_layout(&["opacity"]); - reserve_for(&[&layout]); + let frames = frames_for(&[&layout]); let (modification, modification_layout) = lifted_value(ContextModification::from_sources(ContextFeatures::all(), &[])); let node = crate::context_modification::ContextModificationNode::new( @@ -2646,7 +2679,7 @@ mod tests { &modification_layout, ); - let GPoll::Partial(served) = core_types::record::capture(&node, &ctx) else { + let GPoll::Partial(served) = core_types::record::capture(&node, &ctx, &frames) else { panic!("expected a partial record"); }; assert_eq!(served.element::(), 4.); @@ -2655,6 +2688,7 @@ mod tests { #[test] fn droppable_elements_park_and_clone_out() { + let frames = core_types::record::test_frames(1 << 16); let arena = Arena::new(1024).unwrap(); let generations = []; let scope = scope_fixture(&generations, &arena); @@ -2664,7 +2698,7 @@ mod tests { let layout = Node::::layout(&lift).clone(); let chain = core_types::record::RecordExtract::::new(lift, &layout); - let GPoll::Final(text) = chain.eval(&ctx) else { + let GPoll::Final(text) = chain.eval(&ctx, &frames) else { panic!("expected a final value"); }; assert_eq!(text, "parked"); @@ -2678,17 +2712,21 @@ mod tests { let ctx = ContextImpl::root(&scope); let layout = f64_layout(&["opacity", "length"]); - reserve_for(&[&layout]); - let base = stack::push(0); - stack::pop(base); + let frames = frames_for(&[&layout]); + let base = { + let probe = frames.scope(); + // SAFETY: nothing reads the probe's record; only its address is taken. + let value = unsafe { probe.claim(&layout).finish() }; + layout.rec(&value).ptr() + }; let chain = ForwardRecordNode::new(RecordSource::new(f64_record_source(&layout, 4., vec![("opacity", 0.25)]), &layout, &layout.clone()), &layout); - let GPoll::Final(value) = core_types::record::serve_edge(&chain, &ctx) else { + let GPoll::Final(value) = core_types::record::serve_edge(&chain, &ctx, &frames) else { panic!("expected a final record"); }; let rec = layout.rec(&value); - assert_eq!(rec.ptr(), base.cast_const()); + assert_eq!(rec.ptr(), base, "the served record's pointer is the claimed slot's"); assert_eq!(unsafe { rec.element::() }, 4.); assert_eq!(unsafe { rec.read::(layout.offset_of("opacity", 0).unwrap()) }, 0.25); } @@ -2701,7 +2739,7 @@ mod tests { let ctx = ContextImpl::root(&scope); let layout = f64_layout(&["opacity"]); - reserve_for(&[&layout]); + let frames = frames_for(&[&layout]); let chain = ForwardRecordNode::new( RecordSource::new( @@ -2717,7 +2755,7 @@ mod tests { &layout, ); - let GPoll::Partial(served) = core_types::record::capture(&chain, &ctx) else { + let GPoll::Partial(served) = core_types::record::capture(&chain, &ctx, &frames) else { panic!("expected a partial record"); }; assert_eq!(served.element::(), 4.); @@ -2732,6 +2770,7 @@ mod tests { #[test] fn record_memo_replays_the_deep_copy_on_a_context_hit() { + let frames = core_types::record::test_frames(1 << 16); let arena = Arena::new(1024).unwrap(); let generations = []; let scope = scope_fixture(&generations, &arena); @@ -2742,11 +2781,11 @@ mod tests { let layout = Node::::layout(&lift).clone(); let memo = crate::memo::MemoizeNode::new(lift, &layout); - let GPoll::Final(served) = core_types::record::capture(&memo, &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&memo, &ctx, &frames) else { panic!("expected a final record"); }; assert_eq!(served.element::(), 21.); - let GPoll::Final(served) = core_types::record::capture(&memo, &ctx) else { + let GPoll::Final(served) = core_types::record::capture(&memo, &ctx, &frames) else { panic!("expected a final record"); }; assert_eq!(served.element::(), 21.); @@ -2761,7 +2800,7 @@ mod tests { let ctx = ContextImpl::root(&scope); let layout = f64_layout(&["opacity"]); - reserve_for(&[&layout, &layout]); + let frames = frames_for(&[&layout, &layout]); let source = RecordSourceNode { layout: layout.clone(), @@ -2771,10 +2810,10 @@ mod tests { }; let memo = crate::memo::MemoizeNode::new(source, &layout); - let GPoll::Partial(_) = core_types::record::serve_edge(&memo, &ctx) else { + let GPoll::Partial(_) = core_types::record::serve_edge(&memo, &ctx, &frames) else { panic!("expected a partial record"); }; - let GPoll::Partial(served) = core_types::record::capture(&memo, &ctx) else { + let GPoll::Partial(served) = core_types::record::capture(&memo, &ctx, &frames) else { panic!("expected the replay to keep the partial finality"); }; assert_eq!(served.field::("opacity", 0), 0.5); @@ -2786,7 +2825,7 @@ mod tests { let source_layout = f64_layout(&[]); let labeled = label_layout(&source_layout); - reserve_for(&[&labeled, &labeled]); + let frames = frames_for(&[&labeled, &labeled]); let chain = install( LabelNode::new(bare_source(&source_layout, 1.), ValueSource::new(String::from("a")), &source_layout), @@ -2799,7 +2838,8 @@ mod tests { { let scope = scope_fixture(&generations, &first_arena); let ctx = ContextImpl::root(&scope); - let GPoll::Final(_) = core_types::record::serve_edge(&memo, &ctx) else { + let mut first = frames.reborrow(); + let GPoll::Final(_) = core_types::record::serve_edge(&memo, &ctx, &mut first) else { panic!("expected a final record"); }; } @@ -2807,7 +2847,7 @@ mod tests { let replay_arena = Arena::new(1024).unwrap(); let scope = scope_fixture(&generations, &replay_arena); let ctx = ContextImpl::root(&scope); - let GPoll::Final(value) = core_types::record::serve_edge(&memo, &ctx) else { + let GPoll::Final(value) = core_types::record::serve_edge(&memo, &ctx, &frames) else { panic!("expected a final record"); }; let rec = labeled.rec(&value); diff --git a/node-graph/nodes/graphic/src/record.rs b/node-graph/nodes/graphic/src/record.rs index b9ffd3c4cf..bcb7508584 100644 --- a/node-graph/nodes/graphic/src/record.rs +++ b/node-graph/nodes/graphic/src/record.rs @@ -225,7 +225,7 @@ mod tests { use core_types::context::{ContextImpl, EvalScope, ExtractArena}; use core_types::list::{Item, List}; use core_types::node::Node; - use core_types::record::{self, FrameClaim, Layout, RecordSource, Served, stack}; + use core_types::record::{self, FrameClaim, Layout, RecordSource, Served}; use core_types::value::ValueSource; struct GraphicSource { @@ -234,20 +234,22 @@ mod tests { } impl Node for GraphicSource { - fn serve<'e, 'l>(&self, input: &C, slot: FrameClaim<'l>) -> GPoll> + fn serve<'e, 'l>(&self, input: &C, slot: FrameClaim<'e, 'l>) -> GPoll> where C: ExtractArena, { let (graphic, transform) = &self.rows[input.innermost_index() as usize % self.rows.len()]; - let mut frame = record::FrameBuilder::new(&self.layout, ExtractArena::arena(input)); - frame.element(graphic.clone()); - frame.attr::(*transform); - let Some(value) = frame.finish() else { return GPoll::arena_exhausted() }; - // SAFETY: the builder served a record of this node's layout. - GPoll::Final(unsafe { slot.forward(&value) }) + let mut frame = slot; + let arena = ExtractArena::arena(input); + if frame.element(graphic.clone(), arena).is_none() { + return GPoll::arena_exhausted(); + } + write_attr_at::(&mut frame, &self.layout, *transform); + // SAFETY: the writes above complete the record of this layout. + GPoll::Final(unsafe { frame.finish_served() }) } - fn extent_at<'x>(&self, _input: &C, _level: u8) -> GPoll + fn extent_at<'x>(&self, _input: &C, _level: u8, _frames: &core_types::record::Frames<'x>) -> GPoll where C: ExtractArena, { @@ -259,9 +261,28 @@ mod tests { } } + /// Writes a field at the layout's resolved offset, the wiring-proven pairing + /// a generated node performs. + fn write_field_at(frame: &mut FrameClaim<'_, '_>, layout: &Layout, name: &str, level: u8, value: T) { + let field = layout + .fields + .iter() + .find(|field| field.name == name && field.level == level) + .expect("the layout carries the written field"); + assert_eq!(field.type_id, std::any::TypeId::of::(), "the field was declared at this value type"); + // SAFETY: the offset is this layout's own, at the field's declared type. + unsafe { frame.attr_at(field.offset, value) }; + } + + /// [`write_field_at`] for a census marker at level 0. + fn write_attr_at(frame: &mut FrameClaim<'_, '_>, layout: &Layout, value: A::Value<'static>) + where + A::Value<'static>: Copy + 'static, + { + write_field_at(frame, layout, A::NAME, 0, value); + } fn scope_fixture<'a>(generations: &'a [(SourceId, u64)], arena: &'a Arena) -> EvalScope<'a> { - // SAFETY: between evaluations, nothing served on the stack is live. - unsafe { stack::reserve(1 << 16); } EvalScope::new(Some(0.5), None, None, generations, arena) + EvalScope::new(Some(0.5), None, None, generations, arena) } fn install>>(mut node: N, meta: record::LayoutMeta, inputs: &[Option<&Layout>]) -> N { @@ -294,7 +315,7 @@ mod tests { Graphic::Text(label.to_string()) } - fn group(children: Vec<(Graphic<'static>, DAffine2)>) -> Graphic { + fn group(children: Vec<(Graphic<'static>, DAffine2)>) -> Graphic<'static> { let mut list = List::new(); for (index, (child, transform)) in children.into_iter().enumerate() { list.push(Item::new_from_element(child)); @@ -362,7 +383,7 @@ mod tests { } impl Node for PerRowSource { - fn serve<'e, 'l>(&self, input: &C, slot: FrameClaim<'l>) -> GPoll> + fn serve<'e, 'l>(&self, input: &C, slot: FrameClaim<'e, 'l>) -> GPoll> where C: ExtractArena, { @@ -372,15 +393,17 @@ mod tests { let lane = input.innermost_index(); let graphic = text(&format!("{label}{lane}")); let translated = DAffine2::from_translation(glam::DVec2::new(lane as f64, 0.)); - let mut frame = record::FrameBuilder::new(&self.layout, ExtractArena::arena(input)); - frame.element(graphic); - frame.attr::(translated); - let Some(value) = frame.finish() else { return GPoll::arena_exhausted() }; - // SAFETY: the builder served a record of this node's layout. - GPoll::Final(unsafe { slot.forward(&value) }) + let mut frame = slot; + let arena = ExtractArena::arena(input); + if frame.element(graphic, arena).is_none() { + return GPoll::arena_exhausted(); + } + write_attr_at::(&mut frame, &self.layout, translated); + // SAFETY: the writes above complete the record of this layout. + GPoll::Final(unsafe { frame.finish_served() }) } - fn extent_at<'x>(&self, input: &C, _level: u8) -> GPoll + fn extent_at<'x>(&self, input: &C, _level: u8, _frames: &core_types::record::Frames<'x>) -> GPoll where C: ExtractArena, { @@ -415,6 +438,7 @@ mod tests { #[test] fn map_scans_ragged_rows() { + let frames = core_types::record::test_frames(1 << 16); let arena = Arena::new(1 << 16).unwrap(); let generations = []; let scope = scope_fixture(&generations, &arena); @@ -441,12 +465,12 @@ mod tests { assert_eq!(out.depth, 2); // The extent-fn-less levels report a lower bound; addressing below // proves the lanes are all reachable regardless. - assert_eq!(node.extent_at(&ctx, 1), GPoll::Final(Extent::AtLeast(0))); - assert_eq!(node.extent_at(&ctx, 0), GPoll::Final(Extent::AtLeast(0))); + assert_eq!(node.extent_at(&ctx, 1, &frames.reborrow()), GPoll::Final(Extent::AtLeast(0))); + assert_eq!(node.extent_at(&ctx, 0, &frames.reborrow()), GPoll::Final(Extent::AtLeast(0))); let head = ctx.index_head(); for (lane, &(label, x)) in RAGGED_FLAT.iter().enumerate() { - let GPoll::Final(record) = record::capture(&node, &ctx.promoted(&head, lane as u64)) else { + let GPoll::Final(record) = record::capture(&node, &ctx.promoted(&head, lane as u64), &frames) else { panic!("expected a final record"); }; assert_eq!(text_of(&record.element::()), label, "lane {lane}"); @@ -457,6 +481,7 @@ mod tests { #[test] fn flat_map_matches_flatten_of_map() { + let frames = core_types::record::test_frames(1 << 16); let arena = Arena::new(1 << 16).unwrap(); let generations = []; let scope = scope_fixture(&generations, &arena); @@ -504,18 +529,18 @@ mod tests { assert_eq!(composed_out.depth, 1); // Both spellings report the same lower bound; the lane loop below is // the law. - assert_eq!(flat.extent_at(&ctx, 0), GPoll::Final(Extent::AtLeast(0))); - assert_eq!(composed.extent_at(&ctx, 0), GPoll::Final(Extent::AtLeast(0))); + assert_eq!(flat.extent_at(&ctx, 0, &frames.reborrow()), GPoll::Final(Extent::AtLeast(0))); + assert_eq!(composed.extent_at(&ctx, 0, &frames.reborrow()), GPoll::Final(Extent::AtLeast(0))); let head = ctx.index_head(); for (lane, &(label, x)) in RAGGED_FLAT.iter().enumerate() { let scoped = ctx.promoted(&head, lane as u64); - let GPoll::Final(direct) = record::capture(&flat, &scoped) else { + let GPoll::Final(direct) = record::capture(&flat, &scoped, &frames) else { panic!("expected a final record from flat_map"); }; let direct_label = text_of(&direct.element::()).to_string(); let direct_x: DAffine2 = direct.attr::(); - let GPoll::Final(value) = record::capture(&composed, &scoped) else { + let GPoll::Final(value) = record::capture(&composed, &scoped, &frames) else { panic!("expected a final record from flatten(map)"); }; assert_eq!(text_of(&value.element::()), direct_label, "lane {lane}"); @@ -540,6 +565,7 @@ mod tests { #[test] fn flat_map_batch_matches_per_lane_eval() { + let frames = core_types::record::test_frames(1 << 16); let arena = Arena::new(1 << 16).unwrap(); let generations = []; let scope = scope_fixture(&generations, &arena); @@ -567,14 +593,14 @@ mod tests { let scoped = ctx.promoted(&head, 0); let mut scratch = vec![std::mem::MaybeUninit::::uninit(); 5 * out.lane_stride() / 8]; - let core_types::node::BatchStatus::Filled(batch, ..) = node.eval_batch(&scoped, 0..5, Some(&mut scratch)) else { + let core_types::node::BatchStatus::Filled(batch, ..) = node.eval_batch(&scoped, 0..5, Some(&mut scratch), &frames) else { panic!("expected a filled batch"); }; let batch = batch.into_shared(); assert_eq!(batch.len(), 5); let offset = out.offset_of(::NAME, 0).unwrap(); for lane in 0..5 { - let GPoll::Final(record) = record::capture(&node, &ctx.promoted(&head, lane as u64)) else { + let GPoll::Final(record) = record::capture(&node, &ctx.promoted(&head, lane as u64), &frames) else { panic!("expected a final record"); }; let single = text_of(&record.element::()).to_string(); @@ -587,6 +613,7 @@ mod tests { #[test] fn flatten_expands_one_level() { + let frames = core_types::record::test_frames(1 << 16); let arena = Arena::new(1 << 16).unwrap(); let generations = []; let scope = scope_fixture(&generations, &arena); @@ -596,13 +623,13 @@ mod tests { let node = build!(layout, fixture_rows(), false); let out = Node::::layout(&node).clone(); assert_eq!(out.depth, 1); - assert_eq!(node.extent_at(&ctx, 0), GPoll::Final(Extent::Exactly(3))); + assert_eq!(node.extent_at(&ctx, 0, &frames.reborrow()), GPoll::Final(Extent::Exactly(3))); let head = ctx.index_head(); // Lane 2 is the unexpanded subgroup H, riding as a leaf at G's depth. let expected: [(&str, f64); 2] = [("a", 1.), ("b", 20.5)]; for (lane, &(label, x)) in expected.iter().enumerate() { - let GPoll::Final(record) = record::capture(&node, &ctx.promoted(&head, lane as u64)) else { + let GPoll::Final(record) = record::capture(&node, &ctx.promoted(&head, lane as u64), &frames) else { panic!("expected a final record"); }; assert_eq!(text_of(&record.element::()), label, "lane {lane}"); @@ -610,7 +637,7 @@ mod tests { assert_eq!(transform.translation.x, x, "lane {lane}"); } - let GPoll::Final(record) = record::capture(&node, &ctx.promoted(&head, 2)) else { + let GPoll::Final(record) = record::capture(&node, &ctx.promoted(&head, 2), &frames) else { panic!("expected a final record"); }; let Graphic::Graphic(children) = record.element::() else { @@ -629,13 +656,14 @@ mod tests { #[test] fn a_wire_materializes_into_a_group_for_the_renderer() { + let frames = core_types::record::test_frames(1 << 16); let arena = Arena::new(1 << 16).unwrap(); let generations = []; let scope = scope_fixture(&generations, &arena); let ctx = ContextImpl::root(&scope); let source = core_types::value::LeveledValueSource::new(vec![text("a"), text("b")]); - match graphic_types::boundary::materialize_group(&source, &ctx, &arena) { + match graphic_types::boundary::materialize_group(&source, &ctx, &arena, &frames) { graphic_types::boundary::LevelGroup::Group(group, _) => { let list = graphic_types::graphic::group_to_legacy_list(&group); assert_eq!(list.len(), 2); @@ -646,6 +674,7 @@ mod tests { #[test] fn a_level_batch_converts_to_its_legacy_list() { + let frames = core_types::record::test_frames(1 << 16); let arena = Arena::new(1 << 16).unwrap(); let generations = []; let scope = scope_fixture(&generations, &arena); @@ -653,7 +682,7 @@ mod tests { let source = core_types::value::LeveledValueSource::new(vec![1.5f64, 2.5]); let layout = Node::::layout(&source).clone(); - let record::LevelStatus::Batch(batch, _) = record::materialize_level(&source, &ctx, &arena) else { + let record::LevelStatus::Batch(batch, _) = record::materialize_level(&source, &ctx, &arena, &frames) else { panic!("expected a batch"); }; let legacy = graphic_types::boundary::batch_to_legacy(&layout, batch, &arena).expect("f64 is in the legacy vocabulary"); @@ -665,6 +694,7 @@ mod tests { #[test] fn wrap_collects_the_level_into_a_group() { + let frames = core_types::record::test_frames(1 << 16); let arena = Arena::new(1 << 16).unwrap(); let generations = []; let scope = scope_fixture(&generations, &arena); @@ -679,10 +709,10 @@ mod tests { ); let out = Node::::layout(&node).clone(); assert_eq!(out.depth, 1); - assert_eq!(node.extent_at(&ctx, 0), GPoll::Final(Extent::Exactly(1)), "the group is the level's single lane"); + assert_eq!(node.extent_at(&ctx, 0, &frames.reborrow()), GPoll::Final(Extent::Exactly(1)), "the group is the level's single lane"); let head = ctx.index_head(); - let GPoll::Final(value) = record::serve_edge(&node, &ctx.promoted(&head, 0)) else { + let GPoll::Final(value) = record::serve_edge(&node, &ctx.promoted(&head, 0), &frames) else { panic!("expected a final record"); }; let Graphic::Group(group) = (unsafe { record::borrow_element::(out.rec(&value)) }) else { @@ -702,6 +732,7 @@ mod tests { #[test] fn a_group_element_deep_copies_to_its_owned_form_and_replays() { + let frames = core_types::record::test_frames(1 << 16); let arena = Arena::new(1 << 16).unwrap(); let generations = []; let scope = scope_fixture(&generations, &arena); @@ -717,7 +748,7 @@ mod tests { let out = Node::::layout(&node).clone(); let head = ctx.index_head(); - let GPoll::Final(value) = record::serve_edge(&node, &ctx.promoted(&head, 0)) else { + let GPoll::Final(value) = record::serve_edge(&node, &ctx.promoted(&head, 0), &frames) else { panic!("expected a final record"); }; let copy = unsafe { (out.element.clone_out)(out.rec(&value).ptr()) }; @@ -742,25 +773,28 @@ mod tests { #[test] fn colors_fold_into_evenly_spaced_stops() { + let frames = core_types::record::test_frames(1 << 16); struct ColorSource { layout: Layout, colors: Vec, } impl Node for ColorSource { - fn serve<'e, 'l>(&self, input: &C, slot: FrameClaim<'l>) -> GPoll> + fn serve<'e, 'l>(&self, input: &C, slot: FrameClaim<'e, 'l>) -> GPoll> where C: ExtractArena, { let color = self.colors[input.innermost_index() as usize]; - let mut frame = record::FrameBuilder::new(&self.layout, ExtractArena::arena(input)); - frame.element(color); - let Some(value) = frame.finish() else { return GPoll::arena_exhausted() }; - // SAFETY: the builder served a record of this node's layout. - GPoll::Final(unsafe { slot.forward(&value) }) + let mut frame = slot; + let arena = ExtractArena::arena(input); + if frame.element(color, arena).is_none() { + return GPoll::arena_exhausted(); + } + // SAFETY: the writes above complete the record of this layout. + GPoll::Final(unsafe { frame.finish_served() }) } - fn extent_at<'x>(&self, _input: &C, _level: u8) -> GPoll + fn extent_at<'x>(&self, _input: &C, _level: u8, _frames: &core_types::record::Frames<'x>) -> GPoll where C: ExtractArena, { @@ -782,7 +816,7 @@ mod tests { let build = |colors: Vec| install_flip(ToGradientNode::new(RecordSource::new(ColorSource { layout: layout.clone(), colors }, &layout, &layout), &layout), &out); let stops_of = |colors: Vec| { let node = build(colors); - let GPoll::Final(record) = record::capture(&node, &ctx) else { + let GPoll::Final(record) = record::capture(&node, &ctx, &frames) else { panic!("expected a final record"); }; record.element::() @@ -801,6 +835,7 @@ mod tests { #[test] fn a_group_converts_to_its_legacy_list() { + let frames = core_types::record::test_frames(1 << 16); let arena = Arena::new(1 << 16).unwrap(); let generations = []; let scope = scope_fixture(&generations, &arena); @@ -815,7 +850,7 @@ mod tests { ); let out = Node::::layout(&node).clone(); let head = ctx.index_head(); - let GPoll::Final(value) = record::serve_edge(&node, &ctx.promoted(&head, 0)) else { + let GPoll::Final(value) = record::serve_edge(&node, &ctx.promoted(&head, 0), &frames) else { panic!("expected a final record"); }; let Graphic::Group(group) = (unsafe { record::borrow_element::(out.rec(&value)) }) else { @@ -832,6 +867,7 @@ mod tests { #[test] fn flatten_reverses_wrap() { + let frames = core_types::record::test_frames(1 << 16); let arena = Arena::new(1 << 16).unwrap(); let generations = []; let scope = scope_fixture(&generations, &arena); @@ -849,8 +885,8 @@ mod tests { let group = { // SAFETY: the element is cloned out inside the scope, so no borrow // into the frame escapes it. - let _scope = unsafe { stack::ScopeGuard::enter() }; - let GPoll::Final(value) = record::serve_edge(&wrapped, &ctx.promoted(&head, 0)) else { + let scope = frames.scope(); + let GPoll::Final(value) = record::serve_edge(&wrapped, &ctx.promoted(&head, 0), &scope) else { panic!("expected a final record"); }; let group = unsafe { record::borrow_element::(wrap_out.rec(&value)) }.clone(); @@ -860,11 +896,11 @@ mod tests { // One row holding the wrapped group flattens back to the lanes, the // group's identity transform composed onto each child's. let node = build!(layout, vec![(group, DAffine2::IDENTITY)], false); - assert_eq!(node.extent_at(&ctx, 0), GPoll::Final(Extent::Exactly(2))); + assert_eq!(node.extent_at(&ctx, 0, &frames.reborrow()), GPoll::Final(Extent::Exactly(2))); let head = ctx.index_head(); for (lane, &(label, x)) in [("a", 1.), ("b", 2.)].iter().enumerate() { - let GPoll::Final(record) = record::capture(&node, &ctx.promoted(&head, lane as u64)) else { + let GPoll::Final(record) = record::capture(&node, &ctx.promoted(&head, lane as u64), &frames) else { panic!("expected a final record"); }; assert_eq!(text_of(&record.element::()), label, "lane {lane}"); @@ -875,6 +911,7 @@ mod tests { #[test] fn flatten_fully_composes_the_path() { + let frames = core_types::record::test_frames(1 << 16); let arena = Arena::new(1 << 16).unwrap(); let generations = []; let scope = scope_fixture(&generations, &arena); @@ -884,12 +921,12 @@ mod tests { rows.push((group(vec![]), translation(9.))); let layout = graphic_layout(); let node = build!(layout, rows, true); - assert_eq!(node.extent_at(&ctx, 0), GPoll::Final(Extent::Exactly(3)), "the empty group contributes no leaves"); + assert_eq!(node.extent_at(&ctx, 0, &frames.reborrow()), GPoll::Final(Extent::Exactly(3)), "the empty group contributes no leaves"); let head = ctx.index_head(); let expected: [(&str, f64); 3] = [("a", 1.), ("b", 20.5), ("c", 4300.5)]; for (lane, &(label, x)) in expected.iter().enumerate() { - let GPoll::Final(record) = record::capture(&node, &ctx.promoted(&head, lane as u64)) else { + let GPoll::Final(record) = record::capture(&node, &ctx.promoted(&head, lane as u64), &frames) else { panic!("expected a final record"); }; assert_eq!(text_of(&record.element::()), label, "lane {lane}"); @@ -900,6 +937,7 @@ mod tests { #[test] fn flatten_batch_matches_per_lane_eval() { + let frames = core_types::record::test_frames(1 << 16); let arena = Arena::new(1 << 16).unwrap(); let generations = []; let scope = scope_fixture(&generations, &arena); @@ -912,14 +950,14 @@ mod tests { let scoped = ctx.promoted(&head, 0); let mut scratch = vec![std::mem::MaybeUninit::::uninit(); 3 * out.lane_stride() / 8]; - let core_types::node::BatchStatus::Filled(batch, ..) = node.eval_batch(&scoped, 0..3, Some(&mut scratch)) else { + let core_types::node::BatchStatus::Filled(batch, ..) = node.eval_batch(&scoped, 0..3, Some(&mut scratch), &frames) else { panic!("expected a filled batch"); }; let batch = batch.into_shared(); assert_eq!(batch.len(), 3); let offset = out.offset_of(::NAME, 0).unwrap(); for lane in 0..3 { - let GPoll::Final(record) = record::capture(&node, &ctx.promoted(&head, lane as u64)) else { + let GPoll::Final(record) = record::capture(&node, &ctx.promoted(&head, lane as u64), &frames) else { panic!("expected a final record"); }; let single = text_of(&record.element::()).to_string(); diff --git a/node-graph/nodes/gstd/src/render_node.rs b/node-graph/nodes/gstd/src/render_node.rs index b1c91f3714..db67ea0f35 100644 --- a/node-graph/nodes/gstd/src/render_node.rs +++ b/node-graph/nodes/gstd/src/render_node.rs @@ -261,10 +261,7 @@ mod tests { let probe = core_types::record::LiftedSource::::new(probe); let layout = Node::::layout(&probe).clone(); - // SAFETY: between evaluations, nothing served on the stack is live. - unsafe { - core_types::record::stack::reserve(layout.frame_bytes().max(1 << 12)); - } + let frames = core_types::record::test_frames(layout.frame_bytes().max(1 << 12)); let mut graph = CreateContextNode::new(probe, &layout); // The executor resolves and installs the node's own layout at wiring; // without it the flip tail writes through the default empty layout. @@ -277,7 +274,7 @@ mod tests { lane_invariant: u32::MAX, }, ); - let GPoll::Final(result) = core_types::record::serve_edge(&graph, &ctx) else { + let GPoll::Final(result) = core_types::record::serve_edge(&graph, &ctx, &frames) else { panic!("create_context must complete synchronously"); }; let output: &RenderOutput = unsafe { core_types::record::borrow_element(layout.rec(&result)) }; diff --git a/node-graph/nodes/math/src/lib.rs b/node-graph/nodes/math/src/lib.rs index 92ca398dab..dc3c807d77 100644 --- a/node-graph/nodes/math/src/lib.rs +++ b/node-graph/nodes/math/src/lib.rs @@ -1014,7 +1014,7 @@ mod graphene_test { use core_types::context::{ContextImpl, EvalScope}; use core_types::gpoll::{Finality, GPoll}; use core_types::node::{BatchStatus, Node}; - use core_types::record::{Layout, LiftedSource, RecordValue, serve_edge, stack}; + use core_types::record::{Layout, LiftedSource, RecordValue, serve_edge}; use core_types::registry::{ErasedRecordNode, construct}; use core_types::value::record_value_edge; use std::mem::MaybeUninit; @@ -1023,9 +1023,9 @@ mod graphene_test { EvalScope::new(None, None, None, &[], arena) } - fn reserve_for(layouts: &[&Layout]) { - // SAFETY: between evaluations, nothing served on the stack is live. - unsafe { stack::reserve(layouts.iter().map(|layout| layout.frame_bytes()).sum::().max(1 << 12)); } } + fn frames_for(layouts: &[&Layout]) -> core_types::record::Frames<'static> { + core_types::record::test_frames(layouts.iter().map(|layout| layout.frame_bytes()).sum::().max(1 << 12)) + } /// Lifts a plain-element test source onto a record wire, returned beside its /// element-only layout for the generated node's constructor. @@ -1071,9 +1071,9 @@ mod graphene_test { let (b, lb) = lifted(|_: &ContextImpl| GPoll::Final(2.0f64)); let out = out_layout::(); let graph = installed(AddNode::<_, _, f64, f64>::new(a, b, &la, &lb), &out); - reserve_for(&[&la, &lb, &out]); + let frames = frames_for(&[&la, &lb, &out]); - let GPoll::Final(value) = serve_edge(&graph, &ctx) else { + let GPoll::Final(value) = serve_edge(&graph, &ctx, &frames) else { panic!("expected a final record"); }; assert_eq!(element::(&out, &value), 3.0); @@ -1089,12 +1089,12 @@ mod graphene_test { let (src, ls) = lifted(|_: &ContextImpl| GPoll::Final(10.0f64)); let out = out_layout::(); let node = installed(AddNode::<_, _, f64, f64>::new(index, src, &li, &ls), &out); - reserve_for(&[&li, &ls, &out]); + let frames = frames_for(&[&li, &ls, &out]); let erased: Box = Box::new(node); // One u64 word per lane at the element-only layout. let mut scratch = [const { MaybeUninit::uninit() }; 4]; - let status = erased.eval_batch(&ctx, 2..6, Some(&mut scratch)); + let status = erased.eval_batch(&ctx, 2..6, Some(&mut scratch), &frames); let BatchStatus::Filled(batch, finality, _) = status else { panic!("expected filled, got {status:?}"); }; @@ -1120,9 +1120,9 @@ mod graphene_test { lane_invariant: u32::MAX, }); let edge = wired.downcast_record::().unwrap(); - reserve_for(&[&layout]); + let frames = frames_for(&[&layout]); - let GPoll::Final(value) = serve_edge(&edge, &ctx) else { + let GPoll::Final(value) = serve_edge(&edge, &ctx, &frames) else { panic!("expected a final record"); }; assert!(element::(&layout, &value)); @@ -1166,9 +1166,9 @@ mod graphene_test { lane_invariant: u32::MAX, }); let edge = wired.downcast_record::().unwrap(); - reserve_for(&[&layout]); + let frames = frames_for(&[&layout]); - let GPoll::Final(value) = serve_edge(&edge, &ctx) else { + let GPoll::Final(value) = serve_edge(&edge, &ctx, &frames) else { panic!("expected a final record"); }; assert_eq!(element::(&layout, &value), 4.0); @@ -1213,9 +1213,9 @@ mod graphene_test { let union = core_types::record::Layout::union(&[<, &lf]); let graph = SwitchNode::new(cond, if_true, if_false, &union, &lc); let out = Node::::layout(&graph).clone(); - reserve_for(&[&lc, <, &lf, &out]); + let frames = frames_for(&[&lc, <, &lf, &out]); - let GPoll::Final(value) = serve_edge(&graph, &ctx) else { + let GPoll::Final(value) = serve_edge(&graph, &ctx, &frames) else { panic!("expected a final record"); }; assert_eq!(element::(&out, &value), 1.0); @@ -1239,10 +1239,10 @@ mod graphene_test { let (pa2, lpa2) = lifted(|_: &ContextImpl| GPoll::Partial(7.0f64)); let partial = SwitchNode::new(c2, p2, pa2, &core_types::record::Layout::union(&[&lp2, &lpa2]), &lc2); let out = Node::::layout(&partial).clone(); - reserve_for(&[&lc1, &lp1, &lpa1, &lc2, &lp2, &lpa2, &out]); + let frames = frames_for(&[&lc1, &lp1, &lpa1, &lc2, &lp2, &lpa2, &out]); - assert!(matches!(serve_edge(&pending, &ctx), GPoll::Pending)); - let GPoll::Partial(value) = serve_edge(&partial, &ctx) else { + assert!(matches!(serve_edge(&pending, &ctx, &frames), GPoll::Pending)); + let GPoll::Partial(value) = serve_edge(&partial, &ctx, &frames) else { panic!("expected a partial record"); }; assert_eq!(element::(&out, &value), 7.0); @@ -1260,9 +1260,9 @@ mod graphene_test { let union = core_types::record::Layout::union(&[<, &lf]); let graph = SwitchNode::new(cond, if_true, if_false, &union, &lc); let out = Node::::layout(&graph).clone(); - reserve_for(&[&lc, <, &lf, &out]); + let frames = frames_for(&[&lc, <, &lf, &out]); - let GPoll::Partial(value) = serve_edge(&graph, &ctx) else { + let GPoll::Partial(value) = serve_edge(&graph, &ctx, &frames) else { panic!("expected a partial record"); }; assert_eq!(element::(&out, &value), 1.0); @@ -1278,9 +1278,9 @@ mod graphene_test { let (src, ls) = lifted(|_: &ContextImpl| GPoll::Final(5.0f64)); let out = out_layout::(); let graph = installed(AddNode::<_, _, f64, f64>::new(fallback, src, &lfb, &ls), &out); - reserve_for(&[&lfb, &ls, &out]); + let frames = frames_for(&[&lfb, &ls, &out]); - let GPoll::Fallback(boxed) = serve_edge(&graph, &ctx) else { + let GPoll::Fallback(boxed) = serve_edge(&graph, &ctx, &frames) else { panic!("fallback must propagate with the computed stand-in"); }; assert_eq!(element::(&out, &boxed.0), 5.0); diff --git a/node-graph/nodes/raster/src/image_color_palette.rs b/node-graph/nodes/raster/src/image_color_palette.rs index 1c29b4c73e..844f7bbd4a 100644 --- a/node-graph/nodes/raster/src/image_color_palette.rs +++ b/node-graph/nodes/raster/src/image_color_palette.rs @@ -70,8 +70,8 @@ mod test { #[test] fn test_image_color_palette() { - // SAFETY: between evaluations, nothing served on the stack is live. - unsafe { core_types::record::stack::reserve(1 << 16); } let arena = core_types::arena::Arena::new(1 << 22).unwrap(); + let frames = core_types::record::test_frames(1 << 16); + let arena = core_types::arena::Arena::new(1 << 22).unwrap(); let generations = []; let scope = core_types::context::EvalScope::new(None, None, None, &generations, &arena); let ctx = core_types::context::ContextImpl::root(&scope); @@ -83,7 +83,7 @@ mod test { base64_string: None, }); let source = core_types::value::LeveledValueSource::new(vec![raster]); - let core_types::record::LevelStatus::Batch(batch, _) = core_types::record::materialize_level(&source, &ctx, &arena) else { + let core_types::record::LevelStatus::Batch(batch, _) = core_types::record::materialize_level(&source, &ctx, &arena, &frames) else { panic!("materialize failed") }; let image = unsafe { core_types::node::List::>::new(batch) }; diff --git a/node-graph/nodes/repeat/src/repeat_nodes.rs b/node-graph/nodes/repeat/src/repeat_nodes.rs index 9c3e7e1754..223efc136a 100644 --- a/node-graph/nodes/repeat/src/repeat_nodes.rs +++ b/node-graph/nodes/repeat/src/repeat_nodes.rs @@ -183,7 +183,7 @@ mod test { use core_types::arena::Arena; use core_types::context::{ContextImpl, EvalScope, ExtractArena}; use core_types::node::Node; - use core_types::record::{FieldWrite, FrameBuilder, FrameClaim, Layout, RecordSource, Served, capture, element_write, stack}; + use core_types::record::{FieldWrite, FrameClaim, Layout, RecordSource, Served, capture, element_write}; use core_types::value::ValueSource; use vector_types::subpath::Subpath; @@ -194,16 +194,18 @@ mod test { } impl Node for TransformSource { - fn serve<'e, 'l>(&self, input: &C, slot: FrameClaim<'l>) -> GPoll> + fn serve<'e, 'l>(&self, input: &C, slot: FrameClaim<'e, 'l>) -> GPoll> where C: ExtractArena, { - let mut frame = FrameBuilder::new(&self.layout, ExtractArena::arena(input)); - frame.element(self.element); - frame.attr::(self.transform); - let Some(value) = frame.finish() else { return GPoll::arena_exhausted() }; - // SAFETY: the builder served a record of this node's layout. - GPoll::Final(unsafe { slot.forward(&value) }) + let mut frame = slot; + let arena = ExtractArena::arena(input); + if frame.element(self.element, arena).is_none() { + return GPoll::arena_exhausted(); + } + write_attr_at::(&mut frame, &self.layout, self.transform); + // SAFETY: the writes above complete the record of this layout. + GPoll::Final(unsafe { frame.finish_served() }) } fn layout(&self) -> &Layout { @@ -211,9 +213,28 @@ mod test { } } + /// Writes a field at the layout's resolved offset, the wiring-proven pairing + /// a generated node performs. + fn write_field_at(frame: &mut FrameClaim<'_, '_>, layout: &Layout, name: &str, level: u8, value: T) { + let field = layout + .fields + .iter() + .find(|field| field.name == name && field.level == level) + .expect("the layout carries the written field"); + assert_eq!(field.type_id, std::any::TypeId::of::(), "the field was declared at this value type"); + // SAFETY: the offset is this layout's own, at the field's declared type. + unsafe { frame.attr_at(field.offset, value) }; + } + + /// [`write_field_at`] for a census marker at level 0. + fn write_attr_at(frame: &mut FrameClaim<'_, '_>, layout: &Layout, value: A::Value<'static>) + where + A::Value<'static>: Copy + 'static, + { + write_field_at(frame, layout, A::NAME, 0, value); + } fn scope_fixture<'a>(generations: &'a [(SourceId, u64)], arena: &'a Arena) -> EvalScope<'a> { - // SAFETY: between evaluations, nothing served on the stack is live. - unsafe { stack::reserve(1 << 12); } EvalScope::new(Some(0.5), None, None, generations, arena) + EvalScope::new(Some(0.5), None, None, generations, arena) } struct VectorRows { @@ -222,20 +243,22 @@ mod test { } impl Node for VectorRows { - fn serve<'e, 'l>(&self, input: &C, slot: FrameClaim<'l>) -> GPoll> + fn serve<'e, 'l>(&self, input: &C, slot: FrameClaim<'e, 'l>) -> GPoll> where C: ExtractArena, { let (vector, transform) = &self.rows[input.innermost_index() as usize % self.rows.len()]; - let mut frame = FrameBuilder::new(&self.layout, ExtractArena::arena(input)); - frame.element(vector.clone()); - frame.attr::(*transform); - let Some(value) = frame.finish() else { return GPoll::arena_exhausted() }; - // SAFETY: the builder served a record of this node's layout. - GPoll::Final(unsafe { slot.forward(&value) }) + let mut frame = slot; + let arena = ExtractArena::arena(input); + if frame.element(vector.clone(), arena).is_none() { + return GPoll::arena_exhausted(); + } + write_attr_at::(&mut frame, &self.layout, *transform); + // SAFETY: the writes above complete the record of this layout. + GPoll::Final(unsafe { frame.finish_served() }) } - fn extent_at<'x>(&self, _input: &C, _level: u8) -> GPoll + fn extent_at<'x>(&self, _input: &C, _level: u8, _frames: &core_types::record::Frames<'x>) -> GPoll where C: ExtractArena, { @@ -256,17 +279,19 @@ mod test { } impl Node for PositionProbe { - fn serve<'e, 'l>(&self, input: &C, slot: FrameClaim<'l>) -> GPoll> + fn serve<'e, 'l>(&self, input: &C, slot: FrameClaim<'e, 'l>) -> GPoll> where C: ExtractArena, { let position = input.try_position().and_then(|mut positions| positions.next()).unwrap_or(DVec2::ZERO); - let mut frame = FrameBuilder::new(&self.layout, ExtractArena::arena(input)); - frame.element(position.x); - frame.attr::(DAffine2::IDENTITY); - let Some(value) = frame.finish() else { return GPoll::arena_exhausted() }; - // SAFETY: the builder served a record of this node's layout. - GPoll::Final(unsafe { slot.forward(&value) }) + let mut frame = slot; + let arena = ExtractArena::arena(input); + if frame.element(position.x, arena).is_none() { + return GPoll::arena_exhausted(); + } + write_attr_at::(&mut frame, &self.layout, DAffine2::IDENTITY); + // SAFETY: the writes above complete the record of this layout. + GPoll::Final(unsafe { frame.finish_served() }) } fn layout(&self) -> &Layout { @@ -280,6 +305,7 @@ mod test { #[test] fn repeat_array_composes_the_step_onto_each_copys_transform() { + let frames = core_types::record::test_frames(1 << 16); let arena = Arena::new(1024).unwrap(); let generations = []; let scope = scope_fixture(&generations, &arena); @@ -302,12 +328,12 @@ mod test { Node::::set_layout(&mut node, repeat_array_layout_meta().resolve(&[Some(&layout)])); let leveled = Node::::layout(&node).clone(); assert_eq!(leveled.depth, 1, "the IList return pushed one rank level above the content"); - assert_eq!(node.extent_at(&ctx, 0), GPoll::Final(Extent::Exactly(3))); + assert_eq!(node.extent_at(&ctx, 0, &frames.reborrow()), GPoll::Final(Extent::Exactly(3))); let head = ctx.index_head(); for copy in 0..3u64 { let lane = ctx.promoted(&head, copy); - let GPoll::Final(record) = capture(&node, &lane) else { + let GPoll::Final(record) = capture(&node, &lane, &frames) else { panic!("expected a final record"); }; assert_eq!(record.element::(), 7.); @@ -320,6 +346,7 @@ mod test { #[test] fn repeat_radial_rotates_each_copy_around_the_center() { + let frames = core_types::record::test_frames(1 << 16); let arena = Arena::new(1024).unwrap(); let generations = []; let scope = scope_fixture(&generations, &arena); @@ -335,12 +362,12 @@ mod test { let mut node = RepeatRadialNode::new(RecordSource::new(content, &layout, &layout), ValueSource::new(90.0f64), ValueSource::new(2.0f64), ValueSource::new(4u32), &layout); Node::::set_layout(&mut node, repeat_radial_layout_meta().resolve(&[Some(&layout)])); - assert_eq!(node.extent_at(&ctx, 0), GPoll::Final(Extent::Exactly(4))); + assert_eq!(node.extent_at(&ctx, 0, &frames.reborrow()), GPoll::Final(Extent::Exactly(4))); let head = ctx.index_head(); for copy in 0..4u64 { let lane = ctx.promoted(&head, copy); - let GPoll::Final(record) = capture(&node, &lane) else { + let GPoll::Final(record) = capture(&node, &lane, &frames) else { panic!("expected a final record"); }; assert_eq!(record.element::(), 7.); @@ -354,6 +381,7 @@ mod test { #[test] fn repeat_on_points_lands_each_copy_on_its_transformed_point() { + let frames = core_types::record::test_frames(1 << 16); let arena = Arena::new(1 << 16).unwrap(); let generations = []; let scope = scope_fixture(&generations, &arena); @@ -376,14 +404,18 @@ mod test { Node::::set_layout(&mut node, repeat_on_points_layout_meta().resolve(&[Some(&content_layout)])); let leveled = Node::::layout(&node).clone(); assert_eq!(leveled.depth, 1); - assert_eq!(node.extent_at(&ctx, 0), GPoll::Final(Extent::Exactly(5)), "the pushed level flattens both rows' points"); + assert_eq!( + node.extent_at(&ctx, 0, &frames.reborrow()), + GPoll::Final(Extent::Exactly(5)), + "the pushed level flattens both rows' points" + ); let expected: Vec = row0.iter().map(|&point| row0_transform.transform_point2(point)).chain(row1.iter().copied()).collect(); let head = ctx.index_head(); for (flat, &point) in expected.iter().enumerate() { let lane = ctx.promoted(&head, flat as u64); - let GPoll::Final(record) = capture(&node, &lane) else { + let GPoll::Final(record) = capture(&node, &lane, &frames) else { panic!("expected a final record"); }; // The content saw the pushed position, and the output transform lands on it. @@ -395,6 +427,7 @@ mod test { #[test] fn repeat_on_points_reverse_flips_each_rows_points() { + let frames = core_types::record::test_frames(1 << 16); let arena = Arena::new(1 << 16).unwrap(); let generations = []; let scope = scope_fixture(&generations, &arena); @@ -416,7 +449,7 @@ mod test { let head = ctx.index_head(); for (flat, &point) in expected.iter().enumerate() { let lane = ctx.promoted(&head, flat as u64); - let GPoll::Final(record) = capture(&node, &lane) else { + let GPoll::Final(record) = capture(&node, &lane, &frames) else { panic!("expected a final record"); }; let composed: DAffine2 = record.attr::(); diff --git a/node-graph/nodes/vector/src/vector_nodes.rs b/node-graph/nodes/vector/src/vector_nodes.rs index 9f747abc60..81702cbe74 100644 --- a/node-graph/nodes/vector/src/vector_nodes.rs +++ b/node-graph/nodes/vector/src/vector_nodes.rs @@ -3870,8 +3870,8 @@ mod test { } #[test] fn path_length() { - // SAFETY: between evaluations, nothing served on the stack is live. - unsafe { core_types::record::stack::reserve(1 << 16); } let arena = core_types::arena::Arena::new(1 << 20).unwrap(); + let frames = core_types::record::test_frames(1 << 16); + let arena = core_types::arena::Arena::new(1 << 20).unwrap(); let generations = []; let scope = core_types::context::EvalScope::new(None, None, None, &generations, &arena); let ctx = core_types::context::ContextImpl::root(&scope); @@ -3882,7 +3882,7 @@ mod test { // Element-only lanes read identity lane transforms; the transform term // rides the demo gate. let source = core_types::value::LeveledValueSource::new(vec![row; 5]); - let core_types::record::LevelStatus::Batch(batch, _) = core_types::record::materialize_level(&source, &ctx, &arena) else { + let core_types::record::LevelStatus::Batch(batch, _) = core_types::record::materialize_level(&source, &ctx, &arena, &frames) else { panic!("materialize failed") }; let list = unsafe { core_types::node::List::::new(batch) };