diff --git a/Cargo.lock b/Cargo.lock index 5a62bc852c..09fabbc27a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3329,6 +3329,28 @@ dependencies = [ "libc", ] +[[package]] +name = "migration-core" +version = "0.0.0" +dependencies = [ + "graph-storage", + "rmp-serde", + "serde", + "serde_json", + "thiserror 2.0.18", +] + +[[package]] +name = "migration-runner" +version = "0.0.0" +dependencies = [ + "graph-storage", + "log", + "migration-core", + "serde_json", + "thiserror 2.0.18", +] + [[package]] name = "mime" version = "0.3.17" diff --git a/Cargo.toml b/Cargo.toml index 98ef2d9eee..f085091da2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,8 @@ members = [ "document/graph-storage", "document/container", "document/document-format", + "document/migrations/core", + "document/migrations/runner", "editor", "frontend/wrapper", "libraries/dyn-any", @@ -94,6 +96,8 @@ graph-craft = { path = "node-graph/graph-craft" } graph-storage = { path = "document/graph-storage", default-features = false } document-format = { path = "document/document-format" } document-container = { path = "document/container" } +migration-core = { path = "document/migrations/core" } +migration-runner = { path = "document/migrations/runner" } raster-nodes = { path = "node-graph/nodes/raster" } graphene-std = { path = "node-graph/nodes/gstd" } interpreted-executor = { path = "node-graph/interpreted-executor" } diff --git a/document/graph-storage/src/history.rs b/document/graph-storage/src/history.rs index f091334e2c..d8f1602524 100644 --- a/document/graph-storage/src/history.rs +++ b/document/graph-storage/src/history.rs @@ -10,7 +10,7 @@ use std::collections::HashMap; -use crate::{AttributesWrite, CrdtError, Delta, Rev, TimeStamp}; +use crate::{AttributesWrite, CrdtError, Delta, RegistryDelta, Rev, TimeStamp}; #[derive(Clone, Debug, Default)] pub struct History { @@ -178,3 +178,36 @@ impl History { Ok(()) } } + +/// Recompute every delta's content-addressed `Rev` after delta payloads were rewritten, walking in +/// topological order and remapping parent links (including `Merge` extra parents) along the way. +/// Returns the old → new mapping for remapping cursors (`head`, redo stack, broadcast tip). +pub fn rehash_deltas(deltas: &mut [Delta]) -> HashMap { + fn remap_merge_parents(kind: &mut RegistryDelta, mapping: &HashMap) { + if let RegistryDelta::Merge { extra_parents } = kind { + for parent in extra_parents.iter_mut() { + if let Some(new) = mapping.get(parent) { + *parent = *new; + } + } + } + } + + let mut mapping: HashMap = HashMap::new(); + + for delta in deltas.iter_mut() { + if let Some(parent) = delta.parent.as_mut() + && let Some(new) = mapping.get(parent) + { + *parent = *new; + } + remap_merge_parents(&mut delta.kind, &mapping); + remap_merge_parents(&mut delta.reverse, &mapping); + + let new_id = delta.recomputed_id(); + mapping.insert(delta.id, new_id); + delta.id = new_id; + } + + mapping +} diff --git a/document/graph-storage/src/lib.rs b/document/graph-storage/src/lib.rs index 806c4cb494..3f78d649d2 100644 --- a/document/graph-storage/src/lib.rs +++ b/document/graph-storage/src/lib.rs @@ -21,7 +21,7 @@ pub mod to_runtime; pub use attributes::*; pub use crdt::*; pub use document::*; -pub use history::History; +pub use history::{History, rehash_deltas}; pub use ids::*; pub use model::*; pub use registry::*; diff --git a/document/graph-storage/src/model.rs b/document/graph-storage/src/model.rs index 6b9c1c4663..c41ca9c53b 100644 --- a/document/graph-storage/src/model.rs +++ b/document/graph-storage/src/model.rs @@ -11,6 +11,15 @@ pub struct Node { } impl Node { + pub fn new(implementation: Implementation, inputs: Vec, attributes: Attributes, network: NetworkId) -> Self { + Self { + implementation, + inputs, + attributes, + network, + } + } + pub fn implementation(&self) -> &Implementation { &self.implementation } @@ -24,6 +33,21 @@ impl Node { self.network } + // Mutable access for edits on a registry clone that is diffed back into deltas; + // live-session edits go through the delta paths instead. + pub fn implementation_mut(&mut self) -> &mut Implementation { + &mut self.implementation + } + pub fn inputs_mut(&mut self) -> &mut Vec { + &mut self.inputs + } + pub fn attributes_mut(&mut self) -> &mut Attributes { + &mut self.attributes + } + pub fn set_network(&mut self, network: NetworkId) { + self.network = network; + } + /// True if both nodes agree on every value-bearing field, ignoring slot/attribute timestamps. pub fn value_equal(&self, other: &Self) -> bool { if self.implementation != other.implementation || self.network != other.network { diff --git a/document/graph-storage/src/resources.rs b/document/graph-storage/src/resources.rs index a29a37e2b3..5959e2d918 100644 --- a/document/graph-storage/src/resources.rs +++ b/document/graph-storage/src/resources.rs @@ -45,7 +45,7 @@ impl<'de> Deserialize<'de> for ResourceEntry { } let Raw { mut sources, hash, hash_timestamp } = Raw::deserialize(deserializer)?; - sources.sort_by(|(a, _), (b, _)| a.cmp(b)); + sources.sort_by_key(|(key, _)| *key); sources.dedup_by(|(later_key, later_value), (kept_key, kept_value)| { // `dedup_by` keeps the first of each run; sorting is stable, so resolve duplicates by LWW. if later_key != kept_key { diff --git a/document/graph-storage/src/session.rs b/document/graph-storage/src/session.rs index 3ebb31c625..d435f8b7d6 100644 --- a/document/graph-storage/src/session.rs +++ b/document/graph-storage/src/session.rs @@ -92,6 +92,27 @@ impl Session { Ok(conversion.network_ids) } + /// Diff the working registry against `target` and stage the difference as hot ops, so bulk + /// registry mutations commit through the ordinary hot-op path without hand-built deltas. + pub fn stage_registry_replace(&mut self, target: &Registry) -> Result, CrdtError> { + let ops = crate::delta::compute_deltas(&self.document.working_registry, target); + self.stage_ops(ops) + } + + /// Retire every pending hot op as one gesture. Convenience over [`Session::retire`] for callers + /// that don't track hot-op timestamps themselves. + pub fn retire_all(&mut self) -> Result, CrdtError> { + match self.document.hot_log.iter().map(|hot_op| hot_op.timestamp).max() { + Some(up_to) => self.retire(up_to), + None => Ok(Vec::new()), + } + } + + /// Mint a fresh peer-scoped `NodeId`. Forwards to [`Document::next_node_id`]. + pub fn next_node_id(&mut self) -> NodeId { + self.document.next_node_id() + } + /// Register a content-addressed resource as a single `DataSource::Embedded` source resolved to /// `hash`, staged as one `AddResource` hot op. The caller owns `id` allocation, persists the /// returned hot frame, retires, and persists the bytes into its byte store separately. diff --git a/document/migrations/core/Cargo.toml b/document/migrations/core/Cargo.toml new file mode 100644 index 0000000000..bca3b84dc9 --- /dev/null +++ b/document/migrations/core/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "migration-core" +description = "Trait definitions and selectors for Graphite document migrations" +edition.workspace = true +version.workspace = true +license.workspace = true +authors.workspace = true + +[features] +default = ["typed"] +# Content-tier migrations over the typed registry. Format-tier-only migration crates +# (historic version steps, wasm builds) disable this to drop the graph-storage dependency. +typed = ["dep:graph-storage"] + +[dependencies] +graph-storage = { workspace = true, optional = true } + +serde = { workspace = true } +serde_json = { workspace = true } +rmp-serde = { workspace = true } +thiserror = { workspace = true } diff --git a/document/migrations/core/src/content.rs b/document/migrations/core/src/content.rs new file mode 100644 index 0000000000..456c336f81 --- /dev/null +++ b/document/migrations/core/src/content.rs @@ -0,0 +1,82 @@ +use crate::{MigrationError, MigrationId}; +use graph_storage::{Node, NodeId, Registry, ResourceId}; + +/// Identity of a proto-node declaration, as read from the declaration resource body. +#[derive(Clone, Debug)] +pub struct DeclarationInfo { + pub identifier: String, + /// Explicit behavioral version; 0 for declarations predating versioning. + pub version: u32, +} + +/// Which registry entities a content migration applies to. +#[derive(Clone, Debug)] +pub enum Selector { + /// Every node whose proto-node declaration matches. + Node(NodeSelector), + /// Every node carrying this `ui::reference` attribute value (how legacy wrapper networks + /// like "Brush" are identified). + Reference(&'static str), + /// Once per document, gated by the [`crate::APPLIED_ATTRIBUTE`] provenance list. + Document, +} + +/// Matches nodes by declaration identifier (with historic aliases) and version range. +#[derive(Clone, Debug)] +pub struct NodeSelector { + /// Current declaration identifier plus historic aliases. + pub names: &'static [&'static str], + /// Upgrade target: nodes with a declaration version strictly below this match. + pub below_version: u32, +} + +impl NodeSelector { + /// Whether a node with this declaration identity is matched. + pub fn matches(&self, declaration: &DeclarationInfo) -> bool { + // TODO(Dennis): Matching semantics to decide here: + // - Legacy identifiers can carry generic type arguments (`...MemoNode`); the old system + // compared with `identifier.split('<').next()`. Strip them, or require exact matches? + // - Should `below_version` also gate name-alias matches, or do aliases (which only appear in + // pre-versioning documents, version 0) always match regardless? + let _ = declaration; + todo!("decide and implement declaration matching") + } +} + +/// One matched entity, produced by scanning a [`Selector`] over the registry. +#[derive(Copy, Clone, Debug)] +pub enum Target { + Node(NodeId), + Document, +} + +/// Byte-store and catalog services only the host (editor or CLI) can provide. +pub trait MigrationHost { + /// Read a declaration resource's identity (identifier and version). + fn declaration_info(&self, id: ResourceId) -> Option; + /// The decoded declaration body, for inspection beyond the identity. + fn declaration(&self, id: ResourceId) -> Option; + /// Instantiate the current catalog default node for a declaration identifier. + fn resolve_definition(&mut self, identifier: &str) -> Option; +} + +/// Everything a content migration can reach beyond the registry: host services plus ID minting. +pub trait MigrationContext: MigrationHost { + /// Mint a fresh peer-scoped node ID for inserted nodes. + fn mint_node_id(&mut self) -> NodeId; +} + +/// One node-usage upgrade within the current format version. +/// +/// `migrate` mutates a registry clone in place; the mutation is diffed into deltas and committed to +/// history as one retired gesture, so implementations never construct deltas by hand. Timestamps +/// written into the clone are placeholders the commit path re-stamps. Implementations must be +/// idempotent: a document can round-trip through an editor build that lacks a later migration. +pub trait ContentMigration { + /// Stable identifier recorded in provenance. + fn id(&self) -> MigrationId; + /// Which registry entities to run on. + fn selector(&self) -> Selector; + /// Upgrade one matched target in place. + fn migrate(&self, target: Target, registry: &mut Registry, context: &mut dyn MigrationContext) -> Result<(), MigrationError>; +} diff --git a/document/migrations/core/src/error.rs b/document/migrations/core/src/error.rs new file mode 100644 index 0000000000..9fce702b49 --- /dev/null +++ b/document/migrations/core/src/error.rs @@ -0,0 +1,11 @@ +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum MigrationError { + #[error("failed to decode payload: {0}")] + Decode(String), + #[error("failed to encode payload: {0}")] + Encode(String), + #[error("migration invariant violated: {0}")] + Invariant(String), +} diff --git a/document/migrations/core/src/format.rs b/document/migrations/core/src/format.rs new file mode 100644 index 0000000000..802b4b1c27 --- /dev/null +++ b/document/migrations/core/src/format.rs @@ -0,0 +1,44 @@ +use crate::{MigrationError, MigrationId, Payload}; + +/// How stored history survives a format step. +/// +/// Under `Rewrite`, the migration transforms each record's payload via +/// [`FormatMigration::migrate_delta`]; the runner then recomputes `Rev`s in topological order and +/// remaps parent links and session cursors. Because the rewrite is a pure function of content and +/// `Rev`s are content-addressed, peers applying the same migration converge on identical rewritten +/// history without coordination. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum HistoryPolicy { + /// The shape change does not affect stored deltas. + Untouched, + /// Each history record is rewritten via `migrate_delta`. + Rewrite, + /// No faithful rewrite exists; the document becomes a state-only snapshot. + Truncate, +} + +/// One whole-document format step, migrating version `migrates_from()` to `migrates_from() + 1`. +/// +/// Implementations freeze whatever old struct shapes they need locally (deserialized from +/// [`Payload`]s); the active codebase never carries them. Migrations only transform payload +/// content, never identity fields (`id`, `parent`) — Merkle bookkeeping belongs to the runner. +pub trait FormatMigration { + /// Stable identifier recorded in provenance. + fn id(&self) -> MigrationId; + /// The format version this step upgrades from. + fn migrates_from(&self) -> u32; + /// Transform the serialized registry payload. + fn migrate_registry(&self, registry: &Payload) -> Result; + /// How stored history survives this step. + fn history_policy(&self) -> HistoryPolicy { + HistoryPolicy::Untouched + } + /// Transform one history record. Called only under [`HistoryPolicy::Rewrite`]. + fn migrate_delta(&self, delta: &Payload) -> Result { + Ok(delta.clone()) + } + /// Transform the per-peer session payload. + fn migrate_session(&self, session: &Payload) -> Result { + Ok(session.clone()) + } +} diff --git a/document/migrations/core/src/lib.rs b/document/migrations/core/src/lib.rs new file mode 100644 index 0000000000..3819ee6e5d --- /dev/null +++ b/document/migrations/core/src/lib.rs @@ -0,0 +1,41 @@ +//! Trait definitions for `.gdd` document migrations. +//! +//! Two tiers: a [`FormatMigration`] steps a document's serialized payloads from one format version +//! to the next, and a [`ContentMigration`] (feature `typed`) upgrades node usages on the typed +//! `Registry` within the current version. Migration crates export a [`MigrationSet`] via a plain +//! constructor function; the `migration-runner` crate aggregates and dispatches them. Design +//! rationale lives in `node-graph/rfcs/document-format-migrations.md`. + +#[cfg(feature = "typed")] +pub mod content; +pub mod error; +pub mod format; +pub mod payload; + +#[cfg(feature = "typed")] +pub use content::{ContentMigration, DeclarationInfo, MigrationContext, MigrationHost, NodeSelector, Selector, Target}; +pub use error::MigrationError; +pub use format::{FormatMigration, HistoryPolicy}; +pub use payload::{Payload, PayloadCodec}; + +/// Document attribute holding the list of applied `Document`-selector migration IDs. +pub const APPLIED_ATTRIBUTE: &str = "migrations::applied"; + +/// Stable identifier for one migration, recorded in provenance and used for skip checks. +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] +pub struct MigrationId(pub &'static str); + +impl std::fmt::Display for MigrationId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.0) + } +} + +/// What one migration crate exports: its migrations, in application order. +/// Staged node upgrades must be registered in ascending target-version order. +#[derive(Default)] +pub struct MigrationSet { + pub format: Vec>, + #[cfg(feature = "typed")] + pub content: Vec>, +} diff --git a/document/migrations/core/src/payload.rs b/document/migrations/core/src/payload.rs new file mode 100644 index 0000000000..ab276ad2c8 --- /dev/null +++ b/document/migrations/core/src/payload.rs @@ -0,0 +1,47 @@ +use crate::MigrationError; +use serde::Serialize; +use serde::de::DeserializeOwned; + +/// Codec of a serialized payload. Mirrors the `document-format` codec table entries relevant to +/// migrations so migration crates don't depend on `document-format`; the runner maps between the two. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum PayloadCodec { + Json, + MessagePack, +} + +/// One serialized document payload (the registry, a single history record, the session) plus its +/// codec. History framing is the runner's concern; migrations always see one record per payload. +#[derive(Clone, Debug)] +pub struct Payload { + pub bytes: Vec, + pub codec: PayloadCodec, +} + +impl Payload { + pub fn new(bytes: Vec, codec: PayloadCodec) -> Self { + Self { bytes, codec } + } + + /// Decode into a typed shape, usually a frozen mirror struct owned by the migration crate. + pub fn decode(&self) -> Result { + match self.codec { + PayloadCodec::Json => serde_json::from_slice(&self.bytes).map_err(|e| MigrationError::Decode(e.to_string())), + PayloadCodec::MessagePack => rmp_serde::from_slice(&self.bytes).map_err(|e| MigrationError::Decode(e.to_string())), + } + } + + /// Encode a typed shape with the given codec. + pub fn encode(value: &T, codec: PayloadCodec) -> Result { + let bytes = match codec { + PayloadCodec::Json => serde_json::to_vec(value).map_err(|e| MigrationError::Encode(e.to_string()))?, + PayloadCodec::MessagePack => rmp_serde::to_vec(value).map_err(|e| MigrationError::Encode(e.to_string()))?, + }; + Ok(Self { bytes, codec }) + } + + /// Encode a typed shape, keeping this payload's codec. + pub fn encode_as(&self, value: &T) -> Result { + Self::encode(value, self.codec) + } +} diff --git a/document/migrations/runner/Cargo.toml b/document/migrations/runner/Cargo.toml new file mode 100644 index 0000000000..3f13feda4d --- /dev/null +++ b/document/migrations/runner/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "migration-runner" +description = "Dispatches Graphite document migrations across the format and content tiers" +edition.workspace = true +version.workspace = true +license.workspace = true +authors.workspace = true + +[dependencies] +migration-core = { workspace = true } +graph-storage = { workspace = true } + +serde_json = { workspace = true } +thiserror = { workspace = true } +log = { workspace = true } diff --git a/document/migrations/runner/src/lib.rs b/document/migrations/runner/src/lib.rs new file mode 100644 index 0000000000..c0fd9c13bf --- /dev/null +++ b/document/migrations/runner/src/lib.rs @@ -0,0 +1,221 @@ +//! Dispatches document migrations: format-version chaining over serialized payloads, then +//! delta-expressed content migrations over the typed registry. See +//! `node-graph/rfcs/document-format-migrations.md` for the pipeline design. + +use graph_storage::{AttributesRead, AttributesWrite, CrdtError, Implementation, NodeId, Registry, ResourceId, Rev, Session, TimeStamp, attr}; +use migration_core::{APPLIED_ATTRIBUTE, DeclarationInfo, FormatMigration, HistoryPolicy, MigrationContext, MigrationError, MigrationHost, MigrationSet, Payload, Selector, Target}; + +pub use graph_storage::rehash_deltas; + +#[derive(Debug, thiserror::Error)] +pub enum RunnerError { + #[error("no format migration registered for version {at_version}")] + MissingFormatStep { at_version: u32 }, + #[error("format migration {id} failed: {source}")] + Format { id: &'static str, source: MigrationError }, + #[error("failed to commit migration deltas: {0}")] + Crdt(#[from] CrdtError), +} + +/// The serialized payloads of one document, as handed over by the load path before typed +/// deserialization. History records are unframed: one [`Payload`] per retired delta, in stored +/// (topological) order. +pub struct DocumentPayloads { + pub format_version: u32, + pub registry: Payload, + pub history: Vec, + pub session: Option, +} + +/// What the format tier did to history, so the load path knows whether a `Rev` rehash pass +/// (with session-cursor remap) is required after typed deserialization. Ordered by severity so +/// chained steps combine via `max`. +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum HistoryOutcome { + Untouched, + /// At least one step rewrote records: rehash `Rev`s with [`rehash_deltas`] and remap cursors. + Rewritten, + /// At least one step truncated history: reset cursors, the document is a state-only snapshot. + Truncated, +} + +/// Aggregates registered [`MigrationSet`]s and dispatches them over a document. +pub struct MigrationRunner { + sets: Vec, +} + +impl MigrationRunner { + pub fn new(sets: Vec) -> Self { + Self { sets } + } + + fn format_step(&self, version: u32) -> Option<&dyn FormatMigration> { + self.sets + .iter() + .flat_map(|set| &set.format) + .find(|migration| migration.migrates_from() == version) + .map(|migration| migration.as_ref()) + } + + /// Chain format steps from `payloads.format_version` up to `target_version`, bumping the + /// version once per applied step. A missing step in the chain is a hard error. + pub fn run_format_migrations(&self, payloads: &mut DocumentPayloads, target_version: u32) -> Result { + let mut outcome = HistoryOutcome::Untouched; + + while payloads.format_version < target_version { + let version = payloads.format_version; + let step = self.format_step(version).ok_or(RunnerError::MissingFormatStep { at_version: version })?; + let fail = |source| RunnerError::Format { id: step.id().0, source }; + + payloads.registry = step.migrate_registry(&payloads.registry).map_err(fail)?; + + match step.history_policy() { + HistoryPolicy::Untouched => {} + HistoryPolicy::Rewrite => { + for record in &mut payloads.history { + *record = step.migrate_delta(record).map_err(fail)?; + } + outcome = outcome.max(HistoryOutcome::Rewritten); + } + HistoryPolicy::Truncate => { + payloads.history.clear(); + outcome = HistoryOutcome::Truncated; + } + } + + if let Some(session) = &payloads.session { + payloads.session = Some(step.migrate_session(session).map_err(fail)?); + } + + payloads.format_version += 1; + } + + Ok(outcome) + } + + /// Run content migrations over an open session, committing each migration's changes as one + /// retired gesture. A target that errors has its changes reverted and logged, so one bad node + /// doesn't block the rest of the document or the remaining migrations. Returns the retired revs. + pub fn run_content_migrations(&self, session: &mut Session, host: &mut dyn MigrationHost) -> Result, RunnerError> { + let mut all_revs = Vec::new(); + + for migration in self.sets.iter().flat_map(|set| &set.content) { + let selector = migration.selector(); + + // `Document`-selector migrations have no self-gating selector, so provenance gates reruns + if matches!(selector, Selector::Document) && applied_migrations(session.registry()).iter().any(|applied| applied == migration.id().0) { + continue; + } + + let targets = collect_targets(&selector, session.registry(), host); + if targets.is_empty() { + continue; + } + + // Apply per target on a shared clone, restoring the pre-target state on error + let mut target_registry = session.registry().clone(); + let mut applied = false; + for target in targets { + let backup = target_registry.clone(); + let mut context = SessionContext { + session: &mut *session, + host: &mut *host, + }; + match migration.migrate(target, &mut target_registry, &mut context) { + Ok(()) => applied = true, + Err(error) => { + log::error!("Content migration {} failed on {target:?}: {error}", migration.id()); + target_registry = backup; + } + } + } + if !applied { + continue; + } + + if matches!(selector, Selector::Document) { + record_applied(&mut target_registry, migration.id().0); + } + + // Commit the mutation as deltas and retire them as one undoable gesture + if !session.registry().value_equal(&target_registry) { + session.stage_registry_replace(&target_registry)?; + let revs = session.retire_all()?; + if let Some(last) = revs.last() { + session.mark_interaction_end(*last); + } + all_revs.extend(revs); + } + } + + Ok(all_revs) + } +} + +fn applied_migrations(registry: &Registry) -> Vec { + registry.attributes.get_typed::>(APPLIED_ATTRIBUTE).unwrap_or_default() +} + +fn record_applied(registry: &mut Registry, id: &str) { + let mut applied = applied_migrations(registry); + if !applied.iter().any(|existing| existing == id) { + applied.push(id.to_string()); + // The placeholder timestamp is re-stamped by the commit path + let _ = registry.attributes.set_serialized(APPLIED_ATTRIBUTE, &applied, TimeStamp::default()); + } +} + +/// Scan the registry for the entities a selector matches, in deterministic (sorted) order. +fn collect_targets(selector: &Selector, registry: &Registry, host: &dyn MigrationHost) -> Vec { + let node_targets = |mut nodes: Vec| { + nodes.sort_unstable(); + nodes.into_iter().map(Target::Node).collect() + }; + + match selector { + Selector::Document => vec![Target::Document], + Selector::Reference(name) => node_targets( + registry + .node_instances + .iter() + .filter(|(_, node)| node.attributes().get(attr::node::ui::REFERENCE).is_some_and(|value| value.value.as_str() == Some(name))) + .map(|(id, _)| *id) + .collect(), + ), + Selector::Node(node_selector) => node_targets( + registry + .node_instances + .iter() + .filter(|(_, node)| { + let Implementation::ProtoNode(resource_id) = node.implementation() else { return false }; + host.declaration_info(*resource_id).is_some_and(|info| node_selector.matches(&info)) + }) + .map(|(id, _)| *id) + .collect(), + ), + } +} + +/// Combines the open session (ID minting) with host services into the context migrations see. +struct SessionContext<'a> { + session: &'a mut Session, + host: &'a mut dyn MigrationHost, +} + +impl MigrationHost for SessionContext<'_> { + fn declaration_info(&self, id: ResourceId) -> Option { + self.host.declaration_info(id) + } + fn declaration(&self, id: ResourceId) -> Option { + self.host.declaration(id) + } + fn resolve_definition(&mut self, identifier: &str) -> Option { + self.host.resolve_definition(identifier) + } +} + +impl MigrationContext for SessionContext<'_> { + fn mint_node_id(&mut self) -> NodeId { + self.session.next_node_id() + } +} diff --git a/document/migrations/runner/tests/runner.rs b/document/migrations/runner/tests/runner.rs new file mode 100644 index 0000000000..0b177f10f0 --- /dev/null +++ b/document/migrations/runner/tests/runner.rs @@ -0,0 +1,145 @@ +use graph_storage::{Attributes, AttributesRead, AttributesWrite, Implementation, Network, Node, PeerId, ROOT_NETWORK, Registry, ResourceId, Session, TimeStamp}; +use migration_core::{ + ContentMigration, DeclarationInfo, FormatMigration, HistoryPolicy, MigrationContext, MigrationError, MigrationHost, MigrationId, MigrationSet, Payload, PayloadCodec, Selector, Target, +}; +use migration_runner::{DocumentPayloads, HistoryOutcome, MigrationRunner, RunnerError}; + +struct NoHost; + +impl MigrationHost for NoHost { + fn declaration_info(&self, _id: ResourceId) -> Option { + None + } + fn declaration(&self, _id: ResourceId) -> Option { + None + } + fn resolve_definition(&mut self, _identifier: &str) -> Option { + None + } +} + +/// A v1 → v2 format step that renames a top-level registry key, exercising the frozen-shape +/// decode/encode round trip. +struct RenameKey; + +impl FormatMigration for RenameKey { + fn id(&self) -> MigrationId { + MigrationId("test-rename-key") + } + fn migrates_from(&self) -> u32 { + 1 + } + fn migrate_registry(&self, registry: &Payload) -> Result { + let mut value: serde_json::Value = registry.decode()?; + if let Some(object) = value.as_object_mut() + && let Some(old) = object.remove("old_key") + { + object.insert("new_key".to_string(), old); + } + registry.encode_as(&value) + } + fn history_policy(&self) -> HistoryPolicy { + HistoryPolicy::Truncate + } +} + +fn format_set() -> MigrationSet { + MigrationSet { + format: vec![Box::new(RenameKey)], + ..Default::default() + } +} + +#[test] +fn format_tier_chains_and_bumps_version() { + let runner = MigrationRunner::new(vec![format_set()]); + let mut payloads = DocumentPayloads { + format_version: 1, + registry: Payload::encode(&serde_json::json!({ "old_key": 42 }), PayloadCodec::Json).unwrap(), + history: vec![Payload::new(vec![1, 2, 3], PayloadCodec::MessagePack)], + session: None, + }; + + let outcome = runner.run_format_migrations(&mut payloads, 2).unwrap(); + + assert_eq!(payloads.format_version, 2); + assert_eq!(outcome, HistoryOutcome::Truncated); + assert!(payloads.history.is_empty()); + let registry: serde_json::Value = payloads.registry.decode().unwrap(); + assert_eq!(registry, serde_json::json!({ "new_key": 42 })); +} + +#[test] +fn format_tier_errors_on_missing_step() { + let runner = MigrationRunner::new(vec![format_set()]); + let mut payloads = DocumentPayloads { + format_version: 0, + registry: Payload::encode(&serde_json::json!({}), PayloadCodec::Json).unwrap(), + history: Vec::new(), + session: None, + }; + + let error = runner.run_format_migrations(&mut payloads, 2).unwrap_err(); + assert!(matches!(error, RunnerError::MissingFormatStep { at_version: 0 })); +} + +/// Flags every "Brush"-referenced node with a marker attribute, exercising the delta-expressed +/// clone-mutate-diff-commit path. +struct FlagBrushNodes; + +impl ContentMigration for FlagBrushNodes { + fn id(&self) -> MigrationId { + MigrationId("test-flag-brush") + } + fn selector(&self) -> Selector { + Selector::Reference("Brush") + } + fn migrate(&self, target: Target, registry: &mut Registry, _context: &mut dyn MigrationContext) -> Result<(), MigrationError> { + let Target::Node(node_id) = target else { return Ok(()) }; + let node = registry.node_instances.get_mut(&node_id).ok_or(MigrationError::Invariant("matched node missing".to_string()))?; + node.attributes_mut().set("migrated", serde_json::Value::Bool(true), TimeStamp::default()); + Ok(()) + } +} + +fn seeded_session() -> (Session, graph_storage::NodeId) { + let mut session = Session::with_peer(PeerId(7)); + let node_id = graph_storage::NodeId(99); + + let mut registry = Registry::default(); + registry.networks.insert(ROOT_NETWORK, Network::default()); + let mut attributes = Attributes::new(); + attributes.set(graph_storage::attr::node::ui::REFERENCE, serde_json::Value::String("Brush".to_string()), TimeStamp::default()); + registry + .node_instances + .insert(node_id, Node::new(Implementation::ProtoNode(ResourceId::new()), Vec::new(), attributes, ROOT_NETWORK)); + + session.stage_registry_replace(®istry).unwrap(); + session.retire_all().unwrap(); + + (session, node_id) +} + +#[test] +fn content_tier_commits_migration_as_history_deltas() { + let runner = MigrationRunner::new(vec![MigrationSet { + content: vec![Box::new(FlagBrushNodes)], + ..Default::default() + }]); + + let (mut session, node_id) = seeded_session(); + let history_before = session.history().count(); + + let revs = runner.run_content_migrations(&mut session, &mut NoHost).unwrap(); + + assert!(!revs.is_empty()); + assert!(session.history().count() > history_before); + let node = &session.registry().node_instances[&node_id]; + assert_eq!(node.attributes().get_typed::("migrated"), Some(true)); + + // Idempotency: a second run matches the same selector but changes nothing, so no new deltas + let history_after_first = session.history().count(); + let revs = runner.run_content_migrations(&mut session, &mut NoHost).unwrap(); + assert!(revs.is_empty()); + assert_eq!(session.history().count(), history_after_first); +} diff --git a/node-graph/rfcs/document-format-migrations.md b/node-graph/rfcs/document-format-migrations.md new file mode 100644 index 0000000000..2bb6671c41 --- /dev/null +++ b/node-graph/rfcs/document-format-migrations.md @@ -0,0 +1,93 @@ +# Summary + +A migration system for `.gdd` documents, split into a trait crate (`migration-core`), self-contained migration crates, and a dispatching runner crate (`migration-runner`). Migrations come in two tiers: **format migrations** step a document's serialized payloads from one format version to the next before typed deserialization, and **content migrations** upgrade node usages on the typed `Registry` within the current format version, committed to history as ordinary deltas. Minimal dependencies per migration crate keep historic migrations cheap to build and store, and open the path to shipping them as wasm modules for an online migration service. + +# Motivation + +The legacy `.graphite` machinery mixes three mechanisms (string preprocessing, serde aliases, post-deserialize fixups), all of which keep old runtime shapes alive in the active codebase (see the Migrations section of [document-format.md](document-format.md)). Surveying it yields the operation catalog a replacement must cover: + +- Identifier renames and alias tables (~130 proto-node remaps in `NODE_REPLACEMENTS`). +- Node shape changes: add/drop/permute inputs, staged multi-version upgrades (Morph v1→v2→v3), where the input count acts as an implicit version. +- Value transforms with graph fallout: unit conversions on literal inputs, and conversion-node splices when the input is wired instead. +- Structural rewrites: node splits that preserve the original `NodeId` for reference stability (Blending → Blend Mode/Opacity/Clip), wrapper-network collapses (Brush/Transform/Image), catalog-default resets. +- Data externalization: inline image values extracted into content-addressed resources. +- Metadata normalization: `call_argument` upgrades, layout repair around inserted nodes. + +The new format adds a class the legacy system never had: stored history. Deltas embed node shapes (`AddNode`, removal snapshots), so a shape change must either rewrite stored deltas, which rehashes the Merkle `Rev` chain, or truncate history. + +# Guide-level explanation + +## Two tiers + +**Format migrations** are whole-document version steps: exactly one per `format_version` bump, keyed `migrates_from → migrates_from + 1`. They run before typed deserialization, on serialized payloads (`Payload` = bytes + codec). A format-migration crate freezes whatever old struct shapes it needs *locally*, deserializing payloads into its own mirror types; the active codebase never carries them. Retiring the migration (for example, to the online service) removes the frozen shapes from the repo. The "no old shapes" goal only ever applied to the runtime crates. + +**Content migrations** upgrade node usages within the current format version. They run after deserialization into the typed `Registry`, before `to_runtime`. Selectors: + +- `Node(DeclarationMatch)` — every node whose proto-node declaration matches by identifier (with historic aliases) or by declaration content hash. Declarations are content-addressed resources, so a hash pins an exact node version, replacing the legacy input-count sniffing. +- `Reference(name)` — every node carrying a given `ui::reference` attribute (how legacy wrapper networks like "Brush" are identified). +- `Document` — once per document, for bulk normalization passes. + +## Delta-expressed content migrations + +A content migration mutates a clone of the working registry with plain Rust. The runner diffs the clone against the working registry (`compute_deltas`), stages the difference as ordinary hot ops, and retires them as one gesture authored by the migrating peer. The upgrade is therefore recorded in history, undoable, and converges across peers like any other edit. Migrations never construct deltas by hand, so they cannot emit malformed histories. + +Timestamps inside the mutated clone are irrelevant: the diff is value-only and the commit path stamps every emitted op with fresh clock ticks, so migrations set attribute values with any placeholder timestamp. + +## History under format migrations + +Each format migration declares a `HistoryPolicy`: + +- `Untouched` — the shape change does not affect stored deltas. +- `Rewrite` — the migration transforms each delta record's payload; the runner then recomputes `Rev`s in topological order and remaps parent links and session cursors (`head`, redo stack, `last_broadcast_rev`). Because the rewrite is a pure function of content and `Rev`s are content-addressed, peers applying the same migration converge on identical rewritten history without coordination, the same dedup-by-construction argument `Merge` relies on. +- `Truncate` — no faithful rewrite exists; the document becomes a state-only snapshot (the `include_history: false` export shape). + +Migrations never touch identity fields (`id`, `parent`); Merkle bookkeeping belongs entirely to the runner. + +## Crate layout + +``` +document/migrations/ +├── core/ migration-core: traits, selectors, Payload, errors +├── runner/ migration-runner: version chaining, target scanning, delta commit, Rev rehash +└── one crate per migration era or version step, exporting `fn migrations() -> MigrationSet` +``` + +`migration-core`'s mandatory dependencies are `serde`, `serde_json`, `rmp-serde`, and `thiserror`. The `typed` feature (default) adds `graph-storage` (no default features, itself dependency-light) for the content tier; format-tier-only migration crates build without `graph-storage` entirely. Registration is explicit: each set crate exports a plain constructor and the runner aggregates. No linker-section registry (`inventory`/`linkme`), keeping wasm compilation trivial. + +# Reference-level explanation + +## Runner pipeline + +1. Read `manifest.format_version`. While it is below the target version, apply the format migration whose `migrates_from` matches (a gap is a hard error), transforming registry/history/session payloads per its policy, and bump the version. +2. Deserialize the now-current payloads into typed `graph-storage` structures. If any step declared `Rewrite`, rehash the delta DAG and remap session cursors. +3. For each content migration in registration order: scan the registry for selector matches, clone the working registry, apply the migration per target (reverting a target's changes if it errors, so one bad node doesn't poison the rest), diff, and commit as a migration gesture. +4. Hand off to `to_runtime`. + +Erroring migrations are logged and skipped rather than failing the load, matching the legacy behavior of preferring a partially-upgraded document over no document. + +## Provenance and idempotency + +Node-selector migrations are naturally self-gating: once the declaration is rewritten, the selector no longer matches. `Document`-selector migrations are guarded by a provenance list under the `migrations::applied` document attribute, which the runner appends to after a successful run (through the same diffed commit, so provenance rides history too). Content migrations must still be written idempotently, since a document may round-trip through an editor build that lacks a later migration. + +## Host services + +Content migrations reach the outside world only through a `MigrationContext` trait implemented by the host (editor or CLI): minting peer-scoped IDs, resolving declaration resources to identifiers, and instantiating current catalog defaults. This keeps migration crates independent of the editor. + +## Wasm trajectory + +The format-tier boundary is already bytes-in/bytes-out per payload, so a format migration crate compiles to a wasm module with a small shim and no host callbacks. Content migrations need the typed `Registry` and a `MigrationContext`, so wasm-shipping them means serializing the registry across the boundary and defining a small host-function surface; that design is deferred until the online migration service is scoped. + +# Rationale and alternatives + +**Frozen shapes in migration crates vs. type-erased everything.** An earlier sketch had all migrations operate on `serde_json::Value`. That fails on the actual payloads: `Rev` is a bare `u128` and MessagePack map keys are integers, neither representable in `serde_json::Value`. Typed frozen mirrors sidestep both, and they live in removable migration crates rather than the runtime. + +**Diff-based delta expression vs. hand-written deltas.** Reusing `compute_deltas` means content migrations are ordinary Rust mutations. The cost is a registry clone per migration (plus one per target for error isolation), acceptable at load time. + +**Explicit registration vs. `inventory`.** Linker-section registries complicate wasm and cross-crate builds for zero gain at this scale. + +# Future possibilities + +- CLI `migrate` subcommand for batch upgrades (the runner is already editor-independent). +- Online migration service running retired migration crates as wasm modules, letting active editors drop ancient migrations. +- Declarative rule data (alias tables, input permutations) shippable without code, once the imperative patterns stabilize. +- Per-library format versioning, as in the format RFC.