diff --git a/Cargo.lock b/Cargo.lock index e36ea02788..c70d80b3c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -856,6 +856,20 @@ dependencies = [ "memchr", ] +[[package]] +name = "compression_test" +version = "0.0.0" +dependencies = [ + "document-container", + "document-format", + "document-graph-storage", + "futures", + "lz4_flex", + "rmp-serde", + "zstd", + "zstd-safe", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -1262,7 +1276,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -1541,7 +1555,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -3359,6 +3373,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "lz4_flex" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226" +dependencies = [ + "twox-hash", +] + [[package]] name = "lzma-rust2" version = "0.16.2" @@ -4081,7 +4104,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" dependencies = [ "libc", - "windows-sys 0.45.0", + "windows-sys 0.61.2", ] [[package]] @@ -4609,7 +4632,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -5126,7 +5149,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -5194,7 +5217,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5937,7 +5960,7 @@ dependencies = [ "getrandom 0.3.3", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6419,6 +6442,12 @@ dependencies = [ "core_maths", ] +[[package]] +name = "twox-hash" +version = "2.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5283634e518fe9e82c7b20520bb4bc209009fd16c82077c802f8111ecbb0117a" + [[package]] name = "typed-path" version = "0.12.3" @@ -7225,7 +7254,7 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0978bf7171b3d90bac376700cb56d606feb40f251a475a5d6634613564460b22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -8157,6 +8186,34 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40990edd51aae2c2b6907af74ffb635029d5788228222c4bb811e9351c0caad3" +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.1.0+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "zune-core" version = "0.4.12" diff --git a/Cargo.toml b/Cargo.toml index 194897cbdb..6f13fbbc1f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,7 +33,7 @@ members = [ "tools/crate-hierarchy-viz", "tools/third-party-licenses", "tools/editor-message-tree", - "tools/node-docs", + "tools/node-docs", "tools/compression_test", ] default-members = [ "editor", diff --git a/tools/compression_test/Cargo.toml b/tools/compression_test/Cargo.toml new file mode 100644 index 0000000000..8d095e546b --- /dev/null +++ b/tools/compression_test/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "compression_test" +rust-version.workspace = true +edition.workspace = true +authors.workspace = true +homepage.workspace = true +repository.workspace = true +license.workspace = true +version.workspace = true +readme.workspace = true +publish.workspace = true + +[dependencies] +document-format.workspace = true +document-container.workspace = true +document-graph-storage.workspace = true +rmp-serde.workspace = true +lz4_flex = { version = "0.14.0", features = ["alloc"] } +futures.workspace = true +zstd = "0.13" +zstd-safe = "7" + +[lints] +workspace = true diff --git a/tools/compression_test/src/main.rs b/tools/compression_test/src/main.rs new file mode 100644 index 0000000000..9f1d040feb --- /dev/null +++ b/tools/compression_test/src/main.rs @@ -0,0 +1,310 @@ +use std::collections::HashMap; +use std::io::Write; +use std::path::Path; + +use document_container::{AnyContainer, backends::memory::MemoryBackend}; +use document_format::{Codec, Gdd, GddV1Layout}; +use document_graph_storage::{Delta, NodeInput, RegistryDelta}; +use lz4_flex::frame::{BlockMode, FrameEncoder, FrameInfo}; + +fn main() { + let document_path = Path::new("value-diff-test.gdd"); + let archive = std::fs::read(document_path).unwrap(); + let memory = MemoryBackend::new(); + let container = AnyContainer::Memory(memory); + let gdd = futures::executor::block_on(Gdd::open_from_archive(archive.as_ref(), container, GddV1Layout)).unwrap(); + + let deltas: Vec<&Delta> = gdd.session().history().collect(); + + // What `history.frames` holds today: every delta as a length-prefixed MessagePack frame. + let mut current = Vec::new(); + for delta in &deltas { + Codec::MessagePackFrames.append(&mut current, delta).unwrap(); + } + let frames: Vec> = deltas.iter().map(|delta| rmp_serde::to_vec(delta).unwrap()).collect(); + let frames_total: usize = frames.iter().map(Vec::len).sum(); + + println!("history: {} deltas", deltas.len()); + println!("archive (.gdd, xz): {:>9} bytes", archive.len()); + println!("history.frames as stored today: {:>9} bytes", current.len()); + println!(); + + let report = |label: &str, bytes: usize| { + println!("{label:<42}{bytes:>9} bytes {:>6.2}x smaller than today", current.len() as f64 / bytes as f64); + }; + + report("each delta lz4 independently", independent(&frames)); + report("one linked lz4 chain over the whole history", linked_chain(&frames).iter().sum()); + report("linked lz4 chain per node input", per_input_chains(&deltas, &frames)); + report("whole history as one lz4 frame (reference)", lz4_flex::frame::FrameEncoder::new(Vec::new()).write_and_finish(¤t)); + report("zstd chain level 3 (16 MiB window)", zstd_chain(&frames, 3).iter().sum()); + report("zstd chain level 9 (16 MiB window)", zstd_chain(&frames, 9).iter().sum()); + report("zstd chain level 19 (16 MiB window)", zstd_chain(&frames, 19).iter().sum()); + + // Only the new value: what a store that doesn't persist `reverse` would need. + let kinds: Vec> = deltas.iter().map(|delta| rmp_serde::to_vec(&delta.kind).unwrap()).collect(); + let kinds_total: usize = kinds.iter().map(Vec::len).sum(); + println!(); + println!("`kind` only (dropping `reverse`): {kinds_total:>9} bytes ({frames_total} with reverse)"); + report("linked lz4 chain over `kind` only", linked_chain(&kinds).iter().sum()); + + // Detail for the most-edited large input. + let key = |delta: &Delta| match &delta.kind { + RegistryDelta::ChangeNodeInput { + id, + index, + new_input: NodeInput::Value { .. }, + } => Some((*id, *index)), + _ => None, + }; + let mut per_key: HashMap<_, Vec> = HashMap::new(); + for (i, delta) in deltas.iter().enumerate() { + if let Some(k) = key(delta) { + per_key.entry(k).or_default().push(i); + } + } + let (k, indices) = per_key.iter().max_by_key(|(_, v)| v.iter().map(|&i| frames[i].len()).sum::()).unwrap(); + let payloads: Vec> = indices.iter().map(|&i| frames[i].clone()).collect(); + let ind = payloads.iter().map(|p| lz4_flex::block::compress(p).len()).collect::>(); + let lnk = linked_chain(&payloads); + let zst = zstd_chain(&payloads, 3); + let pred = predecessor_only(&payloads, 3); + println!(); + println!("largest input {k:?}: {} versions", payloads.len()); + println!(" {:>4} {:>8} {:>7} {:>10} {:>10} {:>11} {:>9} {:>10}", "#", "raw", "growth", "lz4-indep", "lz4-suffix", "zstd-suffix", "zstd/new", "pred-only"); + let mut prev_raw = 0; + for (i, ((((p, ind), lnk), zst), pred)) in payloads.iter().zip(&ind).zip(&lnk).zip(&zst).zip(&pred).enumerate().take(40) { + let growth = p.len() - prev_raw; + println!(" {i:>4} {:>8} {growth:>7} {ind:>10} {lnk:>10} {zst:>11} {:>8.1}% {pred:>10}", p.len(), 100.0 * *zst as f64 / growth as f64); + prev_raw = p.len(); + } + let growth_total = payloads.last().unwrap().len(); + let zst_total: usize = zst.iter().sum(); + let pred_total: usize = pred.iter().sum(); + println!(" new bytes total: {growth_total} lz4 suffix total: {} zstd suffix total: {zst_total} predecessor-only total: {pred_total}", lnk.iter().sum::()); + println!(" zstd suffix / new bytes: {:.1}% (compression overhead relative to storing only the raw new bytes)", 100.0 * zst_total as f64 / growth_total as f64); + println!(" predecessor-only / new bytes: {:.1}%", 100.0 * pred_total as f64 / growth_total as f64); + + // Predecessor-only over the whole history: group by input, each version sees only the previous + // version of the same input; non-value deltas are compressed standalone. + let mut by_input: HashMap<_, Vec>> = HashMap::new(); + let mut standalone = Vec::new(); + for (delta, frame) in deltas.iter().zip(&frames) { + match key(delta) { + Some(k) => by_input.entry(k).or_default().push(frame.clone()), + None => standalone.push(frame.clone()), + } + } + let pred_history: usize = by_input.values().map(|p| predecessor_only(p, 3).iter().sum::()).sum::() + standalone.iter().map(|f| predecessor_only(std::slice::from_ref(f), 3)[0]).sum::(); + report("predecessor-only zstd (per input, level 3)", pred_history); + + // Same estimate over the whole history: "new bytes" = growth vs. the previous version of the same + // input, or the full frame for the first version / non-value deltas. + let mut last_len: HashMap<_, usize> = HashMap::new(); + let mut new_bytes_total = 0; + for (delta, frame) in deltas.iter().zip(&frames) { + new_bytes_total += match key(delta) { + Some(k) => { + let prev = last_len.insert(k, frame.len()).unwrap_or(0); + frame.len().saturating_sub(prev) + } + None => frame.len(), + }; + } + let zstd_total: usize = zstd_chain(&frames, 3).iter().sum(); + println!(); + println!("whole history: new bytes {new_bytes_total} zstd chain {zstd_total} ratio {:.1}%", 100.0 * zstd_total as f64 / new_bytes_total as f64); + + // Micro-bench: time to get at the *last* payload. For the streaming chain that means decoding the + // entire stream (block n references everything before it); for predecessor-only it means walking the + // per-input chain from version 0. + let iterations = 100; + println!(); + println!("decode-to-last-payload, {iterations} iterations (debug build):"); + + let (_, chain_largest) = zstd_chain_with_output(&payloads, 3); + let expected_last = payloads.last().unwrap().len(); + bench(&format!("zstd chain, largest input ({} versions, {} B)", payloads.len(), chain_largest.len()), iterations, || { + let decoded = zstd::decode_all(chain_largest.as_slice()).unwrap(); + assert_eq!(decoded.len(), payloads.iter().map(Vec::len).sum::()); + expected_last + }); + + let pred_frames = predecessor_only_frames(&payloads, 3); + let pred_bytes: usize = pred_frames.iter().map(Vec::len).sum(); + bench(&format!("predecessor-only, largest input ({} frames, {} B)", pred_frames.len(), pred_bytes), iterations, || { + let last = decode_predecessor_chain(&pred_frames); + assert_eq!(&last, payloads.last().unwrap()); + last.len() + }); + + let (_, chain_all) = zstd_chain_with_output(&frames, 3); + bench(&format!("zstd chain, whole history ({} deltas, {} B)", frames.len(), chain_all.len()), iterations, || { + let decoded = zstd::decode_all(chain_all.as_slice()).unwrap(); + assert_eq!(decoded.len(), frames_total); + frames.last().unwrap().len() + }); + + let lz4_all = lz4_flex::frame::FrameEncoder::new(Vec::new()).write_and_finish_bytes(¤t); + bench(&format!("lz4 whole history, for reference ({} B)", lz4_all.len()), iterations, || { + let mut out = Vec::with_capacity(current.len()); + std::io::copy(&mut lz4_flex::frame::FrameDecoder::new(lz4_all.as_slice()), &mut out).unwrap(); + out.len() + }); +} + +trait WriteAndFinish { + fn write_and_finish_bytes(self, bytes: &[u8]) -> Vec; + fn write_and_finish(self, bytes: &[u8]) -> usize; +} +impl WriteAndFinish for FrameEncoder> { + fn write_and_finish_bytes(mut self, bytes: &[u8]) -> Vec { + self.write_all(bytes).unwrap(); + self.finish().unwrap() + } + fn write_and_finish(self, bytes: &[u8]) -> usize { + self.write_and_finish_bytes(bytes).len() + } +} + +/// Baseline: every payload compressed on its own, no shared context. +fn independent(payloads: &[Vec]) -> usize { + payloads.iter().map(|p| lz4_flex::block::compress(p).len()).sum() +} + +/// Streaming approach: one linked frame, `flush()` after each payload. +/// The bytes emitted between two flushes are the suffix that would be stored for that block. +fn linked_chain(payloads: &[Vec]) -> Vec { + let mut frame_info = FrameInfo::new(); + frame_info.block_mode = BlockMode::Linked; + let mut encoder = FrameEncoder::with_frame_info(frame_info, Vec::new()); + + let mut sizes = Vec::with_capacity(payloads.len()); + let mut previous = 0; + for p in payloads { + encoder.write_all(p).unwrap(); + encoder.flush().unwrap(); + let len = encoder.get_ref().len(); + sizes.push(len - previous); + previous = len; + } + // The end mark is stored once for the whole chain, not attributed to any block. + encoder.finish().unwrap(); + sizes +} + +/// Same as [`linked_chain`] but with zstd streaming and a window large enough to hold many +/// previous versions (`window_log` 24 = 16 MiB). `flush()` emits a zstd block boundary without +/// resetting the window, so the bytes between flushes are the suffix for that payload. +fn zstd_chain(payloads: &[Vec], level: i32) -> Vec { + zstd_chain_with_output(payloads, level).0 +} + +fn zstd_chain_with_output(payloads: &[Vec], level: i32) -> (Vec, Vec) { + let mut encoder = zstd::Encoder::new(Vec::new(), level).unwrap(); + encoder.window_log(24).unwrap(); + // Long-distance matching finds the "previous payload as prefix" match even far back. + encoder.long_distance_matching(true).unwrap(); + + let mut sizes = Vec::with_capacity(payloads.len()); + let mut previous = 0; + for p in payloads { + encoder.write_all(p).unwrap(); + encoder.flush().unwrap(); + let len = encoder.get_ref().len(); + sizes.push(len - previous); + previous = len; + } + let out = encoder.finish().unwrap(); + (sizes, out) +} + +/// Predecessor-only, but also returning the compressed frames so they can be decoded back. +fn predecessor_only_frames(payloads: &[Vec], level: i32) -> Vec> { + use zstd_safe::{CCtx, CParameter}; + let mut frames = Vec::with_capacity(payloads.len()); + let mut previous: &[u8] = &[]; + for p in payloads { + let mut cctx = CCtx::create(); + cctx.set_parameter(CParameter::CompressionLevel(level)).unwrap(); + cctx.set_parameter(CParameter::WindowLog(24)).unwrap(); + cctx.set_parameter(CParameter::EnableLongDistanceMatching(true)).unwrap(); + cctx.ref_prefix(previous).unwrap(); + let mut out = Vec::with_capacity(zstd_safe::compress_bound(p.len())); + cctx.compress2(&mut out, p).unwrap(); + frames.push(out); + previous = p; + } + frames +} + +/// Walk a predecessor-only chain from the first frame to the last, returning the last payload. +fn decode_predecessor_chain(frames: &[Vec]) -> Vec { + use zstd_safe::DCtx; + let mut previous = Vec::new(); + for frame in frames { + let size = zstd_safe::get_frame_content_size(frame).unwrap().unwrap() as usize; + let mut out = Vec::with_capacity(size); + { + // `DCtx<'a>` borrows the prefix until dropped, so it must go before `previous` is reassigned. + let mut dctx = DCtx::create(); + dctx.set_parameter(zstd_safe::DParameter::WindowLogMax(24)).unwrap(); + dctx.ref_prefix(&previous).unwrap(); + dctx.decompress(&mut out, frame).unwrap(); + } + previous = out; + } + previous +} + +fn bench(label: &str, iterations: u32, mut f: impl FnMut() -> usize) { + let mut checksum = 0; + let start = std::time::Instant::now(); + for _ in 0..iterations { + checksum = checksum.max(f()); + } + let elapsed = start.elapsed(); + println!(" {label:<52} {:>9.3} ms / iter (last payload {checksum} bytes)", elapsed.as_secs_f64() * 1000.0 / iterations as f64); +} + +/// Each payload compressed as its own zstd frame, with only its immediate predecessor supplied as a +/// prefix dictionary (`ZSTD_CCtx_refPrefix`). No stream state is shared: decoding version `n` needs +/// just the raw bytes of version `n-1`. `window_log` must cover prefix + payload for the match to be +/// representable. +fn predecessor_only(payloads: &[Vec], level: i32) -> Vec { + use zstd_safe::{CCtx, CParameter}; + let mut sizes = Vec::with_capacity(payloads.len()); + let mut previous: &[u8] = &[]; + let mut out = Vec::new(); + for p in payloads { + let mut cctx = CCtx::create(); + cctx.set_parameter(CParameter::CompressionLevel(level)).unwrap(); + cctx.set_parameter(CParameter::WindowLog(24)).unwrap(); + cctx.set_parameter(CParameter::EnableLongDistanceMatching(true)).unwrap(); + cctx.ref_prefix(previous).unwrap(); + out.clear(); + out.reserve(zstd_safe::compress_bound(p.len())); + let n = cctx.compress2(&mut out, p).unwrap(); + sizes.push(n); + previous = p; + } + sizes +} + +/// One linked chain per `(node, input)` for value changes; everything else goes into a shared chain. +/// Keeps consecutive versions of the same value adjacent in the window regardless of interleaving. +fn per_input_chains(deltas: &[&Delta], frames: &[Vec]) -> usize { + let mut groups: HashMap, Vec>> = HashMap::new(); + for (delta, frame) in deltas.iter().zip(frames) { + let key = match &delta.kind { + RegistryDelta::ChangeNodeInput { + id, + index, + new_input: NodeInput::Value { .. }, + } => Some((*id, *index)), + _ => None, + }; + groups.entry(key).or_default().push(frame.clone()); + } + groups.values().map(|payloads| linked_chain(payloads).iter().sum::()).sum() +} diff --git a/tools/compression_test/value-diff-test.gdd b/tools/compression_test/value-diff-test.gdd new file mode 100644 index 0000000000..16d2ddd272 Binary files /dev/null and b/tools/compression_test/value-diff-test.gdd differ