Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

chore(rust): re-land DocumentCompactor walker (squash-merge lost it from main)

chopratejas91562491a9ed
diff --git a/crates/headroom-core/src/transforms/smart_crusher/compaction/mod.rs b/crates/headroom-core/src/transforms/smart_crusher/compaction/mod.rs
index 4e5da6d12..5b94dd434 100644
--- a/crates/headroom-core/src/transforms/smart_crusher/compaction/mod.rs
+++ b/crates/headroom-core/src/transforms/smart_crusher/compaction/mod.rs
@@ -27,11 +27,13 @@ pub mod classifier;
pub mod compactor;
pub mod formatter;
pub mod ir;
+pub mod walker;
pub use classifier::{classify_cell, CellClass, ClassifyConfig};
pub use compactor::{compact, CompactConfig};
pub use formatter::{CsvSchemaFormatter, Formatter, JsonFormatter};
pub use ir::{Bucket, CellValue, Compaction, FieldSpec, OpaqueKind, Row, Schema};
+pub use walker::{compact_document, DocumentCompactor};
/// Composed compaction stage: a config + formatter pair.
///
diff --git a/crates/headroom-core/src/transforms/smart_crusher/compaction/walker.rs b/crates/headroom-core/src/transforms/smart_crusher/compaction/walker.rs
new file mode 100644
index 000000000..59e8631c1
--- /dev/null
+++ b/crates/headroom-core/src/transforms/smart_crusher/compaction/walker.rs
@@ -0,0 +1,368 @@
+//! `DocumentCompactor` — recursive walker that finds compactable spots
+//! anywhere in a JSON document and replaces them in place.
+//!
+//! # The whole algorithm in one rule
+//!
+//! ```text
+//! match value {
+//! Object(m) => recurse into each field's value
+//! Array(xs) => recurse into each item, then try TabularCompactor on the array
+//! String(s) => parse-as-JSON-and-recurse / CCR-substitute / leave
+//! scalar => unchanged
+//! }
+//! ```
+//!
+//! # Output shape
+//!
+//! Same JSON shape as input. Compacted spots become **strings** holding
+//! the rendered bytes. The wrapping object/array structure is preserved
+//! exactly — only bulky leaves get replaced.
+//!
+//! Example:
+//!
+//! ```text
+//! input: {"user": "alice", "events": [{...}, {...}, ...]}
+//! output: {"user": "alice", "events": "[50]{id:int,action:string}\n1,click\n..."}
+//! ```
+//!
+//! Nested cases cascade naturally — we recurse into the array's items
+//! BEFORE running TabularCompactor on the array, so inner sub-tables
+//! become strings first and the outer table sees them as cells.
+
+use serde_json::{Map, Value};
+
+use super::classifier::{classify_cell, CellClass};
+use super::compactor::{compact, CompactConfig};
+use super::formatter::{CsvSchemaFormatter, Formatter};
+use super::ir::OpaqueKind;
+
+use sha2::{Digest, Sha256};
+
+/// Walks any JSON value and applies lossless compaction in place.
+///
+/// Reuses the PR2 primitives:
+/// - [`compact`](super::compactor::compact) — array → IR
+/// - [`Formatter`] — IR → bytes
+/// - [`classify_cell`] + opaque-blob detection
+///
+/// The walker itself owns no compaction logic; it just decides
+/// **where** to apply each primitive in the tree.
+pub struct DocumentCompactor {
+ pub config: CompactConfig,
+ pub formatter: Box<dyn Formatter>,
+}
+
+impl Default for DocumentCompactor {
+ fn default() -> Self {
+ Self {
+ config: CompactConfig::default(),
+ formatter: Box::new(CsvSchemaFormatter::new()),
+ }
+ }
+}
+
+impl DocumentCompactor {
+ pub fn new() -> Self {
+ Self::default()
+ }
+
+ pub fn with_formatter(mut self, formatter: Box<dyn Formatter>) -> Self {
+ self.formatter = formatter;
+ self
+ }
+
+ pub fn with_config(mut self, config: CompactConfig) -> Self {
+ self.config = config;
+ self
+ }
+
+ /// Walk and compact. Returns a JSON value with the same shape but
+ /// with compactable spots replaced by rendered strings.
+ pub fn compact(&self, doc: Value) -> Value {
+ walk(doc, self)
+ }
+}
+
+fn walk(v: Value, ctx: &DocumentCompactor) -> Value {
+ match v {
+ Value::Object(map) => walk_object(map, ctx),
+ Value::Array(items) => walk_array(items, ctx),
+ Value::String(s) => walk_string(s, ctx),
+ scalar => scalar,
+ }
+}
+
+fn walk_object(map: Map<String, Value>, ctx: &DocumentCompactor) -> Value {
+ Value::Object(map.into_iter().map(|(k, v)| (k, walk(v, ctx))).collect())
+}
+
+fn walk_array(items: Vec<Value>, ctx: &DocumentCompactor) -> Value {
+ // Recurse into items FIRST so inner sub-tables / opaque markers are
+ // already in their compacted form when the outer compact runs. This
+ // is what makes deep nesting cascade — a stringified-JSON cell
+ // becomes a rendered string before the outer table sees it.
+ let inner: Vec<Value> = items.into_iter().map(|i| walk(i, ctx)).collect();
+
+ // Then try the array as a whole.
+ let c = compact(&inner, &ctx.config);
+ if c.was_compacted() {
+ Value::String(ctx.formatter.format(&c))
+ } else {
+ Value::Array(inner)
+ }
+}
+
+fn walk_string(s: String, ctx: &DocumentCompactor) -> Value {
+ // Stringified-JSON: parse, recurse, replace.
+ if let Some(parsed) = try_parse_json_container(&s) {
+ let recursed = walk(parsed, ctx);
+ return match recursed {
+ // Sub-table won — already a rendered string.
+ Value::String(rendered) => Value::String(rendered),
+ // Sub-recursion didn't compact anything; emit compact JSON.
+ other => Value::String(serde_json::to_string(&other).unwrap_or(s)),
+ };
+ }
+
+ // Long opaque blob: substitute with CCR marker.
+ if let CellClass::Opaque(kind) = classify_cell(&Value::String(s.clone()), &ctx.config.classify)
+ {
+ return Value::String(format_ccr_marker(s.as_bytes(), &kind));
+ }
+
+ Value::String(s)
+}
+
+/// Parse a string as JSON IF it looks like a container (starts with `{`
+/// or `[`) AND parses cleanly to Object/Array. Returns None otherwise —
+/// we don't recurse on bare scalars even if they parse.
+fn try_parse_json_container(s: &str) -> Option<Value> {
+ let trimmed = s.trim_start();
+ if !matches!(trimmed.chars().next(), Some('{') | Some('[')) {
+ return None;
+ }
+ serde_json::from_str::<Value>(s)
+ .ok()
+ .filter(|v| matches!(v, Value::Object(_) | Value::Array(_)))
+}
+
+fn format_ccr_marker(bytes: &[u8], kind: &OpaqueKind) -> String {
+ let mut h = Sha256::new();
+ h.update(bytes);
+ let hash: String = h
+ .finalize()
+ .iter()
+ .take(6)
+ .map(|b| format!("{b:02x}"))
+ .collect();
+ let kind_str = match kind {
+ OpaqueKind::Base64Blob => "base64",
+ OpaqueKind::LongString => "string",
+ OpaqueKind::HtmlChunk => "html",
+ OpaqueKind::Other(s) => s.as_str(),
+ };
+ format!("<<ccr:{},{},{}>>", hash, kind_str, humanize(bytes.len()))
+}
+
+fn humanize(n: usize) -> String {
+ if n < 1024 {
+ return format!("{n}B");
+ }
+ let kb = n as f64 / 1024.0;
+ if kb < 1024.0 {
+ return format!("{kb:.1}KB");
+ }
+ format!("{:.1}MB", kb / 1024.0)
+}
+
+/// Convenience: walk and compact with default config + CSV-schema
+/// formatter. Equivalent to `DocumentCompactor::new().compact(doc)`.
+pub fn compact_document(doc: Value) -> Value {
+ DocumentCompactor::new().compact(doc)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ fn dc() -> DocumentCompactor {
+ DocumentCompactor::new()
+ }
+
+ #[test]
+ fn top_level_array_of_objects_is_compacted() {
+ let doc = json!([
+ {"id": 1, "name": "alice"},
+ {"id": 2, "name": "bob"},
+ {"id": 3, "name": "carol"},
+ ]);
+ let out = dc().compact(doc);
+ match out {
+ Value::String(s) => {
+ assert!(s.starts_with("[3]{"), "got: {s}");
+ assert!(s.contains("name:string"));
+ }
+ other => panic!("expected String, got {other:?}"),
+ }
+ }
+
+ #[test]
+ fn nested_array_in_object_field_is_compacted_in_place() {
+ let doc = json!({
+ "user": "alice",
+ "events": [
+ {"id": 1, "action": "click"},
+ {"id": 2, "action": "hover"},
+ {"id": 3, "action": "submit"},
+ ],
+ });
+ let out = dc().compact(doc);
+ let obj = out.as_object().expect("object preserved");
+ assert_eq!(obj.get("user").and_then(|v| v.as_str()), Some("alice"));
+ let events = obj.get("events").and_then(|v| v.as_str()).expect("string");
+ assert!(events.starts_with("[3]{"), "got: {events}");
+ }
+
+ #[test]
+ fn deeply_nested_arrays_compact_at_every_level() {
+ let doc = json!({
+ "outer": {
+ "middle": {
+ "rows": [
+ {"a": 1, "b": "x"},
+ {"a": 2, "b": "y"},
+ ],
+ },
+ },
+ });
+ let out = dc().compact(doc);
+ let inner = out
+ .pointer("/outer/middle/rows")
+ .and_then(|v| v.as_str())
+ .expect("rows compacted to string");
+ assert!(inner.starts_with("[2]{"), "got: {inner}");
+ }
+
+ #[test]
+ fn stringified_json_in_field_is_parsed_and_compacted() {
+ let inner = r#"[{"x":1},{"x":2},{"x":3}]"#;
+ let doc = json!({
+ "id": "abc",
+ "payload": inner,
+ });
+ let out = dc().compact(doc);
+ let payload = out
+ .pointer("/payload")
+ .and_then(|v| v.as_str())
+ .expect("payload compacted");
+ assert!(payload.starts_with("[3]{"), "got: {payload}");
+ }
+
+ #[test]
+ fn long_opaque_string_at_top_level_becomes_ccr_marker() {
+ let big = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=".repeat(8);
+ let out = dc().compact(Value::String(big));
+ match out {
+ Value::String(s) => assert!(
+ s.starts_with("<<ccr:") && s.contains(",base64,"),
+ "got: {s}"
+ ),
+ other => panic!("expected String, got {other:?}"),
+ }
+ }
+
+ #[test]
+ fn long_opaque_string_inside_object_field_becomes_ccr_marker() {
+ let big = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=".repeat(8);
+ let doc = json!({"id": 1, "blob": big});
+ let out = dc().compact(doc);
+ let blob = out.pointer("/blob").and_then(|v| v.as_str()).unwrap();
+ assert!(blob.starts_with("<<ccr:"), "got: {blob}");
+ }
+
+ #[test]
+ fn pure_scalar_object_unchanged() {
+ let doc = json!({"a": 1, "b": "short", "c": true, "d": null});
+ let out = dc().compact(doc.clone());
+ assert_eq!(out, doc);
+ }
+
+ #[test]
+ fn mixed_doc_only_compactable_parts_change() {
+ let doc = json!({
+ "user_id": 42,
+ "tag": "active",
+ "events": [
+ {"id": 1, "kind": "x"},
+ {"id": 2, "kind": "y"},
+ ],
+ "config": {"region": "us", "tier": "gold"},
+ });
+ let out = dc().compact(doc);
+ // user_id and tag preserved as scalars.
+ assert_eq!(out.pointer("/user_id"), Some(&json!(42)));
+ assert_eq!(out.pointer("/tag"), Some(&json!("active")));
+ // config preserved as object (not an array, can't tabulate).
+ assert!(out
+ .pointer("/config")
+ .map(|v| v.is_object())
+ .unwrap_or(false));
+ // events compacted to a string.
+ assert!(out
+ .pointer("/events")
+ .and_then(|v| v.as_str())
+ .unwrap()
+ .starts_with("[2]{"));
+ }
+
+ #[test]
+ fn cascading_recursion_outer_table_sees_inner_compacted_string() {
+ // Each row has a stringified-JSON `payload`. After the walker
+ // recurses into items, each payload is a rendered sub-table
+ // string. The outer compact then builds a 3-row × 2-col table
+ // where the payload column holds the inner renderings.
+ let doc = json!([
+ {"id": 1, "payload": r#"[{"x":1},{"x":2},{"x":3}]"#},
+ {"id": 2, "payload": r#"[{"x":4},{"x":5}]"#},
+ ]);
+ let out = dc().compact(doc);
+ match out {
+ Value::String(s) => {
+ assert!(s.starts_with("[2]{"), "outer table: {s}");
+ // The inner-rendered sub-tables show up CSV-quoted in
+ // the payload column.
+ assert!(s.contains("[3]{") || s.contains("\"[3]{"));
+ }
+ other => panic!("expected String, got {other:?}"),
+ }
+ }
+
+ #[test]
+ fn array_of_scalars_left_alone() {
+ // Compactor declines non-object arrays → walker returns the
+ // recursed array unchanged.
+ let doc = json!([1, 2, 3, "four", 5.0]);
+ let out = dc().compact(doc.clone());
+ assert_eq!(out, doc);
+ }
+
+ #[test]
+ fn empty_object_unchanged() {
+ let doc = json!({});
+ assert_eq!(dc().compact(doc.clone()), doc);
+ }
+
+ #[test]
+ fn empty_array_unchanged() {
+ let doc = json!([]);
+ assert_eq!(dc().compact(doc.clone()), doc);
+ }
+
+ #[test]
+ fn malformed_stringified_json_left_alone() {
+ let doc = json!({"payload": "{not valid json"});
+ let out = dc().compact(doc.clone());
+ assert_eq!(out, doc);
+ }
+}