Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix: preserve null and empty strings in lossless CSV compaction (#3665)
Pramod Ubbala7ad9f809fa1b
diff --git a/crates/headroom-core/src/transforms/smart_crusher/compaction/formatter.rs b/crates/headroom-core/src/transforms/smart_crusher/compaction/formatter.rsindex 73e264036..dcb085e7b 100644--- a/crates/headroom-core/src/transforms/smart_crusher/compaction/formatter.rs+++ b/crates/headroom-core/src/transforms/smart_crusher/compaction/formatter.rs@@ -337,11 +337,14 @@ fn humanize_bytes(n: usize) -> String {fn json_scalar_to_csv(v: &Value) -> String {match v {- Value::Null => String::new(),+ // Keep null distinct from CellValue::Missing (empty field) and+ // Value::String("") (quoted empty field). Literal string "null"+ // is quoted below so these cases remain byte-distinguishable.+ Value::Null => "null".to_string(),Value::Bool(b) => if *b { "true" } else { "false" }.to_string(),Value::Number(n) => n.to_string(),Value::String(s) => {- if needs_csv_quote(s) {+ if s.is_empty() || s == "null" || needs_csv_quote(s) {csv_quote(s)} else {s.clone()@@ -726,6 +729,43 @@ mod tests {assert!(!without.contains("__dropped"));}+ #[test]+ fn csv_formatter_distinguishes_missing_null_empty_and_literal_null() {+ let c = Compaction::Table {+ schema: Schema {+ fields: vec![super::super::ir::FieldSpec {+ name: "label".into(),+ type_tag: "string".into(),+ nullable: true,+ }],+ },+ rows: vec![+ Row::new(vec![CellValue::Missing]),+ Row::new(vec![CellValue::Scalar(Value::Null)]),+ Row::new(vec![CellValue::Scalar(Value::String(String::new()))]),+ Row::new(vec![CellValue::Scalar(Value::String("null".into()))]),+ ],+ original_count: 4,+ };+ let out = CsvSchemaFormatter::new().format(&c);+ assert_eq!(out, "[4]{label:string?}\n\nnull\n\"\"\n\"null\"\n");+ }++ #[test]+ fn csv_formatter_nullable_swap_is_not_identical() {+ let a = vec![+ json!({"id": 1, "label": ""}),+ json!({"id": 2, "label": null}),+ ];+ let b = vec![+ json!({"id": 1, "label": null}),+ json!({"id": 2, "label": ""}),+ ];+ let fa = CsvSchemaFormatter::new().format(&compact(&a, &cfg()));+ let fb = CsvSchemaFormatter::new().format(&compact(&b, &cfg()));+ assert_ne!(fa, fb);+ }+#[test]fn estimate_matches_format_len() {let items = vec![json!({"a": 1}), json!({"a": 2})];