Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(smart-crusher): keep string cells in json columns from decoding as other types (#4053)

Suliman Abdulrazzaq3db1a9444340
diff --git a/crates/headroom-core/src/transforms/smart_crusher/compaction/formatter.rs b/crates/headroom-core/src/transforms/smart_crusher/compaction/formatter.rs
index dcb085e7b..22771448f 100644
--- a/crates/headroom-core/src/transforms/smart_crusher/compaction/formatter.rs
+++ b/crates/headroom-core/src/transforms/smart_crusher/compaction/formatter.rs
@@ -282,17 +282,31 @@ fn write_table(
}
out.push('\n');
+ // Cells of a `json` column are decoded with `json.loads`, so a string
+ // that looks like a JSON literal needs its own quoting there.
+ let json_cols: Vec<bool> = schema.fields.iter().map(|f| f.type_tag == "json").collect();
+
// Rows.
for row in rows {
- let cells: Vec<String> = row.0.iter().map(format_cell).collect();
+ let cells: Vec<String> = row
+ .0
+ .iter()
+ .enumerate()
+ .map(|(i, c)| format_cell(c, json_cols.get(i).copied().unwrap_or(false)))
+ .collect();
out.push_str(&cells.join(","));
out.push('\n');
}
}
-fn format_cell(c: &CellValue) -> String {
+fn format_cell(c: &CellValue, json_col: bool) -> String {
match c {
CellValue::Missing => String::new(),
+ CellValue::Scalar(Value::String(s)) if json_col && is_ambiguous_json_text(s) => {
+ // Render as a JSON string literal (then CSV-quoted) so the
+ // decoder gets this string back, not the value it spells.
+ csv_quote(&serde_json::to_string(s).unwrap_or_default())
+ }
CellValue::Scalar(v) => json_scalar_to_csv(v),
CellValue::Nested(sub) => {
// Render nested as compact JSON; CSV-quote because it
@@ -356,6 +370,17 @@ fn json_scalar_to_csv(v: &Value) -> String {
}
}
+/// True when `json.loads` could parse `s` into something other than the
+/// string itself. A conservative superset: after trimming JSON white
+/// space, `s` starts with `[`, `{`, `"`, `-` or a digit, or is one of the
+/// bare literals Python accepts (`true`, `false`, `null`, `NaN`,
+/// `Infinity`).
+fn is_ambiguous_json_text(s: &str) -> bool {
+ let t = s.trim_matches(|c| matches!(c, ' ' | '\t' | '\n' | '\r'));
+ matches!(t.chars().next(), Some('[' | '{' | '"' | '-' | '0'..='9'))
+ || matches!(t, "true" | "false" | "null" | "NaN" | "Infinity")
+}
+
fn needs_csv_quote(s: &str) -> bool {
s.contains(',') || s.contains('"') || s.contains('\n') || s.contains('\r')
}
@@ -766,6 +791,118 @@ mod tests {
assert_ne!(fa, fb);
}
+ /// Wire text of the single data row of a one-column table.
+ fn one_cell_wire(type_tag: &str, cell: Value) -> String {
+ let c = Compaction::Table {
+ schema: Schema {
+ fields: vec![super::super::ir::FieldSpec {
+ name: "v".into(),
+ type_tag: type_tag.into(),
+ nullable: false,
+ }],
+ },
+ rows: vec![Row::new(vec![CellValue::Scalar(cell)])],
+ original_count: 1,
+ };
+ let out = CsvSchemaFormatter::new().format(&c);
+ out.lines().nth(1).unwrap().to_string()
+ }
+
+ #[test]
+ fn csv_formatter_json_column_quotes_strings_that_parse_as_json() {
+ // The cells are built by hand to pin the formatter's contract. Through
+ // `compact`, a string that parses as a JSON array or object (`[1]`,
+ // `{"a":1}`) never reaches the formatter as a string: the classifier
+ // un-escapes it first (`CellClass::StringifiedJson`).
+ // (string value, the JSON string literal it is rendered as)
+ let cases = [
+ ("null", r#""null""#),
+ ("true", r#""true""#),
+ ("false", r#""false""#),
+ ("1", r#""1""#),
+ ("-1", r#""-1""#),
+ ("1.5", r#""1.5""#),
+ ("NaN", r#""NaN""#),
+ ("Infinity", r#""Infinity""#),
+ ("-Infinity", r#""-Infinity""#),
+ (" 2 ", r#"" 2 ""#),
+ ("[1]", r#""[1]""#),
+ (r#"{"a":1}"#, r#""{\"a\":1}""#),
+ (r#""q""#, r#""\"q\"""#),
+ ];
+ for (s, literal) in cases {
+ let wire = one_cell_wire("json", Value::String(s.into()));
+ assert_eq!(wire, csv_quote(literal), "json column, string {s:?}");
+ // The control: other column types keep the old rendering.
+ let control = one_cell_wire("string", Value::String(s.into()));
+ assert_eq!(
+ control,
+ json_scalar_to_csv(&Value::String(s.into())),
+ "string column, string {s:?}"
+ );
+ }
+ assert_eq!(
+ one_cell_wire("json", Value::String("null".into())),
+ r#""""null""""#
+ );
+ assert_eq!(
+ one_cell_wire("string", Value::String("null".into())),
+ r#""null""#
+ );
+ assert_eq!(
+ one_cell_wire("string", Value::String("true".into())),
+ "true"
+ );
+ }
+
+ #[test]
+ fn csv_formatter_json_column_leaves_other_strings_alone() {
+ let cases = [
+ ("hello", "hello"),
+ ("a,b", r#""a,b""#),
+ (r#"q"uote"#, r#""q""uote""#),
+ (" ", " "),
+ ("x-y", "x-y"),
+ ("", r#""""#),
+ ];
+ for (s, wire) in cases {
+ assert_eq!(
+ one_cell_wire("json", Value::String(s.into())),
+ wire,
+ "json column, string {s:?}"
+ );
+ }
+ }
+
+ #[test]
+ fn csv_formatter_json_column_tells_strings_from_values() {
+ let items = vec![
+ json!({"id": 1, "v": "null"}),
+ json!({"id": 2, "v": null}),
+ json!({"id": 3, "v": "true"}),
+ json!({"id": 4, "v": true}),
+ json!({"id": 5, "v": "1"}),
+ json!({"id": 6, "v": 1}),
+ json!({"id": 7, "v": "hello"}),
+ ];
+ let c = compact(&items, &cfg());
+ let out = CsvSchemaFormatter::new().format(&c);
+ let lines: Vec<&str> = out.lines().collect();
+ assert_eq!(lines[0], "[7]{id:int,v:json?}", "got: {out}");
+ assert_eq!(lines[1], r#"1,"""null""""#);
+ assert_eq!(lines[2], "2,null");
+ assert_eq!(lines[3], r#"3,"""true""""#);
+ assert_eq!(lines[4], "4,true");
+ assert_eq!(lines[5], r#"5,"""1""""#);
+ assert_eq!(lines[6], "6,1");
+ assert_eq!(lines[7], "7,hello");
+ // The string "true" and the bool true no longer share a wire form.
+ assert_ne!(
+ lines[3].split_once(',').unwrap().1,
+ lines[4].split_once(',').unwrap().1
+ );
+ }
+
#[test]
fn estimate_matches_format_len() {
let items = vec![json!({"a": 1}), json!({"a": 2})];
diff --git a/tests/test_transforms/test_compaction_codec.py b/tests/test_transforms/test_compaction_codec.py
index 67d100f56..45476b59e 100644
--- a/tests/test_transforms/test_compaction_codec.py
+++ b/tests/test_transforms/test_compaction_codec.py
@@ -173,6 +173,29 @@ def test_trailing_newline_beyond_declared_count_is_trimmed() -> None:
assert expand_compacted("[2]{k:string?}\na\nb\n\n") == [{"k": "a"}, {"k": "b"}]
+def test_json_column_string_literals_stay_strings_and_values_keep_their_type() -> None:
+ # In a `json` column the native encoder writes a string that looks like a
+ # JSON literal as a quoted JSON string (`"""null"""` is the string `null`),
+ # while real values stay bare. The decoder must tell the two apart.
+ block = (
+ "[9]{id:int,v:json?}\n"
+ '1,"""null"""\n'
+ "2,null\n"
+ '3,"""true"""\n'
+ "4,true\n"
+ '5,"""1"""\n'
+ "6,1\n"
+ '7,"""NaN"""\n'
+ '8,"""[1]"""\n'
+ "9,hello\n"
+ )
+ decoded = expand_compacted(block)
+ assert decoded is not None
+ values = [row["v"] for row in decoded]
+ assert values == ["null", None, "true", True, "1", 1, "NaN", "[1]", "hello"]
+ assert [type(v) for v in values] == [str, type(None), str, bool, str, int, str, str, str]
+
+
def test_single_column_null_survives_value_factoring() -> None:
block = "[3]{k:string?}\nx\ny\nnull\n"
factored = factor_values(block)