Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(tabular): split fixed-width rows on column boundaries, not on double spaces (#4541)

HetCreepb80245ea73e4
diff --git a/headroom/transforms/tabular_ingest.py b/headroom/transforms/tabular_ingest.py
index 2ea713601..e3938b350 100644
--- a/headroom/transforms/tabular_ingest.py
+++ b/headroom/transforms/tabular_ingest.py
@@ -115,14 +115,54 @@ def parse_markdown_table(content: str) -> tuple[list[str], list[list[str]]]:
def parse_fixed_width(content: str) -> tuple[list[str], list[list[str]]]:
- """Parse whitespace-aligned columns (best-effort, ≥ 2 spaces as a gap)."""
- lines = [ln for ln in content.split("\n") if ln.strip()]
- if len(lines) < 2:
+ """Parse whitespace-aligned columns by character position.
+
+ A gutter is a run of two or more positions that are blank on every line,
+ so a column is a run of positions where some line has text, and a single
+ space inside it (``First Name``, ``Alice Brown``) stays inside the cell.
+ Splitting on runs of spaces instead would let a narrow gutter merge two
+ cells while a double space inside another cell splits it, leaving the cell
+ count right and the values under the wrong columns. Returns no rows when
+ the text does not fit that grid: a header cell that is empty or holds a 2+
+ space gap means two columns were only one space apart somewhere, so the
+ caller passes the text through.
+
+ The occupied positions are collected from the characters actually present,
+ so the cost is the input length, not rows x longest row.
+ """
+ lines = [ln.rstrip() for ln in content.split("\n") if ln.strip()]
+ if len(lines) < 2 or any("\t" in ln for ln in lines):
return [], []
- splitter = re.compile(r"\s{2,}")
- headers = splitter.split(lines[0].strip())
- rows = [splitter.split(ln.strip()) for ln in lines[1:]]
- return headers, rows
+ width = max(len(ln) for ln in lines)
+ occupied = bytearray(width)
+ for ln in lines:
+ for i, ch in enumerate(ln):
+ if ch != " ":
+ occupied[i] = 1
+ spans: list[tuple[int, int]] = []
+ start = -1
+ blank_run = 0
+ for i in range(width + 2):
+ filled = i < width and occupied[i]
+ if filled:
+ if start < 0:
+ start = i
+ blank_run = 0
+ continue
+ blank_run += 1
+ if start >= 0 and blank_run == 2:
+ spans.append((start, i - 1))
+ start = -1
+ if start >= 0:
+ spans.append((start, width))
+
+ def cells(ln: str) -> list[str]:
+ return [ln[a:b].strip() for a, b in spans]
+
+ headers = cells(lines[0])
+ if any(not h or " " in h for h in headers):
+ return [], []
+ return headers, [cells(ln) for ln in lines[1:]]
def to_records(headers: list[str], rows: list[list[str]]) -> list[dict[str, str]]:
@@ -175,8 +215,7 @@ def parse_tabular(
# column of each name, and the others are dropped. Pass through verbatim.
if len(set(headers)) != width:
return None
- # The detector finds fixed-width columns by single-space gutters, but the
- # parser splits on 2+ spaces, so a table like GNU `ls -l` can come back as
+ # A table like GNU `ls -l` has no usable column grid and can come back as
# one cell per line. That is not a table; leave it verbatim.
if fmt == "fixed_width" and width < 2:
return None
diff --git a/tests/test_transforms_tabular.py b/tests/test_transforms_tabular.py
index fd8b28ce2..777ae8736 100644
--- a/tests/test_transforms_tabular.py
+++ b/tests/test_transforms_tabular.py
@@ -615,6 +615,69 @@ def test_parse_fixed_width() -> None:
assert rows[0] == ["Alice", "30", "NYC"]
+def test_parse_fixed_width_never_splits_on_double_spaces_inside_a_cell() -> None:
+ # One-space gutter between USER and COMMAND on the 4444 row, and a double
+ # space inside that row's COMMAND cell. A 2+ space split returned
+ # ['4444', 'sysadmin1 sleep', '60'] (COMMAND=60), still three cells. A
+ # one-space gap cannot be told apart from a space inside a cell, so the
+ # table is not parsed and passes through verbatim.
+ table = (
+ "PID USER COMMAND\n"
+ "1 root /sbin/init\n"
+ "22 www-data nginx\n"
+ "4444 sysadmin1 sleep 60\n"
+ "5 root cron"
+ )
+ assert parse_fixed_width(table) == ([], [])
+
+
+def test_parse_fixed_width_keeps_an_empty_cell_empty() -> None:
+ # An empty STATUS next to a NOTE with a double space used to read as
+ # STATUS='two', NOTE='words'. Here the other NOTE cells are longer, so the
+ # grid is unambiguous and the empty cell stays empty.
+ table = (
+ "NAME STATUS NOTE\n"
+ "alpha ok fine-and-dandy-things\n"
+ "beta two words\n"
+ "gamma ok fine-and-dandy-things"
+ )
+ headers, rows = parse_fixed_width(table)
+ assert headers == ["NAME", "STATUS", "NOTE"]
+ assert rows[1] == ["beta", "", "two words"]
+
+
+def test_parse_fixed_width_never_shifts_a_value_when_the_grid_is_ambiguous() -> None:
+ # 'fine' covers one of the two spaces inside 'two words', so no 2-wide
+ # gutter opens there and NOTE keeps the whole value. The one wrong answer
+ # is putting 'two' under STATUS.
+ table = (
+ "NAME STATUS NOTE\nalpha ok fine\nbeta two words\ngamma ok fine"
+ )
+ headers, rows = parse_fixed_width(table)
+ assert headers == ["NAME", "STATUS", "NOTE"]
+ assert rows[1] == ["beta", "", "two words"]
+
+
+def test_parse_fixed_width_refuses_a_row_that_crosses_a_column_boundary() -> None:
+ table = "NAME STATUS\nalphabetical ok\nbeta ok"
+ assert parse_fixed_width(table) == ([], [])
+
+
+def test_parse_tabular_fixed_width_control_matches_kubectl_columns(monkeypatch) -> None:
+ import headroom.transforms.tabular_ingest as ti
+
+ monkeypatch.setattr(
+ ti,
+ "detect_content_type",
+ lambda _c: DetectionResult(ContentType.TABULAR, 0.9, {"format": "fixed_width"}),
+ )
+ parsed = ti.parse_tabular(KUBECTL)
+ assert parsed is not None
+ headers, rows, _fmt = parsed
+ assert headers == ["NAME", "READY", "STATUS", "RESTARTS", "AGE"]
+ assert rows[2] == ["worker-5c8d7f9b8-lq2vx", "1/1", "Running", "2", "5h"]
+
+
def test_to_records_empty_headers_returns_empty() -> None:
assert to_records([], [["a", "b"]]) == []
@@ -1126,3 +1189,30 @@ def test_load_spreadsheet_missing_file(tmp_path) -> None:
with pytest.raises(FileNotFoundError):
load_spreadsheet(tmp_path / "nope.xlsx")
+
+
+def test_parse_fixed_width_keeps_multiword_headers_and_values_whole():
+ text = (
+ "First Name AGE CITY\n"
+ "Alice Brown 30 LA\n"
+ "Carol Green 25 SF\n"
+ "David White 40 NY\n"
+ "Frank Young 32 TX"
+ )
+ headers, rows = parse_fixed_width(text)
+ assert headers == ["First Name", "AGE", "CITY"]
+ assert rows[0] == ["Alice Brown", "30", "LA"]
+
+
+def test_parse_fixed_width_cost_follows_input_length():
+ # Many short rows plus one very long cell: checking every row at every
+ # position of the longest row was ~20M checks for 33 KB of input.
+ import time
+
+ text = "KEY FLAG VALUE\n" + "a b c\n" * 1000 + "a b " + "z" * 20000
+ t0 = time.perf_counter()
+ headers, rows = parse_fixed_width(text)
+ elapsed = time.perf_counter() - t0
+ assert headers == ["KEY", "FLAG", "VALUE"]
+ assert rows[-1][2] == "z" * 20000
+ assert elapsed < 0.05, f"parse took {elapsed:.3f}s for {len(text)} chars"