Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(tabular): split fixed-width rows on column boundaries, not on double spaces (#4541)
HetCreepb80245ea73e4
diff --git a/headroom/transforms/tabular_ingest.py b/headroom/transforms/tabular_ingest.pyindex 2ea713601..e3938b350 100644--- a/headroom/transforms/tabular_ingest.py+++ b/headroom/transforms/tabular_ingest.py@@ -115,14 +115,54 @@ def parse_markdown_table(content: str) -> tuple[list[str], list[list[str]]]:def parse_fixed_width(content: str) -> tuple[list[str], list[list[str]]]:- """Parse whitespace-aligned columns (best-effort, ≥ 2 spaces as a gap)."""- lines = [ln for ln in content.split("\n") if ln.strip()]- if len(lines) < 2:+ """Parse whitespace-aligned columns by character position.++ A gutter is a run of two or more positions that are blank on every line,+ so a column is a run of positions where some line has text, and a single+ space inside it (``First Name``, ``Alice Brown``) stays inside the cell.+ Splitting on runs of spaces instead would let a narrow gutter merge two+ cells while a double space inside another cell splits it, leaving the cell+ count right and the values under the wrong columns. Returns no rows when+ the text does not fit that grid: a header cell that is empty or holds a 2++ space gap means two columns were only one space apart somewhere, so the+ caller passes the text through.++ The occupied positions are collected from the characters actually present,+ so the cost is the input length, not rows x longest row.+ """+ lines = [ln.rstrip() for ln in content.split("\n") if ln.strip()]+ if len(lines) < 2 or any("\t" in ln for ln in lines):return [], []- splitter = re.compile(r"\s{2,}")- headers = splitter.split(lines[0].strip())- rows = [splitter.split(ln.strip()) for ln in lines[1:]]- return headers, rows+ width = max(len(ln) for ln in lines)+ occupied = bytearray(width)+ for ln in lines:+ for i, ch in enumerate(ln):+ if ch != " ":+ occupied[i] = 1+ spans: list[tuple[int, int]] = []+ start = -1+ blank_run = 0+ for i in range(width + 2):+ filled = i < width and occupied[i]+ if filled:+ if start < 0:+ start = i+ blank_run = 0+ continue+ blank_run += 1+ if start >= 0 and blank_run == 2:+ spans.append((start, i - 1))+ start = -1+ if start >= 0:+ spans.append((start, width))++ def cells(ln: str) -> list[str]:+ return [ln[a:b].strip() for a, b in spans]++ headers = cells(lines[0])+ if any(not h or " " in h for h in headers):+ return [], []+ return headers, [cells(ln) for ln in lines[1:]]def to_records(headers: list[str], rows: list[list[str]]) -> list[dict[str, str]]:@@ -175,8 +215,7 @@ def parse_tabular(# column of each name, and the others are dropped. Pass through verbatim.if len(set(headers)) != width:return None- # The detector finds fixed-width columns by single-space gutters, but the- # parser splits on 2+ spaces, so a table like GNU `ls -l` can come back as+ # A table like GNU `ls -l` has no usable column grid and can come back as# one cell per line. That is not a table; leave it verbatim.if fmt == "fixed_width" and width < 2:return Nonediff --git a/tests/test_transforms_tabular.py b/tests/test_transforms_tabular.pyindex fd8b28ce2..777ae8736 100644--- a/tests/test_transforms_tabular.py+++ b/tests/test_transforms_tabular.py@@ -615,6 +615,69 @@ def test_parse_fixed_width() -> None:assert rows[0] == ["Alice", "30", "NYC"]+def test_parse_fixed_width_never_splits_on_double_spaces_inside_a_cell() -> None:+ # One-space gutter between USER and COMMAND on the 4444 row, and a double+ # space inside that row's COMMAND cell. A 2+ space split returned+ # ['4444', 'sysadmin1 sleep', '60'] (COMMAND=60), still three cells. A+ # one-space gap cannot be told apart from a space inside a cell, so the+ # table is not parsed and passes through verbatim.+ table = (+ "PID USER COMMAND\n"+ "1 root /sbin/init\n"+ "22 www-data nginx\n"+ "4444 sysadmin1 sleep 60\n"+ "5 root cron"+ )+ assert parse_fixed_width(table) == ([], [])+++def test_parse_fixed_width_keeps_an_empty_cell_empty() -> None:+ # An empty STATUS next to a NOTE with a double space used to read as+ # STATUS='two', NOTE='words'. Here the other NOTE cells are longer, so the+ # grid is unambiguous and the empty cell stays empty.+ table = (+ "NAME STATUS NOTE\n"+ "alpha ok fine-and-dandy-things\n"+ "beta two words\n"+ "gamma ok fine-and-dandy-things"+ )+ headers, rows = parse_fixed_width(table)+ assert headers == ["NAME", "STATUS", "NOTE"]+ assert rows[1] == ["beta", "", "two words"]+++def test_parse_fixed_width_never_shifts_a_value_when_the_grid_is_ambiguous() -> None:+ # 'fine' covers one of the two spaces inside 'two words', so no 2-wide+ # gutter opens there and NOTE keeps the whole value. The one wrong answer+ # is putting 'two' under STATUS.+ table = (+ "NAME STATUS NOTE\nalpha ok fine\nbeta two words\ngamma ok fine"+ )+ headers, rows = parse_fixed_width(table)+ assert headers == ["NAME", "STATUS", "NOTE"]+ assert rows[1] == ["beta", "", "two words"]+++def test_parse_fixed_width_refuses_a_row_that_crosses_a_column_boundary() -> None:+ table = "NAME STATUS\nalphabetical ok\nbeta ok"+ assert parse_fixed_width(table) == ([], [])+++def test_parse_tabular_fixed_width_control_matches_kubectl_columns(monkeypatch) -> None:+ import headroom.transforms.tabular_ingest as ti++ monkeypatch.setattr(+ ti,+ "detect_content_type",+ lambda _c: DetectionResult(ContentType.TABULAR, 0.9, {"format": "fixed_width"}),+ )+ parsed = ti.parse_tabular(KUBECTL)+ assert parsed is not None+ headers, rows, _fmt = parsed+ assert headers == ["NAME", "READY", "STATUS", "RESTARTS", "AGE"]+ assert rows[2] == ["worker-5c8d7f9b8-lq2vx", "1/1", "Running", "2", "5h"]++def test_to_records_empty_headers_returns_empty() -> None:assert to_records([], [["a", "b"]]) == []@@ -1126,3 +1189,30 @@ def test_load_spreadsheet_missing_file(tmp_path) -> None:with pytest.raises(FileNotFoundError):load_spreadsheet(tmp_path / "nope.xlsx")+++def test_parse_fixed_width_keeps_multiword_headers_and_values_whole():+ text = (+ "First Name AGE CITY\n"+ "Alice Brown 30 LA\n"+ "Carol Green 25 SF\n"+ "David White 40 NY\n"+ "Frank Young 32 TX"+ )+ headers, rows = parse_fixed_width(text)+ assert headers == ["First Name", "AGE", "CITY"]+ assert rows[0] == ["Alice Brown", "30", "LA"]+++def test_parse_fixed_width_cost_follows_input_length():+ # Many short rows plus one very long cell: checking every row at every+ # position of the longest row was ~20M checks for 33 KB of input.+ import time++ text = "KEY FLAG VALUE\n" + "a b c\n" * 1000 + "a b " + "z" * 20000+ t0 = time.perf_counter()+ headers, rows = parse_fixed_width(text)+ elapsed = time.perf_counter() - t0+ assert headers == ["KEY", "FLAG", "VALUE"]+ assert rows[-1][2] == "z" * 20000+ assert elapsed < 0.05, f"parse took {elapsed:.3f}s for {len(text)} chars"