Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(tabular): keep empty records and separator-like Markdown body rows (#4582)
Parideboy5cebb90db1f1
diff --git a/headroom/transforms/tabular_ingest.py b/headroom/transforms/tabular_ingest.pyindex 35dcfd0ef..32967136c 100644--- a/headroom/transforms/tabular_ingest.py+++ b/headroom/transforms/tabular_ingest.py@@ -74,7 +74,7 @@ def parse_csv(content: str, delimiter: str = ",") -> tuple[list[str], list[list["""reader = csv.reader(io.StringIO(content), delimiter=delimiter)try:- parsed = [row for row in reader if any(cell.strip() for cell in row)]+ parsed = [row for row in reader if row and not (len(row) == 1 and not row[0].strip())]except csv.Error:return [], []if not parsed:@@ -97,7 +97,10 @@ def parse_markdown_table(content: str) -> tuple[list[str], list[list[str]]]:if len(lines) < 2:return [], []headers = split_row(lines[0])- rows = [split_row(ln) for ln in lines[1:] if not is_separator(ln)]+ body_lines = lines[1:]+ if body_lines and is_separator(body_lines[0]):+ body_lines = body_lines[1:]+ rows = [split_row(ln) for ln in body_lines]return headers, rowsdiff --git a/tests/test_transforms_tabular.py b/tests/test_transforms_tabular.pyindex e465db8bc..d1f3b962b 100644--- a/tests/test_transforms_tabular.py+++ b/tests/test_transforms_tabular.py@@ -410,6 +410,30 @@ def test_parse_markdown_table_drops_separator() -> None:assert all("---" not in cell for row in rows for cell in row)+def test_parse_csv_keeps_empty_records() -> None:+ headers, rows = parse_csv("a,b\n1,2\n,\n3,4")+ assert headers == ["a", "b"]+ assert rows == [["1", "2"], ["", ""], ["3", "4"]]+++def test_parse_csv_keeps_empty_records_tsv() -> None:+ headers, rows = parse_csv("a\tb\n1\t2\n\t\n3\t4", delimiter="\t")+ assert headers == ["a", "b"]+ assert rows == [["1", "2"], ["", ""], ["3", "4"]]+++def test_parse_csv_keeps_quoted_empty_record() -> None:+ headers, rows = parse_csv('a,b\n"",""\n1,2')+ assert headers == ["a", "b"]+ assert rows[0] == ["", ""]+++def test_parse_markdown_table_keeps_separator_like_body_row() -> None:+ headers, rows = parse_markdown_table("| a | b |\n|---|---|\n| 1 | 2 |\n| --- | --- |")+ assert headers == ["a", "b"]+ assert rows == [["1", "2"], ["---", "---"]]++def test_parse_tabular_rejects_ragged_fixed_width(monkeypatch) -> None:# Rows with differing cell counts can't be zipped under the headers# without misattributing columns (#1652) — must pass through.@@ -562,7 +586,7 @@ def test_parse_tabular_rejects_single_column_fixed_width(monkeypatch) -> None:def test_parse_tabular_none_when_no_data_rows_survive() -> None:# Detected as a markdown table, but it is header + separator rows only:# nothing survives as a data row, so parse_tabular bails to None.- assert parse_tabular("| a | b |\n| --- | --- |\n| --- | --- |") is None+ assert parse_tabular("| a | b |\n| --- | --- |") is Nonedef test_compression_ratio_zero_for_empty_original() -> None: