Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(transforms): detect fixed-width command output as tabular and keep it out of Kompress (#3786)

Parideboy6c9aef1dec71
diff --git a/headroom/transforms/content_detector.py b/headroom/transforms/content_detector.py
index 2d657108b..45689e84a 100644
--- a/headroom/transforms/content_detector.py
+++ b/headroom/transforms/content_detector.py
@@ -245,7 +245,13 @@ def detect_content_type(content: str) -> DetectionResult:
if code_result and code_result.confidence >= 0.5:
return code_result
- # 9. Fallback to plain text
+ # 9. Space-aligned command output (`ls -l`, `ps aux`, `docker ps`). Last,
+ # so it only claims content that would otherwise be plain text.
+ fixed_width_result = _try_detect_fixed_width(content)
+ if fixed_width_result:
+ return fixed_width_result
+
+ # 10. Fallback to plain text
return DetectionResult(ContentType.PLAIN_TEXT, 0.5, {})
@@ -716,6 +722,82 @@ def _try_detect_tabular(content: str) -> DetectionResult | None:
return _try_detect_delimited(lines)
+# Fixed-width (space-aligned) command output: `ls -l`, `ps aux`, `df -h`,
+# `docker ps`, `kubectl get`. These rows have no delimiter, so without this
+# check they fall through to PLAIN_TEXT and the prose compressor drops fields
+# out of individual rows with nothing marking which row lost what (#3652).
+_FIXED_WIDTH_LIST_RE = re.compile(r"^\s*(?:[-*+•]|\d{1,3}[.)])\s")
+_FIXED_WIDTH_PROSE_END_RE = re.compile(r"[A-Za-z][.!?][\"')\]]?$")
+_FIXED_WIDTH_CODE_ENDS = ("{", "}", ";", "(", ")", ",", ":", "\\")
+_FIXED_WIDTH_CODE_STARTS = ("#", "//", "/*", "--")
+_FIXED_WIDTH_MAX_COLS = 400
+
+
+def _fixed_width_gutters(lines: list[str]) -> int:
+ """Count the column gaps shared by at least 90% of ``lines``.
+
+ A gutter is a character position that holds a space strictly inside the
+ text (after the first non-space character; lines are right-stripped) on
+ nearly every line. Adjacent gutter positions count as one gap, so a table
+ with N columns has N - 1 gaps. Prose and code line up by accident on one
+ position at most, not on several.
+ """
+ need = -(-9 * len(lines) // 10) # ceil(0.9 * n)
+ hits = [0] * _FIXED_WIDTH_MAX_COLS
+ for ln in lines:
+ start = len(ln) - len(ln.lstrip(" "))
+ for i in range(start, min(len(ln), _FIXED_WIDTH_MAX_COLS)):
+ if ln[i] == " ":
+ hits[i] += 1
+ gaps = 0
+ in_gap = False
+ for count in hits:
+ is_gutter = count >= need
+ if is_gutter and not in_gap:
+ gaps += 1
+ in_gap = is_gutter
+ return gaps
+
+
+def _try_detect_fixed_width(content: str) -> DetectionResult | None:
+ """Detect space-aligned columns (>= 3 of them) in command output.
+
+ Runs last, only for content that would otherwise be PLAIN_TEXT, so it
+ never takes content away from another detector. A false positive costs
+ savings, not correctness: the tabular compressor passes rows it cannot
+ split cleanly through verbatim.
+ """
+ lines = [ln.rstrip() for ln in content.split("\n") if ln.strip()][:50]
+ if len(lines) < 4 or any("\t" in ln for ln in lines):
+ return None
+
+ n = len(lines)
+ if sum(1 for ln in lines if _FIXED_WIDTH_LIST_RE.match(ln)) / n >= 0.5:
+ return None
+ if sum(1 for ln in lines if _FIXED_WIDTH_PROSE_END_RE.search(ln)) / n >= 0.3:
+ return None
+ code_like = sum(
+ 1
+ for ln in lines
+ if ln.endswith(_FIXED_WIDTH_CODE_ENDS) or ln.lstrip().startswith(_FIXED_WIDTH_CODE_STARTS)
+ )
+ if code_like / n >= 0.3:
+ return None
+
+ gaps = _fixed_width_gutters(lines)
+ if gaps < 2 and n >= 5:
+ # A one-line preamble (`total 480` above `ls -l` rows) has no gaps of
+ # its own; measure the rows without it.
+ gaps = _fixed_width_gutters(lines[1:])
+ if gaps < 2:
+ return None
+ return DetectionResult(
+ ContentType.TABULAR,
+ 0.7,
+ {"format": "fixed_width", "columns": gaps + 1},
+ )
+
+
def _try_parse_toml(content: str) -> bool:
"""True if `content` parses as TOML (stdlib tomllib, or the tomli backport)."""
try:
diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.py
index 2a9c0f8a3..65ec0b37a 100644
--- a/headroom/transforms/content_router.py
+++ b/headroom/transforms/content_router.py
@@ -1173,10 +1173,16 @@ def _read_output_should_be_protected(text: Any) -> bool:
if not isinstance(text, str) or not text:
return False
try:
- return _detect_content(text).content_type not in _RELEASABLE_READ_TYPES
+ detection = _detect_content(text)
except Exception:
# Detection failure → protect (preserve the byte-exact default).
return True
+ if detection.metadata.get("format") == "fixed_width":
+ # Space-aligned columns are also what hand-aligned files look like
+ # (/etc/fstab, a block of C #defines), so a READ of one stays
+ # byte-exact. Only delimited and markdown tables are released.
+ return True
+ return detection.content_type not in _RELEASABLE_READ_TYPES
def _create_content_signature(
@@ -3941,10 +3947,13 @@ class ContentRouter(Transform):
# If compression succeeded, record to TOIN
if compressed is not None and compressed_tokens is not None:
+ # TABULAR is deliberately absent: a table the tabular compressor
+ # declined (ragged rows, #1652) or could not shrink stays verbatim.
+ # Kompress drops words inside rows, so it would remove fields from
+ # some rows and not others with nothing marking which (#3652).
fallback_eligible_strategy = strategy in {
CompressionStrategy.SMART_CRUSHER,
CompressionStrategy.CODE_AWARE,
- CompressionStrategy.TABULAR,
CompressionStrategy.CONFIG,
}
fallback_no_savings = compressed == content or compressed_tokens >= original_tokens
@@ -4024,8 +4033,8 @@ class ContentRouter(Transform):
# actually shorter — never inflating, never doing worse than the
# strategy output. DIFF is excluded (Kompress corrupts ``git
# apply``); TEXT/KOMPRESS already ran Kompress; CODE_AWARE has its
- # own inline no-shrink fallback; SMART_CRUSHER/TABULAR use the
- # zero-savings fallback above.
+ # own inline no-shrink fallback; SMART_CRUSHER uses the
+ # zero-savings fallback above; TABULAR never goes to Kompress.
if (
self._lossless_then_lossy
and compressed is not None
diff --git a/headroom/transforms/tabular_ingest.py b/headroom/transforms/tabular_ingest.py
index aabf4ad80..4767090f2 100644
--- a/headroom/transforms/tabular_ingest.py
+++ b/headroom/transforms/tabular_ingest.py
@@ -153,6 +153,11 @@ def parse_tabular(
width = len(headers)
if any(len(row) != width for row in rows):
return None
+ # The detector finds fixed-width columns by single-space gutters, but the
+ # parser splits on 2+ spaces, so a table like GNU `ls -l` can come back as
+ # one cell per line. That is not a table; leave it verbatim.
+ if fmt == "fixed_width" and width < 2:
+ return None
return headers, rows, fmt
diff --git a/tests/test_transforms_tabular.py b/tests/test_transforms_tabular.py
index 78123e181..377761a5d 100644
--- a/tests/test_transforms_tabular.py
+++ b/tests/test_transforms_tabular.py
@@ -9,6 +9,7 @@ from __future__ import annotations
import datetime
import importlib.util
+import random
import pytest
@@ -25,6 +26,7 @@ from headroom.transforms.content_router import (
CompressionStrategy,
ContentRouter,
ContentRouterConfig,
+ _read_output_should_be_protected,
)
from headroom.transforms.tabular_ingest import (
TabularCompressionResult,
@@ -93,6 +95,107 @@ def test_does_not_misroute_to_tabular(content: str, expected: ContentType) -> No
assert detect_content_type(content).content_type is expected
+# Detection: fixed-width command output (#3652) ------------------------------
+
+
+def _ls_issue_payload() -> str:
+ # The exact payload from issue #3652.
+ rng = random.Random(1)
+ rows = [
+ f"-rw-r--r-- 1 tejas staff {rng.randint(1000, 99999)} Sep {d} 09:{d:02d} file_{d}.py"
+ for d in range(1, 60)
+ ]
+ return "total 480\n" + "\n".join(rows)
+
+
+LS_MACOS = (
+ "total 64\n"
+ "drwxr-xr-x 12 tejas staff 384 Sep 18 09:01 .\n"
+ "drwxr-xr-x 5 tejas staff 160 Sep 17 11:20 ..\n"
+ "-rw-r--r-- 1 tejas staff 1834 Sep 18 09:01 README.md\n"
+ "-rw-r--r-- 1 tejas staff 18611 Sep 18 09:01 setup.py\n"
+ "drwxr-xr-x 8 tejas staff 256 Sep 18 09:01 src"
+)
+KUBECTL = (
+ "NAME READY STATUS RESTARTS AGE\n"
+ "api-7d9f8b6c4-2xkqp 1/1 Running 0 3d\n"
+ "api-7d9f8b6c4-9wz7m 1/1 Running 0 3d\n"
+ "worker-5c8d7f9b8-lq2vx 1/1 Running 2 5h\n"
+ "redis-0 1/1 Running 0 12d"
+)
+PS_AUX = (
+ "USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND\n"
+ "root 1 0.0 0.1 168100 11520 ? Ss Sep17 0:04 /sbin/init\n"
+ "root 2 0.0 0.0 0 0 ? S Sep17 0:00 [kthreadd]\n"
+ "tejas 4121 1.2 2.3 912344 190220 pts/0 Sl+ 09:01 0:12 python app.py\n"
+ "tejas 4188 0.0 0.0 10072 3300 pts/1 R+ 09:05 0:00 ps aux"
+)
+DF_H = (
+ "Filesystem Size Used Avail Use% Mounted on\n"
+ "/dev/nvme0n1p2 468G 201G 244G 46% /\n"
+ "tmpfs 16G 1.2M 16G 1% /dev/shm\n"
+ "/dev/nvme0n1p1 511M 6.1M 505M 2% /boot/efi\n"
+ "tmpfs 3.2G 2.4M 3.2G 1% /run/user/1000"
+)
+
+
+ "content",
+ [_ls_issue_payload(), LS_MACOS, KUBECTL, PS_AUX, DF_H],
+ ids=["ls_issue", "ls_macos", "kubectl", "ps_aux", "df_h"],
+)
+def test_detects_fixed_width_command_output(content: str) -> None:
+ result = detect_content_type(content)
+ assert result.content_type is ContentType.TABULAR
+ assert result.metadata["format"] == "fixed_width"
+ assert result.metadata["columns"] >= 3
+
+
+ "content,expected",
+ [
+ pytest.param(
+ "- first item in the list\n- second item in the list\n- third item in the list\n- fourth item in the list",
+ ContentType.PLAIN_TEXT,
+ id="bullets",
+ ),
+ pytest.param(
+ "1. install the package\n2. run the proxy\n3. wrap the agent\n4. check the stats",
+ ContentType.PLAIN_TEXT,
+ id="numbered",
+ ),
+ pytest.param(
+ "SELECT id, name\nFROM users\nWHERE active = 1\nORDER BY name;\n-- comment\nLIMIT 10;",
+ ContentType.PLAIN_TEXT,
+ id="sql",
+ ),
+ pytest.param(
+ "#define FOO 1\n#define BAR 2\n#define BAZ 3\n#define QUX 4",
+ ContentType.PLAIN_TEXT,
+ id="c_defines",
+ ),
+ pytest.param(
+ 'On branch main\nChanges not staged for commit:\n (use "git add <file>..." to update what will be committed)\n'
+ + "\n".join(f"\tmodified: src/m_{i}.py" for i in range(10)),
+ ContentType.PLAIN_TEXT,
+ id="git_status",
+ ),
+ pytest.param(
+ "3aa5012 perf(memory/budget): precompute word sets once\nc81378c fix(grok): preserve xAI model context metadata\nb0c19a2 fix(security): reject unauthenticated public proxy binds\n871bbde fix(proxy): reject Anthropic batch operations on Copilot\na29162b fix(dashboard): separate rolling cache economics by owner",
+ ContentType.PLAIN_TEXT,
+ id="git_log",
+ ),
+ pytest.param(
+ "Headroom compresses tool output before it reaches the model, which saves\ntokens on long agent sessions. The router picks a compressor per content\ntype, and plain prose goes to Kompress, an ML model that drops words it\npredicts the reader can do without. That is fine for prose and wrong for\nrecords, where every field matters to whatever command runs next, so the\ndetector has to tell the two apart before anything is dropped at all.",
+ ContentType.PLAIN_TEXT,
+ id="wrapped_prose",
+ ),
+ ],
+)
+def test_fixed_width_does_not_claim_non_tables(content: str, expected: ContentType) -> None:
+ assert detect_content_type(content).content_type is expected
+
+
# Detection — edge branches --------------------------------------------------
@@ -251,8 +354,8 @@ def test_parse_fixed_width_too_short_returns_empty() -> None:
def test_parse_tabular_dispatches_fixed_width(monkeypatch) -> None:
- # The detector currently emits only csv/markdown, so drive the fixed_width
- # dispatch branch directly with a stubbed detection result.
+ # Drive the fixed_width dispatch branch directly with a stubbed detection
+ # result, independent of the detector's thresholds.
import headroom.transforms.tabular_ingest as ti
monkeypatch.setattr(
@@ -266,6 +369,20 @@ def test_parse_tabular_dispatches_fixed_width(monkeypatch) -> None:
assert rows[0] == ["Alice", "30"]
+def test_parse_tabular_rejects_single_column_fixed_width(monkeypatch) -> None:
+ import headroom.transforms.tabular_ingest as ti
+
+ monkeypatch.setattr(
+ ti,
+ "detect_content_type",
+ lambda _c: DetectionResult(ContentType.TABULAR, 0.9, {"format": "fixed_width"}),
+ )
+ # Single-space rows split into one cell each; that is not a table.
+ assert (
+ ti.parse_tabular("-rw-r--r-- 1 a b 1 f\n-rw-r--r-- 1 a b 2 g\n-rw-r--r-- 1 a b 3 h") is None
+ )
+
+
def test_parse_tabular_none_when_no_data_rows_survive() -> None:
# Detected as a markdown table, but it is header + separator rows only:
# nothing survives as a data row, so parse_tabular bails to None.
@@ -342,6 +459,47 @@ def test_router_respects_disable_flag() -> None:
assert result.tokens_saved == 0
+# Router: tables never fall back to Kompress (#3652) -------------------------
+
+
+def _record_kompress_calls(monkeypatch) -> list[str]:
+ calls: list[str] = []
+
+ def fake(self, content, context, question=None, target_ratio=None):
+ calls.append(content)
+ return "x", 1 # would "win" on savings if the router ever called it
+
+ monkeypatch.setenv("HEADROOM_DETECT_BACKEND", "python")
+ monkeypatch.setattr(ContentRouter, "_try_ml_compressor", fake)
+ return calls
+
+
+def test_router_keeps_ls_output_verbatim(monkeypatch) -> None:
+ calls = _record_kompress_calls(monkeypatch)
+ payload = _ls_issue_payload()
+ result = ContentRouter(ContentRouterConfig()).compress(payload)
+ assert result.compressed == payload
+ assert calls == []
+ assert result.strategy_used is CompressionStrategy.TABULAR
+
+
+def test_router_does_not_kompress_a_ragged_csv(monkeypatch) -> None:
+ calls = _record_kompress_calls(monkeypatch)
+ csv = "id,name,city\n" + "\n".join(f"{i},user_{i},city_{i % 5}" for i in range(30))
+ csv += "\n99,extra,field,here"
+ assert detect_content_type(csv).content_type is ContentType.TABULAR
+ result = ContentRouter(ContentRouterConfig()).compress(csv)
+ assert result.compressed == csv
+ assert calls == []
+
+
+def test_fixed_width_read_stays_protected(monkeypatch) -> None:
+ monkeypatch.setenv("HEADROOM_DETECT_BACKEND", "python")
+ assert _read_output_should_be_protected(_ls_issue_payload()) is True
+ csv = "id,name,city\n" + "\n".join(f"{i},user_{i},city_{i % 5}" for i in range(30))
+ assert _read_output_should_be_protected(csv) is False
+
+
# Binary spreadsheet ingestion -----------------------------------------------