Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(relevance): bound segment size by max_chars (#2518)

Omar Gerardo0ef7b2a7fdd7
diff --git a/headroom/transforms/relevance_split.py b/headroom/transforms/relevance_split.py
index 166c2b8c2..121dc666c 100644
--- a/headroom/transforms/relevance_split.py
+++ b/headroom/transforms/relevance_split.py
@@ -12,10 +12,10 @@ Kompressed marker-free; in CCR mode the same DROP tail can be dropped with a
retrieval marker. Nothing here emits markers or calls a compressor.
Segmentation is boundary-aware, not line-based: blank lines delimit records,
-indented continuation lines stay attached to their parent (so stack traces and
-pretty-printed blobs are scored as one unit), and dense blank-free streams
-(grep, tight logs) are packed into small fixed windows. The partition is
-lossless -- ``"".join(segment(content)) == content`` -- so KEEP runs
+indented continuation lines stay attached to their parent (so short stack
+traces and pretty-printed blobs are scored as one unit), and dense blank-free
+streams (grep, tight logs) are packed into small fixed windows. The partition
+is lossless -- ``"".join(segment(content)) == content`` -- so KEEP runs
reconstruct the original bytes exactly.
"""
@@ -53,8 +53,12 @@ def segment(content: str, *, window: int = 8, max_chars: int = 1200) -> list[str
Lossless partition: ``"".join(segment(content)) == content``. Blank lines
delimit records; oversized or dense blank-free blocks are packed into
- windows of at most ``window`` lines / ``max_chars`` chars, with indented
- continuation lines held to their window so multi-line units aren't cut.
+ windows of at most ``window`` lines and ``max_chars`` chars, with indented
+ continuation lines held past the line window (still inside ``max_chars``) so
+ multi-line units aren't cut mid-run. One budget covers the whole segment, so
+ a long indented run is windowed rather than collapsed into one record. A
+ single line longer than ``max_chars`` is emitted whole, since splitting it
+ would break the partition.
"""
lines = content.splitlines(keepends=True)
if len(lines) <= 1:
@@ -76,15 +80,22 @@ def segment(content: str, *, window: int = 8, max_chars: int = 1200) -> list[str
# their window so stack traces / pretty JSON aren't split mid-unit.
segments: list[str] = []
for block in blocks:
- if len(block) <= window and sum(len(x) for x in block) <= max_chars:
- segments.append("".join(block))
- continue
i = 0
n = len(block)
while i < n:
- j = min(i + window, n)
- while j < n and block[j][:1] in (" ", "\t"):
- j += 1 # don't cut off an indented continuation run
+ # Always take at least one line (a single over-long line is atomic --
+ # splitting it would break the lossless partition), then grow while the
+ # line window or an indented continuation asks for it AND the char
+ # budget still fits. One budget covers the whole segment, so max_chars
+ # binds every multi-line record instead of only the continuation.
+ j, size = i + 1, len(block[i])
+ while (
+ j < n
+ and (j - i < window or block[j][:1] in (" ", "\t"))
+ and size + len(block[j]) <= max_chars
+ ):
+ size += len(block[j])
+ j += 1
segments.append("".join(block[i:j]))
i = j
return segments
@@ -155,6 +166,12 @@ def plan_relevance_split(
caller applies one disposition per run. Returns a single KEEP run -- i.e.
no split -- when the query is empty, the content is a single record, or it
segments into more than ``max_records`` records (a latency guard).
+
+ Note that bounding long runs makes uniformly-indented output (YAML,
+ container logs, indented JSON) segment into several records with no head
+ line, so a blob that previously stayed one whole KEEP record can now be
+ dropped in part or in full. Recoverability of a DROP run is the caller's
+ concern, not this module's.
"""
if not query.strip():
return [(True, content)]
diff --git a/tests/test_relevance_split.py b/tests/test_relevance_split.py
index 6cc560ca9..6f7e5ef5b 100644
--- a/tests/test_relevance_split.py
+++ b/tests/test_relevance_split.py
@@ -42,13 +42,70 @@ def test_segment_windows_dense_stream_losslessly():
def test_segment_keeps_indented_continuation_attached():
- # window=1 forces splitting, but indented continuation lines must stay
- # with their head line (stack-trace / pretty-JSON safety).
+ # window=1 forces splitting, but a SHORT (sub-budget) indented continuation
+ # stays with its head line (stack-trace / pretty-JSON safety). This holds
+ # only while the run fits in max_chars; a longer run is bounded and its later
+ # segments start headless, pinned by the test two below.
text = "ERROR boom\n File a.py line 1\n File b.py line 2\nnext record\n"
segs = segment(text, window=1)
assert "".join(segs) == text
for s in segs:
- assert not s.startswith((" ", "\t")) # every segment starts at a head line
+ assert not s.startswith((" ", "\t")) # sub-budget run: each segment has its head
+
+
+def test_segment_bounds_indented_continuation_by_max_chars():
+ # A long indented run (deep stack trace / indented log payload) must NOT
+ # collapse into one mega-segment: it is bounded by max_chars, not just by
+ # indentation, so the blob splits into multiple scorable records instead of
+ # being scored all-or-nothing as a single record.
+ text = "HEADER (not indented)\n" + "".join(f" frame line {i:03d}\n" for i in range(300))
+ segs = segment(text, window=8, max_chars=1200)
+ assert "".join(segs) == text # partition stays lossless
+ assert len(segs) > 2 # split into records, not one mega-segment
+ assert all(len(s) <= 1200 for s in segs) # every segment held to max_chars
+
+
+def test_segment_long_indented_run_yields_headless_segments():
+ # The cost of bounding a long run: once the budget is spent mid-run the next
+ # segment necessarily begins on an indented continuation line, so it is
+ # scored without its head line. Pinned so the trade-off is deliberate rather
+ # than an accident, and so it is not mistaken for the head-line property the
+ # sub-budget test above asserts.
+ text = "ERROR boom\n" + "".join(f" frame {i:03d} in handler\n" for i in range(200))
+ segs = segment(text, window=8, max_chars=1200)
+ assert "".join(segs) == text
+ assert len(segs) > 2
+ assert any(s.startswith((" ", "\t")) for s in segs)
+
+
+def test_segment_bounds_block_of_long_lines():
+ # The budget covers the whole segment, not only the continuation, so a block
+ # of plain (non-indented) long lines is windowed by chars rather than by line
+ # count alone. Without that, `window` lines of long text sail past max_chars.
+ text = "".join("L" * 500 + "\n" for _ in range(8))
+ segs = segment(text, window=8, max_chars=1200)
+ assert "".join(segs) == text
+ assert all(len(s) <= 1200 for s in segs)
+
+
+def test_segment_keeps_over_long_single_line_atomic():
+ # A single line longer than max_chars cannot be split without breaking the
+ # lossless partition, so it is emitted whole and the bound yields to it.
+ text = "X" * 5000 + "\n" + "short tail\n"
+ segs = segment(text, window=8, max_chars=1200)
+ assert "".join(segs) == text
+ assert len(segs[0]) == 5001
+
+
+def test_segment_bounds_continuation_across_blank_delimited_blocks():
+ # Blank lines split blocks (Pass 1); a long indented run inside one block is
+ # still windowed under max_chars (Pass 2). The whole partition stays lossless
+ # across both passes with short surrounding paragraphs intact.
+ block = "head\n" + "".join(f"\tdetail {i:03d}\n" for i in range(200))
+ text = "intro paragraph\n\n" + block + "\ntail note\n"
+ segs = segment(text, window=8, max_chars=1200)
+ assert "".join(segs) == text
+ assert all(len(s) <= 1200 for s in segs)
def test_split_keeps_relevant_drops_irrelevant():