Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(kompress): keep the boolean connectives that hold a condition together (#3687)

sxhbf290ba9a9de
diff --git a/headroom/transforms/kompress_compressor.py b/headroom/transforms/kompress_compressor.py
index 6a076ee97..c220f7cdb 100644
--- a/headroom/transforms/kompress_compressor.py
+++ b/headroom/transforms/kompress_compressor.py
@@ -68,6 +68,16 @@ _KOMPRESS_MUST_KEEP_RE = re.compile(
r"|(?i:\b(?:not|never|none|cannot|can't|don't|doesn't|didn't|won't|shouldn't"
r"|mustn't|isn't|aren't|avoid|refuse|prohibited|forbidden|disallow|unless"
r"|except|without|must|should|shall|required|always|only|mandatory)\b)"
+ # Boolean connectives, for the same reason as the directive words above:
+ # they decide WHICH predicates have to hold, so dropping one does not
+ # weaken the statement, it changes the condition. Negation was pinned but
+ # conjunction was not, so one line could lose its `or` and keep its `not`
+ # (issue #3545: 12 of 40 `or` lost from a repeated Python return line,
+ # while the same word survived 48/48 in prose). The surviving text is the
+ # dangerous part -- `a == b not c.startswith(d)` still reads as code, and
+ # nothing marks it as altered. Uppercase `AND`/`OR` were already held by
+ # the ALLCAPS class; only the lowercase forms leaked.
+ r"|(?i:\b(?:and|or|nor|xor)\b)"
)
_KOMPRESS_MUST_KEEP_ENV = "HEADROOM_KOMPRESS_MUST_KEEP"
KOMPRESS_BACKEND_ENV = "HEADROOM_KOMPRESS_BACKEND"
diff --git a/tests/test_kompress_must_keep.py b/tests/test_kompress_must_keep.py
index 9bab82792..953cdc04c 100644
--- a/tests/test_kompress_must_keep.py
+++ b/tests/test_kompress_must_keep.py
@@ -81,6 +81,63 @@ class TestMustKeepRegex:
assert not _KOMPRESS_MUST_KEEP_RE.search("process")
assert not _KOMPRESS_MUST_KEEP_RE.search("raised")
+ def test_boolean_connectives(self):
+ assert _KOMPRESS_MUST_KEEP_RE.search("or")
+ assert _KOMPRESS_MUST_KEEP_RE.search("and")
+ assert _KOMPRESS_MUST_KEEP_RE.search("nor")
+ assert _KOMPRESS_MUST_KEEP_RE.search("xor")
+ assert _KOMPRESS_MUST_KEEP_RE.search("or,")
+ assert _KOMPRESS_MUST_KEEP_RE.search("(and)")
+ # Tokens that merely contain a connective are not connectives.
+ for word in ("orange", "random", "android", "core", "sort", "concat", "node"):
+ assert not _KOMPRESS_MUST_KEEP_RE.search(word), word
+
+
+class TestBooleanConnectivesCompression:
+ """`and`/`or` hold a condition together, so dropping one changes the
+ predicate rather than degrading the sentence. Issue #3545 measured 12 of 40
+ `or` lost from a repeated Python return line on the released compressor,
+ while `not` from the very same line survived because negation is already
+ pinned -- the result still reads as a line of Python and nothing marks it
+ as altered."""
+
+ CODE = 'return name == "__init__" or not name.startswith("_") alpha beta gamma delta'
+
+ def _compress(self, monkeypatch, env):
+ _install_fake_kompress(monkeypatch)
+ if env:
+ monkeypatch.delenv(_KOMPRESS_MUST_KEEP_ENV, raising=False)
+ else:
+ monkeypatch.setenv(_KOMPRESS_MUST_KEEP_ENV, "0")
+ compressor = KompressCompressor(KompressConfig(enable_ccr=False, min_input_words=10))
+ monkeypatch.setattr(compressor, "_should_batch_single_content", lambda *a, **k: False)
+ return compressor.compress(self.CODE)
+
+ def test_keeps_both_operands_of_a_negated_disjunction(self, monkeypatch):
+ kept = self._compress(monkeypatch, env=True).compressed.split()
+ assert "not" in kept
+ assert "or" in kept
+
+ def test_batch_path_keeps_connectives_too(self, monkeypatch):
+ _install_fake_kompress(monkeypatch)
+ monkeypatch.delenv(_KOMPRESS_MUST_KEEP_ENV, raising=False)
+ compressor = KompressCompressor(KompressConfig(enable_ccr=False, min_input_words=10))
+ monkeypatch.setattr(compressor, "_should_use_sequential_fallback", lambda: False)
+
+ [result] = compressor.compress_batch(
+ ["alpha beta gamma delta epsilon zeta eta theta iota kappa one and two or three"],
+ batch_size=8,
+ )
+
+ kept = result.compressed.split()
+ assert "and" in kept
+ assert "or" in kept
+
+ def test_disabling_must_keep_leaves_the_connective_to_the_model(self, monkeypatch):
+ kept = self._compress(monkeypatch, env=False).compressed.split()
+ assert "or" not in kept
+ assert "not" not in kept
+
class TestMustKeepEnvVar:
def test_env_var_name(self):