Python
"""OPE-27 — auto-compaction pure functions: trigger math, boundary picking, mechanical
extraction, summarizer seam, trim fallback, outbound view. No engine involved."""
import json
import time
import pytest
from coworker import compaction
from coworker import compaction as C
from coworker.compaction import (
CompactionState,
DEFAULT_CAP_TOKENS,
DEFAULT_CONTEXT_WINDOW,
apply_to_outbound,
build_state,
compacted_block,
estimate_tokens,
extract_user_messages,
extract_working_state,
is_context_overflow,
pick_boundary,
should_compact,
summarize_span,
summarizer_messages,
trigger_tokens,
trim_state,
)
# -- message builders ---------------------------------------------------------
def user(text):
return {"role": "user", "content": text, "ts": 1.0}
_call_seq = 0
def assistant(text="", tool_calls=None):
global _call_seq
msg = {"role": "assistant", "content": text, "ts": 1.0}
if tool_calls:
calls = []
for name, args in tool_calls:
calls.append(
{
"id": f"c{_call_seq}",
"type": "function",
"function": {"name": name, "arguments": json.dumps(args)},
}
)
_call_seq += 1
msg["tool_calls"] = calls
return msg
def tool(call_id, content):
return {
"role": "tool",
"tool_call_id": call_id,
"content": content if isinstance(content, str) else json.dumps(content),
"ts": 1.0,
}
def tool_turn(name, args, result):
"""[assistant tool-call, matching tool result] with a properly paired call id."""
a = assistant(tool_calls=[(name, args)])
return [a, tool(a["tool_calls"][0]["id"], result)]
def convo(turns=6, bulk=2000):
"""system + N user/assistant turns with bulky assistant text."""
msgs = [{"role": "system", "content": "You are a coworker."}]
for i in range(turns):
msgs.append(user(f"request {i}"))
msgs.append(assistant(f"answer {i} " + "x" * bulk))
return msgs
class FakeSummarizer:
# The default must satisfy the OPE-189 quality gate (all eight sections, past the
# minimum length) — an unusable summary is now refused, not stored. GOOD_SUMMARY is
# defined further down the module and resolved when a fake is constructed.
def __init__(self, text=None, fail_times=0):
self.text = GOOD_SUMMARY if text is None else text
self.fail_times = fail_times
self.calls = []
def complete(self, *, model, messages, tools=None, **settings):
self.calls.append({"model": model, "messages": messages, "tools": tools, **settings})
if self.fail_times > 0:
self.fail_times -= 1
raise RuntimeError("summarizer down")
class Turn:
pass
t = Turn()
t.text = self.text
return t
# -- trigger math -------------------------------------------------------------
def test_trigger_is_min_of_pct_and_cap():
assert trigger_tokens(100_000) == 80_000
assert trigger_tokens(1_000_000) == DEFAULT_CAP_TOKENS # the 250k cap wins
assert trigger_tokens(None) == int(0.8 * DEFAULT_CONTEXT_WINDOW)
# both knobs are user-overridable
assert trigger_tokens(100_000, threshold_pct=0.5, cap_tokens=40_000) == 40_000
assert trigger_tokens(100_000, threshold_pct=0.5, cap_tokens=999_999) == 50_000
def test_should_compact_crosses_threshold():
assert not should_compact(79_999, 100_000)
assert should_compact(80_000, 100_000)
def test_estimate_tokens_is_chars_over_four():
msgs = [user("a" * 400)]
est = estimate_tokens(msgs)
assert 100 <= est <= 120 # 400 chars of content + json overhead, /4
# -- boundary -----------------------------------------------------------------
def test_boundary_prefers_earliest_user_turn_that_fits():
msgs = convo(turns=6)
per_turn = estimate_tokens(msgs[1:3])
boundary = pick_boundary(msgs, keep_tokens=per_turn * 2 + 10)
assert msgs[boundary]["role"] == "user"
assert msgs[boundary]["content"] == "request 4" # newest two turns survive
def test_boundary_falls_inside_a_giant_final_turn():
# One user turn followed by a huge tool loop: the turn alone exceeds the budget,
# so the cut lands on an assistant (iteration) boundary inside it — never a tool row.
msgs = [{"role": "system", "content": "s"}, user("go")]
for i in range(8):
a = assistant("step " + "y" * 3000, tool_calls=[("run_shell", {"command": f"cmd{i}"})])
msgs += [a, tool(a["tool_calls"][0]["id"], {"exit_code": 0, "out": "z" * 3000})]
boundary = pick_boundary(msgs, keep_tokens=estimate_tokens(msgs[-3:]))
assert msgs[boundary]["role"] == "assistant"
def test_boundary_none_when_nothing_to_summarize():
msgs = [{"role": "system", "content": "s"}, user("hi"), assistant("hello")]
assert pick_boundary(msgs, keep_tokens=10_000_000) is None
# -- mechanical extraction ----------------------------------------------------
def test_working_state_files_commands_tools():
span = [
user("write it"),
*tool_turn("write_file", {"path": "a.py", "content": "x"}, {"ok": True}),
*tool_turn("run_shell", {"command": "pytest -q"}, {"exit_code": 1}),
*tool_turn("write_file", {"path": "b.py", "content": "y"}, {"ok": True}),
*tool_turn("write_file", {"path": "a.py", "content": "x2"}, {"ok": True}),
]
block = extract_working_state(span)
# deduped, most recent first
assert block.index("- a.py") < block.index("- b.py")
assert block.count("a.py") == 1
assert "pytest -q" in block and "[exit 1]" in block
assert "run_shell" in block and "write_file" in block
def test_working_state_empty_span():
assert extract_working_state([user("hi"), assistant("yo")]) == ""
def test_user_messages_extracted_verbatim_and_uncut():
# OPE-189: extraction no longer clips — a 1,400-char task statement reached the block
# cut mid-sentence, losing the output path and the field names the grader checks.
span = [
user("first ask"),
assistant("a"),
user([{"type": "text", "text": "second"}, {"type": "image_url", "image_url": {}}]),
assistant("b"),
user("bulk " + "z" * 2000),
]
out = extract_user_messages(span)
assert out[0] == "first ask"
assert out[1] == "second [image]"
assert out[2] == "bulk " + "z" * 2000 # verbatim; fitting is fit_user_messages' job
# -- summarizer seam ----------------------------------------------------------
def test_summarizer_messages_clip_tool_results_and_fold_prior():
span = [user("go"), *tool_turn("read_file", {"path": "big.txt"}, "huge " * 500)]
msgs = summarizer_messages(span, prior_summary="OLD SUMMARY")
body = msgs[1]["content"]
assert "OLD SUMMARY" in body
# The 2500-char tool result got clipped hard; the tail instruction is the only other bulk.
assert len(body) < 3000 + len(C.SUMMARY_TAIL_INSTRUCTION)
assert msgs[0]["role"] == "system" and "Primary request and intent" in msgs[0]["content"]
def test_summarizer_instruction_comes_after_the_transcript():
"""OPE-189: recency decides. With the instruction only up front, the last thing the
model read was the coworker mid-task and it continued that voice instead of
summarizing. The transcript is delimited and the instruction restated last."""
span = [user("go"), *tool_turn("run_shell", {"command": "ls"}, "a.txt b.txt")]
body = summarizer_messages(span)[1]["content"]
assert body.startswith("**--- BEGIN TRANSCRIPT TO SUMMARIZE ---**")
assert body.rstrip().endswith(C.SUMMARY_TAIL_INSTRUCTION)
assert body.index("[tool result]") < body.index("**--- END OF TRANSCRIPT ---**")
assert "## 1. Primary request and intent" in C.SUMMARY_TAIL_INSTRUCTION
GOOD_SUMMARY = "\n".join(
f"## {i}. {name}\ndetail line with enough substance to clear the minimum length bar"
for i, name in enumerate(
[
"Primary request and intent",
"Key concepts and decisions",
"Artifacts and files",
"Errors and fixes",
"All user messages",
"Pending tasks",
"Current work",
"Next step",
],
start=1,
)
)
def test_summarize_span_passes_model_and_budget():
fake = FakeSummarizer(text=GOOD_SUMMARY)
out = summarize_span(fake, "prov:model-x", [user("hi")], max_tokens=9999)
assert out == GOOD_SUMMARY
assert fake.calls[0]["model"] == "prov:model-x"
assert fake.calls[0]["tools"] is None
assert fake.calls[0]["max_tokens"] == 9999
def test_summarize_span_refuses_an_unusable_summary():
"""OPE-189: 'not empty' accepted a 61-char next-action sentence as the coworker's whole
memory of 75 turns. Empty, too short, and role-slipped replies are all refused so the
caller's retry (then trim) policy runs."""
for bad in (
" ",
"The output got truncated. Let me re-run the remaining checks.",
"## Artifacts and files\n" + "detail " * 200, # long, but not the anchor section
):
with pytest.raises(RuntimeError):
summarize_span(FakeSummarizer(text=bad), "m", [user("hi")])
def test_summary_quality_problem_names_the_fault():
assert C.summary_quality_problem(GOOD_SUMMARY) is None
assert C.summary_quality_problem("") == "empty"
assert "too short" in C.summary_quality_problem("## 1. Primary request and intent")
slipped = "Let me re-run the remaining checks. " * 20
assert "does not open the sections" in C.summary_quality_problem(slipped)
# -- build + repeated compaction ----------------------------------------------
def test_build_state_and_outbound_view():
msgs = convo(turns=6)
fake = FakeSummarizer(text=GOOD_SUMMARY + "\nthe gist")
state = build_state(
msgs, provider=fake, model="m", keep_tokens=estimate_tokens(msgs[-4:]) + 10
)
assert state is not None and not state.trimmed
assert state.user_messages[0] == "request 0"
out = apply_to_outbound(msgs, state)
assert out[0]["role"] == "system" # instructions survive
assert "<compacted-history>" in out[1]["content"]
assert "the gist" in out[1]["content"]
assert "request 0" in out[1]["content"] # mechanical user-message list
assert out[2] is msgs[state.boundary_index] # verbatim tail, canonical untouched
assert len(msgs) == 13 # canonical history unchanged
def test_repeated_compaction_summarizes_prior_plus_new_turns():
msgs = convo(turns=4)
fake = FakeSummarizer()
first = build_state(msgs, provider=fake, model="m", keep_tokens=estimate_tokens(msgs[-4:]) + 10)
# session grows
for i in range(4, 8):
msgs.append(user(f"request {i}"))
msgs.append(assistant(f"answer {i} " + "x" * 2000))
second = build_state(
msgs, provider=fake, model="m",
keep_tokens=estimate_tokens(msgs[-4:]) + 10, prior=first,
)
assert second is not None and second.boundary_index > first.boundary_index
# the second summarizer call folds the prior summary in
assert "previous compaction summary" in fake.calls[1]["messages"][1]["content"]
# user messages accumulate across compactions
assert "request 0" in second.user_messages[0]
assert any("request 5" in u for u in second.user_messages)
def test_build_state_none_when_boundary_stale():
msgs = convo(turns=3)
fake = FakeSummarizer()
state = build_state(msgs, provider=fake, model="m", keep_tokens=estimate_tokens(msgs[-2:]) + 10)
again = build_state(
msgs, provider=fake, model="m",
keep_tokens=10_000_000, prior=state,
)
assert again is None # nothing new fits below the prior boundary
# -- trim fallback ------------------------------------------------------------
def test_trim_advances_boundary_and_keeps_user_messages():
msgs = convo(turns=10)
state = trim_state(msgs)
assert state is not None and state.trimmed
assert msgs[state.boundary_index]["role"] in ("user", "assistant")
assert state.user_messages # preserved mechanically even without a summary
assert "trimmed" in state.summary_text
out = apply_to_outbound(msgs, state)
assert len(out) < len(msgs) + 1
def test_trim_from_prior_state_never_lands_on_tool_row():
msgs = [{"role": "system", "content": "s"}, user("go")]
for i in range(10):
msgs += tool_turn("run_shell", {"command": f"c{i}"}, {"exit_code": 0})
prior = trim_state(msgs)
later = trim_state(msgs, prior=prior)
assert later.boundary_index > prior.boundary_index
assert msgs[later.boundary_index]["role"] != "tool"
def test_trim_none_when_too_small():
assert trim_state([user("hi"), assistant("yo")]) is None
# -- state round-trip + overflow detection ------------------------------------
def test_state_dict_round_trip():
state = CompactionState(
boundary_index=7, summary_text="s", working_state="w",
user_messages=["u1"], created_at=1.5, model_used="m", trimmed=True,
)
assert CompactionState.from_dict(state.as_dict()) == state
assert CompactionState.from_dict(None) is None
assert CompactionState.from_dict({}) is None
def test_apply_to_outbound_noop_on_stale_or_missing_state():
msgs = convo(turns=2)
assert apply_to_outbound(msgs, None) is msgs
stale = CompactionState(boundary_index=999, summary_text="s", working_state="")
assert apply_to_outbound(msgs, stale) is msgs
def test_is_context_overflow():
assert is_context_overflow(Exception("Error 400: maximum context length is 128000 tokens"))
assert is_context_overflow(Exception("context_length_exceeded"))
assert is_context_overflow(Exception("Prompt is too long: 210000 tokens > limit"))
assert not is_context_overflow(Exception("rate limit exceeded"))
assert not is_context_overflow(Exception("connection reset"))
def test_user_messages_pin_the_first_and_budget_the_rest():
"""OPE-189: the list must not grow forever, but the OLD rule (newest 40) silently threw
away the opening message — the one that says why the session exists. Now the first is
pinned and the newest fill a token budget; the gap is in the middle."""
msgs = [{"role": "system", "content": "s"}]
msgs.append({"role": "user", "content": "THE ORIGINAL TASK: " + "spec " * 100})
msgs.append({"role": "assistant", "content": "starting"})
for i in range(300):
msgs.append({"role": "user", "content": f"ask {i} " + "filler " * 40})
msgs.append({"role": "assistant", "content": f"answer {i}"})
state = None
while True:
nxt = trim_state(msgs, prior=state, fraction=0.4, user_budget_tokens=2_000)
if nxt is None:
break
state = nxt
assert state is not None
assert state.user_messages[0].startswith("THE ORIGINAL TASK:") # pinned, never dropped
# The newest preserved message is from the end of the run (the very last turns stay in
# the verbatim tail rather than the block, so this is the newest that was compacted).
assert state.user_messages[-1].startswith("ask 29")
assert state.user_messages_dropped > 0 # the middle is what gives
block = compacted_block(state)
assert "older user messages omitted here" in block
assert block.index("THE ORIGINAL TASK") < block.index("omitted here")
restored = CompactionState.from_dict(state.as_dict())
assert restored.user_messages == state.user_messages
assert restored.user_messages_dropped == state.user_messages_dropped
def test_fit_user_messages_pins_first_clips_giants_and_counts_drops():
budget = 100 # tokens, i.e. ~400 chars
msgs = ["FIRST " + "a" * 200, "old " + "b" * 400, "mid", "newest"]
kept, dropped = C.fit_user_messages(msgs, prior_dropped=3, budget_tokens=budget)
assert kept[0] == msgs[0] # pinned whole: well under the pin cap
assert kept[-1] == "newest"
assert dropped >= 3 # running total carries forward
assert "mid" in kept # cheap messages still fit
# A first message far over the pin cap is clipped, never dropped.
huge = "X" * 100_000
kept, _ = C.fit_user_messages([huge, "later"], prior_dropped=0, budget_tokens=budget)
assert kept[0].startswith("XXX") and kept[0].endswith("…")
assert len(kept[0]) <= C._USER_PIN_MAX_TOKENS * 4
# A single newest message over budget is clipped to fit, not dropped.
kept, _ = C.fit_user_messages(["first", huge], prior_dropped=0, budget_tokens=budget)
assert len(kept) == 2 and kept[1].endswith("…")
def test_user_message_budget_scales_with_the_trigger():
assert C.user_message_budget(250_000) == 20_000 # at the default trigger
assert C.user_message_budget(60_000) == 4_800 # a lowered trigger keeps its saving
assert C.user_message_budget(1_000) == C._USER_BUDGET_MIN # floor
assert C.user_message_budget(10_000_000) == C._USER_BUDGET_MAX # ceiling
# -- attachments --------------------------------------------------------------
def _image_turn(chars: int = 1_500_000, text: str = "what's in this screenshot?") -> dict:
return {
"role": "user",
"content": [
{"type": "text", "text": text},
{
"type": "image_url",
"image_url": {"url": "data:image/png;base64," + "A" * chars},
},
],
}
def _pdf_turn(chars: int = 2_000_000) -> dict:
return {
"role": "user",
"content": [
{"type": "text", "text": "summarize this"},
{
"type": "file",
"file": {
"filename": "report.pdf",
"file_data": "data:application/pdf;base64," + "B" * chars,
},
},
],
}
def test_image_part_is_not_charged_its_base64_length():
"""A data-URL image must not dominate the signal: chars/4 read a 1.5 MB screenshot as
~375k tokens, so a single attachment sat permanently above cap_tokens and re-triggered
compaction on every iteration of the turn loop.
Bracketed on both sides. The upper bound is the defect being fixed; the lower bound
guards the inverse defect, an attachment charged ~nothing, which would keep a history
that really is over the cap permanently under it. A one-sided `< 5_000` certifies an
estimate that collapsed to 26 just as readily as the correct ~1.5k.
"""
msgs = [{"role": "system", "content": "s"}, _image_turn()]
est = estimate_tokens(msgs)
assert 1_000 < est < 5_000
assert not should_compact(est, 200_000)
def test_pdf_part_is_not_charged_its_base64_length():
msgs = [{"role": "system", "content": "s"}, _pdf_turn()]
est = estimate_tokens(msgs)
assert 2_000 < est < 10_000 # bracketed both sides; see the image test
def test_attachment_estimate_does_not_track_data_url_length():
"""The property the fix establishes is *independence* from base64 length, so assert it
differentially rather than at one sampled size. The range spans what attachments.py
actually admits (MAX_IMAGE_CHARS 12 M, MAX_PDF_CHARS 15 M) — a bound checked only at
1.5 MB says nothing about the sizes the GUI's own picker allows through.
"""
images = {estimate_tokens([_image_turn(n)]) for n in (100_000, 1_500_000, 12_000_000)}
assert len(images) == 1, f"image estimate varied with data-URL length: {sorted(images)}"
pdfs = {estimate_tokens([_pdf_turn(n)]) for n in (100_000, 2_000_000, 15_000_000)}
assert len(pdfs) == 1, f"file estimate varied with data-URL length: {sorted(pdfs)}"
def test_text_alongside_an_attachment_is_still_counted():
"""Only `image_url` and `file` parts get the flat charge; every other part falls to the
`else` branch and must still be measured. A long pasted prompt next to a screenshot is
the ordinary case, and charging it zero is the same under-count in a different place.
"""
small = estimate_tokens([_image_turn(text="hi")])
large = estimate_tokens([_image_turn(text="x" * 40_000)])
assert large - small >= 9_000 # 40k chars / 4, less serialization slack
def test_message_keys_outside_content_are_still_counted():
"""`_message_chars` measures the non-content keys separately once it takes the list
branch. `tool_call_id`, `name` and friends are real tokens on the wire."""
base = {"role": "user", "content": [{"type": "text", "text": "hi"}]}
tagged = {**base, "name": "z" * 4_000}
assert estimate_tokens([tagged]) - estimate_tokens([base]) >= 900
def test_text_only_estimate_is_unchanged_by_part_handling():
"""Text-only histories must measure exactly as before — the whole trigger threshold is
calibrated against that number."""
msgs = convo(turns=6)
assert estimate_tokens(msgs) == sum(len(json.dumps(m, default=str)) for m in msgs) // 4
def _long_history(turns: int = 300) -> list[dict]:
msgs = [{"role": "system", "content": "s"}]
for i in range(turns):
a = assistant("done " + "z" * 300, tool_calls=[("read_file", {"path": "a.py"})])
msgs += [
user(f"task {i} " + "x" * 400),
a,
tool(a["tool_calls"][0]["id"], {"exit_code": 0, "out": "y" * 4000}),
]
return msgs
def test_boundary_search_measures_each_message_a_bounded_number_of_times(monkeypatch):
"""The complexity claim, asserted without a clock.
Measuring each candidate with `estimate_tokens(messages[i:])` re-serialized the whole
tail per candidate — ~1 s of blocking CPU on a 1 200-message session, on the turn loop.
Counting the per-message measurements separates O(n) from O(n²) exactly, on any
machine and under any CI load, where a wall-clock bound only does so probabilistically.
"""
msgs = _long_history()
calls = 0
real = compaction._message_chars
def counting(msg):
nonlocal calls
calls += 1
return real(msg)
monkeypatch.setattr(compaction, "_message_chars", counting)
boundary = pick_boundary(msgs, keep_tokens=40_000)
assert boundary is not None
assert calls <= 3 * len(msgs) # one backward pass; quadratic ran ~150x this
def test_boundary_search_is_linear_in_history_length():
"""Wall-clock companion to the call-count test above: catches a regression that keeps
the call count linear but makes each call expensive."""
msgs = _long_history()
started = time.perf_counter()
boundary = pick_boundary(msgs, keep_tokens=40_000)
elapsed = time.perf_counter() - started
assert boundary is not None
assert estimate_tokens(msgs[boundary:]) <= 40_000
assert elapsed < 0.25 # linear lands ~5 ms here; the quadratic version took ~700 ms
def test_falls_back_to_the_last_assistant_when_no_candidate_fits():
"""The `assistants[-1]` fallback, on the one path that reaches it: a history with no
user turns at all (the `if users:` block short-circuits every other route there).
Pins which end of the list the fallback takes. `assistants[0]` would cut at the very
front and summarize the whole history; the intent is the opposite — keep as little as
possible when even the newest turn will not fit.
"""
msgs = [{"role": "system", "content": "s"}]
for i in range(6):
a = assistant("step " + "y" * 3000, tool_calls=[("run_shell", {"command": f"c{i}"})])
msgs += [a, tool(a["tool_calls"][0]["id"], {"exit_code": 0, "out": "z" * 3000})]
_, assistants = compaction._turn_starts(msgs, start=1)
assert len(assistants) > 2 # otherwise [-1] and [-2] coincide and the check is inert
boundary = pick_boundary(msgs, keep_tokens=10) # nothing fits in 10 tokens
assert boundary == assistants[-1]
assert msgs[boundary]["role"] == "assistant"
def test_history_that_wholly_fits_is_not_compacted():
"""`_fit` returning a legitimate index 0 means the entire history fits inside
keep_tokens, so there is nothing to summarize. The `or` this replaced coerced that 0 to
the `assistants[-1]` fallback and compacted a history of a few dozen tokens.
"""
msgs = [{"role": "assistant", "content": f"a{i}"} for i in range(3)]
assert estimate_tokens(msgs) < 100
assert pick_boundary(msgs, keep_tokens=10_000) is None
# -- what these checks do NOT cover -------------------------------------------
#
# * **The absolute accuracy of the flat estimates.** 1_500 / 3_000 are bracketed to an
# order of magnitude, not anchored to a provider's published image-token formula. A
# systematically wrong-but-plausible constant (say 600, or 4_000) passes everything here.
# Anchoring to the documented formula for the vision models actually in use would turn
# this from a range check into a real oracle.
# * **Multiple attachments in one message.** Every case above carries a single image or
# file; N-attachment accumulation is unexercised.
# * **Providers that report real usage.** estimate_tokens is the fallback signal only, so
# none of this constrains behaviour on providers that meter properly.
# * **Boundary quality, as opposed to boundary cost.** The complexity checks assert the
# search is linear and that the kept tail fits; they do not assert the boundary chosen is
# the *best* one, and the earliest-first preference is covered only by the pre-existing
# test_boundary_prefers_earliest_user_turn_that_fits.
# * **The exact-threshold boundary.** Nothing pins `suffix[i] // 4 <= keep_tokens` at
# equality, so an off-by-one to `<` survives; every case here sits clear of the threshold.
# Known from a mutation pass rather than assumed.