Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(learn): count each tool call once per tool_call_id when detecting loops (#3462)
Nicholas Chae4088a65bb7bc
diff --git a/headroom/learn/fixtures.py b/headroom/learn/fixtures.pyindex 517fa77a0..40c42dae3 100644--- a/headroom/learn/fixtures.py+++ b/headroom/learn/fixtures.py@@ -21,11 +21,17 @@ def _tc(command: str,output: str,*,+ session_id: str,msg_index: int,is_error: bool = False,error_category: ErrorCategory = ErrorCategory.UNKNOWN,) -> ToolCall:- """Build a ToolCall, keying input on the field the tool's summary reads."""+ """Build a ToolCall, keying input on the field the tool's summary reads.++ ``tool_call_id`` is scoped by session: ``detect_loops`` counts a call once+ per id, so an id unique only within a session would make two sessions+ running the same loop look like one session replayed twice.+ """if name.lower() in ("bash", "shell"):input_data = {"command": command}elif name.lower() in ("read",):@@ -36,7 +42,7 @@ def _tc(input_data = {"command": command}return ToolCall(name=name,- tool_call_id=f"tc_{msg_index}",+ tool_call_id=f"{session_id}_tc_{msg_index}",input_data=input_data,output=output,is_error=is_error,@@ -67,7 +73,7 @@ def refetch_loop_session(# Same base command; only the output-limit varies — the loop signature.command = f"grep -rn 'TimeoutError' logs/ | head -{limit}"output = "logs/app.log:" + ("x" * (bytes_per_call - 20)) + "\n(truncated)"- calls.append(_tc("Bash", command, output, msg_index=i * 2))+ calls.append(_tc("Bash", command, output, session_id=session_id, msg_index=i * 2))limit += 50 # agent asks for more next time — still truncatedreturn SessionData(session_id=session_id, tool_calls=calls)@@ -85,6 +91,7 @@ def error_loop_session("Bash","python3 run_tests.py","python3: command not found",+ session_id=session_id,msg_index=i * 2,is_error=True,error_category=ErrorCategory.COMMAND_NOT_FOUND,@@ -102,10 +109,11 @@ def one_off_error_session(session_id: str = "one-off") -> SessionData:"Read","/etc/missing.conf","Error: file not found",+ session_id=session_id,msg_index=0,is_error=True,error_category=ErrorCategory.FILE_NOT_FOUND,),- _tc("Bash", "ls -la", "total 8\ndrwxr-xr-x", msg_index=1),+ _tc("Bash", "ls -la", "total 8\ndrwxr-xr-x", session_id=session_id, msg_index=1),],)diff --git a/headroom/learn/loops.py b/headroom/learn/loops.pyindex d88e920d2..3c48d0928 100644--- a/headroom/learn/loops.py+++ b/headroom/learn/loops.py@@ -106,6 +106,24 @@ def _tokens(tc: ToolCall) -> int:return nbytes // _BYTES_PER_TOKEN+def _without_replays(calls: list[ToolCall], seen: set[str]) -> list[ToolCall]:+ """Drop calls whose ``tool_call_id`` is already in ``seen``, recording the rest.++ ``seen`` is mutated, so the caller chooses the scope a replay is judged+ against: a fresh set collapses a transcript's own repeated turns, while a+ set carried across sessions suppresses a resume's replay of earlier ones.+ An id-less call is always kept — nothing identifies it as a replay.+ """+ kept: list[ToolCall] = []+ for call in calls:+ if call.tool_call_id:+ if call.tool_call_id in seen:+ continue+ seen.add(call.tool_call_id)+ kept.append(call)+ return kept++def detect_loops(sessions: list[SessionData],*,@@ -117,8 +135,30 @@ def detect_loops(a within-conversation phenomenon; the same command in two unrelatedsessions is not a loop). Groups meeting ``min_occurrences`` become``LoopPattern`` results, sorted by measured wasted tokens descending.++ A call is counted once per ``tool_call_id``. A resumed conversation can be+ written as a fresh transcript that replays earlier turns, which presents the+ same provider-assigned call to the scanner more than once; without this the+ replayed turns would inflate the loop. Calls carrying no id are always+ counted, since nothing identifies them as replays.++ This makes ``tool_call_id`` uniqueness a scanner contract: an id must be+ unique across sessions, not just within one, or two unrelated sessions look+ like one replayed twice. Scanners that synthesize ids scope them by session+ (see ``GeminiPlugin`` and ``OpenCodePlugin``).++ Dedup runs *before* the threshold, twice over. A session is screened on its+ distinct calls, because the question the threshold asks — did this+ conversation repeat itself? — is not answered by one call written into the+ transcript three times. Only then are its calls merged, deduped again+ against the calls already collected for that signature so a resume that+ replays an earlier session adds only what is new. Screening the raw group+ instead would let three sessions that each merely replayed one call+ contribute one real call apiece and clear the bar together, reporting a loop+ no conversation ran."""groups: dict[str, list[ToolCall]] = {}+ seen_ids: dict[str, set[str]] = {}for session in sessions:per_session: dict[str, list[ToolCall]] = {}for tc in session.tool_calls:@@ -126,11 +166,17 @@ def detect_loops(# Merge each session's qualifying groups into the global view keyed by# signature so cross-session recurrence of the SAME loop accumulates.for sig, calls in per_session.items():- if len(calls) >= min_occurrences:- groups.setdefault(sig, []).extend(calls)+ distinct = _without_replays(calls, set())+ if len(distinct) < min_occurrences:+ continue+ bucket = groups.setdefault(sig, [])+ bucket.extend(_without_replays(distinct, seen_ids.setdefault(sig, set())))loops: list[LoopPattern] = []for sig, calls in groups.items():+ # No post-merge threshold re-check: every group here was seeded by a+ # session that cleared the bar on its own distinct calls, and merging+ # only ever adds.count = len(calls)is_error_loop = sum(1 for c in calls if c.is_error) >= (count / 2)if is_error_loop:diff --git a/tests/test_learn/test_loop_weighting.py b/tests/test_learn/test_loop_weighting.pyindex 982ab2310..0ac01d5e4 100644--- a/tests/test_learn/test_loop_weighting.py+++ b/tests/test_learn/test_loop_weighting.py@@ -16,6 +16,8 @@ surfaced, were ranked no higher than a one-off rule. These tests pin:from pathlib import Pathfrom unittest.mock import MagicMock, patch+import pytest+from headroom.learn.analyzer import SessionAnalyzer, _build_digestfrom headroom.learn.fixtures import (error_loop_session,@@ -31,6 +33,8 @@ from headroom.learn.models import (ProjectInfo,Recommendation,RecommendationTarget,+ SessionData,+ ToolCall,)@@ -195,3 +199,225 @@ class TestAnalyzeEndToEnd:# After weighting, the loop guardrail ranks first despite the LLM's order.assert result.recommendations[0].is_loop_guardrail is Trueassert "loop" in result.recommendations[0].section.lower()+++# =============================================================================+# signature identity (regressions)+# =============================================================================+++def _call(name: str, tool_call_id: str, input_data: dict, *, msg_index: int) -> ToolCall:+ """A successful tool call with a fixed 40 KB output."""+ output = "x" * 40_000+ return ToolCall(+ name=name,+ tool_call_id=tool_call_id,+ input_data=input_data,+ output=output,+ is_error=False,+ msg_index=msg_index,+ output_bytes=len(output),+ )+++def _mixed_calls() -> list[ToolCall]:+ return [+ _call("Read", "call_r0", {"file_path": "/repo/a.py"}, msg_index=0),+ _call("Read", "call_r1", {"file_path": "/repo/a.py"}, msg_index=1),+ _call("Read", "call_r2", {"file_path": "/repo/a.py"}, msg_index=2),+ _call("Bash", "call_b0", {"command": "rg alpha /repo"}, msg_index=3),+ _call("Bash", "call_b1", {"command": "rg alpha /repo"}, msg_index=4),+ _call("Bash", "call_b2", {"command": "rg alpha /repo"}, msg_index=5),+ _call("Grep", "call_g0", {"pattern": "beta"}, msg_index=6),+ ]+++class TestReplayedTranscriptsDoNotDoubleCount:+ """A resume that replays prior history must not recount the same calls.++ ``detect_loops`` accumulates a signature across sessions so a recurring loop+ adds up. Providers whose resume writes a *new* transcript replaying earlier+ turns therefore present the same tool call more than once; identity comes+ from ``tool_call_id``, which the provider assigns.+ """++ def _reads(self) -> list[ToolCall]:+ return [+ _call("Read", f"call_r{i}", {"file_path": "/repo/docs/design.md"}, msg_index=i)+ for i in range(3)+ ]++ def test_replayed_calls_counted_once(self):+ reads = self._reads()+ single = detect_loops([SessionData(session_id="rollout-1", tool_calls=list(reads))])+ replayed = detect_loops(+ [+ SessionData(session_id="rollout-1", tool_calls=list(reads)),+ SessionData(session_id="rollout-2-resume", tool_calls=list(reads)),+ ]+ )+ assert replayed[0].count == single[0].count+ assert replayed[0].wasted_tokens == single[0].wasted_tokens++ def test_new_calls_in_a_resume_still_accumulate(self):+ # Dedup must not swallow genuinely new repetitions in the resumed run.+ reads = self._reads()+ extra = [+ _call("Read", f"call_r{i}", {"file_path": "/repo/docs/design.md"}, msg_index=i)+ for i in range(3, 6)+ ]+ replayed = detect_loops(+ [+ SessionData(session_id="rollout-1", tool_calls=list(reads)),+ SessionData(session_id="rollout-2-resume", tool_calls=list(reads) + extra),+ ]+ )+ assert replayed[0].count == 6++ def test_calls_without_ids_are_not_deduped(self):+ # An id-less scanner must keep counting repetitions rather than collapse.+ calls = [+ _call("Read", "", {"file_path": "/repo/docs/design.md"}, msg_index=i) for i in range(3)+ ]+ loops = detect_loops(+ [+ SessionData(session_id="a", tool_calls=list(calls)),+ SessionData(session_id="b", tool_calls=list(calls)),+ ]+ )+ assert loops[0].count == 6++ @pytest.mark.parametrize("replays", [1, 2, 3, 5])+ def test_replaying_a_transcript_never_changes_the_count(self, replays):+ calls = _mixed_calls()+ sessions = [+ SessionData(session_id=f"rollout-{i}", tool_calls=list(calls)) for i in range(replays)+ ]++ once = detect_loops([SessionData(session_id="rollout-0", tool_calls=list(calls))])+ many = detect_loops(sessions)++ assert sorted(lp.count for lp in many) == sorted(lp.count for lp in once)+++class TestDedupRespectsTheOccurrenceThreshold:+ """Dedup must not leave behind loops that no longer meet the bar.++ ``detect_loops`` screens a group against ``min_occurrences`` before removing+ replayed calls, so a group can clear the bar only *because* of duplicates and+ still be reported once they are gone. Real transcripts do repeat a+ ``tool_use_id`` within one file, so this is reachable from the default+ Claude Code path.+ """++ def test_a_group_that_only_duplicates_is_not_a_loop(self):+ # One call, written into the transcript three times.+ calls = [+ _call("Read", "toolu_SAME", {"file_path": "/repo/design.md"}, msg_index=i)+ for i in range(3)+ ]++ assert detect_loops([SessionData(session_id="s", tool_calls=calls)]) == []++ def test_partial_dedup_still_has_to_clear_the_bar(self):+ # Three calls, two distinct — below DEFAULT_MIN_OCCURRENCES once deduped.+ calls = [+ _call("Read", "toolu_A", {"file_path": "/repo/design.md"}, msg_index=0),+ _call("Read", "toolu_A", {"file_path": "/repo/design.md"}, msg_index=1),+ _call("Read", "toolu_B", {"file_path": "/repo/design.md"}, msg_index=2),+ ]++ assert detect_loops([SessionData(session_id="s", tool_calls=calls)]) == []++ def test_a_real_loop_padded_with_duplicates_is_still_reported(self):+ # Dedup must subtract the replays without dropping the genuine loop.+ distinct = [+ _call("Read", f"toolu_{i}", {"file_path": "/repo/design.md"}, msg_index=i)+ for i in range(3)+ ]+ padded = distinct + [+ _call("Read", "toolu_0", {"file_path": "/repo/design.md"}, msg_index=9)+ ]++ loops = detect_loops([SessionData(session_id="s", tool_calls=padded)])++ assert [lp.count for lp in loops] == [3]++ def test_no_loop_is_reported_with_zero_measured_waste(self):+ """A zero-waste loop is the tell that a group survived on duplicates.++ ``format_loops_for_digest`` bills every entry to the LLM as HIGHEST+ PRIORITY, and ``SessionAnalyzer.analyze`` treats a non-empty loop list as+ reason enough to call the model, so a zero-waste entry buys an LLM round+ trip for a session with nothing to report.+ """+ calls = [+ _call("Read", "toolu_SAME", {"file_path": "/repo/design.md"}, msg_index=i)+ for i in range(4)+ ]++ loops = detect_loops([SessionData(session_id="s", tool_calls=calls)])++ assert [lp for lp in loops if lp.wasted_tokens == 0] == []+++class TestFixturesSatisfyTheDedupContract:+ """Dedup keys on ``tool_call_id``, so the shipped fixtures must scope theirs.++ ``detect_loops`` accumulates a signature across sessions. An id that is only+ unique *within* a session makes two unrelated sessions look like one replayed+ twice, silently halving a real loop.+ """++ def test_the_same_loop_in_two_sessions_accumulates(self):+ monday = refetch_loop_session("session-monday")+ tuesday = refetch_loop_session("session-tuesday")++ one = detect_loops([monday])+ both = detect_loops([monday, tuesday])++ assert both[0].count == one[0].count * 2++ def test_fixture_ids_are_unique_across_sessions(self):+ monday = {c.tool_call_id for c in refetch_loop_session("session-monday").tool_calls}+ tuesday = {c.tool_call_id for c in refetch_loop_session("session-tuesday").tool_calls}++ assert monday.isdisjoint(tuesday)+++class TestSubThresholdSessionsDoNotCombine:+ """A session that only looped in replays must not contribute at all.++ ``min_occurrences`` is screened against the *pre-dedup* per-session group, so+ a session whose repetitions are all replays of one call still qualifies and+ hands its single real call to the global view. Three such sessions then clear+ the bar together, reporting a loop no conversation actually ran.+ """++ def _replay_padded_session(self, session_id: str, call_id: str) -> SessionData:+ # One real Read, written into this transcript three times.+ return SessionData(+ session_id=session_id,+ tool_calls=[+ _call("Read", call_id, {"file_path": "/repo/design.md"}, msg_index=i)+ for i in range(3)+ ],+ )++ def test_each_session_alone_is_not_a_loop(self):+ sessions = [+ self._replay_padded_session("s1", "toolu_s1"),+ self._replay_padded_session("s2", "toolu_s2"),+ self._replay_padded_session("s3", "toolu_s3"),+ ]++ assert [detect_loops([s]) for s in sessions] == [[], [], []]++ def test_sub_threshold_sessions_do_not_combine_into_a_loop(self):+ sessions = [+ self._replay_padded_session("s1", "toolu_s1"),+ self._replay_padded_session("s2", "toolu_s2"),+ self._replay_padded_session("s3", "toolu_s3"),+ ]++ assert detect_loops(sessions) == []