Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(learn): count each tool call once per tool_call_id when detecting loops (#3462)

Nicholas Chae4088a65bb7bc
diff --git a/headroom/learn/fixtures.py b/headroom/learn/fixtures.py
index 517fa77a0..40c42dae3 100644
--- a/headroom/learn/fixtures.py
+++ b/headroom/learn/fixtures.py
@@ -21,11 +21,17 @@ def _tc(
command: str,
output: str,
*,
+ session_id: str,
msg_index: int,
is_error: bool = False,
error_category: ErrorCategory = ErrorCategory.UNKNOWN,
) -> ToolCall:
- """Build a ToolCall, keying input on the field the tool's summary reads."""
+ """Build a ToolCall, keying input on the field the tool's summary reads.
+
+ ``tool_call_id`` is scoped by session: ``detect_loops`` counts a call once
+ per id, so an id unique only within a session would make two sessions
+ running the same loop look like one session replayed twice.
+ """
if name.lower() in ("bash", "shell"):
input_data = {"command": command}
elif name.lower() in ("read",):
@@ -36,7 +42,7 @@ def _tc(
input_data = {"command": command}
return ToolCall(
name=name,
- tool_call_id=f"tc_{msg_index}",
+ tool_call_id=f"{session_id}_tc_{msg_index}",
input_data=input_data,
output=output,
is_error=is_error,
@@ -67,7 +73,7 @@ def refetch_loop_session(
# Same base command; only the output-limit varies — the loop signature.
command = f"grep -rn 'TimeoutError' logs/ | head -{limit}"
output = "logs/app.log:" + ("x" * (bytes_per_call - 20)) + "\n(truncated)"
- calls.append(_tc("Bash", command, output, msg_index=i * 2))
+ calls.append(_tc("Bash", command, output, session_id=session_id, msg_index=i * 2))
limit += 50 # agent asks for more next time — still truncated
return SessionData(session_id=session_id, tool_calls=calls)
@@ -85,6 +91,7 @@ def error_loop_session(
"Bash",
"python3 run_tests.py",
"python3: command not found",
+ session_id=session_id,
msg_index=i * 2,
is_error=True,
error_category=ErrorCategory.COMMAND_NOT_FOUND,
@@ -102,10 +109,11 @@ def one_off_error_session(session_id: str = "one-off") -> SessionData:
"Read",
"/etc/missing.conf",
"Error: file not found",
+ session_id=session_id,
msg_index=0,
is_error=True,
error_category=ErrorCategory.FILE_NOT_FOUND,
),
- _tc("Bash", "ls -la", "total 8\ndrwxr-xr-x", msg_index=1),
+ _tc("Bash", "ls -la", "total 8\ndrwxr-xr-x", session_id=session_id, msg_index=1),
],
)
diff --git a/headroom/learn/loops.py b/headroom/learn/loops.py
index d88e920d2..3c48d0928 100644
--- a/headroom/learn/loops.py
+++ b/headroom/learn/loops.py
@@ -106,6 +106,24 @@ def _tokens(tc: ToolCall) -> int:
return nbytes // _BYTES_PER_TOKEN
+def _without_replays(calls: list[ToolCall], seen: set[str]) -> list[ToolCall]:
+ """Drop calls whose ``tool_call_id`` is already in ``seen``, recording the rest.
+
+ ``seen`` is mutated, so the caller chooses the scope a replay is judged
+ against: a fresh set collapses a transcript's own repeated turns, while a
+ set carried across sessions suppresses a resume's replay of earlier ones.
+ An id-less call is always kept — nothing identifies it as a replay.
+ """
+ kept: list[ToolCall] = []
+ for call in calls:
+ if call.tool_call_id:
+ if call.tool_call_id in seen:
+ continue
+ seen.add(call.tool_call_id)
+ kept.append(call)
+ return kept
+
+
def detect_loops(
sessions: list[SessionData],
*,
@@ -117,8 +135,30 @@ def detect_loops(
a within-conversation phenomenon; the same command in two unrelated
sessions is not a loop). Groups meeting ``min_occurrences`` become
``LoopPattern`` results, sorted by measured wasted tokens descending.
+
+ A call is counted once per ``tool_call_id``. A resumed conversation can be
+ written as a fresh transcript that replays earlier turns, which presents the
+ same provider-assigned call to the scanner more than once; without this the
+ replayed turns would inflate the loop. Calls carrying no id are always
+ counted, since nothing identifies them as replays.
+
+ This makes ``tool_call_id`` uniqueness a scanner contract: an id must be
+ unique across sessions, not just within one, or two unrelated sessions look
+ like one replayed twice. Scanners that synthesize ids scope them by session
+ (see ``GeminiPlugin`` and ``OpenCodePlugin``).
+
+ Dedup runs *before* the threshold, twice over. A session is screened on its
+ distinct calls, because the question the threshold asks — did this
+ conversation repeat itself? — is not answered by one call written into the
+ transcript three times. Only then are its calls merged, deduped again
+ against the calls already collected for that signature so a resume that
+ replays an earlier session adds only what is new. Screening the raw group
+ instead would let three sessions that each merely replayed one call
+ contribute one real call apiece and clear the bar together, reporting a loop
+ no conversation ran.
"""
groups: dict[str, list[ToolCall]] = {}
+ seen_ids: dict[str, set[str]] = {}
for session in sessions:
per_session: dict[str, list[ToolCall]] = {}
for tc in session.tool_calls:
@@ -126,11 +166,17 @@ def detect_loops(
# Merge each session's qualifying groups into the global view keyed by
# signature so cross-session recurrence of the SAME loop accumulates.
for sig, calls in per_session.items():
- if len(calls) >= min_occurrences:
- groups.setdefault(sig, []).extend(calls)
+ distinct = _without_replays(calls, set())
+ if len(distinct) < min_occurrences:
+ continue
+ bucket = groups.setdefault(sig, [])
+ bucket.extend(_without_replays(distinct, seen_ids.setdefault(sig, set())))
loops: list[LoopPattern] = []
for sig, calls in groups.items():
+ # No post-merge threshold re-check: every group here was seeded by a
+ # session that cleared the bar on its own distinct calls, and merging
+ # only ever adds.
count = len(calls)
is_error_loop = sum(1 for c in calls if c.is_error) >= (count / 2)
if is_error_loop:
diff --git a/tests/test_learn/test_loop_weighting.py b/tests/test_learn/test_loop_weighting.py
index 982ab2310..0ac01d5e4 100644
--- a/tests/test_learn/test_loop_weighting.py
+++ b/tests/test_learn/test_loop_weighting.py
@@ -16,6 +16,8 @@ surfaced, were ranked no higher than a one-off rule. These tests pin:
from pathlib import Path
from unittest.mock import MagicMock, patch
+import pytest
+
from headroom.learn.analyzer import SessionAnalyzer, _build_digest
from headroom.learn.fixtures import (
error_loop_session,
@@ -31,6 +33,8 @@ from headroom.learn.models import (
ProjectInfo,
Recommendation,
RecommendationTarget,
+ SessionData,
+ ToolCall,
)
@@ -195,3 +199,225 @@ class TestAnalyzeEndToEnd:
# After weighting, the loop guardrail ranks first despite the LLM's order.
assert result.recommendations[0].is_loop_guardrail is True
assert "loop" in result.recommendations[0].section.lower()
+
+
+# =============================================================================
+# signature identity (regressions)
+# =============================================================================
+
+
+def _call(name: str, tool_call_id: str, input_data: dict, *, msg_index: int) -> ToolCall:
+ """A successful tool call with a fixed 40 KB output."""
+ output = "x" * 40_000
+ return ToolCall(
+ name=name,
+ tool_call_id=tool_call_id,
+ input_data=input_data,
+ output=output,
+ is_error=False,
+ msg_index=msg_index,
+ output_bytes=len(output),
+ )
+
+
+def _mixed_calls() -> list[ToolCall]:
+ return [
+ _call("Read", "call_r0", {"file_path": "/repo/a.py"}, msg_index=0),
+ _call("Read", "call_r1", {"file_path": "/repo/a.py"}, msg_index=1),
+ _call("Read", "call_r2", {"file_path": "/repo/a.py"}, msg_index=2),
+ _call("Bash", "call_b0", {"command": "rg alpha /repo"}, msg_index=3),
+ _call("Bash", "call_b1", {"command": "rg alpha /repo"}, msg_index=4),
+ _call("Bash", "call_b2", {"command": "rg alpha /repo"}, msg_index=5),
+ _call("Grep", "call_g0", {"pattern": "beta"}, msg_index=6),
+ ]
+
+
+class TestReplayedTranscriptsDoNotDoubleCount:
+ """A resume that replays prior history must not recount the same calls.
+
+ ``detect_loops`` accumulates a signature across sessions so a recurring loop
+ adds up. Providers whose resume writes a *new* transcript replaying earlier
+ turns therefore present the same tool call more than once; identity comes
+ from ``tool_call_id``, which the provider assigns.
+ """
+
+ def _reads(self) -> list[ToolCall]:
+ return [
+ _call("Read", f"call_r{i}", {"file_path": "/repo/docs/design.md"}, msg_index=i)
+ for i in range(3)
+ ]
+
+ def test_replayed_calls_counted_once(self):
+ reads = self._reads()
+ single = detect_loops([SessionData(session_id="rollout-1", tool_calls=list(reads))])
+ replayed = detect_loops(
+ [
+ SessionData(session_id="rollout-1", tool_calls=list(reads)),
+ SessionData(session_id="rollout-2-resume", tool_calls=list(reads)),
+ ]
+ )
+ assert replayed[0].count == single[0].count
+ assert replayed[0].wasted_tokens == single[0].wasted_tokens
+
+ def test_new_calls_in_a_resume_still_accumulate(self):
+ # Dedup must not swallow genuinely new repetitions in the resumed run.
+ reads = self._reads()
+ extra = [
+ _call("Read", f"call_r{i}", {"file_path": "/repo/docs/design.md"}, msg_index=i)
+ for i in range(3, 6)
+ ]
+ replayed = detect_loops(
+ [
+ SessionData(session_id="rollout-1", tool_calls=list(reads)),
+ SessionData(session_id="rollout-2-resume", tool_calls=list(reads) + extra),
+ ]
+ )
+ assert replayed[0].count == 6
+
+ def test_calls_without_ids_are_not_deduped(self):
+ # An id-less scanner must keep counting repetitions rather than collapse.
+ calls = [
+ _call("Read", "", {"file_path": "/repo/docs/design.md"}, msg_index=i) for i in range(3)
+ ]
+ loops = detect_loops(
+ [
+ SessionData(session_id="a", tool_calls=list(calls)),
+ SessionData(session_id="b", tool_calls=list(calls)),
+ ]
+ )
+ assert loops[0].count == 6
+
+ @pytest.mark.parametrize("replays", [1, 2, 3, 5])
+ def test_replaying_a_transcript_never_changes_the_count(self, replays):
+ calls = _mixed_calls()
+ sessions = [
+ SessionData(session_id=f"rollout-{i}", tool_calls=list(calls)) for i in range(replays)
+ ]
+
+ once = detect_loops([SessionData(session_id="rollout-0", tool_calls=list(calls))])
+ many = detect_loops(sessions)
+
+ assert sorted(lp.count for lp in many) == sorted(lp.count for lp in once)
+
+
+class TestDedupRespectsTheOccurrenceThreshold:
+ """Dedup must not leave behind loops that no longer meet the bar.
+
+ ``detect_loops`` screens a group against ``min_occurrences`` before removing
+ replayed calls, so a group can clear the bar only *because* of duplicates and
+ still be reported once they are gone. Real transcripts do repeat a
+ ``tool_use_id`` within one file, so this is reachable from the default
+ Claude Code path.
+ """
+
+ def test_a_group_that_only_duplicates_is_not_a_loop(self):
+ # One call, written into the transcript three times.
+ calls = [
+ _call("Read", "toolu_SAME", {"file_path": "/repo/design.md"}, msg_index=i)
+ for i in range(3)
+ ]
+
+ assert detect_loops([SessionData(session_id="s", tool_calls=calls)]) == []
+
+ def test_partial_dedup_still_has_to_clear_the_bar(self):
+ # Three calls, two distinct — below DEFAULT_MIN_OCCURRENCES once deduped.
+ calls = [
+ _call("Read", "toolu_A", {"file_path": "/repo/design.md"}, msg_index=0),
+ _call("Read", "toolu_A", {"file_path": "/repo/design.md"}, msg_index=1),
+ _call("Read", "toolu_B", {"file_path": "/repo/design.md"}, msg_index=2),
+ ]
+
+ assert detect_loops([SessionData(session_id="s", tool_calls=calls)]) == []
+
+ def test_a_real_loop_padded_with_duplicates_is_still_reported(self):
+ # Dedup must subtract the replays without dropping the genuine loop.
+ distinct = [
+ _call("Read", f"toolu_{i}", {"file_path": "/repo/design.md"}, msg_index=i)
+ for i in range(3)
+ ]
+ padded = distinct + [
+ _call("Read", "toolu_0", {"file_path": "/repo/design.md"}, msg_index=9)
+ ]
+
+ loops = detect_loops([SessionData(session_id="s", tool_calls=padded)])
+
+ assert [lp.count for lp in loops] == [3]
+
+ def test_no_loop_is_reported_with_zero_measured_waste(self):
+ """A zero-waste loop is the tell that a group survived on duplicates.
+
+ ``format_loops_for_digest`` bills every entry to the LLM as HIGHEST
+ PRIORITY, and ``SessionAnalyzer.analyze`` treats a non-empty loop list as
+ reason enough to call the model, so a zero-waste entry buys an LLM round
+ trip for a session with nothing to report.
+ """
+ calls = [
+ _call("Read", "toolu_SAME", {"file_path": "/repo/design.md"}, msg_index=i)
+ for i in range(4)
+ ]
+
+ loops = detect_loops([SessionData(session_id="s", tool_calls=calls)])
+
+ assert [lp for lp in loops if lp.wasted_tokens == 0] == []
+
+
+class TestFixturesSatisfyTheDedupContract:
+ """Dedup keys on ``tool_call_id``, so the shipped fixtures must scope theirs.
+
+ ``detect_loops`` accumulates a signature across sessions. An id that is only
+ unique *within* a session makes two unrelated sessions look like one replayed
+ twice, silently halving a real loop.
+ """
+
+ def test_the_same_loop_in_two_sessions_accumulates(self):
+ monday = refetch_loop_session("session-monday")
+ tuesday = refetch_loop_session("session-tuesday")
+
+ one = detect_loops([monday])
+ both = detect_loops([monday, tuesday])
+
+ assert both[0].count == one[0].count * 2
+
+ def test_fixture_ids_are_unique_across_sessions(self):
+ monday = {c.tool_call_id for c in refetch_loop_session("session-monday").tool_calls}
+ tuesday = {c.tool_call_id for c in refetch_loop_session("session-tuesday").tool_calls}
+
+ assert monday.isdisjoint(tuesday)
+
+
+class TestSubThresholdSessionsDoNotCombine:
+ """A session that only looped in replays must not contribute at all.
+
+ ``min_occurrences`` is screened against the *pre-dedup* per-session group, so
+ a session whose repetitions are all replays of one call still qualifies and
+ hands its single real call to the global view. Three such sessions then clear
+ the bar together, reporting a loop no conversation actually ran.
+ """
+
+ def _replay_padded_session(self, session_id: str, call_id: str) -> SessionData:
+ # One real Read, written into this transcript three times.
+ return SessionData(
+ session_id=session_id,
+ tool_calls=[
+ _call("Read", call_id, {"file_path": "/repo/design.md"}, msg_index=i)
+ for i in range(3)
+ ],
+ )
+
+ def test_each_session_alone_is_not_a_loop(self):
+ sessions = [
+ self._replay_padded_session("s1", "toolu_s1"),
+ self._replay_padded_session("s2", "toolu_s2"),
+ self._replay_padded_session("s3", "toolu_s3"),
+ ]
+
+ assert [detect_loops([s]) for s in sessions] == [[], [], []]
+
+ def test_sub_threshold_sessions_do_not_combine_into_a_loop(self):
+ sessions = [
+ self._replay_padded_session("s1", "toolu_s1"),
+ self._replay_padded_session("s2", "toolu_s2"),
+ self._replay_padded_session("s3", "toolu_s3"),
+ ]
+
+ assert detect_loops(sessions) == []