Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

refactor(proxy): extract tool injection tracker (#2002)

JD Davisd1c484b16414
diff --git a/headroom/proxy/helpers.py b/headroom/proxy/helpers.py
index d4b51f280..f3c95e2a8 100644
--- a/headroom/proxy/helpers.py
+++ b/headroom/proxy/helpers.py
@@ -56,6 +56,7 @@ from headroom.proxy.tool_injection_config import (
from headroom.proxy.tool_injection_config import (
get_tool_tracker_max_sessions as _get_tool_tracker_max_sessions,
)
+from headroom.proxy.tool_injection_tracker import SessionToolTracker as _SessionToolTracker
if TYPE_CHECKING:
import httpx
@@ -1884,133 +1885,13 @@ def serialize_tool_definition_canonical(tool_definition: dict[str, Any]) -> byte
return serialize_body_canonical(tool_definition)
-class SessionToolTracker:
- """Bounded LRU tracker recording per-session memory-tool injection state.
-
- Once memory injects tool definitions into a session, future requests
- in that session always inject the byte-equal same definitions —
- never toggling on/off mid-session (guide §6.3 #2). The first turn's
- canonical bytes are stored as the golden definition; subsequent
- turns reuse those bytes verbatim.
-
- State per session: ordered list of (tool_name → golden_bytes)
- pairs. Order is preserved so the rebuilt tool list matches the
- original injection order.
-
- Bounded by ``max_sessions`` (default 1000) via ``OrderedDict`` LRU
- eviction: hits move-to-end; overflow pops oldest. Reentrant lock so
- future callers from inside another locked method don't self-deadlock
- (mirrors `SessionBetaTracker` / `CompressionCache` pattern).
-
- The tracker is provider-aware: the same ``session_id`` for Anthropic
- and OpenAI keeps independent state (the tool schemas differ in
- format).
- """
+class SessionToolTracker(_SessionToolTracker):
+ """Env-aware compatibility wrapper for the pure session tool tracker."""
def __init__(self, max_sessions: int | None = None) -> None:
if max_sessions is None:
max_sessions = get_tool_tracker_max_sessions()
- if max_sessions <= 0:
- raise ValueError("max_sessions must be > 0")
- self._max_sessions: int = max_sessions
- self._lock = threading.RLock()
- # Value is an OrderedDict[tool_name -> golden_definition_bytes].
- # Storing per-tool bytes (not the entire tools list) keeps the
- # tracker resilient to non-memory tool list changes by the client
- # (which are the client's responsibility, not ours to gate).
- self._sessions: OrderedDict[tuple[str, str], OrderedDict[str, bytes]] = OrderedDict()
-
- @property
- def active_sessions(self) -> int:
- with self._lock:
- return len(self._sessions)
-
- def _key(self, provider: str, session_id: str) -> tuple[str, str]:
- return (provider, session_id)
-
- def should_inject(self, provider: str, session_id: str) -> bool:
- """Return True iff this session has previously injected memory tools.
-
- Used by the sticky-on path: when memory is disabled this turn but
- the session previously injected, we still inject the golden bytes.
- """
- if not provider:
- raise ValueError("provider must be non-empty")
- if not session_id:
- raise ValueError("session_id must be non-empty")
- with self._lock:
- entry = self._sessions.get(self._key(provider, session_id))
- if entry is None:
- return False
- # LRU touch on read so the carry-over decision keeps the
- # session in the hot set.
- self._sessions.move_to_end(self._key(provider, session_id))
- return len(entry) > 0
-
- def get_golden_definitions(
- self, provider: str, session_id: str
- ) -> list[tuple[str, bytes]] | None:
- """Return the previously-recorded (name, bytes) pairs for the session.
-
- Returns ``None`` when the session has never injected memory tools.
- Callers replay the bytes verbatim into ``body["tools"]``.
- """
- if not provider:
- raise ValueError("provider must be non-empty")
- if not session_id:
- raise ValueError("session_id must be non-empty")
- with self._lock:
- entry = self._sessions.get(self._key(provider, session_id))
- if entry is None:
- return None
- self._sessions.move_to_end(self._key(provider, session_id))
- # Snapshot — never expose internal storage directly.
- return [(name, golden_bytes) for name, golden_bytes in entry.items()]
-
- def record_injection(
- self,
- provider: str,
- session_id: str,
- tool_name: str,
- tool_definition_bytes: bytes,
- ) -> None:
- """Record the golden bytes for a single memory tool in this session.
-
- First-write wins: re-recording the same ``tool_name`` for an
- existing session is a no-op (prevents drift if the canonical
- serialization output changed between deploys mid-session). For
- a *new* session, record fresh. LRU bound applies on every write.
- """
- if not provider:
- raise ValueError("provider must be non-empty")
- if not session_id:
- raise ValueError("session_id must be non-empty")
- if not tool_name:
- raise ValueError("tool_name must be non-empty")
- if not tool_definition_bytes:
- raise ValueError("tool_definition_bytes must be non-empty")
-
- key = self._key(provider, session_id)
-
- with self._lock:
- entry = self._sessions.get(key)
- if entry is None:
- entry = OrderedDict()
- self._sessions[key] = entry
-
- # First-write wins: only record if not already pinned.
- if tool_name not in entry:
- entry[tool_name] = tool_definition_bytes
-
- # LRU touch + bound enforcement.
- self._sessions.move_to_end(key)
- while len(self._sessions) > self._max_sessions:
- self._sessions.popitem(last=False)
-
- def reset(self) -> None:
- """Clear all session state (test helper)."""
- with self._lock:
- self._sessions.clear()
+ super().__init__(max_sessions=max_sessions)
# Process-wide singleton. Lazily replaced by tests via
diff --git a/headroom/proxy/tool_injection_tracker.py b/headroom/proxy/tool_injection_tracker.py
new file mode 100644
index 000000000..2b80a0b41
--- /dev/null
+++ b/headroom/proxy/tool_injection_tracker.py
@@ -0,0 +1,93 @@
+"""Session-scoped state for sticky memory tool injection."""
+
+from __future__ import annotations
+
+import threading
+from collections import OrderedDict
+
+
+class SessionToolTracker:
+ """Bounded LRU tracker recording per-session memory-tool injection state."""
+
+ def __init__(self, max_sessions: int) -> None:
+ if max_sessions <= 0:
+ raise ValueError("max_sessions must be > 0")
+ self._max_sessions: int = max_sessions
+ self._lock = threading.RLock()
+ self._sessions: OrderedDict[tuple[str, str], OrderedDict[str, bytes]] = OrderedDict()
+
+ @property
+ def active_sessions(self) -> int:
+ with self._lock:
+ return len(self._sessions)
+
+ def _key(self, provider: str, session_id: str) -> tuple[str, str]:
+ return (provider, session_id)
+
+ def should_inject(self, provider: str, session_id: str) -> bool:
+ """Return True when this session has previously injected memory tools."""
+
+ if not provider:
+ raise ValueError("provider must be non-empty")
+ if not session_id:
+ raise ValueError("session_id must be non-empty")
+ key = self._key(provider, session_id)
+ with self._lock:
+ entry = self._sessions.get(key)
+ if entry is None:
+ return False
+ self._sessions.move_to_end(key)
+ return len(entry) > 0
+
+ def get_golden_definitions(
+ self, provider: str, session_id: str
+ ) -> list[tuple[str, bytes]] | None:
+ """Return the previously recorded (tool_name, bytes) pairs for a session."""
+
+ if not provider:
+ raise ValueError("provider must be non-empty")
+ if not session_id:
+ raise ValueError("session_id must be non-empty")
+ key = self._key(provider, session_id)
+ with self._lock:
+ entry = self._sessions.get(key)
+ if entry is None:
+ return None
+ self._sessions.move_to_end(key)
+ return [(name, golden_bytes) for name, golden_bytes in entry.items()]
+
+ def record_injection(
+ self,
+ provider: str,
+ session_id: str,
+ tool_name: str,
+ tool_definition_bytes: bytes,
+ ) -> None:
+ """Record golden bytes for a memory tool in this provider/session."""
+
+ if not provider:
+ raise ValueError("provider must be non-empty")
+ if not session_id:
+ raise ValueError("session_id must be non-empty")
+ if not tool_name:
+ raise ValueError("tool_name must be non-empty")
+ if not tool_definition_bytes:
+ raise ValueError("tool_definition_bytes must be non-empty")
+
+ key = self._key(provider, session_id)
+ with self._lock:
+ entry = self._sessions.get(key)
+ if entry is None:
+ entry = OrderedDict()
+ self._sessions[key] = entry
+ if tool_name not in entry:
+ entry[tool_name] = tool_definition_bytes
+ self._sessions.move_to_end(key)
+ while len(self._sessions) > self._max_sessions:
+ self._sessions.popitem(last=False)
+
+ def reset(self) -> None:
+ """Clear all session state."""
+
+ with self._lock:
+ self._sessions.clear()
diff --git a/tests/test_tool_injection_tracker.py b/tests/test_tool_injection_tracker.py
new file mode 100644
index 000000000..07b5abff3
--- /dev/null
+++ b/tests/test_tool_injection_tracker.py
@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+import pytest
+
+from headroom.proxy.tool_injection_tracker import SessionToolTracker
+
+
+def test_tracker_reports_unknown_session_as_not_injected() -> None:
+ tracker = SessionToolTracker(max_sessions=10)
+
+ assert tracker.should_inject("anthropic", "s-1") is False
+
+
+def test_tracker_records_and_returns_golden_bytes_in_order() -> None:
+ tracker = SessionToolTracker(max_sessions=10)
+
+ tracker.record_injection("anthropic", "s-1", "memory_save", b"save")
+ tracker.record_injection("anthropic", "s-1", "memory_search", b"search")
+
+ assert tracker.should_inject("anthropic", "s-1") is True
+ assert tracker.get_golden_definitions("anthropic", "s-1") == [
+ ("memory_save", b"save"),
+ ("memory_search", b"search"),
+ ]
+
+
+def test_tracker_first_write_wins_per_tool_name() -> None:
+ tracker = SessionToolTracker(max_sessions=10)
+
+ tracker.record_injection("anthropic", "s-1", "memory_save", b"original")
+ tracker.record_injection("anthropic", "s-1", "memory_save", b"drift")
+
+ assert tracker.get_golden_definitions("anthropic", "s-1") == [("memory_save", b"original")]
+
+
+def test_tracker_keeps_provider_namespaces_independent() -> None:
+ tracker = SessionToolTracker(max_sessions=10)
+
+ tracker.record_injection("anthropic", "shared", "memory_save", b"anthropic")
+ tracker.record_injection("openai", "shared", "memory_save", b"openai")
+
+ assert tracker.get_golden_definitions("anthropic", "shared") == [("memory_save", b"anthropic")]
+ assert tracker.get_golden_definitions("openai", "shared") == [("memory_save", b"openai")]
+
+
+def test_tracker_evicts_least_recently_used_session() -> None:
+ tracker = SessionToolTracker(max_sessions=2)
+ tracker.record_injection("anthropic", "s-1", "memory_save", b"a")
+ tracker.record_injection("anthropic", "s-2", "memory_save", b"b")
+
+ assert tracker.should_inject("anthropic", "s-1") is True
+ tracker.record_injection("anthropic", "s-3", "memory_save", b"c")
+
+ assert tracker.active_sessions == 2
+ assert tracker.should_inject("anthropic", "s-1") is True
+ assert tracker.should_inject("anthropic", "s-2") is False
+ assert tracker.should_inject("anthropic", "s-3") is True
+
+
+def test_tracker_validates_constructor_and_record_inputs() -> None:
+ with pytest.raises(ValueError, match="max_sessions"):
+ SessionToolTracker(max_sessions=0)
+
+ tracker = SessionToolTracker(max_sessions=10)
+ with pytest.raises(ValueError, match="provider"):
+ tracker.should_inject("", "s-1")
+ with pytest.raises(ValueError, match="session_id"):
+ tracker.get_golden_definitions("anthropic", "")
+ with pytest.raises(ValueError, match="tool_name"):
+ tracker.record_injection("anthropic", "s-1", "", b"bytes")
+ with pytest.raises(ValueError, match="tool_definition_bytes"):
+ tracker.record_injection("anthropic", "s-1", "memory_save", b"")