Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
refactor(proxy): extract tool injection tracker (#2002)
JD Davisd1c484b16414
diff --git a/headroom/proxy/helpers.py b/headroom/proxy/helpers.pyindex d4b51f280..f3c95e2a8 100644--- a/headroom/proxy/helpers.py+++ b/headroom/proxy/helpers.py@@ -56,6 +56,7 @@ from headroom.proxy.tool_injection_config import (from headroom.proxy.tool_injection_config import (get_tool_tracker_max_sessions as _get_tool_tracker_max_sessions,)+from headroom.proxy.tool_injection_tracker import SessionToolTracker as _SessionToolTrackerif TYPE_CHECKING:import httpx@@ -1884,133 +1885,13 @@ def serialize_tool_definition_canonical(tool_definition: dict[str, Any]) -> bytereturn serialize_body_canonical(tool_definition)-class SessionToolTracker:- """Bounded LRU tracker recording per-session memory-tool injection state.-- Once memory injects tool definitions into a session, future requests- in that session always inject the byte-equal same definitions —- never toggling on/off mid-session (guide §6.3 #2). The first turn's- canonical bytes are stored as the golden definition; subsequent- turns reuse those bytes verbatim.-- State per session: ordered list of (tool_name → golden_bytes)- pairs. Order is preserved so the rebuilt tool list matches the- original injection order.-- Bounded by ``max_sessions`` (default 1000) via ``OrderedDict`` LRU- eviction: hits move-to-end; overflow pops oldest. Reentrant lock so- future callers from inside another locked method don't self-deadlock- (mirrors `SessionBetaTracker` / `CompressionCache` pattern).-- The tracker is provider-aware: the same ``session_id`` for Anthropic- and OpenAI keeps independent state (the tool schemas differ in- format).- """+class SessionToolTracker(_SessionToolTracker):+ """Env-aware compatibility wrapper for the pure session tool tracker."""def __init__(self, max_sessions: int | None = None) -> None:if max_sessions is None:max_sessions = get_tool_tracker_max_sessions()- if max_sessions <= 0:- raise ValueError("max_sessions must be > 0")- self._max_sessions: int = max_sessions- self._lock = threading.RLock()- # Value is an OrderedDict[tool_name -> golden_definition_bytes].- # Storing per-tool bytes (not the entire tools list) keeps the- # tracker resilient to non-memory tool list changes by the client- # (which are the client's responsibility, not ours to gate).- self._sessions: OrderedDict[tuple[str, str], OrderedDict[str, bytes]] = OrderedDict()-- @property- def active_sessions(self) -> int:- with self._lock:- return len(self._sessions)-- def _key(self, provider: str, session_id: str) -> tuple[str, str]:- return (provider, session_id)-- def should_inject(self, provider: str, session_id: str) -> bool:- """Return True iff this session has previously injected memory tools.-- Used by the sticky-on path: when memory is disabled this turn but- the session previously injected, we still inject the golden bytes.- """- if not provider:- raise ValueError("provider must be non-empty")- if not session_id:- raise ValueError("session_id must be non-empty")- with self._lock:- entry = self._sessions.get(self._key(provider, session_id))- if entry is None:- return False- # LRU touch on read so the carry-over decision keeps the- # session in the hot set.- self._sessions.move_to_end(self._key(provider, session_id))- return len(entry) > 0-- def get_golden_definitions(- self, provider: str, session_id: str- ) -> list[tuple[str, bytes]] | None:- """Return the previously-recorded (name, bytes) pairs for the session.-- Returns ``None`` when the session has never injected memory tools.- Callers replay the bytes verbatim into ``body["tools"]``.- """- if not provider:- raise ValueError("provider must be non-empty")- if not session_id:- raise ValueError("session_id must be non-empty")- with self._lock:- entry = self._sessions.get(self._key(provider, session_id))- if entry is None:- return None- self._sessions.move_to_end(self._key(provider, session_id))- # Snapshot — never expose internal storage directly.- return [(name, golden_bytes) for name, golden_bytes in entry.items()]-- def record_injection(- self,- provider: str,- session_id: str,- tool_name: str,- tool_definition_bytes: bytes,- ) -> None:- """Record the golden bytes for a single memory tool in this session.-- First-write wins: re-recording the same ``tool_name`` for an- existing session is a no-op (prevents drift if the canonical- serialization output changed between deploys mid-session). For- a *new* session, record fresh. LRU bound applies on every write.- """- if not provider:- raise ValueError("provider must be non-empty")- if not session_id:- raise ValueError("session_id must be non-empty")- if not tool_name:- raise ValueError("tool_name must be non-empty")- if not tool_definition_bytes:- raise ValueError("tool_definition_bytes must be non-empty")-- key = self._key(provider, session_id)-- with self._lock:- entry = self._sessions.get(key)- if entry is None:- entry = OrderedDict()- self._sessions[key] = entry-- # First-write wins: only record if not already pinned.- if tool_name not in entry:- entry[tool_name] = tool_definition_bytes-- # LRU touch + bound enforcement.- self._sessions.move_to_end(key)- while len(self._sessions) > self._max_sessions:- self._sessions.popitem(last=False)-- def reset(self) -> None:- """Clear all session state (test helper)."""- with self._lock:- self._sessions.clear()+ super().__init__(max_sessions=max_sessions)# Process-wide singleton. Lazily replaced by tests viadiff --git a/headroom/proxy/tool_injection_tracker.py b/headroom/proxy/tool_injection_tracker.pynew file mode 100644index 000000000..2b80a0b41--- /dev/null+++ b/headroom/proxy/tool_injection_tracker.py@@ -0,0 +1,93 @@+"""Session-scoped state for sticky memory tool injection."""++from __future__ import annotations++import threading+from collections import OrderedDict+++class SessionToolTracker:+ """Bounded LRU tracker recording per-session memory-tool injection state."""++ def __init__(self, max_sessions: int) -> None:+ if max_sessions <= 0:+ raise ValueError("max_sessions must be > 0")+ self._max_sessions: int = max_sessions+ self._lock = threading.RLock()+ self._sessions: OrderedDict[tuple[str, str], OrderedDict[str, bytes]] = OrderedDict()++ @property+ def active_sessions(self) -> int:+ with self._lock:+ return len(self._sessions)++ def _key(self, provider: str, session_id: str) -> tuple[str, str]:+ return (provider, session_id)++ def should_inject(self, provider: str, session_id: str) -> bool:+ """Return True when this session has previously injected memory tools."""++ if not provider:+ raise ValueError("provider must be non-empty")+ if not session_id:+ raise ValueError("session_id must be non-empty")+ key = self._key(provider, session_id)+ with self._lock:+ entry = self._sessions.get(key)+ if entry is None:+ return False+ self._sessions.move_to_end(key)+ return len(entry) > 0++ def get_golden_definitions(+ self, provider: str, session_id: str+ ) -> list[tuple[str, bytes]] | None:+ """Return the previously recorded (tool_name, bytes) pairs for a session."""++ if not provider:+ raise ValueError("provider must be non-empty")+ if not session_id:+ raise ValueError("session_id must be non-empty")+ key = self._key(provider, session_id)+ with self._lock:+ entry = self._sessions.get(key)+ if entry is None:+ return None+ self._sessions.move_to_end(key)+ return [(name, golden_bytes) for name, golden_bytes in entry.items()]++ def record_injection(+ self,+ provider: str,+ session_id: str,+ tool_name: str,+ tool_definition_bytes: bytes,+ ) -> None:+ """Record golden bytes for a memory tool in this provider/session."""++ if not provider:+ raise ValueError("provider must be non-empty")+ if not session_id:+ raise ValueError("session_id must be non-empty")+ if not tool_name:+ raise ValueError("tool_name must be non-empty")+ if not tool_definition_bytes:+ raise ValueError("tool_definition_bytes must be non-empty")++ key = self._key(provider, session_id)+ with self._lock:+ entry = self._sessions.get(key)+ if entry is None:+ entry = OrderedDict()+ self._sessions[key] = entry+ if tool_name not in entry:+ entry[tool_name] = tool_definition_bytes+ self._sessions.move_to_end(key)+ while len(self._sessions) > self._max_sessions:+ self._sessions.popitem(last=False)++ def reset(self) -> None:+ """Clear all session state."""++ with self._lock:+ self._sessions.clear()diff --git a/tests/test_tool_injection_tracker.py b/tests/test_tool_injection_tracker.pynew file mode 100644index 000000000..07b5abff3--- /dev/null+++ b/tests/test_tool_injection_tracker.py@@ -0,0 +1,72 @@+from __future__ import annotations++import pytest++from headroom.proxy.tool_injection_tracker import SessionToolTracker+++def test_tracker_reports_unknown_session_as_not_injected() -> None:+ tracker = SessionToolTracker(max_sessions=10)++ assert tracker.should_inject("anthropic", "s-1") is False+++def test_tracker_records_and_returns_golden_bytes_in_order() -> None:+ tracker = SessionToolTracker(max_sessions=10)++ tracker.record_injection("anthropic", "s-1", "memory_save", b"save")+ tracker.record_injection("anthropic", "s-1", "memory_search", b"search")++ assert tracker.should_inject("anthropic", "s-1") is True+ assert tracker.get_golden_definitions("anthropic", "s-1") == [+ ("memory_save", b"save"),+ ("memory_search", b"search"),+ ]+++def test_tracker_first_write_wins_per_tool_name() -> None:+ tracker = SessionToolTracker(max_sessions=10)++ tracker.record_injection("anthropic", "s-1", "memory_save", b"original")+ tracker.record_injection("anthropic", "s-1", "memory_save", b"drift")++ assert tracker.get_golden_definitions("anthropic", "s-1") == [("memory_save", b"original")]+++def test_tracker_keeps_provider_namespaces_independent() -> None:+ tracker = SessionToolTracker(max_sessions=10)++ tracker.record_injection("anthropic", "shared", "memory_save", b"anthropic")+ tracker.record_injection("openai", "shared", "memory_save", b"openai")++ assert tracker.get_golden_definitions("anthropic", "shared") == [("memory_save", b"anthropic")]+ assert tracker.get_golden_definitions("openai", "shared") == [("memory_save", b"openai")]+++def test_tracker_evicts_least_recently_used_session() -> None:+ tracker = SessionToolTracker(max_sessions=2)+ tracker.record_injection("anthropic", "s-1", "memory_save", b"a")+ tracker.record_injection("anthropic", "s-2", "memory_save", b"b")++ assert tracker.should_inject("anthropic", "s-1") is True+ tracker.record_injection("anthropic", "s-3", "memory_save", b"c")++ assert tracker.active_sessions == 2+ assert tracker.should_inject("anthropic", "s-1") is True+ assert tracker.should_inject("anthropic", "s-2") is False+ assert tracker.should_inject("anthropic", "s-3") is True+++def test_tracker_validates_constructor_and_record_inputs() -> None:+ with pytest.raises(ValueError, match="max_sessions"):+ SessionToolTracker(max_sessions=0)++ tracker = SessionToolTracker(max_sessions=10)+ with pytest.raises(ValueError, match="provider"):+ tracker.should_inject("", "s-1")+ with pytest.raises(ValueError, match="session_id"):+ tracker.get_golden_definitions("anthropic", "")+ with pytest.raises(ValueError, match="tool_name"):+ tracker.record_injection("anthropic", "s-1", "", b"bytes")+ with pytest.raises(ValueError, match="tool_definition_bytes"):+ tracker.record_injection("anthropic", "s-1", "memory_save", b"")