Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix: close TOIN feedback loop — retrievals now flow back for learning
Tejas Chopraacddeff3840c
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.pyindex 1d1ba3749..3ee6fdaa6 100644--- a/headroom/proxy/server.py+++ b/headroom/proxy/server.py@@ -2139,9 +2139,7 @@ class HeadroomProxy:elif isinstance(existing_system, list):# system is a list of content blocks (e.g., with cache_control).# Append memory context as a new text block — never overwrite.- body["system"] = existing_system + [- {"type": "text", "text": context}- ]+ body["system"] = existing_system + [{"type": "text", "text": context}]else:# No existing system prompt — set as stringbody["system"] = context@@ -2487,7 +2485,9 @@ class HeadroomProxy:# CCR Tool Injection: Inject retrieval tool if compression occurredtools = body.get("tools")_original_tools = tools # Preserve for diagnostic / future retry- if (self.config.ccr_inject_tool or self.config.ccr_inject_system_instructions) and not _bypass:+ if (+ self.config.ccr_inject_tool or self.config.ccr_inject_system_instructions+ ) and not _bypass:# Create fresh injector to avoid state leakage between requestsinjector = CCRToolInjector(provider="anthropic",@@ -2837,13 +2837,9 @@ class HeadroomProxy:with open(debug_file, "w") as f:json.dump(debug_payload, f, indent=2, default=str)- logger.warning(- f"[{request_id}] Full debug dump: {debug_file}"- )+ logger.warning(f"[{request_id}] Full debug dump: {debug_file}")except Exception as dump_err:- logger.error(- f"[{request_id}] Failed to write debug dump: {dump_err}"- )+ logger.error(f"[{request_id}] Failed to write debug dump: {dump_err}")# Parse response for CCR handlingresp_json = None@@ -4497,6 +4493,53 @@ class HeadroomProxy:return events+ def _record_ccr_feedback_from_response(+ self, response: dict, provider: str, request_id: str+ ) -> None:+ """Extract headroom_retrieve tool calls from a response and record feedback.++ This closes the TOIN feedback loop for streaming responses where+ the proxy can't intercept and handle retrieval calls inline.+ """+ from headroom.cache.compression_store import get_compression_store++ content = response.get("content", [])+ if not isinstance(content, list):+ return++ store = get_compression_store()++ for block in content:+ if not isinstance(block, dict):+ continue+ if block.get("type") != "tool_use":+ continue+ if block.get("name") != "headroom_retrieve":+ continue++ input_data = block.get("input", {})+ hash_key = input_data.get("hash")+ query = input_data.get("query")++ if not hash_key:+ continue++ logger.info(+ f"[{request_id}] CCR Feedback: Recording retrieval "+ f"hash={hash_key[:8]}... query={query!r}"+ )++ # Call store.retrieve()/search() for the side effect of triggering+ # the feedback chain: _log_retrieval -> process_pending_feedback+ # -> toin.record_retrieval(). We discard the returned content.+ try:+ if query:+ store.search(hash_key, query)+ else:+ store.retrieve(hash_key, query=None)+ except Exception as e:+ logger.debug(f"[{request_id}] CCR Feedback recording failed: {e}")+async def _stream_response(self,url: str,@@ -4643,6 +4686,19 @@ class HeadroomProxy:f"[{request_id}] Memory: Tool calls executed silently ""(streaming mode — no continuation)")++ # CCR Feedback: Record headroom_retrieve tool calls for TOIN learning.+ # In streaming mode, the client handles actual retrieval, but we+ # still need to record the event so TOIN learns which fields matter.+ if self.config.ccr_inject_tool and full_sse_data:+ ccr_parsed = (+ parsed_response+ if parsed_response+ else self._parse_sse_to_response(full_sse_data, provider)+ )+ if ccr_parsed:+ self._record_ccr_feedback_from_response(ccr_parsed, provider, request_id)+except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as e:logger.error(f"[{request_id}] Connection error to upstream API: {e}")error_event = {@@ -5171,7 +5227,9 @@ class HeadroomProxy:# CCR Tool Injection: Inject retrieval tool if compression occurredtools = body.get("tools")_original_tools = tools # Preserve for diagnostic / future retry- if (self.config.ccr_inject_tool or self.config.ccr_inject_system_instructions) and not _bypass:+ if (+ self.config.ccr_inject_tool or self.config.ccr_inject_system_instructions+ ) and not _bypass:injector = CCRToolInjector(provider="openai",inject_tool=self.config.ccr_inject_tool,@@ -5371,13 +5429,9 @@ class HeadroomProxy:with open(debug_file, "w") as f:json.dump(debug_payload, f, indent=2, default=str)- logger.warning(- f"[{request_id}] Full debug dump: {debug_file}"- )+ logger.warning(f"[{request_id}] Full debug dump: {debug_file}")except Exception as dump_err:- logger.error(- f"[{request_id}] Failed to write debug dump: {dump_err}"- )+ logger.error(f"[{request_id}] Failed to write debug dump: {dump_err}")total_latency = (time.time() - start_time) * 1000diff --git a/headroom/telemetry/toin.py b/headroom/telemetry/toin.pyindex e376a4222..4b7eb7061 100644--- a/headroom/telemetry/toin.py+++ b/headroom/telemetry/toin.py@@ -149,6 +149,9 @@ class ToolPattern:# This enables zero-latency signal detection without hardcoded patternsfield_semantics: dict[str, FieldSemantics] = field(default_factory=dict)+ # === Observation Counter ===+ observations: int = 0 # How many times get_recommendation() was called for this pattern+# === Confidence ===sample_size: int = 0user_count: int = 0 # Number of unique users (anonymized)@@ -198,6 +201,7 @@ class ToolPattern:"preserve_fields": self.preserve_fields,# Field-level semantics (TOIN Evolution)"field_semantics": {k: v.to_dict() for k, v in self.field_semantics.items()},+ "observations": self.observations,"sample_size": self.sample_size,"user_count": self.user_count,"confidence": self.confidence,@@ -236,6 +240,7 @@ class ToolPattern:"optimal_max_items","skip_compression_recommended","preserve_fields",+ "observations","sample_size","user_count","confidence",@@ -873,6 +878,10 @@ class ToolIntelligenceNetwork:reason="No pattern data for this tool type",)+ # Track observation: TOIN was consulted for this pattern+ pattern.observations += 1+ self._dirty = True+# Not enough samples for reliable recommendationif pattern.sample_size < self._config.min_samples_for_recommendation:hint = CompressionHint(diff --git a/tests/test_toin_feedback.py b/tests/test_toin_feedback.pynew file mode 100644index 000000000..62d382f71--- /dev/null+++ b/tests/test_toin_feedback.py@@ -0,0 +1,412 @@+"""Tests for TOIN feedback loop: headroom_retrieve calls flow back to TOIN."""++from __future__ import annotations++from unittest.mock import MagicMock, patch++import pytest++from headroom.telemetry import (+ TOINConfig,+ ToolIntelligenceNetwork,+ ToolPattern,+ ToolSignature,+ reset_toin,+)++[email protected](autouse=True)+def reset_globals(monkeypatch, tmp_path):+ """Reset global state before each test."""+ temp_toin_path = str(tmp_path / "toin_test.json")+ monkeypatch.setenv("HEADROOM_TOIN_PATH", temp_toin_path)+ reset_toin()+ yield+ reset_toin()+++def _make_config(min_samples: int = 5) -> TOINConfig:+ """Create a TOIN config for testing."""+ return TOINConfig(+ enabled=True,+ min_samples_for_recommendation=min_samples,+ storage_path="", # disable persistence+ )+++def _make_signature(structure_hash: str = "test_hash_123") -> ToolSignature:+ """Create a minimal ToolSignature for testing."""+ return ToolSignature(+ structure_hash=structure_hash,+ field_count=3,+ has_nested_objects=False,+ has_arrays=True,+ max_depth=1,+ )+++class TestGetRecommendationObservations:+ """Bug 1: get_recommendation() should increment observations counter."""++ def test_increments_observations_when_pattern_exists(self):+ """get_recommendation() should increment observations when pattern exists."""+ config = _make_config(min_samples=5)+ toin = ToolIntelligenceNetwork(config=config)+ sig = _make_signature("obs_test_hash")++ # Record enough compressions to create a pattern with sufficient samples+ for _ in range(15):+ toin.record_compression(+ tool_signature=sig,+ original_count=100,+ compressed_count=20,+ original_tokens=5000,+ compressed_tokens=1000,+ strategy="top_n",+ )++ # Get recommendation+ toin.get_recommendation(sig)++ # Check observations incremented+ pattern = toin._patterns[sig.structure_hash]+ assert pattern.observations == 1++ # Call again+ toin.get_recommendation(sig)+ assert pattern.observations == 2++ def test_increments_observations_even_below_min_samples(self):+ """observations increments even when sample_size < min_samples."""+ config = _make_config(min_samples=100)+ toin = ToolIntelligenceNetwork(config=config)+ sig = _make_signature("low_sample_hash")++ # Record just a few compressions (below min_samples)+ for _ in range(3):+ toin.record_compression(+ tool_signature=sig,+ original_count=50,+ compressed_count=10,+ original_tokens=2000,+ compressed_tokens=500,+ strategy="top_n",+ )++ result = toin.get_recommendation(sig)+ assert result.source == "local" # Not enough samples++ pattern = toin._patterns[sig.structure_hash]+ assert pattern.observations == 1++ def test_no_increment_for_unknown_pattern(self):+ """get_recommendation() should NOT increment for unknown patterns."""+ config = _make_config()+ toin = ToolIntelligenceNetwork(config=config)+ sig = _make_signature("nonexistent_hash")++ result = toin.get_recommendation(sig)+ assert result.source == "default"+ assert result.reason == "No pattern data for this tool type"++ # No pattern exists, nothing to increment+ assert "nonexistent_hash" not in toin._patterns++ def test_observations_survives_serialization(self):+ """observations field should serialize and deserialize correctly."""+ pattern = ToolPattern(+ tool_signature_hash="serial_test",+ total_compressions=10,+ observations=42,+ )++ d = pattern.to_dict()+ assert d["observations"] == 42++ restored = ToolPattern.from_dict(d)+ assert restored.observations == 42++ def test_observations_defaults_to_zero(self):+ """observations defaults to 0 for new patterns."""+ pattern = ToolPattern(tool_signature_hash="new_pattern")+ assert pattern.observations == 0+++class TestRecordRetrievalPopulatesFields:+ """Bug 1 related: record_retrieval with query_fields populates field data."""++ def test_record_retrieval_populates_fields(self):+ """record_retrieval with query_fields should populate field_retrieval_frequency."""+ config = _make_config()+ toin = ToolIntelligenceNetwork(config=config)+ sig_hash = "retrieval_field_test"++ # Record some compressions first to create the pattern+ sig = _make_signature(sig_hash)+ for _ in range(5):+ toin.record_compression(+ tool_signature=sig,+ original_count=50,+ compressed_count=10,+ original_tokens=2000,+ compressed_tokens=500,+ strategy="top_n",+ )++ # Record multiple retrievals with same field+ for _ in range(5):+ toin.record_retrieval(+ sig_hash,+ "search",+ query="error_message:timeout",+ query_fields=["error_message"],+ )++ pattern = toin._patterns[sig_hash]+ assert pattern.total_retrievals == 5+ assert pattern.search_retrievals == 5+ assert len(pattern.field_retrieval_frequency) > 0+++class TestCCRFeedbackExtraction:+ """Bug 2: _record_ccr_feedback_from_response extracts headroom_retrieve calls."""++ def test_extract_headroom_retrieve_from_response(self):+ """Should detect headroom_retrieve tool_use blocks in response content."""+ response = {+ "content": [+ {"type": "text", "text": "Let me retrieve that."},+ {+ "type": "tool_use",+ "id": "toolu_123",+ "name": "headroom_retrieve",+ "input": {"hash": "abc123def456", "query": "error fields"},+ },+ ]+ }++ # Extract tool calls the same way _record_ccr_feedback_from_response does+ content = response.get("content", [])+ retrieve_calls = []+ for block in content:+ if not isinstance(block, dict):+ continue+ if block.get("type") == "tool_use" and block.get("name") == "headroom_retrieve":+ input_data = block.get("input", {})+ if input_data.get("hash"):+ retrieve_calls.append(input_data)++ assert len(retrieve_calls) == 1+ assert retrieve_calls[0]["hash"] == "abc123def456"+ assert retrieve_calls[0]["query"] == "error fields"++ def test_ignore_non_retrieve_tool_calls(self):+ """Should ignore tool_use blocks that are not headroom_retrieve."""+ response = {+ "content": [+ {+ "type": "tool_use",+ "id": "toolu_456",+ "name": "some_other_tool",+ "input": {"data": "something"},+ },+ {+ "type": "tool_use",+ "id": "toolu_789",+ "name": "headroom_retrieve",+ "input": {"hash": "xyz789", "query": None},+ },+ ]+ }++ content = response.get("content", [])+ retrieve_calls = []+ for block in content:+ if not isinstance(block, dict):+ continue+ if block.get("type") == "tool_use" and block.get("name") == "headroom_retrieve":+ input_data = block.get("input", {})+ if input_data.get("hash"):+ retrieve_calls.append(input_data)++ assert len(retrieve_calls) == 1+ assert retrieve_calls[0]["hash"] == "xyz789"++ def test_empty_content_does_not_crash(self):+ """Should handle empty or missing content gracefully."""+ for response in [+ {"content": []},+ {"content": "not a list"},+ {},+ ]:+ content = response.get("content", [])+ if not isinstance(content, list):+ continue+ # Should not raise+ for _block in content:+ pass++ def test_missing_hash_skipped(self):+ """Should skip headroom_retrieve calls without a hash."""+ response = {+ "content": [+ {+ "type": "tool_use",+ "id": "toolu_000",+ "name": "headroom_retrieve",+ "input": {"query": "some query"}, # No hash+ },+ ]+ }++ content = response.get("content", [])+ retrieve_calls = []+ for block in content:+ if not isinstance(block, dict):+ continue+ if block.get("type") == "tool_use" and block.get("name") == "headroom_retrieve":+ input_data = block.get("input", {})+ if input_data.get("hash"):+ retrieve_calls.append(input_data)++ assert len(retrieve_calls) == 0+++class TestStreamingFeedbackIntegration:+ """Bug 2: Full feedback loop — streaming headroom_retrieve reaches TOIN."""++ def test_record_ccr_feedback_calls_store_search(self):+ """_record_ccr_feedback_from_response should call store.search for queries."""+ from headroom.proxy.server import HeadroomProxy++ response = {+ "content": [+ {+ "type": "tool_use",+ "id": "toolu_001",+ "name": "headroom_retrieve",+ "input": {"hash": "feedbackhash1", "query": "error details"},+ },+ ]+ }++ mock_store = MagicMock()+ with patch(+ "headroom.cache.compression_store.get_compression_store",+ return_value=mock_store,+ ):+ # Create a minimal proxy to test the method+ proxy = HeadroomProxy.__new__(HeadroomProxy)+ proxy.config = MagicMock()+ proxy.config.ccr_inject_tool = True++ proxy._record_ccr_feedback_from_response(response, "anthropic", "req-test-001")++ mock_store.search.assert_called_once_with("feedbackhash1", "error details")++ def test_record_ccr_feedback_calls_store_retrieve_no_query(self):+ """_record_ccr_feedback_from_response should call store.retrieve when no query."""+ from headroom.proxy.server import HeadroomProxy++ response = {+ "content": [+ {+ "type": "tool_use",+ "id": "toolu_002",+ "name": "headroom_retrieve",+ "input": {"hash": "feedbackhash2"},+ },+ ]+ }++ mock_store = MagicMock()+ with patch(+ "headroom.cache.compression_store.get_compression_store",+ return_value=mock_store,+ ):+ proxy = HeadroomProxy.__new__(HeadroomProxy)+ proxy.config = MagicMock()+ proxy.config.ccr_inject_tool = True++ proxy._record_ccr_feedback_from_response(response, "anthropic", "req-test-002")++ mock_store.retrieve.assert_called_once_with("feedbackhash2", query=None)++ def test_record_ccr_feedback_handles_store_exception(self):+ """_record_ccr_feedback_from_response should not raise on store errors."""+ from headroom.proxy.server import HeadroomProxy++ response = {+ "content": [+ {+ "type": "tool_use",+ "id": "toolu_003",+ "name": "headroom_retrieve",+ "input": {"hash": "feedbackhash3", "query": "test"},+ },+ ]+ }++ mock_store = MagicMock()+ mock_store.search.side_effect = RuntimeError("store unavailable")+ with patch(+ "headroom.cache.compression_store.get_compression_store",+ return_value=mock_store,+ ):+ proxy = HeadroomProxy.__new__(HeadroomProxy)+ proxy.config = MagicMock()+ proxy.config.ccr_inject_tool = True++ # Should not raise+ proxy._record_ccr_feedback_from_response(response, "anthropic", "req-test-003")+++class TestParseSSEToolUse:+ """Bug 3: _parse_sse_to_response correctly handles tool_use blocks."""++ def test_parse_sse_extracts_tool_use(self):+ """SSE with tool_use content_block should be parsed correctly."""+ from headroom.proxy.server import HeadroomProxy++ sse_data = (+ 'data: {"type":"message_start","message":{"id":"msg_01","model":"claude-3-5-sonnet-20241022","role":"assistant","stop_reason":null,"usage":{"input_tokens":100,"output_tokens":0}}}\n'+ "\n"+ 'data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}\n'+ "\n"+ 'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Let me retrieve that."}}\n'+ "\n"+ 'data: {"type":"content_block_stop","index":0}\n'+ "\n"+ 'data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_abc","name":"headroom_retrieve"}}\n'+ "\n"+ 'data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\\"hash\\": \\"abc123\\", \\"query\\": \\"error\\"}"}}\n'+ "\n"+ 'data: {"type":"content_block_stop","index":1}\n'+ "\n"+ 'data: {"type":"message_delta","delta":{"stop_reason":"tool_use"},"usage":{"output_tokens":50}}\n'+ )++ proxy = HeadroomProxy.__new__(HeadroomProxy)+ result = proxy._parse_sse_to_response(sse_data, "anthropic")++ assert result is not None+ assert len(result["content"]) == 2++ text_block = result["content"][0]+ assert text_block["type"] == "text"+ assert "retrieve" in text_block["text"]++ tool_block = result["content"][1]+ assert tool_block["type"] == "tool_use"+ assert tool_block["name"] == "headroom_retrieve"+ assert tool_block["id"] == "toolu_abc"+ assert tool_block["input"]["hash"] == "abc123"+ assert tool_block["input"]["query"] == "error"++ def test_parse_sse_non_anthropic_returns_none(self):+ """Non-anthropic provider should return None."""+ from headroom.proxy.server import HeadroomProxy++ proxy = HeadroomProxy.__new__(HeadroomProxy)+ result = proxy._parse_sse_to_response("data: {}", "openai")+ assert result is None