Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix: close TOIN feedback loop — retrievals now flow back for learning

Tejas Chopraacddeff3840c
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py
index 1d1ba3749..3ee6fdaa6 100644
--- a/headroom/proxy/server.py
+++ b/headroom/proxy/server.py
@@ -2139,9 +2139,7 @@ class HeadroomProxy:
elif isinstance(existing_system, list):
# system is a list of content blocks (e.g., with cache_control).
# Append memory context as a new text block — never overwrite.
- body["system"] = existing_system + [
- {"type": "text", "text": context}
- ]
+ body["system"] = existing_system + [{"type": "text", "text": context}]
else:
# No existing system prompt — set as string
body["system"] = context
@@ -2487,7 +2485,9 @@ class HeadroomProxy:
# CCR Tool Injection: Inject retrieval tool if compression occurred
tools = body.get("tools")
_original_tools = tools # Preserve for diagnostic / future retry
- if (self.config.ccr_inject_tool or self.config.ccr_inject_system_instructions) and not _bypass:
+ if (
+ self.config.ccr_inject_tool or self.config.ccr_inject_system_instructions
+ ) and not _bypass:
# Create fresh injector to avoid state leakage between requests
injector = CCRToolInjector(
provider="anthropic",
@@ -2837,13 +2837,9 @@ class HeadroomProxy:
with open(debug_file, "w") as f:
json.dump(debug_payload, f, indent=2, default=str)
- logger.warning(
- f"[{request_id}] Full debug dump: {debug_file}"
- )
+ logger.warning(f"[{request_id}] Full debug dump: {debug_file}")
except Exception as dump_err:
- logger.error(
- f"[{request_id}] Failed to write debug dump: {dump_err}"
- )
+ logger.error(f"[{request_id}] Failed to write debug dump: {dump_err}")
# Parse response for CCR handling
resp_json = None
@@ -4497,6 +4493,53 @@ class HeadroomProxy:
return events
+ def _record_ccr_feedback_from_response(
+ self, response: dict, provider: str, request_id: str
+ ) -> None:
+ """Extract headroom_retrieve tool calls from a response and record feedback.
+
+ This closes the TOIN feedback loop for streaming responses where
+ the proxy can't intercept and handle retrieval calls inline.
+ """
+ from headroom.cache.compression_store import get_compression_store
+
+ content = response.get("content", [])
+ if not isinstance(content, list):
+ return
+
+ store = get_compression_store()
+
+ for block in content:
+ if not isinstance(block, dict):
+ continue
+ if block.get("type") != "tool_use":
+ continue
+ if block.get("name") != "headroom_retrieve":
+ continue
+
+ input_data = block.get("input", {})
+ hash_key = input_data.get("hash")
+ query = input_data.get("query")
+
+ if not hash_key:
+ continue
+
+ logger.info(
+ f"[{request_id}] CCR Feedback: Recording retrieval "
+ f"hash={hash_key[:8]}... query={query!r}"
+ )
+
+ # Call store.retrieve()/search() for the side effect of triggering
+ # the feedback chain: _log_retrieval -> process_pending_feedback
+ # -> toin.record_retrieval(). We discard the returned content.
+ try:
+ if query:
+ store.search(hash_key, query)
+ else:
+ store.retrieve(hash_key, query=None)
+ except Exception as e:
+ logger.debug(f"[{request_id}] CCR Feedback recording failed: {e}")
+
async def _stream_response(
self,
url: str,
@@ -4643,6 +4686,19 @@ class HeadroomProxy:
f"[{request_id}] Memory: Tool calls executed silently "
"(streaming mode — no continuation)"
)
+
+ # CCR Feedback: Record headroom_retrieve tool calls for TOIN learning.
+ # In streaming mode, the client handles actual retrieval, but we
+ # still need to record the event so TOIN learns which fields matter.
+ if self.config.ccr_inject_tool and full_sse_data:
+ ccr_parsed = (
+ parsed_response
+ if parsed_response
+ else self._parse_sse_to_response(full_sse_data, provider)
+ )
+ if ccr_parsed:
+ self._record_ccr_feedback_from_response(ccr_parsed, provider, request_id)
+
except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as e:
logger.error(f"[{request_id}] Connection error to upstream API: {e}")
error_event = {
@@ -5171,7 +5227,9 @@ class HeadroomProxy:
# CCR Tool Injection: Inject retrieval tool if compression occurred
tools = body.get("tools")
_original_tools = tools # Preserve for diagnostic / future retry
- if (self.config.ccr_inject_tool or self.config.ccr_inject_system_instructions) and not _bypass:
+ if (
+ self.config.ccr_inject_tool or self.config.ccr_inject_system_instructions
+ ) and not _bypass:
injector = CCRToolInjector(
provider="openai",
inject_tool=self.config.ccr_inject_tool,
@@ -5371,13 +5429,9 @@ class HeadroomProxy:
with open(debug_file, "w") as f:
json.dump(debug_payload, f, indent=2, default=str)
- logger.warning(
- f"[{request_id}] Full debug dump: {debug_file}"
- )
+ logger.warning(f"[{request_id}] Full debug dump: {debug_file}")
except Exception as dump_err:
- logger.error(
- f"[{request_id}] Failed to write debug dump: {dump_err}"
- )
+ logger.error(f"[{request_id}] Failed to write debug dump: {dump_err}")
total_latency = (time.time() - start_time) * 1000
diff --git a/headroom/telemetry/toin.py b/headroom/telemetry/toin.py
index e376a4222..4b7eb7061 100644
--- a/headroom/telemetry/toin.py
+++ b/headroom/telemetry/toin.py
@@ -149,6 +149,9 @@ class ToolPattern:
# This enables zero-latency signal detection without hardcoded patterns
field_semantics: dict[str, FieldSemantics] = field(default_factory=dict)
+ # === Observation Counter ===
+ observations: int = 0 # How many times get_recommendation() was called for this pattern
+
# === Confidence ===
sample_size: int = 0
user_count: int = 0 # Number of unique users (anonymized)
@@ -198,6 +201,7 @@ class ToolPattern:
"preserve_fields": self.preserve_fields,
# Field-level semantics (TOIN Evolution)
"field_semantics": {k: v.to_dict() for k, v in self.field_semantics.items()},
+ "observations": self.observations,
"sample_size": self.sample_size,
"user_count": self.user_count,
"confidence": self.confidence,
@@ -236,6 +240,7 @@ class ToolPattern:
"optimal_max_items",
"skip_compression_recommended",
"preserve_fields",
+ "observations",
"sample_size",
"user_count",
"confidence",
@@ -873,6 +878,10 @@ class ToolIntelligenceNetwork:
reason="No pattern data for this tool type",
)
+ # Track observation: TOIN was consulted for this pattern
+ pattern.observations += 1
+ self._dirty = True
+
# Not enough samples for reliable recommendation
if pattern.sample_size < self._config.min_samples_for_recommendation:
hint = CompressionHint(
diff --git a/tests/test_toin_feedback.py b/tests/test_toin_feedback.py
new file mode 100644
index 000000000..62d382f71
--- /dev/null
+++ b/tests/test_toin_feedback.py
@@ -0,0 +1,412 @@
+"""Tests for TOIN feedback loop: headroom_retrieve calls flow back to TOIN."""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from headroom.telemetry import (
+ TOINConfig,
+ ToolIntelligenceNetwork,
+ ToolPattern,
+ ToolSignature,
+ reset_toin,
+)
+
+
[email protected](autouse=True)
+def reset_globals(monkeypatch, tmp_path):
+ """Reset global state before each test."""
+ temp_toin_path = str(tmp_path / "toin_test.json")
+ monkeypatch.setenv("HEADROOM_TOIN_PATH", temp_toin_path)
+ reset_toin()
+ yield
+ reset_toin()
+
+
+def _make_config(min_samples: int = 5) -> TOINConfig:
+ """Create a TOIN config for testing."""
+ return TOINConfig(
+ enabled=True,
+ min_samples_for_recommendation=min_samples,
+ storage_path="", # disable persistence
+ )
+
+
+def _make_signature(structure_hash: str = "test_hash_123") -> ToolSignature:
+ """Create a minimal ToolSignature for testing."""
+ return ToolSignature(
+ structure_hash=structure_hash,
+ field_count=3,
+ has_nested_objects=False,
+ has_arrays=True,
+ max_depth=1,
+ )
+
+
+class TestGetRecommendationObservations:
+ """Bug 1: get_recommendation() should increment observations counter."""
+
+ def test_increments_observations_when_pattern_exists(self):
+ """get_recommendation() should increment observations when pattern exists."""
+ config = _make_config(min_samples=5)
+ toin = ToolIntelligenceNetwork(config=config)
+ sig = _make_signature("obs_test_hash")
+
+ # Record enough compressions to create a pattern with sufficient samples
+ for _ in range(15):
+ toin.record_compression(
+ tool_signature=sig,
+ original_count=100,
+ compressed_count=20,
+ original_tokens=5000,
+ compressed_tokens=1000,
+ strategy="top_n",
+ )
+
+ # Get recommendation
+ toin.get_recommendation(sig)
+
+ # Check observations incremented
+ pattern = toin._patterns[sig.structure_hash]
+ assert pattern.observations == 1
+
+ # Call again
+ toin.get_recommendation(sig)
+ assert pattern.observations == 2
+
+ def test_increments_observations_even_below_min_samples(self):
+ """observations increments even when sample_size < min_samples."""
+ config = _make_config(min_samples=100)
+ toin = ToolIntelligenceNetwork(config=config)
+ sig = _make_signature("low_sample_hash")
+
+ # Record just a few compressions (below min_samples)
+ for _ in range(3):
+ toin.record_compression(
+ tool_signature=sig,
+ original_count=50,
+ compressed_count=10,
+ original_tokens=2000,
+ compressed_tokens=500,
+ strategy="top_n",
+ )
+
+ result = toin.get_recommendation(sig)
+ assert result.source == "local" # Not enough samples
+
+ pattern = toin._patterns[sig.structure_hash]
+ assert pattern.observations == 1
+
+ def test_no_increment_for_unknown_pattern(self):
+ """get_recommendation() should NOT increment for unknown patterns."""
+ config = _make_config()
+ toin = ToolIntelligenceNetwork(config=config)
+ sig = _make_signature("nonexistent_hash")
+
+ result = toin.get_recommendation(sig)
+ assert result.source == "default"
+ assert result.reason == "No pattern data for this tool type"
+
+ # No pattern exists, nothing to increment
+ assert "nonexistent_hash" not in toin._patterns
+
+ def test_observations_survives_serialization(self):
+ """observations field should serialize and deserialize correctly."""
+ pattern = ToolPattern(
+ tool_signature_hash="serial_test",
+ total_compressions=10,
+ observations=42,
+ )
+
+ d = pattern.to_dict()
+ assert d["observations"] == 42
+
+ restored = ToolPattern.from_dict(d)
+ assert restored.observations == 42
+
+ def test_observations_defaults_to_zero(self):
+ """observations defaults to 0 for new patterns."""
+ pattern = ToolPattern(tool_signature_hash="new_pattern")
+ assert pattern.observations == 0
+
+
+class TestRecordRetrievalPopulatesFields:
+ """Bug 1 related: record_retrieval with query_fields populates field data."""
+
+ def test_record_retrieval_populates_fields(self):
+ """record_retrieval with query_fields should populate field_retrieval_frequency."""
+ config = _make_config()
+ toin = ToolIntelligenceNetwork(config=config)
+ sig_hash = "retrieval_field_test"
+
+ # Record some compressions first to create the pattern
+ sig = _make_signature(sig_hash)
+ for _ in range(5):
+ toin.record_compression(
+ tool_signature=sig,
+ original_count=50,
+ compressed_count=10,
+ original_tokens=2000,
+ compressed_tokens=500,
+ strategy="top_n",
+ )
+
+ # Record multiple retrievals with same field
+ for _ in range(5):
+ toin.record_retrieval(
+ sig_hash,
+ "search",
+ query="error_message:timeout",
+ query_fields=["error_message"],
+ )
+
+ pattern = toin._patterns[sig_hash]
+ assert pattern.total_retrievals == 5
+ assert pattern.search_retrievals == 5
+ assert len(pattern.field_retrieval_frequency) > 0
+
+
+class TestCCRFeedbackExtraction:
+ """Bug 2: _record_ccr_feedback_from_response extracts headroom_retrieve calls."""
+
+ def test_extract_headroom_retrieve_from_response(self):
+ """Should detect headroom_retrieve tool_use blocks in response content."""
+ response = {
+ "content": [
+ {"type": "text", "text": "Let me retrieve that."},
+ {
+ "type": "tool_use",
+ "id": "toolu_123",
+ "name": "headroom_retrieve",
+ "input": {"hash": "abc123def456", "query": "error fields"},
+ },
+ ]
+ }
+
+ # Extract tool calls the same way _record_ccr_feedback_from_response does
+ content = response.get("content", [])
+ retrieve_calls = []
+ for block in content:
+ if not isinstance(block, dict):
+ continue
+ if block.get("type") == "tool_use" and block.get("name") == "headroom_retrieve":
+ input_data = block.get("input", {})
+ if input_data.get("hash"):
+ retrieve_calls.append(input_data)
+
+ assert len(retrieve_calls) == 1
+ assert retrieve_calls[0]["hash"] == "abc123def456"
+ assert retrieve_calls[0]["query"] == "error fields"
+
+ def test_ignore_non_retrieve_tool_calls(self):
+ """Should ignore tool_use blocks that are not headroom_retrieve."""
+ response = {
+ "content": [
+ {
+ "type": "tool_use",
+ "id": "toolu_456",
+ "name": "some_other_tool",
+ "input": {"data": "something"},
+ },
+ {
+ "type": "tool_use",
+ "id": "toolu_789",
+ "name": "headroom_retrieve",
+ "input": {"hash": "xyz789", "query": None},
+ },
+ ]
+ }
+
+ content = response.get("content", [])
+ retrieve_calls = []
+ for block in content:
+ if not isinstance(block, dict):
+ continue
+ if block.get("type") == "tool_use" and block.get("name") == "headroom_retrieve":
+ input_data = block.get("input", {})
+ if input_data.get("hash"):
+ retrieve_calls.append(input_data)
+
+ assert len(retrieve_calls) == 1
+ assert retrieve_calls[0]["hash"] == "xyz789"
+
+ def test_empty_content_does_not_crash(self):
+ """Should handle empty or missing content gracefully."""
+ for response in [
+ {"content": []},
+ {"content": "not a list"},
+ {},
+ ]:
+ content = response.get("content", [])
+ if not isinstance(content, list):
+ continue
+ # Should not raise
+ for _block in content:
+ pass
+
+ def test_missing_hash_skipped(self):
+ """Should skip headroom_retrieve calls without a hash."""
+ response = {
+ "content": [
+ {
+ "type": "tool_use",
+ "id": "toolu_000",
+ "name": "headroom_retrieve",
+ "input": {"query": "some query"}, # No hash
+ },
+ ]
+ }
+
+ content = response.get("content", [])
+ retrieve_calls = []
+ for block in content:
+ if not isinstance(block, dict):
+ continue
+ if block.get("type") == "tool_use" and block.get("name") == "headroom_retrieve":
+ input_data = block.get("input", {})
+ if input_data.get("hash"):
+ retrieve_calls.append(input_data)
+
+ assert len(retrieve_calls) == 0
+
+
+class TestStreamingFeedbackIntegration:
+ """Bug 2: Full feedback loop — streaming headroom_retrieve reaches TOIN."""
+
+ def test_record_ccr_feedback_calls_store_search(self):
+ """_record_ccr_feedback_from_response should call store.search for queries."""
+ from headroom.proxy.server import HeadroomProxy
+
+ response = {
+ "content": [
+ {
+ "type": "tool_use",
+ "id": "toolu_001",
+ "name": "headroom_retrieve",
+ "input": {"hash": "feedbackhash1", "query": "error details"},
+ },
+ ]
+ }
+
+ mock_store = MagicMock()
+ with patch(
+ "headroom.cache.compression_store.get_compression_store",
+ return_value=mock_store,
+ ):
+ # Create a minimal proxy to test the method
+ proxy = HeadroomProxy.__new__(HeadroomProxy)
+ proxy.config = MagicMock()
+ proxy.config.ccr_inject_tool = True
+
+ proxy._record_ccr_feedback_from_response(response, "anthropic", "req-test-001")
+
+ mock_store.search.assert_called_once_with("feedbackhash1", "error details")
+
+ def test_record_ccr_feedback_calls_store_retrieve_no_query(self):
+ """_record_ccr_feedback_from_response should call store.retrieve when no query."""
+ from headroom.proxy.server import HeadroomProxy
+
+ response = {
+ "content": [
+ {
+ "type": "tool_use",
+ "id": "toolu_002",
+ "name": "headroom_retrieve",
+ "input": {"hash": "feedbackhash2"},
+ },
+ ]
+ }
+
+ mock_store = MagicMock()
+ with patch(
+ "headroom.cache.compression_store.get_compression_store",
+ return_value=mock_store,
+ ):
+ proxy = HeadroomProxy.__new__(HeadroomProxy)
+ proxy.config = MagicMock()
+ proxy.config.ccr_inject_tool = True
+
+ proxy._record_ccr_feedback_from_response(response, "anthropic", "req-test-002")
+
+ mock_store.retrieve.assert_called_once_with("feedbackhash2", query=None)
+
+ def test_record_ccr_feedback_handles_store_exception(self):
+ """_record_ccr_feedback_from_response should not raise on store errors."""
+ from headroom.proxy.server import HeadroomProxy
+
+ response = {
+ "content": [
+ {
+ "type": "tool_use",
+ "id": "toolu_003",
+ "name": "headroom_retrieve",
+ "input": {"hash": "feedbackhash3", "query": "test"},
+ },
+ ]
+ }
+
+ mock_store = MagicMock()
+ mock_store.search.side_effect = RuntimeError("store unavailable")
+ with patch(
+ "headroom.cache.compression_store.get_compression_store",
+ return_value=mock_store,
+ ):
+ proxy = HeadroomProxy.__new__(HeadroomProxy)
+ proxy.config = MagicMock()
+ proxy.config.ccr_inject_tool = True
+
+ # Should not raise
+ proxy._record_ccr_feedback_from_response(response, "anthropic", "req-test-003")
+
+
+class TestParseSSEToolUse:
+ """Bug 3: _parse_sse_to_response correctly handles tool_use blocks."""
+
+ def test_parse_sse_extracts_tool_use(self):
+ """SSE with tool_use content_block should be parsed correctly."""
+ from headroom.proxy.server import HeadroomProxy
+
+ sse_data = (
+ 'data: {"type":"message_start","message":{"id":"msg_01","model":"claude-3-5-sonnet-20241022","role":"assistant","stop_reason":null,"usage":{"input_tokens":100,"output_tokens":0}}}\n'
+ "\n"
+ 'data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}\n'
+ "\n"
+ 'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Let me retrieve that."}}\n'
+ "\n"
+ 'data: {"type":"content_block_stop","index":0}\n'
+ "\n"
+ 'data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_abc","name":"headroom_retrieve"}}\n'
+ "\n"
+ 'data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\\"hash\\": \\"abc123\\", \\"query\\": \\"error\\"}"}}\n'
+ "\n"
+ 'data: {"type":"content_block_stop","index":1}\n'
+ "\n"
+ 'data: {"type":"message_delta","delta":{"stop_reason":"tool_use"},"usage":{"output_tokens":50}}\n'
+ )
+
+ proxy = HeadroomProxy.__new__(HeadroomProxy)
+ result = proxy._parse_sse_to_response(sse_data, "anthropic")
+
+ assert result is not None
+ assert len(result["content"]) == 2
+
+ text_block = result["content"][0]
+ assert text_block["type"] == "text"
+ assert "retrieve" in text_block["text"]
+
+ tool_block = result["content"][1]
+ assert tool_block["type"] == "tool_use"
+ assert tool_block["name"] == "headroom_retrieve"
+ assert tool_block["id"] == "toolu_abc"
+ assert tool_block["input"]["hash"] == "abc123"
+ assert tool_block["input"]["query"] == "error"
+
+ def test_parse_sse_non_anthropic_returns_none(self):
+ """Non-anthropic provider should return None."""
+ from headroom.proxy.server import HeadroomProxy
+
+ proxy = HeadroomProxy.__new__(HeadroomProxy)
+ result = proxy._parse_sse_to_response("data: {}", "openai")
+ assert result is None