Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
chore(transforms): retire query_echo (already disabled in production)
chopratejaseac5204b9d34
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.pyindex e50da5c65..f1636da07 100644--- a/headroom/proxy/handlers/anthropic.py+++ b/headroom/proxy/handlers/anthropic.py@@ -1198,10 +1198,6 @@ class AnthropicHandlerMixin:if remembered_event.headers is not None:headers = remembered_event.headers- # Query Echo: disabled — hurts prefix caching in long conversations.- # The echo changes every turn, invalidating the cached prefix.- # To re-enable, uncomment and set query_echo_enabled on ProxyConfig.-# Update bodybody["messages"] = optimized_messagesif tools is not None:diff --git a/headroom/proxy/handlers/gemini.py b/headroom/proxy/handlers/gemini.pyindex 4aa246ba9..b95594489 100644--- a/headroom/proxy/handlers/gemini.py+++ b/headroom/proxy/handlers/gemini.py@@ -308,8 +308,6 @@ class GeminiHandlerMixin:tokens_saved = original_tokens - optimized_tokensoptimization_latency = (time.time() - start_time) * 1000- # Query Echo: disabled — hurts prefix caching in long conversations.-# Memory: inject context for Gemini requestsif self.memory_handler and memory_user_id:try:diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex 03496c95d..5599d09d4 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -513,7 +513,6 @@ class OpenAIHandlerMixin:f"[{request_id}] CCR: Tool already present (MCP?), skipped injection for hashes: {injector.detected_hashes}")- # Query Echo: disabled — hurts prefix caching in long conversations.if is_cache_mode(self.config.mode):optimized_messages, restored_count = self._restore_frozen_prefix(original_client_messages,diff --git a/headroom/transforms/query_echo.py b/headroom/transforms/query_echo.pydeleted file mode 100644index 599eb1d40..000000000--- a/headroom/transforms/query_echo.py+++ /dev/null@@ -1,123 +0,0 @@-"""Query Echo — re-injects the user's question after compressed tool outputs.--After Headroom compresses tool outputs, the user's original question may be-thousands of tokens away from where the LLM generates its answer. Attention-to the question decays with distance (the "lost in the middle" problem).--Query Echo solves this by appending a brief reminder of the user's question-after the last compressed content block. This:--1. Improves answer quality from compressed data (fresh attention on the question)-2. Makes CCR retrieval triggers more accurate (LLM is more aware of what's missing)-3. Feeds cleaner signals to TOIN (more accurate retrievals → better learning)--The echo is:-- Cache-safe: always placed after the cache boundary (in the "new tokens" region)-- Compression-ratio-proportional: only triggers when compression was >30%-- Provider-agnostic: appends to content strings, works for all providers-- Cheap: ~50 tokens overhead, vs thousands of tokens saved by compression-"""--from __future__ import annotations--import logging--logger = logging.getLogger(__name__)---def extract_user_query(messages: list[dict]) -> str:- """Extract the last substantive user question from messages.-- Scans backwards through messages to find the last user message- that contains a real question (not just "yes", "continue", etc.).-- Works for both Anthropic and OpenAI message formats.- """- for msg in reversed(messages):- if msg.get("role") != "user":- continue-- content = msg.get("content", "")-- # Anthropic: content can be a list of blocks- if isinstance(content, list):- for block in content:- if isinstance(block, dict) and block.get("type") == "text":- text = str(block.get("text", "")).strip()- if len(text) > 10: # Skip trivial messages- return text- continue-- # OpenAI/generic: content is a string- if isinstance(content, str):- text = content.strip()- if len(text) > 10:- return text-- return ""---def inject_query_echo(- messages: list[dict],- query: str,- tokens_saved: int,- original_tokens: int,- min_compression_ratio: float = 0.3,- max_query_chars: int = 200,-) -> bool:- """Inject a query reminder after the last compressed tool output.-- Args:- messages: The optimized messages list (will be modified in-place).- query: The user's question to echo.- tokens_saved: Tokens saved by compression.- original_tokens: Original token count before compression.- min_compression_ratio: Minimum compression ratio to trigger echo (0.0–1.0).- max_query_chars: Maximum characters of query to include in echo.-- Returns:- True if echo was injected, False if skipped.- """- if not query or original_tokens == 0:- return False-- compression_ratio = tokens_saved / original_tokens- if compression_ratio < min_compression_ratio:- return False-- # Truncate and sanitize query- safe_query = query[:max_query_chars].replace("\n", " ").replace('"', "'").strip()- if not safe_query:- return False-- echo = f"\n\n[Recall: User asked: '{safe_query}']"-- # Walk backwards through messages, find last compressed content- for msg in reversed(messages):- content = msg.get("content", "")-- # String content (OpenAI tool messages, Anthropic string content)- if isinstance(content, str) and "compressed" in content:- msg["content"] = content + echo- logger.debug(- "Query echo injected (ratio=%.1f%%, query=%s...)",- compression_ratio * 100,- safe_query[:50],- )- return True-- # List of content blocks (Anthropic format)- if isinstance(content, list):- for block in reversed(content):- if not isinstance(block, dict):- continue- block_content = block.get("content", "")- if isinstance(block_content, str) and "compressed" in block_content:- block["content"] = block_content + echo- logger.debug(- "Query echo injected in content block (ratio=%.1f%%)",- compression_ratio * 100,- )- return True-- return Falsediff --git a/tests/test_query_echo.py b/tests/test_query_echo.pydeleted file mode 100644index 9eea8d147..000000000--- a/tests/test_query_echo.py+++ /dev/null@@ -1,301 +0,0 @@-"""Tests for Query Echo — unit tests + integration eval with real API.--Unit tests: verify echo injection logic across message formats.-Integration: verify echo actually improves LLM answers on compressed data.-"""--from __future__ import annotations--import json-import os--import pytest--from headroom.transforms.query_echo import extract_user_query, inject_query_echo-from tests._dotenv import autouse_apply_env, load_env_overrides--_env_overrides = load_env_overrides()-ANTHROPIC_KEY = os.environ.get("ANTHROPIC_API_KEY") or _env_overrides.get("ANTHROPIC_API_KEY", "")-apply_dotenv = autouse_apply_env(_env_overrides)---# =============================================================================-# Unit Tests: extract_user_query-# =============================================================================---class TestExtractUserQuery:- def test_simple_string_message(self):- messages = [- {"role": "user", "content": "What are the test failures?"},- {"role": "assistant", "content": "Let me check."},- ]- assert extract_user_query(messages) == "What are the test failures?"-- def test_anthropic_content_blocks(self):- messages = [- {- "role": "user",- "content": [- {"type": "text", "text": "Find the auth bug in these results"},- {"type": "tool_result", "tool_use_id": "x", "content": "data..."},- ],- },- ]- assert extract_user_query(messages) == "Find the auth bug in these results"-- def test_skips_trivial_messages(self):- messages = [- {"role": "user", "content": "What are the top 3 errors in this log output?"},- {"role": "assistant", "content": "Looking..."},- {"role": "user", "content": "yes"},- ]- # "yes" is too short (<10 chars), should get the earlier question- assert "errors" in extract_user_query(messages)-- def test_empty_messages(self):- assert extract_user_query([]) == ""-- def test_no_user_messages(self):- messages = [{"role": "assistant", "content": "Hello"}]- assert extract_user_query(messages) == ""-- def test_multi_turn_gets_last_substantive(self):- messages = [- {"role": "user", "content": "Tell me about the weather"},- {"role": "assistant", "content": "It's sunny."},- {"role": "user", "content": "What about the authentication failures in CI?"},- {"role": "assistant", "content": "Let me check."},- ]- assert "authentication" in extract_user_query(messages)---# =============================================================================-# Unit Tests: inject_query_echo-# =============================================================================---class TestInjectQueryEcho:- def test_injects_after_compressed_content(self):- messages = [- {"role": "user", "content": "What are the failures?"},- {- "role": "tool",- "content": '[{"status":"pass"}]\n[50 items compressed to 5. Retrieve more: hash=abc123]',- },- ]- result = inject_query_echo(- messages, "What are the failures?", tokens_saved=500, original_tokens=1000- )- assert result is True- assert "[Recall:" in messages[-1]["content"]- assert "failures" in messages[-1]["content"]-- def test_skips_when_no_compression(self):- messages = [- {"role": "user", "content": "hello"},- {"role": "tool", "content": "some data"},- ]- result = inject_query_echo(messages, "hello", tokens_saved=0, original_tokens=100)- assert result is False-- def test_skips_when_low_compression(self):- messages = [- {"role": "tool", "content": "data\n[10 items compressed to 8]"},- ]- result = inject_query_echo(messages, "query", tokens_saved=20, original_tokens=100)- # 20% compression — below 30% threshold- assert result is False-- def test_triggers_at_high_compression(self):- messages = [- {- "role": "tool",- "content": "data\n[500 items compressed to 20. Retrieve more: hash=abc]",- },- ]- result = inject_query_echo(- messages, "What are the errors?", tokens_saved=800, original_tokens=1000- )- assert result is True- assert "[Recall:" in messages[0]["content"]-- def test_anthropic_content_blocks(self):- messages = [- {- "role": "user",- "content": [- {- "type": "tool_result",- "tool_use_id": "1",- "content": '[{"x":1}]\n[100 items compressed to 10. hash=abc123]',- },- {- "type": "tool_result",- "tool_use_id": "2",- "content": '[{"y":2}]\n[200 items compressed to 15. hash=def456]',- },- ],- },- ]- result = inject_query_echo(- messages, "Find the errors", tokens_saved=600, original_tokens=1000- )- assert result is True- # Echo should be on the LAST compressed block- last_block = messages[0]["content"][-1]- assert "[Recall:" in last_block["content"]-- def test_truncates_long_query(self):- long_query = "x" * 500- messages = [- {"role": "tool", "content": "data\n[100 items compressed to 5. hash=abc]"},- ]- inject_query_echo(messages, long_query, tokens_saved=500, original_tokens=1000)- # Should be truncated to 200 chars- assert len(messages[0]["content"]) < 800-- def test_sanitizes_quotes(self):- messages = [- {"role": "tool", "content": "data\n[100 items compressed to 5. hash=abc]"},- ]- inject_query_echo(- messages, 'Find "auth_error" in logs', tokens_saved=500, original_tokens=1000- )- # Double quotes replaced with single quotes- assert '"auth_error"' not in messages[0]["content"]- assert "'auth_error'" in messages[0]["content"]-- def test_no_echo_without_compressed_marker(self):- """If no message contains 'compressed', no echo injected."""- messages = [- {"role": "tool", "content": "just plain tool output without compression"},- ]- result = inject_query_echo(messages, "query", tokens_saved=500, original_tokens=1000)- assert result is False-- def test_empty_query_skipped(self):- messages = [- {"role": "tool", "content": "data\n[100 items compressed]"},- ]- result = inject_query_echo(messages, "", tokens_saved=500, original_tokens=1000)- assert result is False---# =============================================================================-# Integration Eval: Does echo improve answer quality?-# =============================================================================---def _call_claude(messages, max_tokens=200):- import httpx-- resp = httpx.post(- "https://api.anthropic.com/v1/messages",- headers={- "X-Api-Key": ANTHROPIC_KEY,- "anthropic-version": "2023-06-01",- "Content-Type": "application/json",- },- json={- "model": "claude-sonnet-4-5-20250929",- "max_tokens": max_tokens,- "messages": messages,- },- timeout=30,- )- return resp.json()---def _make_compressed_test_data():- """Create realistic compressed test results where answer is in omitted data."""- # Visible: 10 passing tests- visible = [- {"test": f"test_module_{i}", "status": "passed", "duration_ms": 50 + i} for i in range(10)- ]- compressed_output = json.dumps(visible, indent=2)-- # The important info is in the COMPRESSED summary- summary = "87 passed, 2 failed, 1 error; notable: test_auth (fail); test_db (timeout)"- compressed_output += (- f"\n[90 items compressed to 10. Omitted: {summary}."- f" Retrieve more: hash=abc123def456789012345678."- f" Expires in 5m.]"- )- return compressed_output--[email protected](not ANTHROPIC_KEY, reason="ANTHROPIC_API_KEY not set")-class TestQueryEchoIntegration:- def test_with_echo_finds_failures(self):- """WITH echo → LLM should identify failures from compressed data."""- compressed = _make_compressed_test_data()- question = "What are the test failures and what went wrong?"-- # Add echo at the end- echoed_content = compressed + f"\n\n[Recall: User asked: '{question}']"-- messages = [- {- "role": "user",- "content": f"Here are the CI test results:\n\n{echoed_content}\n\nAnswer my question.",- },- ]- resp = _call_claude(messages)- text = resp.get("content", [{}])[0].get("text", "").lower()-- print(f"\n WITH echo response: {text[:200]}")-- has_failure_info = any(w in text for w in ["fail", "error", "timeout", "auth", "db"])- assert has_failure_info, f"LLM missed failures with echo: {text[:300]}"-- def test_without_echo_baseline(self):- """WITHOUT echo → baseline for comparison."""- compressed = _make_compressed_test_data()- question = "What are the test failures and what went wrong?"-- messages = [- {- "role": "user",- "content": f"Here are the CI test results:\n\n{compressed}\n\n{question}",- },- ]- resp = _call_claude(messages)- text = resp.get("content", [{}])[0].get("text", "").lower()-- print(f"\n WITHOUT echo response: {text[:200]}")-- # Not asserting — this is the baseline. May or may not find failures.- has_failure_info = any(w in text for w in ["fail", "error", "timeout"])- print(f" Found failure info without echo: {has_failure_info}")-- def test_echo_with_specific_lookup(self):- """Echo helps LLM find specific data in compressed results."""- # Compressed config data — visible items are all 'production'- visible = [- {"env": "production", "key": "DB_URL", "value": "postgres://prod:5432/app"},- {"env": "production", "key": "REDIS_URL", "value": "redis://prod:6379"},- {"env": "production", "key": "API_KEY", "value": "sk-prod-xxx"},- ]- compressed = json.dumps(visible, indent=2)- compressed += (- "\n[45 items compressed to 3. Omitted: 30 staging, 12 development."- " Retrieve more: hash=cfg123def456789012345678."- " Expires in 5m.]"- )-- question = "What is the staging database URL?"- echoed = compressed + f"\n\n[Recall: User asked: '{question}']"-- messages = [- {"role": "user", "content": f"Application configs:\n\n{echoed}\n\nAnswer precisely."},- ]- resp = _call_claude(messages)- text = resp.get("content", [{}])[0].get("text", "").lower()-- print(f"\n Staging lookup with echo: {text[:200]}")-- # With echo + summary mentioning "30 staging", LLM should know- # staging data exists in compressed items- knows_staging = "staging" in text or "compressed" in text or "retrieve" in text- assert knows_staging, f"LLM didn't reference staging data: {text[:300]}"