Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
refactor(memory): isolate query construction policy (#1950)
JD Davis235c986c9cc7
diff --git a/headroom/proxy/memory_query.py b/headroom/proxy/memory_query.pyindex f052f0a63..9ba58682f 100644--- a/headroom/proxy/memory_query.py+++ b/headroom/proxy/memory_query.py@@ -24,12 +24,10 @@ from __future__ import annotationsfrom dataclasses import dataclassfrom typing import Any-# Section delimiters surfaced in the embedding input so the embedder-# sees structured context rather than a wall of run-on text. Kept short-# so they don't dominate the embedding signal.-_USER_DELIM = "### USER ###\n"-_ASSISTANT_DELIM = "\n### PRIOR_ASSISTANT ###\n"-_TOOL_DELIM = "\n### TOOL_OUTPUT ###\n"+from .memory_query_policy import (+ extract_memory_query_sources,+ render_embedding_input,+)@dataclass(frozen=True)@@ -59,16 +57,11 @@ class MemoryQuery:embedder's positional weighting often emphasizes the tail ofthe input."""- parts: list[str] = []- for asst in self.recent_assistant_turns:- if asst:- parts.append(_ASSISTANT_DELIM + asst)- for tool_out in self.recent_tool_outputs:- if tool_out:- parts.append(_TOOL_DELIM + tool_out)- if self.user_text:- parts.append(_USER_DELIM + self.user_text)- return "".join(parts)+ return render_embedding_input(+ user_text=self.user_text,+ recent_tool_outputs=self.recent_tool_outputs,+ recent_assistant_turns=self.recent_assistant_turns,+ )@classmethoddef from_messages(@@ -91,70 +84,14 @@ class MemoryQuery:Handles both OpenAI shape (``role: tool``) and Anthropic shape(``tool_result`` content block inside a ``role: user`` message)."""- if not messages:- return cls(- user_text="",- recent_tool_outputs=(),- recent_assistant_turns=(),- conversation_id=conversation_id,- )-- latest_user = ""- assistant_turns: list[str] = []- tool_outputs: list[str] = []-- # Walk messages backward so we naturally find the LATEST entries- # first; preserve chronological order in the output by reversing- # the collected lists at the end.- for msg in reversed(messages):- role = msg.get("role")- content = msg.get("content", "")-- if role == "user":- # Distinguish "real user text" from "Anthropic tool_result- # masquerading as a user message". Anthropic uses- # role=user with content=[{type: tool_result, ...}].- if isinstance(content, list):- for block in content:- if isinstance(block, dict) and block.get("type") == "tool_result":- tool_text = block.get("content", "")- if isinstance(tool_text, list):- # Nested content blocks; flatten text fields.- tool_text = "\n".join(- b.get("text", "") for b in tool_text if isinstance(b, dict)- )- if tool_text and len(tool_outputs) < lookback_tools:- tool_outputs.append(str(tool_text))- # Anthropic tool_result is NOT a real user turn —- # don't use it as the user_text source. Continue- # walking back for the actual user message.- elif isinstance(content, str):- if not latest_user:- latest_user = content-- elif role == "assistant":- if isinstance(content, str) and content:- if len(assistant_turns) < lookback_assistant:- assistant_turns.append(content)- elif isinstance(content, list):- text_parts = [- b.get("text", "")- for b in content- if isinstance(b, dict) and b.get("type") == "text"- ]- joined = "\n".join(p for p in text_parts if p)- if joined and len(assistant_turns) < lookback_assistant:- assistant_turns.append(joined)-- elif role == "tool":- # OpenAI shape: tool messages carry the result.- if isinstance(content, str) and content and len(tool_outputs) < lookback_tools:- tool_outputs.append(content)-- # Reverse to restore chronological order (we walked backward).+ user_text, recent_tool_outputs, recent_assistant_turns = extract_memory_query_sources(+ messages,+ lookback_assistant=lookback_assistant,+ lookback_tools=lookback_tools,+ )return cls(- user_text=latest_user,- recent_tool_outputs=tuple(reversed(tool_outputs)),- recent_assistant_turns=tuple(reversed(assistant_turns)),+ user_text=user_text,+ recent_tool_outputs=recent_tool_outputs,+ recent_assistant_turns=recent_assistant_turns,conversation_id=conversation_id,)diff --git a/headroom/proxy/memory_query_policy.py b/headroom/proxy/memory_query_policy.pynew file mode 100644index 000000000..263605ebc--- /dev/null+++ b/headroom/proxy/memory_query_policy.py@@ -0,0 +1,105 @@+"""Pure policy helpers for building memory retrieval queries."""++from __future__ import annotations++from typing import Any++# Section delimiters surfaced in the embedding input so the embedder+# sees structured context rather than a wall of run-on text. Kept short+# so they don't dominate the embedding signal.+USER_DELIM = "### USER ###\n"+ASSISTANT_DELIM = "\n### PRIOR_ASSISTANT ###\n"+TOOL_DELIM = "\n### TOOL_OUTPUT ###\n"+++def render_embedding_input(+ *,+ user_text: str,+ recent_tool_outputs: tuple[str, ...],+ recent_assistant_turns: tuple[str, ...],+) -> str:+ """Concatenate memory query sources into a delimited embedding input."""+ parts: list[str] = []+ for asst in recent_assistant_turns:+ if asst:+ parts.append(ASSISTANT_DELIM + asst)+ for tool_out in recent_tool_outputs:+ if tool_out:+ parts.append(TOOL_DELIM + tool_out)+ if user_text:+ parts.append(USER_DELIM + user_text)+ return "".join(parts)+++def extract_memory_query_sources(+ messages: list[dict[str, Any]] | None,+ *,+ lookback_assistant: int = 2,+ lookback_tools: int = 3,+) -> tuple[str, tuple[str, ...], tuple[str, ...]]:+ """Extract user, tool, and assistant sources from chat-style messages.++ Returns ``(user_text, recent_tool_outputs, recent_assistant_turns)``.+ """+ if not messages:+ return "", (), ()++ latest_user = ""+ assistant_turns: list[str] = []+ tool_outputs: list[str] = []++ for msg in reversed(messages):+ role = msg.get("role")+ content = msg.get("content", "")++ if role == "user":+ if isinstance(content, list):+ _append_anthropic_tool_results(+ content,+ tool_outputs=tool_outputs,+ lookback_tools=lookback_tools,+ )+ elif isinstance(content, str) and not latest_user:+ latest_user = content++ elif role == "assistant":+ assistant_text = _assistant_text(content)+ if assistant_text and len(assistant_turns) < lookback_assistant:+ assistant_turns.append(assistant_text)++ elif role == "tool":+ if isinstance(content, str) and content and len(tool_outputs) < lookback_tools:+ tool_outputs.append(content)++ return (+ latest_user,+ tuple(reversed(tool_outputs)),+ tuple(reversed(assistant_turns)),+ )+++def _append_anthropic_tool_results(+ content: list[Any],+ *,+ tool_outputs: list[str],+ lookback_tools: int,+) -> None:+ for block in content:+ if not isinstance(block, dict) or block.get("type") != "tool_result":+ continue+ tool_text = block.get("content", "")+ if isinstance(tool_text, list):+ tool_text = "\n".join(b.get("text", "") for b in tool_text if isinstance(b, dict))+ if tool_text and len(tool_outputs) < lookback_tools:+ tool_outputs.append(str(tool_text))+++def _assistant_text(content: Any) -> str:+ if isinstance(content, str):+ return content+ if isinstance(content, list):+ text_parts = [+ b.get("text", "") for b in content if isinstance(b, dict) and b.get("type") == "text"+ ]+ return "\n".join(p for p in text_parts if p)+ return ""diff --git a/tests/test_memory_query_policy.py b/tests/test_memory_query_policy.pynew file mode 100644index 000000000..ea88617d7--- /dev/null+++ b/tests/test_memory_query_policy.py@@ -0,0 +1,56 @@+"""Tests for pure memory query construction policy."""++from __future__ import annotations++from headroom.proxy.memory_query_policy import (+ extract_memory_query_sources,+ render_embedding_input,+)+++def test_render_embedding_input_orders_sources_for_embedding() -> None:+ rendered = render_embedding_input(+ user_text="latest user",+ recent_tool_outputs=("tool output",),+ recent_assistant_turns=("assistant context",),+ )++ assert rendered.index("assistant context") < rendered.index("tool output")+ assert rendered.index("tool output") < rendered.index("latest user")+++def test_extract_sources_uses_latest_user_and_recent_context_in_order() -> None:+ messages = [+ {"role": "user", "content": "first"},+ {"role": "assistant", "content": "a1"},+ {"role": "tool", "content": "t1"},+ {"role": "assistant", "content": "a2"},+ {"role": "tool", "content": "t2"},+ {"role": "user", "content": "second"},+ ]++ user_text, tool_outputs, assistant_turns = extract_memory_query_sources(+ messages,+ lookback_assistant=2,+ lookback_tools=2,+ )++ assert user_text == "second"+ assert tool_outputs == ("t1", "t2")+ assert assistant_turns == ("a1", "a2")+++def test_extract_sources_handles_anthropic_tool_result_without_user_text() -> None:+ messages = [+ {"role": "user", "content": "real user"},+ {+ "role": "user",+ "content": [{"type": "tool_result", "content": [{"type": "text", "text": "nested"}]}],+ },+ ]++ user_text, tool_outputs, assistant_turns = extract_memory_query_sources(messages)++ assert user_text == "real user"+ assert tool_outputs == ("nested",)+ assert assistant_turns == ()