Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

refactor(memory): isolate query construction policy (#1950)

JD Davis235c986c9cc7
diff --git a/headroom/proxy/memory_query.py b/headroom/proxy/memory_query.py
index f052f0a63..9ba58682f 100644
--- a/headroom/proxy/memory_query.py
+++ b/headroom/proxy/memory_query.py
@@ -24,12 +24,10 @@ from __future__ import annotations
from dataclasses import dataclass
from typing import Any
-# Section delimiters surfaced in the embedding input so the embedder
-# sees structured context rather than a wall of run-on text. Kept short
-# so they don't dominate the embedding signal.
-_USER_DELIM = "### USER ###\n"
-_ASSISTANT_DELIM = "\n### PRIOR_ASSISTANT ###\n"
-_TOOL_DELIM = "\n### TOOL_OUTPUT ###\n"
+from .memory_query_policy import (
+ extract_memory_query_sources,
+ render_embedding_input,
+)
@dataclass(frozen=True)
@@ -59,16 +57,11 @@ class MemoryQuery:
embedder's positional weighting often emphasizes the tail of
the input.
"""
- parts: list[str] = []
- for asst in self.recent_assistant_turns:
- if asst:
- parts.append(_ASSISTANT_DELIM + asst)
- for tool_out in self.recent_tool_outputs:
- if tool_out:
- parts.append(_TOOL_DELIM + tool_out)
- if self.user_text:
- parts.append(_USER_DELIM + self.user_text)
- return "".join(parts)
+ return render_embedding_input(
+ user_text=self.user_text,
+ recent_tool_outputs=self.recent_tool_outputs,
+ recent_assistant_turns=self.recent_assistant_turns,
+ )
@classmethod
def from_messages(
@@ -91,70 +84,14 @@ class MemoryQuery:
Handles both OpenAI shape (``role: tool``) and Anthropic shape
(``tool_result`` content block inside a ``role: user`` message).
"""
- if not messages:
- return cls(
- user_text="",
- recent_tool_outputs=(),
- recent_assistant_turns=(),
- conversation_id=conversation_id,
- )
-
- latest_user = ""
- assistant_turns: list[str] = []
- tool_outputs: list[str] = []
-
- # Walk messages backward so we naturally find the LATEST entries
- # first; preserve chronological order in the output by reversing
- # the collected lists at the end.
- for msg in reversed(messages):
- role = msg.get("role")
- content = msg.get("content", "")
-
- if role == "user":
- # Distinguish "real user text" from "Anthropic tool_result
- # masquerading as a user message". Anthropic uses
- # role=user with content=[{type: tool_result, ...}].
- if isinstance(content, list):
- for block in content:
- if isinstance(block, dict) and block.get("type") == "tool_result":
- tool_text = block.get("content", "")
- if isinstance(tool_text, list):
- # Nested content blocks; flatten text fields.
- tool_text = "\n".join(
- b.get("text", "") for b in tool_text if isinstance(b, dict)
- )
- if tool_text and len(tool_outputs) < lookback_tools:
- tool_outputs.append(str(tool_text))
- # Anthropic tool_result is NOT a real user turn —
- # don't use it as the user_text source. Continue
- # walking back for the actual user message.
- elif isinstance(content, str):
- if not latest_user:
- latest_user = content
-
- elif role == "assistant":
- if isinstance(content, str) and content:
- if len(assistant_turns) < lookback_assistant:
- assistant_turns.append(content)
- elif isinstance(content, list):
- text_parts = [
- b.get("text", "")
- for b in content
- if isinstance(b, dict) and b.get("type") == "text"
- ]
- joined = "\n".join(p for p in text_parts if p)
- if joined and len(assistant_turns) < lookback_assistant:
- assistant_turns.append(joined)
-
- elif role == "tool":
- # OpenAI shape: tool messages carry the result.
- if isinstance(content, str) and content and len(tool_outputs) < lookback_tools:
- tool_outputs.append(content)
-
- # Reverse to restore chronological order (we walked backward).
+ user_text, recent_tool_outputs, recent_assistant_turns = extract_memory_query_sources(
+ messages,
+ lookback_assistant=lookback_assistant,
+ lookback_tools=lookback_tools,
+ )
return cls(
- user_text=latest_user,
- recent_tool_outputs=tuple(reversed(tool_outputs)),
- recent_assistant_turns=tuple(reversed(assistant_turns)),
+ user_text=user_text,
+ recent_tool_outputs=recent_tool_outputs,
+ recent_assistant_turns=recent_assistant_turns,
conversation_id=conversation_id,
)
diff --git a/headroom/proxy/memory_query_policy.py b/headroom/proxy/memory_query_policy.py
new file mode 100644
index 000000000..263605ebc
--- /dev/null
+++ b/headroom/proxy/memory_query_policy.py
@@ -0,0 +1,105 @@
+"""Pure policy helpers for building memory retrieval queries."""
+
+from __future__ import annotations
+
+from typing import Any
+
+# Section delimiters surfaced in the embedding input so the embedder
+# sees structured context rather than a wall of run-on text. Kept short
+# so they don't dominate the embedding signal.
+USER_DELIM = "### USER ###\n"
+ASSISTANT_DELIM = "\n### PRIOR_ASSISTANT ###\n"
+TOOL_DELIM = "\n### TOOL_OUTPUT ###\n"
+
+
+def render_embedding_input(
+ *,
+ user_text: str,
+ recent_tool_outputs: tuple[str, ...],
+ recent_assistant_turns: tuple[str, ...],
+) -> str:
+ """Concatenate memory query sources into a delimited embedding input."""
+ parts: list[str] = []
+ for asst in recent_assistant_turns:
+ if asst:
+ parts.append(ASSISTANT_DELIM + asst)
+ for tool_out in recent_tool_outputs:
+ if tool_out:
+ parts.append(TOOL_DELIM + tool_out)
+ if user_text:
+ parts.append(USER_DELIM + user_text)
+ return "".join(parts)
+
+
+def extract_memory_query_sources(
+ messages: list[dict[str, Any]] | None,
+ *,
+ lookback_assistant: int = 2,
+ lookback_tools: int = 3,
+) -> tuple[str, tuple[str, ...], tuple[str, ...]]:
+ """Extract user, tool, and assistant sources from chat-style messages.
+
+ Returns ``(user_text, recent_tool_outputs, recent_assistant_turns)``.
+ """
+ if not messages:
+ return "", (), ()
+
+ latest_user = ""
+ assistant_turns: list[str] = []
+ tool_outputs: list[str] = []
+
+ for msg in reversed(messages):
+ role = msg.get("role")
+ content = msg.get("content", "")
+
+ if role == "user":
+ if isinstance(content, list):
+ _append_anthropic_tool_results(
+ content,
+ tool_outputs=tool_outputs,
+ lookback_tools=lookback_tools,
+ )
+ elif isinstance(content, str) and not latest_user:
+ latest_user = content
+
+ elif role == "assistant":
+ assistant_text = _assistant_text(content)
+ if assistant_text and len(assistant_turns) < lookback_assistant:
+ assistant_turns.append(assistant_text)
+
+ elif role == "tool":
+ if isinstance(content, str) and content and len(tool_outputs) < lookback_tools:
+ tool_outputs.append(content)
+
+ return (
+ latest_user,
+ tuple(reversed(tool_outputs)),
+ tuple(reversed(assistant_turns)),
+ )
+
+
+def _append_anthropic_tool_results(
+ content: list[Any],
+ *,
+ tool_outputs: list[str],
+ lookback_tools: int,
+) -> None:
+ for block in content:
+ if not isinstance(block, dict) or block.get("type") != "tool_result":
+ continue
+ tool_text = block.get("content", "")
+ if isinstance(tool_text, list):
+ tool_text = "\n".join(b.get("text", "") for b in tool_text if isinstance(b, dict))
+ if tool_text and len(tool_outputs) < lookback_tools:
+ tool_outputs.append(str(tool_text))
+
+
+def _assistant_text(content: Any) -> str:
+ if isinstance(content, str):
+ return content
+ if isinstance(content, list):
+ text_parts = [
+ b.get("text", "") for b in content if isinstance(b, dict) and b.get("type") == "text"
+ ]
+ return "\n".join(p for p in text_parts if p)
+ return ""
diff --git a/tests/test_memory_query_policy.py b/tests/test_memory_query_policy.py
new file mode 100644
index 000000000..ea88617d7
--- /dev/null
+++ b/tests/test_memory_query_policy.py
@@ -0,0 +1,56 @@
+"""Tests for pure memory query construction policy."""
+
+from __future__ import annotations
+
+from headroom.proxy.memory_query_policy import (
+ extract_memory_query_sources,
+ render_embedding_input,
+)
+
+
+def test_render_embedding_input_orders_sources_for_embedding() -> None:
+ rendered = render_embedding_input(
+ user_text="latest user",
+ recent_tool_outputs=("tool output",),
+ recent_assistant_turns=("assistant context",),
+ )
+
+ assert rendered.index("assistant context") < rendered.index("tool output")
+ assert rendered.index("tool output") < rendered.index("latest user")
+
+
+def test_extract_sources_uses_latest_user_and_recent_context_in_order() -> None:
+ messages = [
+ {"role": "user", "content": "first"},
+ {"role": "assistant", "content": "a1"},
+ {"role": "tool", "content": "t1"},
+ {"role": "assistant", "content": "a2"},
+ {"role": "tool", "content": "t2"},
+ {"role": "user", "content": "second"},
+ ]
+
+ user_text, tool_outputs, assistant_turns = extract_memory_query_sources(
+ messages,
+ lookback_assistant=2,
+ lookback_tools=2,
+ )
+
+ assert user_text == "second"
+ assert tool_outputs == ("t1", "t2")
+ assert assistant_turns == ("a1", "a2")
+
+
+def test_extract_sources_handles_anthropic_tool_result_without_user_text() -> None:
+ messages = [
+ {"role": "user", "content": "real user"},
+ {
+ "role": "user",
+ "content": [{"type": "tool_result", "content": [{"type": "text", "text": "nested"}]}],
+ },
+ ]
+
+ user_text, tool_outputs, assistant_turns = extract_memory_query_sources(messages)
+
+ assert user_text == "real user"
+ assert tool_outputs == ("nested",)
+ assert assistant_turns == ()