Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
Merge remote-tracking branch 'origin/main' into refactor/memory-ranker-and-image-decision
chopratejas7a0f80d0f45a
diff --cc headroom/proxy/memory_handler.pyindex 10386fd19,07f9e6f13..fb977636f--- a/headroom/proxy/memory_handler.py+++ b/headroom/proxy/memory_handler.py@@@ -623,7 -623,8 +623,9 @@@ class MemoryHandlermessages: list[dict[str, Any]],request_context: RequestContext | None = None,*,+ ranker: Any | None = None,+ query: Any | None = None,+ budget: Any | None = None,) -> str | None:"""Search memories and format as context injection.@@@ -634,18 -636,18 +637,30 @@@request_context: Optional request envelope (headers, systemprompt, base user id). When provided, memory retrievalis scoped to the resolved workspace / project so memories- from unrelated projects can never bleed in (GH #462).+ from unrelated projects can never bleed in (GH #462). When+ omitted, behaves as before this fix — single-bucket search+ against the legacy backend. Production handlers always+ pass it; tests / mocks can keep the simpler call shape.+ ranker: Optional :class:`~headroom.proxy.memory_ranker.MemoryRanker`+ — re-ranks the backend's cosine-only candidates by an+ additional signal (recency, source, access count, …).- When ``None`` (default), behaviour is identical to pre-- this-PR: pure cosine + min_similarity floor. When- provided, candidates are adapted to- :class:`MemoryCandidate`, re-ranked, then re-filtered by- ``min_similarity`` on the boosted score.++ When ``None`` (default), behaviour is pure cosine +++ ``budget.min_similarity`` floor. When provided, candidates++ are adapted to :class:`MemoryCandidate`, re-ranked, then++ re-filtered by ``budget.min_similarity`` on the boosted++ score.+ query: Optional :class:`MemoryQuery` — multi-source, full-+ fidelity retrieval query. When provided, takes precedence+ over the ``messages``-derived query. Constructed at the+ handler from latest user msg + recent tool outputs ++ recent assistant turns; preserves full input fidelity (no+ 500-char truncation).+ budget: Optional :class:`MemoryInjectionBudget` — bounds the+ returned formatted block by tokens / entries / min+ similarity. When ``None``, defaults are taken from+ ``self.config`` so the existing top_k / min_similarity- contract is preserved.++ contract is preserved. Both the no-ranker and the with-++ ranker paths honour the same budget.Returns:Formatted context string, or None if no relevant memories.@@@ -699,48 -717,28 +730,58 @@@)return None- # Filter by minimum similarity using the budget.- filtered_results = [r for r in results if r.score >= effective_budget.min_similarity]-- if not filtered_results:- logger.debug(- f"Memory: {len(results)} memories found but none above threshold "- f"{effective_budget.min_similarity}"- )- return None-- # Cap entry count via the budget (defence-in-depth — backend- # already gets top_k=max_entries but this enforces it on- # post-filter results too).- filtered_results = filtered_results[: effective_budget.max_entries]-- # Format as context.- memory_lines = []- for i, result in enumerate(filtered_results, 1):- memory_lines.append(f"{i}. {result.memory.content}")- if hasattr(result, "related_entities") and result.related_entities:- entities_str = ", ".join(result.related_entities[:3])- memory_lines.append(f" (Related: {entities_str})")+ # Optional re-rank: when a MemoryRanker is provided, adapt+ # results to MemoryCandidate, re-rank, then filter by- # min_similarity on the BOOSTED score. The re-rank can- # promote a fresh weak-cosine memory above a stale strong-- # cosine one (RecencyBoostRanker default behaviour).++ # ``budget.min_similarity`` on the BOOSTED score. The re-rank++ # can promote a fresh weak-cosine memory above a stale++ # strong-cosine one (RecencyBoostRanker default behaviour).++ # Cap by ``budget.max_entries`` after filtering so the budget++ # contract is honoured on both branches.+ if ranker is not None:+ from headroom.proxy.memory_ranker import MemoryCandidate++ candidates = [MemoryCandidate.from_backend_result(r) for r in results]+ ranked = ranker.rank(candidates)+ # Filter on the post-rank score (the ranker may have+ # boosted or attenuated original cosine values).- ranked = [c for c in ranked if c.score >= self.config.min_similarity]++ ranked = [c for c in ranked if c.score >= effective_budget.min_similarity]+ if not ranked:+ logger.debug(+ f"Memory: {len(results)} memories found but none above threshold "- f"{self.config.min_similarity} after re-rank"++ f"{effective_budget.min_similarity} after re-rank"+ )+ return None++ ranked = ranked[: effective_budget.max_entries]+ memory_lines = []+ for i, candidate in enumerate(ranked, 1):+ memory_lines.append(f"{i}. {candidate.content}")+ if candidate.related_entities:+ entities_str = ", ".join(candidate.related_entities[:3])+ memory_lines.append(f" (Related: {entities_str})")+ else:- # Pre-PR-this behaviour: pure cosine, no boost.- filtered_results = [r for r in results if r.score >= self.config.min_similarity]++ # No ranker: pure cosine + budget min_similarity floor.++ filtered_results = [++ r for r in results if r.score >= effective_budget.min_similarity++ ]++ if not filtered_results:+ logger.debug(+ f"Memory: {len(results)} memories found but none above threshold "- f"{self.config.min_similarity}"++ f"{effective_budget.min_similarity}"+ )+ return None+++ # Cap entry count via the budget (defence-in-depth —++ # backend already gets top_k=max_entries but this enforces++ # it on post-filter results too).++ filtered_results = filtered_results[: effective_budget.max_entries]+++ memory_lines = []+ for i, result in enumerate(filtered_results, 1):+ memory_lines.append(f"{i}. {result.memory.content}")+ if hasattr(result, "related_entities") and result.related_entities:+ entities_str = ", ".join(result.related_entities[:3])+ memory_lines.append(f" (Related: {entities_str})")except Exception as e:logger.warning(f"Memory: Search failed for user {effective_user_id}: {e}")