Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
Merge remote-tracking branch 'origin/main' into refactor/memory-tool-handles
chopratejas002ed97c541f
diff --cc headroom/proxy/memory_handler.pyindex 7e43661a3,fb977636f..c3392890c--- a/headroom/proxy/memory_handler.py+++ b/headroom/proxy/memory_handler.py@@@ -743,35 -730,58 +756,66 @@@ class MemoryHandler)return None- # Filter by minimum similarity using the budget.- filtered_results = [r for r in results if r.score >= effective_budget.min_similarity]-- if not filtered_results:- logger.debug(- f"Memory: {len(results)} memories found but none above threshold "- f"{effective_budget.min_similarity}"- )- return None-- # Cap entry count via the budget (defence-in-depth — backend- # already gets top_k=max_entries but this enforces it on- # post-filter results too).- filtered_results = filtered_results[: effective_budget.max_entries]-- # Format as context. Each row prefixes the memory ID so the- # model can address it directly (e.g.,- # ``memory_update(id, ...)``) without first calling- # ``memory_search`` to discover IDs. Pre-this-PR the block- # only carried content; the model had to round-trip through- # search to do any UPDATE / DELETE on a row visible in the- # auto-injected tail.- memory_lines = []- for i, result in enumerate(filtered_results, 1):- memory_id = getattr(result.memory, "id", None) or "?"- memory_lines.append(f"{i}. [{memory_id}] {result.memory.content}")- if hasattr(result, "related_entities") and result.related_entities:- entities_str = ", ".join(result.related_entities[:3])- memory_lines.append(f" (Related: {entities_str})")+ # Optional re-rank: when a MemoryRanker is provided, adapt+ # results to MemoryCandidate, re-rank, then filter by+ # ``budget.min_similarity`` on the BOOSTED score. The re-rank+ # can promote a fresh weak-cosine memory above a stale+ # strong-cosine one (RecencyBoostRanker default behaviour).+ # Cap by ``budget.max_entries`` after filtering so the budget+ # contract is honoured on both branches.++ # Each rendered row carries the memory ID in [brackets] so++ # the model can address it directly via memory_update /++ # memory_delete without round-tripping through memory_search.++ # Both branches below render the same `i. [id] content` shape++ # so the format is stable regardless of whether a ranker is++ # in play.+ if ranker is not None:+ from headroom.proxy.memory_ranker import MemoryCandidate++ candidates = [MemoryCandidate.from_backend_result(r) for r in results]+ ranked = ranker.rank(candidates)+ # Filter on the post-rank score (the ranker may have+ # boosted or attenuated original cosine values).+ ranked = [c for c in ranked if c.score >= effective_budget.min_similarity]+ if not ranked:+ logger.debug(+ f"Memory: {len(results)} memories found but none above threshold "+ f"{effective_budget.min_similarity} after re-rank"+ )+ return None+ ranked = ranked[: effective_budget.max_entries]+ memory_lines = []+ for i, candidate in enumerate(ranked, 1):- memory_lines.append(f"{i}. {candidate.content}")++ memory_id = candidate.id or "?"++ memory_lines.append(f"{i}. [{memory_id}] {candidate.content}")+ if candidate.related_entities:+ entities_str = ", ".join(candidate.related_entities[:3])+ memory_lines.append(f" (Related: {entities_str})")+ else:+ # No ranker: pure cosine + budget min_similarity floor.+ filtered_results = [+ r for r in results if r.score >= effective_budget.min_similarity+ ]++ if not filtered_results:+ logger.debug(+ f"Memory: {len(results)} memories found but none above threshold "+ f"{effective_budget.min_similarity}"+ )+ return None++ # Cap entry count via the budget (defence-in-depth —+ # backend already gets top_k=max_entries but this enforces+ # it on post-filter results too).+ filtered_results = filtered_results[: effective_budget.max_entries]++ memory_lines = []+ for i, result in enumerate(filtered_results, 1):- memory_lines.append(f"{i}. {result.memory.content}")++ memory_id = getattr(result.memory, "id", None) or "?"++ memory_lines.append(f"{i}. [{memory_id}] {result.memory.content}")+ if hasattr(result, "related_entities") and result.related_entities:+ entities_str = ", ".join(result.related_entities[:3])+ memory_lines.append(f" (Related: {entities_str})")except Exception as e:logger.warning(f"Memory: Search failed for user {effective_user_id}: {e}")diff --cc headroom/proxy/memory_ranker.pyindex 000000000,ef5aea09f..09e37b9a7mode 000000,100644..100644--- a/headroom/proxy/memory_ranker.py+++ b/headroom/proxy/memory_ranker.py@@@ -1,0 -1,198 +1,206 @@@+ """``MemoryRanker``: pluggable re-ranker for memory candidates.++ Pre-this-PR Headroom ranked memory candidates by pure cosine+ similarity. Every other memory system we surveyed+ (Letta / Mem0 / Cognee / Supermemory) re-ranks beyond cosine — recency,+ source weight, access count, and decay are table-stakes for not+ returning 6-month-old "winners" when fresh signal exists.++ This module ships the first ranker — :class:`RecencyBoostRanker` —+ plus the :class:`MemoryRanker` protocol that future rankers+ (source-weight, access-count) plug into.++ The ranker is **pure**: ``rank(candidates) -> ranked_candidates``,+ no I/O, no state, no mutation of inputs. Same Rust-port shape as+ ``CompressionDecision`` and ``MemoryDecision``.++ Performance: O(N) over candidates where N = top_k. One ``math.exp()``+ per candidate. Sub-microsecond per request — no embedding compute,+ no network, no disk.+ """++ from __future__ import annotations++ import math+ from dataclasses import dataclass, field+ from datetime import datetime, timezone+ from typing import Protocol++ # Use ``timezone.utc`` (always available) instead of ``datetime.UTC``+ # (Python 3.11+) so this module imports cleanly on older interpreters.+ _UTC = timezone.utc+++ @dataclass(frozen=True)+ class MemoryCandidate:+ """Immutable retrieval candidate as it flows through the ranker.++ The shape is the **proxy-side internal contract** — the backend's+ return type (typically ``MemoryResult`` with nested+ ``MemoryResult.memory.created_at``) is adapted into this flatter+ shape at the ranker boundary so the ranker stays backend-agnostic.++ ``score`` is the cosine similarity as returned by the backend.+ Rankers MAY mutate ``score`` by returning a new candidate with an+ updated score (frozen dataclass means they cannot mutate in place).+ """++ content: str+ score: float+ created_at: datetime | None = None+ source: str | None = None # e.g. "memory_save" | "traffic_learner" | "inline"+ related_entities: tuple[str, ...] = field(default_factory=tuple)++ # Backend memory ID. Empty string when not preserved (test fixtures,++ # legacy callers). Rendered as ``[id]`` in the auto-tail block so the++ # model can pass it to memory_update / memory_delete directly.++ id: str = ""++ @classmethod+ def from_backend_result(cls, result: object) -> MemoryCandidate:+ """Adapter from backend ``MemoryResult`` shape to ``MemoryCandidate``.++ The backend returns objects with ``.score``, ``.memory.content``,- and (optionally) ``.memory.created_at`` (str ISO timestamp) +- ``.related_entities``. This adapter flattens that to the- ranker's expected shape and parses the timestamp to ``datetime``.++ ``.memory.id``, and (optionally) ``.memory.created_at`` (str ISO++ timestamp) + ``.related_entities``. This adapter flattens that to++ the ranker's expected shape and parses the timestamp to++ ``datetime``.++ Missing / unparseable timestamps → ``None`` (recency-neutral).++ Missing IDs → ``""`` (rendered as ``[?]`` in the auto-tail block).+ """+ score = float(getattr(result, "score", 0.0))+ memory = getattr(result, "memory", None)+ content = str(getattr(memory, "content", "")) if memory is not None else ""++ memory_id = str(getattr(memory, "id", "") or "") if memory is not None else ""+ raw_dt = getattr(memory, "created_at", None) if memory is not None else None+ created_at = _parse_created_at(raw_dt)+ raw_related = getattr(result, "related_entities", None) or ()+ related = tuple(str(x) for x in raw_related)+ source_meta = getattr(memory, "metadata", None) or {}+ source = source_meta.get("source") if isinstance(source_meta, dict) else None+ return cls(+ content=content,+ score=score,+ created_at=created_at,+ source=source,+ related_entities=related,++ id=memory_id,+ )+++ def _parse_created_at(value: object) -> datetime | None:+ """Best-effort parse of a timestamp into a UTC-aware datetime.++ Accepts ``datetime`` (returned as-is, UTC-normalized) or ISO-8601+ string (with or without trailing ``Z``). Anything else → ``None``+ so the ranker treats the candidate as recency-neutral.+ """+ if value is None:+ return None+ if isinstance(value, datetime):+ return value if value.tzinfo is not None else value.replace(tzinfo=_UTC)+ if isinstance(value, str):+ try:+ return datetime.fromisoformat(value.replace("Z", "+00:00"))+ except ValueError:+ return None+ return None+++ class MemoryRanker(Protocol):+ """Re-ranks retrieval candidates. Pure function.++ Implementations MUST:+ * Not mutate the input list or its elements+ * Be deterministic (same input → same output) for prefix-cache+ stability across consecutive turns+ * Be backend-agnostic — work with any+ :class:`MemoryCandidate`, regardless of which backend produced it+ """++ def rank(self, candidates: list[MemoryCandidate]) -> list[MemoryCandidate]: ...+++ @dataclass(frozen=True)+ class RecencyBoostRanker:+ """Re-ranker applying an exponential recency decay to cosine scores.++ Final score: ``cosine × exp(-age_days / decay_days)``.++ At ``decay_days=30``:+ * age = 0 days → factor 1.000+ * age = 15 days → factor 0.607+ * age = 30 days → factor 0.368+ * age = 60 days → factor 0.135+ * age = 90 days → factor 0.050++ Tuned so a fresh memory with weak cosine doesn't dominate (factor+ decays gradually), but a 6-month-old strong-cosine can't dominate+ either (factor approaches zero). Operators tune ``decay_days`` for+ their codebase's rate of change — 7 days for rapidly-evolving+ repos, 90 days for stable archival.++ Backwards-compat: candidates with ``created_at=None`` get factor+ 1.0 — treated as recency-neutral. Lets a backend during a+ migration return some rows with timestamps and some without+ without breaking the ranker.++ Defensive: negative ages (clock skew on future-timestamped rows)+ are clamped to factor 1.0 — a clock-skewed candidate cannot+ outrank a real fresh one with the same cosine.+ """++ decay_days: float = 30.0++ def rank(self, candidates: list[MemoryCandidate]) -> list[MemoryCandidate]:+ """Re-rank by ``score × recency_factor``. Pure; input unchanged.++ Sorts descending by boosted score. Ties broken by input order+ (Python's sort is stable) — deterministic output for prefix-+ cache stability across turns.+ """+ if not candidates:+ return []++ now = datetime.now(_UTC)+ boosted: list[tuple[int, MemoryCandidate, float]] = []+ for idx, c in enumerate(candidates):+ factor = self._recency_factor(now, c.created_at)+ boosted.append((idx, c, c.score * factor))++ # Sort descending by boosted score; stable on ties via the+ # captured idx — same input order preserved on ties for+ # deterministic output across turns.+ boosted.sort(key=lambda triple: (-triple[2], triple[0]))++ # Return new candidates with the boosted score so downstream+ # consumers (logging, budget filter) see the post-boost number.+ return [+ MemoryCandidate(+ content=c.content,+ score=new_score,+ created_at=c.created_at,+ source=c.source,+ related_entities=c.related_entities,+ )+ for _, c, new_score in boosted+ ]++ def _recency_factor(self, now: datetime, created_at: datetime | None) -> float:+ """Compute the recency multiplier for a single candidate.++ ``None`` timestamp → 1.0 (recency-neutral, backwards-compat).+ Future timestamps → 1.0 (clock-skew defence).+ Otherwise: ``exp(-age_days / decay_days)``.+ """+ if created_at is None:+ return 1.0+ # Normalize to UTC-aware for safe subtraction.+ if created_at.tzinfo is None:+ created_at = created_at.replace(tzinfo=_UTC)+ delta = now - created_at+ age_days = delta.total_seconds() / 86400.0+ if age_days <= 0:+ # Future-dated row (clock skew). Clamp to neutral.+ return 1.0+ return math.exp(-age_days / self.decay_days)diff --cc tests/test_memory_ranker.pyindex 000000000,eacfcabb2..58e2d5d9dmode 000000,100644..100644--- a/tests/test_memory_ranker.py+++ b/tests/test_memory_ranker.py@@@ -1,0 -1,243 +1,286 @@@+ """Tests for :class:`headroom.proxy.memory_ranker.MemoryRanker` ++ :class:`RecencyBoostRanker`.++ Pre-this-PR Headroom ranked memory candidates by pure cosine+ similarity. Every other memory system we surveyed (Letta, Mem0,+ Cognee, Supermemory) re-ranks beyond cosine — recency / source /+ access-count / decay are table-stakes. The pure-cosine baseline+ returns 6-month-old memories with 0.9 similarity ahead of fresh+ memories with 0.5 — wrong for most use cases.++ ``RecencyBoostRanker`` is the first ranker we ship: a pure-function+ ``score = cosine × exp(-age_days / decay_days)`` re-ranker. Default+ ``decay_days=30`` (half-life ~21 days). Other rankers (source-weight,+ access-count) plug into the same :class:`MemoryRanker` protocol in+ follow-on PRs.++ Performance: O(N) over candidates where N = top_k = ~10. One ``exp()``+ per candidate. Sub-microsecond per request — no embedding compute, no+ I/O. The ranker is pure and Rust-portable.+ """++ from __future__ import annotations++ from dataclasses import FrozenInstanceError+ from datetime import datetime, timedelta, timezone++ from headroom.proxy.memory_ranker import (+ MemoryCandidate,+ RecencyBoostRanker,+ )++ _UTC = timezone.utc++ # ── Helpers ───────────────────────────────────────────────────────────+++ def _candidate(content: str, score: float, age_days: float = 0.0) -> MemoryCandidate:+ """Build a MemoryCandidate at the given cosine score and age."""+ created = datetime.now(_UTC) - timedelta(days=age_days)+ return MemoryCandidate(content=content, score=score, created_at=created)+++ def _candidate_no_timestamp(content: str, score: float) -> MemoryCandidate:+ """Build a MemoryCandidate without a created_at (back-compat shape)."""+ return MemoryCandidate(content=content, score=score, created_at=None)+++ # ── MemoryCandidate value-type contract ──────────────────────────────+++ def test_candidate_is_frozen() -> None:+ """Frozen so a ranker can't mutate a candidate's score and lie about+ which candidates it returned."""+ c = _candidate("x", 0.9, age_days=0)+ try:+ c.score = 0.1 # type: ignore[misc]+ except FrozenInstanceError:+ pass+ else:+ raise AssertionError("MemoryCandidate must be frozen")++++def test_from_backend_result_preserves_memory_id() -> None:++ """The adapter must carry ``memory.id`` through to MemoryCandidate.id++ so the auto-tail block can render it as the bracketed handle the++ model uses for memory_update / memory_delete. Pre-this-fix the++ adapter dropped the ID, which silently regressed the [id] auto-tail++ format on the ranker path."""++++ class _Mem:++ id = "mem_abc_123"++ content = "User prefers Python."++ created_at = None++ metadata = {"source": "memory_save"}++++ class _Result:++ memory = _Mem()++ score = 0.91++ related_entities = ("python",)++++ cand = MemoryCandidate.from_backend_result(_Result())++ assert cand.id == "mem_abc_123"++ assert cand.content == "User prefers Python."++ assert cand.score == 0.91++++++def test_from_backend_result_handles_missing_id() -> None:++ """Defensive: legacy backend rows without an ID become ``id=""``;++ the auto-tail formatter renders ``[?]`` for those rows, no crash."""++++ class _Mem:++ # no .id attribute++ content = "legacy row"++ created_at = None++ metadata = {}++++ class _Result:++ memory = _Mem()++ score = 0.5++ related_entities = ()++++ cand = MemoryCandidate.from_backend_result(_Result())++ assert cand.id == ""+++++ # ── RecencyBoostRanker contract ──────────────────────────────────────+++ def test_ranker_is_frozen() -> None:+ """The ranker config is itself immutable — operators set+ ``decay_days`` at construction; runtime cannot edit it."""+ r = RecencyBoostRanker()+ try:+ r.decay_days = 99 # type: ignore[misc]+ except FrozenInstanceError:+ pass+ else:+ raise AssertionError("RecencyBoostRanker must be frozen")+++ def test_ranker_default_decay_is_thirty_days() -> None:+ """30-day decay is the conservative default. At 30 days, factor is+ ~0.37 (e^{-1}); at 90 days, ~0.05. Tuned so a fresh memory with+ weak cosine doesn't dominate, but a 6-month-old strong-cosine+ can't dominate either."""+ assert RecencyBoostRanker().decay_days == 30.0+++ def test_ranker_default_decay_is_configurable() -> None:+ """Operators can tune decay; e.g., 7 days for an aggressive+ recency bias on rapidly-evolving codebases."""+ r = RecencyBoostRanker(decay_days=7.0)+ assert r.decay_days == 7.0+++ def test_ranker_returns_list_preserving_shape() -> None:+ """Output is a list of candidates (re-ranked). Length matches+ input length — the ranker does NOT filter, only re-orders. The+ budget filters; the ranker ranks."""+ candidates = [_candidate("a", 0.9), _candidate("b", 0.5)]+ out = RecencyBoostRanker().rank(candidates)+ assert len(out) == 2+ assert {c.content for c in out} == {"a", "b"}+++ # ── Recency boost behaviour ──────────────────────────────────────────+++ def test_equal_cosine_younger_wins() -> None:+ """Two candidates with identical cosine score — the younger one+ wins because its recency factor is closer to 1.0."""+ fresh = _candidate("fresh", 0.5, age_days=0)+ old = _candidate("old", 0.5, age_days=60)+ out = RecencyBoostRanker().rank([old, fresh])+ assert out[0].content == "fresh"+ assert out[1].content == "old"+++ def test_old_strong_cosine_can_still_beat_young_weak_cosine() -> None:+ """The boost is multiplicative, not absolute — a 60-day-old memory+ with 0.9 cosine (0.9 × 0.135 ≈ 0.12) still loses to a 0-day-old+ memory with 0.5 cosine (0.5 × 1.0 = 0.5). But a 5-day-old memory+ with 0.9 (0.9 × 0.847 ≈ 0.76) beats a 0-day-old with 0.5."""+ very_old_strong = _candidate("old_strong", 0.9, age_days=60)+ fresh_weak = _candidate("fresh_weak", 0.5, age_days=0)+ out = RecencyBoostRanker().rank([very_old_strong, fresh_weak])+ # fresh_weak should win because 60-day decay flattens the strong cosine+ assert out[0].content == "fresh_weak"++ # Versus: slightly-old strong beats fresh weak+ slightly_old_strong = _candidate("slightly_old_strong", 0.9, age_days=5)+ fresh_weak2 = _candidate("fresh_weak2", 0.5, age_days=0)+ out2 = RecencyBoostRanker().rank([fresh_weak2, slightly_old_strong])+ assert out2[0].content == "slightly_old_strong"+++ def test_decay_rate_changes_winner() -> None:+ """An aggressive decay_days=7 makes a 30-day-old memory much+ weaker than a default decay_days=30. Locks the configurability+ contract."""+ old_strong = _candidate("old_strong", 0.9, age_days=30)+ fresh_weak = _candidate("fresh_weak", 0.6, age_days=0)++ # decay_days=30: old × e^{-1} ≈ 0.331; fresh = 0.6 → fresh wins+ r_default = RecencyBoostRanker(decay_days=30.0)+ out_default = r_default.rank([old_strong, fresh_weak])+ assert out_default[0].content == "fresh_weak"++ # decay_days=120 (loose): old × e^{-0.25} ≈ 0.701; fresh = 0.6 → old wins+ r_loose = RecencyBoostRanker(decay_days=120.0)+ out_loose = r_loose.rank([old_strong, fresh_weak])+ assert out_loose[0].content == "old_strong"+++ def test_zero_age_memory_keeps_full_cosine() -> None:+ """At age=0 days, the recency factor is e^0 = 1.0 — the boosted+ score equals the original cosine. Fresh memories see no penalty."""+ fresh = _candidate("fresh", 0.7, age_days=0)+ out = RecencyBoostRanker().rank([fresh])+ # Compare with tolerance — datetime.now() drift between+ # _candidate() and rank() is microseconds, so factor ~ 1.0.+ assert out[0].score == 0.7 or abs(out[0].score - 0.7) < 1e-3+++ def test_candidate_without_timestamp_keeps_pure_cosine() -> None:+ """Backwards-compat: pre-this-PR candidates may not have a+ ``created_at`` (older rows / older backends). NULL timestamp+ means "treat as recency-neutral" — factor 1.0. Pure cosine."""+ no_ts = _candidate_no_timestamp("legacy", 0.8)+ out = RecencyBoostRanker().rank([no_ts])+ assert out[0].score == 0.8+++ def test_mixed_with_and_without_timestamps() -> None:+ """A backend that returns SOME candidates with timestamps and+ SOME without (e.g., during a migration) must still produce a+ sensible ranking. NULL-timestamp candidates get factor 1.0,+ timestamped ones get their decay."""+ fresh_ts = _candidate("fresh_ts", 0.6, age_days=0)+ old_ts = _candidate("old_ts", 0.6, age_days=60)+ no_ts_neutral = _candidate_no_timestamp("no_ts", 0.6)+ out = RecencyBoostRanker().rank([old_ts, no_ts_neutral, fresh_ts])+ # fresh_ts (~0.6) and no_ts (=0.6) tied at top — old_ts decayed.+ assert out[-1].content == "old_ts"+ assert {out[0].content, out[1].content} == {"fresh_ts", "no_ts"}+++ # ── Stability + edge cases ───────────────────────────────────────────+++ def test_empty_input_returns_empty_output() -> None:+ """No candidates → no candidates. Boundary case."""+ assert RecencyBoostRanker().rank([]) == []+++ def test_ranking_is_stable_for_identical_candidates() -> None:+ """Two candidates with identical content + score + age → stable+ order (no spurious reshuffling). Important for prefix-cache+ stability: a deterministic ranker means consecutive turns inject+ the same memory in the same order, preserving byte-stable+ output."""+ a = _candidate("same", 0.5, age_days=10)+ b = _candidate("same", 0.5, age_days=10)+ out = RecencyBoostRanker().rank([a, b])+ assert len(out) == 2+++ def test_negative_age_treated_as_zero() -> None:+ """Defensive: a candidate with a future ``created_at`` (clock+ skew) shouldn't crash or give a > 1.0 factor. ``exp(-age/decay)``+ with negative age gives > 1; we clamp to 1.0 so a clock-skewed+ candidate can't outrank a real fresh one with score=1.0+ artifically."""+ future = _candidate("future", 0.5, age_days=-10) # 10 days in future+ fresh = _candidate("fresh", 0.5, age_days=0)+ out = RecencyBoostRanker().rank([future, fresh])+ # Both should have factor 1.0 (clamped) — score equal → stable order+ assert {out[0].content, out[1].content} == {"future", "fresh"}+ assert out[0].score == 0.5 or abs(out[0].score - 0.5) < 1e-3+++ # ── Rust-port shape ─────────────────────────────────────────────────+++ def test_ranker_is_pure_no_side_effects() -> None:+ """Calling rank() twice with the same input gives the same+ output. No state on the ranker; no I/O. Rust-portable."""+ candidates = [_candidate("a", 0.7, age_days=5), _candidate("b", 0.5, age_days=20)]+ r = RecencyBoostRanker()+ out1 = r.rank(candidates)+ out2 = r.rank(candidates)+ assert [c.content for c in out1] == [c.content for c in out2]+ # Inputs preserved — ranker did not mutate+ assert candidates[0].content == "a"+ assert candidates[1].content == "b"+++ def test_ranker_does_not_mutate_input_list() -> None:+ """Defence-in-depth: the input list and its elements must be+ unchanged after ranking. Frozen candidates make element mutation+ impossible; the list order itself must also be preserved."""+ a = _candidate("a", 0.5, age_days=20)+ b = _candidate("b", 0.5, age_days=5)+ candidates = [a, b]+ RecencyBoostRanker().rank(candidates)+ assert candidates == [a, b] # original list order preserved