Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(memory): keep learned evidence until persistence succeeds (#4229)
Mansurshakh Japarov662a7f6fd836
diff --git a/docs/content/docs/proxy.mdx b/docs/content/docs/proxy.mdxindex 4f8c00bb6..63bdf3856 100644--- a/docs/content/docs/proxy.mdx+++ b/docs/content/docs/proxy.mdx@@ -137,7 +137,7 @@ HEADROOM_SAVINGS_PROFILE=agent-90 headroom proxy --port 8787| `--memory-top-k` | `10` | Number of memories to inject as context || `--learn` | `false` | Enable live traffic learning; implies `--memory` || `--no-learn` | `false` | Explicitly disable traffic learning |-| `--min-evidence` | `5` | Minimum observations before a learned pattern is persisted |+| `--min-evidence` | `5` | Minimum observations before a learned pattern is persisted; `1` saves the first sighting || `--codex-wire-debug` | `false` | Write local Codex wire snapshots and matching proxy log traces || `HEADROOM_COMPRESS_PASSTHROUGH` | `0` | Also compress custom proxy paths that fall through to the catch-all handler (OpenAI Responses-shaped bodies, path ends in `/responses`). No `--compress-passthrough` flag on `headroom proxy`; the direct module entry point (`python -m headroom.proxy.server`) does accept one |@@ -148,6 +148,15 @@ headroom proxy --codex-wire-debugHEADROOM_COMPRESS_PASSTHROUGH=1 headroom proxy```+Live learning includes observations received while a save is queued or in flight.+A pattern is marked as saved only after its evidence total is persisted. A full+queue, failed save or cancellation retains the accumulated evidence for the next+sighting and the shutdown snapshot.++With LocalBackend, recovery reuses a row that committed before indexing failed+instead of inserting another memory, and refreshes its indexes. Reconciliation+is idempotent even when a cancelled SQLite update finishes late.+Automatic memory lookup renders a separate, bounded embedding query:| Input | Maximum characters |diff --git a/headroom/memory/traffic_learner.py b/headroom/memory/traffic_learner.pyindex 7d2c95be9..f8a197e96 100644--- a/headroom/memory/traffic_learner.py+++ b/headroom/memory/traffic_learner.py@@ -26,10 +26,12 @@ import osimport reimport sqlite3import time+import uuidfrom collections import OrderedDictfrom dataclasses import dataclass, fieldfrom datetime import datetime, timezonefrom enum import Enum+from itertools import chainfrom pathlib import Pathfrom typing import TYPE_CHECKING, Any, ClassVar@@ -539,6 +541,10 @@ class TrafficLearner:# Background save queueself._save_queue: asyncio.Queue[ExtractedPattern] = asyncio.Queue(maxsize=100)+ # Keep queued/in-flight dedup separate from confirmed persistence.+ self._queued_patterns: dict[str, ExtractedPattern] = {}+ # Keep an attempt's idempotency key until its whole evidence total is durable.+ self._save_tokens: dict[str, str] = {}self._save_task: asyncio.Task[None] | None = Noneself._stopping = False@@ -565,7 +571,7 @@ class TrafficLearner:# Hydrate persisted dedup state before workers spin up so cross-session# re-sightings bump existing rows instead of creating duplicates.await self._hydrate_persisted_state()- # Rehydrate sub-threshold evidence from the previous process. Without+ # Rehydrate pending evidence from the previous process. Without# this every restart zeroed pending counts, so users with short or# restart-heavy sessions could never cross min_evidence.await self._load_pending_state()@@ -593,27 +599,15 @@ class TrafficLearner:except asyncio.CancelledError:pass- # Drain any patterns left in the queue (worker may have been cancelled mid-flight)+ # Failed or cancelled saves remain pending for the shutdown snapshot.while not self._save_queue.empty():+ pattern = self._save_queue.get_nowait()try:- pattern = self._save_queue.get_nowait()- if self._backend is not None:- await self._backend.save_memory(- content=pattern.content,- user_id=self._user_id,- importance=pattern.importance,- metadata={- "source": "traffic_learner",- "category": pattern.category.value,- "evidence_count": pattern.evidence_count,- **pattern.metadata,- },- )- self._patterns_saved += 1- except Exception:- break+ await self._save_pattern(pattern)+ except Exception as exc:+ logger.warning("Traffic learner shutdown save failed: %r", exc)- # Snapshot sub-threshold evidence so it survives the restart.+ # Snapshot pending evidence so it survives the restart.await self._save_pending_state()# Final flush on shutdown — bypass debounce.@@ -1288,6 +1282,11 @@ class TrafficLearner:self._patterns_extracted += 1self._flush_dirty = Trueh = pattern.content_hash+ queued = self._queued_patterns.get(h)+ if queued is not None:+ queued.evidence_count += 1+ queued.metadata["last_seen_at"] = datetime.now(timezone.utc).isoformat()+ return# Already saved — bump the persisted row's evidence_count rather# than creating a duplicate.@@ -1308,42 +1307,86 @@ class TrafficLearner:# Bound the pending accumulator so one-off patterns can't grow it# without limit; drop the least-recently-corroborated pending entry.if len(self._pattern_counts) >= self._max_pending_patterns:- self._pattern_counts.popitem(last=False)+ evicted, _ = self._pattern_counts.popitem(last=False)+ self._save_tokens.pop(evicted, None)self._pattern_counts[h] = (pattern, 1)- return # First sighting — wait for more evidence# Check if evidence threshold met- _, count = self._pattern_counts[h]+ pattern, count = self._pattern_counts[h]+ now_iso = datetime.now(timezone.utc).isoformat()+ pattern.metadata.setdefault("first_seen_at", now_iso)+ pattern.metadata["last_seen_at"] = now_isoif count >= self._min_evidence:- # Ready to save- del self._pattern_counts[h]- self._saved_hashes.add(h)- # Trim saved hashes to prevent unbounded growth, and drop the evicted- # hash's persisted-id entry in lockstep. ``_persisted_ids`` is only- # ever read behind an ``h in _saved_hashes`` guard (the dedup check- # above), so an id for a hash no longer tracked is dead weight;- # without this it grew one entry per distinct persisted pattern for- # the whole process lifetime while ``_saved_hashes`` stayed bounded.- if len(self._saved_hashes) > self._dedup_window:- # Remove oldest (arbitrary, set is unordered, but prevents growth)- evicted = self._saved_hashes.pop()- self._persisted_ids.pop(evicted, None)-- # Persist the real accumulated count, not the dataclass default.pattern.evidence_count = counttry:self._save_queue.put_nowait(pattern)except asyncio.QueueFull:- logger.debug("Traffic learner save queue full, dropping pattern")+ logger.debug("Traffic learner save queue full, keeping pending evidence")+ return+ del self._pattern_counts[h]+ self._queued_patterns[h] = pattern- async def _save_worker(self) -> None:- """Background worker that persists patterns to memory backend."""- while True:+ def _restore_pattern(self, pattern: ExtractedPattern) -> None:+ """Keep corroborated evidence eligible after an unsuccessful save."""+ h = pattern.content_hash+ if (+ h not in self._pattern_counts+ and len(self._pattern_counts) >= self._max_pending_patterns+ ):+ evicted, _ = self._pattern_counts.popitem(last=False)+ self._save_tokens.pop(evicted, None)+ self._pattern_counts[h] = (pattern, pattern.evidence_count)+ self._pattern_counts.move_to_end(h)+ self._flush_dirty = True++ async def _find_persisted_pattern(+ self, pattern: ExtractedPattern, save_token: str+ ) -> tuple[str, int] | None:+ """Resolve an uncertain local save before another insert is allowed."""+ db_path = _resolve_backend_db_path(self._backend)+ if db_path is None or not db_path.exists():+ return None++ def _read() -> tuple[str, int] | None:+ conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)try:- pattern = await self._save_queue.get()- if self._backend is None:- continue+ row = conn.execute(+ "SELECT id, COALESCE(json_extract(metadata, '$.learner_saved_evidence'), 0) "+ "FROM memories WHERE user_id = ? AND content = ? "+ "AND json_extract(metadata, '$.source') = 'traffic_learner' "+ "AND json_extract(metadata, '$.category') = ? "+ "AND json_extract(metadata, '$.content_hash') = ? "+ "AND json_extract(metadata, '$.learner_save_token') = ? "+ "ORDER BY json_extract(metadata, '$.last_seen_at') DESC, id DESC LIMIT 1",+ (+ self._user_id,+ pattern.content,+ pattern.category.value,+ pattern.content_hash,+ save_token,+ ),+ ).fetchone()+ finally:+ conn.close()+ return (row[0], int(row[1])) if row is not None else None++ return await asyncio.to_thread(_read)+ async def _save_pattern(self, pattern: ExtractedPattern) -> None:+ h = pattern.content_hash+ try:+ if self._backend is None:+ self._restore_pattern(pattern)+ return+ existing: tuple[str, int] | None = None+ save_token = self._save_tokens.get(h)+ if save_token is None:+ save_token = uuid.uuid4().hex+ self._save_tokens[h] = save_token+ else:+ existing = await self._find_persisted_pattern(pattern, save_token)+ if existing is None:+ saved_count = pattern.evidence_countnow_iso = datetime.now(timezone.utc).isoformat()memory = await self._backend.save_memory(content=pattern.content,@@ -1352,27 +1395,62 @@ class TrafficLearner:metadata={"source": "traffic_learner","category": pattern.category.value,- "evidence_count": pattern.evidence_count,+ "evidence_count": saved_count,"first_seen_at": now_iso,"last_seen_at": now_iso,**pattern.metadata,+ "content_hash": h,+ "learner_save_token": save_token,+ "learner_saved_evidence": saved_count,},)- self._patterns_saved += 1- # Track id so future re-sightings bump this row — but only while- # the hash is still within the dedup window. If it was evicted- # from ``_saved_hashes`` between enqueue and now, recording its id- # would re-leak an entry that can never be read again (the dedup- # read at the top of ``_accumulate`` is gated on ``_saved_hashes``).memory_id = getattr(memory, "id", None)- if memory_id is not None and pattern.content_hash in self._saved_hashes:- self._persisted_ids[pattern.content_hash] = memory_id- logger.debug(f"Traffic learner saved pattern: {pattern.content[:80]}")+ else:+ memory_id, saved_count = existing++ # Observations can arrive during either the save or its index refresh.+ # Keep ownership queued until every observed count is durable.+ refresh = existing is not None+ while refresh or pattern.evidence_count > saved_count:+ if memory_id is None:+ raise RuntimeError("Saved learner pattern has no memory id")+ target_count = pattern.evidence_count+ last_seen = pattern.metadata.get("last_seen_at")+ updated = await self._bump_persisted_evidence(+ memory_id,+ count=0,+ last_seen_at=last_seen if isinstance(last_seen, str) else None,+ save_attempt=(save_token, target_count),+ )+ if not updated:+ raise RuntimeError("Learner evidence update was not persisted")+ saved_count = target_count+ refresh = False+ except (Exception, asyncio.CancelledError):+ self._restore_pattern(pattern)+ raise+ else:+ self._patterns_saved += 1+ self._saved_hashes.add(h)+ self._save_tokens.pop(h, None)+ if len(self._saved_hashes) > self._dedup_window:+ self._persisted_ids.pop(self._saved_hashes.pop(), None)+ if memory_id is not None and h in self._saved_hashes:+ self._persisted_ids[h] = memory_id+ logger.debug("Traffic learner saved pattern: %s", pattern.content[:80])+ finally:+ self._queued_patterns.pop(h, None)+ async def _save_worker(self) -> None:+ """Background worker that persists patterns to memory backend."""+ while True:+ try:+ pattern = await self._save_queue.get()+ await self._save_pattern(pattern)except asyncio.CancelledError:break- except Exception as e:- logger.warning(f"Traffic learner save failed: {e}")+ except Exception as exc:+ logger.warning("Traffic learner save failed: %r", exc)async def _hydrate_persisted_state(self) -> None:"""Load existing traffic_learner rows into _saved_hashes / _persisted_ids.@@ -1446,14 +1524,14 @@ class TrafficLearner:self._persisted_ids[h] = memory_iddef _pending_state_path(self) -> Path | None:- """Sidecar JSON next to memory.db holding sub-threshold evidence."""+ """Sidecar JSON next to memory.db holding unsaved evidence."""db_path = _resolve_backend_db_path(self._backend)if db_path is None:return Nonereturn db_path.with_name("pending_patterns.json")async def _save_pending_state(self) -> None:- """Snapshot the pending accumulator so evidence survives restarts.+ """Snapshot pending and queued evidence so it survives restarts.Best-effort: failures are logged at debug and never propagate — theaccumulator degrades gracefully back to per-process behavior.@@ -1470,8 +1548,12 @@ class TrafficLearner:"entity_refs": pattern.entity_refs,"metadata": pattern.metadata,"content_hash": h,+ "save_token": self._save_tokens.get(h),}- for h, (pattern, count) in self._pattern_counts.items()+ for h, (pattern, count) in chain(+ self._pattern_counts.items(),+ ((h, (p, p.evidence_count)) for h, p in self._queued_patterns.items()),+ )]def _write() -> None:@@ -1486,12 +1568,12 @@ class TrafficLearner:logger.debug("Traffic learner pending-state save failed: %s", e)async def _load_pending_state(self) -> None:- """Rehydrate sub-threshold evidence saved by a previous process.+ """Rehydrate unsaved evidence from a previous process.- Runs after _hydrate_persisted_state so entries whose pattern crossed- the threshold in the meantime (hash now in _saved_hashes) are skipped;- their re-sightings bump the persisted row instead. Unreadable or- malformed files are ignored — the next snapshot overwrites them.+ Runs after _hydrate_persisted_state. Confirmed patterns are skipped,+ but an interrupted save's token keeps its pending evidence eligible+ for reconciliation with the committed row. Unreadable or malformed+ files are ignored — the next snapshot overwrites them."""path = self._pending_state_path()if path is None or not path.exists():@@ -1507,8 +1589,16 @@ class TrafficLearner:for entry in entries[: self._max_pending_patterns]:try:h = entry["content_hash"]- if not h or h in self._saved_hashes or h in self._pattern_counts:+ if not h or h in self._pattern_counts:continue+ save_token = entry.get("save_token")+ if not isinstance(save_token, str) or not save_token:+ save_token = None+ if h in self._saved_hashes:+ if save_token is None:+ continue+ self._saved_hashes.discard(h)+ self._persisted_ids.pop(h, None)pattern = ExtractedPattern(category=PatternCategory(entry["category"]),content=entry["content"],@@ -1518,30 +1608,59 @@ class TrafficLearner:content_hash=h,)self._pattern_counts[h] = (pattern, max(1, int(entry.get("count", 1))))+ if save_token is not None:+ self._save_tokens[h] = save_tokenexcept Exception:continue- async def _bump_persisted_evidence(self, memory_id: str) -> None:- """Atomically increment a persisted row's metadata.evidence_count."""+ async def _bump_persisted_evidence(+ self,+ memory_id: str,+ count: int = 1,+ last_seen_at: str | None = None,+ *,+ save_attempt: tuple[str, int] | None = None,+ ) -> bool:+ """Atomically add evidence to a persisted row and refresh its indexes."""db_path = _resolve_backend_db_path(self._backend)if db_path is None or not db_path.exists():- return+ return False- now_iso = datetime.now(timezone.utc).isoformat()+ now_iso = last_seen_at or datetime.now(timezone.utc).isoformat()def _bump() -> bool:from ..fileperms import connect_private_sqliteconn = connect_private_sqlite(db_path, what="memory store")try:- cursor = conn.execute(- "UPDATE memories SET metadata = json_set("- "metadata, '$.evidence_count', "- "COALESCE(json_extract(metadata, '$.evidence_count'), 0) + 1, "- "'$.last_seen_at', ?"- ") WHERE id = ?",- (now_iso, memory_id),- )+ if save_attempt is None:+ cursor = conn.execute(+ "UPDATE memories SET metadata = json_set("+ "metadata, '$.evidence_count', "+ "COALESCE(json_extract(metadata, '$.evidence_count'), 0) + ?, "+ "'$.last_seen_at', "+ "MAX(COALESCE(json_extract(metadata, '$.last_seen_at'), ''), ?)"+ ") WHERE id = ?",+ (count, now_iso, memory_id),+ )+ else:+ # A cancelled worker's SQLite write can still finish. Apply+ # only evidence beyond this save attempt's durable watermark.+ save_token, target = save_attempt+ cursor = conn.execute(+ "UPDATE memories SET metadata = json_set("+ "metadata, '$.evidence_count', "+ "COALESCE(json_extract(metadata, '$.evidence_count'), 0) + "+ "MAX(0, ? - COALESCE("+ "json_extract(metadata, '$.learner_saved_evidence'), 0)), "+ "'$.learner_saved_evidence', MAX(COALESCE("+ "json_extract(metadata, '$.learner_saved_evidence'), 0), ?), "+ "'$.last_seen_at', "+ "MAX(COALESCE(json_extract(metadata, '$.last_seen_at'), ''), ?)"+ ") WHERE id = ? "+ "AND json_extract(metadata, '$.learner_save_token') = ?",+ (target, target, now_iso, memory_id, save_token),+ )conn.commit()return cursor.rowcount > 0finally:@@ -1551,7 +1670,7 @@ class TrafficLearner:updated = await asyncio.to_thread(_bump)except Exception as e:logger.debug("Traffic learner evidence bump failed for %s: %s", memory_id, e)- return+ return Falserefresh = getattr(self._backend, "refresh_memory_indexes", None)if updated and refresh is not None:@@ -1563,6 +1682,9 @@ class TrafficLearner:memory_id,e,)+ if save_attempt is not None:+ raise+ return updated# =========================================================================# Convenience: Extract from Anthropic messages formatdiff --git a/tests/test_memory/test_traffic_learner.py b/tests/test_memory/test_traffic_learner.pyindex fa7eaecb9..789ee6b05 100644--- a/tests/test_memory/test_traffic_learner.py+++ b/tests/test_memory/test_traffic_learner.py@@ -521,87 +521,6 @@ class TestTrafficLearner:asyncio.run(feed_one_offs())assert len(learner._pattern_counts) <= 8 # capped, not 500- def test_pending_accumulator_lru_still_promotes_corroborated_pattern(self):- """Capping the accumulator must not break promotion: a pattern- corroborated to ``min_evidence`` without interruption is still removed- from pending and recorded in ``_saved_hashes``."""- import asyncio-- learner = TrafficLearner(backend=None, min_evidence=3, max_pending_patterns=100)- pattern = ExtractedPattern(- category=PatternCategory.PREFERENCE,- content="corroborated preference",- importance=0.5,- )-- async def corroborate() -> None:- await learner._accumulate(pattern) # count 1- await learner._accumulate(pattern) # count 2- assert pattern.content_hash in learner._pattern_counts- await learner._accumulate(pattern) # count 3 == min_evidence -> promote-- asyncio.run(corroborate())- assert pattern.content_hash not in learner._pattern_counts # removed on promotion- assert pattern.content_hash in learner._saved_hashes-- def test_persisted_ids_bounded_in_lockstep_with_saved_hashes(self):- """``_persisted_ids`` (content_hash -> memory row id) must not outgrow- ``_saved_hashes``. The dedup read only consults ``_persisted_ids`` behind- an ``h in _saved_hashes`` guard, so an id whose hash has been evicted from- the dedup window is dead weight; previously it accumulated one entry per- distinct persisted pattern for the whole process lifetime while- ``_saved_hashes`` stayed trimmed to ``dedup_window``.- """- import asyncio-- learner = TrafficLearner(backend=None, min_evidence=1, dedup_window=4)-- async def feed() -> None:- for i in range(200):- pattern = ExtractedPattern(- category=PatternCategory.PREFERENCE,- content=f"distinct preference number {i}",- importance=0.5,- )- await learner._accumulate(pattern) # first sight -> pending- await learner._accumulate(pattern) # second -> promote to saved- # Emulate the async save worker recording the row id, using the- # same "still tracked" guard the worker now applies.- if pattern.content_hash in learner._saved_hashes:- learner._persisted_ids[pattern.content_hash] = f"mem-{i}"-- asyncio.run(feed())-- # _saved_hashes stays bounded (existing behavior).- assert len(learner._saved_hashes) <= 4- # _persisted_ids no longer leaks: it never holds an id for a hash that is- # no longer in the dedup window, so it stays bounded too.- assert set(learner._persisted_ids).issubset(learner._saved_hashes)- assert len(learner._persisted_ids) <= 4 # not 200-- @pytest.mark.asyncio- async def test_dedup(self, learner: TrafficLearner):- """Test that identical patterns are deduplicated."""- # Same pattern twice- for _ in range(2):- await learner.on_tool_result(- tool_name="Bash",- tool_input={"command": "ruff check ."},- tool_output="command not found: ruff",- is_error=True,- )- await learner.on_tool_result(- tool_name="Bash",- tool_input={"command": ".venv/bin/ruff check ."},- tool_output="OK",- is_error=False,- )-- # Should not double-count the same pattern- stats = learner.get_stats()- # First extraction saves, second is deduped- assert stats["patterns_extracted"] >= 1-@pytest.mark.asyncioasync def test_extract_tool_results_from_messages(self, learner: TrafficLearner):"""Test extraction of tool results from Anthropic message format."""@@ -1166,9 +1085,9 @@ class _FakeBackend:conn = _sql.connect(self._db_path)try:conn.execute(- "INSERT INTO memories (id, content, metadata, entity_refs, importance) "- "VALUES (?,?,?,?,?)",- (mid, content, _json.dumps(metadata), "[]", importance),+ "INSERT INTO memories (id, content, user_id, metadata, entity_refs, importance) "+ "VALUES (?,?,?,?,?,?)",+ (mid, content, user_id, _json.dumps(metadata), "[]", importance),)conn.commit()finally:@@ -1183,6 +1102,7 @@ def _init_db(path):conn.execute("CREATE TABLE memories (""id TEXT PRIMARY KEY, content TEXT NOT NULL, "+ "user_id TEXT NOT NULL DEFAULT 'default', ""metadata TEXT NOT NULL DEFAULT '{}', ""entity_refs TEXT NOT NULL DEFAULT '[]', ""importance REAL NOT NULL DEFAULT 0.5, "@@ -1220,7 +1140,466 @@ async def _wait_for_saved(learner: TrafficLearner, count: int, db_path) -> None:)+async def _wait_for_evidence(db_path, content, count):+ for _ in range(100):+ found = [+ meta["evidence_count"]+ for _, text, meta in _read_traffic_rows(db_path)+ if text == content+ ]+ if found == [count]:+ return+ await asyncio.sleep(0.02)+ raise AssertionError(f"Expected one row with evidence {count}; got {found}")+++class _ObservedBackend(_FakeBackend):+ def __init__(self, db_path):+ super().__init__(db_path)+ self.entered = asyncio.Event()+ self.attempted = asyncio.Event()+ self.release = None+ self.committed = asyncio.Event()+ self.after_commit_release = None+ self.fail_after_commit = False++ async def save_memory(self, **kwargs):+ self.entered.set()+ try:+ if self.release is not None:+ await self.release.wait()+ memory = await super().save_memory(**kwargs)+ self.committed.set()+ if self.after_commit_release is not None:+ await self.after_commit_release.wait()+ if self.fail_after_commit:+ raise RuntimeError("indexing failed after commit")+ return memory+ finally:+ self.attempted.set()+++class TestSaveRecovery:+ @staticmethod+ async def _feed(learner, content, count):+ for _ in range(count):+ await learner._accumulate(+ ExtractedPattern(+ category=PatternCategory.PREFERENCE,+ content=content,+ importance=0.7,+ )+ )++ @pytest.mark.asyncio+ async def test_failed_save_retries_with_accumulated_evidence(self, tmp_path):+ import sqlite3++ db = tmp_path / "memory.db"+ _init_db(db)+ with sqlite3.connect(db) as conn:+ conn.execute(+ "CREATE TRIGGER reject_save BEFORE INSERT ON memories "+ "BEGIN SELECT RAISE(ABORT, 'backend unavailable'); END"+ )+ backend = _ObservedBackend(db)+ backend.release = asyncio.Event()+ learner = TrafficLearner(backend=backend, min_evidence=3)+ await learner.start()+ try:+ await self._feed(learner, "Use uv for Python packages.", 3)+ await asyncio.wait_for(backend.entered.wait(), 2)+ await self._feed(learner, "Use uv for Python packages.", 2)+ backend.release.set()+ await asyncio.wait_for(backend.attempted.wait(), 2)+ assert _read_traffic_rows(db) == []+ with sqlite3.connect(db) as conn:+ conn.execute("DROP TRIGGER reject_save")+ await self._feed(learner, "Use uv for Python packages.", 1)+ await _wait_for_saved(learner, 1, db)+ finally:+ await learner.stop()+ rows = _read_traffic_rows(db)+ assert len(rows) == 1+ assert rows[0][1] == "Use uv for Python packages."+ assert rows[0][2]["evidence_count"] == 6++ @pytest.mark.asyncio+ @pytest.mark.parametrize("in_flight", [False, True])+ async def test_queued_and_inflight_observations_reach_the_saved_row(self, tmp_path, in_flight):+ db = tmp_path / "memory.db"+ _init_db(db)+ backend = _ObservedBackend(db)+ backend.release = asyncio.Event()+ learner = TrafficLearner(backend=backend, min_evidence=3)+ content = "Use uv for Python packages."+ try:+ if in_flight:+ await learner.start()+ await self._feed(learner, content, 3)+ await asyncio.wait_for(backend.entered.wait(), 2)+ await self._feed(learner, content, 2)+ backend.release.set()+ else:+ await self._feed(learner, content, 5)+ backend.release.set()+ await learner.start()+ await _wait_for_evidence(db, content, 5)+ finally:+ backend.release.set()+ await learner.stop()+ rows = _read_traffic_rows(db)+ assert len(rows) == 1+ assert rows[0][2]["evidence_count"] == 5+ assert _parse_iso_timestamp(rows[0][2]["last_seen_at"]) >= _parse_iso_timestamp(+ rows[0][2]["first_seen_at"]+ )++ @pytest.mark.asyncio+ @pytest.mark.parametrize("cancelled", [False, True])+ async def test_postcommit_failure_or_cancellation_reuses_the_committed_row(+ self, tmp_path, cancelled+ ):+ db = tmp_path / "memory.db"+ _init_db(db)+ backend = _ObservedBackend(db)+ backend.after_commit_release = asyncio.Event()+ backend.fail_after_commit = not cancelled+ learner = TrafficLearner(backend=backend, min_evidence=3)+ content = "Use uv for Python packages."+ await learner.start()+ try:+ await self._feed(learner, content, 3)+ await asyncio.wait_for(backend.committed.wait(), 2)+ committed = _read_traffic_rows(db)+ assert len(committed) == 1+ assert committed[0][2]["evidence_count"] == 3+ await self._feed(learner, content, 2)+ if cancelled:+ await learner.stop()+ learner = TrafficLearner(backend=backend, min_evidence=3)+ await learner.start()+ else:+ backend.after_commit_release.set()+ await asyncio.wait_for(backend.attempted.wait(), 2)+ backend.fail_after_commit = False+ backend.after_commit_release.set()+ await self._feed(learner, content, 1)+ await _wait_for_evidence(db, content, 6)+ finally:+ backend.after_commit_release.set()+ await learner.stop()+ rows = _read_traffic_rows(db)+ assert len(rows) == 1+ assert rows[0][0] == committed[0][0]+ assert rows[0][2]["evidence_count"] == 6++ @pytest.mark.asyncio+ async def test_repeated_index_repair_failures_remain_pending(self, tmp_path):+ db = tmp_path / "memory.db"+ _init_db(db)+ backend = _ObservedBackend(db)+ backend.fail_after_commit = True+ refresh = backend.refresh_memory_indexes+ refresh_attempted = asyncio.Event()+ failures_left = 2++ async def failing_refresh(memory_id):+ nonlocal failures_left+ try:+ if failures_left:+ failures_left -= 1+ raise RuntimeError("index unavailable")+ await refresh(memory_id)+ finally:+ refresh_attempted.set()++ backend.refresh_memory_indexes = failing_refresh+ learner = TrafficLearner(backend=backend, min_evidence=3)+ content = "Use uv for Python packages."+ await learner.start()+ try:+ await self._feed(learner, content, 3)+ await asyncio.wait_for(backend.attempted.wait(), 2)+ committed = _read_traffic_rows(db)+ assert len(committed) == 1+ memory_id, _, metadata = committed[0]+ h = metadata["content_hash"]+ save_token = metadata["learner_save_token"]+ assert h not in learner._saved_hashes+ assert learner._pattern_counts[h][1] == 3+ assert learner._save_tokens[h] == save_token++ for extra, expected in ((2, 5), (1, 6)):+ refresh_attempted.clear()+ await self._feed(learner, content, extra)+ await asyncio.wait_for(refresh_attempted.wait(), 2)+ rows = _read_traffic_rows(db)+ assert len(rows) == 1+ assert rows[0][0] == memory_id+ assert rows[0][2]["evidence_count"] == expected+ assert rows[0][2]["learner_saved_evidence"] == expected+ assert h not in learner._saved_hashes+ assert learner._pattern_counts[h][1] == expected+ assert learner._save_tokens[h] == save_token++ refresh_attempted.clear()+ await self._feed(learner, content, 1)+ await asyncio.wait_for(refresh_attempted.wait(), 2)+ rows = _read_traffic_rows(db)+ assert len(rows) == 1+ assert rows[0][0] == memory_id+ assert rows[0][2]["evidence_count"] == 7+ assert rows[0][2]["learner_saved_evidence"] == 7+ assert h in learner._saved_hashes+ assert h not in learner._pattern_counts+ assert h not in learner._save_tokens+ finally:+ await learner.stop()++ @pytest.mark.asyncio+ async def test_shutdown_keeps_failed_evidence_and_drains_other_queued_saves(self, tmp_path):+ import sqlite3++ db = tmp_path / "memory.db"+ _init_db(db)+ failed = "Use uv for Python packages."+ saved = "Keep pull request descriptions short."+ with sqlite3.connect(db) as conn:+ conn.execute(+ "CREATE TRIGGER reject_one_save BEFORE INSERT ON memories "+ "WHEN NEW.content = 'Use uv for Python packages.' "+ "BEGIN SELECT RAISE(ABORT, 'backend unavailable'); END"+ )+ backend = _FakeBackend(db)+ learner = TrafficLearner(backend=backend, min_evidence=3)+ await self._feed(learner, failed, 3)+ await self._feed(learner, saved, 3)+ await learner.stop()+ assert {content: meta["evidence_count"] for _, content, meta in _read_traffic_rows(db)} == {+ saved: 3+ }++ with sqlite3.connect(db) as conn:+ conn.execute("DROP TRIGGER reject_one_save")+ restored = TrafficLearner(backend=backend, min_evidence=3)+ await restored.start()+ try:+ await self._feed(restored, failed, 1)+ await _wait_for_evidence(db, failed, 4)+ finally:+ await restored.stop()+ assert {content: meta["evidence_count"] for _, content, meta in _read_traffic_rows(db)} == {+ failed: 4,+ saved: 3,+ }++ @pytest.mark.asyncio+ async def test_reconciliation_retries_a_failed_sql_update_without_losing_evidence(+ self, tmp_path+ ):+ import sqlite3++ db = tmp_path / "memory.db"+ _init_db(db)+ backend = _ObservedBackend(db)+ backend.fail_after_commit = True+ learner = TrafficLearner(backend=backend, min_evidence=3)+ reconciled = asyncio.Event()+ bump = learner._bump_persisted_evidence++ async def observed_bump(*args, **kwargs):+ try:+ return await bump(*args, **kwargs)+ finally:+ reconciled.set()++ learner._bump_persisted_evidence = observed_bump+ content = "Use uv for Python packages."+ await learner.start()+ try:+ await self._feed(learner, content, 3)+ await asyncio.wait_for(backend.attempted.wait(), 2)+ committed = _read_traffic_rows(db)+ assert len(committed) == 1+ memory_id, _, metadata = committed[0]+ h = metadata["content_hash"]+ token = metadata["learner_save_token"]+ with sqlite3.connect(db) as conn:+ conn.execute(+ "CREATE TRIGGER reject_update BEFORE UPDATE ON memories "+ "BEGIN SELECT RAISE(ABORT, 'backend unavailable'); END"+ )+ await self._feed(learner, content, 2)+ await asyncio.wait_for(reconciled.wait(), 2)+ rows = _read_traffic_rows(db)+ assert len(rows) == 1+ assert rows[0][0] == memory_id+ assert rows[0][2]["evidence_count"] == 3+ assert rows[0][2]["learner_saved_evidence"] == 3+ assert learner._pattern_counts[h][1] == 5+ assert learner._save_tokens[h] == token+ assert h not in learner._saved_hashes++ with sqlite3.connect(db) as conn:+ conn.execute("DROP TRIGGER reject_update")+ reconciled.clear()+ await self._feed(learner, content, 1)+ await asyncio.wait_for(reconciled.wait(), 2)+ rows = _read_traffic_rows(db)+ assert len(rows) == 1+ assert rows[0][0] == memory_id+ assert rows[0][2]["evidence_count"] == 6+ assert rows[0][2]["learner_saved_evidence"] == 6+ assert h in learner._saved_hashes+ assert h not in learner._pattern_counts+ assert h not in learner._save_tokens+ finally:+ await learner.stop()++ @pytest.mark.asyncio+ async def test_confirmed_evidence_survives_best_effort_index_refresh_failure(self, tmp_path):+ db = tmp_path / "memory.db"+ _init_db(db)+ backend = _FakeBackend(db)+ learner = TrafficLearner(backend=backend, min_evidence=3)+ content = "Use uv for Python packages."+ await learner.start()+ try:+ await self._feed(learner, content, 3)+ await _wait_for_saved(learner, 1, db)+ committed = _read_traffic_rows(db)+ assert len(committed) == 1+ memory_id, _, metadata = committed[0]+ h = metadata["content_hash"]++ async def failing_refresh(memory_id):+ raise RuntimeError("index unavailable")++ backend.refresh_memory_indexes = failing_refresh+ await self._feed(learner, content, 2)+ rows = _read_traffic_rows(db)+ assert len(rows) == 1+ assert rows[0][0] == memory_id+ assert rows[0][2]["evidence_count"] == 5+ assert h in learner._saved_hashes+ assert h not in learner._pattern_counts+ assert h not in learner._save_tokens+ finally:+ await learner.stop()++ @pytest.mark.asyncio+ async def test_late_reconciliation_preserves_other_and_already_counted_sightings(+ self, tmp_path+ ):+ db = tmp_path / "memory.db"+ _init_db(db)+ backend = _FakeBackend(db)+ memory = await backend.save_memory(+ content="Use uv for Python packages.",+ user_id="default",+ importance=0.7,+ metadata={+ "source": "traffic_learner",+ "evidence_count": 3,+ "learner_save_token": "save-attempt",+ "learner_saved_evidence": 3,+ },+ )+ learner = TrafficLearner(backend=backend, min_evidence=3)+ await learner._bump_persisted_evidence(+ memory.id, last_seen_at="2024-01-02T00:00:00+00:00", save_attempt=("save-attempt", 5)+ )+ await learner._bump_persisted_evidence(memory.id, last_seen_at="2024-01-04T00:00:00+00:00")+ await learner._bump_persisted_evidence(+ memory.id, last_seen_at="2024-01-03T00:00:00+00:00", save_attempt=("save-attempt", 6)+ )+ await learner._bump_persisted_evidence(+ memory.id, last_seen_at="2024-01-02T00:00:00+00:00", save_attempt=("save-attempt", 5)+ )+ rows = _read_traffic_rows(db)+ assert len(rows) == 1+ assert rows[0][2]["evidence_count"] == 7+ assert rows[0][2]["learner_saved_evidence"] == 6+ assert rows[0][2]["last_seen_at"] == "2024-01-04T00:00:00+00:00"++ @pytest.mark.asyncio+ async def test_full_queue_preserves_evidence_for_a_later_sighting(self, tmp_path):+ db = tmp_path / "memory.db"+ _init_db(db)+ learner = TrafficLearner(backend=_FakeBackend(db), min_evidence=3)+ learner._save_queue = asyncio.Queue(maxsize=1)+ await self._feed(learner, "Use uv for Python packages.", 3)+ await self._feed(learner, "Keep pull request descriptions short.", 3)+ await learner.start()+ try:+ await _wait_for_saved(learner, 1, db)+ await self._feed(learner, "Keep pull request descriptions short.", 1)+ await _wait_for_saved(learner, 2, db)+ finally:+ await learner.stop()+ assert {content: meta["evidence_count"] for _, content, meta in _read_traffic_rows(db)} == {+ "Use uv for Python packages.": 3,+ "Keep pull request descriptions short.": 4,+ }++ @pytest.mark.asyncio+ async def test_cancelled_save_survives_restart(self, tmp_path):+ db = tmp_path / "memory.db"+ _init_db(db)+ backend = _ObservedBackend(db)+ backend.release = asyncio.Event()+ learner = TrafficLearner(backend=backend, min_evidence=3)+ await learner.start()+ await self._feed(learner, "Use uv for Python packages.", 3)+ await asyncio.wait_for(backend.entered.wait(), 2)+ await self._feed(learner, "Use uv for Python packages.", 2)+ await learner.stop()+ restarted = TrafficLearner(backend=_FakeBackend(db), min_evidence=3)+ await restarted.start()+ try:+ await self._feed(restarted, "Use uv for Python packages.", 1)+ await _wait_for_saved(restarted, 1, db)+ finally:+ await restarted.stop()+ rows = _read_traffic_rows(db)+ assert len(rows) == 1+ assert rows[0][2]["evidence_count"] == 6++class TestEvidencePersistence:+ @pytest.mark.asyncio+ @pytest.mark.parametrize("min_evidence", [1, 2, 5])+ async def test_persists_at_configured_evidence_threshold(self, tmp_path, min_evidence):+ db = tmp_path / "memory.db"+ _init_db(db)+ learner = TrafficLearner(backend=_FakeBackend(db), min_evidence=min_evidence)+ await learner.start()+ try:+ for _ in range(min_evidence - 1):+ await learner._accumulate(+ ExtractedPattern(+ category=PatternCategory.PREFERENCE,+ content="Use uv for Python packages.",+ importance=0.7,+ )+ )+ assert _read_traffic_rows(db) == []+ await learner._accumulate(+ ExtractedPattern(+ category=PatternCategory.PREFERENCE,+ content="Use uv for Python packages.",+ importance=0.7,+ )+ )+ await _wait_for_saved(learner, 1, db)+ finally:+ await learner.stop()+ rows = _read_traffic_rows(db)+ assert len(rows) == 1+ assert rows[0][1] == "Use uv for Python packages."+ assert rows[0][2]["evidence_count"] == min_evidence+@pytest.mark.asyncioasync def test_save_persists_actual_evidence_count(self, tmp_path):"""The count written to the DB reflects total sightings, not the default 1."""diff --git a/wiki/memory.md b/wiki/memory.mdindex 2806b3de3..54e0145cf 100644--- a/wiki/memory.md+++ b/wiki/memory.md@@ -85,6 +85,14 @@ with `--memory-db-path` for a custom location.> memory store should pass `--memory-db-path` explicitly. See the> [Filesystem Contract](filesystem-contract.md) for the rationale.+### Traffic Learning Recovery++The live traffic learner retains sightings received while a save is queued or+running. If SQLite commits but indexing fails, a later attempt reuses the same+row and adds only the missing observations. The pattern and its save-attempt+token stay pending if index repair fails again; the learner marks it saved only+after that repair succeeds.+### User IdentityUser ID is auto-detected from `$USER` (your OS username). Override per-request with the `x-headroom-user-id` header. All memories are scoped to the user — multiple developers on the same project have separate memory stores.