Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(memory): keep learned evidence until persistence succeeds (#4229)

Mansurshakh Japarov662a7f6fd836
diff --git a/docs/content/docs/proxy.mdx b/docs/content/docs/proxy.mdx
index 4f8c00bb6..63bdf3856 100644
--- a/docs/content/docs/proxy.mdx
+++ b/docs/content/docs/proxy.mdx
@@ -137,7 +137,7 @@ HEADROOM_SAVINGS_PROFILE=agent-90 headroom proxy --port 8787
| `--memory-top-k` | `10` | Number of memories to inject as context |
| `--learn` | `false` | Enable live traffic learning; implies `--memory` |
| `--no-learn` | `false` | Explicitly disable traffic learning |
-| `--min-evidence` | `5` | Minimum observations before a learned pattern is persisted |
+| `--min-evidence` | `5` | Minimum observations before a learned pattern is persisted; `1` saves the first sighting |
| `--codex-wire-debug` | `false` | Write local Codex wire snapshots and matching proxy log traces |
| `HEADROOM_COMPRESS_PASSTHROUGH` | `0` | Also compress custom proxy paths that fall through to the catch-all handler (OpenAI Responses-shaped bodies, path ends in `/responses`). No `--compress-passthrough` flag on `headroom proxy`; the direct module entry point (`python -m headroom.proxy.server`) does accept one |
@@ -148,6 +148,15 @@ headroom proxy --codex-wire-debug
HEADROOM_COMPRESS_PASSTHROUGH=1 headroom proxy
```
+Live learning includes observations received while a save is queued or in flight.
+A pattern is marked as saved only after its evidence total is persisted. A full
+queue, failed save or cancellation retains the accumulated evidence for the next
+sighting and the shutdown snapshot.
+
+With LocalBackend, recovery reuses a row that committed before indexing failed
+instead of inserting another memory, and refreshes its indexes. Reconciliation
+is idempotent even when a cancelled SQLite update finishes late.
+
Automatic memory lookup renders a separate, bounded embedding query:
| Input | Maximum characters |
diff --git a/headroom/memory/traffic_learner.py b/headroom/memory/traffic_learner.py
index 7d2c95be9..f8a197e96 100644
--- a/headroom/memory/traffic_learner.py
+++ b/headroom/memory/traffic_learner.py
@@ -26,10 +26,12 @@ import os
import re
import sqlite3
import time
+import uuid
from collections import OrderedDict
from dataclasses import dataclass, field
from datetime import datetime, timezone
from enum import Enum
+from itertools import chain
from pathlib import Path
from typing import TYPE_CHECKING, Any, ClassVar
@@ -539,6 +541,10 @@ class TrafficLearner:
# Background save queue
self._save_queue: asyncio.Queue[ExtractedPattern] = asyncio.Queue(maxsize=100)
+ # Keep queued/in-flight dedup separate from confirmed persistence.
+ self._queued_patterns: dict[str, ExtractedPattern] = {}
+ # Keep an attempt's idempotency key until its whole evidence total is durable.
+ self._save_tokens: dict[str, str] = {}
self._save_task: asyncio.Task[None] | None = None
self._stopping = False
@@ -565,7 +571,7 @@ class TrafficLearner:
# Hydrate persisted dedup state before workers spin up so cross-session
# re-sightings bump existing rows instead of creating duplicates.
await self._hydrate_persisted_state()
- # Rehydrate sub-threshold evidence from the previous process. Without
+ # Rehydrate pending evidence from the previous process. Without
# this every restart zeroed pending counts, so users with short or
# restart-heavy sessions could never cross min_evidence.
await self._load_pending_state()
@@ -593,27 +599,15 @@ class TrafficLearner:
except asyncio.CancelledError:
pass
- # Drain any patterns left in the queue (worker may have been cancelled mid-flight)
+ # Failed or cancelled saves remain pending for the shutdown snapshot.
while not self._save_queue.empty():
+ pattern = self._save_queue.get_nowait()
try:
- pattern = self._save_queue.get_nowait()
- if self._backend is not None:
- await self._backend.save_memory(
- content=pattern.content,
- user_id=self._user_id,
- importance=pattern.importance,
- metadata={
- "source": "traffic_learner",
- "category": pattern.category.value,
- "evidence_count": pattern.evidence_count,
- **pattern.metadata,
- },
- )
- self._patterns_saved += 1
- except Exception:
- break
+ await self._save_pattern(pattern)
+ except Exception as exc:
+ logger.warning("Traffic learner shutdown save failed: %r", exc)
- # Snapshot sub-threshold evidence so it survives the restart.
+ # Snapshot pending evidence so it survives the restart.
await self._save_pending_state()
# Final flush on shutdown — bypass debounce.
@@ -1288,6 +1282,11 @@ class TrafficLearner:
self._patterns_extracted += 1
self._flush_dirty = True
h = pattern.content_hash
+ queued = self._queued_patterns.get(h)
+ if queued is not None:
+ queued.evidence_count += 1
+ queued.metadata["last_seen_at"] = datetime.now(timezone.utc).isoformat()
+ return
# Already saved — bump the persisted row's evidence_count rather
# than creating a duplicate.
@@ -1308,42 +1307,86 @@ class TrafficLearner:
# Bound the pending accumulator so one-off patterns can't grow it
# without limit; drop the least-recently-corroborated pending entry.
if len(self._pattern_counts) >= self._max_pending_patterns:
- self._pattern_counts.popitem(last=False)
+ evicted, _ = self._pattern_counts.popitem(last=False)
+ self._save_tokens.pop(evicted, None)
self._pattern_counts[h] = (pattern, 1)
- return # First sighting — wait for more evidence
# Check if evidence threshold met
- _, count = self._pattern_counts[h]
+ pattern, count = self._pattern_counts[h]
+ now_iso = datetime.now(timezone.utc).isoformat()
+ pattern.metadata.setdefault("first_seen_at", now_iso)
+ pattern.metadata["last_seen_at"] = now_iso
if count >= self._min_evidence:
- # Ready to save
- del self._pattern_counts[h]
- self._saved_hashes.add(h)
- # Trim saved hashes to prevent unbounded growth, and drop the evicted
- # hash's persisted-id entry in lockstep. ``_persisted_ids`` is only
- # ever read behind an ``h in _saved_hashes`` guard (the dedup check
- # above), so an id for a hash no longer tracked is dead weight;
- # without this it grew one entry per distinct persisted pattern for
- # the whole process lifetime while ``_saved_hashes`` stayed bounded.
- if len(self._saved_hashes) > self._dedup_window:
- # Remove oldest (arbitrary, set is unordered, but prevents growth)
- evicted = self._saved_hashes.pop()
- self._persisted_ids.pop(evicted, None)
-
- # Persist the real accumulated count, not the dataclass default.
pattern.evidence_count = count
try:
self._save_queue.put_nowait(pattern)
except asyncio.QueueFull:
- logger.debug("Traffic learner save queue full, dropping pattern")
+ logger.debug("Traffic learner save queue full, keeping pending evidence")
+ return
+ del self._pattern_counts[h]
+ self._queued_patterns[h] = pattern
- async def _save_worker(self) -> None:
- """Background worker that persists patterns to memory backend."""
- while True:
+ def _restore_pattern(self, pattern: ExtractedPattern) -> None:
+ """Keep corroborated evidence eligible after an unsuccessful save."""
+ h = pattern.content_hash
+ if (
+ h not in self._pattern_counts
+ and len(self._pattern_counts) >= self._max_pending_patterns
+ ):
+ evicted, _ = self._pattern_counts.popitem(last=False)
+ self._save_tokens.pop(evicted, None)
+ self._pattern_counts[h] = (pattern, pattern.evidence_count)
+ self._pattern_counts.move_to_end(h)
+ self._flush_dirty = True
+
+ async def _find_persisted_pattern(
+ self, pattern: ExtractedPattern, save_token: str
+ ) -> tuple[str, int] | None:
+ """Resolve an uncertain local save before another insert is allowed."""
+ db_path = _resolve_backend_db_path(self._backend)
+ if db_path is None or not db_path.exists():
+ return None
+
+ def _read() -> tuple[str, int] | None:
+ conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
try:
- pattern = await self._save_queue.get()
- if self._backend is None:
- continue
+ row = conn.execute(
+ "SELECT id, COALESCE(json_extract(metadata, '$.learner_saved_evidence'), 0) "
+ "FROM memories WHERE user_id = ? AND content = ? "
+ "AND json_extract(metadata, '$.source') = 'traffic_learner' "
+ "AND json_extract(metadata, '$.category') = ? "
+ "AND json_extract(metadata, '$.content_hash') = ? "
+ "AND json_extract(metadata, '$.learner_save_token') = ? "
+ "ORDER BY json_extract(metadata, '$.last_seen_at') DESC, id DESC LIMIT 1",
+ (
+ self._user_id,
+ pattern.content,
+ pattern.category.value,
+ pattern.content_hash,
+ save_token,
+ ),
+ ).fetchone()
+ finally:
+ conn.close()
+ return (row[0], int(row[1])) if row is not None else None
+
+ return await asyncio.to_thread(_read)
+ async def _save_pattern(self, pattern: ExtractedPattern) -> None:
+ h = pattern.content_hash
+ try:
+ if self._backend is None:
+ self._restore_pattern(pattern)
+ return
+ existing: tuple[str, int] | None = None
+ save_token = self._save_tokens.get(h)
+ if save_token is None:
+ save_token = uuid.uuid4().hex
+ self._save_tokens[h] = save_token
+ else:
+ existing = await self._find_persisted_pattern(pattern, save_token)
+ if existing is None:
+ saved_count = pattern.evidence_count
now_iso = datetime.now(timezone.utc).isoformat()
memory = await self._backend.save_memory(
content=pattern.content,
@@ -1352,27 +1395,62 @@ class TrafficLearner:
metadata={
"source": "traffic_learner",
"category": pattern.category.value,
- "evidence_count": pattern.evidence_count,
+ "evidence_count": saved_count,
"first_seen_at": now_iso,
"last_seen_at": now_iso,
**pattern.metadata,
+ "content_hash": h,
+ "learner_save_token": save_token,
+ "learner_saved_evidence": saved_count,
},
)
- self._patterns_saved += 1
- # Track id so future re-sightings bump this row — but only while
- # the hash is still within the dedup window. If it was evicted
- # from ``_saved_hashes`` between enqueue and now, recording its id
- # would re-leak an entry that can never be read again (the dedup
- # read at the top of ``_accumulate`` is gated on ``_saved_hashes``).
memory_id = getattr(memory, "id", None)
- if memory_id is not None and pattern.content_hash in self._saved_hashes:
- self._persisted_ids[pattern.content_hash] = memory_id
- logger.debug(f"Traffic learner saved pattern: {pattern.content[:80]}")
+ else:
+ memory_id, saved_count = existing
+
+ # Observations can arrive during either the save or its index refresh.
+ # Keep ownership queued until every observed count is durable.
+ refresh = existing is not None
+ while refresh or pattern.evidence_count > saved_count:
+ if memory_id is None:
+ raise RuntimeError("Saved learner pattern has no memory id")
+ target_count = pattern.evidence_count
+ last_seen = pattern.metadata.get("last_seen_at")
+ updated = await self._bump_persisted_evidence(
+ memory_id,
+ count=0,
+ last_seen_at=last_seen if isinstance(last_seen, str) else None,
+ save_attempt=(save_token, target_count),
+ )
+ if not updated:
+ raise RuntimeError("Learner evidence update was not persisted")
+ saved_count = target_count
+ refresh = False
+ except (Exception, asyncio.CancelledError):
+ self._restore_pattern(pattern)
+ raise
+ else:
+ self._patterns_saved += 1
+ self._saved_hashes.add(h)
+ self._save_tokens.pop(h, None)
+ if len(self._saved_hashes) > self._dedup_window:
+ self._persisted_ids.pop(self._saved_hashes.pop(), None)
+ if memory_id is not None and h in self._saved_hashes:
+ self._persisted_ids[h] = memory_id
+ logger.debug("Traffic learner saved pattern: %s", pattern.content[:80])
+ finally:
+ self._queued_patterns.pop(h, None)
+ async def _save_worker(self) -> None:
+ """Background worker that persists patterns to memory backend."""
+ while True:
+ try:
+ pattern = await self._save_queue.get()
+ await self._save_pattern(pattern)
except asyncio.CancelledError:
break
- except Exception as e:
- logger.warning(f"Traffic learner save failed: {e}")
+ except Exception as exc:
+ logger.warning("Traffic learner save failed: %r", exc)
async def _hydrate_persisted_state(self) -> None:
"""Load existing traffic_learner rows into _saved_hashes / _persisted_ids.
@@ -1446,14 +1524,14 @@ class TrafficLearner:
self._persisted_ids[h] = memory_id
def _pending_state_path(self) -> Path | None:
- """Sidecar JSON next to memory.db holding sub-threshold evidence."""
+ """Sidecar JSON next to memory.db holding unsaved evidence."""
db_path = _resolve_backend_db_path(self._backend)
if db_path is None:
return None
return db_path.with_name("pending_patterns.json")
async def _save_pending_state(self) -> None:
- """Snapshot the pending accumulator so evidence survives restarts.
+ """Snapshot pending and queued evidence so it survives restarts.
Best-effort: failures are logged at debug and never propagate — the
accumulator degrades gracefully back to per-process behavior.
@@ -1470,8 +1548,12 @@ class TrafficLearner:
"entity_refs": pattern.entity_refs,
"metadata": pattern.metadata,
"content_hash": h,
+ "save_token": self._save_tokens.get(h),
}
- for h, (pattern, count) in self._pattern_counts.items()
+ for h, (pattern, count) in chain(
+ self._pattern_counts.items(),
+ ((h, (p, p.evidence_count)) for h, p in self._queued_patterns.items()),
+ )
]
def _write() -> None:
@@ -1486,12 +1568,12 @@ class TrafficLearner:
logger.debug("Traffic learner pending-state save failed: %s", e)
async def _load_pending_state(self) -> None:
- """Rehydrate sub-threshold evidence saved by a previous process.
+ """Rehydrate unsaved evidence from a previous process.
- Runs after _hydrate_persisted_state so entries whose pattern crossed
- the threshold in the meantime (hash now in _saved_hashes) are skipped;
- their re-sightings bump the persisted row instead. Unreadable or
- malformed files are ignored — the next snapshot overwrites them.
+ Runs after _hydrate_persisted_state. Confirmed patterns are skipped,
+ but an interrupted save's token keeps its pending evidence eligible
+ for reconciliation with the committed row. Unreadable or malformed
+ files are ignored — the next snapshot overwrites them.
"""
path = self._pending_state_path()
if path is None or not path.exists():
@@ -1507,8 +1589,16 @@ class TrafficLearner:
for entry in entries[: self._max_pending_patterns]:
try:
h = entry["content_hash"]
- if not h or h in self._saved_hashes or h in self._pattern_counts:
+ if not h or h in self._pattern_counts:
continue
+ save_token = entry.get("save_token")
+ if not isinstance(save_token, str) or not save_token:
+ save_token = None
+ if h in self._saved_hashes:
+ if save_token is None:
+ continue
+ self._saved_hashes.discard(h)
+ self._persisted_ids.pop(h, None)
pattern = ExtractedPattern(
category=PatternCategory(entry["category"]),
content=entry["content"],
@@ -1518,30 +1608,59 @@ class TrafficLearner:
content_hash=h,
)
self._pattern_counts[h] = (pattern, max(1, int(entry.get("count", 1))))
+ if save_token is not None:
+ self._save_tokens[h] = save_token
except Exception:
continue
- async def _bump_persisted_evidence(self, memory_id: str) -> None:
- """Atomically increment a persisted row's metadata.evidence_count."""
+ async def _bump_persisted_evidence(
+ self,
+ memory_id: str,
+ count: int = 1,
+ last_seen_at: str | None = None,
+ *,
+ save_attempt: tuple[str, int] | None = None,
+ ) -> bool:
+ """Atomically add evidence to a persisted row and refresh its indexes."""
db_path = _resolve_backend_db_path(self._backend)
if db_path is None or not db_path.exists():
- return
+ return False
- now_iso = datetime.now(timezone.utc).isoformat()
+ now_iso = last_seen_at or datetime.now(timezone.utc).isoformat()
def _bump() -> bool:
from ..fileperms import connect_private_sqlite
conn = connect_private_sqlite(db_path, what="memory store")
try:
- cursor = conn.execute(
- "UPDATE memories SET metadata = json_set("
- "metadata, '$.evidence_count', "
- "COALESCE(json_extract(metadata, '$.evidence_count'), 0) + 1, "
- "'$.last_seen_at', ?"
- ") WHERE id = ?",
- (now_iso, memory_id),
- )
+ if save_attempt is None:
+ cursor = conn.execute(
+ "UPDATE memories SET metadata = json_set("
+ "metadata, '$.evidence_count', "
+ "COALESCE(json_extract(metadata, '$.evidence_count'), 0) + ?, "
+ "'$.last_seen_at', "
+ "MAX(COALESCE(json_extract(metadata, '$.last_seen_at'), ''), ?)"
+ ") WHERE id = ?",
+ (count, now_iso, memory_id),
+ )
+ else:
+ # A cancelled worker's SQLite write can still finish. Apply
+ # only evidence beyond this save attempt's durable watermark.
+ save_token, target = save_attempt
+ cursor = conn.execute(
+ "UPDATE memories SET metadata = json_set("
+ "metadata, '$.evidence_count', "
+ "COALESCE(json_extract(metadata, '$.evidence_count'), 0) + "
+ "MAX(0, ? - COALESCE("
+ "json_extract(metadata, '$.learner_saved_evidence'), 0)), "
+ "'$.learner_saved_evidence', MAX(COALESCE("
+ "json_extract(metadata, '$.learner_saved_evidence'), 0), ?), "
+ "'$.last_seen_at', "
+ "MAX(COALESCE(json_extract(metadata, '$.last_seen_at'), ''), ?)"
+ ") WHERE id = ? "
+ "AND json_extract(metadata, '$.learner_save_token') = ?",
+ (target, target, now_iso, memory_id, save_token),
+ )
conn.commit()
return cursor.rowcount > 0
finally:
@@ -1551,7 +1670,7 @@ class TrafficLearner:
updated = await asyncio.to_thread(_bump)
except Exception as e:
logger.debug("Traffic learner evidence bump failed for %s: %s", memory_id, e)
- return
+ return False
refresh = getattr(self._backend, "refresh_memory_indexes", None)
if updated and refresh is not None:
@@ -1563,6 +1682,9 @@ class TrafficLearner:
memory_id,
e,
)
+ if save_attempt is not None:
+ raise
+ return updated
# =========================================================================
# Convenience: Extract from Anthropic messages format
diff --git a/tests/test_memory/test_traffic_learner.py b/tests/test_memory/test_traffic_learner.py
index fa7eaecb9..789ee6b05 100644
--- a/tests/test_memory/test_traffic_learner.py
+++ b/tests/test_memory/test_traffic_learner.py
@@ -521,87 +521,6 @@ class TestTrafficLearner:
asyncio.run(feed_one_offs())
assert len(learner._pattern_counts) <= 8 # capped, not 500
- def test_pending_accumulator_lru_still_promotes_corroborated_pattern(self):
- """Capping the accumulator must not break promotion: a pattern
- corroborated to ``min_evidence`` without interruption is still removed
- from pending and recorded in ``_saved_hashes``."""
- import asyncio
-
- learner = TrafficLearner(backend=None, min_evidence=3, max_pending_patterns=100)
- pattern = ExtractedPattern(
- category=PatternCategory.PREFERENCE,
- content="corroborated preference",
- importance=0.5,
- )
-
- async def corroborate() -> None:
- await learner._accumulate(pattern) # count 1
- await learner._accumulate(pattern) # count 2
- assert pattern.content_hash in learner._pattern_counts
- await learner._accumulate(pattern) # count 3 == min_evidence -> promote
-
- asyncio.run(corroborate())
- assert pattern.content_hash not in learner._pattern_counts # removed on promotion
- assert pattern.content_hash in learner._saved_hashes
-
- def test_persisted_ids_bounded_in_lockstep_with_saved_hashes(self):
- """``_persisted_ids`` (content_hash -> memory row id) must not outgrow
- ``_saved_hashes``. The dedup read only consults ``_persisted_ids`` behind
- an ``h in _saved_hashes`` guard, so an id whose hash has been evicted from
- the dedup window is dead weight; previously it accumulated one entry per
- distinct persisted pattern for the whole process lifetime while
- ``_saved_hashes`` stayed trimmed to ``dedup_window``.
- """
- import asyncio
-
- learner = TrafficLearner(backend=None, min_evidence=1, dedup_window=4)
-
- async def feed() -> None:
- for i in range(200):
- pattern = ExtractedPattern(
- category=PatternCategory.PREFERENCE,
- content=f"distinct preference number {i}",
- importance=0.5,
- )
- await learner._accumulate(pattern) # first sight -> pending
- await learner._accumulate(pattern) # second -> promote to saved
- # Emulate the async save worker recording the row id, using the
- # same "still tracked" guard the worker now applies.
- if pattern.content_hash in learner._saved_hashes:
- learner._persisted_ids[pattern.content_hash] = f"mem-{i}"
-
- asyncio.run(feed())
-
- # _saved_hashes stays bounded (existing behavior).
- assert len(learner._saved_hashes) <= 4
- # _persisted_ids no longer leaks: it never holds an id for a hash that is
- # no longer in the dedup window, so it stays bounded too.
- assert set(learner._persisted_ids).issubset(learner._saved_hashes)
- assert len(learner._persisted_ids) <= 4 # not 200
-
- @pytest.mark.asyncio
- async def test_dedup(self, learner: TrafficLearner):
- """Test that identical patterns are deduplicated."""
- # Same pattern twice
- for _ in range(2):
- await learner.on_tool_result(
- tool_name="Bash",
- tool_input={"command": "ruff check ."},
- tool_output="command not found: ruff",
- is_error=True,
- )
- await learner.on_tool_result(
- tool_name="Bash",
- tool_input={"command": ".venv/bin/ruff check ."},
- tool_output="OK",
- is_error=False,
- )
-
- # Should not double-count the same pattern
- stats = learner.get_stats()
- # First extraction saves, second is deduped
- assert stats["patterns_extracted"] >= 1
-
@pytest.mark.asyncio
async def test_extract_tool_results_from_messages(self, learner: TrafficLearner):
"""Test extraction of tool results from Anthropic message format."""
@@ -1166,9 +1085,9 @@ class _FakeBackend:
conn = _sql.connect(self._db_path)
try:
conn.execute(
- "INSERT INTO memories (id, content, metadata, entity_refs, importance) "
- "VALUES (?,?,?,?,?)",
- (mid, content, _json.dumps(metadata), "[]", importance),
+ "INSERT INTO memories (id, content, user_id, metadata, entity_refs, importance) "
+ "VALUES (?,?,?,?,?,?)",
+ (mid, content, user_id, _json.dumps(metadata), "[]", importance),
)
conn.commit()
finally:
@@ -1183,6 +1102,7 @@ def _init_db(path):
conn.execute(
"CREATE TABLE memories ("
"id TEXT PRIMARY KEY, content TEXT NOT NULL, "
+ "user_id TEXT NOT NULL DEFAULT 'default', "
"metadata TEXT NOT NULL DEFAULT '{}', "
"entity_refs TEXT NOT NULL DEFAULT '[]', "
"importance REAL NOT NULL DEFAULT 0.5, "
@@ -1220,7 +1140,466 @@ async def _wait_for_saved(learner: TrafficLearner, count: int, db_path) -> None:
)
+async def _wait_for_evidence(db_path, content, count):
+ for _ in range(100):
+ found = [
+ meta["evidence_count"]
+ for _, text, meta in _read_traffic_rows(db_path)
+ if text == content
+ ]
+ if found == [count]:
+ return
+ await asyncio.sleep(0.02)
+ raise AssertionError(f"Expected one row with evidence {count}; got {found}")
+
+
+class _ObservedBackend(_FakeBackend):
+ def __init__(self, db_path):
+ super().__init__(db_path)
+ self.entered = asyncio.Event()
+ self.attempted = asyncio.Event()
+ self.release = None
+ self.committed = asyncio.Event()
+ self.after_commit_release = None
+ self.fail_after_commit = False
+
+ async def save_memory(self, **kwargs):
+ self.entered.set()
+ try:
+ if self.release is not None:
+ await self.release.wait()
+ memory = await super().save_memory(**kwargs)
+ self.committed.set()
+ if self.after_commit_release is not None:
+ await self.after_commit_release.wait()
+ if self.fail_after_commit:
+ raise RuntimeError("indexing failed after commit")
+ return memory
+ finally:
+ self.attempted.set()
+
+
+class TestSaveRecovery:
+ @staticmethod
+ async def _feed(learner, content, count):
+ for _ in range(count):
+ await learner._accumulate(
+ ExtractedPattern(
+ category=PatternCategory.PREFERENCE,
+ content=content,
+ importance=0.7,
+ )
+ )
+
+ @pytest.mark.asyncio
+ async def test_failed_save_retries_with_accumulated_evidence(self, tmp_path):
+ import sqlite3
+
+ db = tmp_path / "memory.db"
+ _init_db(db)
+ with sqlite3.connect(db) as conn:
+ conn.execute(
+ "CREATE TRIGGER reject_save BEFORE INSERT ON memories "
+ "BEGIN SELECT RAISE(ABORT, 'backend unavailable'); END"
+ )
+ backend = _ObservedBackend(db)
+ backend.release = asyncio.Event()
+ learner = TrafficLearner(backend=backend, min_evidence=3)
+ await learner.start()
+ try:
+ await self._feed(learner, "Use uv for Python packages.", 3)
+ await asyncio.wait_for(backend.entered.wait(), 2)
+ await self._feed(learner, "Use uv for Python packages.", 2)
+ backend.release.set()
+ await asyncio.wait_for(backend.attempted.wait(), 2)
+ assert _read_traffic_rows(db) == []
+ with sqlite3.connect(db) as conn:
+ conn.execute("DROP TRIGGER reject_save")
+ await self._feed(learner, "Use uv for Python packages.", 1)
+ await _wait_for_saved(learner, 1, db)
+ finally:
+ await learner.stop()
+ rows = _read_traffic_rows(db)
+ assert len(rows) == 1
+ assert rows[0][1] == "Use uv for Python packages."
+ assert rows[0][2]["evidence_count"] == 6
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("in_flight", [False, True])
+ async def test_queued_and_inflight_observations_reach_the_saved_row(self, tmp_path, in_flight):
+ db = tmp_path / "memory.db"
+ _init_db(db)
+ backend = _ObservedBackend(db)
+ backend.release = asyncio.Event()
+ learner = TrafficLearner(backend=backend, min_evidence=3)
+ content = "Use uv for Python packages."
+ try:
+ if in_flight:
+ await learner.start()
+ await self._feed(learner, content, 3)
+ await asyncio.wait_for(backend.entered.wait(), 2)
+ await self._feed(learner, content, 2)
+ backend.release.set()
+ else:
+ await self._feed(learner, content, 5)
+ backend.release.set()
+ await learner.start()
+ await _wait_for_evidence(db, content, 5)
+ finally:
+ backend.release.set()
+ await learner.stop()
+ rows = _read_traffic_rows(db)
+ assert len(rows) == 1
+ assert rows[0][2]["evidence_count"] == 5
+ assert _parse_iso_timestamp(rows[0][2]["last_seen_at"]) >= _parse_iso_timestamp(
+ rows[0][2]["first_seen_at"]
+ )
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("cancelled", [False, True])
+ async def test_postcommit_failure_or_cancellation_reuses_the_committed_row(
+ self, tmp_path, cancelled
+ ):
+ db = tmp_path / "memory.db"
+ _init_db(db)
+ backend = _ObservedBackend(db)
+ backend.after_commit_release = asyncio.Event()
+ backend.fail_after_commit = not cancelled
+ learner = TrafficLearner(backend=backend, min_evidence=3)
+ content = "Use uv for Python packages."
+ await learner.start()
+ try:
+ await self._feed(learner, content, 3)
+ await asyncio.wait_for(backend.committed.wait(), 2)
+ committed = _read_traffic_rows(db)
+ assert len(committed) == 1
+ assert committed[0][2]["evidence_count"] == 3
+ await self._feed(learner, content, 2)
+ if cancelled:
+ await learner.stop()
+ learner = TrafficLearner(backend=backend, min_evidence=3)
+ await learner.start()
+ else:
+ backend.after_commit_release.set()
+ await asyncio.wait_for(backend.attempted.wait(), 2)
+ backend.fail_after_commit = False
+ backend.after_commit_release.set()
+ await self._feed(learner, content, 1)
+ await _wait_for_evidence(db, content, 6)
+ finally:
+ backend.after_commit_release.set()
+ await learner.stop()
+ rows = _read_traffic_rows(db)
+ assert len(rows) == 1
+ assert rows[0][0] == committed[0][0]
+ assert rows[0][2]["evidence_count"] == 6
+
+ @pytest.mark.asyncio
+ async def test_repeated_index_repair_failures_remain_pending(self, tmp_path):
+ db = tmp_path / "memory.db"
+ _init_db(db)
+ backend = _ObservedBackend(db)
+ backend.fail_after_commit = True
+ refresh = backend.refresh_memory_indexes
+ refresh_attempted = asyncio.Event()
+ failures_left = 2
+
+ async def failing_refresh(memory_id):
+ nonlocal failures_left
+ try:
+ if failures_left:
+ failures_left -= 1
+ raise RuntimeError("index unavailable")
+ await refresh(memory_id)
+ finally:
+ refresh_attempted.set()
+
+ backend.refresh_memory_indexes = failing_refresh
+ learner = TrafficLearner(backend=backend, min_evidence=3)
+ content = "Use uv for Python packages."
+ await learner.start()
+ try:
+ await self._feed(learner, content, 3)
+ await asyncio.wait_for(backend.attempted.wait(), 2)
+ committed = _read_traffic_rows(db)
+ assert len(committed) == 1
+ memory_id, _, metadata = committed[0]
+ h = metadata["content_hash"]
+ save_token = metadata["learner_save_token"]
+ assert h not in learner._saved_hashes
+ assert learner._pattern_counts[h][1] == 3
+ assert learner._save_tokens[h] == save_token
+
+ for extra, expected in ((2, 5), (1, 6)):
+ refresh_attempted.clear()
+ await self._feed(learner, content, extra)
+ await asyncio.wait_for(refresh_attempted.wait(), 2)
+ rows = _read_traffic_rows(db)
+ assert len(rows) == 1
+ assert rows[0][0] == memory_id
+ assert rows[0][2]["evidence_count"] == expected
+ assert rows[0][2]["learner_saved_evidence"] == expected
+ assert h not in learner._saved_hashes
+ assert learner._pattern_counts[h][1] == expected
+ assert learner._save_tokens[h] == save_token
+
+ refresh_attempted.clear()
+ await self._feed(learner, content, 1)
+ await asyncio.wait_for(refresh_attempted.wait(), 2)
+ rows = _read_traffic_rows(db)
+ assert len(rows) == 1
+ assert rows[0][0] == memory_id
+ assert rows[0][2]["evidence_count"] == 7
+ assert rows[0][2]["learner_saved_evidence"] == 7
+ assert h in learner._saved_hashes
+ assert h not in learner._pattern_counts
+ assert h not in learner._save_tokens
+ finally:
+ await learner.stop()
+
+ @pytest.mark.asyncio
+ async def test_shutdown_keeps_failed_evidence_and_drains_other_queued_saves(self, tmp_path):
+ import sqlite3
+
+ db = tmp_path / "memory.db"
+ _init_db(db)
+ failed = "Use uv for Python packages."
+ saved = "Keep pull request descriptions short."
+ with sqlite3.connect(db) as conn:
+ conn.execute(
+ "CREATE TRIGGER reject_one_save BEFORE INSERT ON memories "
+ "WHEN NEW.content = 'Use uv for Python packages.' "
+ "BEGIN SELECT RAISE(ABORT, 'backend unavailable'); END"
+ )
+ backend = _FakeBackend(db)
+ learner = TrafficLearner(backend=backend, min_evidence=3)
+ await self._feed(learner, failed, 3)
+ await self._feed(learner, saved, 3)
+ await learner.stop()
+ assert {content: meta["evidence_count"] for _, content, meta in _read_traffic_rows(db)} == {
+ saved: 3
+ }
+
+ with sqlite3.connect(db) as conn:
+ conn.execute("DROP TRIGGER reject_one_save")
+ restored = TrafficLearner(backend=backend, min_evidence=3)
+ await restored.start()
+ try:
+ await self._feed(restored, failed, 1)
+ await _wait_for_evidence(db, failed, 4)
+ finally:
+ await restored.stop()
+ assert {content: meta["evidence_count"] for _, content, meta in _read_traffic_rows(db)} == {
+ failed: 4,
+ saved: 3,
+ }
+
+ @pytest.mark.asyncio
+ async def test_reconciliation_retries_a_failed_sql_update_without_losing_evidence(
+ self, tmp_path
+ ):
+ import sqlite3
+
+ db = tmp_path / "memory.db"
+ _init_db(db)
+ backend = _ObservedBackend(db)
+ backend.fail_after_commit = True
+ learner = TrafficLearner(backend=backend, min_evidence=3)
+ reconciled = asyncio.Event()
+ bump = learner._bump_persisted_evidence
+
+ async def observed_bump(*args, **kwargs):
+ try:
+ return await bump(*args, **kwargs)
+ finally:
+ reconciled.set()
+
+ learner._bump_persisted_evidence = observed_bump
+ content = "Use uv for Python packages."
+ await learner.start()
+ try:
+ await self._feed(learner, content, 3)
+ await asyncio.wait_for(backend.attempted.wait(), 2)
+ committed = _read_traffic_rows(db)
+ assert len(committed) == 1
+ memory_id, _, metadata = committed[0]
+ h = metadata["content_hash"]
+ token = metadata["learner_save_token"]
+ with sqlite3.connect(db) as conn:
+ conn.execute(
+ "CREATE TRIGGER reject_update BEFORE UPDATE ON memories "
+ "BEGIN SELECT RAISE(ABORT, 'backend unavailable'); END"
+ )
+ await self._feed(learner, content, 2)
+ await asyncio.wait_for(reconciled.wait(), 2)
+ rows = _read_traffic_rows(db)
+ assert len(rows) == 1
+ assert rows[0][0] == memory_id
+ assert rows[0][2]["evidence_count"] == 3
+ assert rows[0][2]["learner_saved_evidence"] == 3
+ assert learner._pattern_counts[h][1] == 5
+ assert learner._save_tokens[h] == token
+ assert h not in learner._saved_hashes
+
+ with sqlite3.connect(db) as conn:
+ conn.execute("DROP TRIGGER reject_update")
+ reconciled.clear()
+ await self._feed(learner, content, 1)
+ await asyncio.wait_for(reconciled.wait(), 2)
+ rows = _read_traffic_rows(db)
+ assert len(rows) == 1
+ assert rows[0][0] == memory_id
+ assert rows[0][2]["evidence_count"] == 6
+ assert rows[0][2]["learner_saved_evidence"] == 6
+ assert h in learner._saved_hashes
+ assert h not in learner._pattern_counts
+ assert h not in learner._save_tokens
+ finally:
+ await learner.stop()
+
+ @pytest.mark.asyncio
+ async def test_confirmed_evidence_survives_best_effort_index_refresh_failure(self, tmp_path):
+ db = tmp_path / "memory.db"
+ _init_db(db)
+ backend = _FakeBackend(db)
+ learner = TrafficLearner(backend=backend, min_evidence=3)
+ content = "Use uv for Python packages."
+ await learner.start()
+ try:
+ await self._feed(learner, content, 3)
+ await _wait_for_saved(learner, 1, db)
+ committed = _read_traffic_rows(db)
+ assert len(committed) == 1
+ memory_id, _, metadata = committed[0]
+ h = metadata["content_hash"]
+
+ async def failing_refresh(memory_id):
+ raise RuntimeError("index unavailable")
+
+ backend.refresh_memory_indexes = failing_refresh
+ await self._feed(learner, content, 2)
+ rows = _read_traffic_rows(db)
+ assert len(rows) == 1
+ assert rows[0][0] == memory_id
+ assert rows[0][2]["evidence_count"] == 5
+ assert h in learner._saved_hashes
+ assert h not in learner._pattern_counts
+ assert h not in learner._save_tokens
+ finally:
+ await learner.stop()
+
+ @pytest.mark.asyncio
+ async def test_late_reconciliation_preserves_other_and_already_counted_sightings(
+ self, tmp_path
+ ):
+ db = tmp_path / "memory.db"
+ _init_db(db)
+ backend = _FakeBackend(db)
+ memory = await backend.save_memory(
+ content="Use uv for Python packages.",
+ user_id="default",
+ importance=0.7,
+ metadata={
+ "source": "traffic_learner",
+ "evidence_count": 3,
+ "learner_save_token": "save-attempt",
+ "learner_saved_evidence": 3,
+ },
+ )
+ learner = TrafficLearner(backend=backend, min_evidence=3)
+ await learner._bump_persisted_evidence(
+ memory.id, last_seen_at="2024-01-02T00:00:00+00:00", save_attempt=("save-attempt", 5)
+ )
+ await learner._bump_persisted_evidence(memory.id, last_seen_at="2024-01-04T00:00:00+00:00")
+ await learner._bump_persisted_evidence(
+ memory.id, last_seen_at="2024-01-03T00:00:00+00:00", save_attempt=("save-attempt", 6)
+ )
+ await learner._bump_persisted_evidence(
+ memory.id, last_seen_at="2024-01-02T00:00:00+00:00", save_attempt=("save-attempt", 5)
+ )
+ rows = _read_traffic_rows(db)
+ assert len(rows) == 1
+ assert rows[0][2]["evidence_count"] == 7
+ assert rows[0][2]["learner_saved_evidence"] == 6
+ assert rows[0][2]["last_seen_at"] == "2024-01-04T00:00:00+00:00"
+
+ @pytest.mark.asyncio
+ async def test_full_queue_preserves_evidence_for_a_later_sighting(self, tmp_path):
+ db = tmp_path / "memory.db"
+ _init_db(db)
+ learner = TrafficLearner(backend=_FakeBackend(db), min_evidence=3)
+ learner._save_queue = asyncio.Queue(maxsize=1)
+ await self._feed(learner, "Use uv for Python packages.", 3)
+ await self._feed(learner, "Keep pull request descriptions short.", 3)
+ await learner.start()
+ try:
+ await _wait_for_saved(learner, 1, db)
+ await self._feed(learner, "Keep pull request descriptions short.", 1)
+ await _wait_for_saved(learner, 2, db)
+ finally:
+ await learner.stop()
+ assert {content: meta["evidence_count"] for _, content, meta in _read_traffic_rows(db)} == {
+ "Use uv for Python packages.": 3,
+ "Keep pull request descriptions short.": 4,
+ }
+
+ @pytest.mark.asyncio
+ async def test_cancelled_save_survives_restart(self, tmp_path):
+ db = tmp_path / "memory.db"
+ _init_db(db)
+ backend = _ObservedBackend(db)
+ backend.release = asyncio.Event()
+ learner = TrafficLearner(backend=backend, min_evidence=3)
+ await learner.start()
+ await self._feed(learner, "Use uv for Python packages.", 3)
+ await asyncio.wait_for(backend.entered.wait(), 2)
+ await self._feed(learner, "Use uv for Python packages.", 2)
+ await learner.stop()
+ restarted = TrafficLearner(backend=_FakeBackend(db), min_evidence=3)
+ await restarted.start()
+ try:
+ await self._feed(restarted, "Use uv for Python packages.", 1)
+ await _wait_for_saved(restarted, 1, db)
+ finally:
+ await restarted.stop()
+ rows = _read_traffic_rows(db)
+ assert len(rows) == 1
+ assert rows[0][2]["evidence_count"] == 6
+
+
class TestEvidencePersistence:
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("min_evidence", [1, 2, 5])
+ async def test_persists_at_configured_evidence_threshold(self, tmp_path, min_evidence):
+ db = tmp_path / "memory.db"
+ _init_db(db)
+ learner = TrafficLearner(backend=_FakeBackend(db), min_evidence=min_evidence)
+ await learner.start()
+ try:
+ for _ in range(min_evidence - 1):
+ await learner._accumulate(
+ ExtractedPattern(
+ category=PatternCategory.PREFERENCE,
+ content="Use uv for Python packages.",
+ importance=0.7,
+ )
+ )
+ assert _read_traffic_rows(db) == []
+ await learner._accumulate(
+ ExtractedPattern(
+ category=PatternCategory.PREFERENCE,
+ content="Use uv for Python packages.",
+ importance=0.7,
+ )
+ )
+ await _wait_for_saved(learner, 1, db)
+ finally:
+ await learner.stop()
+ rows = _read_traffic_rows(db)
+ assert len(rows) == 1
+ assert rows[0][1] == "Use uv for Python packages."
+ assert rows[0][2]["evidence_count"] == min_evidence
+
@pytest.mark.asyncio
async def test_save_persists_actual_evidence_count(self, tmp_path):
"""The count written to the DB reflects total sightings, not the default 1."""
diff --git a/wiki/memory.md b/wiki/memory.md
index 2806b3de3..54e0145cf 100644
--- a/wiki/memory.md
+++ b/wiki/memory.md
@@ -85,6 +85,14 @@ with `--memory-db-path` for a custom location.
> memory store should pass `--memory-db-path` explicitly. See the
> [Filesystem Contract](filesystem-contract.md) for the rationale.
+### Traffic Learning Recovery
+
+The live traffic learner retains sightings received while a save is queued or
+running. If SQLite commits but indexing fails, a later attempt reuses the same
+row and adds only the missing observations. The pattern and its save-attempt
+token stay pending if index repair fails again; the learner marks it saved only
+after that repair succeeds.
+
### User Identity
User ID is auto-detected from `$USER` (your OS username). Override per-request with the `x-headroom-user-id` header. All memories are scoped to the user — multiple developers on the same project have separate memory stores.