Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(learn): persist real evidence_count and bump on re-sighting

Garm3e290b734b92
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1ee141352..f64ee5d4d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,38 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
+### Added
+- **Live flush of traffic-learned patterns to CLAUDE.md / MEMORY.md** — the
+ `TrafficLearner` now writes to agent-native context files continuously
+ during proxy operation, not just at shutdown. A new dirty-flag debounced
+ `_flush_worker` (10s window, `FLUSH_DEBOUNCE_SECONDS`) calls
+ `flush_to_file()` whenever `_accumulate()` marks the learner dirty, so
+ patterns surface in `CLAUDE.md` / `MEMORY.md` near real-time. Flushes
+ read both persisted rows (via `_load_persisted_patterns_from_sqlite`)
+ and the in-memory accumulator, bucket patterns by project via the learn
+ plugin registry (`plugin.discover_projects()` + longest-path anchoring
+ in `_project_for_pattern`), and route by `PatternCategory` to the
+ correct file (`_patterns_to_recommendations` +
+ `_CATEGORY_TO_TARGET`). Live flushes require `evidence_count >= 2`;
+ the shutdown flush accepts single-evidence rows.
+
+### Fixed
+- **Traffic-learner evidence count stuck at 1; duplicate DB rows across
+ restarts.** `_accumulate` queued patterns with the default
+ `ExtractedPattern.evidence_count = 1` regardless of how many times the
+ pattern was actually seen, so every persisted row landed at `1` and
+ never crossed the live-flush gate (`evidence_count >= 2`). Worse, once
+ a pattern was in `_saved_hashes` it was early-returned on every
+ re-sighting, and `_saved_hashes` reset on process restart — so a second
+ sighting in a later session inserted a duplicate row rather than
+ bumping the existing one. Now: `_accumulate` writes the real
+ accumulated count at save time, `start()` hydrates `_saved_hashes` +
+ a new `_persisted_ids` map from the DB, and re-sightings bump the
+ persisted row's `metadata.evidence_count` via an atomic `json_set`
+ `UPDATE` (`_bump_persisted_evidence`). `_load_persisted_patterns_from_sqlite`
+ now filters via `json_extract(metadata, '$.source')` instead of a
+ LIKE on the raw JSON string, so rows survive metadata rewrites.
+
### Added
- **Telemetry stack & install-mode identity fields** — anonymous beacon now
reports `headroom_stack` (how Headroom is invoked: `proxy`, `wrap_claude`,
diff --git a/headroom/memory/traffic_learner.py b/headroom/memory/traffic_learner.py
index 40bfff55e..0a7d10e9b 100644
--- a/headroom/memory/traffic_learner.py
+++ b/headroom/memory/traffic_learner.py
@@ -193,6 +193,10 @@ class TrafficLearner:
# Dedup: hashes of patterns already saved to DB
self._saved_hashes: set[str] = set()
+ # content_hash → memory.id for persisted rows. Lets re-sightings
+ # bump the existing row's evidence_count instead of creating a
+ # duplicate row.
+ self._persisted_ids: dict[str, str] = {}
self._dedup_window = dedup_window
# Stats
@@ -225,6 +229,9 @@ class TrafficLearner:
async def start(self) -> None:
"""Start the background save worker and flush worker."""
+ # Hydrate persisted dedup state before workers spin up so cross-session
+ # re-sightings bump existing rows instead of creating duplicates.
+ await self._hydrate_persisted_state()
if self._save_task is None or self._save_task.done():
self._save_task = asyncio.create_task(self._save_worker())
if self._flush_task is None or self._flush_task.done():
@@ -712,8 +719,12 @@ class TrafficLearner:
self._flush_dirty = True
h = pattern.content_hash
- # Already saved — skip
+ # Already saved — bump the persisted row's evidence_count rather
+ # than creating a duplicate.
if h in self._saved_hashes:
+ memory_id = self._persisted_ids.get(h)
+ if memory_id is not None:
+ await self._bump_persisted_evidence(memory_id)
return
# Accumulate evidence
@@ -736,6 +747,8 @@ class TrafficLearner:
# Remove oldest (arbitrary, set is unordered, but prevents growth)
self._saved_hashes.pop()
+ # Persist the real accumulated count, not the dataclass default.
+ pattern.evidence_count = count
try:
self._save_queue.put_nowait(pattern)
except asyncio.QueueFull:
@@ -749,7 +762,7 @@ class TrafficLearner:
if self._backend is None:
continue
- await self._backend.save_memory(
+ memory = await self._backend.save_memory(
content=pattern.content,
user_id=self._user_id,
importance=pattern.importance,
@@ -761,6 +774,10 @@ class TrafficLearner:
},
)
self._patterns_saved += 1
+ # Track id so future re-sightings bump this row.
+ memory_id = getattr(memory, "id", None)
+ if memory_id is not None:
+ self._persisted_ids[pattern.content_hash] = memory_id
logger.debug(f"Traffic learner saved pattern: {pattern.content[:80]}")
except asyncio.CancelledError:
@@ -768,6 +785,77 @@ class TrafficLearner:
except Exception as e:
logger.warning(f"Traffic learner save failed: {e}")
+ async def _hydrate_persisted_state(self) -> None:
+ """Load existing traffic_learner rows into _saved_hashes / _persisted_ids.
+
+ Runs once at start() so re-sightings across process restarts bump the
+ existing row rather than inserting a duplicate. Read-only; if the DB
+ is absent or unreadable we simply skip.
+ """
+ db_path = _resolve_backend_db_path(self._backend)
+ if db_path is None or not db_path.exists():
+ return
+
+ def _read() -> list[tuple[str, str]]:
+ uri = f"file:{db_path}?mode=ro"
+ try:
+ conn = sqlite3.connect(uri, uri=True)
+ except sqlite3.OperationalError:
+ return []
+ try:
+ rows = conn.execute(
+ "SELECT id, content FROM memories "
+ "WHERE json_extract(metadata, '$.source') = 'traffic_learner'"
+ ).fetchall()
+ except sqlite3.DatabaseError:
+ return []
+ finally:
+ try:
+ conn.close()
+ except Exception:
+ pass
+ return [(row[0], row[1] or "") for row in rows]
+
+ try:
+ rows = await asyncio.to_thread(_read)
+ except Exception as e:
+ logger.debug("Traffic learner hydrate failed: %s", e)
+ return
+
+ for memory_id, content in rows:
+ if not content:
+ continue
+ h = hashlib.sha256(content.encode()).hexdigest()[:16]
+ self._saved_hashes.add(h)
+ # If multiple rows share the same content (legacy duplicates),
+ # last-wins — we only need one id to target the bump.
+ self._persisted_ids[h] = memory_id
+
+ async def _bump_persisted_evidence(self, memory_id: str) -> None:
+ """Atomically increment a persisted row's metadata.evidence_count."""
+ db_path = _resolve_backend_db_path(self._backend)
+ if db_path is None or not db_path.exists():
+ return
+
+ def _bump() -> None:
+ conn = sqlite3.connect(str(db_path))
+ try:
+ conn.execute(
+ "UPDATE memories SET metadata = json_set("
+ "metadata, '$.evidence_count', "
+ "COALESCE(json_extract(metadata, '$.evidence_count'), 0) + 1"
+ ") WHERE id = ?",
+ (memory_id,),
+ )
+ conn.commit()
+ finally:
+ conn.close()
+
+ try:
+ await asyncio.to_thread(_bump)
+ except Exception as e:
+ logger.debug("Traffic learner evidence bump failed for %s: %s", memory_id, e)
+
# =========================================================================
# Convenience: Extract from Anthropic messages format
# =========================================================================
@@ -921,7 +1009,7 @@ def _load_persisted_patterns_from_sqlite(db_path: Path) -> list[ExtractedPattern
rows = conn.execute(
"SELECT content, metadata, entity_refs, importance "
"FROM memories "
- 'WHERE metadata LIKE \'%"source": "traffic_learner"%\''
+ "WHERE json_extract(metadata, '$.source') = 'traffic_learner'"
).fetchall()
except sqlite3.DatabaseError:
conn.close()
diff --git a/tests/test_memory/test_traffic_learner.py b/tests/test_memory/test_traffic_learner.py
index 0157861be..1f54914ad 100644
--- a/tests/test_memory/test_traffic_learner.py
+++ b/tests/test_memory/test_traffic_learner.py
@@ -555,3 +555,201 @@ class TestFlushDebounce:
# NOT call flush on every sleep tick — cap is generous.
assert call_count <= 5, f"Expected few flushes, got {call_count}"
assert call_count >= 1, "Expected at least one flush during the burst"
+
+
+# =============================================================================
+# Evidence-count persistence & re-sighting bumps
+# =============================================================================
+
+
+class _FakeBackend:
+ """Minimal LocalBackend stand-in that persists to a real SQLite file.
+
+ Provides just enough surface area for TrafficLearner: `_config.db_path`
+ (read by `_resolve_backend_db_path`) and an `async save_memory` that
+ inserts a row and returns an object with `.id`.
+ """
+
+ def __init__(self, db_path):
+ import types as _types
+
+ self._config = _types.SimpleNamespace(db_path=str(db_path))
+ self._db_path = str(db_path)
+
+ async def save_memory(
+ self,
+ *,
+ content: str,
+ user_id: str,
+ importance: float,
+ metadata: dict,
+ ):
+ import json as _json
+ import sqlite3 as _sql
+ import types as _types
+ import uuid
+
+ mid = str(uuid.uuid4())
+ conn = _sql.connect(self._db_path)
+ try:
+ conn.execute(
+ "INSERT INTO memories (id, content, metadata, entity_refs, importance) "
+ "VALUES (?,?,?,?,?)",
+ (mid, content, _json.dumps(metadata), "[]", importance),
+ )
+ conn.commit()
+ finally:
+ conn.close()
+ return _types.SimpleNamespace(id=mid)
+
+
+def _init_db(path):
+ import sqlite3 as _sql
+
+ conn = _sql.connect(path)
+ conn.execute(
+ "CREATE TABLE memories ("
+ "id TEXT PRIMARY KEY, content TEXT NOT NULL, "
+ "metadata TEXT NOT NULL DEFAULT '{}', "
+ "entity_refs TEXT NOT NULL DEFAULT '[]', "
+ "importance REAL NOT NULL DEFAULT 0.5)"
+ )
+ conn.commit()
+ conn.close()
+
+
+def _read_traffic_rows(db_path):
+ import json as _json
+ import sqlite3 as _sql
+
+ conn = _sql.connect(db_path)
+ try:
+ rows = conn.execute(
+ "SELECT id, content, metadata FROM memories "
+ "WHERE json_extract(metadata, '$.source') = 'traffic_learner'"
+ ).fetchall()
+ finally:
+ conn.close()
+ return [(r[0], r[1], _json.loads(r[2])) for r in rows]
+
+
+async def _wait_for_saved(learner: TrafficLearner, count: int, db_path) -> None:
+ """Wait until at least `count` traffic_learner rows exist in the DB."""
+ import asyncio as _asyncio
+
+ for _ in range(100):
+ if len(_read_traffic_rows(db_path)) >= count:
+ return
+ await _asyncio.sleep(0.02)
+ raise AssertionError(
+ f"Timeout waiting for {count} saved row(s); got {len(_read_traffic_rows(db_path))}"
+ )
+
+
+class TestEvidencePersistence:
+ @pytest.mark.asyncio
+ async def test_save_persists_actual_evidence_count(self, tmp_path):
+ """The count written to the DB reflects total sightings, not the default 1."""
+ db = tmp_path / "memory.db"
+ _init_db(db)
+ backend = _FakeBackend(db)
+ learner = TrafficLearner(backend=backend, min_evidence=3)
+ await learner.start()
+
+ pattern_kwargs = {
+ "category": PatternCategory.ENVIRONMENT,
+ "content": "Use /usr/bin/python3 for system scripts.",
+ "importance": 0.6,
+ }
+ for _ in range(3):
+ await learner._accumulate(ExtractedPattern(**pattern_kwargs))
+ await _wait_for_saved(learner, 1, db)
+ await learner.stop()
+
+ rows = _read_traffic_rows(db)
+ assert len(rows) == 1
+ assert rows[0][2]["evidence_count"] == 3
+
+ @pytest.mark.asyncio
+ async def test_resighting_bumps_persisted_row(self, tmp_path):
+ """Sightings after save bump the existing row instead of creating duplicates."""
+ db = tmp_path / "memory.db"
+ _init_db(db)
+ backend = _FakeBackend(db)
+ learner = TrafficLearner(backend=backend, min_evidence=2)
+ await learner.start()
+
+ def mk() -> ExtractedPattern:
+ return ExtractedPattern(
+ category=PatternCategory.PREFERENCE,
+ content="User preference: terse replies.",
+ importance=0.7,
+ )
+
+ # Two sightings → save with evidence_count=2.
+ await learner._accumulate(mk())
+ await learner._accumulate(mk())
+ await _wait_for_saved(learner, 1, db)
+
+ # Three more sightings → three bumps.
+ for _ in range(3):
+ await learner._accumulate(mk())
+ await learner.stop()
+
+ rows = _read_traffic_rows(db)
+ assert len(rows) == 1, "re-sightings must not create duplicate rows"
+ assert rows[0][2]["evidence_count"] == 5
+
+ @pytest.mark.asyncio
+ async def test_hydrate_prevents_cross_session_duplicates(self, tmp_path):
+ """A second session re-sighting an already-persisted pattern bumps, doesn't insert."""
+ import json as _json
+ import sqlite3 as _sql
+
+ db = tmp_path / "memory.db"
+ _init_db(db)
+
+ # Session 1 row pre-seeded directly.
+ seeded_content = "Command `foo` fails; use `bar` instead."
+ conn = _sql.connect(db)
+ conn.execute(
+ "INSERT INTO memories (id, content, metadata, entity_refs, importance) "
+ "VALUES (?,?,?,?,?)",
+ (
+ "seed-id",
+ seeded_content,
+ _json.dumps(
+ {
+ "source": "traffic_learner",
+ "category": "error_recovery",
+ "evidence_count": 2,
+ }
+ ),
+ "[]",
+ 0.7,
+ ),
+ )
+ conn.commit()
+ conn.close()
+
+ # Session 2: fresh learner, hydrates from DB on start().
+ backend = _FakeBackend(db)
+ learner = TrafficLearner(backend=backend, min_evidence=2)
+ await learner.start()
+
+ def mk() -> ExtractedPattern:
+ return ExtractedPattern(
+ category=PatternCategory.ERROR_RECOVERY,
+ content=seeded_content,
+ importance=0.7,
+ )
+
+ # Two sightings: both should bump the seeded row (no duplicates).
+ await learner._accumulate(mk())
+ await learner._accumulate(mk())
+ await learner.stop()
+
+ rows = _read_traffic_rows(db)
+ assert len(rows) == 1
+ assert rows[0][0] == "seed-id"
+ assert rows[0][2]["evidence_count"] == 4