Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(learn): persist real evidence_count and bump on re-sighting
Garm3e290b734b92
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 1ee141352..f64ee5d4d 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -7,6 +7,38 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0## [Unreleased]+### Added+- **Live flush of traffic-learned patterns to CLAUDE.md / MEMORY.md** — the+ `TrafficLearner` now writes to agent-native context files continuously+ during proxy operation, not just at shutdown. A new dirty-flag debounced+ `_flush_worker` (10s window, `FLUSH_DEBOUNCE_SECONDS`) calls+ `flush_to_file()` whenever `_accumulate()` marks the learner dirty, so+ patterns surface in `CLAUDE.md` / `MEMORY.md` near real-time. Flushes+ read both persisted rows (via `_load_persisted_patterns_from_sqlite`)+ and the in-memory accumulator, bucket patterns by project via the learn+ plugin registry (`plugin.discover_projects()` + longest-path anchoring+ in `_project_for_pattern`), and route by `PatternCategory` to the+ correct file (`_patterns_to_recommendations` ++ `_CATEGORY_TO_TARGET`). Live flushes require `evidence_count >= 2`;+ the shutdown flush accepts single-evidence rows.++### Fixed+- **Traffic-learner evidence count stuck at 1; duplicate DB rows across+ restarts.** `_accumulate` queued patterns with the default+ `ExtractedPattern.evidence_count = 1` regardless of how many times the+ pattern was actually seen, so every persisted row landed at `1` and+ never crossed the live-flush gate (`evidence_count >= 2`). Worse, once+ a pattern was in `_saved_hashes` it was early-returned on every+ re-sighting, and `_saved_hashes` reset on process restart — so a second+ sighting in a later session inserted a duplicate row rather than+ bumping the existing one. Now: `_accumulate` writes the real+ accumulated count at save time, `start()` hydrates `_saved_hashes` ++ a new `_persisted_ids` map from the DB, and re-sightings bump the+ persisted row's `metadata.evidence_count` via an atomic `json_set`+ `UPDATE` (`_bump_persisted_evidence`). `_load_persisted_patterns_from_sqlite`+ now filters via `json_extract(metadata, '$.source')` instead of a+ LIKE on the raw JSON string, so rows survive metadata rewrites.+### Added- **Telemetry stack & install-mode identity fields** — anonymous beacon nowreports `headroom_stack` (how Headroom is invoked: `proxy`, `wrap_claude`,diff --git a/headroom/memory/traffic_learner.py b/headroom/memory/traffic_learner.pyindex 40bfff55e..0a7d10e9b 100644--- a/headroom/memory/traffic_learner.py+++ b/headroom/memory/traffic_learner.py@@ -193,6 +193,10 @@ class TrafficLearner:# Dedup: hashes of patterns already saved to DBself._saved_hashes: set[str] = set()+ # content_hash → memory.id for persisted rows. Lets re-sightings+ # bump the existing row's evidence_count instead of creating a+ # duplicate row.+ self._persisted_ids: dict[str, str] = {}self._dedup_window = dedup_window# Stats@@ -225,6 +229,9 @@ class TrafficLearner:async def start(self) -> None:"""Start the background save worker and flush worker."""+ # Hydrate persisted dedup state before workers spin up so cross-session+ # re-sightings bump existing rows instead of creating duplicates.+ await self._hydrate_persisted_state()if self._save_task is None or self._save_task.done():self._save_task = asyncio.create_task(self._save_worker())if self._flush_task is None or self._flush_task.done():@@ -712,8 +719,12 @@ class TrafficLearner:self._flush_dirty = Trueh = pattern.content_hash- # Already saved — skip+ # Already saved — bump the persisted row's evidence_count rather+ # than creating a duplicate.if h in self._saved_hashes:+ memory_id = self._persisted_ids.get(h)+ if memory_id is not None:+ await self._bump_persisted_evidence(memory_id)return# Accumulate evidence@@ -736,6 +747,8 @@ class TrafficLearner:# Remove oldest (arbitrary, set is unordered, but prevents growth)self._saved_hashes.pop()+ # Persist the real accumulated count, not the dataclass default.+ pattern.evidence_count = counttry:self._save_queue.put_nowait(pattern)except asyncio.QueueFull:@@ -749,7 +762,7 @@ class TrafficLearner:if self._backend is None:continue- await self._backend.save_memory(+ memory = await self._backend.save_memory(content=pattern.content,user_id=self._user_id,importance=pattern.importance,@@ -761,6 +774,10 @@ class TrafficLearner:},)self._patterns_saved += 1+ # Track id so future re-sightings bump this row.+ memory_id = getattr(memory, "id", None)+ if memory_id is not None:+ self._persisted_ids[pattern.content_hash] = memory_idlogger.debug(f"Traffic learner saved pattern: {pattern.content[:80]}")except asyncio.CancelledError:@@ -768,6 +785,77 @@ class TrafficLearner:except Exception as e:logger.warning(f"Traffic learner save failed: {e}")+ async def _hydrate_persisted_state(self) -> None:+ """Load existing traffic_learner rows into _saved_hashes / _persisted_ids.++ Runs once at start() so re-sightings across process restarts bump the+ existing row rather than inserting a duplicate. Read-only; if the DB+ is absent or unreadable we simply skip.+ """+ db_path = _resolve_backend_db_path(self._backend)+ if db_path is None or not db_path.exists():+ return++ def _read() -> list[tuple[str, str]]:+ uri = f"file:{db_path}?mode=ro"+ try:+ conn = sqlite3.connect(uri, uri=True)+ except sqlite3.OperationalError:+ return []+ try:+ rows = conn.execute(+ "SELECT id, content FROM memories "+ "WHERE json_extract(metadata, '$.source') = 'traffic_learner'"+ ).fetchall()+ except sqlite3.DatabaseError:+ return []+ finally:+ try:+ conn.close()+ except Exception:+ pass+ return [(row[0], row[1] or "") for row in rows]++ try:+ rows = await asyncio.to_thread(_read)+ except Exception as e:+ logger.debug("Traffic learner hydrate failed: %s", e)+ return++ for memory_id, content in rows:+ if not content:+ continue+ h = hashlib.sha256(content.encode()).hexdigest()[:16]+ self._saved_hashes.add(h)+ # If multiple rows share the same content (legacy duplicates),+ # last-wins — we only need one id to target the bump.+ self._persisted_ids[h] = memory_id++ async def _bump_persisted_evidence(self, memory_id: str) -> None:+ """Atomically increment a persisted row's metadata.evidence_count."""+ db_path = _resolve_backend_db_path(self._backend)+ if db_path is None or not db_path.exists():+ return++ def _bump() -> None:+ conn = sqlite3.connect(str(db_path))+ try:+ conn.execute(+ "UPDATE memories SET metadata = json_set("+ "metadata, '$.evidence_count', "+ "COALESCE(json_extract(metadata, '$.evidence_count'), 0) + 1"+ ") WHERE id = ?",+ (memory_id,),+ )+ conn.commit()+ finally:+ conn.close()++ try:+ await asyncio.to_thread(_bump)+ except Exception as e:+ logger.debug("Traffic learner evidence bump failed for %s: %s", memory_id, e)+# =========================================================================# Convenience: Extract from Anthropic messages format# =========================================================================@@ -921,7 +1009,7 @@ def _load_persisted_patterns_from_sqlite(db_path: Path) -> list[ExtractedPatternrows = conn.execute("SELECT content, metadata, entity_refs, importance ""FROM memories "- 'WHERE metadata LIKE \'%"source": "traffic_learner"%\''+ "WHERE json_extract(metadata, '$.source') = 'traffic_learner'").fetchall()except sqlite3.DatabaseError:conn.close()diff --git a/tests/test_memory/test_traffic_learner.py b/tests/test_memory/test_traffic_learner.pyindex 0157861be..1f54914ad 100644--- a/tests/test_memory/test_traffic_learner.py+++ b/tests/test_memory/test_traffic_learner.py@@ -555,3 +555,201 @@ class TestFlushDebounce:# NOT call flush on every sleep tick — cap is generous.assert call_count <= 5, f"Expected few flushes, got {call_count}"assert call_count >= 1, "Expected at least one flush during the burst"+++# =============================================================================+# Evidence-count persistence & re-sighting bumps+# =============================================================================+++class _FakeBackend:+ """Minimal LocalBackend stand-in that persists to a real SQLite file.++ Provides just enough surface area for TrafficLearner: `_config.db_path`+ (read by `_resolve_backend_db_path`) and an `async save_memory` that+ inserts a row and returns an object with `.id`.+ """++ def __init__(self, db_path):+ import types as _types++ self._config = _types.SimpleNamespace(db_path=str(db_path))+ self._db_path = str(db_path)++ async def save_memory(+ self,+ *,+ content: str,+ user_id: str,+ importance: float,+ metadata: dict,+ ):+ import json as _json+ import sqlite3 as _sql+ import types as _types+ import uuid++ mid = str(uuid.uuid4())+ conn = _sql.connect(self._db_path)+ try:+ conn.execute(+ "INSERT INTO memories (id, content, metadata, entity_refs, importance) "+ "VALUES (?,?,?,?,?)",+ (mid, content, _json.dumps(metadata), "[]", importance),+ )+ conn.commit()+ finally:+ conn.close()+ return _types.SimpleNamespace(id=mid)+++def _init_db(path):+ import sqlite3 as _sql++ conn = _sql.connect(path)+ conn.execute(+ "CREATE TABLE memories ("+ "id TEXT PRIMARY KEY, content TEXT NOT NULL, "+ "metadata TEXT NOT NULL DEFAULT '{}', "+ "entity_refs TEXT NOT NULL DEFAULT '[]', "+ "importance REAL NOT NULL DEFAULT 0.5)"+ )+ conn.commit()+ conn.close()+++def _read_traffic_rows(db_path):+ import json as _json+ import sqlite3 as _sql++ conn = _sql.connect(db_path)+ try:+ rows = conn.execute(+ "SELECT id, content, metadata FROM memories "+ "WHERE json_extract(metadata, '$.source') = 'traffic_learner'"+ ).fetchall()+ finally:+ conn.close()+ return [(r[0], r[1], _json.loads(r[2])) for r in rows]+++async def _wait_for_saved(learner: TrafficLearner, count: int, db_path) -> None:+ """Wait until at least `count` traffic_learner rows exist in the DB."""+ import asyncio as _asyncio++ for _ in range(100):+ if len(_read_traffic_rows(db_path)) >= count:+ return+ await _asyncio.sleep(0.02)+ raise AssertionError(+ f"Timeout waiting for {count} saved row(s); got {len(_read_traffic_rows(db_path))}"+ )+++class TestEvidencePersistence:+ @pytest.mark.asyncio+ async def test_save_persists_actual_evidence_count(self, tmp_path):+ """The count written to the DB reflects total sightings, not the default 1."""+ db = tmp_path / "memory.db"+ _init_db(db)+ backend = _FakeBackend(db)+ learner = TrafficLearner(backend=backend, min_evidence=3)+ await learner.start()++ pattern_kwargs = {+ "category": PatternCategory.ENVIRONMENT,+ "content": "Use /usr/bin/python3 for system scripts.",+ "importance": 0.6,+ }+ for _ in range(3):+ await learner._accumulate(ExtractedPattern(**pattern_kwargs))+ await _wait_for_saved(learner, 1, db)+ await learner.stop()++ rows = _read_traffic_rows(db)+ assert len(rows) == 1+ assert rows[0][2]["evidence_count"] == 3++ @pytest.mark.asyncio+ async def test_resighting_bumps_persisted_row(self, tmp_path):+ """Sightings after save bump the existing row instead of creating duplicates."""+ db = tmp_path / "memory.db"+ _init_db(db)+ backend = _FakeBackend(db)+ learner = TrafficLearner(backend=backend, min_evidence=2)+ await learner.start()++ def mk() -> ExtractedPattern:+ return ExtractedPattern(+ category=PatternCategory.PREFERENCE,+ content="User preference: terse replies.",+ importance=0.7,+ )++ # Two sightings → save with evidence_count=2.+ await learner._accumulate(mk())+ await learner._accumulate(mk())+ await _wait_for_saved(learner, 1, db)++ # Three more sightings → three bumps.+ for _ in range(3):+ await learner._accumulate(mk())+ await learner.stop()++ rows = _read_traffic_rows(db)+ assert len(rows) == 1, "re-sightings must not create duplicate rows"+ assert rows[0][2]["evidence_count"] == 5++ @pytest.mark.asyncio+ async def test_hydrate_prevents_cross_session_duplicates(self, tmp_path):+ """A second session re-sighting an already-persisted pattern bumps, doesn't insert."""+ import json as _json+ import sqlite3 as _sql++ db = tmp_path / "memory.db"+ _init_db(db)++ # Session 1 row pre-seeded directly.+ seeded_content = "Command `foo` fails; use `bar` instead."+ conn = _sql.connect(db)+ conn.execute(+ "INSERT INTO memories (id, content, metadata, entity_refs, importance) "+ "VALUES (?,?,?,?,?)",+ (+ "seed-id",+ seeded_content,+ _json.dumps(+ {+ "source": "traffic_learner",+ "category": "error_recovery",+ "evidence_count": 2,+ }+ ),+ "[]",+ 0.7,+ ),+ )+ conn.commit()+ conn.close()++ # Session 2: fresh learner, hydrates from DB on start().+ backend = _FakeBackend(db)+ learner = TrafficLearner(backend=backend, min_evidence=2)+ await learner.start()++ def mk() -> ExtractedPattern:+ return ExtractedPattern(+ category=PatternCategory.ERROR_RECOVERY,+ content=seeded_content,+ importance=0.7,+ )++ # Two sightings: both should bump the seeded row (no duplicates).+ await learner._accumulate(mk())+ await learner._accumulate(mk())+ await learner.stop()++ rows = _read_traffic_rows(db)+ assert len(rows) == 1+ assert rows[0][0] == "seed-id"+ assert rows[0][2]["evidence_count"] == 4