Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
perf(memory): reindex FTS5 a page at a time instead of a record at a time (#3466)
trs-803e36aefb432e
diff --git a/benchmarks/memory_fts_reindex_benchmark.py b/benchmarks/memory_fts_reindex_benchmark.pynew file mode 100644index 000000000..75ba77ec4--- /dev/null+++ b/benchmarks/memory_fts_reindex_benchmark.py@@ -0,0 +1,85 @@+#!/usr/bin/env python3+"""Measure FTS5 reindex throughput: per-record commits vs batched transactions."""++from __future__ import annotations++import argparse+import asyncio+import json+import statistics+import sys+import tempfile+import time+from pathlib import Path++# Run as a script sys.path[0] is benchmarks/, so an editable install of another+# checkout would win and we would silently measure the wrong tree.+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))++from headroom.cli.memory import _REINDEX_PAGE_SIZE+from headroom.memory.adapters.fts5 import FTS5TextIndex+from headroom.memory.models import Memory+++def make_memories(count: int, content_words: int) -> list[Memory]:+ body = " ".join(f"token{n}" for n in range(content_words))+ return [+ Memory(+ id=f"mem-{index:06d}",+ content=f"{body} unique{index}",+ user_id="bench-user",+ session_id=f"session-{index % 7}",+ )+ for index in range(count)+ ]+++def benchmark(count: int, content_words: int, page_size: int, runs: int) -> dict[str, object]:+ memories = make_memories(count, content_words)+ per_record: list[float] = []+ batched: list[float] = []++ for _ in range(runs):+ # Fresh temporary database per mode per run so neither inherits the+ # other's page cache or index state.+ with tempfile.TemporaryDirectory(prefix="headroom-fts-bench-") as tmp:+ fts = FTS5TextIndex(db_path=str(Path(tmp) / "memory.db"))+ start = time.perf_counter()+ for memory in memories:+ asyncio.run(fts.index_memory(memory))+ per_record.append((time.perf_counter() - start) * 1000)++ with tempfile.TemporaryDirectory(prefix="headroom-fts-bench-") as tmp:+ fts = FTS5TextIndex(db_path=str(Path(tmp) / "memory.db"))+ start = time.perf_counter()+ for offset in range(0, len(memories), page_size):+ asyncio.run(fts.index_batch_memories(memories[offset : offset + page_size]))+ batched.append((time.perf_counter() - start) * 1000)++ return {+ "records": count,+ "content_words": content_words,+ "page_size": page_size,+ "runs": runs,+ "per_record_ms": statistics.median(per_record),+ "batched_ms": statistics.median(batched),+ "speedup": round(statistics.median(per_record) / statistics.median(batched), 1),+ }+++def main() -> None:+ parser = argparse.ArgumentParser()+ parser.add_argument("--records", type=int, nargs="+", default=[1000])+ parser.add_argument("--content-words", type=int, default=40)+ parser.add_argument("--page-size", type=int, default=_REINDEX_PAGE_SIZE)+ parser.add_argument("--runs", type=int, default=3)+ args = parser.parse_args()++ results = [+ benchmark(count, args.content_words, args.page_size, args.runs) for count in args.records+ ]+ print(json.dumps(results, indent=2))+++if __name__ == "__main__":+ main()diff --git a/headroom/cli/memory.py b/headroom/cli/memory.pyindex 9a930233a..e981f13ce 100644--- a/headroom/cli/memory.py+++ b/headroom/cli/memory.py@@ -1212,14 +1212,31 @@ def reindex_memories(ctx: click.Context, db_path: str) -> None:print_error(f"Failed to clear FTS5 index: {exc}")sys.exit(1)+ # Index a page at a time. One transaction per page instead of per record+ # is ~100x faster, while still bounding how much the journal grows and+ # how much a single failure can take with it. On a failed page, retry+ # that page record by record so valid records still land and the broken+ # one is still named -- the index was already cleared above, so a page+ # that only rolled back would leave a silent hole in search coverage.fts_indexed = 0- for mem in memories:+ for start in range(0, len(memories), _REINDEX_PAGE_SIZE):+ page = memories[start : start + _REINDEX_PAGE_SIZE]try:- asyncio.run(fts.index_memory(mem))- fts_indexed += 1+ fts_indexed += asyncio.run(fts.index_batch_memories(page))+ continueexcept Exception as exc:- print_warning(f"FTS5: failed to index {mem.id[:8]}: {exc}")- ok = False+ # Not a failure on its own: nothing from this page committed,+ # and the per-record retry below still gets to index it. Exit+ # status stays tied to whether records actually got indexed.+ print_warning(f"FTS5: page rolled back ({exc}); retrying record by record")++ for mem in page:+ try:+ asyncio.run(fts.index_memory(mem))+ fts_indexed += 1+ except Exception as exc:+ print_warning(f"FTS5: failed to index {mem.id[:8]}: {exc}")+ ok = False# --- Vector: remove orphaned entries (requires sqlite-vec) ---vector_db = db.parent / f"{db.stem}_vectors.db"diff --git a/tests/test_cli_memory_index_sync.py b/tests/test_cli_memory_index_sync.pyindex b4578d05f..16b50503e 100644--- a/tests/test_cli_memory_index_sync.py+++ b/tests/test_cli_memory_index_sync.py@@ -506,3 +506,154 @@ def test_purge_command_exits_nonzero_when_index_sync_fails(tmp_path, monkeypatchassert result.exit_code == 1assert asyncio.run(store.get(memory.id)) is Noneassert "index sync incomplete" in result.output+++# ---------------------------------------------------------------------------+# reindex batching+# ---------------------------------------------------------------------------+++def _always_raises(message: str):+ async def _raise(*_args, **_kwargs):+ raise RuntimeError(message)++ return _raise+++def _fts_rows(db_path: Path) -> dict[str, tuple[str, str, str, str]]:+ with sqlite3.connect(str(db_path)) as conn:+ rows = conn.execute(+ "SELECT memory_id, content, user_id, session_id, category FROM memory_fts"+ ).fetchall()+ return {r[0]: (r[1], r[2], r[3], r[4]) for r in rows}+++def _seeded_store(db_path: Path, count: int) -> list[Memory]:+ store = SQLiteMemoryStore(str(db_path))+ memories = []+ for i in range(count):+ mem = _make_memory(f"id-{i}", f"content {i}")+ mem.session_id = f"session-{i % 3}"+ memories.append(mem)+ asyncio.run(store.save(mem))+ return memories+++def test_reindex_batches_pages_instead_of_indexing_record_by_record(tmp_path, monkeypatch):+ """The batch API does the work; the per-record path is only a fallback."""+ db_path = tmp_path / "memory.db"+ _seeded_store(db_path, 5)+ monkeypatch.setattr(memory_cli, "_REINDEX_PAGE_SIZE", 2)++ batch_sizes: list[int] = []+ real_batch = FTS5TextIndex.index_batch_memories++ async def spy_batch(self, memories):+ batch_sizes.append(len(memories))+ return await real_batch(self, memories)++ monkeypatch.setattr(FTS5TextIndex, "index_batch_memories", spy_batch)+ monkeypatch.setattr(+ FTS5TextIndex,+ "index_memory",+ lambda *_a, **_k: pytest.fail("reindex fell back to per-record indexing"),+ )++ result = CliRunner().invoke(main, ["memory", "reindex", "--db-path", str(db_path)])++ assert result.exit_code == 0, result.output+ assert batch_sizes == [2, 2, 1]+ assert "Re-indexed 5/5 memories" in result.output+++def test_reindex_batched_output_matches_per_record_output(tmp_path, monkeypatch):+ """Every indexed field and FTS query result is identical either way."""+ batched_db = tmp_path / "batched.db"+ per_record_db = tmp_path / "per_record.db"+ for db_path in (batched_db, per_record_db):+ _seeded_store(db_path, 7)+ monkeypatch.setattr(memory_cli, "_REINDEX_PAGE_SIZE", 3)++ batched = CliRunner().invoke(main, ["memory", "reindex", "--db-path", str(batched_db)])+ assert batched.exit_code == 0, batched.output++ # Force the per-record path by making every batch fail.+ monkeypatch.setattr(+ FTS5TextIndex,+ "index_batch_memories",+ _always_raises("batching disabled for parity check"),+ )+ per_record = CliRunner().invoke(main, ["memory", "reindex", "--db-path", str(per_record_db)])+ assert per_record.exit_code == 0, per_record.output++ assert _fts_rows(batched_db) == _fts_rows(per_record_db)+ batched_hits = FTS5TextIndex(db_path=str(batched_db)).search("content", k=10)+ per_record_hits = FTS5TextIndex(db_path=str(per_record_db)).search("content", k=10)+ assert [h.memory_id for h in batched_hits] == [h.memory_id for h in per_record_hits]+++def test_reindex_empty_store_indexes_nothing_and_succeeds(tmp_path):+ db_path = tmp_path / "memory.db"+ SQLiteMemoryStore(str(db_path))++ result = CliRunner().invoke(main, ["memory", "reindex", "--db-path", str(db_path)])++ assert result.exit_code == 0, result.output+ assert _fts_ids(db_path) == set()+ assert "Re-indexed 0/0 memories" in result.output+++def test_reindex_indexes_more_records_than_one_page(tmp_path, monkeypatch):+ db_path = tmp_path / "memory.db"+ memories = _seeded_store(db_path, 7)+ monkeypatch.setattr(memory_cli, "_REINDEX_PAGE_SIZE", 3)++ result = CliRunner().invoke(main, ["memory", "reindex", "--db-path", str(db_path)])++ assert result.exit_code == 0, result.output+ assert _fts_ids(db_path) == {m.id for m in memories}+ assert "Re-indexed 7/7 memories" in result.output+++def test_reindex_failed_batch_still_indexes_valid_records_in_that_page(tmp_path, monkeypatch):+ """A rolled-back page must not leave a hole: valid records still land."""+ db_path = tmp_path / "memory.db"+ memories = _seeded_store(db_path, 4)+ monkeypatch.setattr(memory_cli, "_REINDEX_PAGE_SIZE", 4)+ monkeypatch.setattr(+ FTS5TextIndex, "index_batch_memories", _always_raises("simulated batch failure")+ )++ result = CliRunner().invoke(main, ["memory", "reindex", "--db-path", str(db_path)])++ assert result.exit_code == 0, result.output+ assert _fts_ids(db_path) == {m.id for m in memories}+ assert "page rolled back" in result.output+ assert "Re-indexed 4/4 memories" in result.output+++def test_reindex_failed_record_is_named_and_exits_nonzero(tmp_path, monkeypatch):+ """Per-record diagnosability survives batching."""+ db_path = tmp_path / "memory.db"+ memories = _seeded_store(db_path, 3)+ monkeypatch.setattr(memory_cli, "_REINDEX_PAGE_SIZE", 3)+ monkeypatch.setattr(+ FTS5TextIndex, "index_batch_memories", _always_raises("simulated batch failure")+ )++ broken = memories[1].id+ real_index_memory = FTS5TextIndex.index_memory++ async def index_or_fail(self, memory):+ if memory.id == broken:+ raise RuntimeError("simulated record failure")+ await real_index_memory(self, memory)++ monkeypatch.setattr(FTS5TextIndex, "index_memory", index_or_fail)++ result = CliRunner().invoke(main, ["memory", "reindex", "--db-path", str(db_path)])++ assert result.exit_code == 1+ assert f"failed to index {broken[:8]}" in result.output+ assert _fts_ids(db_path) == {memories[0].id, memories[2].id}+ assert "Re-indexed 2/3 memories" in result.output