Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

perf(memory): reindex FTS5 a page at a time instead of a record at a time (#3466)

trs-803e36aefb432e
diff --git a/benchmarks/memory_fts_reindex_benchmark.py b/benchmarks/memory_fts_reindex_benchmark.py
new file mode 100644
index 000000000..75ba77ec4
--- /dev/null
+++ b/benchmarks/memory_fts_reindex_benchmark.py
@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+"""Measure FTS5 reindex throughput: per-record commits vs batched transactions."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import statistics
+import sys
+import tempfile
+import time
+from pathlib import Path
+
+# Run as a script sys.path[0] is benchmarks/, so an editable install of another
+# checkout would win and we would silently measure the wrong tree.
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from headroom.cli.memory import _REINDEX_PAGE_SIZE
+from headroom.memory.adapters.fts5 import FTS5TextIndex
+from headroom.memory.models import Memory
+
+
+def make_memories(count: int, content_words: int) -> list[Memory]:
+ body = " ".join(f"token{n}" for n in range(content_words))
+ return [
+ Memory(
+ id=f"mem-{index:06d}",
+ content=f"{body} unique{index}",
+ user_id="bench-user",
+ session_id=f"session-{index % 7}",
+ )
+ for index in range(count)
+ ]
+
+
+def benchmark(count: int, content_words: int, page_size: int, runs: int) -> dict[str, object]:
+ memories = make_memories(count, content_words)
+ per_record: list[float] = []
+ batched: list[float] = []
+
+ for _ in range(runs):
+ # Fresh temporary database per mode per run so neither inherits the
+ # other's page cache or index state.
+ with tempfile.TemporaryDirectory(prefix="headroom-fts-bench-") as tmp:
+ fts = FTS5TextIndex(db_path=str(Path(tmp) / "memory.db"))
+ start = time.perf_counter()
+ for memory in memories:
+ asyncio.run(fts.index_memory(memory))
+ per_record.append((time.perf_counter() - start) * 1000)
+
+ with tempfile.TemporaryDirectory(prefix="headroom-fts-bench-") as tmp:
+ fts = FTS5TextIndex(db_path=str(Path(tmp) / "memory.db"))
+ start = time.perf_counter()
+ for offset in range(0, len(memories), page_size):
+ asyncio.run(fts.index_batch_memories(memories[offset : offset + page_size]))
+ batched.append((time.perf_counter() - start) * 1000)
+
+ return {
+ "records": count,
+ "content_words": content_words,
+ "page_size": page_size,
+ "runs": runs,
+ "per_record_ms": statistics.median(per_record),
+ "batched_ms": statistics.median(batched),
+ "speedup": round(statistics.median(per_record) / statistics.median(batched), 1),
+ }
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--records", type=int, nargs="+", default=[1000])
+ parser.add_argument("--content-words", type=int, default=40)
+ parser.add_argument("--page-size", type=int, default=_REINDEX_PAGE_SIZE)
+ parser.add_argument("--runs", type=int, default=3)
+ args = parser.parse_args()
+
+ results = [
+ benchmark(count, args.content_words, args.page_size, args.runs) for count in args.records
+ ]
+ print(json.dumps(results, indent=2))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/headroom/cli/memory.py b/headroom/cli/memory.py
index 9a930233a..e981f13ce 100644
--- a/headroom/cli/memory.py
+++ b/headroom/cli/memory.py
@@ -1212,14 +1212,31 @@ def reindex_memories(ctx: click.Context, db_path: str) -> None:
print_error(f"Failed to clear FTS5 index: {exc}")
sys.exit(1)
+ # Index a page at a time. One transaction per page instead of per record
+ # is ~100x faster, while still bounding how much the journal grows and
+ # how much a single failure can take with it. On a failed page, retry
+ # that page record by record so valid records still land and the broken
+ # one is still named -- the index was already cleared above, so a page
+ # that only rolled back would leave a silent hole in search coverage.
fts_indexed = 0
- for mem in memories:
+ for start in range(0, len(memories), _REINDEX_PAGE_SIZE):
+ page = memories[start : start + _REINDEX_PAGE_SIZE]
try:
- asyncio.run(fts.index_memory(mem))
- fts_indexed += 1
+ fts_indexed += asyncio.run(fts.index_batch_memories(page))
+ continue
except Exception as exc:
- print_warning(f"FTS5: failed to index {mem.id[:8]}: {exc}")
- ok = False
+ # Not a failure on its own: nothing from this page committed,
+ # and the per-record retry below still gets to index it. Exit
+ # status stays tied to whether records actually got indexed.
+ print_warning(f"FTS5: page rolled back ({exc}); retrying record by record")
+
+ for mem in page:
+ try:
+ asyncio.run(fts.index_memory(mem))
+ fts_indexed += 1
+ except Exception as exc:
+ print_warning(f"FTS5: failed to index {mem.id[:8]}: {exc}")
+ ok = False
# --- Vector: remove orphaned entries (requires sqlite-vec) ---
vector_db = db.parent / f"{db.stem}_vectors.db"
diff --git a/tests/test_cli_memory_index_sync.py b/tests/test_cli_memory_index_sync.py
index b4578d05f..16b50503e 100644
--- a/tests/test_cli_memory_index_sync.py
+++ b/tests/test_cli_memory_index_sync.py
@@ -506,3 +506,154 @@ def test_purge_command_exits_nonzero_when_index_sync_fails(tmp_path, monkeypatch
assert result.exit_code == 1
assert asyncio.run(store.get(memory.id)) is None
assert "index sync incomplete" in result.output
+
+
+# ---------------------------------------------------------------------------
+# reindex batching
+# ---------------------------------------------------------------------------
+
+
+def _always_raises(message: str):
+ async def _raise(*_args, **_kwargs):
+ raise RuntimeError(message)
+
+ return _raise
+
+
+def _fts_rows(db_path: Path) -> dict[str, tuple[str, str, str, str]]:
+ with sqlite3.connect(str(db_path)) as conn:
+ rows = conn.execute(
+ "SELECT memory_id, content, user_id, session_id, category FROM memory_fts"
+ ).fetchall()
+ return {r[0]: (r[1], r[2], r[3], r[4]) for r in rows}
+
+
+def _seeded_store(db_path: Path, count: int) -> list[Memory]:
+ store = SQLiteMemoryStore(str(db_path))
+ memories = []
+ for i in range(count):
+ mem = _make_memory(f"id-{i}", f"content {i}")
+ mem.session_id = f"session-{i % 3}"
+ memories.append(mem)
+ asyncio.run(store.save(mem))
+ return memories
+
+
+def test_reindex_batches_pages_instead_of_indexing_record_by_record(tmp_path, monkeypatch):
+ """The batch API does the work; the per-record path is only a fallback."""
+ db_path = tmp_path / "memory.db"
+ _seeded_store(db_path, 5)
+ monkeypatch.setattr(memory_cli, "_REINDEX_PAGE_SIZE", 2)
+
+ batch_sizes: list[int] = []
+ real_batch = FTS5TextIndex.index_batch_memories
+
+ async def spy_batch(self, memories):
+ batch_sizes.append(len(memories))
+ return await real_batch(self, memories)
+
+ monkeypatch.setattr(FTS5TextIndex, "index_batch_memories", spy_batch)
+ monkeypatch.setattr(
+ FTS5TextIndex,
+ "index_memory",
+ lambda *_a, **_k: pytest.fail("reindex fell back to per-record indexing"),
+ )
+
+ result = CliRunner().invoke(main, ["memory", "reindex", "--db-path", str(db_path)])
+
+ assert result.exit_code == 0, result.output
+ assert batch_sizes == [2, 2, 1]
+ assert "Re-indexed 5/5 memories" in result.output
+
+
+def test_reindex_batched_output_matches_per_record_output(tmp_path, monkeypatch):
+ """Every indexed field and FTS query result is identical either way."""
+ batched_db = tmp_path / "batched.db"
+ per_record_db = tmp_path / "per_record.db"
+ for db_path in (batched_db, per_record_db):
+ _seeded_store(db_path, 7)
+ monkeypatch.setattr(memory_cli, "_REINDEX_PAGE_SIZE", 3)
+
+ batched = CliRunner().invoke(main, ["memory", "reindex", "--db-path", str(batched_db)])
+ assert batched.exit_code == 0, batched.output
+
+ # Force the per-record path by making every batch fail.
+ monkeypatch.setattr(
+ FTS5TextIndex,
+ "index_batch_memories",
+ _always_raises("batching disabled for parity check"),
+ )
+ per_record = CliRunner().invoke(main, ["memory", "reindex", "--db-path", str(per_record_db)])
+ assert per_record.exit_code == 0, per_record.output
+
+ assert _fts_rows(batched_db) == _fts_rows(per_record_db)
+ batched_hits = FTS5TextIndex(db_path=str(batched_db)).search("content", k=10)
+ per_record_hits = FTS5TextIndex(db_path=str(per_record_db)).search("content", k=10)
+ assert [h.memory_id for h in batched_hits] == [h.memory_id for h in per_record_hits]
+
+
+def test_reindex_empty_store_indexes_nothing_and_succeeds(tmp_path):
+ db_path = tmp_path / "memory.db"
+ SQLiteMemoryStore(str(db_path))
+
+ result = CliRunner().invoke(main, ["memory", "reindex", "--db-path", str(db_path)])
+
+ assert result.exit_code == 0, result.output
+ assert _fts_ids(db_path) == set()
+ assert "Re-indexed 0/0 memories" in result.output
+
+
+def test_reindex_indexes_more_records_than_one_page(tmp_path, monkeypatch):
+ db_path = tmp_path / "memory.db"
+ memories = _seeded_store(db_path, 7)
+ monkeypatch.setattr(memory_cli, "_REINDEX_PAGE_SIZE", 3)
+
+ result = CliRunner().invoke(main, ["memory", "reindex", "--db-path", str(db_path)])
+
+ assert result.exit_code == 0, result.output
+ assert _fts_ids(db_path) == {m.id for m in memories}
+ assert "Re-indexed 7/7 memories" in result.output
+
+
+def test_reindex_failed_batch_still_indexes_valid_records_in_that_page(tmp_path, monkeypatch):
+ """A rolled-back page must not leave a hole: valid records still land."""
+ db_path = tmp_path / "memory.db"
+ memories = _seeded_store(db_path, 4)
+ monkeypatch.setattr(memory_cli, "_REINDEX_PAGE_SIZE", 4)
+ monkeypatch.setattr(
+ FTS5TextIndex, "index_batch_memories", _always_raises("simulated batch failure")
+ )
+
+ result = CliRunner().invoke(main, ["memory", "reindex", "--db-path", str(db_path)])
+
+ assert result.exit_code == 0, result.output
+ assert _fts_ids(db_path) == {m.id for m in memories}
+ assert "page rolled back" in result.output
+ assert "Re-indexed 4/4 memories" in result.output
+
+
+def test_reindex_failed_record_is_named_and_exits_nonzero(tmp_path, monkeypatch):
+ """Per-record diagnosability survives batching."""
+ db_path = tmp_path / "memory.db"
+ memories = _seeded_store(db_path, 3)
+ monkeypatch.setattr(memory_cli, "_REINDEX_PAGE_SIZE", 3)
+ monkeypatch.setattr(
+ FTS5TextIndex, "index_batch_memories", _always_raises("simulated batch failure")
+ )
+
+ broken = memories[1].id
+ real_index_memory = FTS5TextIndex.index_memory
+
+ async def index_or_fail(self, memory):
+ if memory.id == broken:
+ raise RuntimeError("simulated record failure")
+ await real_index_memory(self, memory)
+
+ monkeypatch.setattr(FTS5TextIndex, "index_memory", index_or_fail)
+
+ result = CliRunner().invoke(main, ["memory", "reindex", "--db-path", str(db_path)])
+
+ assert result.exit_code == 1
+ assert f"failed to index {broken[:8]}" in result.output
+ assert _fts_ids(db_path) == {memories[0].id, memories[2].id}
+ assert "Re-indexed 2/3 memories" in result.output