The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(sessions): optimize-storage paces for a writer that opens state.db mid-run; failed runs exit 1
Tekniumd569365e87e2
diff --git a/hermes_cli/sessions_cmd.py b/hermes_cli/sessions_cmd.pyindex bbe79a1baeb..4881375d607 100644--- a/hermes_cli/sessions_cmd.py+++ b/hermes_cli/sessions_cmd.py@@ -980,15 +980,20 @@ def _cmd_clean_markers(db, args):print(f"✓ Cleared {report['rows_affected']} row(s).")+_BACKFILL_LABELS = {"backfill": "Rebuilding index", "backfill_cjk": "Building CJK index"}+_COUNTED_PHASES = frozenset({*_BACKFILL_LABELS, "teardown"})++def _optimize_progress_line(info) -> str:phase, pct = info.get("phase"), f"{info.get('percent', 0):3d}%"counts = f"({info.get('indexed', 0):,}/{info.get('total', 0):,}"- if phase == "backfill":- return f" Rebuilding index: {pct} {counts})"+ paced = " — paced: another process is using the database" if info.get("paced") else ""+ if phase in _BACKFILL_LABELS:+ return f" {_BACKFILL_LABELS[phase]}: {pct} {counts} messages){paced}"if phase == "teardown":table = str(info.get("table", "")).removeprefix("fts_v22_trash_")return (f" Reclaiming old index: table {info.get('table_index', 0)}/{info.get('table_count', 0)} "- f"({table}) {pct} {counts} rows)")+ f"({table}) {pct} {counts} rows){paced}")elapsed = int(info.get("elapsed", 0))return (f" Compacting database (VACUUM): {elapsed // 60}m {elapsed % 60:02d}s elapsed, "f"{info.get('size_bytes', 0) / 2**20:,.1f} MB → ~{info.get('estimated_bytes', 0) / 2**20:,.1f} MB")@@ -1008,7 +1013,7 @@ class _OptimizeProgress:# The phase or table moved on, so the previous one ran to completion.self.close(completed=True)self._key = key- if phase == "done" or (phase in ("backfill", "teardown") and not info.get("total") and not info.get("table")):+ if phase == "done" or (phase in _COUNTED_PHASES and not info.get("total") and not info.get("table")):returnself._last = infoself._draw(_optimize_progress_line(info))@@ -1021,19 +1026,13 @@ class _OptimizeProgress:def close(self, completed: bool = False) -> None:last = self._last- if completed and last and last.get("phase") in ("backfill", "teardown"):+ if completed and last and last.get("phase") in _COUNTED_PHASES:self._draw(_optimize_progress_line({**last, "percent": 100, "indexed": last.get("total", 0)}))if self._line:print(flush=True)self._last, self._line = None, ""-def _store_has_foreign_holder(db_path) -> bool:- from hermes_state_holders import held_store_refusal-- return held_store_refusal(db_path, command="optimize-storage") is not None--def _cmd_optimize_storage(db, args):db_path = db.db_pathif not db.fts_optimize_available():@@ -1068,15 +1067,12 @@ def _cmd_optimize_storage(db, args):if resp not in ("y", "yes"):print("Cancelled.")return- # The holder check was skipped under --force: if someone really holds the store, keep the old- # duty-cycle pause so their writes are not starved (slower, but they stay responsive).- yield_to_writers = bool(getattr(args, "force", False)) and _store_has_foreign_holder(db_path)- if yield_to_writers:- print(" Another process is using the database (--force): pacing the rebuild so it stays responsive.")+ # optimize_fts_storage re-scans for holders itself (--force past one, or a writer that opens the+ # store mid-run) and paces while one is there; the progress line says so.progress = _OptimizeProgress()print("Optimizing search-index storage…")try:- result = db.optimize_fts_storage(progress_cb=progress, vacuum=do_vacuum, yield_to_writers=yield_to_writers)+ result = db.optimize_fts_storage(progress_cb=progress, vacuum=do_vacuum)except KeyboardInterrupt:progress.close()print("Interrupted — no data was lost; re-run to resume.")@@ -1084,11 +1080,11 @@ def _cmd_optimize_storage(db, args):except Exception as e:progress.close()print(f"Error: optimization failed: {e}\nNo data was lost. Re-run to resume.")- return+ return 1progress.close()if not result.get("ok"):print(f"Could not optimize: {result.get('reason', 'unknown')}")- return+ return 1print("✓ Search index optimized.")_print_size_change(db, before_mb, prefix=" ")if result.get("vacuumed") is False:diff --git a/hermes_cli/subcommands/sessions.py b/hermes_cli/subcommands/sessions.pyindex 4dc9388e1d9..9f6ce24044c 100644--- a/hermes_cli/subcommands/sessions.py+++ b/hermes_cli/subcommands/sessions.py@@ -172,8 +172,8 @@ def build_sessions_parser(subparsers, *, cmd_sessions: Callable) -> None:"large fraction of state.db (the old layout stored duplicate ""copies of every message and indexed tool output). Runs ""foreground with progress for every phase (refusing while a gateway "- "or other Hermes process holds state.db) and VACUUMs at the end. Safe to interrupt and "- "re-run — it resumes where it left off. No conversation data is "+ "or other Hermes process holds state.db, pacing itself if one opens it mid-run) and "+ "VACUUMs at the end. Safe to interrupt and re-run — it resumes where it left off. No conversation data is ""changed; only the search index is rebuilt.")_flag(sessions_optimize_storage, "--no-vacuum", default=False,help="Skip the final VACUUM (index is rebuilt but freed pages aren't returned to the OS until a later VACUUM)")diff --git a/hermes_state_fts.py b/hermes_state_fts.pyindex 00b51a86696..f3204fed500 100644--- a/hermes_state_fts.py+++ b/hermes_state_fts.py@@ -538,15 +538,16 @@ class SessionFtsSetupMixin:# One blocking rebuild held the write lock ~16 min on a 25 GB DB, so the# backfill and teardown run in chunks: each its own bounded transaction (WAL# growth stays small), resumable from its marker (claimed by CAS), and a- # progress tick. They run back to back with no pause: the only driver is the- # foreground `hermes sessions optimize-storage`, which refuses while any other- # process holds state.db (held_store_refusal, #110054), so there is no writer- # to yield to; a fixed 0.2 s pause after each few-ms chunk was >95% of the- # wall time. Chunk size (in ROWS, never an id span: a pruned store has id gaps)- # adapts toward TARGET seconds, since rows cost ~1000x more in trigram_data- # than in docsize. `--force` past a live holder (yield_to_writers) keeps the- # duty cycle instead: chunks near PACED_TARGET, a pause of max(MIN_PAUSE,- # cost x DUTY_FACTOR) after each, so that holder's writes are not starved.+ # progress tick. They run back to back with no pause while no other process+ # holds state.db: the foreground `hermes sessions optimize-storage` refuses to+ # start under a holder (held_store_refusal, #110054), and a fixed 0.2 s pause+ # after each few-ms chunk was >95% of the wall time. Chunk size (in ROWS, never+ # an id span: a pruned store has id gaps) adapts toward TARGET seconds, since+ # rows cost ~1000x more in trigram_data than in docsize. A side thread re-runs+ # the holder scan every _HOLDER_SCAN_INTERVAL_S (`--force` past a holder, or a+ # writer that opens the store mid-run): while one is there the duty cycle comes+ # back — chunks near PACED_TARGET, a pause of max(MIN_PAUSE, cost x+ # DUTY_FACTOR) after each — so that holder's writes are not starved._FTS_REBUILD_CHUNK_ROWS = 500 # first chunk of every phase and trash table; later ones adapt_FTS_REBUILD_CHUNK_MIN_ROWS = 50_FTS_REBUILD_CHUNK_MAX_ROWS = 100_000diff --git a/hermes_state_search.py b/hermes_state_search.pyindex efdefc1c076..bfc75aecf7a 100644--- a/hermes_state_search.py+++ b/hermes_state_search.py@@ -12,6 +12,7 @@ import sqlite3import threadingimport timefrom dataclasses import dataclass, field+from functools import partialfrom typing import Any, Callable, Collection, Dict, List, Optional, Tuplefrom agent.skill_commands import describe_skill_invocation@@ -28,16 +29,35 @@ from hermes_state_fts import fts_layout_current, fts_layout_driftlogger = logging.getLogger("hermes_state")+@dataclass+class _BackfillProgress:+ """How many messages a backfill pass (``backfill`` = base + trigram, ``backfill_cjk``) has+ indexed out of the messages at or below its high water. Counts rows, never ids: a pruned+ store's id gaps would otherwise read 0% then jump to 99%. The step engine adds the rows each+ window covered (:meth:`SessionSearchMixin._rebuild_step`)."""++ phase: str+ total: int = 0+ indexed: int = 0++ def info(self) -> dict[str, Any]:+ done, total = self.indexed, self.total+ return {"phase": self.phase, "percent": min(100, 100 * done // total) if total else 100,+ "indexed": min(done, total), "total": total}++@dataclassclass _TeardownProgress:"""Which demoted shadow table optimize-storage is draining and how far (``progress_cb``- payload). Owned and kept current by :meth:`SessionSearchMixin._fts_teardown_trash_step`."""+ payload). Owned and kept current by :meth:`SessionSearchMixin._fts_teardown_trash_step`.+ ``counted=False`` (no progress callback) skips the per-table ``COUNT(*)``."""table: str = ""table_index: int = 0table_count: int = 0total: int = 0drained: int = 0+ counted: bool = Truedef info(self) -> dict[str, Any]:done, total = self.drained, self.total@@ -61,6 +81,43 @@ class _VacuumProgress:"estimated_bytes": self.estimated_bytes}+class _HolderWatch:+ """Re-runs the foreign-holder scan (the one the held-store refusal and ``--force`` use) every+ *interval* seconds on a side thread while optimize-storage rebuilds and tears down. The chunk+ loop only reads :attr:`held`, so a quiet store pays nothing per chunk, while a writer that opens+ state.db mid-run (Desktop launched, gateway restarted) moves the loop onto the paced cycle within+ one interval instead of waiting behind back-to-back chunks."""++ def __init__(self, scan: Callable[[], list], interval: float):+ self._scan, self._interval = scan, interval+ self._held, self._stop = threading.Event(), threading.Event()+ self._thread: Optional[threading.Thread] = None++ @property+ def held(self) -> bool:+ return self._held.is_set()++ def _refresh(self) -> None:+ (self._held.set if self._scan() else self._held.clear)()++ def _run(self) -> None:+ while not self._stop.wait(self._interval):+ self._refresh()++ def __enter__(self) -> "_HolderWatch":+ self._refresh() # a holder already there paces from the first chunk+ # copy_context().run: the watcher keeps the caller's profile scope (HX012).+ self._thread = threading.Thread(target=contextvars.copy_context().run, args=(self._run,),+ name="optimize-holder-watch", daemon=True)+ self._thread.start()+ return self++ def __exit__(self, *_exc) -> None:+ self._stop.set()+ if self._thread is not None:+ self._thread.join()++def _search_slow_ms() -> float:"""``sessions.search_slow_ms`` for the served profile (default 1000; 0 logs every call)."""@@ -352,26 +409,28 @@ class SessionSearchMixin:_delete_meta(conn, f"{prefix}_high_water", f"{prefix}_progress")self._execute_write(_do)- def fts_rebuild_step(self) -> bool:+ def fts_rebuild_step(self, progress: Optional[_BackfillProgress] = None) -> bool:"""Backfill one chunk (``_fts_chunk_rows`` messages) of the deferred FTS rebuild; True whilework remains. Chunks are claimed atomically inside the write transaction, so concurrentprocesses interleave instead of duplicating rows. A failed chunk raises (nothing committed;- the caller decides whether to retry)."""+ the caller decides whether to retry). *progress* gains the messages the chunk covered."""if not self._fts_enabled:return Falseinserts = [self._CHUNK_INSERT_SQL.format(table="messages_fts", extra="")]if self._trigram_available:inserts.append(self._TRIGRAM_CHUNK_INSERT_SQL)- return self._rebuild_step("fts_rebuild", inserts, finish=self._fts_rebuild_finish, finish_when_empty=True)+ return self._rebuild_step("fts_rebuild", inserts, finish=self._fts_rebuild_finish, finish_when_empty=True,+ progress=progress)- def fts_cjk_rebuild_step(self) -> bool:+ def fts_cjk_rebuild_step(self, progress: Optional[_BackfillProgress] = None) -> bool:"""Backfill one chunk of the CJK index. True while work remains."""if not self._fts_enabled or not self._fts_cjk_loaded:return Falseinsert = self._CHUNK_INSERT_SQL.format(table="messages_fts_cjk", extra=" AND role <> 'tool'")- return self._rebuild_step("fts_cjk_rebuild", [insert], finish=self._fts_cjk_rebuild_finish)+ return self._rebuild_step("fts_cjk_rebuild", [insert], finish=self._fts_cjk_rebuild_finish, progress=progress)- def _rebuild_step(self, prefix: str, insert_sqls: list[str], *, finish, finish_when_empty: bool = False) -> bool:+ def _rebuild_step(self, prefix: str, insert_sqls: list[str], *, finish, finish_when_empty: bool = False,+ progress: Optional[_BackfillProgress] = None) -> bool:"""Shared chunk engine for the base and CJK deferred backfills. ``finish_when_empty``finalizes a high_water <= 0 marker (empty messages table) instead of leaving it pending."""high_water_raw = self.get_meta(f"{prefix}_high_water")@@ -385,22 +444,29 @@ class SessionSearchMixin:# the claim: two workers cannot read the same value.row = _meta_row(conn, f"{prefix}_progress")if row is None:- return False # finished (or cleared) by another process- progress = int(row[0])- if progress >= high_water:- return False+ return False, 0 # finished (or cleared) by another process+ done = int(row[0])+ if done >= high_water:+ return False, 0# The window is the next `chunk` messages, not `chunk` ids: an id gap (pruned# sessions) neither shrinks a chunk nor lets the next one span a million dense ids.nth = conn.execute("SELECT id FROM messages WHERE id > ? AND id <= ? ORDER BY id LIMIT 1 OFFSET ?",- (progress, high_water, chunk - 1)).fetchone()+ (done, high_water, chunk - 1)).fetchone()upper = int(nth[0]) if nth is not None else high_water+ covered = chunk if nth is not None else 0+ if nth is None and progress is not None: # the last, short window (< chunk rows)+ covered = conn.execute("SELECT COUNT(*) FROM messages WHERE id > ? AND id <= ?",+ (done, upper)).fetchone()[0]for sql in insert_sqls:- conn.execute(sql, (progress, upper))+ conn.execute(sql, (done, upper))# Progress lands in the same transaction as its rows (crash-atomic).conn.execute("UPDATE state_meta SET value = ? WHERE key = ?", (str(upper), f"{prefix}_progress"))- return upper < high_water+ return upper < high_water, covered- more = self._execute_write(_do)+ more, covered = self._execute_write(_do)+ if progress is not None:+ # Counted after the commit: _execute_write may replay _do on contention.+ progress.indexed += coveredif more is False:status = self._rebuild_status(prefix)if (finish_when_empty and high_water <= 0) or (@@ -489,7 +555,7 @@ class SessionSearchMixin:table_count = progress.table_count or len(trash)# COUNT before publishing the table: a failed COUNT must leave progress on the previous# table so the retry runs this start step again instead of inheriting its totals.- total = int(conn.execute(f"SELECT COUNT(*) FROM {tbl} {where}", params).fetchone()[0])+ total = int(conn.execute(f"SELECT COUNT(*) FROM {tbl} {where}", params).fetchone()[0]) if progress.counted else 0progress.table_count, progress.total, progress.drained = table_count, total, 0progress.table_index = max(1, table_count - len(trash) + 1)self._fts_chunk_rows = self._FTS_REBUILD_CHUNK_ROWS@@ -663,22 +729,30 @@ class SessionSearchMixin:raise sqlite3.OperationalError(failure_message)self._conn.commit()- def _backfill_progress_info(self, phase: str = "backfill") -> dict[str, Any]:- st = self.fts_rebuild_status() or self.fts_cjk_rebuild_status()- return {"phase": phase, "percent": st["percent"] if st else 100,- "indexed": st["indexed"] if st else 0, "total": st["total"] if st else 0}+ def _backfill_progress(self, phase: str, prefix: str) -> Optional[_BackfillProgress]:+ """Messages at or below *prefix*'s high water, and how many of them the resume marker already+ covers (one scan per pass); None when the pass has nothing pending."""+ st = self._rebuild_status(prefix)+ if st is None:+ return None+ with self._read_ctx() as conn:+ total, done = conn.execute("SELECT COUNT(*), COALESCE(SUM(id <= ?), 0) FROM messages WHERE id <= ?",+ (st["indexed"], st["total"])).fetchone()+ return _BackfillProgress(phase=phase, total=int(total), indexed=int(done))# A chunk that fails this many times in a row ends the run (each lock failure already waited out# _execute_write's patience); a full disk or I/O error never heals by retrying._OPTIMIZE_CHUNK_ATTEMPTS = 3_VACUUM_TICK_S = 1.0-- def _optimize_drive(self, step: Callable[[], bool], emit: Callable[[], None], *, paced: bool) -> None:- """Run *step* to completion, emitting after every chunk. Back to back (no other writer: see- the chunk engine note in hermes_state_fts) unless *paced*, which keeps the duty cycle for a- holder ``--force`` ran past. Chunk size adapts to its measured cost; a step may restart it- (teardown on each new table)."""- target = self._FTS_REBUILD_PACED_TARGET_S if paced else self._FTS_REBUILD_CHUNK_TARGET_S+ # How often the side thread re-scans for another process holding state.db (~15 ms on Linux).+ _HOLDER_SCAN_INTERVAL_S = 2.0++ def _optimize_drive(self, step: Callable[[], bool], info: Optional[Callable[[], dict[str, Any]]],+ emit: Callable[[dict[str, Any]], None], watch: _HolderWatch) -> None:+ """Run *step* to completion, emitting ``info()`` (plus ``paced``) after every chunk. Back to+ back while no other process holds state.db (see the chunk engine note in hermes_state_fts);+ while *watch* sees a holder, the duty cycle (short chunks + a pause) keeps its writes moving.+ Chunk size adapts to its measured cost; a step may restart it (teardown on each new table)."""self._fts_chunk_rows = self._FTS_REBUILD_CHUNK_ROWSfailures = 0while True:@@ -695,8 +769,10 @@ class SessionSearchMixin:failures = 0if not more:return- cost = time.monotonic() - t0- emit()+ cost, paced = time.monotonic() - t0, watch.held+ if info is not None:+ emit({**info(), "paced": paced})+ target = self._FTS_REBUILD_PACED_TARGET_S if paced else self._FTS_REBUILD_CHUNK_TARGET_Sself._fts_chunk_rows = self._next_chunk_rows(self._fts_chunk_rows, cost, target)if paced:time.sleep(max(self._FTS_REBUILD_MIN_PAUSE, cost * self._FTS_REBUILD_DUTY_FACTOR))@@ -772,14 +848,14 @@ class SessionSearchMixin:def optimize_fts_storage(self, *, progress_cb: Optional[Callable[[dict[str, Any]], None]] = None, vacuum: bool = True,- yield_to_writers: bool = False,) -> dict[str, Any]:"""Repair an older FTS layout into the current v23 shape, foreground and to completion:legacy-v22 inline -> external-content, or a v23 ``messages_fts_trigram`` that still stores``tool_calls``. Re-running resumes; a chunk that keeps failing raises (nothing is lost).- ``progress_cb`` receives {"phase", "percent", "indexed", "total"} after every chunk (plus- :meth:`_TeardownProgress.info` / :meth:`_VacuumProgress.info` fields) and every second of the- VACUUM. ``yield_to_writers`` (``--force`` past a live holder) paces chunks so its writes get+ ``progress_cb`` receives {"phase", "percent", "indexed", "total", "paced"} after every chunk+ (``backfill``/``backfill_cjk`` count messages; plus :meth:`_TeardownProgress.info` fields)+ and every second of the VACUUM (:meth:`_VacuumProgress.info`). Whenever another process holds+ state.db (re-scanned every ``_HOLDER_SCAN_INTERVAL_S``) chunks are paced so its writes getthrough. A missing trigram tokenizer is not fatal (CJK falls back to LIKE)."""if not self._fts_enabled:return {"ok": False, "reason": "fts5_unavailable"}@@ -805,14 +881,18 @@ class SessionSearchMixin:self._ensure_cjk_schema_committed()emit = progress_cb or (lambda _info: None)- teardown = _TeardownProgress()- # Phase 1: base backfill; 1b: CJK-bigram backfill (own marker pair).- emit(self._backfill_progress_info())- for step in (self.fts_rebuild_step, self.fts_cjk_rebuild_step):- self._optimize_drive(step, lambda: emit(self._backfill_progress_info()), paced=yield_to_writers)- # Phase 2: tear down the demoted legacy shadow tables in chunks.- self._optimize_drive(lambda: self._fts_teardown_trash_step(teardown), lambda: emit(teardown.info()),- paced=yield_to_writers)+ teardown = _TeardownProgress(counted=progress_cb is not None)+ passes = (("backfill", "fts_rebuild", self.fts_rebuild_step), ("backfill_cjk", "fts_cjk_rebuild",+ self.fts_cjk_rebuild_step))+ with _HolderWatch(self._foreign_state_db_holders, self._HOLDER_SCAN_INTERVAL_S) as watch:+ # Phase 1: base backfill; 1b: CJK-bigram backfill (own marker pair, own progress line).+ for phase, prefix, step in passes:+ bf = self._backfill_progress(phase, prefix) if progress_cb else None+ if bf is not None:+ emit({**bf.info(), "paced": watch.held})+ self._optimize_drive(partial(step, bf) if bf else step, bf.info if bf else None, emit, watch)+ # Phase 2: tear down the demoted legacy shadow tables in chunks.+ self._optimize_drive(partial(self._fts_teardown_trash_step, teardown), teardown.info, emit, watch)with self._read_ctx() as conn:still_pending = _meta_row(conn, "fts_rebuild_high_water") is not Nonestill_trash = self._has_fts_trash(conn)@@ -831,7 +911,7 @@ class SessionSearchMixin:# A concurrent process changed state since the pre-vacuum check; a re-run can still settle.logger.warning("FTS storage optimization settle refused (%s)", refusal)return {"ok": False, "reason": refusal, "vacuumed": vacuum_ok}- emit(self._backfill_progress_info("done"))+ emit({"phase": "done", "percent": 100, "indexed": 0, "total": 0})logger.info("FTS storage optimization complete (layout v%d).", FTS_STORAGE_VERSION)return {"ok": True, "vacuumed": vacuum_ok}diff --git a/tests/hermes_state/test_fts_optimize_progress.py b/tests/hermes_state/test_fts_optimize_progress.pyindex 33137e7db8f..763d85c8690 100644--- a/tests/hermes_state/test_fts_optimize_progress.py+++ b/tests/hermes_state/test_fts_optimize_progress.py@@ -6,11 +6,13 @@ import itertoolsimport sqlite3import threadingimport time+from argparse import Namespaceimport pytest+from hermes_cli.sessions_cmd import _cmd_optimize_storage, _optimize_progress_linefrom hermes_state import SessionDB-from hermes_state_common import LEGACY_FTS_SQL, SCHEMA_SQL+from hermes_state_common import LEGACY_FTS_SQL, LEGACY_FTS_TRIGRAM_SQL, SCHEMA_SQLGAP_AFTER, GAP_IDS = 600, 100_000 # a pruned store: ids jump after the first 600 messages@@ -19,7 +21,7 @@ def _legacy_inline_store(db_path, n=1200):ids = [i + 1 + (GAP_IDS if i >= GAP_AFTER else 0) for i in range(n)]raw = sqlite3.connect(str(db_path))try:- raw.executescript(SCHEMA_SQL + LEGACY_FTS_SQL)+ raw.executescript(SCHEMA_SQL + LEGACY_FTS_SQL + LEGACY_FTS_TRIGRAM_SQL)raw.execute("INSERT INTO sessions (id, source, started_at) VALUES ('s1', 'cli', ?)", (time.time(),))raw.executemany("INSERT INTO messages (id, session_id, timestamp, role, content) VALUES (?, 's1', ?, 'user', ?)",@@ -68,6 +70,10 @@ def test_progress_reports_teardown_rows_and_vacuum_liveness(legacy_db, monkeypat# The ticker is stopped and joined on success and on failure: nothing can report after return.assert not [t for t in threading.enumerate() if t.name == "optimize-vacuum-ticker"]assert events[-1]["phase"] == "done"+ # Backfill counts messages, not ids: after the first 500-row chunk the 100k-id gap still reads 500/1,200.+ backfill = [e for e in events if e["phase"] == "backfill"]+ assert {e["total"] for e in backfill} == {1200} and backfill[0]["indexed"] == 0+ assert (backfill[1]["indexed"], backfill[1]["percent"]) == (500, 41)vacuum = [e for e in events if e["phase"] == "vacuum"]assert vacuum[0]["size_bytes"] >= vacuum[0]["estimated_bytes"] > 0teardown = [e for e in events if e["phase"] == "teardown"]@@ -76,7 +82,9 @@ def test_progress_reports_teardown_rows_and_vacuum_liveness(legacy_db, monkeypatassert docsize[0]["total"] == 1200 and docsize[0]["indexed"] == 0assert docsize[-1]["indexed"] == 1200 and docsize[-1]["percent"] == 100count = teardown[0]["table_count"]- assert count >= 2 and {e["table_index"] for e in teardown} <= set(range(1, count + 1))+ # Both legacy indexes (base + trigram) are reclaimed: five shadow tables each.+ assert count == 10 and {e["table_index"] for e in teardown} <= set(range(1, count + 1))+ assert any(e["table"].endswith("messages_fts_trigram_data") and e["total"] > 0 for e in teardown)assert legacy_db.search_messages("quokka")@@ -87,8 +95,8 @@ def test_foreground_optimize_never_sleeps_and_chunks_by_rows_across_id_gaps(legamonkeypatch.setattr(SessionDB, "_FTS_REBUILD_CHUNK_MAX_ROWS", 500)markers, real_step = [0], legacy_db.fts_rebuild_step- def observed_step():- more = real_step()+ def observed_step(*args):+ more = real_step(*args)status = legacy_db.fts_rebuild_status()markers.append(status["indexed"] if status else GAP_IDS + 1200)return more@@ -103,21 +111,57 @@ def test_foreground_optimize_never_sleeps_and_chunks_by_rows_across_id_gaps(legaassert windows == [500, 500, 200]-def test_chunk_that_keeps_failing_ends_the_run_and_a_rerun_resumes(legacy_db, monkeypatch):+def test_chunk_that_keeps_failing_ends_the_run_and_a_rerun_resumes(legacy_db, monkeypatch, capsys):monkeypatch.setattr(time, "sleep", lambda _s: None) # the retry backoffcalls, real_step = [], legacy_db.fts_rebuild_step- def disk_fills_during_backfill():+ def disk_fills_during_backfill(*args):if not calls:pages = legacy_db._conn.execute("PRAGMA page_count").fetchone()[0]legacy_db._conn.execute(f"PRAGMA max_page_count = {pages + 5}")calls.append(1)assert len(calls) < 50, "optimize kept retrying a chunk that can never succeed"- return real_step()+ return real_step(*args)legacy_db.fts_rebuild_step = disk_fills_during_backfillwith pytest.raises(sqlite3.OperationalError, match="full"):legacy_db.optimize_fts_storage(vacuum=False)+ # The CLI reports the failure and exits non-zero (main() reads None as success).+ calls.clear()+ assert _cmd_optimize_storage(legacy_db, Namespace(yes=True, no_vacuum=True, force=False)) == 1+ assert "Error: optimization failed: database or disk is full" in capsys.readouterr().outlegacy_db._conn.execute("PRAGMA max_page_count = 1073741823")assert legacy_db.optimize_fts_storage(vacuum=False)["ok"] is Trueassert len(legacy_db.search_messages("quokka", limit=2000)) == 1200+++def test_writer_that_opens_the_store_mid_run_gets_the_paced_cycle(legacy_db, monkeypatch):+ slept, scans, events, writer_opened = [], [], [], threading.Event()+ monkeypatch.setattr(time, "sleep", slept.append)+ monkeypatch.setattr(SessionDB, "_HOLDER_SCAN_INTERVAL_S", 0.01)+ monkeypatch.setattr(SessionDB, "_FTS_REBUILD_CHUNK_MAX_ROWS", 100)++ def holder_scan():+ scans.append(1)+ if len(scans) == 1:+ return [] # quiet when the run starts: no pacing+ writer_opened.set()+ return [(4242, str(legacy_db.db_path))]++ monkeypatch.setattr(legacy_db, "_foreign_state_db_holders", holder_scan)+ real_step = legacy_db.fts_rebuild_step++ def step_then_writer_opens(*args):+ more = real_step(*args)+ assert writer_opened.wait(10) # e.g. Desktop launched after the first chunk+ return more++ legacy_db.fts_rebuild_step = step_then_writer_opens+ assert legacy_db.optimize_fts_storage(progress_cb=events.append, vacuum=False)["ok"] is True+ chunks = [e for e in events if e["phase"] != "done"]+ assert chunks[0]["paced"] is False and all(e["paced"] for e in chunks[1:])+ # Every chunk after the writer appeared yields the write lock for at least the duty-cycle floor.+ teardown = [e for e in chunks if e["phase"] == "teardown"]+ assert len(slept) >= len(teardown) > 0 and min(slept) >= SessionDB._FTS_REBUILD_MIN_PAUSE+ assert "paced: another process" in _optimize_progress_line(chunks[-1])+ assert not [t for t in threading.enumerate() if t.name == "optimize-holder-watch"]diff --git a/website/docs/user-guide/session-storage-recovery.md b/website/docs/user-guide/session-storage-recovery.mdindex ff545834a63..17ee06cdffb 100644--- a/website/docs/user-guide/session-storage-recovery.md+++ b/website/docs/user-guide/session-storage-recovery.md@@ -78,7 +78,9 @@ Override with --force if you accept the risk.`--dry-run` previews are never blocked. `--force` runs anyway — use it only when you knowthe listed processes are idle (a reader you started yourself, for example). The same check-runs when you type `sessions optimize` in the Desktop console.+runs when you type `sessions optimize` in the Desktop console. `optimize-storage` keeps watching+while it runs: whenever another process holds the database (under `--force`, or one that opens it+mid-run) it pauses between chunks so that process's writes get through, which makes it much slower.`hermes sessions prune` is the exception while the gateway is running: the gateway owns`state.db`, so prune shows its preview from a read-only view and the gateway itself deletes