Skip to content

The agent that grows with you

fix(sessions): optimize-storage paces for a writer that opens state.db mid-run; failed runs exit 1

Tekniumd569365e87e2
diff --git a/hermes_cli/sessions_cmd.py b/hermes_cli/sessions_cmd.py
index bbe79a1baeb..4881375d607 100644
--- a/hermes_cli/sessions_cmd.py
+++ b/hermes_cli/sessions_cmd.py
@@ -980,15 +980,20 @@ def _cmd_clean_markers(db, args):
print(f"✓ Cleared {report['rows_affected']} row(s).")
+_BACKFILL_LABELS = {"backfill": "Rebuilding index", "backfill_cjk": "Building CJK index"}
+_COUNTED_PHASES = frozenset({*_BACKFILL_LABELS, "teardown"})
+
+
def _optimize_progress_line(info) -> str:
phase, pct = info.get("phase"), f"{info.get('percent', 0):3d}%"
counts = f"({info.get('indexed', 0):,}/{info.get('total', 0):,}"
- if phase == "backfill":
- return f" Rebuilding index: {pct} {counts})"
+ paced = " — paced: another process is using the database" if info.get("paced") else ""
+ if phase in _BACKFILL_LABELS:
+ return f" {_BACKFILL_LABELS[phase]}: {pct} {counts} messages){paced}"
if phase == "teardown":
table = str(info.get("table", "")).removeprefix("fts_v22_trash_")
return (f" Reclaiming old index: table {info.get('table_index', 0)}/{info.get('table_count', 0)} "
- f"({table}) {pct} {counts} rows)")
+ f"({table}) {pct} {counts} rows){paced}")
elapsed = int(info.get("elapsed", 0))
return (f" Compacting database (VACUUM): {elapsed // 60}m {elapsed % 60:02d}s elapsed, "
f"{info.get('size_bytes', 0) / 2**20:,.1f} MB → ~{info.get('estimated_bytes', 0) / 2**20:,.1f} MB")
@@ -1008,7 +1013,7 @@ class _OptimizeProgress:
# The phase or table moved on, so the previous one ran to completion.
self.close(completed=True)
self._key = key
- if phase == "done" or (phase in ("backfill", "teardown") and not info.get("total") and not info.get("table")):
+ if phase == "done" or (phase in _COUNTED_PHASES and not info.get("total") and not info.get("table")):
return
self._last = info
self._draw(_optimize_progress_line(info))
@@ -1021,19 +1026,13 @@ class _OptimizeProgress:
def close(self, completed: bool = False) -> None:
last = self._last
- if completed and last and last.get("phase") in ("backfill", "teardown"):
+ if completed and last and last.get("phase") in _COUNTED_PHASES:
self._draw(_optimize_progress_line({**last, "percent": 100, "indexed": last.get("total", 0)}))
if self._line:
print(flush=True)
self._last, self._line = None, ""
-def _store_has_foreign_holder(db_path) -> bool:
- from hermes_state_holders import held_store_refusal
-
- return held_store_refusal(db_path, command="optimize-storage") is not None
-
-
def _cmd_optimize_storage(db, args):
db_path = db.db_path
if not db.fts_optimize_available():
@@ -1068,15 +1067,12 @@ def _cmd_optimize_storage(db, args):
if resp not in ("y", "yes"):
print("Cancelled.")
return
- # The holder check was skipped under --force: if someone really holds the store, keep the old
- # duty-cycle pause so their writes are not starved (slower, but they stay responsive).
- yield_to_writers = bool(getattr(args, "force", False)) and _store_has_foreign_holder(db_path)
- if yield_to_writers:
- print(" Another process is using the database (--force): pacing the rebuild so it stays responsive.")
+ # optimize_fts_storage re-scans for holders itself (--force past one, or a writer that opens the
+ # store mid-run) and paces while one is there; the progress line says so.
progress = _OptimizeProgress()
print("Optimizing search-index storage…")
try:
- result = db.optimize_fts_storage(progress_cb=progress, vacuum=do_vacuum, yield_to_writers=yield_to_writers)
+ result = db.optimize_fts_storage(progress_cb=progress, vacuum=do_vacuum)
except KeyboardInterrupt:
progress.close()
print("Interrupted — no data was lost; re-run to resume.")
@@ -1084,11 +1080,11 @@ def _cmd_optimize_storage(db, args):
except Exception as e:
progress.close()
print(f"Error: optimization failed: {e}\nNo data was lost. Re-run to resume.")
- return
+ return 1
progress.close()
if not result.get("ok"):
print(f"Could not optimize: {result.get('reason', 'unknown')}")
- return
+ return 1
print("✓ Search index optimized.")
_print_size_change(db, before_mb, prefix=" ")
if result.get("vacuumed") is False:
diff --git a/hermes_cli/subcommands/sessions.py b/hermes_cli/subcommands/sessions.py
index 4dc9388e1d9..9f6ce24044c 100644
--- a/hermes_cli/subcommands/sessions.py
+++ b/hermes_cli/subcommands/sessions.py
@@ -172,8 +172,8 @@ def build_sessions_parser(subparsers, *, cmd_sessions: Callable) -> None:
"large fraction of state.db (the old layout stored duplicate "
"copies of every message and indexed tool output). Runs "
"foreground with progress for every phase (refusing while a gateway "
- "or other Hermes process holds state.db) and VACUUMs at the end. Safe to interrupt and "
- "re-run — it resumes where it left off. No conversation data is "
+ "or other Hermes process holds state.db, pacing itself if one opens it mid-run) and "
+ "VACUUMs at the end. Safe to interrupt and re-run — it resumes where it left off. No conversation data is "
"changed; only the search index is rebuilt.")
_flag(sessions_optimize_storage, "--no-vacuum", default=False,
help="Skip the final VACUUM (index is rebuilt but freed pages aren't returned to the OS until a later VACUUM)")
diff --git a/hermes_state_fts.py b/hermes_state_fts.py
index 00b51a86696..f3204fed500 100644
--- a/hermes_state_fts.py
+++ b/hermes_state_fts.py
@@ -538,15 +538,16 @@ class SessionFtsSetupMixin:
# One blocking rebuild held the write lock ~16 min on a 25 GB DB, so the
# backfill and teardown run in chunks: each its own bounded transaction (WAL
# growth stays small), resumable from its marker (claimed by CAS), and a
- # progress tick. They run back to back with no pause: the only driver is the
- # foreground `hermes sessions optimize-storage`, which refuses while any other
- # process holds state.db (held_store_refusal, #110054), so there is no writer
- # to yield to; a fixed 0.2 s pause after each few-ms chunk was >95% of the
- # wall time. Chunk size (in ROWS, never an id span: a pruned store has id gaps)
- # adapts toward TARGET seconds, since rows cost ~1000x more in trigram_data
- # than in docsize. `--force` past a live holder (yield_to_writers) keeps the
- # duty cycle instead: chunks near PACED_TARGET, a pause of max(MIN_PAUSE,
- # cost x DUTY_FACTOR) after each, so that holder's writes are not starved.
+ # progress tick. They run back to back with no pause while no other process
+ # holds state.db: the foreground `hermes sessions optimize-storage` refuses to
+ # start under a holder (held_store_refusal, #110054), and a fixed 0.2 s pause
+ # after each few-ms chunk was >95% of the wall time. Chunk size (in ROWS, never
+ # an id span: a pruned store has id gaps) adapts toward TARGET seconds, since
+ # rows cost ~1000x more in trigram_data than in docsize. A side thread re-runs
+ # the holder scan every _HOLDER_SCAN_INTERVAL_S (`--force` past a holder, or a
+ # writer that opens the store mid-run): while one is there the duty cycle comes
+ # back — chunks near PACED_TARGET, a pause of max(MIN_PAUSE, cost x
+ # DUTY_FACTOR) after each — so that holder's writes are not starved.
_FTS_REBUILD_CHUNK_ROWS = 500 # first chunk of every phase and trash table; later ones adapt
_FTS_REBUILD_CHUNK_MIN_ROWS = 50
_FTS_REBUILD_CHUNK_MAX_ROWS = 100_000
diff --git a/hermes_state_search.py b/hermes_state_search.py
index efdefc1c076..bfc75aecf7a 100644
--- a/hermes_state_search.py
+++ b/hermes_state_search.py
@@ -12,6 +12,7 @@ import sqlite3
import threading
import time
from dataclasses import dataclass, field
+from functools import partial
from typing import Any, Callable, Collection, Dict, List, Optional, Tuple
from agent.skill_commands import describe_skill_invocation
@@ -28,16 +29,35 @@ from hermes_state_fts import fts_layout_current, fts_layout_drift
logger = logging.getLogger("hermes_state")
+@dataclass
+class _BackfillProgress:
+ """How many messages a backfill pass (``backfill`` = base + trigram, ``backfill_cjk``) has
+ indexed out of the messages at or below its high water. Counts rows, never ids: a pruned
+ store's id gaps would otherwise read 0% then jump to 99%. The step engine adds the rows each
+ window covered (:meth:`SessionSearchMixin._rebuild_step`)."""
+
+ phase: str
+ total: int = 0
+ indexed: int = 0
+
+ def info(self) -> dict[str, Any]:
+ done, total = self.indexed, self.total
+ return {"phase": self.phase, "percent": min(100, 100 * done // total) if total else 100,
+ "indexed": min(done, total), "total": total}
+
+
@dataclass
class _TeardownProgress:
"""Which demoted shadow table optimize-storage is draining and how far (``progress_cb``
- payload). Owned and kept current by :meth:`SessionSearchMixin._fts_teardown_trash_step`."""
+ payload). Owned and kept current by :meth:`SessionSearchMixin._fts_teardown_trash_step`.
+ ``counted=False`` (no progress callback) skips the per-table ``COUNT(*)``."""
table: str = ""
table_index: int = 0
table_count: int = 0
total: int = 0
drained: int = 0
+ counted: bool = True
def info(self) -> dict[str, Any]:
done, total = self.drained, self.total
@@ -61,6 +81,43 @@ class _VacuumProgress:
"estimated_bytes": self.estimated_bytes}
+class _HolderWatch:
+ """Re-runs the foreign-holder scan (the one the held-store refusal and ``--force`` use) every
+ *interval* seconds on a side thread while optimize-storage rebuilds and tears down. The chunk
+ loop only reads :attr:`held`, so a quiet store pays nothing per chunk, while a writer that opens
+ state.db mid-run (Desktop launched, gateway restarted) moves the loop onto the paced cycle within
+ one interval instead of waiting behind back-to-back chunks."""
+
+ def __init__(self, scan: Callable[[], list], interval: float):
+ self._scan, self._interval = scan, interval
+ self._held, self._stop = threading.Event(), threading.Event()
+ self._thread: Optional[threading.Thread] = None
+
+ @property
+ def held(self) -> bool:
+ return self._held.is_set()
+
+ def _refresh(self) -> None:
+ (self._held.set if self._scan() else self._held.clear)()
+
+ def _run(self) -> None:
+ while not self._stop.wait(self._interval):
+ self._refresh()
+
+ def __enter__(self) -> "_HolderWatch":
+ self._refresh() # a holder already there paces from the first chunk
+ # copy_context().run: the watcher keeps the caller's profile scope (HX012).
+ self._thread = threading.Thread(target=contextvars.copy_context().run, args=(self._run,),
+ name="optimize-holder-watch", daemon=True)
+ self._thread.start()
+ return self
+
+ def __exit__(self, *_exc) -> None:
+ self._stop.set()
+ if self._thread is not None:
+ self._thread.join()
+
+
def _search_slow_ms() -> float:
"""``sessions.search_slow_ms`` for the served profile (default 1000; 0 logs every call)."""
@@ -352,26 +409,28 @@ class SessionSearchMixin:
_delete_meta(conn, f"{prefix}_high_water", f"{prefix}_progress")
self._execute_write(_do)
- def fts_rebuild_step(self) -> bool:
+ def fts_rebuild_step(self, progress: Optional[_BackfillProgress] = None) -> bool:
"""Backfill one chunk (``_fts_chunk_rows`` messages) of the deferred FTS rebuild; True while
work remains. Chunks are claimed atomically inside the write transaction, so concurrent
processes interleave instead of duplicating rows. A failed chunk raises (nothing committed;
- the caller decides whether to retry)."""
+ the caller decides whether to retry). *progress* gains the messages the chunk covered."""
if not self._fts_enabled:
return False
inserts = [self._CHUNK_INSERT_SQL.format(table="messages_fts", extra="")]
if self._trigram_available:
inserts.append(self._TRIGRAM_CHUNK_INSERT_SQL)
- return self._rebuild_step("fts_rebuild", inserts, finish=self._fts_rebuild_finish, finish_when_empty=True)
+ return self._rebuild_step("fts_rebuild", inserts, finish=self._fts_rebuild_finish, finish_when_empty=True,
+ progress=progress)
- def fts_cjk_rebuild_step(self) -> bool:
+ def fts_cjk_rebuild_step(self, progress: Optional[_BackfillProgress] = None) -> bool:
"""Backfill one chunk of the CJK index. True while work remains."""
if not self._fts_enabled or not self._fts_cjk_loaded:
return False
insert = self._CHUNK_INSERT_SQL.format(table="messages_fts_cjk", extra=" AND role <> 'tool'")
- return self._rebuild_step("fts_cjk_rebuild", [insert], finish=self._fts_cjk_rebuild_finish)
+ return self._rebuild_step("fts_cjk_rebuild", [insert], finish=self._fts_cjk_rebuild_finish, progress=progress)
- def _rebuild_step(self, prefix: str, insert_sqls: list[str], *, finish, finish_when_empty: bool = False) -> bool:
+ def _rebuild_step(self, prefix: str, insert_sqls: list[str], *, finish, finish_when_empty: bool = False,
+ progress: Optional[_BackfillProgress] = None) -> bool:
"""Shared chunk engine for the base and CJK deferred backfills. ``finish_when_empty``
finalizes a high_water <= 0 marker (empty messages table) instead of leaving it pending."""
high_water_raw = self.get_meta(f"{prefix}_high_water")
@@ -385,22 +444,29 @@ class SessionSearchMixin:
# the claim: two workers cannot read the same value.
row = _meta_row(conn, f"{prefix}_progress")
if row is None:
- return False # finished (or cleared) by another process
- progress = int(row[0])
- if progress >= high_water:
- return False
+ return False, 0 # finished (or cleared) by another process
+ done = int(row[0])
+ if done >= high_water:
+ return False, 0
# The window is the next `chunk` messages, not `chunk` ids: an id gap (pruned
# sessions) neither shrinks a chunk nor lets the next one span a million dense ids.
nth = conn.execute("SELECT id FROM messages WHERE id > ? AND id <= ? ORDER BY id LIMIT 1 OFFSET ?",
- (progress, high_water, chunk - 1)).fetchone()
+ (done, high_water, chunk - 1)).fetchone()
upper = int(nth[0]) if nth is not None else high_water
+ covered = chunk if nth is not None else 0
+ if nth is None and progress is not None: # the last, short window (< chunk rows)
+ covered = conn.execute("SELECT COUNT(*) FROM messages WHERE id > ? AND id <= ?",
+ (done, upper)).fetchone()[0]
for sql in insert_sqls:
- conn.execute(sql, (progress, upper))
+ conn.execute(sql, (done, upper))
# Progress lands in the same transaction as its rows (crash-atomic).
conn.execute("UPDATE state_meta SET value = ? WHERE key = ?", (str(upper), f"{prefix}_progress"))
- return upper < high_water
+ return upper < high_water, covered
- more = self._execute_write(_do)
+ more, covered = self._execute_write(_do)
+ if progress is not None:
+ # Counted after the commit: _execute_write may replay _do on contention.
+ progress.indexed += covered
if more is False:
status = self._rebuild_status(prefix)
if (finish_when_empty and high_water <= 0) or (
@@ -489,7 +555,7 @@ class SessionSearchMixin:
table_count = progress.table_count or len(trash)
# COUNT before publishing the table: a failed COUNT must leave progress on the previous
# table so the retry runs this start step again instead of inheriting its totals.
- total = int(conn.execute(f"SELECT COUNT(*) FROM {tbl} {where}", params).fetchone()[0])
+ total = int(conn.execute(f"SELECT COUNT(*) FROM {tbl} {where}", params).fetchone()[0]) if progress.counted else 0
progress.table_count, progress.total, progress.drained = table_count, total, 0
progress.table_index = max(1, table_count - len(trash) + 1)
self._fts_chunk_rows = self._FTS_REBUILD_CHUNK_ROWS
@@ -663,22 +729,30 @@ class SessionSearchMixin:
raise sqlite3.OperationalError(failure_message)
self._conn.commit()
- def _backfill_progress_info(self, phase: str = "backfill") -> dict[str, Any]:
- st = self.fts_rebuild_status() or self.fts_cjk_rebuild_status()
- return {"phase": phase, "percent": st["percent"] if st else 100,
- "indexed": st["indexed"] if st else 0, "total": st["total"] if st else 0}
+ def _backfill_progress(self, phase: str, prefix: str) -> Optional[_BackfillProgress]:
+ """Messages at or below *prefix*'s high water, and how many of them the resume marker already
+ covers (one scan per pass); None when the pass has nothing pending."""
+ st = self._rebuild_status(prefix)
+ if st is None:
+ return None
+ with self._read_ctx() as conn:
+ total, done = conn.execute("SELECT COUNT(*), COALESCE(SUM(id <= ?), 0) FROM messages WHERE id <= ?",
+ (st["indexed"], st["total"])).fetchone()
+ return _BackfillProgress(phase=phase, total=int(total), indexed=int(done))
# A chunk that fails this many times in a row ends the run (each lock failure already waited out
# _execute_write's patience); a full disk or I/O error never heals by retrying.
_OPTIMIZE_CHUNK_ATTEMPTS = 3
_VACUUM_TICK_S = 1.0
-
- def _optimize_drive(self, step: Callable[[], bool], emit: Callable[[], None], *, paced: bool) -> None:
- """Run *step* to completion, emitting after every chunk. Back to back (no other writer: see
- the chunk engine note in hermes_state_fts) unless *paced*, which keeps the duty cycle for a
- holder ``--force`` ran past. Chunk size adapts to its measured cost; a step may restart it
- (teardown on each new table)."""
- target = self._FTS_REBUILD_PACED_TARGET_S if paced else self._FTS_REBUILD_CHUNK_TARGET_S
+ # How often the side thread re-scans for another process holding state.db (~15 ms on Linux).
+ _HOLDER_SCAN_INTERVAL_S = 2.0
+
+ def _optimize_drive(self, step: Callable[[], bool], info: Optional[Callable[[], dict[str, Any]]],
+ emit: Callable[[dict[str, Any]], None], watch: _HolderWatch) -> None:
+ """Run *step* to completion, emitting ``info()`` (plus ``paced``) after every chunk. Back to
+ back while no other process holds state.db (see the chunk engine note in hermes_state_fts);
+ while *watch* sees a holder, the duty cycle (short chunks + a pause) keeps its writes moving.
+ Chunk size adapts to its measured cost; a step may restart it (teardown on each new table)."""
self._fts_chunk_rows = self._FTS_REBUILD_CHUNK_ROWS
failures = 0
while True:
@@ -695,8 +769,10 @@ class SessionSearchMixin:
failures = 0
if not more:
return
- cost = time.monotonic() - t0
- emit()
+ cost, paced = time.monotonic() - t0, watch.held
+ if info is not None:
+ emit({**info(), "paced": paced})
+ target = self._FTS_REBUILD_PACED_TARGET_S if paced else self._FTS_REBUILD_CHUNK_TARGET_S
self._fts_chunk_rows = self._next_chunk_rows(self._fts_chunk_rows, cost, target)
if paced:
time.sleep(max(self._FTS_REBUILD_MIN_PAUSE, cost * self._FTS_REBUILD_DUTY_FACTOR))
@@ -772,14 +848,14 @@ class SessionSearchMixin:
def optimize_fts_storage(
self, *, progress_cb: Optional[Callable[[dict[str, Any]], None]] = None, vacuum: bool = True,
- yield_to_writers: bool = False,
) -> dict[str, Any]:
"""Repair an older FTS layout into the current v23 shape, foreground and to completion:
legacy-v22 inline -> external-content, or a v23 ``messages_fts_trigram`` that still stores
``tool_calls``. Re-running resumes; a chunk that keeps failing raises (nothing is lost).
- ``progress_cb`` receives {"phase", "percent", "indexed", "total"} after every chunk (plus
- :meth:`_TeardownProgress.info` / :meth:`_VacuumProgress.info` fields) and every second of the
- VACUUM. ``yield_to_writers`` (``--force`` past a live holder) paces chunks so its writes get
+ ``progress_cb`` receives {"phase", "percent", "indexed", "total", "paced"} after every chunk
+ (``backfill``/``backfill_cjk`` count messages; plus :meth:`_TeardownProgress.info` fields)
+ and every second of the VACUUM (:meth:`_VacuumProgress.info`). Whenever another process holds
+ state.db (re-scanned every ``_HOLDER_SCAN_INTERVAL_S``) chunks are paced so its writes get
through. A missing trigram tokenizer is not fatal (CJK falls back to LIKE)."""
if not self._fts_enabled:
return {"ok": False, "reason": "fts5_unavailable"}
@@ -805,14 +881,18 @@ class SessionSearchMixin:
self._ensure_cjk_schema_committed()
emit = progress_cb or (lambda _info: None)
- teardown = _TeardownProgress()
- # Phase 1: base backfill; 1b: CJK-bigram backfill (own marker pair).
- emit(self._backfill_progress_info())
- for step in (self.fts_rebuild_step, self.fts_cjk_rebuild_step):
- self._optimize_drive(step, lambda: emit(self._backfill_progress_info()), paced=yield_to_writers)
- # Phase 2: tear down the demoted legacy shadow tables in chunks.
- self._optimize_drive(lambda: self._fts_teardown_trash_step(teardown), lambda: emit(teardown.info()),
- paced=yield_to_writers)
+ teardown = _TeardownProgress(counted=progress_cb is not None)
+ passes = (("backfill", "fts_rebuild", self.fts_rebuild_step), ("backfill_cjk", "fts_cjk_rebuild",
+ self.fts_cjk_rebuild_step))
+ with _HolderWatch(self._foreign_state_db_holders, self._HOLDER_SCAN_INTERVAL_S) as watch:
+ # Phase 1: base backfill; 1b: CJK-bigram backfill (own marker pair, own progress line).
+ for phase, prefix, step in passes:
+ bf = self._backfill_progress(phase, prefix) if progress_cb else None
+ if bf is not None:
+ emit({**bf.info(), "paced": watch.held})
+ self._optimize_drive(partial(step, bf) if bf else step, bf.info if bf else None, emit, watch)
+ # Phase 2: tear down the demoted legacy shadow tables in chunks.
+ self._optimize_drive(partial(self._fts_teardown_trash_step, teardown), teardown.info, emit, watch)
with self._read_ctx() as conn:
still_pending = _meta_row(conn, "fts_rebuild_high_water") is not None
still_trash = self._has_fts_trash(conn)
@@ -831,7 +911,7 @@ class SessionSearchMixin:
# A concurrent process changed state since the pre-vacuum check; a re-run can still settle.
logger.warning("FTS storage optimization settle refused (%s)", refusal)
return {"ok": False, "reason": refusal, "vacuumed": vacuum_ok}
- emit(self._backfill_progress_info("done"))
+ emit({"phase": "done", "percent": 100, "indexed": 0, "total": 0})
logger.info("FTS storage optimization complete (layout v%d).", FTS_STORAGE_VERSION)
return {"ok": True, "vacuumed": vacuum_ok}
diff --git a/tests/hermes_state/test_fts_optimize_progress.py b/tests/hermes_state/test_fts_optimize_progress.py
index 33137e7db8f..763d85c8690 100644
--- a/tests/hermes_state/test_fts_optimize_progress.py
+++ b/tests/hermes_state/test_fts_optimize_progress.py
@@ -6,11 +6,13 @@ import itertools
import sqlite3
import threading
import time
+from argparse import Namespace
import pytest
+from hermes_cli.sessions_cmd import _cmd_optimize_storage, _optimize_progress_line
from hermes_state import SessionDB
-from hermes_state_common import LEGACY_FTS_SQL, SCHEMA_SQL
+from hermes_state_common import LEGACY_FTS_SQL, LEGACY_FTS_TRIGRAM_SQL, SCHEMA_SQL
GAP_AFTER, GAP_IDS = 600, 100_000 # a pruned store: ids jump after the first 600 messages
@@ -19,7 +21,7 @@ def _legacy_inline_store(db_path, n=1200):
ids = [i + 1 + (GAP_IDS if i >= GAP_AFTER else 0) for i in range(n)]
raw = sqlite3.connect(str(db_path))
try:
- raw.executescript(SCHEMA_SQL + LEGACY_FTS_SQL)
+ raw.executescript(SCHEMA_SQL + LEGACY_FTS_SQL + LEGACY_FTS_TRIGRAM_SQL)
raw.execute("INSERT INTO sessions (id, source, started_at) VALUES ('s1', 'cli', ?)", (time.time(),))
raw.executemany(
"INSERT INTO messages (id, session_id, timestamp, role, content) VALUES (?, 's1', ?, 'user', ?)",
@@ -68,6 +70,10 @@ def test_progress_reports_teardown_rows_and_vacuum_liveness(legacy_db, monkeypat
# The ticker is stopped and joined on success and on failure: nothing can report after return.
assert not [t for t in threading.enumerate() if t.name == "optimize-vacuum-ticker"]
assert events[-1]["phase"] == "done"
+ # Backfill counts messages, not ids: after the first 500-row chunk the 100k-id gap still reads 500/1,200.
+ backfill = [e for e in events if e["phase"] == "backfill"]
+ assert {e["total"] for e in backfill} == {1200} and backfill[0]["indexed"] == 0
+ assert (backfill[1]["indexed"], backfill[1]["percent"]) == (500, 41)
vacuum = [e for e in events if e["phase"] == "vacuum"]
assert vacuum[0]["size_bytes"] >= vacuum[0]["estimated_bytes"] > 0
teardown = [e for e in events if e["phase"] == "teardown"]
@@ -76,7 +82,9 @@ def test_progress_reports_teardown_rows_and_vacuum_liveness(legacy_db, monkeypat
assert docsize[0]["total"] == 1200 and docsize[0]["indexed"] == 0
assert docsize[-1]["indexed"] == 1200 and docsize[-1]["percent"] == 100
count = teardown[0]["table_count"]
- assert count >= 2 and {e["table_index"] for e in teardown} <= set(range(1, count + 1))
+ # Both legacy indexes (base + trigram) are reclaimed: five shadow tables each.
+ assert count == 10 and {e["table_index"] for e in teardown} <= set(range(1, count + 1))
+ assert any(e["table"].endswith("messages_fts_trigram_data") and e["total"] > 0 for e in teardown)
assert legacy_db.search_messages("quokka")
@@ -87,8 +95,8 @@ def test_foreground_optimize_never_sleeps_and_chunks_by_rows_across_id_gaps(lega
monkeypatch.setattr(SessionDB, "_FTS_REBUILD_CHUNK_MAX_ROWS", 500)
markers, real_step = [0], legacy_db.fts_rebuild_step
- def observed_step():
- more = real_step()
+ def observed_step(*args):
+ more = real_step(*args)
status = legacy_db.fts_rebuild_status()
markers.append(status["indexed"] if status else GAP_IDS + 1200)
return more
@@ -103,21 +111,57 @@ def test_foreground_optimize_never_sleeps_and_chunks_by_rows_across_id_gaps(lega
assert windows == [500, 500, 200]
-def test_chunk_that_keeps_failing_ends_the_run_and_a_rerun_resumes(legacy_db, monkeypatch):
+def test_chunk_that_keeps_failing_ends_the_run_and_a_rerun_resumes(legacy_db, monkeypatch, capsys):
monkeypatch.setattr(time, "sleep", lambda _s: None) # the retry backoff
calls, real_step = [], legacy_db.fts_rebuild_step
- def disk_fills_during_backfill():
+ def disk_fills_during_backfill(*args):
if not calls:
pages = legacy_db._conn.execute("PRAGMA page_count").fetchone()[0]
legacy_db._conn.execute(f"PRAGMA max_page_count = {pages + 5}")
calls.append(1)
assert len(calls) < 50, "optimize kept retrying a chunk that can never succeed"
- return real_step()
+ return real_step(*args)
legacy_db.fts_rebuild_step = disk_fills_during_backfill
with pytest.raises(sqlite3.OperationalError, match="full"):
legacy_db.optimize_fts_storage(vacuum=False)
+ # The CLI reports the failure and exits non-zero (main() reads None as success).
+ calls.clear()
+ assert _cmd_optimize_storage(legacy_db, Namespace(yes=True, no_vacuum=True, force=False)) == 1
+ assert "Error: optimization failed: database or disk is full" in capsys.readouterr().out
legacy_db._conn.execute("PRAGMA max_page_count = 1073741823")
assert legacy_db.optimize_fts_storage(vacuum=False)["ok"] is True
assert len(legacy_db.search_messages("quokka", limit=2000)) == 1200
+
+
+def test_writer_that_opens_the_store_mid_run_gets_the_paced_cycle(legacy_db, monkeypatch):
+ slept, scans, events, writer_opened = [], [], [], threading.Event()
+ monkeypatch.setattr(time, "sleep", slept.append)
+ monkeypatch.setattr(SessionDB, "_HOLDER_SCAN_INTERVAL_S", 0.01)
+ monkeypatch.setattr(SessionDB, "_FTS_REBUILD_CHUNK_MAX_ROWS", 100)
+
+ def holder_scan():
+ scans.append(1)
+ if len(scans) == 1:
+ return [] # quiet when the run starts: no pacing
+ writer_opened.set()
+ return [(4242, str(legacy_db.db_path))]
+
+ monkeypatch.setattr(legacy_db, "_foreign_state_db_holders", holder_scan)
+ real_step = legacy_db.fts_rebuild_step
+
+ def step_then_writer_opens(*args):
+ more = real_step(*args)
+ assert writer_opened.wait(10) # e.g. Desktop launched after the first chunk
+ return more
+
+ legacy_db.fts_rebuild_step = step_then_writer_opens
+ assert legacy_db.optimize_fts_storage(progress_cb=events.append, vacuum=False)["ok"] is True
+ chunks = [e for e in events if e["phase"] != "done"]
+ assert chunks[0]["paced"] is False and all(e["paced"] for e in chunks[1:])
+ # Every chunk after the writer appeared yields the write lock for at least the duty-cycle floor.
+ teardown = [e for e in chunks if e["phase"] == "teardown"]
+ assert len(slept) >= len(teardown) > 0 and min(slept) >= SessionDB._FTS_REBUILD_MIN_PAUSE
+ assert "paced: another process" in _optimize_progress_line(chunks[-1])
+ assert not [t for t in threading.enumerate() if t.name == "optimize-holder-watch"]
diff --git a/website/docs/user-guide/session-storage-recovery.md b/website/docs/user-guide/session-storage-recovery.md
index ff545834a63..17ee06cdffb 100644
--- a/website/docs/user-guide/session-storage-recovery.md
+++ b/website/docs/user-guide/session-storage-recovery.md
@@ -78,7 +78,9 @@ Override with --force if you accept the risk.
`--dry-run` previews are never blocked. `--force` runs anyway — use it only when you know
the listed processes are idle (a reader you started yourself, for example). The same check
-runs when you type `sessions optimize` in the Desktop console.
+runs when you type `sessions optimize` in the Desktop console. `optimize-storage` keeps watching
+while it runs: whenever another process holds the database (under `--force`, or one that opens it
+mid-run) it pauses between chunks so that process's writes get through, which makes it much slower.
`hermes sessions prune` is the exception while the gateway is running: the gateway owns
`state.db`, so prune shows its preview from a read-only view and the gateway itself deletes