Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
fix(watches): recover from transient store errors instead of fusing permanently (#2251)
Call White10990362ddf5
diff --git a/core/scheduled_tasks.py b/core/scheduled_tasks.pyindex a2273f7f4..7991b2500 100644--- a/core/scheduled_tasks.py+++ b/core/scheduled_tasks.py@@ -1344,6 +1344,10 @@ class ScheduledTaskStore:it durable would mean writing to the very database that was just provenunwritable. It is cleared only by the reload that repairs the mirror (``load``),so a reload that fails again keeps retrying on every later tick.++ A reload that fails after the probe reported a commit sets the same flag: the+ probe has consumed that commit, and without the flag a task created during a+ store outage stays invisible until some unrelated commit."""with self._reload_lock:@@ -1365,7 +1369,11 @@ class ScheduledTaskStore:return Falsereturn Trueif changed:- self.load()+ try:+ self.load()+ except Exception:+ self._reload_required = True+ raisereturn changedsignature = _path_signature(self.path)if signature == self._signature and not self._reload_required:diff --git a/core/watches.py b/core/watches.pyindex c709d1f24..13c7c65d0 100644--- a/core/watches.py+++ b/core/watches.py@@ -7,6 +7,7 @@ import jsonimport loggingimport osimport threading+import timefrom dataclasses import asdict, dataclass, field, replacefrom datetime import datetime, timezonefrom functools import partial@@ -118,7 +119,15 @@ FOLLOW_UP_SLOT_READY = "ready"FOLLOW_UP_SLOT_STOPPED = "stopped"FOLLOW_UP_SLOT_LIFETIME_EXPIRED = "lifetime_expired"WATCH_RECONCILE_INTERVAL_SECONDS = 2.0-WATCH_STORE_RECONCILE_FUSE_FAILURES = 3+#: Consecutive reconcile store errors that pause reconciliation. A paused lane+#: stops its fast retries and probes the store once per regular scan instead,+#: resuming on the first scan that reconciles cleanly.+WATCH_STORE_RECONCILE_PAUSE_FAILURES = 3+#: Backoff for one watch whose own store call raised: the watch sits out this+#: long before reconcile restarts it, doubling per consecutive failure. A cycle+#: result that commits proves its store path again and clears the backoff.+WATCH_STORE_RETRY_INITIAL_SECONDS = 30.0+WATCH_STORE_RETRY_MAX_SECONDS = 600.0WATCH_RECOVERY_ENTRY_TIMEOUT_SECONDS = 2 * DEFAULT_PROCESS_TERMINATE_TIMEOUT_SECONDS@@ -383,6 +392,10 @@ class ManagedWatchStore:SQLite anyway, so there is nothing for it to survive. Making it durable wouldalso mean writing to the database that was just proven unwritable. Only ``load``clears it, so a reload that fails again is retried on every later tick.++ A reload that fails after the probe reported a commit sets the same flag: the+ probe has consumed that commit, and without the flag a definition created+ during a store outage stays invisible until some unrelated commit."""with self._mirror_lock:@@ -395,13 +408,17 @@ class ManagedWatchStore:# Still unreachable. Keep the flag and the incomplete mirror, and# report "nothing changed" -- the retry is the next tick's.logger.exception(- "Could not reload managed watches after a lost write; the live "- "store stays incomplete until a later attempt succeeds"+ "Could not reload managed watches; the live store stays "+ "stale until a later attempt succeeds")return Falsereturn Trueif changed:- self.load()+ try:+ self.load()+ except Exception:+ self._reload_required = True+ raisereturn changedsignature = _path_signature(self.path)if signature == self._signature and not self._reload_required:@@ -1074,8 +1091,7 @@ class _ManagedWatchRuntimeWorkHandler(RuntimeWorkHandler):)watches: tuple[ManagedWatch, ...] = ()store_error: Exception | None = None- fused = self.service._store_error_fused- if recovery.recovered and not fused:+ if recovery.recovered:try:self.service.store.maybe_reload()watches = tuple(@@ -1092,7 +1108,6 @@ class _ManagedWatchRuntimeWorkHandler(RuntimeWorkHandler):"unblocked": unblocked,"watches": watches,"store_error": store_error,- "fused": fused,},rearm_after_process=False,)@@ -1116,29 +1131,29 @@ class _ManagedWatchRuntimeWorkHandler(RuntimeWorkHandler):if observation["unblocked"]:self.service._runtime_state_dirty = Truestore_error = observation.get("store_error")+ if store_error is None:+ try:+ # Another shared-store reader may have consumed maybe_reload's change+ # result. Reconcile the snapshot, not which reader refreshed it; the+ # active-task map already makes unchanged reconciliation idempotent.+ if self.service.reconcile_watches(observation["watches"]):+ self.service._runtime_state_dirty = True+ except Exception as exc:+ store_error = excif store_error is not None:- self.service._handle_reconcile_store_error(store_error)- return self.service._store_error_fused- if observation.get("fused") or self.service._store_error_fused:+ if not self.service._handle_reconcile_store_error(store_error):+ return False+ # Paused: skip the fast retry. The next regular scan probes the store+ # again and resumes reconciliation once it reads cleanly.if self.service._runtime_state_dirty:await self.service._persist_runtime_state()- return not self.service._runtime_state_dirtyreturn True- try:- # Another shared-store reader may have consumed maybe_reload's change- # result. Reconcile the snapshot, not which reader refreshed it; the- # active-task map already makes unchanged reconciliation idempotent.- if self.service.reconcile_watches(observation["watches"]):- self.service._runtime_state_dirty = True+ self.service._note_store_reconcile_ok()+ if self.service._runtime_state_dirty:+ await self.service._persist_runtime_state()if self.service._runtime_state_dirty:- await self.service._persist_runtime_state()- if self.service._runtime_state_dirty:- return False- self.service._store_reconcile_failures = 0- return True- except Exception as exc:- self.service._handle_reconcile_store_error(exc)- return self.service._store_error_fused+ return False+ return Trueclass ManagedWatchService:@@ -1161,10 +1176,11 @@ class ManagedWatchService:self._active_pids: dict[str, int] = {}self._active_process_identities: dict[str, PersistedProcessIdentity] = {}self._watch_started_at: dict[str, str] = {}- self._fused_watch_ids: set[str] = set()+ #: watch_id -> (backoff seconds, monotonic retry deadline) after that+ #: watch's own store call raised.+ self._watch_store_retries: dict[str, tuple[float, float]] = {}self._recovery_blocked_watch_ids: set[str] = set()self._unreaped_runtime_entries: dict[str, dict[str, Any]] = {}- self._store_error_fused = Falseself._store_reconcile_failures = 0self._recovery_pending = Trueself._requires_service_lease = runtime.service_instance_lock_attached_to_process()@@ -1608,9 +1624,6 @@ class ManagedWatchService:continueself._recovery_pending = Falseself._runtime_state_dirty = True- if self._store_error_fused:- await asyncio.sleep(WATCH_RECONCILE_INTERVAL_SECONDS)- continuetry:if self._recovery_blocked_watch_ids:blocked = {@@ -1631,7 +1644,7 @@ class ManagedWatchService:self._runtime_state_dirty = Trueif self._runtime_state_dirty:self._write_runtime_state()- self._store_reconcile_failures = 0+ self._note_store_reconcile_ok()except asyncio.CancelledError:raiseexcept Exception as exc:@@ -1644,19 +1657,22 @@ class ManagedWatchService:) -> bool:if not self._owns_service_instance():return False- if self._store_error_fused:- return Falsewatches = self.store.list_watches() if watches is None else watchesdesired_ids = {watch.id for watch in watches if watch.enabled}+ for watch_id in self._watch_store_retries.keys() - desired_ids:+ del self._watch_store_retries[watch_id]changed = Falsefor watch in watches:if (not watch.enabledor watch.id in self._active_tasks- or watch.id in self._fused_watch_idsor watch.id in self._recovery_blocked_watch_ids):continue+ retry_in = self._watch_store_retry_in(watch.id)+ if retry_in > 0:+ self._schedule_runtime_work_wake(retry_in)+ continuetask = asyncio.create_task(self._run_watch(watch.id))self._active_tasks[watch.id] = tasktask.add_done_callback(lambda _task, watch_id=watch.id: self._on_watch_done(watch_id))@@ -1749,38 +1765,74 @@ class ManagedWatchService:self._runtime_state_dirty = Truelogger.exception("Failed to persist watch runtime state")- def _fuse_store_after_error(self, operation: str, exc: Exception, *, watch_id: str | None = None) -> None:- if watch_id is not None:- self._fused_watch_ids.add(watch_id)- self._store_error_fused = True+ def _back_off_watch_after_store_error(self, operation: str, exc: Exception, *, watch_id: str) -> None:+ """Keep one watch out of reconcile after its own store call raised.++ Only this watch backs off: a store error seen by one watch says nothing+ about the definitions reconcile has yet to start.+ """++ previous = self._watch_store_retries.get(watch_id)+ delay = (+ WATCH_STORE_RETRY_INITIAL_SECONDS+ if previous is None+ else min(previous[0] * 2, WATCH_STORE_RETRY_MAX_SECONDS)+ )+ self._watch_store_retries[watch_id] = (delay, time.monotonic() + delay)logger.error(- "Disabling watch store reconciliation after persistent store error "- "(watch_id=%s operation=%s): %s",+ "Watch %s stopped after a store error (operation=%s); retrying in %.0fs: %s",watch_id,operation,+ delay,exc,exc_info=True,)- def _handle_reconcile_store_error(self, exc: Exception) -> None:+ def _watch_store_retry_in(self, watch_id: str) -> float:+ retry = self._watch_store_retries.get(watch_id)+ if retry is None:+ return 0.0+ return max(0.0, retry[1] - time.monotonic())++ def _handle_reconcile_store_error(self, exc: Exception) -> bool:+ """Count one reconcile store error; ``True`` means reconciliation is paused."""+self._store_reconcile_failures += 1- if self._store_reconcile_failures >= WATCH_STORE_RECONCILE_FUSE_FAILURES:- self._fuse_store_after_error("reconcile", exc)- return- logger.warning(- "Managed watch reconcile failed; will retry "- "(attempt=%s/%s): %s",- self._store_reconcile_failures,- WATCH_STORE_RECONCILE_FUSE_FAILURES,- exc,- exc_info=True,- )+ failures = self._store_reconcile_failures+ if failures < WATCH_STORE_RECONCILE_PAUSE_FAILURES:+ logger.warning(+ "Managed watch reconcile failed; will retry (attempt=%s/%s): %s",+ failures,+ WATCH_STORE_RECONCILE_PAUSE_FAILURES,+ exc,+ exc_info=True,+ )+ elif failures == WATCH_STORE_RECONCILE_PAUSE_FAILURES:+ logger.error(+ "Pausing managed watch reconciliation after %s consecutive store errors; "+ "probing the store on each regular scan until it recovers: %s",+ failures,+ exc,+ exc_info=True,+ )+ else:+ logger.debug("Managed watch store still unavailable (attempt=%s): %s", failures, exc)+ return failures >= WATCH_STORE_RECONCILE_PAUSE_FAILURES++ def _note_store_reconcile_ok(self) -> None:+ if self._store_reconcile_failures >= WATCH_STORE_RECONCILE_PAUSE_FAILURES:+ logger.info(+ "Managed watch store recovered after %s failed reconcile attempts; "+ "reconciliation resumed",+ self._store_reconcile_failures,+ )+ self._store_reconcile_failures = 0def _watch_store_call(self, watch_id: str, operation: str, callback, *, guarded: bool = False) -> bool:"""Run a store call for ``watch_id``; ``False`` means "do not proceed".TWO ways a store call can fail to happen, and the supervisor has to stop for- BOTH. An exception fuses the store, as before. ``guarded=True`` says the+ BOTH. An exception backs the watch off, as before. ``guarded=True`` says thecallback's OWN return value is the answer as well: ``mark_cycle_start`` and``mark_cycle_result`` are compare-and-set writes (HFR-261) that return``False`` when a ``/new`` reclaim or an archive committed after the payload@@ -1798,8 +1850,8 @@ class ManagedWatchService:result = callback()if guarded and result is False:# NOT a store error: the row is fine, this write simply lost to a- # concurrent lifecycle change. Fusing would disable reconciliation for a- # healthy database, so the watch is stopped and the store left alone.+ # concurrent lifecycle change. Backing off would restart a watch whose+ # row has been torn down, so the watch is simply stopped.logger.warning("Watch %s stopping: the store refused %s because a guarded lifecycle ""or follow-up admission prerequisite changed",@@ -1826,7 +1878,7 @@ class ManagedWatchService:guarded=guarded,)except Exception as exc:- self._fuse_store_after_error(operation, exc, watch_id=watch_id)+ self._back_off_watch_after_store_error(operation, exc, watch_id=watch_id)return Falsedef _current_asyncio_task(self) -> Optional["asyncio.Task[Any]"]:@@ -1957,18 +2009,21 @@ class ManagedWatchService:result: _CycleResult,) -> bool:try:- return await self._run_runtime_sync(+ committed = await self._run_runtime_sync(self._commit_success_cycle,watch,result,)except Exception as exc:- self._fuse_store_after_error(+ self._back_off_watch_after_store_error("commit successful watch cycle",exc,watch_id=watch.id,)return False+ if committed:+ self._watch_store_retries.pop(watch.id, None)+ return committedasync def _wait_for_follow_up_slot(self,@@ -2003,7 +2058,7 @@ class ManagedWatchService:watch_id,)except Exception as exc:- self._fuse_store_after_error(+ self._back_off_watch_after_store_error("read follow-up fence",exc,watch_id=watch_id,@@ -2030,7 +2085,7 @@ class ManagedWatchService:try:run = await self._run_runtime_sync(self.request_store.get_run, run_id)except Exception as exc:- self._fuse_store_after_error(+ self._back_off_watch_after_store_error("read previous follow-up",exc,watch_id=watch_id,@@ -2138,7 +2193,7 @@ class ManagedWatchService:while self._running:if not self._owns_service_instance():return- if watch_id in self._fused_watch_ids:+ if self._watch_store_retry_in(watch_id) > 0:returnif not await self._watch_store_call_async(watch_id,@@ -2270,7 +2325,7 @@ class ManagedWatchService:# so ``vibe watch show`` can say the cycle ran and found nothing.## Through the async wrapper like every other result branch: a store- # failure here must fuse the store and stop the watch, not escape+ # failure here must back the watch off and stop it, not escape# ``_run_cycle`` and kill the task silently -- reconcile would restart# the same quiet cycle and hide the storage problem instead of# surfacing it.@@ -2584,18 +2639,21 @@ class ManagedWatchService:**kwargs: Any,) -> bool:try:- return await self._run_runtime_sync(+ committed = await self._run_runtime_sync(self._commit_cycle_result,watch,**kwargs,)except Exception as exc:- self._fuse_store_after_error(+ self._back_off_watch_after_store_error("mark_cycle_result",exc,watch_id=watch.id,)return False+ if committed:+ self._watch_store_retries.pop(watch.id, None)+ return committeddef _stop_watch_for_missing_cwd(self, watch: ManagedWatch, *, error_text: str) -> None:watch_label = watch.name or watch.id@@ -2624,7 +2682,7 @@ class ManagedWatchService:error_text=error_text,)except Exception as exc:- self._fuse_store_after_error(+ self._back_off_watch_after_store_error("mark_cycle_result",exc,watch_id=watch.id,diff --git a/tests/test_scheduled_tasks.py b/tests/test_scheduled_tasks.pyindex a731dcb5f..e7cfde137 100644--- a/tests/test_scheduled_tasks.py+++ b/tests/test_scheduled_tasks.py@@ -16496,6 +16496,52 @@ def test_a_dropped_task_mirror_recovers_with_no_unrelated_commit_to_wake_it() ->)+def test_a_task_committed_before_a_failed_reload_is_not_lost_with_the_probe() -> None:+ """The commit a failed reload consumed must still reach the mirror.++ The watch store's twin. ``PRAGMA data_version`` reports another connection's+ commit once. When the reload it triggers fails -- as every read does while the+ disk is full -- a later probe no longer sees that commit, so a task created+ during the outage stayed enabled in SQLite and unscheduled until an unrelated+ commit or a restart. Nothing commits between the failure and the recovery+ below, so the mirror can only heal because the store remembered to reload.+ """+ from storage.db import SqliteInvalidationProbe, create_sqlite_engine++ store = ScheduledTaskStore()+ assert store._sqlite is not None, "this test is about the SQLite probe"+ store.maybe_reload()+ assert store.maybe_reload() is False, "the store's probe is not settled"++ created = ScheduledTaskStore().add_task(**_TASK_FIXTURE_PAYLOAD)+ witness_engine = create_sqlite_engine(store._sqlite.db_path)+ witness = SqliteInvalidationProbe(witness_engine)+ witness.has_external_write()+ assert witness.has_external_write() is False, "the witness probe is not settled"++ fault = _fail_the_definition_write_and_the_reload(store._sqlite.engine)+ try:+ with pytest.raises(Exception): # noqa: B017 - the fault type is the injected one's+ store.maybe_reload()+ assert fault["reads"] >= 1, "the probe did not report the commit, so no reload failed"+ assert store.get_task(created.id) is None, "the mirror already holds the new task"++ fault["live"] = False+ assert witness.has_external_write() is False, (+ "something COMMITTED after the failed reload; a data_version bump heals "+ "the mirror on its own, so this test would pass without the fix"+ )+ assert store.maybe_reload() is True, (+ "the failed reload consumed the only change signal, so the task created "+ "during the outage is never scheduled"+ )+ assert [item.id for item in store.list_tasks()] == [created.id]+ assert store.maybe_reload() is False, "the repairing reload must clear the flag"+ finally:+ witness.close()+ witness_engine.dispose()++# --- Command tasks: a definition that runs a subprocess instead of an Agent turn ---diff --git a/tests/test_watches.py b/tests/test_watches.pyindex f34fdac93..c836a448e 100644--- a/tests/test_watches.py+++ b/tests/test_watches.py@@ -128,22 +128,28 @@ async def _start_watch_service(service: ManagedWatchService) -> None:await startup_task-async def _await_fused_watch(+async def _await_backed_off_watch(service: ManagedWatchService,watch_id: str,timeout: float = 10.0,) -> None:- """Wait for a watch to be fused instead of sleeping a fixed interval.+ """Wait for a watch to back off instead of sleeping a fixed interval.- Fusing needs a real waiter subprocess to start and its result to reach the- raising store, so the wait covers a process spawn plus interpreter startup.- A fixed 0.12s was enough locally and not on a loaded CI runner, where the- assertion saw an empty fused set.+ Backing off needs a real waiter subprocess to start and its result to reach+ the raising store, so the wait covers a process spawn plus interpreter+ startup. A fixed 0.12s was enough locally and not on a loaded CI runner."""deadline = time.monotonic() + timeout- while watch_id not in service._fused_watch_ids:- assert time.monotonic() < deadline, "watch was never fused after the store error"+ while watch_id not in service._watch_store_retries:+ assert time.monotonic() < deadline, "watch never backed off after the store error"+ await asyncio.sleep(0.02)+++async def _await_condition(predicate, message: str, timeout: float = 10.0) -> None:+ deadline = time.monotonic() + timeout+ while not predicate():+ assert time.monotonic() < deadline, messageawait asyncio.sleep(0.02)@@ -2366,67 +2372,105 @@ def test_managed_watch_service_forever_non_retry_error_disables_and_enqueues_faiassert pending[0].prompt.startswith("Investigate the failure.\n\nWatch 'Broken forever waiter' stopped because the waiter exited with code 1.")-def test_managed_watch_service_fuses_watch_after_store_error(tmp_path: Path) -> None:- class FailingResultStore(ManagedWatchStore):+def test_managed_watch_service_backs_off_one_watch_after_its_store_error(+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """A store error stops one watch for a doubling backoff, not the whole service.++ The watch sits out each backoff instead of restarting into the same error, and+ runs again once the store accepts its cycle result. A watch created meanwhile+ starts at once: a store error one watch saw says nothing about the others, and+ a service-wide fuse left every Watch created after a brief disk-full episode+ unstarted until the service restarted.+ """++ monkeypatch.setattr("core.watches.WATCH_RECONCILE_INTERVAL_SECONDS", 0.01)+ monkeypatch.setattr("core.watches.WATCH_STORE_RETRY_INITIAL_SECONDS", 0.2)++ class FlakyResultStore(ManagedWatchStore):def __init__(self, path: Path):super().__init__(path)- self.starts = 0+ self.starts: dict[str, list[float]] = {}+ self.failures: dict[str, list[float]] = {}+ self.failing: dict[str, int] = {}def mark_cycle_start(self, watch_id: str) -> bool:- self.starts += 1+ self.starts.setdefault(watch_id, []).append(time.monotonic())return super().mark_cycle_start(watch_id)- def mark_cycle_result(self, *args, **kwargs) -> bool:- raise RuntimeError("database disk image is malformed")+ def mark_cycle_result(self, watch_id: str, *args, **kwargs) -> bool:+ if self.failing.get(watch_id, 0) > 0:+ self.failing[watch_id] -= 1+ self.failures.setdefault(watch_id, []).append(time.monotonic())+ raise RuntimeError("unable to open database file")+ return super().mark_cycle_result(watch_id, *args, **kwargs)++ def add_retrying_watch(name: str) -> watches_module.ManagedWatch:+ return store.add_watch(+ name=name,+ session_key="slack::channel::C123",+ command=[sys.executable, "-c", "import sys; sys.exit(75)"],+ shell_command=None,+ prefix="Should not storm.",+ cwd=None,+ mode="forever",+ timeout_seconds=5,+ lifetime_timeout_seconds=0,+ retry_exit_codes=[75],+ retry_delay_seconds=0.01,+ post_to=None,+ deliver_key=None,+ )- store = FailingResultStore(tmp_path / "watches.json")- request_store = TaskExecutionStore(tmp_path / "task_requests")- runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")- watch = store.add_watch(- name="Broken persistence",- session_key="slack::channel::C123",- command=[sys.executable, "-c", "import sys; sys.exit(75)"],- shell_command=None,- prefix="Should not storm.",- cwd=None,- mode="forever",- timeout_seconds=5,- lifetime_timeout_seconds=0,- retry_exit_codes=[75],- retry_delay_seconds=0.01,- post_to=None,- deliver_key=None,- )+ store = FlakyResultStore(tmp_path / "watches.json")+ broken = add_retrying_watch("Broken persistence")+ store.failing[broken.id] = 2service = ManagedWatchService(controller=SimpleNamespace(),store=store,- request_store=request_store,- runtime_store=runtime_store,+ request_store=TaskExecutionStore(tmp_path / "task_requests"),+ runtime_store=WatchRuntimeStateStore(tmp_path / "watch_runtime.json"),)async def _run() -> None:await _start_watch_service(service)- await _await_fused_watch(service, watch.id)- # Settle briefly: a fused store must not let the cycle start again.- await asyncio.sleep(0.08)+ await _await_backed_off_watch(service, broken.id)+ created = add_retrying_watch("Created during the outage")+ await _await_condition(+ lambda: created.id in store.starts,+ "a watch created after another watch's store error never started",+ )+ await _await_condition(+ lambda: store.failing[broken.id] == 0 and broken.id not in service._watch_store_retries,+ "the backed-off watch never committed a cycle after the store recovered",+ )await service.stop()asyncio.run(_run())- assert store.starts == 1- assert service._store_error_fused is True- assert request_store.list_pending() == []+ starts = store.starts[broken.id]+ failures = store.failures[broken.id]+ assert len(failures) == 2+ assert len(starts) >= 3+ assert starts[1] - failures[0] >= 0.2, "the watch restarted before its backoff elapsed"+ assert starts[2] - failures[1] >= 0.4, "the second backoff did not double"-def test_managed_watch_service_fuses_quiet_cycle_after_store_error(tmp_path: Path) -> None:+def test_managed_watch_service_backs_off_quiet_cycle_after_store_error(+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+) -> None:"""A quiet cycle commits through the same wrapper as every other result branch.Called synchronously, a raising ``mark_cycle_result`` escapes ``_run_cycle`` and- kills the watch task with the store unfused and the cycle unrecorded. Reconcile- then restarts the same quiet cycle and repeats it, so a real storage failure- never reaches the user as one.+ kills the watch task with no backoff and the cycle unrecorded. Reconcile then+ restarts the same quiet cycle and repeats it, so a real storage failure never+ reaches the user as one."""+ monkeypatch.setattr("core.watches.WATCH_RECONCILE_INTERVAL_SECONDS", 0.01)+class FailingResultStore(ManagedWatchStore):def __init__(self, path: Path):super().__init__(path)@@ -2466,57 +2510,84 @@ def test_managed_watch_service_fuses_quiet_cycle_after_store_error(tmp_path: Patasync def _run() -> None:await _start_watch_service(service)- await _await_fused_watch(service, watch.id)- # Settle briefly: a fused store must not let the cycle start again.+ await _await_backed_off_watch(service, watch.id)+ # Settle across several reconcile ticks: the backoff must keep the quiet+ # cycle from starting again.await asyncio.sleep(0.08)await service.stop()asyncio.run(_run())- assert store.starts == 1, "the fused store let the quiet cycle restart"- assert service._store_error_fused is True+ assert store.starts == 1, "reconcile restarted the quiet cycle inside its backoff"# A quiet cycle authorises no hook, and a failed one must not invent it.assert request_store.list_pending() == []-def test_managed_watch_service_fuses_reconcile_after_store_read_error(tmp_path: Path, monkeypatch) -> None:+def test_managed_watch_service_resumes_reconcile_after_store_outage(+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+ caplog: pytest.LogCaptureFixture,+) -> None:+ """The polling loop pauses on a store outage and resumes when the store heals.++ A pause quiets the logs; it must not stop the probing. Before the fix the+ loop stopped reading after three errors, so a watch the store listed again+ once the outage ended never started until the service restarted.+ """+monkeypatch.setattr("core.watches.WATCH_RECONCILE_INTERVAL_SECONDS", 0.01)+ outage_reads = watches_module.WATCH_STORE_RECONCILE_PAUSE_FAILURES + 3- class FailingListStore(ManagedWatchStore):+ class OutageListStore(ManagedWatchStore):def __init__(self, path: Path):super().__init__(path)- self.calls = 0+ self.failures_remaining = outage_readsdef list_watches(self):- self.calls += 1- raise RuntimeError("database disk image is malformed")+ if self.failures_remaining > 0:+ self.failures_remaining -= 1+ raise RuntimeError("unable to open database file")+ return super().list_watches()- store = FailingListStore(tmp_path / "watches.json")- request_store = TaskExecutionStore(tmp_path / "task_requests")- runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")+ store = OutageListStore(tmp_path / "watches.json")+ watch = store.add_watch(+ name="Waiting out the outage",+ session_key="slack::channel::C123",+ command=[sys.executable, "-c", "import time; time.sleep(30)"],+ shell_command=None,+ prefix="Done.",+ cwd=None,+ mode="once",+ timeout_seconds=60,+ lifetime_timeout_seconds=0,+ retry_exit_codes=[],+ retry_delay_seconds=0,+ post_to=None,+ deliver_key=None,+ )service = ManagedWatchService(controller=SimpleNamespace(),store=store,- request_store=request_store,- runtime_store=runtime_store,+ request_store=TaskExecutionStore(tmp_path / "task_requests"),+ runtime_store=WatchRuntimeStateStore(tmp_path / "watch_runtime.json"),)async def _run() -> None:- service._running = True- task = asyncio.create_task(service._watch_store())- await asyncio.sleep(0.05)- service._running = False- task.cancel()- try:- await task- except asyncio.CancelledError:- pass+ await _start_watch_service(service)+ await _await_condition(+ lambda: watch.id in service._active_tasks,+ "the watch never started after the store outage ended",+ )+ await service.stop()- asyncio.run(_run())+ with caplog.at_level(logging.INFO, logger="core.watches"):+ asyncio.run(_run())- assert service._store_error_fused is True- assert store.calls == 3- assert service._active_tasks == {}+ assert store.failures_remaining == 0+ assert service._store_reconcile_failures == 0+ pauses = [record for record in caplog.records if "Pausing managed watch reconciliation" in record.message]+ assert len(pauses) == 1, "the pause must be reported once, not per probe"+ assert any("reconciliation resumed" in record.message for record in caplog.records)def test_managed_watch_service_idle_tick_does_not_write_runtime_state(tmp_path: Path, monkeypatch) -> None:@@ -3387,7 +3458,6 @@ def test_hfr_179_blocked_watch_recovery_arms_generation_scoped_recheck("unblocked": (),"watches": (),"store_error": None,- "fused": False,},rearm_after_process=False,)@@ -3419,7 +3489,6 @@ def test_hfr_179_pending_watch_recovery_uses_watch_lane_cadence("unblocked": (),"watches": (),"store_error": None,- "fused": False,},rearm_after_process=False,)@@ -3429,10 +3498,20 @@ def test_hfr_179_pending_watch_recovery_uses_watch_lane_cadence(assert delayed == [(token, watches_module.WATCH_RECONCILE_INTERVAL_SECONDS)]-def test_hfr_179_watch_store_fuse_stops_generation_reads_and_retries(+def test_hfr_179_paused_watch_store_keeps_probing_and_resumes(tmp_path: Path,monkeypatch: pytest.MonkeyPatch,+ caplog: pytest.LogCaptureFixture,) -> None:+ """A store outage pauses the watch lane; it must not end it.++ Consecutive read errors pause reconciliation: the lane drops its fast retry+ and reports the pause once. Every later scan still probes the store, so the+ lane reconciles again on the first clean read. The old one-way fuse stopped+ reading for good, and Watches created after a brief "unable to open database+ file" episode never started until the service restarted.+ """+service = ManagedWatchService(controller=SimpleNamespace(),store=ManagedWatchStore(tmp_path / "watches.json"),@@ -3441,52 +3520,69 @@ def test_hfr_179_watch_store_fuse_stops_generation_reads_and_retries()service._recovery_pending = Falseservice._runtime_state_dirty = False+ watch = service.store.add_watch(**_WATCH_FIXTURE_PAYLOAD)+ healthy = Falsereads = 0+ reload = service.store.maybe_reload- def fail_read() -> bool:+ def flaky_read() -> bool:nonlocal readsreads += 1- raise OSError("watch store unavailable")+ if not healthy:+ raise OSError("unable to open database file")+ return reload()- monkeypatch.setattr(service.store, "maybe_reload", fail_read)- handler = _ManagedWatchRuntimeWorkHandler(service)+ reconciled: list[list[str]] = []- async def _run() -> None:- outcomes = []- for _ in range(watches_module.WATCH_STORE_RECONCILE_FUSE_FAILURES):- items, _has_more = await asyncio.to_thread(- handler.scan,- limit=1,- occupied=frozenset(),- cursor=None,- )- outcomes.append(await handler.process(items[0]))+ def record_reconcile(watches) -> bool:+ reconciled.append([item.id for item in watches])+ return False- assert outcomes[:-1] == [False] * (len(outcomes) - 1)- assert outcomes[-1] is True- assert service._store_error_fused is True- assert reads == watches_module.WATCH_STORE_RECONCILE_FUSE_FAILURES+ persisted = 0- persisted = 0+ async def persist_runtime_state() -> None:+ nonlocal persisted+ persisted += 1+ service._runtime_state_dirty = False- async def persist_runtime_state() -> None:- nonlocal persisted- persisted += 1- service._runtime_state_dirty = False+ monkeypatch.setattr(service.store, "maybe_reload", flaky_read)+ monkeypatch.setattr(service, "reconcile_watches", record_reconcile)+ monkeypatch.setattr(service, "_persist_runtime_state", persist_runtime_state)+ handler = _ManagedWatchRuntimeWorkHandler(service)+ pause_after = watches_module.WATCH_STORE_RECONCILE_PAUSE_FAILURES- monkeypatch.setattr(service, "_persist_runtime_state", persist_runtime_state)- service._runtime_state_dirty = True+ async def scan_and_process() -> bool:items, _has_more = await asyncio.to_thread(handler.scan,limit=1,occupied=frozenset(),cursor=None,)- assert await handler.process(items[0]) is True- assert reads == watches_module.WATCH_STORE_RECONCILE_FUSE_FAILURES- assert persisted == 1+ return await handler.process(items[0])- asyncio.run(_run())+ async def _run() -> None:+ nonlocal healthy+ outcomes = [await scan_and_process() for _ in range(pause_after)]+ # ``False`` asks the supervisor for a fast retry; the pause stops asking.+ assert outcomes == [False] * (pause_after - 1) + [True]++ service._runtime_state_dirty = True+ assert await scan_and_process() is True, "a paused lane must not fast-retry"+ assert reads == pause_after + 1, "the paused lane stopped probing the store"+ assert persisted == 1, "runtime state must still persist while paused"+ assert reconciled == []++ healthy = True+ assert await scan_and_process() is True+ assert reconciled == [[watch.id]], "the lane did not resume once the store read cleanly"+ assert service._store_reconcile_failures == 0++ with caplog.at_level(logging.DEBUG, logger="core.watches"):+ asyncio.run(_run())++ errors = [record for record in caplog.records if record.levelno >= logging.ERROR]+ assert len(errors) == 1 and "Pausing managed watch reconciliation" in errors[0].message+ assert any("reconciliation resumed" in record.message for record in caplog.records)def test_watch_runtime_state_persistence_failure_rearms_maintenance(@@ -3892,7 +3988,6 @@ def test_managed_watch_service_retries_transient_reconcile_errors(tmp_path: Pathasyncio.run(_run())- assert service._store_error_fused is Falseassert service._store_reconcile_failures == 0@@ -3913,7 +4008,6 @@ def test_managed_watch_service_start_retries_initial_reconcile_error(tmp_path: Pasync def _run() -> None:await _start_watch_service(service)- assert service._store_error_fused is Falseassert 1 <= service._store_reconcile_failures < 3await service.stop()@@ -5376,3 +5470,49 @@ def test_managed_watch_service_names_the_watch_in_the_cycle_environment(tmp_pathasyncio.run(_run())assert echoed.read_text(encoding="utf-8") == watch.id+++def test_a_watch_committed_before_a_failed_reload_is_not_lost_with_the_probe() -> None:+ """The commit a failed reload consumed must still reach the mirror.++ ``PRAGMA data_version`` reports another connection's commit once. When the+ reload it triggers fails -- as every read does while the disk is full -- a+ later probe no longer sees that commit, so a watch created during the outage+ stayed enabled in SQLite and invisible to reconcile until an unrelated commit+ or a restart. Nothing commits between the failure and the recovery below, so+ the mirror can only heal because the store remembered it must reload.+ """+ from storage.db import SqliteInvalidationProbe, create_sqlite_engine++ store = ManagedWatchStore()+ assert store._sqlite is not None, "this test is about the SQLite probe"+ store.maybe_reload()+ assert store.maybe_reload() is False, "the store's probe is not settled"++ created = ManagedWatchStore().add_watch(**_WATCH_FIXTURE_PAYLOAD)+ witness_engine = create_sqlite_engine(store._sqlite.db_path)+ witness = SqliteInvalidationProbe(witness_engine)+ witness.has_external_write()+ assert witness.has_external_write() is False, "the witness probe is not settled"++ fault = _fail_the_definition_write_and_the_reload(store._sqlite.engine)+ try:+ with pytest.raises(Exception): # noqa: B017 - the fault type is the injected one's+ store.maybe_reload()+ assert fault["reads"] >= 1, "the probe did not report the commit, so no reload failed"+ assert store.get_watch(created.id) is None, "the mirror already holds the new watch"++ fault["live"] = False+ assert witness.has_external_write() is False, (+ "something COMMITTED after the failed reload; a data_version bump heals "+ "the mirror on its own, so this test would pass without the fix"+ )+ assert store.maybe_reload() is True, (+ "the failed reload consumed the only change signal, so the watch created "+ "during the outage stays invisible to reconcile"+ )+ assert [item.id for item in store.list_watches()] == [created.id]+ assert store.maybe_reload() is False, "the repairing reload must clear the flag"+ finally:+ witness.close()+ witness_engine.dispose()