Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

fix(watches): recover from transient store errors instead of fusing permanently (#2251)

Call White10990362ddf5
diff --git a/core/scheduled_tasks.py b/core/scheduled_tasks.py
index a2273f7f4..7991b2500 100644
--- a/core/scheduled_tasks.py
+++ b/core/scheduled_tasks.py
@@ -1344,6 +1344,10 @@ class ScheduledTaskStore:
it durable would mean writing to the very database that was just proven
unwritable. It is cleared only by the reload that repairs the mirror (``load``),
so a reload that fails again keeps retrying on every later tick.
+
+ A reload that fails after the probe reported a commit sets the same flag: the
+ probe has consumed that commit, and without the flag a task created during a
+ store outage stays invisible until some unrelated commit.
"""
with self._reload_lock:
@@ -1365,7 +1369,11 @@ class ScheduledTaskStore:
return False
return True
if changed:
- self.load()
+ try:
+ self.load()
+ except Exception:
+ self._reload_required = True
+ raise
return changed
signature = _path_signature(self.path)
if signature == self._signature and not self._reload_required:
diff --git a/core/watches.py b/core/watches.py
index c709d1f24..13c7c65d0 100644
--- a/core/watches.py
+++ b/core/watches.py
@@ -7,6 +7,7 @@ import json
import logging
import os
import threading
+import time
from dataclasses import asdict, dataclass, field, replace
from datetime import datetime, timezone
from functools import partial
@@ -118,7 +119,15 @@ FOLLOW_UP_SLOT_READY = "ready"
FOLLOW_UP_SLOT_STOPPED = "stopped"
FOLLOW_UP_SLOT_LIFETIME_EXPIRED = "lifetime_expired"
WATCH_RECONCILE_INTERVAL_SECONDS = 2.0
-WATCH_STORE_RECONCILE_FUSE_FAILURES = 3
+#: Consecutive reconcile store errors that pause reconciliation. A paused lane
+#: stops its fast retries and probes the store once per regular scan instead,
+#: resuming on the first scan that reconciles cleanly.
+WATCH_STORE_RECONCILE_PAUSE_FAILURES = 3
+#: Backoff for one watch whose own store call raised: the watch sits out this
+#: long before reconcile restarts it, doubling per consecutive failure. A cycle
+#: result that commits proves its store path again and clears the backoff.
+WATCH_STORE_RETRY_INITIAL_SECONDS = 30.0
+WATCH_STORE_RETRY_MAX_SECONDS = 600.0
WATCH_RECOVERY_ENTRY_TIMEOUT_SECONDS = 2 * DEFAULT_PROCESS_TERMINATE_TIMEOUT_SECONDS
@@ -383,6 +392,10 @@ class ManagedWatchStore:
SQLite anyway, so there is nothing for it to survive. Making it durable would
also mean writing to the database that was just proven unwritable. Only ``load``
clears it, so a reload that fails again is retried on every later tick.
+
+ A reload that fails after the probe reported a commit sets the same flag: the
+ probe has consumed that commit, and without the flag a definition created
+ during a store outage stays invisible until some unrelated commit.
"""
with self._mirror_lock:
@@ -395,13 +408,17 @@ class ManagedWatchStore:
# Still unreachable. Keep the flag and the incomplete mirror, and
# report "nothing changed" -- the retry is the next tick's.
logger.exception(
- "Could not reload managed watches after a lost write; the live "
- "store stays incomplete until a later attempt succeeds"
+ "Could not reload managed watches; the live store stays "
+ "stale until a later attempt succeeds"
)
return False
return True
if changed:
- self.load()
+ try:
+ self.load()
+ except Exception:
+ self._reload_required = True
+ raise
return changed
signature = _path_signature(self.path)
if signature == self._signature and not self._reload_required:
@@ -1074,8 +1091,7 @@ class _ManagedWatchRuntimeWorkHandler(RuntimeWorkHandler):
)
watches: tuple[ManagedWatch, ...] = ()
store_error: Exception | None = None
- fused = self.service._store_error_fused
- if recovery.recovered and not fused:
+ if recovery.recovered:
try:
self.service.store.maybe_reload()
watches = tuple(
@@ -1092,7 +1108,6 @@ class _ManagedWatchRuntimeWorkHandler(RuntimeWorkHandler):
"unblocked": unblocked,
"watches": watches,
"store_error": store_error,
- "fused": fused,
},
rearm_after_process=False,
)
@@ -1116,29 +1131,29 @@ class _ManagedWatchRuntimeWorkHandler(RuntimeWorkHandler):
if observation["unblocked"]:
self.service._runtime_state_dirty = True
store_error = observation.get("store_error")
+ if store_error is None:
+ try:
+ # Another shared-store reader may have consumed maybe_reload's change
+ # result. Reconcile the snapshot, not which reader refreshed it; the
+ # active-task map already makes unchanged reconciliation idempotent.
+ if self.service.reconcile_watches(observation["watches"]):
+ self.service._runtime_state_dirty = True
+ except Exception as exc:
+ store_error = exc
if store_error is not None:
- self.service._handle_reconcile_store_error(store_error)
- return self.service._store_error_fused
- if observation.get("fused") or self.service._store_error_fused:
+ if not self.service._handle_reconcile_store_error(store_error):
+ return False
+ # Paused: skip the fast retry. The next regular scan probes the store
+ # again and resumes reconciliation once it reads cleanly.
if self.service._runtime_state_dirty:
await self.service._persist_runtime_state()
- return not self.service._runtime_state_dirty
return True
- try:
- # Another shared-store reader may have consumed maybe_reload's change
- # result. Reconcile the snapshot, not which reader refreshed it; the
- # active-task map already makes unchanged reconciliation idempotent.
- if self.service.reconcile_watches(observation["watches"]):
- self.service._runtime_state_dirty = True
+ self.service._note_store_reconcile_ok()
+ if self.service._runtime_state_dirty:
+ await self.service._persist_runtime_state()
if self.service._runtime_state_dirty:
- await self.service._persist_runtime_state()
- if self.service._runtime_state_dirty:
- return False
- self.service._store_reconcile_failures = 0
- return True
- except Exception as exc:
- self.service._handle_reconcile_store_error(exc)
- return self.service._store_error_fused
+ return False
+ return True
class ManagedWatchService:
@@ -1161,10 +1176,11 @@ class ManagedWatchService:
self._active_pids: dict[str, int] = {}
self._active_process_identities: dict[str, PersistedProcessIdentity] = {}
self._watch_started_at: dict[str, str] = {}
- self._fused_watch_ids: set[str] = set()
+ #: watch_id -> (backoff seconds, monotonic retry deadline) after that
+ #: watch's own store call raised.
+ self._watch_store_retries: dict[str, tuple[float, float]] = {}
self._recovery_blocked_watch_ids: set[str] = set()
self._unreaped_runtime_entries: dict[str, dict[str, Any]] = {}
- self._store_error_fused = False
self._store_reconcile_failures = 0
self._recovery_pending = True
self._requires_service_lease = runtime.service_instance_lock_attached_to_process()
@@ -1608,9 +1624,6 @@ class ManagedWatchService:
continue
self._recovery_pending = False
self._runtime_state_dirty = True
- if self._store_error_fused:
- await asyncio.sleep(WATCH_RECONCILE_INTERVAL_SECONDS)
- continue
try:
if self._recovery_blocked_watch_ids:
blocked = {
@@ -1631,7 +1644,7 @@ class ManagedWatchService:
self._runtime_state_dirty = True
if self._runtime_state_dirty:
self._write_runtime_state()
- self._store_reconcile_failures = 0
+ self._note_store_reconcile_ok()
except asyncio.CancelledError:
raise
except Exception as exc:
@@ -1644,19 +1657,22 @@ class ManagedWatchService:
) -> bool:
if not self._owns_service_instance():
return False
- if self._store_error_fused:
- return False
watches = self.store.list_watches() if watches is None else watches
desired_ids = {watch.id for watch in watches if watch.enabled}
+ for watch_id in self._watch_store_retries.keys() - desired_ids:
+ del self._watch_store_retries[watch_id]
changed = False
for watch in watches:
if (
not watch.enabled
or watch.id in self._active_tasks
- or watch.id in self._fused_watch_ids
or watch.id in self._recovery_blocked_watch_ids
):
continue
+ retry_in = self._watch_store_retry_in(watch.id)
+ if retry_in > 0:
+ self._schedule_runtime_work_wake(retry_in)
+ continue
task = asyncio.create_task(self._run_watch(watch.id))
self._active_tasks[watch.id] = task
task.add_done_callback(lambda _task, watch_id=watch.id: self._on_watch_done(watch_id))
@@ -1749,38 +1765,74 @@ class ManagedWatchService:
self._runtime_state_dirty = True
logger.exception("Failed to persist watch runtime state")
- def _fuse_store_after_error(self, operation: str, exc: Exception, *, watch_id: str | None = None) -> None:
- if watch_id is not None:
- self._fused_watch_ids.add(watch_id)
- self._store_error_fused = True
+ def _back_off_watch_after_store_error(self, operation: str, exc: Exception, *, watch_id: str) -> None:
+ """Keep one watch out of reconcile after its own store call raised.
+
+ Only this watch backs off: a store error seen by one watch says nothing
+ about the definitions reconcile has yet to start.
+ """
+
+ previous = self._watch_store_retries.get(watch_id)
+ delay = (
+ WATCH_STORE_RETRY_INITIAL_SECONDS
+ if previous is None
+ else min(previous[0] * 2, WATCH_STORE_RETRY_MAX_SECONDS)
+ )
+ self._watch_store_retries[watch_id] = (delay, time.monotonic() + delay)
logger.error(
- "Disabling watch store reconciliation after persistent store error "
- "(watch_id=%s operation=%s): %s",
+ "Watch %s stopped after a store error (operation=%s); retrying in %.0fs: %s",
watch_id,
operation,
+ delay,
exc,
exc_info=True,
)
- def _handle_reconcile_store_error(self, exc: Exception) -> None:
+ def _watch_store_retry_in(self, watch_id: str) -> float:
+ retry = self._watch_store_retries.get(watch_id)
+ if retry is None:
+ return 0.0
+ return max(0.0, retry[1] - time.monotonic())
+
+ def _handle_reconcile_store_error(self, exc: Exception) -> bool:
+ """Count one reconcile store error; ``True`` means reconciliation is paused."""
+
self._store_reconcile_failures += 1
- if self._store_reconcile_failures >= WATCH_STORE_RECONCILE_FUSE_FAILURES:
- self._fuse_store_after_error("reconcile", exc)
- return
- logger.warning(
- "Managed watch reconcile failed; will retry "
- "(attempt=%s/%s): %s",
- self._store_reconcile_failures,
- WATCH_STORE_RECONCILE_FUSE_FAILURES,
- exc,
- exc_info=True,
- )
+ failures = self._store_reconcile_failures
+ if failures < WATCH_STORE_RECONCILE_PAUSE_FAILURES:
+ logger.warning(
+ "Managed watch reconcile failed; will retry (attempt=%s/%s): %s",
+ failures,
+ WATCH_STORE_RECONCILE_PAUSE_FAILURES,
+ exc,
+ exc_info=True,
+ )
+ elif failures == WATCH_STORE_RECONCILE_PAUSE_FAILURES:
+ logger.error(
+ "Pausing managed watch reconciliation after %s consecutive store errors; "
+ "probing the store on each regular scan until it recovers: %s",
+ failures,
+ exc,
+ exc_info=True,
+ )
+ else:
+ logger.debug("Managed watch store still unavailable (attempt=%s): %s", failures, exc)
+ return failures >= WATCH_STORE_RECONCILE_PAUSE_FAILURES
+
+ def _note_store_reconcile_ok(self) -> None:
+ if self._store_reconcile_failures >= WATCH_STORE_RECONCILE_PAUSE_FAILURES:
+ logger.info(
+ "Managed watch store recovered after %s failed reconcile attempts; "
+ "reconciliation resumed",
+ self._store_reconcile_failures,
+ )
+ self._store_reconcile_failures = 0
def _watch_store_call(self, watch_id: str, operation: str, callback, *, guarded: bool = False) -> bool:
"""Run a store call for ``watch_id``; ``False`` means "do not proceed".
TWO ways a store call can fail to happen, and the supervisor has to stop for
- BOTH. An exception fuses the store, as before. ``guarded=True`` says the
+ BOTH. An exception backs the watch off, as before. ``guarded=True`` says the
callback's OWN return value is the answer as well: ``mark_cycle_start`` and
``mark_cycle_result`` are compare-and-set writes (HFR-261) that return
``False`` when a ``/new`` reclaim or an archive committed after the payload
@@ -1798,8 +1850,8 @@ class ManagedWatchService:
result = callback()
if guarded and result is False:
# NOT a store error: the row is fine, this write simply lost to a
- # concurrent lifecycle change. Fusing would disable reconciliation for a
- # healthy database, so the watch is stopped and the store left alone.
+ # concurrent lifecycle change. Backing off would restart a watch whose
+ # row has been torn down, so the watch is simply stopped.
logger.warning(
"Watch %s stopping: the store refused %s because a guarded lifecycle "
"or follow-up admission prerequisite changed",
@@ -1826,7 +1878,7 @@ class ManagedWatchService:
guarded=guarded,
)
except Exception as exc:
- self._fuse_store_after_error(operation, exc, watch_id=watch_id)
+ self._back_off_watch_after_store_error(operation, exc, watch_id=watch_id)
return False
def _current_asyncio_task(self) -> Optional["asyncio.Task[Any]"]:
@@ -1957,18 +2009,21 @@ class ManagedWatchService:
result: _CycleResult,
) -> bool:
try:
- return await self._run_runtime_sync(
+ committed = await self._run_runtime_sync(
self._commit_success_cycle,
watch,
result,
)
except Exception as exc:
- self._fuse_store_after_error(
+ self._back_off_watch_after_store_error(
"commit successful watch cycle",
exc,
watch_id=watch.id,
)
return False
+ if committed:
+ self._watch_store_retries.pop(watch.id, None)
+ return committed
async def _wait_for_follow_up_slot(
self,
@@ -2003,7 +2058,7 @@ class ManagedWatchService:
watch_id,
)
except Exception as exc:
- self._fuse_store_after_error(
+ self._back_off_watch_after_store_error(
"read follow-up fence",
exc,
watch_id=watch_id,
@@ -2030,7 +2085,7 @@ class ManagedWatchService:
try:
run = await self._run_runtime_sync(self.request_store.get_run, run_id)
except Exception as exc:
- self._fuse_store_after_error(
+ self._back_off_watch_after_store_error(
"read previous follow-up",
exc,
watch_id=watch_id,
@@ -2138,7 +2193,7 @@ class ManagedWatchService:
while self._running:
if not self._owns_service_instance():
return
- if watch_id in self._fused_watch_ids:
+ if self._watch_store_retry_in(watch_id) > 0:
return
if not await self._watch_store_call_async(
watch_id,
@@ -2270,7 +2325,7 @@ class ManagedWatchService:
# so ``vibe watch show`` can say the cycle ran and found nothing.
#
# Through the async wrapper like every other result branch: a store
- # failure here must fuse the store and stop the watch, not escape
+ # failure here must back the watch off and stop it, not escape
# ``_run_cycle`` and kill the task silently -- reconcile would restart
# the same quiet cycle and hide the storage problem instead of
# surfacing it.
@@ -2584,18 +2639,21 @@ class ManagedWatchService:
**kwargs: Any,
) -> bool:
try:
- return await self._run_runtime_sync(
+ committed = await self._run_runtime_sync(
self._commit_cycle_result,
watch,
**kwargs,
)
except Exception as exc:
- self._fuse_store_after_error(
+ self._back_off_watch_after_store_error(
"mark_cycle_result",
exc,
watch_id=watch.id,
)
return False
+ if committed:
+ self._watch_store_retries.pop(watch.id, None)
+ return committed
def _stop_watch_for_missing_cwd(self, watch: ManagedWatch, *, error_text: str) -> None:
watch_label = watch.name or watch.id
@@ -2624,7 +2682,7 @@ class ManagedWatchService:
error_text=error_text,
)
except Exception as exc:
- self._fuse_store_after_error(
+ self._back_off_watch_after_store_error(
"mark_cycle_result",
exc,
watch_id=watch.id,
diff --git a/tests/test_scheduled_tasks.py b/tests/test_scheduled_tasks.py
index a731dcb5f..e7cfde137 100644
--- a/tests/test_scheduled_tasks.py
+++ b/tests/test_scheduled_tasks.py
@@ -16496,6 +16496,52 @@ def test_a_dropped_task_mirror_recovers_with_no_unrelated_commit_to_wake_it() ->
)
+def test_a_task_committed_before_a_failed_reload_is_not_lost_with_the_probe() -> None:
+ """The commit a failed reload consumed must still reach the mirror.
+
+ The watch store's twin. ``PRAGMA data_version`` reports another connection's
+ commit once. When the reload it triggers fails -- as every read does while the
+ disk is full -- a later probe no longer sees that commit, so a task created
+ during the outage stayed enabled in SQLite and unscheduled until an unrelated
+ commit or a restart. Nothing commits between the failure and the recovery
+ below, so the mirror can only heal because the store remembered to reload.
+ """
+ from storage.db import SqliteInvalidationProbe, create_sqlite_engine
+
+ store = ScheduledTaskStore()
+ assert store._sqlite is not None, "this test is about the SQLite probe"
+ store.maybe_reload()
+ assert store.maybe_reload() is False, "the store's probe is not settled"
+
+ created = ScheduledTaskStore().add_task(**_TASK_FIXTURE_PAYLOAD)
+ witness_engine = create_sqlite_engine(store._sqlite.db_path)
+ witness = SqliteInvalidationProbe(witness_engine)
+ witness.has_external_write()
+ assert witness.has_external_write() is False, "the witness probe is not settled"
+
+ fault = _fail_the_definition_write_and_the_reload(store._sqlite.engine)
+ try:
+ with pytest.raises(Exception): # noqa: B017 - the fault type is the injected one's
+ store.maybe_reload()
+ assert fault["reads"] >= 1, "the probe did not report the commit, so no reload failed"
+ assert store.get_task(created.id) is None, "the mirror already holds the new task"
+
+ fault["live"] = False
+ assert witness.has_external_write() is False, (
+ "something COMMITTED after the failed reload; a data_version bump heals "
+ "the mirror on its own, so this test would pass without the fix"
+ )
+ assert store.maybe_reload() is True, (
+ "the failed reload consumed the only change signal, so the task created "
+ "during the outage is never scheduled"
+ )
+ assert [item.id for item in store.list_tasks()] == [created.id]
+ assert store.maybe_reload() is False, "the repairing reload must clear the flag"
+ finally:
+ witness.close()
+ witness_engine.dispose()
+
+
# --- Command tasks: a definition that runs a subprocess instead of an Agent turn ---
diff --git a/tests/test_watches.py b/tests/test_watches.py
index f34fdac93..c836a448e 100644
--- a/tests/test_watches.py
+++ b/tests/test_watches.py
@@ -128,22 +128,28 @@ async def _start_watch_service(service: ManagedWatchService) -> None:
await startup_task
-async def _await_fused_watch(
+async def _await_backed_off_watch(
service: ManagedWatchService,
watch_id: str,
timeout: float = 10.0,
) -> None:
- """Wait for a watch to be fused instead of sleeping a fixed interval.
+ """Wait for a watch to back off instead of sleeping a fixed interval.
- Fusing needs a real waiter subprocess to start and its result to reach the
- raising store, so the wait covers a process spawn plus interpreter startup.
- A fixed 0.12s was enough locally and not on a loaded CI runner, where the
- assertion saw an empty fused set.
+ Backing off needs a real waiter subprocess to start and its result to reach
+ the raising store, so the wait covers a process spawn plus interpreter
+ startup. A fixed 0.12s was enough locally and not on a loaded CI runner.
"""
deadline = time.monotonic() + timeout
- while watch_id not in service._fused_watch_ids:
- assert time.monotonic() < deadline, "watch was never fused after the store error"
+ while watch_id not in service._watch_store_retries:
+ assert time.monotonic() < deadline, "watch never backed off after the store error"
+ await asyncio.sleep(0.02)
+
+
+async def _await_condition(predicate, message: str, timeout: float = 10.0) -> None:
+ deadline = time.monotonic() + timeout
+ while not predicate():
+ assert time.monotonic() < deadline, message
await asyncio.sleep(0.02)
@@ -2366,67 +2372,105 @@ def test_managed_watch_service_forever_non_retry_error_disables_and_enqueues_fai
assert pending[0].prompt.startswith("Investigate the failure.\n\nWatch 'Broken forever waiter' stopped because the waiter exited with code 1.")
-def test_managed_watch_service_fuses_watch_after_store_error(tmp_path: Path) -> None:
- class FailingResultStore(ManagedWatchStore):
+def test_managed_watch_service_backs_off_one_watch_after_its_store_error(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """A store error stops one watch for a doubling backoff, not the whole service.
+
+ The watch sits out each backoff instead of restarting into the same error, and
+ runs again once the store accepts its cycle result. A watch created meanwhile
+ starts at once: a store error one watch saw says nothing about the others, and
+ a service-wide fuse left every Watch created after a brief disk-full episode
+ unstarted until the service restarted.
+ """
+
+ monkeypatch.setattr("core.watches.WATCH_RECONCILE_INTERVAL_SECONDS", 0.01)
+ monkeypatch.setattr("core.watches.WATCH_STORE_RETRY_INITIAL_SECONDS", 0.2)
+
+ class FlakyResultStore(ManagedWatchStore):
def __init__(self, path: Path):
super().__init__(path)
- self.starts = 0
+ self.starts: dict[str, list[float]] = {}
+ self.failures: dict[str, list[float]] = {}
+ self.failing: dict[str, int] = {}
def mark_cycle_start(self, watch_id: str) -> bool:
- self.starts += 1
+ self.starts.setdefault(watch_id, []).append(time.monotonic())
return super().mark_cycle_start(watch_id)
- def mark_cycle_result(self, *args, **kwargs) -> bool:
- raise RuntimeError("database disk image is malformed")
+ def mark_cycle_result(self, watch_id: str, *args, **kwargs) -> bool:
+ if self.failing.get(watch_id, 0) > 0:
+ self.failing[watch_id] -= 1
+ self.failures.setdefault(watch_id, []).append(time.monotonic())
+ raise RuntimeError("unable to open database file")
+ return super().mark_cycle_result(watch_id, *args, **kwargs)
+
+ def add_retrying_watch(name: str) -> watches_module.ManagedWatch:
+ return store.add_watch(
+ name=name,
+ session_key="slack::channel::C123",
+ command=[sys.executable, "-c", "import sys; sys.exit(75)"],
+ shell_command=None,
+ prefix="Should not storm.",
+ cwd=None,
+ mode="forever",
+ timeout_seconds=5,
+ lifetime_timeout_seconds=0,
+ retry_exit_codes=[75],
+ retry_delay_seconds=0.01,
+ post_to=None,
+ deliver_key=None,
+ )
- store = FailingResultStore(tmp_path / "watches.json")
- request_store = TaskExecutionStore(tmp_path / "task_requests")
- runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
- watch = store.add_watch(
- name="Broken persistence",
- session_key="slack::channel::C123",
- command=[sys.executable, "-c", "import sys; sys.exit(75)"],
- shell_command=None,
- prefix="Should not storm.",
- cwd=None,
- mode="forever",
- timeout_seconds=5,
- lifetime_timeout_seconds=0,
- retry_exit_codes=[75],
- retry_delay_seconds=0.01,
- post_to=None,
- deliver_key=None,
- )
+ store = FlakyResultStore(tmp_path / "watches.json")
+ broken = add_retrying_watch("Broken persistence")
+ store.failing[broken.id] = 2
service = ManagedWatchService(
controller=SimpleNamespace(),
store=store,
- request_store=request_store,
- runtime_store=runtime_store,
+ request_store=TaskExecutionStore(tmp_path / "task_requests"),
+ runtime_store=WatchRuntimeStateStore(tmp_path / "watch_runtime.json"),
)
async def _run() -> None:
await _start_watch_service(service)
- await _await_fused_watch(service, watch.id)
- # Settle briefly: a fused store must not let the cycle start again.
- await asyncio.sleep(0.08)
+ await _await_backed_off_watch(service, broken.id)
+ created = add_retrying_watch("Created during the outage")
+ await _await_condition(
+ lambda: created.id in store.starts,
+ "a watch created after another watch's store error never started",
+ )
+ await _await_condition(
+ lambda: store.failing[broken.id] == 0 and broken.id not in service._watch_store_retries,
+ "the backed-off watch never committed a cycle after the store recovered",
+ )
await service.stop()
asyncio.run(_run())
- assert store.starts == 1
- assert service._store_error_fused is True
- assert request_store.list_pending() == []
+ starts = store.starts[broken.id]
+ failures = store.failures[broken.id]
+ assert len(failures) == 2
+ assert len(starts) >= 3
+ assert starts[1] - failures[0] >= 0.2, "the watch restarted before its backoff elapsed"
+ assert starts[2] - failures[1] >= 0.4, "the second backoff did not double"
-def test_managed_watch_service_fuses_quiet_cycle_after_store_error(tmp_path: Path) -> None:
+def test_managed_watch_service_backs_off_quiet_cycle_after_store_error(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
"""A quiet cycle commits through the same wrapper as every other result branch.
Called synchronously, a raising ``mark_cycle_result`` escapes ``_run_cycle`` and
- kills the watch task with the store unfused and the cycle unrecorded. Reconcile
- then restarts the same quiet cycle and repeats it, so a real storage failure
- never reaches the user as one.
+ kills the watch task with no backoff and the cycle unrecorded. Reconcile then
+ restarts the same quiet cycle and repeats it, so a real storage failure never
+ reaches the user as one.
"""
+ monkeypatch.setattr("core.watches.WATCH_RECONCILE_INTERVAL_SECONDS", 0.01)
+
class FailingResultStore(ManagedWatchStore):
def __init__(self, path: Path):
super().__init__(path)
@@ -2466,57 +2510,84 @@ def test_managed_watch_service_fuses_quiet_cycle_after_store_error(tmp_path: Pat
async def _run() -> None:
await _start_watch_service(service)
- await _await_fused_watch(service, watch.id)
- # Settle briefly: a fused store must not let the cycle start again.
+ await _await_backed_off_watch(service, watch.id)
+ # Settle across several reconcile ticks: the backoff must keep the quiet
+ # cycle from starting again.
await asyncio.sleep(0.08)
await service.stop()
asyncio.run(_run())
- assert store.starts == 1, "the fused store let the quiet cycle restart"
- assert service._store_error_fused is True
+ assert store.starts == 1, "reconcile restarted the quiet cycle inside its backoff"
# A quiet cycle authorises no hook, and a failed one must not invent it.
assert request_store.list_pending() == []
-def test_managed_watch_service_fuses_reconcile_after_store_read_error(tmp_path: Path, monkeypatch) -> None:
+def test_managed_watch_service_resumes_reconcile_after_store_outage(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ """The polling loop pauses on a store outage and resumes when the store heals.
+
+ A pause quiets the logs; it must not stop the probing. Before the fix the
+ loop stopped reading after three errors, so a watch the store listed again
+ once the outage ended never started until the service restarted.
+ """
+
monkeypatch.setattr("core.watches.WATCH_RECONCILE_INTERVAL_SECONDS", 0.01)
+ outage_reads = watches_module.WATCH_STORE_RECONCILE_PAUSE_FAILURES + 3
- class FailingListStore(ManagedWatchStore):
+ class OutageListStore(ManagedWatchStore):
def __init__(self, path: Path):
super().__init__(path)
- self.calls = 0
+ self.failures_remaining = outage_reads
def list_watches(self):
- self.calls += 1
- raise RuntimeError("database disk image is malformed")
+ if self.failures_remaining > 0:
+ self.failures_remaining -= 1
+ raise RuntimeError("unable to open database file")
+ return super().list_watches()
- store = FailingListStore(tmp_path / "watches.json")
- request_store = TaskExecutionStore(tmp_path / "task_requests")
- runtime_store = WatchRuntimeStateStore(tmp_path / "watch_runtime.json")
+ store = OutageListStore(tmp_path / "watches.json")
+ watch = store.add_watch(
+ name="Waiting out the outage",
+ session_key="slack::channel::C123",
+ command=[sys.executable, "-c", "import time; time.sleep(30)"],
+ shell_command=None,
+ prefix="Done.",
+ cwd=None,
+ mode="once",
+ timeout_seconds=60,
+ lifetime_timeout_seconds=0,
+ retry_exit_codes=[],
+ retry_delay_seconds=0,
+ post_to=None,
+ deliver_key=None,
+ )
service = ManagedWatchService(
controller=SimpleNamespace(),
store=store,
- request_store=request_store,
- runtime_store=runtime_store,
+ request_store=TaskExecutionStore(tmp_path / "task_requests"),
+ runtime_store=WatchRuntimeStateStore(tmp_path / "watch_runtime.json"),
)
async def _run() -> None:
- service._running = True
- task = asyncio.create_task(service._watch_store())
- await asyncio.sleep(0.05)
- service._running = False
- task.cancel()
- try:
- await task
- except asyncio.CancelledError:
- pass
+ await _start_watch_service(service)
+ await _await_condition(
+ lambda: watch.id in service._active_tasks,
+ "the watch never started after the store outage ended",
+ )
+ await service.stop()
- asyncio.run(_run())
+ with caplog.at_level(logging.INFO, logger="core.watches"):
+ asyncio.run(_run())
- assert service._store_error_fused is True
- assert store.calls == 3
- assert service._active_tasks == {}
+ assert store.failures_remaining == 0
+ assert service._store_reconcile_failures == 0
+ pauses = [record for record in caplog.records if "Pausing managed watch reconciliation" in record.message]
+ assert len(pauses) == 1, "the pause must be reported once, not per probe"
+ assert any("reconciliation resumed" in record.message for record in caplog.records)
def test_managed_watch_service_idle_tick_does_not_write_runtime_state(tmp_path: Path, monkeypatch) -> None:
@@ -3387,7 +3458,6 @@ def test_hfr_179_blocked_watch_recovery_arms_generation_scoped_recheck(
"unblocked": (),
"watches": (),
"store_error": None,
- "fused": False,
},
rearm_after_process=False,
)
@@ -3419,7 +3489,6 @@ def test_hfr_179_pending_watch_recovery_uses_watch_lane_cadence(
"unblocked": (),
"watches": (),
"store_error": None,
- "fused": False,
},
rearm_after_process=False,
)
@@ -3429,10 +3498,20 @@ def test_hfr_179_pending_watch_recovery_uses_watch_lane_cadence(
assert delayed == [(token, watches_module.WATCH_RECONCILE_INTERVAL_SECONDS)]
-def test_hfr_179_watch_store_fuse_stops_generation_reads_and_retries(
+def test_hfr_179_paused_watch_store_keeps_probing_and_resumes(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
+ caplog: pytest.LogCaptureFixture,
) -> None:
+ """A store outage pauses the watch lane; it must not end it.
+
+ Consecutive read errors pause reconciliation: the lane drops its fast retry
+ and reports the pause once. Every later scan still probes the store, so the
+ lane reconciles again on the first clean read. The old one-way fuse stopped
+ reading for good, and Watches created after a brief "unable to open database
+ file" episode never started until the service restarted.
+ """
+
service = ManagedWatchService(
controller=SimpleNamespace(),
store=ManagedWatchStore(tmp_path / "watches.json"),
@@ -3441,52 +3520,69 @@ def test_hfr_179_watch_store_fuse_stops_generation_reads_and_retries(
)
service._recovery_pending = False
service._runtime_state_dirty = False
+ watch = service.store.add_watch(**_WATCH_FIXTURE_PAYLOAD)
+ healthy = False
reads = 0
+ reload = service.store.maybe_reload
- def fail_read() -> bool:
+ def flaky_read() -> bool:
nonlocal reads
reads += 1
- raise OSError("watch store unavailable")
+ if not healthy:
+ raise OSError("unable to open database file")
+ return reload()
- monkeypatch.setattr(service.store, "maybe_reload", fail_read)
- handler = _ManagedWatchRuntimeWorkHandler(service)
+ reconciled: list[list[str]] = []
- async def _run() -> None:
- outcomes = []
- for _ in range(watches_module.WATCH_STORE_RECONCILE_FUSE_FAILURES):
- items, _has_more = await asyncio.to_thread(
- handler.scan,
- limit=1,
- occupied=frozenset(),
- cursor=None,
- )
- outcomes.append(await handler.process(items[0]))
+ def record_reconcile(watches) -> bool:
+ reconciled.append([item.id for item in watches])
+ return False
- assert outcomes[:-1] == [False] * (len(outcomes) - 1)
- assert outcomes[-1] is True
- assert service._store_error_fused is True
- assert reads == watches_module.WATCH_STORE_RECONCILE_FUSE_FAILURES
+ persisted = 0
- persisted = 0
+ async def persist_runtime_state() -> None:
+ nonlocal persisted
+ persisted += 1
+ service._runtime_state_dirty = False
- async def persist_runtime_state() -> None:
- nonlocal persisted
- persisted += 1
- service._runtime_state_dirty = False
+ monkeypatch.setattr(service.store, "maybe_reload", flaky_read)
+ monkeypatch.setattr(service, "reconcile_watches", record_reconcile)
+ monkeypatch.setattr(service, "_persist_runtime_state", persist_runtime_state)
+ handler = _ManagedWatchRuntimeWorkHandler(service)
+ pause_after = watches_module.WATCH_STORE_RECONCILE_PAUSE_FAILURES
- monkeypatch.setattr(service, "_persist_runtime_state", persist_runtime_state)
- service._runtime_state_dirty = True
+ async def scan_and_process() -> bool:
items, _has_more = await asyncio.to_thread(
handler.scan,
limit=1,
occupied=frozenset(),
cursor=None,
)
- assert await handler.process(items[0]) is True
- assert reads == watches_module.WATCH_STORE_RECONCILE_FUSE_FAILURES
- assert persisted == 1
+ return await handler.process(items[0])
- asyncio.run(_run())
+ async def _run() -> None:
+ nonlocal healthy
+ outcomes = [await scan_and_process() for _ in range(pause_after)]
+ # ``False`` asks the supervisor for a fast retry; the pause stops asking.
+ assert outcomes == [False] * (pause_after - 1) + [True]
+
+ service._runtime_state_dirty = True
+ assert await scan_and_process() is True, "a paused lane must not fast-retry"
+ assert reads == pause_after + 1, "the paused lane stopped probing the store"
+ assert persisted == 1, "runtime state must still persist while paused"
+ assert reconciled == []
+
+ healthy = True
+ assert await scan_and_process() is True
+ assert reconciled == [[watch.id]], "the lane did not resume once the store read cleanly"
+ assert service._store_reconcile_failures == 0
+
+ with caplog.at_level(logging.DEBUG, logger="core.watches"):
+ asyncio.run(_run())
+
+ errors = [record for record in caplog.records if record.levelno >= logging.ERROR]
+ assert len(errors) == 1 and "Pausing managed watch reconciliation" in errors[0].message
+ assert any("reconciliation resumed" in record.message for record in caplog.records)
def test_watch_runtime_state_persistence_failure_rearms_maintenance(
@@ -3892,7 +3988,6 @@ def test_managed_watch_service_retries_transient_reconcile_errors(tmp_path: Path
asyncio.run(_run())
- assert service._store_error_fused is False
assert service._store_reconcile_failures == 0
@@ -3913,7 +4008,6 @@ def test_managed_watch_service_start_retries_initial_reconcile_error(tmp_path: P
async def _run() -> None:
await _start_watch_service(service)
- assert service._store_error_fused is False
assert 1 <= service._store_reconcile_failures < 3
await service.stop()
@@ -5376,3 +5470,49 @@ def test_managed_watch_service_names_the_watch_in_the_cycle_environment(tmp_path
asyncio.run(_run())
assert echoed.read_text(encoding="utf-8") == watch.id
+
+
+def test_a_watch_committed_before_a_failed_reload_is_not_lost_with_the_probe() -> None:
+ """The commit a failed reload consumed must still reach the mirror.
+
+ ``PRAGMA data_version`` reports another connection's commit once. When the
+ reload it triggers fails -- as every read does while the disk is full -- a
+ later probe no longer sees that commit, so a watch created during the outage
+ stayed enabled in SQLite and invisible to reconcile until an unrelated commit
+ or a restart. Nothing commits between the failure and the recovery below, so
+ the mirror can only heal because the store remembered it must reload.
+ """
+ from storage.db import SqliteInvalidationProbe, create_sqlite_engine
+
+ store = ManagedWatchStore()
+ assert store._sqlite is not None, "this test is about the SQLite probe"
+ store.maybe_reload()
+ assert store.maybe_reload() is False, "the store's probe is not settled"
+
+ created = ManagedWatchStore().add_watch(**_WATCH_FIXTURE_PAYLOAD)
+ witness_engine = create_sqlite_engine(store._sqlite.db_path)
+ witness = SqliteInvalidationProbe(witness_engine)
+ witness.has_external_write()
+ assert witness.has_external_write() is False, "the witness probe is not settled"
+
+ fault = _fail_the_definition_write_and_the_reload(store._sqlite.engine)
+ try:
+ with pytest.raises(Exception): # noqa: B017 - the fault type is the injected one's
+ store.maybe_reload()
+ assert fault["reads"] >= 1, "the probe did not report the commit, so no reload failed"
+ assert store.get_watch(created.id) is None, "the mirror already holds the new watch"
+
+ fault["live"] = False
+ assert witness.has_external_write() is False, (
+ "something COMMITTED after the failed reload; a data_version bump heals "
+ "the mirror on its own, so this test would pass without the fix"
+ )
+ assert store.maybe_reload() is True, (
+ "the failed reload consumed the only change signal, so the watch created "
+ "during the outage stays invisible to reconcile"
+ )
+ assert [item.id for item in store.list_watches()] == [created.id]
+ assert store.maybe_reload() is False, "the repairing reload must clear the flag"
+ finally:
+ witness.close()
+ witness_engine.dispose()