Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

fix(harness): reconcile watches independently of reload hints (#1996)

Call White2f2009dc6fce
diff --git a/core/watches.py b/core/watches.py
index 317f25fe3..c101b36ef 100644
--- a/core/watches.py
+++ b/core/watches.py
@@ -1069,13 +1069,12 @@ class _ManagedWatchRuntimeWorkHandler(RuntimeWorkHandler):
if recovery.recovered and blocked
else ()
)
- changed = False
watches: tuple[ManagedWatch, ...] = ()
store_error: Exception | None = None
fused = self.service._store_error_fused
if recovery.recovered and not fused:
try:
- changed = self.service.store.maybe_reload()
+ self.service.store.maybe_reload()
watches = tuple(
ManagedWatch.from_dict(watch.to_dict())
for watch in self.service.store.list_watches()
@@ -1088,7 +1087,6 @@ class _ManagedWatchRuntimeWorkHandler(RuntimeWorkHandler):
{
"recovery": recovery,
"unblocked": unblocked,
- "changed": changed,
"watches": watches,
"store_error": store_error,
"fused": fused,
@@ -1113,11 +1111,9 @@ class _ManagedWatchRuntimeWorkHandler(RuntimeWorkHandler):
)
self.service._recovery_pending = False
if observation["unblocked"]:
- self.service._reconcile_dirty = True
self.service._runtime_state_dirty = True
store_error = observation.get("store_error")
if store_error is not None:
- self.service._reconcile_dirty = True
self.service._handle_reconcile_store_error(store_error)
return self.service._store_error_fused
if observation.get("fused") or self.service._store_error_fused:
@@ -1126,18 +1122,18 @@ class _ManagedWatchRuntimeWorkHandler(RuntimeWorkHandler):
return not self.service._runtime_state_dirty
return True
try:
- if observation["changed"] or self.service._reconcile_dirty:
- if self.service.reconcile_watches(observation["watches"]):
- self.service._runtime_state_dirty = True
+ # Another shared-store reader may have consumed maybe_reload's change
+ # result. Reconcile the snapshot, not which reader refreshed it; the
+ # active-task map already makes unchanged reconciliation idempotent.
+ if self.service.reconcile_watches(observation["watches"]):
+ self.service._runtime_state_dirty = True
if self.service._runtime_state_dirty:
await self.service._persist_runtime_state()
if self.service._runtime_state_dirty:
return False
self.service._store_reconcile_failures = 0
- self.service._reconcile_dirty = False
return True
except Exception as exc:
- self.service._reconcile_dirty = True
self.service._handle_reconcile_store_error(exc)
return self.service._store_error_fused
@@ -1158,6 +1154,7 @@ class ManagedWatchService:
self._startup_task: Optional[asyncio.Task] = None
self._reconcile_task: Optional[asyncio.Task] = None
self._active_tasks: dict[str, asyncio.Task] = {}
+ self._canceling_watch_ids: set[str] = set()
self._active_pids: dict[str, int] = {}
self._active_process_identities: dict[str, PersistedProcessIdentity] = {}
self._watch_started_at: dict[str, str] = {}
@@ -1168,7 +1165,6 @@ class ManagedWatchService:
self._store_reconcile_failures = 0
self._recovery_pending = True
self._requires_service_lease = runtime.service_instance_lock_attached_to_process()
- self._reconcile_dirty = True
self._runtime_state_dirty = True
self._runtime_state_revision = 0
self._store_worker_lock = threading.Lock()
@@ -1242,8 +1238,6 @@ class ManagedWatchService:
notify = getattr(supervisor, "notify", None)
if callable(notify):
notify(RuntimeWorkLane.WATCH_DEFINITIONS)
- else:
- self._reconcile_dirty = True
def _schedule_runtime_work_wake(self, delay: float) -> None:
supervisor = getattr(self.controller, "runtime_work_supervisor", None)
@@ -1281,9 +1275,7 @@ class ManagedWatchService:
if self.reconcile_watches():
self._runtime_state_dirty = True
self._write_runtime_state()
- self._reconcile_dirty = False
except Exception as exc:
- self._reconcile_dirty = True
self._handle_reconcile_store_error(exc)
if self._running and self._owns_service_instance() and self._supports_runtime_work_lane():
self._register_runtime_work_lane()
@@ -1586,6 +1578,7 @@ class ManagedWatchService:
if self._active_tasks:
await asyncio.gather(*self._active_tasks.values(), return_exceptions=True)
self._active_tasks.clear()
+ self._canceling_watch_ids.clear()
self._active_pids.clear()
self._active_process_identities.clear()
self._watch_started_at.clear()
@@ -1614,7 +1607,6 @@ class ManagedWatchService:
await asyncio.sleep(WATCH_RECONCILE_INTERVAL_SECONDS)
continue
self._recovery_pending = False
- self._reconcile_dirty = True
self._runtime_state_dirty = True
if self._store_error_fused:
await asyncio.sleep(WATCH_RECONCILE_INTERVAL_SECONDS)
@@ -1633,20 +1625,16 @@ class ManagedWatchService:
else:
unblocked = ()
if self._apply_recovery_unblocked(unblocked):
- self._reconcile_dirty = True
self._runtime_state_dirty = True
- should_reconcile = self.store.maybe_reload() or self._reconcile_dirty
- if should_reconcile:
- if self.reconcile_watches():
- self._runtime_state_dirty = True
+ self.store.maybe_reload()
+ if self.reconcile_watches():
+ self._runtime_state_dirty = True
if self._runtime_state_dirty:
self._write_runtime_state()
self._store_reconcile_failures = 0
- self._reconcile_dirty = False
except asyncio.CancelledError:
raise
except Exception as exc:
- self._reconcile_dirty = True
self._handle_reconcile_store_error(exc)
await asyncio.sleep(WATCH_RECONCILE_INTERVAL_SECONDS)
@@ -1677,18 +1665,26 @@ class ManagedWatchService:
for watch_id, task in list(self._active_tasks.items()):
if watch_id in desired_ids:
continue
- task.cancel()
- changed = True
+ if self._cancel_watch(watch_id, task):
+ changed = True
return changed
+ def _cancel_watch(self, watch_id: str, task: asyncio.Task) -> bool:
+ # Repeated scans or shutdown must not interrupt asynchronous teardown.
+ # Track our request explicitly, including on Python 3.10.
+ if watch_id in self._canceling_watch_ids or not task.cancel():
+ return False
+ self._canceling_watch_ids.add(watch_id)
+ return True
+
def _on_watch_done(self, watch_id: str) -> None:
self._active_tasks.pop(watch_id, None)
+ self._canceling_watch_ids.discard(watch_id)
self._active_pids.pop(watch_id, None)
self._active_process_identities.pop(watch_id, None)
self._watch_started_at.pop(watch_id, None)
self._write_runtime_state()
- self._reconcile_dirty = True
def _runtime_state_payload(self) -> dict[str, Any]:
"""Assemble the loop-owned runtime projection without storage I/O."""
@@ -1847,9 +1843,9 @@ class ManagedWatchService:
self._reconcile_task.cancel()
if self._legacy_probe_task and self._legacy_probe_task is not current_task:
self._legacy_probe_task.cancel()
- for task in list(self._active_tasks.values()):
+ for watch_id, task in list(self._active_tasks.items()):
if task is not current_task:
- task.cancel()
+ self._cancel_watch(watch_id, task)
self._runtime_state_dirty = True
self._write_runtime_state()
diff --git a/docs/plans/watch-reconcile-current-snapshot.md b/docs/plans/watch-reconcile-current-snapshot.md
new file mode 100644
index 000000000..b8d09f87c
--- /dev/null
+++ b/docs/plans/watch-reconcile-current-snapshot.md
@@ -0,0 +1,38 @@
+# Watch reconciliation against current snapshots
+
+Issue: #1995.
+
+## Change contract
+
+- An enabled, eligible Watch must reach scheduling regardless of which reader
+ refreshes the shared store first. No resume, unrelated write, or restart is
+ required.
+- A change arriving between scan and apply must converge on the next scan.
+- Repeated unchanged snapshots must keep the same active worker, avoid idle
+ runtime-state writes, and retain recovery, fuse, service-lease, and guarded
+ lifecycle-write protections.
+- Apply the same contract to the runtime-work lane and legacy polling loop.
+
+## Approach
+
+Always reconcile a successfully read snapshot through the existing idempotent
+`reconcile_watches`. Store reload detection only controls cache freshness; it
+cannot acknowledge scheduling work. Remove the redundant reconciliation dirty
+flag instead of introducing another revision counter or notification path.
+The runtime-work scan already copies every Watch; the legacy loop now checks
+its in-memory list each tick. Unchanged reconciliation performs no durable write.
+Already-cancelling workers retain their original cancellation so later scans
+cannot interrupt their asynchronous teardown.
+
+## Evidence and boundaries
+
+HFR-485 covers file-backed and isolated SQLite-backed stores, reader-before-scan,
+reader-between-scan-and-apply, legacy polling, retained worker identity, and
+disable-after-reload. It uses real stores, scheduler handlers, and reconciliation,
+with blocking coroutine workers instead of external waiter processes.
+Existing HFR-179 tests cover single-flight store access and deferred wakes.
+
+The prior tests checked reload and reconciliation independently, not whether a
+different reader could consume the reload result before reconciliation.
+No schema, CLI timeout, external notification, or deployment change is needed.
+Live IM delivery and deployed-service verification remain separate acceptance work.
diff --git a/tests/scenarios/harness_failure_recovery/catalog.yaml b/tests/scenarios/harness_failure_recovery/catalog.yaml
index 63f89d411..2c468c44c 100644
--- a/tests/scenarios/harness_failure_recovery/catalog.yaml
+++ b/tests/scenarios/harness_failure_recovery/catalog.yaml
@@ -4382,6 +4382,16 @@ scenarios:
# refused for a merely backlogged predecessor leaves its registration alone.
# An edit that keeps the definition recurring still runs, at its refreshed value.
test: tests/test_scheduled_tasks.py::test_hfr_484_in_flight_cron_fire_cannot_spend_a_replacement_one_shot
+ - id: HFR-485
+ name: Watch scheduling survives another reader consuming the store reload
+ status: covered
+ kind: concurrency
+ layer: scenario
+ backend: shared
+ # File and SQLite stores, both scheduler paths, and changes before or after
+ # snapshot capture must converge without a resume or another definition write.
+ # Unchanged snapshots retain worker identity; consumed disables still stop it.
+ test: tests/test_watches.py::test_hfr_485_watch_reconciles_after_another_reader_consumes_reload
manual_evidence:
- local Incus backend process death after prompt acceptance
- local Incus controller restart with persisted queued successor
diff --git a/tests/scenarios/harness_failure_recovery/observations.yaml b/tests/scenarios/harness_failure_recovery/observations.yaml
index 89c5f7278..bb53dffd6 100644
--- a/tests/scenarios/harness_failure_recovery/observations.yaml
+++ b/tests/scenarios/harness_failure_recovery/observations.yaml
@@ -311,3 +311,17 @@ observations:
the handoff before any backend started. The binding transaction must atomically
adopt the freshly reserved Session only for create_per_run rows; a Run already
bound to another Session must remain a hard refusal.
+ - id: HFR-OBS-026
+ date: 2026-09-16
+ source: issue 1995 and isolated file/SQLite shared-reader reproduction
+ result: confirmed
+ detail: >-
+ maybe_reload acknowledges cache freshness, not scheduler reconciliation.
+ An existing Watch can refresh the shared mirror before the scheduler scans,
+ leaving an enabled new Watch in every snapshot but never scheduled. Repeated
+ scans cannot recover when their reload result is used as the admission gate.
+ HFR-485 exercises real stores and worker-task ownership across both scheduler
+ paths; only external waiter execution is replaced by a blocking coroutine.
+ Prior reload and single-flight tests did not connect that reader ordering to
+ actual scheduling. Always reconciling the captured snapshot retains existing
+ worker identity and avoids adding a second change-acknowledgement mechanism.
diff --git a/tests/test_watches.py b/tests/test_watches.py
index aa1781d2d..fc54f383a 100644
--- a/tests/test_watches.py
+++ b/tests/test_watches.py
@@ -2486,7 +2486,6 @@ def test_managed_watch_service_idle_tick_does_not_write_runtime_state(tmp_path:
)
service._running = True
service._recovery_pending = False
- service._reconcile_dirty = False
service._runtime_state_dirty = False
async def _run() -> None:
@@ -2502,7 +2501,7 @@ def test_managed_watch_service_idle_tick_does_not_write_runtime_state(tmp_path:
asyncio.run(_run())
assert store.reloads > 0
- assert store.lists == 0
+ assert store.lists == store.reloads
assert runtime_store.writes == 0
@@ -2591,7 +2590,8 @@ def test_managed_watch_service_start_reaps_matching_stale_worker_before_reconcil
asyncio.run(_run())
- assert events == [("terminate", 4321), ("reconcile", None)]
+ assert events[0] == ("terminate", 4321)
+ assert events[1:] and all(event == ("reconcile", None) for event in events[1:])
def test_managed_watch_service_start_does_not_reap_reused_pid(
@@ -2635,7 +2635,7 @@ def test_managed_watch_service_start_does_not_reap_reused_pid(
asyncio.run(_run())
- assert reconciles == 1
+ assert reconciles >= 1
assert watch.id not in service._recovery_blocked_watch_ids
@@ -2808,7 +2808,7 @@ def test_managed_watch_service_start_ignores_dead_recorded_pid(
asyncio.run(_run())
- assert reconciles == 1
+ assert reconciles >= 1
def test_managed_watch_service_start_reaps_group_after_leader_exit(
@@ -2851,7 +2851,8 @@ def test_managed_watch_service_start_reaps_group_after_leader_exit(
asyncio.run(_run())
- assert events == ["terminate-group", "reconcile"]
+ assert events[0] == "terminate-group"
+ assert events[1:] and all(event == "reconcile" for event in events[1:])
def test_managed_watch_service_does_not_reap_unverified_group_after_leader_exit(
@@ -3054,6 +3055,152 @@ def test_hfr_179_watch_store_phases_are_single_flight_and_apply_on_loop(
asyncio.run(_run())
[email protected]("backend", ["file", "sqlite"])
[email protected]("ordering", ["before_scan", "after_scan", "legacy"])
+def test_hfr_485_watch_reconciles_after_another_reader_consumes_reload(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ backend: str,
+ ordering: str,
+) -> None:
+ """Issue #1995: a refreshed mirror is not an acknowledged scheduling change."""
+ monkeypatch.setattr(watches_module, "_publish_watch_definitions_updated", lambda: None)
+ monkeypatch.setattr(watches_module, "WATCH_RECONCILE_INTERVAL_SECONDS", 0.01)
+ reader = ManagedWatchStore(tmp_path / "watches.json")
+ writer = ManagedWatchStore(tmp_path / "watches.json")
+ sqlite_stores = []
+ if backend == "sqlite":
+ for store in (reader, writer):
+ store._sqlite = SQLiteBackgroundTaskStore(tmp_path / "isolated.sqlite")
+ sqlite_stores.append(store._sqlite)
+ store.load()
+ reader.maybe_reload()
+ assert reader.maybe_reload() is False
+ service = ManagedWatchService(
+ controller=SimpleNamespace(),
+ store=reader,
+ request_store=TaskExecutionStore(tmp_path / "task_requests"),
+ runtime_store=WatchRuntimeStateStore(tmp_path / "watch_runtime.json"),
+ )
+ service._recovery_pending = False
+ handler = _ManagedWatchRuntimeWorkHandler(service)
+ started: list[str] = []
+ runtime_writes: list[dict] = []
+ write_runtime = service.runtime_store.write
+
+ def record_runtime_write(payload: dict) -> None:
+ runtime_writes.append(payload)
+ write_runtime(payload)
+
+ monkeypatch.setattr(service.runtime_store, "write", record_runtime_write)
+
+ async def _run() -> None:
+ new_started = asyncio.Event()
+ release_workers = asyncio.Event()
+ cleanup_started = asyncio.Event()
+ finish_cleanup = asyncio.Event()
+
+ async def worker(watch_id: str) -> None:
+ started.append(watch_id)
+ if watch_id == "new-watch":
+ new_started.set()
+ try:
+ await release_workers.wait()
+ finally:
+ if watch_id == "new-watch":
+ cleanup_started.set()
+ await finish_cleanup.wait()
+
+ monkeypatch.setattr(service, "_run_watch", worker)
+
+ async def scan():
+ items, _ = await asyncio.to_thread(
+ handler.scan, limit=1, occupied=frozenset(), cursor=None,
+ )
+ return items[0]
+
+ service._running = True
+ legacy_task = None
+ try:
+ writer.upsert_watch(watches_module.ManagedWatch(
+ id="existing-watch", name="Existing", session_key="",
+ ))
+ assert await handler.process(await scan()) is True
+ await asyncio.sleep(0)
+ assert started == ["existing-watch"]
+ original_task = service._active_tasks["existing-watch"]
+
+ # Also cover a definition arriving after scan captured its old snapshot.
+ stale_item = await scan() if ordering == "after_scan" else None
+ writer.upsert_watch(watches_module.ManagedWatch(
+ id="new-watch", name="新 Watch", session_key="",
+ ))
+ assert await service._watch_store_call_async(
+ "existing-watch", "reload before follow-up fence", reader.maybe_reload,
+ ) is True
+ assert reader.get_watch("new-watch") is not None
+ assert reader.maybe_reload() is False
+
+ if ordering == "legacy":
+ legacy_task = asyncio.create_task(service._watch_store())
+ else:
+ if stale_item is not None:
+ assert await handler.process(stale_item) is True
+ current = await scan()
+ assert {watch.id for watch in current.observation["watches"]} == {
+ "existing-watch", "new-watch",
+ }
+ assert await handler.process(current) is True
+ await asyncio.wait_for(new_started.wait(), timeout=1)
+
+ writes_before_idle = len(runtime_writes)
+ for _ in range(3):
+ assert await handler.process(await scan()) is True
+ await asyncio.sleep(0)
+ assert started == ["existing-watch", "new-watch"]
+ assert service._active_tasks["existing-watch"] is original_task
+ assert len(runtime_writes) == writes_before_idle
+
+ # A consumed disabling change must still retire the existing owner.
+ writer.set_enabled("new-watch", False)
+ assert await service._watch_store_call_async(
+ "existing-watch", "reload", reader.maybe_reload,
+ ) is True
+ new_task = service._active_tasks["new-watch"]
+ assert await handler.process(await scan()) is True
+ await asyncio.wait_for(cleanup_started.wait(), timeout=1)
+ writes_before_cleanup = len(runtime_writes)
+ for _ in range(3):
+ assert await handler.process(await scan()) is True
+ assert not new_task.done()
+ assert len(runtime_writes) == writes_before_cleanup
+ finish_cleanup.set()
+ await asyncio.gather(new_task, return_exceptions=True)
+ assert new_task.cancelled()
+ assert service._active_tasks["existing-watch"] is original_task
+ writer.set_enabled("new-watch", True)
+ new_started.clear()
+ assert await handler.process(await scan()) is True
+ await asyncio.wait_for(new_started.wait(), timeout=1)
+ assert started == ["existing-watch", "new-watch", "new-watch"]
+ assert service._active_tasks["new-watch"] is not new_task
+ finally:
+ service._running = False
+ finish_cleanup.set()
+ pending = list(service._active_tasks.values())
+ if legacy_task is not None:
+ pending.append(legacy_task)
+ for task in pending:
+ task.cancel()
+ await asyncio.gather(*pending, return_exceptions=True)
+
+ try:
+ asyncio.run(_run())
+ finally:
+ for store in sqlite_stores:
+ store.close()
+
+
def test_hfr_179_watch_wake_during_reconcile_replays_after_owner_release(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
@@ -3066,7 +3213,6 @@ def test_hfr_179_watch_wake_during_reconcile_replays_after_owner_release(
runtime_store=WatchRuntimeStateStore(tmp_path / "watch_runtime.json"),
)
service._recovery_pending = False
- service._reconcile_dirty = False
service._runtime_state_dirty = True
reload_calls = 0
persist_entered = asyncio.Event()
@@ -3125,7 +3271,6 @@ def test_hfr_179_blocked_watch_recovery_arms_generation_scoped_recheck(
token = object()
service._runtime_work_token = token # type: ignore[assignment]
service._recovery_pending = False
- service._reconcile_dirty = False
service._runtime_state_dirty = False
service._recovery_blocked_watch_ids.add("watch-a")
handler = _ManagedWatchRuntimeWorkHandler(service)
@@ -3134,7 +3279,6 @@ def test_hfr_179_blocked_watch_recovery_arms_generation_scoped_recheck(
{
"recovery": _StaleWorkerRecovery(True),
"unblocked": (),
- "changed": False,
"watches": (),
"store_error": None,
"fused": False,
@@ -3167,7 +3311,6 @@ def test_hfr_179_pending_watch_recovery_uses_watch_lane_cadence(
{
"recovery": _StaleWorkerRecovery(False),
"unblocked": (),
- "changed": False,
"watches": (),
"store_error": None,
"fused": False,
@@ -3191,7 +3334,6 @@ def test_hfr_179_watch_store_fuse_stops_generation_reads_and_retries(
runtime_store=WatchRuntimeStateStore(tmp_path / "watch_runtime.json"),
)
service._recovery_pending = False
- service._reconcile_dirty = False
service._runtime_state_dirty = False
reads = 0
@@ -3252,7 +3394,6 @@ def test_watch_runtime_state_persistence_failure_rearms_maintenance(
runtime_store=WatchRuntimeStateStore(tmp_path / "watch_runtime.json"),
)
service._recovery_pending = False
- service._reconcile_dirty = False
service._runtime_state_dirty = True
async def _persist_but_remain_dirty() -> None:
@@ -3265,7 +3406,6 @@ def test_watch_runtime_state_persistence_failure_rearms_maintenance(
{
"recovery": _StaleWorkerRecovery(True),
"unblocked": (),
- "changed": False,
"watches": (),
},
rearm_after_process=False,