Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
fix(harness): reconcile watches independently of reload hints (#1996)
Call White2f2009dc6fce
diff --git a/core/watches.py b/core/watches.pyindex 317f25fe3..c101b36ef 100644--- a/core/watches.py+++ b/core/watches.py@@ -1069,13 +1069,12 @@ class _ManagedWatchRuntimeWorkHandler(RuntimeWorkHandler):if recovery.recovered and blockedelse ())- changed = Falsewatches: tuple[ManagedWatch, ...] = ()store_error: Exception | None = Nonefused = self.service._store_error_fusedif recovery.recovered and not fused:try:- changed = self.service.store.maybe_reload()+ self.service.store.maybe_reload()watches = tuple(ManagedWatch.from_dict(watch.to_dict())for watch in self.service.store.list_watches()@@ -1088,7 +1087,6 @@ class _ManagedWatchRuntimeWorkHandler(RuntimeWorkHandler):{"recovery": recovery,"unblocked": unblocked,- "changed": changed,"watches": watches,"store_error": store_error,"fused": fused,@@ -1113,11 +1111,9 @@ class _ManagedWatchRuntimeWorkHandler(RuntimeWorkHandler):)self.service._recovery_pending = Falseif observation["unblocked"]:- self.service._reconcile_dirty = Trueself.service._runtime_state_dirty = Truestore_error = observation.get("store_error")if store_error is not None:- self.service._reconcile_dirty = Trueself.service._handle_reconcile_store_error(store_error)return self.service._store_error_fusedif observation.get("fused") or self.service._store_error_fused:@@ -1126,18 +1122,18 @@ class _ManagedWatchRuntimeWorkHandler(RuntimeWorkHandler):return not self.service._runtime_state_dirtyreturn Truetry:- if observation["changed"] or self.service._reconcile_dirty:- if self.service.reconcile_watches(observation["watches"]):- self.service._runtime_state_dirty = True+ # Another shared-store reader may have consumed maybe_reload's change+ # result. Reconcile the snapshot, not which reader refreshed it; the+ # active-task map already makes unchanged reconciliation idempotent.+ if self.service.reconcile_watches(observation["watches"]):+ self.service._runtime_state_dirty = Trueif self.service._runtime_state_dirty:await self.service._persist_runtime_state()if self.service._runtime_state_dirty:return Falseself.service._store_reconcile_failures = 0- self.service._reconcile_dirty = Falsereturn Trueexcept Exception as exc:- self.service._reconcile_dirty = Trueself.service._handle_reconcile_store_error(exc)return self.service._store_error_fused@@ -1158,6 +1154,7 @@ class ManagedWatchService:self._startup_task: Optional[asyncio.Task] = Noneself._reconcile_task: Optional[asyncio.Task] = Noneself._active_tasks: dict[str, asyncio.Task] = {}+ self._canceling_watch_ids: set[str] = set()self._active_pids: dict[str, int] = {}self._active_process_identities: dict[str, PersistedProcessIdentity] = {}self._watch_started_at: dict[str, str] = {}@@ -1168,7 +1165,6 @@ class ManagedWatchService:self._store_reconcile_failures = 0self._recovery_pending = Trueself._requires_service_lease = runtime.service_instance_lock_attached_to_process()- self._reconcile_dirty = Trueself._runtime_state_dirty = Trueself._runtime_state_revision = 0self._store_worker_lock = threading.Lock()@@ -1242,8 +1238,6 @@ class ManagedWatchService:notify = getattr(supervisor, "notify", None)if callable(notify):notify(RuntimeWorkLane.WATCH_DEFINITIONS)- else:- self._reconcile_dirty = Truedef _schedule_runtime_work_wake(self, delay: float) -> None:supervisor = getattr(self.controller, "runtime_work_supervisor", None)@@ -1281,9 +1275,7 @@ class ManagedWatchService:if self.reconcile_watches():self._runtime_state_dirty = Trueself._write_runtime_state()- self._reconcile_dirty = Falseexcept Exception as exc:- self._reconcile_dirty = Trueself._handle_reconcile_store_error(exc)if self._running and self._owns_service_instance() and self._supports_runtime_work_lane():self._register_runtime_work_lane()@@ -1586,6 +1578,7 @@ class ManagedWatchService:if self._active_tasks:await asyncio.gather(*self._active_tasks.values(), return_exceptions=True)self._active_tasks.clear()+ self._canceling_watch_ids.clear()self._active_pids.clear()self._active_process_identities.clear()self._watch_started_at.clear()@@ -1614,7 +1607,6 @@ class ManagedWatchService:await asyncio.sleep(WATCH_RECONCILE_INTERVAL_SECONDS)continueself._recovery_pending = False- self._reconcile_dirty = Trueself._runtime_state_dirty = Trueif self._store_error_fused:await asyncio.sleep(WATCH_RECONCILE_INTERVAL_SECONDS)@@ -1633,20 +1625,16 @@ class ManagedWatchService:else:unblocked = ()if self._apply_recovery_unblocked(unblocked):- self._reconcile_dirty = Trueself._runtime_state_dirty = True- should_reconcile = self.store.maybe_reload() or self._reconcile_dirty- if should_reconcile:- if self.reconcile_watches():- self._runtime_state_dirty = True+ self.store.maybe_reload()+ if self.reconcile_watches():+ self._runtime_state_dirty = Trueif self._runtime_state_dirty:self._write_runtime_state()self._store_reconcile_failures = 0- self._reconcile_dirty = Falseexcept asyncio.CancelledError:raiseexcept Exception as exc:- self._reconcile_dirty = Trueself._handle_reconcile_store_error(exc)await asyncio.sleep(WATCH_RECONCILE_INTERVAL_SECONDS)@@ -1677,18 +1665,26 @@ class ManagedWatchService:for watch_id, task in list(self._active_tasks.items()):if watch_id in desired_ids:continue- task.cancel()- changed = True+ if self._cancel_watch(watch_id, task):+ changed = Truereturn changed+ def _cancel_watch(self, watch_id: str, task: asyncio.Task) -> bool:+ # Repeated scans or shutdown must not interrupt asynchronous teardown.+ # Track our request explicitly, including on Python 3.10.+ if watch_id in self._canceling_watch_ids or not task.cancel():+ return False+ self._canceling_watch_ids.add(watch_id)+ return True+def _on_watch_done(self, watch_id: str) -> None:self._active_tasks.pop(watch_id, None)+ self._canceling_watch_ids.discard(watch_id)self._active_pids.pop(watch_id, None)self._active_process_identities.pop(watch_id, None)self._watch_started_at.pop(watch_id, None)self._write_runtime_state()- self._reconcile_dirty = Truedef _runtime_state_payload(self) -> dict[str, Any]:"""Assemble the loop-owned runtime projection without storage I/O."""@@ -1847,9 +1843,9 @@ class ManagedWatchService:self._reconcile_task.cancel()if self._legacy_probe_task and self._legacy_probe_task is not current_task:self._legacy_probe_task.cancel()- for task in list(self._active_tasks.values()):+ for watch_id, task in list(self._active_tasks.items()):if task is not current_task:- task.cancel()+ self._cancel_watch(watch_id, task)self._runtime_state_dirty = Trueself._write_runtime_state()diff --git a/docs/plans/watch-reconcile-current-snapshot.md b/docs/plans/watch-reconcile-current-snapshot.mdnew file mode 100644index 000000000..b8d09f87c--- /dev/null+++ b/docs/plans/watch-reconcile-current-snapshot.md@@ -0,0 +1,38 @@+# Watch reconciliation against current snapshots++Issue: #1995.++## Change contract++- An enabled, eligible Watch must reach scheduling regardless of which reader+ refreshes the shared store first. No resume, unrelated write, or restart is+ required.+- A change arriving between scan and apply must converge on the next scan.+- Repeated unchanged snapshots must keep the same active worker, avoid idle+ runtime-state writes, and retain recovery, fuse, service-lease, and guarded+ lifecycle-write protections.+- Apply the same contract to the runtime-work lane and legacy polling loop.++## Approach++Always reconcile a successfully read snapshot through the existing idempotent+`reconcile_watches`. Store reload detection only controls cache freshness; it+cannot acknowledge scheduling work. Remove the redundant reconciliation dirty+flag instead of introducing another revision counter or notification path.+The runtime-work scan already copies every Watch; the legacy loop now checks+its in-memory list each tick. Unchanged reconciliation performs no durable write.+Already-cancelling workers retain their original cancellation so later scans+cannot interrupt their asynchronous teardown.++## Evidence and boundaries++HFR-485 covers file-backed and isolated SQLite-backed stores, reader-before-scan,+reader-between-scan-and-apply, legacy polling, retained worker identity, and+disable-after-reload. It uses real stores, scheduler handlers, and reconciliation,+with blocking coroutine workers instead of external waiter processes.+Existing HFR-179 tests cover single-flight store access and deferred wakes.++The prior tests checked reload and reconciliation independently, not whether a+different reader could consume the reload result before reconciliation.+No schema, CLI timeout, external notification, or deployment change is needed.+Live IM delivery and deployed-service verification remain separate acceptance work.diff --git a/tests/scenarios/harness_failure_recovery/catalog.yaml b/tests/scenarios/harness_failure_recovery/catalog.yamlindex 63f89d411..2c468c44c 100644--- a/tests/scenarios/harness_failure_recovery/catalog.yaml+++ b/tests/scenarios/harness_failure_recovery/catalog.yaml@@ -4382,6 +4382,16 @@ scenarios:# refused for a merely backlogged predecessor leaves its registration alone.# An edit that keeps the definition recurring still runs, at its refreshed value.test: tests/test_scheduled_tasks.py::test_hfr_484_in_flight_cron_fire_cannot_spend_a_replacement_one_shot+ - id: HFR-485+ name: Watch scheduling survives another reader consuming the store reload+ status: covered+ kind: concurrency+ layer: scenario+ backend: shared+ # File and SQLite stores, both scheduler paths, and changes before or after+ # snapshot capture must converge without a resume or another definition write.+ # Unchanged snapshots retain worker identity; consumed disables still stop it.+ test: tests/test_watches.py::test_hfr_485_watch_reconciles_after_another_reader_consumes_reloadmanual_evidence:- local Incus backend process death after prompt acceptance- local Incus controller restart with persisted queued successordiff --git a/tests/scenarios/harness_failure_recovery/observations.yaml b/tests/scenarios/harness_failure_recovery/observations.yamlindex 89c5f7278..bb53dffd6 100644--- a/tests/scenarios/harness_failure_recovery/observations.yaml+++ b/tests/scenarios/harness_failure_recovery/observations.yaml@@ -311,3 +311,17 @@ observations:the handoff before any backend started. The binding transaction must atomicallyadopt the freshly reserved Session only for create_per_run rows; a Run alreadybound to another Session must remain a hard refusal.+ - id: HFR-OBS-026+ date: 2026-09-16+ source: issue 1995 and isolated file/SQLite shared-reader reproduction+ result: confirmed+ detail: >-+ maybe_reload acknowledges cache freshness, not scheduler reconciliation.+ An existing Watch can refresh the shared mirror before the scheduler scans,+ leaving an enabled new Watch in every snapshot but never scheduled. Repeated+ scans cannot recover when their reload result is used as the admission gate.+ HFR-485 exercises real stores and worker-task ownership across both scheduler+ paths; only external waiter execution is replaced by a blocking coroutine.+ Prior reload and single-flight tests did not connect that reader ordering to+ actual scheduling. Always reconciling the captured snapshot retains existing+ worker identity and avoids adding a second change-acknowledgement mechanism.diff --git a/tests/test_watches.py b/tests/test_watches.pyindex aa1781d2d..fc54f383a 100644--- a/tests/test_watches.py+++ b/tests/test_watches.py@@ -2486,7 +2486,6 @@ def test_managed_watch_service_idle_tick_does_not_write_runtime_state(tmp_path:)service._running = Trueservice._recovery_pending = False- service._reconcile_dirty = Falseservice._runtime_state_dirty = Falseasync def _run() -> None:@@ -2502,7 +2501,7 @@ def test_managed_watch_service_idle_tick_does_not_write_runtime_state(tmp_path:asyncio.run(_run())assert store.reloads > 0- assert store.lists == 0+ assert store.lists == store.reloadsassert runtime_store.writes == 0@@ -2591,7 +2590,8 @@ def test_managed_watch_service_start_reaps_matching_stale_worker_before_reconcilasyncio.run(_run())- assert events == [("terminate", 4321), ("reconcile", None)]+ assert events[0] == ("terminate", 4321)+ assert events[1:] and all(event == ("reconcile", None) for event in events[1:])def test_managed_watch_service_start_does_not_reap_reused_pid(@@ -2635,7 +2635,7 @@ def test_managed_watch_service_start_does_not_reap_reused_pid(asyncio.run(_run())- assert reconciles == 1+ assert reconciles >= 1assert watch.id not in service._recovery_blocked_watch_ids@@ -2808,7 +2808,7 @@ def test_managed_watch_service_start_ignores_dead_recorded_pid(asyncio.run(_run())- assert reconciles == 1+ assert reconciles >= 1def test_managed_watch_service_start_reaps_group_after_leader_exit(@@ -2851,7 +2851,8 @@ def test_managed_watch_service_start_reaps_group_after_leader_exit(asyncio.run(_run())- assert events == ["terminate-group", "reconcile"]+ assert events[0] == "terminate-group"+ assert events[1:] and all(event == "reconcile" for event in events[1:])def test_managed_watch_service_does_not_reap_unverified_group_after_leader_exit(@@ -3054,6 +3055,152 @@ def test_hfr_179_watch_store_phases_are_single_flight_and_apply_on_loop(asyncio.run(_run())[email protected]("backend", ["file", "sqlite"])[email protected]("ordering", ["before_scan", "after_scan", "legacy"])+def test_hfr_485_watch_reconciles_after_another_reader_consumes_reload(+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+ backend: str,+ ordering: str,+) -> None:+ """Issue #1995: a refreshed mirror is not an acknowledged scheduling change."""+ monkeypatch.setattr(watches_module, "_publish_watch_definitions_updated", lambda: None)+ monkeypatch.setattr(watches_module, "WATCH_RECONCILE_INTERVAL_SECONDS", 0.01)+ reader = ManagedWatchStore(tmp_path / "watches.json")+ writer = ManagedWatchStore(tmp_path / "watches.json")+ sqlite_stores = []+ if backend == "sqlite":+ for store in (reader, writer):+ store._sqlite = SQLiteBackgroundTaskStore(tmp_path / "isolated.sqlite")+ sqlite_stores.append(store._sqlite)+ store.load()+ reader.maybe_reload()+ assert reader.maybe_reload() is False+ service = ManagedWatchService(+ controller=SimpleNamespace(),+ store=reader,+ request_store=TaskExecutionStore(tmp_path / "task_requests"),+ runtime_store=WatchRuntimeStateStore(tmp_path / "watch_runtime.json"),+ )+ service._recovery_pending = False+ handler = _ManagedWatchRuntimeWorkHandler(service)+ started: list[str] = []+ runtime_writes: list[dict] = []+ write_runtime = service.runtime_store.write++ def record_runtime_write(payload: dict) -> None:+ runtime_writes.append(payload)+ write_runtime(payload)++ monkeypatch.setattr(service.runtime_store, "write", record_runtime_write)++ async def _run() -> None:+ new_started = asyncio.Event()+ release_workers = asyncio.Event()+ cleanup_started = asyncio.Event()+ finish_cleanup = asyncio.Event()++ async def worker(watch_id: str) -> None:+ started.append(watch_id)+ if watch_id == "new-watch":+ new_started.set()+ try:+ await release_workers.wait()+ finally:+ if watch_id == "new-watch":+ cleanup_started.set()+ await finish_cleanup.wait()++ monkeypatch.setattr(service, "_run_watch", worker)++ async def scan():+ items, _ = await asyncio.to_thread(+ handler.scan, limit=1, occupied=frozenset(), cursor=None,+ )+ return items[0]++ service._running = True+ legacy_task = None+ try:+ writer.upsert_watch(watches_module.ManagedWatch(+ id="existing-watch", name="Existing", session_key="",+ ))+ assert await handler.process(await scan()) is True+ await asyncio.sleep(0)+ assert started == ["existing-watch"]+ original_task = service._active_tasks["existing-watch"]++ # Also cover a definition arriving after scan captured its old snapshot.+ stale_item = await scan() if ordering == "after_scan" else None+ writer.upsert_watch(watches_module.ManagedWatch(+ id="new-watch", name="新 Watch", session_key="",+ ))+ assert await service._watch_store_call_async(+ "existing-watch", "reload before follow-up fence", reader.maybe_reload,+ ) is True+ assert reader.get_watch("new-watch") is not None+ assert reader.maybe_reload() is False++ if ordering == "legacy":+ legacy_task = asyncio.create_task(service._watch_store())+ else:+ if stale_item is not None:+ assert await handler.process(stale_item) is True+ current = await scan()+ assert {watch.id for watch in current.observation["watches"]} == {+ "existing-watch", "new-watch",+ }+ assert await handler.process(current) is True+ await asyncio.wait_for(new_started.wait(), timeout=1)++ writes_before_idle = len(runtime_writes)+ for _ in range(3):+ assert await handler.process(await scan()) is True+ await asyncio.sleep(0)+ assert started == ["existing-watch", "new-watch"]+ assert service._active_tasks["existing-watch"] is original_task+ assert len(runtime_writes) == writes_before_idle++ # A consumed disabling change must still retire the existing owner.+ writer.set_enabled("new-watch", False)+ assert await service._watch_store_call_async(+ "existing-watch", "reload", reader.maybe_reload,+ ) is True+ new_task = service._active_tasks["new-watch"]+ assert await handler.process(await scan()) is True+ await asyncio.wait_for(cleanup_started.wait(), timeout=1)+ writes_before_cleanup = len(runtime_writes)+ for _ in range(3):+ assert await handler.process(await scan()) is True+ assert not new_task.done()+ assert len(runtime_writes) == writes_before_cleanup+ finish_cleanup.set()+ await asyncio.gather(new_task, return_exceptions=True)+ assert new_task.cancelled()+ assert service._active_tasks["existing-watch"] is original_task+ writer.set_enabled("new-watch", True)+ new_started.clear()+ assert await handler.process(await scan()) is True+ await asyncio.wait_for(new_started.wait(), timeout=1)+ assert started == ["existing-watch", "new-watch", "new-watch"]+ assert service._active_tasks["new-watch"] is not new_task+ finally:+ service._running = False+ finish_cleanup.set()+ pending = list(service._active_tasks.values())+ if legacy_task is not None:+ pending.append(legacy_task)+ for task in pending:+ task.cancel()+ await asyncio.gather(*pending, return_exceptions=True)++ try:+ asyncio.run(_run())+ finally:+ for store in sqlite_stores:+ store.close()++def test_hfr_179_watch_wake_during_reconcile_replays_after_owner_release(tmp_path: Path,monkeypatch: pytest.MonkeyPatch,@@ -3066,7 +3213,6 @@ def test_hfr_179_watch_wake_during_reconcile_replays_after_owner_release(runtime_store=WatchRuntimeStateStore(tmp_path / "watch_runtime.json"),)service._recovery_pending = False- service._reconcile_dirty = Falseservice._runtime_state_dirty = Truereload_calls = 0persist_entered = asyncio.Event()@@ -3125,7 +3271,6 @@ def test_hfr_179_blocked_watch_recovery_arms_generation_scoped_recheck(token = object()service._runtime_work_token = token # type: ignore[assignment]service._recovery_pending = False- service._reconcile_dirty = Falseservice._runtime_state_dirty = Falseservice._recovery_blocked_watch_ids.add("watch-a")handler = _ManagedWatchRuntimeWorkHandler(service)@@ -3134,7 +3279,6 @@ def test_hfr_179_blocked_watch_recovery_arms_generation_scoped_recheck({"recovery": _StaleWorkerRecovery(True),"unblocked": (),- "changed": False,"watches": (),"store_error": None,"fused": False,@@ -3167,7 +3311,6 @@ def test_hfr_179_pending_watch_recovery_uses_watch_lane_cadence({"recovery": _StaleWorkerRecovery(False),"unblocked": (),- "changed": False,"watches": (),"store_error": None,"fused": False,@@ -3191,7 +3334,6 @@ def test_hfr_179_watch_store_fuse_stops_generation_reads_and_retries(runtime_store=WatchRuntimeStateStore(tmp_path / "watch_runtime.json"),)service._recovery_pending = False- service._reconcile_dirty = Falseservice._runtime_state_dirty = Falsereads = 0@@ -3252,7 +3394,6 @@ def test_watch_runtime_state_persistence_failure_rearms_maintenance(runtime_store=WatchRuntimeStateStore(tmp_path / "watch_runtime.json"),)service._recovery_pending = False- service._reconcile_dirty = Falseservice._runtime_state_dirty = Trueasync def _persist_but_remain_dirty() -> None:@@ -3265,7 +3406,6 @@ def test_watch_runtime_state_persistence_failure_rearms_maintenance({"recovery": _StaleWorkerRecovery(True),"unblocked": (),- "changed": False,"watches": (),},rearm_after_process=False,