Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.
Python agent-orchestrationai-agentai-coding-assistantclaude-codecoding-agentcomputer-usedesktop-applocal-firstmcppersonal-jarvisvoice-assistantvoice-coding
fix(missions): sweep a mission at once when its owner process died
rubenluetke10-beep0e97ba7c1ecd
diff --git a/docs/BUGS.md b/docs/BUGS.mdindex 2a6a29f161..01b4e8353b 100644--- a/docs/BUGS.md+++ b/docs/BUGS.md@@ -15566,3 +15566,27 @@ monitor. The Screen Context window probe reads this instead of the rawforeground window. macOS and X11 need no walk: their probes never report thefloating overlays. Guards: `tests/unit/platform/test_foreground_app_window.py`,`tests/unit/screen_context/test_ports.py`.++## BUG-229: a mission killed by an app restart stayed "running" for 37 minutes, then failed without an artifact (MEDIUM, FIXED 2026-10-02)++**Symptom.** An artifact build (a GitHub-stars history dashboard, mission+`01a0fcc3`) showed as running, then ended as `crash_recovery` /+Failed 38 minutes later with no artifact and no partial output.++**Cause.** The desktop app process was ended without a shutdown 85 s after+the worker started (15:19:40; no crash record, the next launcher started at+15:19:44). The Claude worker died with it before writing any file. The new+instance's recovery sweep then judged the mission only by its timestamps: its+last heartbeat was younger than `RECOVERY_STALE_AFTER_MS` (30 min), so it was+"presumed owned by a live instance" and skipped on every sweep — although no+process was running it any more. The re-sweep finally failed it at 15:56:42.++**Fix.** The orchestrator stamps its process identity (pid + process start+time) next to every mission heartbeat, starting the moment `run_mission`+begins (`missions.owner_pid` / `owner_start_ms`, migrated in place).+`startup_recover` asks `jarvis/missions/ownership.py` whether that owner is+alive: a provably dead owner (or a pid now reused by another process) is+swept at once with an `error_detail` naming the exited process. An alive or+unknown owner keeps the old freshness guard, so a second instance still never+sweeps a mission a live first instance is running. The restart that killed+the worker is outside this fix. Guard: `tests/missions/test_recovery_owner.py`.diff --git a/jarvis/missions/event_store.py b/jarvis/missions/event_store.pyindex ac9ba7566c..fd870a8e2e 100644--- a/jarvis/missions/event_store.py+++ b/jarvis/missions/event_store.py@@ -22,6 +22,7 @@ import aiosqlitefrom .event_bus import MissionBusfrom .events import EventEnvelope+from .ownership import current_process_identitylog = logging.getLogger(__name__)@@ -83,6 +84,12 @@ class MissionEventStore:"ALTER TABLE missions ADD COLUMN last_heartbeat_ms INTEGER NOT NULL DEFAULT 0")log.info("MissionEventStore: migration applied — added 'last_heartbeat_ms'")+ for column in ("owner_pid", "owner_start_ms"):+ if column not in existing_cols:+ await self.conn.execute(+ f"ALTER TABLE missions ADD COLUMN {column} INTEGER NOT NULL DEFAULT 0"+ )+ log.info("MissionEventStore: migration applied — added '%s'", column)async def close(self) -> None:if self._conn is not None:@@ -374,11 +381,32 @@ class MissionEventStore:so startup recovery can distinguish a busy-but-silent worker from agenuinely orphaned mission. Deliberately not an event: it must not bloatthe event log or wake the flight-recorder wildcard subscriber.++ The calling process also stamps itself as the mission's owner, so+ recovery can tell a mission whose owner died in a restart from one a+ live instance is still running (:mod:`jarvis.missions.ownership`)."""+ owner_pid, owner_start_ms = current_process_identity()await self.conn.execute(- "UPDATE missions SET last_heartbeat_ms = ? WHERE id = ?",- (ts_ms, mission_id),+ "UPDATE missions SET last_heartbeat_ms = ?, owner_pid = ?, "+ "owner_start_ms = ? WHERE id = ?",+ (ts_ms, owner_pid, owner_start_ms, mission_id),+ )++ async def get_owner(self, mission_id: str) -> tuple[int, int]:+ """``(pid, start_ms)`` of the process that last heartbeat a mission.++ ``(0, 0)`` when no owner was stamped (old row, or no heartbeat yet).+ """+ cur = await self.conn.execute(+ "SELECT owner_pid, owner_start_ms FROM missions WHERE id = ?",+ (mission_id,),)+ row = await cur.fetchone()+ await cur.close()+ if row is None:+ return (0, 0)+ return (int(row[0] or 0), int(row[1] or 0))async def get_heartbeat(self, mission_id: str) -> int:"""Last heartbeat ms for a mission, or 0 if none/unknown."""diff --git a/jarvis/missions/kontrollierer/orchestrator.py b/jarvis/missions/kontrollierer/orchestrator.pyindex 2b0ce221a7..67001ef652 100644--- a/jarvis/missions/kontrollierer/orchestrator.py+++ b/jarvis/missions/kontrollierer/orchestrator.py@@ -988,6 +988,12 @@ class Kontrollierer:# PENDING -> RUNNINGawait self._safe_transition(mission_id, MissionState.RUNNING, "kontrollierer-start")+ # Claim ownership before planning so recovery can tell, from the very+ # first second, whether the process running this mission still exists.+ try:+ await self._manager.store.touch_heartbeat(mission_id, now_ms())+ except Exception as hb_exc: # noqa: BLE001 - ownership stamp is advisory; freshness guard still applies+ logger.debug("Initial mission heartbeat failed (non-fatal): %s", hb_exc)# Decomposer: determine the plan.try:diff --git a/jarvis/missions/missions_schema.sql b/jarvis/missions/missions_schema.sqlindex f296a1c66b..673199427a 100644--- a/jarvis/missions/missions_schema.sql+++ b/jarvis/missions/missions_schema.sql@@ -20,7 +20,13 @@ CREATE TABLE IF NOT EXISTS missions (-- Computer-Use) is never swept as orphaned. Not an event: must not bloat-- the event log or wake the flight-recorder wildcard subscriber.-- Existing DBs are upgraded via _apply_migrations() in event_store.py.- last_heartbeat_ms INTEGER NOT NULL DEFAULT 0+ last_heartbeat_ms INTEGER NOT NULL DEFAULT 0,+ -- Identity of the process that wrote the heartbeat (pid + process start+ -- time in ms). Recovery sweeps a mission at once when this owner is+ -- provably dead instead of waiting out the freshness window+ -- (jarvis/missions/ownership.py). 0 = unknown.+ owner_pid INTEGER NOT NULL DEFAULT 0,+ owner_start_ms INTEGER NOT NULL DEFAULT 0);CREATE INDEX IF NOT EXISTS idx_missions_state ON missions(state);diff --git a/jarvis/missions/ownership.py b/jarvis/missions/ownership.pynew file mode 100644index 0000000000..b9be13fe33--- /dev/null+++ b/jarvis/missions/ownership.py@@ -0,0 +1,97 @@+"""Which process owns a running mission, and is that process still alive?++Recovery used to judge an unfinished mission only by how recently it was+active: anything with an event or heartbeat younger than+``RECOVERY_STALE_AFTER_MS`` (30 min) was "presumed owned by a live instance".+That guess is right when a second instance shares ``missions.db`` with a live+first one, but wrong after a plain app restart: the old process — and with it+the mission's worker — is gone, yet the new instance kept the mission RUNNING+for the full 30 minutes plus one re-sweep interval (live forensic 2026-10-02,+mission 01a0fcc3: worker killed by a restart at 15:19:40, swept to+FAILED('crash_recovery') only at 15:56:42, no artifact ever written).++The live orchestrator therefore stamps its own process identity next to the+heartbeat. Recovery asks :func:`owner_is_alive` about that stamp: a provably+dead owner means the mission is orphaned right now, whatever its timestamps+say. Anything uncertain (no stamp yet, psutil missing, access denied) answers+``None`` and recovery falls back to the old freshness guard, so a mission a+live instance is running is never swept on a guess.++The identity is ``(pid, start_ms)``. The process start time guards against PID+reuse: a new, unrelated process that inherited the dead owner's PID has a+different start time and does not count as the owner.+"""+from __future__ import annotations++import logging+import os++log = logging.getLogger(__name__)++# psutil rounds create_time differently across platforms; two readings of the+# same process can differ by a few milliseconds.+_START_TIME_TOLERANCE_MS = 1_000++_current_identity: tuple[int, int] | None = None+++def _process_start_ms(pid: int) -> int | None:+ """Start time of ``pid`` in epoch milliseconds, or None if unknowable."""+ try:+ import psutil+ except ImportError:+ return None+ try:+ return int(psutil.Process(pid).create_time() * 1000)+ except psutil.NoSuchProcess:+ raise+ except psutil.Error as exc:+ log.debug("mission ownership: cannot read start time of pid %s: %s", pid, exc)+ return None+++def current_process_identity() -> tuple[int, int]:+ """``(pid, start_ms)`` of this process; ``start_ms`` is 0 when unknown."""+ global _current_identity+ pid = os.getpid()+ if _current_identity is None or _current_identity[0] != pid:+ try:+ start_ms = _process_start_ms(pid) or 0+ except Exception as exc: # noqa: BLE001 - our own pid always exists; stay best-effort+ log.debug("mission ownership: own start time unavailable: %s", exc)+ start_ms = 0+ _current_identity = (pid, start_ms)+ return _current_identity+++def owner_is_alive(pid: int, start_ms: int) -> bool | None:+ """Whether the process that stamped ``(pid, start_ms)`` is still running.++ Returns ``True`` (alive), ``False`` (provably gone, or its PID now belongs+ to a different process) or ``None`` (no stamp or no way to tell).+ """+ if pid <= 0:+ return None+ if pid == os.getpid():+ own_start = current_process_identity()[1]+ if not start_ms or not own_start:+ return True+ return abs(own_start - start_ms) <= _START_TIME_TOLERANCE_MS+ try:+ import psutil+ except ImportError:+ return None+ try:+ actual_start = _process_start_ms(pid)+ except psutil.NoSuchProcess:+ return False+ except Exception as exc: # noqa: BLE001 - an unreadable owner must count as "unknown", not "dead"+ log.debug("mission ownership: liveness of pid %s unknown: %s", pid, exc)+ return None+ if actual_start is None:+ # The process exists but its start time is unreadable (access denied):+ # it may well be the owner, so do not declare it dead.+ return None+ if not start_ms:+ return None+ return abs(actual_start - start_ms) <= _START_TIME_TOLERANCE_MSdiff --git a/jarvis/missions/recovery.py b/jarvis/missions/recovery.pyindex dc701f6040..8529937ec6 100644--- a/jarvis/missions/recovery.py+++ b/jarvis/missions/recovery.py@@ -27,6 +27,14 @@ cloud-first headless-VPS case alike):2. ACTIVE-GUARD — a mission whose last event is younger than ``stale_after_ms``is presumed owned by a LIVE orchestrator and is SKIPPED. Only genuinelystale, orphaned missions are swept to FAILED('crash_recovery').++OWNER-AWARE (live forensic 2026-10-02, mission 01a0fcc3): the active-guard+alone kept a mission RUNNING for 37 minutes after an app restart had killed its+worker — the new instance "presumed" the dead old process still owned it. The+orchestrator now stamps its process identity with each heartbeat; a mission+whose stamped owner is provably dead is swept immediately, ahead of the+active-guard (:mod:`jarvis.missions.ownership`). An unknown owner keeps the+conservative freshness guard."""from __future__ import annotations@@ -44,6 +52,7 @@ from .events import (MissionTimedOut,now_ms,)+from .ownership import owner_is_alivefrom .state_machine import MissionStatelog = logging.getLogger(__name__)@@ -99,14 +108,17 @@ async def startup_recover(stale_after_ms: int = RECOVERY_STALE_AFTER_MS,now: int | None = None,now_fn: Callable[[], int] = now_ms,+ owner_alive_fn: Callable[[int, int], bool | None] = owner_is_alive,) -> list[str]:"""Recover genuinely-orphaned missions; never touch live or finished ones.For each non-terminal mission:1. If its event log carries a terminal event, reconcile the header tothat state (no new event emitted) and skip the sweep.- 2. Else if its last event is younger than ``stale_after_ms``, skip it —- a live orchestrator is presumed to own it.+ 2. Else if the process stamped as its owner is provably dead, sweep it+ now — nothing can finish it. Otherwise, if its last event is younger+ than ``stale_after_ms``, skip it — a live orchestrator is presumed+ to own it.3. Else mark it FAILED('crash_recovery'), emitting (in order):MissionStateChanged(to=FAILED) then MissionFailed.@@ -118,6 +130,8 @@ async def startup_recover(against (defaults to ``now_fn()``); injectable for tests. Note``now=0`` means the epoch (1970), NOT "use the default".now_fn: clock function (default :func:`now_ms`).+ owner_alive_fn: liveness probe for a stamped ``(pid, start_ms)``+ owner (default :func:`owner_is_alive`); injectable for tests.Returns:The list of mission_ids actually swept to FAILED (empty if nothing was@@ -130,6 +144,7 @@ async def startup_recover(interrupted_ids: list[str] = []reconciled_ids: list[str] = []skipped_active: list[str] = []+ owner_dead_ids: list[str] = []for mission_id, prompt, last_state in stale:events = await store.events_for_mission(mission_id)@@ -164,12 +179,32 @@ async def startup_recover(# activity to protect, so fall through to the sweep (do NOT treat# eventless + heartbeat-zero as active — that would leave it stuck# non-terminal forever).+ #+ # Owner check first: a mission whose stamped owner process is gone+ # (an app restart killed it, together with its worker) is orphaned+ # no matter how fresh its last heartbeat is.+ owner_pid, owner_start_ms = await store.get_owner(mission_id)+ owner_dead = owner_alive_fn(owner_pid, owner_start_ms) is Falselast_event_ts = events[-1].ts_ms if events else 0heartbeat_ts = await store.get_heartbeat(mission_id)freshness = max(last_event_ts, heartbeat_ts)- if stale_after_ms > 0 and freshness > 0 and (now_ts - freshness) < stale_after_ms:+ if (+ not owner_dead+ and stale_after_ms > 0+ and freshness > 0+ and (now_ts - freshness) < stale_after_ms+ ):skipped_active.append(mission_id)continue+ error_detail = (+ f"The process running this mission (pid {owner_pid}) exited before "+ "it finished — the app was restarted or closed while the worker "+ "was running."+ if owner_dead+ else None+ )+ if owner_dead:+ owner_dead_ids.append(mission_id)# 3. Genuinely orphaned → sweep to FAILED.# Distinguish "had delivered work" (interrupted delivery) from a bare@@ -199,6 +234,7 @@ async def startup_recover(error_class=error_class,last_state=last_state,partial_artifacts=delivered,+ error_detail=error_detail,),)await store.append_and_publish(fail_env)@@ -222,6 +258,13 @@ async def startup_recover(len(crash_ids),crash_ids,)+ if owner_dead_ids:+ log.warning(+ "Mission recovery: %d of those had a dead owner process (swept "+ "without waiting for the staleness window): %s",+ len(owner_dead_ids),+ owner_dead_ids,+ )if interrupted_ids:log.warning("Mission recovery: marked %d mission(s) FAILED('interrupted') "diff --git a/tests/missions/test_recovery_owner.py b/tests/missions/test_recovery_owner.pynew file mode 100644index 0000000000..b8ef08e803--- /dev/null+++ b/tests/missions/test_recovery_owner.py@@ -0,0 +1,164 @@+"""Owner-aware crash recovery (live forensic 2026-10-02, mission 01a0fcc3).++An artifact mission's worker was killed when the desktop app restarted at+15:19:40, 85 s after dispatch. The new instance's recovery sweep saw a heartbeat+younger than 30 minutes and "presumed" a live instance still owned the mission,+so it stayed RUNNING/CRITIQUING — with no worker behind it — until 15:56:42,+when it was finally swept to FAILED('crash_recovery') with no artifact.++The fix: the orchestrator stamps its process identity with each heartbeat, and+recovery sweeps a mission at once when that owner process is provably dead,+while an alive or unknown owner keeps the conservative freshness guard.+"""+from __future__ import annotations++import os+import subprocess+import sys+from pathlib import Path++import pytest_asyncio++from jarvis.core.process_utils import NO_WINDOW_CREATIONFLAGS+from jarvis.missions.events import now_ms+from jarvis.missions.manager import MissionManager+from jarvis.missions.ownership import current_process_identity, owner_is_alive+from jarvis.missions.recovery import startup_recover+from jarvis.missions.state_machine import MissionState+++@pytest_asyncio.fixture+async def open_store(tmp_missions_db: Path):+ m = MissionManager(tmp_missions_db)+ await m.start(recover=False)+ try:+ yield m+ finally:+ await m.stop()+++async def _critiquing_mission_owned_by(+ m: MissionManager, owner_pid: int, owner_start_ms: int+) -> str:+ """A mission mid-run whose heartbeat is 5 s old, stamped by ``owner_pid``."""+ mid = await m.dispatch(prompt="artifact build")+ await m.transition_state(mid, MissionState.RUNNING, reason="kontrollierer-start")+ await m.transition_state(mid, MissionState.CRITIQUING, reason="iter-0-start")+ await m.store.touch_heartbeat(mid, now_ms() - 5_000)+ await m.store.conn.execute(+ "UPDATE missions SET owner_pid = ?, owner_start_ms = ? WHERE id = ?",+ (owner_pid, owner_start_ms, mid),+ )+ return mid+++def _exited_pid() -> int:+ """PID of a real process that has already exited."""+ proc = subprocess.run(+ [sys.executable, "-c", "import os; print(os.getpid())"],+ capture_output=True,+ encoding="utf-8",+ timeout=30,+ check=True,+ creationflags=NO_WINDOW_CREATIONFLAGS,+ )+ return int(proc.stdout.strip())+++def _dead_owner(pid: int, start_ms: int) -> bool | None:+ return False+++def _alive_owner(pid: int, start_ms: int) -> bool | None:+ return True+++def _unknown_owner(pid: int, start_ms: int) -> bool | None:+ return None+++async def test_dead_owner_is_swept_despite_fresh_heartbeat(+ open_store: MissionManager,+) -> None:+ """The 01a0fcc3 case: fresh heartbeat, but the process that wrote it died+ in a restart. Recovery must fail the mission now, not 30+ minutes later."""+ mid = await _critiquing_mission_owned_by(open_store, 424242, 1_790_947_000_000)++ recovered = await startup_recover(open_store.store, owner_alive_fn=_dead_owner)++ assert recovered == [mid]+ view = await open_store.mission(mid)+ assert view is not None and view.state == MissionState.FAILED+ events = await open_store.store.events_for_mission(mid)+ failed = [e.payload for e in events if e.payload.event_type == "MissionFailed"]+ assert len(failed) == 1+ assert failed[0].reason == "crash_recovery"+ assert failed[0].last_state == MissionState.CRITIQUING.value+ assert failed[0].error_detail and "424242" in failed[0].error_detail+++async def test_alive_owner_keeps_the_freshness_guard(open_store: MissionManager) -> None:+ """A second instance sharing missions.db must never sweep a mission a live+ first instance is running (the 019e6fea false-FAILED bug stays fixed)."""+ mid = await _critiquing_mission_owned_by(open_store, 424242, 1_790_947_000_000)++ recovered = await startup_recover(open_store.store, owner_alive_fn=_alive_owner)++ assert recovered == []+ view = await open_store.mission(mid)+ assert view is not None and view.state == MissionState.CRITIQUING+++async def test_unknown_owner_keeps_the_freshness_guard(open_store: MissionManager) -> None:+ """No stamp, no psutil, access denied: never sweep on a guess."""+ mid = await _critiquing_mission_owned_by(open_store, 0, 0)++ recovered = await startup_recover(open_store.store, owner_alive_fn=_unknown_owner)++ assert recovered == []+ view = await open_store.mission(mid)+ assert view is not None and view.state == MissionState.CRITIQUING+++async def test_heartbeat_stamps_this_process_as_owner(open_store: MissionManager) -> None:+ mid = await open_store.dispatch(prompt="artifact build")++ assert await open_store.store.get_owner(mid) == (0, 0)+ await open_store.store.touch_heartbeat(mid, now_ms())++ assert await open_store.store.get_owner(mid) == current_process_identity()+ assert await open_store.store.get_owner("no-such-mission") == (0, 0)+++async def test_mission_heartbeat_by_this_live_process_is_not_swept(+ open_store: MissionManager,+) -> None:+ """End to end with the real liveness probe: the periodic re-sweep runs in+ the same process that owns the mission and must leave it alone."""+ mid = await open_store.dispatch(prompt="artifact build")+ await open_store.transition_state(mid, MissionState.RUNNING, reason="r")+ await open_store.store.touch_heartbeat(mid, now_ms())++ assert await startup_recover(open_store.store) == []+++async def test_mission_owned_by_an_exited_process_is_swept(+ open_store: MissionManager,+) -> None:+ """End to end with the real liveness probe and a real process that has+ exited — the restart that killed mission 01a0fcc3's owner."""+ mid = await _critiquing_mission_owned_by(open_store, _exited_pid(), now_ms() - 60_000)++ recovered = await startup_recover(open_store.store)++ assert recovered == [mid]+++def test_owner_is_alive_probe() -> None:+ pid, start_ms = current_process_identity()+ assert pid == os.getpid()+ assert owner_is_alive(pid, start_ms) is True+ assert owner_is_alive(0, 0) is None+ if start_ms:+ # Same pid, different start time: the pid was reused by a new process.+ assert owner_is_alive(pid, start_ms - 3_600_000) is False