Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

fix(missions): sweep a mission at once when its owner process died

rubenluetke10-beep0e97ba7c1ecd
diff --git a/docs/BUGS.md b/docs/BUGS.md
index 2a6a29f161..01b4e8353b 100644
--- a/docs/BUGS.md
+++ b/docs/BUGS.md
@@ -15566,3 +15566,27 @@ monitor. The Screen Context window probe reads this instead of the raw
foreground window. macOS and X11 need no walk: their probes never report the
floating overlays. Guards: `tests/unit/platform/test_foreground_app_window.py`,
`tests/unit/screen_context/test_ports.py`.
+
+## BUG-229: a mission killed by an app restart stayed "running" for 37 minutes, then failed without an artifact (MEDIUM, FIXED 2026-10-02)
+
+**Symptom.** An artifact build (a GitHub-stars history dashboard, mission
+`01a0fcc3`) showed as running, then ended as `crash_recovery` /
+Failed 38 minutes later with no artifact and no partial output.
+
+**Cause.** The desktop app process was ended without a shutdown 85 s after
+the worker started (15:19:40; no crash record, the next launcher started at
+15:19:44). The Claude worker died with it before writing any file. The new
+instance's recovery sweep then judged the mission only by its timestamps: its
+last heartbeat was younger than `RECOVERY_STALE_AFTER_MS` (30 min), so it was
+"presumed owned by a live instance" and skipped on every sweep — although no
+process was running it any more. The re-sweep finally failed it at 15:56:42.
+
+**Fix.** The orchestrator stamps its process identity (pid + process start
+time) next to every mission heartbeat, starting the moment `run_mission`
+begins (`missions.owner_pid` / `owner_start_ms`, migrated in place).
+`startup_recover` asks `jarvis/missions/ownership.py` whether that owner is
+alive: a provably dead owner (or a pid now reused by another process) is
+swept at once with an `error_detail` naming the exited process. An alive or
+unknown owner keeps the old freshness guard, so a second instance still never
+sweeps a mission a live first instance is running. The restart that killed
+the worker is outside this fix. Guard: `tests/missions/test_recovery_owner.py`.
diff --git a/jarvis/missions/event_store.py b/jarvis/missions/event_store.py
index ac9ba7566c..fd870a8e2e 100644
--- a/jarvis/missions/event_store.py
+++ b/jarvis/missions/event_store.py
@@ -22,6 +22,7 @@ import aiosqlite
from .event_bus import MissionBus
from .events import EventEnvelope
+from .ownership import current_process_identity
log = logging.getLogger(__name__)
@@ -83,6 +84,12 @@ class MissionEventStore:
"ALTER TABLE missions ADD COLUMN last_heartbeat_ms INTEGER NOT NULL DEFAULT 0"
)
log.info("MissionEventStore: migration applied — added 'last_heartbeat_ms'")
+ for column in ("owner_pid", "owner_start_ms"):
+ if column not in existing_cols:
+ await self.conn.execute(
+ f"ALTER TABLE missions ADD COLUMN {column} INTEGER NOT NULL DEFAULT 0"
+ )
+ log.info("MissionEventStore: migration applied — added '%s'", column)
async def close(self) -> None:
if self._conn is not None:
@@ -374,11 +381,32 @@ class MissionEventStore:
so startup recovery can distinguish a busy-but-silent worker from a
genuinely orphaned mission. Deliberately not an event: it must not bloat
the event log or wake the flight-recorder wildcard subscriber.
+
+ The calling process also stamps itself as the mission's owner, so
+ recovery can tell a mission whose owner died in a restart from one a
+ live instance is still running (:mod:`jarvis.missions.ownership`).
"""
+ owner_pid, owner_start_ms = current_process_identity()
await self.conn.execute(
- "UPDATE missions SET last_heartbeat_ms = ? WHERE id = ?",
- (ts_ms, mission_id),
+ "UPDATE missions SET last_heartbeat_ms = ?, owner_pid = ?, "
+ "owner_start_ms = ? WHERE id = ?",
+ (ts_ms, owner_pid, owner_start_ms, mission_id),
+ )
+
+ async def get_owner(self, mission_id: str) -> tuple[int, int]:
+ """``(pid, start_ms)`` of the process that last heartbeat a mission.
+
+ ``(0, 0)`` when no owner was stamped (old row, or no heartbeat yet).
+ """
+ cur = await self.conn.execute(
+ "SELECT owner_pid, owner_start_ms FROM missions WHERE id = ?",
+ (mission_id,),
)
+ row = await cur.fetchone()
+ await cur.close()
+ if row is None:
+ return (0, 0)
+ return (int(row[0] or 0), int(row[1] or 0))
async def get_heartbeat(self, mission_id: str) -> int:
"""Last heartbeat ms for a mission, or 0 if none/unknown."""
diff --git a/jarvis/missions/kontrollierer/orchestrator.py b/jarvis/missions/kontrollierer/orchestrator.py
index 2b0ce221a7..67001ef652 100644
--- a/jarvis/missions/kontrollierer/orchestrator.py
+++ b/jarvis/missions/kontrollierer/orchestrator.py
@@ -988,6 +988,12 @@ class Kontrollierer:
# PENDING -> RUNNING
await self._safe_transition(mission_id, MissionState.RUNNING, "kontrollierer-start")
+ # Claim ownership before planning so recovery can tell, from the very
+ # first second, whether the process running this mission still exists.
+ try:
+ await self._manager.store.touch_heartbeat(mission_id, now_ms())
+ except Exception as hb_exc: # noqa: BLE001 - ownership stamp is advisory; freshness guard still applies
+ logger.debug("Initial mission heartbeat failed (non-fatal): %s", hb_exc)
# Decomposer: determine the plan.
try:
diff --git a/jarvis/missions/missions_schema.sql b/jarvis/missions/missions_schema.sql
index f296a1c66b..673199427a 100644
--- a/jarvis/missions/missions_schema.sql
+++ b/jarvis/missions/missions_schema.sql
@@ -20,7 +20,13 @@ CREATE TABLE IF NOT EXISTS missions (
-- Computer-Use) is never swept as orphaned. Not an event: must not bloat
-- the event log or wake the flight-recorder wildcard subscriber.
-- Existing DBs are upgraded via _apply_migrations() in event_store.py.
- last_heartbeat_ms INTEGER NOT NULL DEFAULT 0
+ last_heartbeat_ms INTEGER NOT NULL DEFAULT 0,
+ -- Identity of the process that wrote the heartbeat (pid + process start
+ -- time in ms). Recovery sweeps a mission at once when this owner is
+ -- provably dead instead of waiting out the freshness window
+ -- (jarvis/missions/ownership.py). 0 = unknown.
+ owner_pid INTEGER NOT NULL DEFAULT 0,
+ owner_start_ms INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX IF NOT EXISTS idx_missions_state ON missions(state);
diff --git a/jarvis/missions/ownership.py b/jarvis/missions/ownership.py
new file mode 100644
index 0000000000..b9be13fe33
--- /dev/null
+++ b/jarvis/missions/ownership.py
@@ -0,0 +1,97 @@
+"""Which process owns a running mission, and is that process still alive?
+
+Recovery used to judge an unfinished mission only by how recently it was
+active: anything with an event or heartbeat younger than
+``RECOVERY_STALE_AFTER_MS`` (30 min) was "presumed owned by a live instance".
+That guess is right when a second instance shares ``missions.db`` with a live
+first one, but wrong after a plain app restart: the old process — and with it
+the mission's worker — is gone, yet the new instance kept the mission RUNNING
+for the full 30 minutes plus one re-sweep interval (live forensic 2026-10-02,
+mission 01a0fcc3: worker killed by a restart at 15:19:40, swept to
+FAILED('crash_recovery') only at 15:56:42, no artifact ever written).
+
+The live orchestrator therefore stamps its own process identity next to the
+heartbeat. Recovery asks :func:`owner_is_alive` about that stamp: a provably
+dead owner means the mission is orphaned right now, whatever its timestamps
+say. Anything uncertain (no stamp yet, psutil missing, access denied) answers
+``None`` and recovery falls back to the old freshness guard, so a mission a
+live instance is running is never swept on a guess.
+
+The identity is ``(pid, start_ms)``. The process start time guards against PID
+reuse: a new, unrelated process that inherited the dead owner's PID has a
+different start time and does not count as the owner.
+"""
+from __future__ import annotations
+
+import logging
+import os
+
+log = logging.getLogger(__name__)
+
+# psutil rounds create_time differently across platforms; two readings of the
+# same process can differ by a few milliseconds.
+_START_TIME_TOLERANCE_MS = 1_000
+
+_current_identity: tuple[int, int] | None = None
+
+
+def _process_start_ms(pid: int) -> int | None:
+ """Start time of ``pid`` in epoch milliseconds, or None if unknowable."""
+ try:
+ import psutil
+ except ImportError:
+ return None
+ try:
+ return int(psutil.Process(pid).create_time() * 1000)
+ except psutil.NoSuchProcess:
+ raise
+ except psutil.Error as exc:
+ log.debug("mission ownership: cannot read start time of pid %s: %s", pid, exc)
+ return None
+
+
+def current_process_identity() -> tuple[int, int]:
+ """``(pid, start_ms)`` of this process; ``start_ms`` is 0 when unknown."""
+ global _current_identity
+ pid = os.getpid()
+ if _current_identity is None or _current_identity[0] != pid:
+ try:
+ start_ms = _process_start_ms(pid) or 0
+ except Exception as exc: # noqa: BLE001 - our own pid always exists; stay best-effort
+ log.debug("mission ownership: own start time unavailable: %s", exc)
+ start_ms = 0
+ _current_identity = (pid, start_ms)
+ return _current_identity
+
+
+def owner_is_alive(pid: int, start_ms: int) -> bool | None:
+ """Whether the process that stamped ``(pid, start_ms)`` is still running.
+
+ Returns ``True`` (alive), ``False`` (provably gone, or its PID now belongs
+ to a different process) or ``None`` (no stamp or no way to tell).
+ """
+ if pid <= 0:
+ return None
+ if pid == os.getpid():
+ own_start = current_process_identity()[1]
+ if not start_ms or not own_start:
+ return True
+ return abs(own_start - start_ms) <= _START_TIME_TOLERANCE_MS
+ try:
+ import psutil
+ except ImportError:
+ return None
+ try:
+ actual_start = _process_start_ms(pid)
+ except psutil.NoSuchProcess:
+ return False
+ except Exception as exc: # noqa: BLE001 - an unreadable owner must count as "unknown", not "dead"
+ log.debug("mission ownership: liveness of pid %s unknown: %s", pid, exc)
+ return None
+ if actual_start is None:
+ # The process exists but its start time is unreadable (access denied):
+ # it may well be the owner, so do not declare it dead.
+ return None
+ if not start_ms:
+ return None
+ return abs(actual_start - start_ms) <= _START_TIME_TOLERANCE_MS
diff --git a/jarvis/missions/recovery.py b/jarvis/missions/recovery.py
index dc701f6040..8529937ec6 100644
--- a/jarvis/missions/recovery.py
+++ b/jarvis/missions/recovery.py
@@ -27,6 +27,14 @@ cloud-first headless-VPS case alike):
2. ACTIVE-GUARD — a mission whose last event is younger than ``stale_after_ms``
is presumed owned by a LIVE orchestrator and is SKIPPED. Only genuinely
stale, orphaned missions are swept to FAILED('crash_recovery').
+
+OWNER-AWARE (live forensic 2026-10-02, mission 01a0fcc3): the active-guard
+alone kept a mission RUNNING for 37 minutes after an app restart had killed its
+worker — the new instance "presumed" the dead old process still owned it. The
+orchestrator now stamps its process identity with each heartbeat; a mission
+whose stamped owner is provably dead is swept immediately, ahead of the
+active-guard (:mod:`jarvis.missions.ownership`). An unknown owner keeps the
+conservative freshness guard.
"""
from __future__ import annotations
@@ -44,6 +52,7 @@ from .events import (
MissionTimedOut,
now_ms,
)
+from .ownership import owner_is_alive
from .state_machine import MissionState
log = logging.getLogger(__name__)
@@ -99,14 +108,17 @@ async def startup_recover(
stale_after_ms: int = RECOVERY_STALE_AFTER_MS,
now: int | None = None,
now_fn: Callable[[], int] = now_ms,
+ owner_alive_fn: Callable[[int, int], bool | None] = owner_is_alive,
) -> list[str]:
"""Recover genuinely-orphaned missions; never touch live or finished ones.
For each non-terminal mission:
1. If its event log carries a terminal event, reconcile the header to
that state (no new event emitted) and skip the sweep.
- 2. Else if its last event is younger than ``stale_after_ms``, skip it —
- a live orchestrator is presumed to own it.
+ 2. Else if the process stamped as its owner is provably dead, sweep it
+ now — nothing can finish it. Otherwise, if its last event is younger
+ than ``stale_after_ms``, skip it — a live orchestrator is presumed
+ to own it.
3. Else mark it FAILED('crash_recovery'), emitting (in order):
MissionStateChanged(to=FAILED) then MissionFailed.
@@ -118,6 +130,8 @@ async def startup_recover(
against (defaults to ``now_fn()``); injectable for tests. Note
``now=0`` means the epoch (1970), NOT "use the default".
now_fn: clock function (default :func:`now_ms`).
+ owner_alive_fn: liveness probe for a stamped ``(pid, start_ms)``
+ owner (default :func:`owner_is_alive`); injectable for tests.
Returns:
The list of mission_ids actually swept to FAILED (empty if nothing was
@@ -130,6 +144,7 @@ async def startup_recover(
interrupted_ids: list[str] = []
reconciled_ids: list[str] = []
skipped_active: list[str] = []
+ owner_dead_ids: list[str] = []
for mission_id, prompt, last_state in stale:
events = await store.events_for_mission(mission_id)
@@ -164,12 +179,32 @@ async def startup_recover(
# activity to protect, so fall through to the sweep (do NOT treat
# eventless + heartbeat-zero as active — that would leave it stuck
# non-terminal forever).
+ #
+ # Owner check first: a mission whose stamped owner process is gone
+ # (an app restart killed it, together with its worker) is orphaned
+ # no matter how fresh its last heartbeat is.
+ owner_pid, owner_start_ms = await store.get_owner(mission_id)
+ owner_dead = owner_alive_fn(owner_pid, owner_start_ms) is False
last_event_ts = events[-1].ts_ms if events else 0
heartbeat_ts = await store.get_heartbeat(mission_id)
freshness = max(last_event_ts, heartbeat_ts)
- if stale_after_ms > 0 and freshness > 0 and (now_ts - freshness) < stale_after_ms:
+ if (
+ not owner_dead
+ and stale_after_ms > 0
+ and freshness > 0
+ and (now_ts - freshness) < stale_after_ms
+ ):
skipped_active.append(mission_id)
continue
+ error_detail = (
+ f"The process running this mission (pid {owner_pid}) exited before "
+ "it finished — the app was restarted or closed while the worker "
+ "was running."
+ if owner_dead
+ else None
+ )
+ if owner_dead:
+ owner_dead_ids.append(mission_id)
# 3. Genuinely orphaned → sweep to FAILED.
# Distinguish "had delivered work" (interrupted delivery) from a bare
@@ -199,6 +234,7 @@ async def startup_recover(
error_class=error_class,
last_state=last_state,
partial_artifacts=delivered,
+ error_detail=error_detail,
),
)
await store.append_and_publish(fail_env)
@@ -222,6 +258,13 @@ async def startup_recover(
len(crash_ids),
crash_ids,
)
+ if owner_dead_ids:
+ log.warning(
+ "Mission recovery: %d of those had a dead owner process (swept "
+ "without waiting for the staleness window): %s",
+ len(owner_dead_ids),
+ owner_dead_ids,
+ )
if interrupted_ids:
log.warning(
"Mission recovery: marked %d mission(s) FAILED('interrupted') "
diff --git a/tests/missions/test_recovery_owner.py b/tests/missions/test_recovery_owner.py
new file mode 100644
index 0000000000..b8ef08e803
--- /dev/null
+++ b/tests/missions/test_recovery_owner.py
@@ -0,0 +1,164 @@
+"""Owner-aware crash recovery (live forensic 2026-10-02, mission 01a0fcc3).
+
+An artifact mission's worker was killed when the desktop app restarted at
+15:19:40, 85 s after dispatch. The new instance's recovery sweep saw a heartbeat
+younger than 30 minutes and "presumed" a live instance still owned the mission,
+so it stayed RUNNING/CRITIQUING — with no worker behind it — until 15:56:42,
+when it was finally swept to FAILED('crash_recovery') with no artifact.
+
+The fix: the orchestrator stamps its process identity with each heartbeat, and
+recovery sweeps a mission at once when that owner process is provably dead,
+while an alive or unknown owner keeps the conservative freshness guard.
+"""
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest_asyncio
+
+from jarvis.core.process_utils import NO_WINDOW_CREATIONFLAGS
+from jarvis.missions.events import now_ms
+from jarvis.missions.manager import MissionManager
+from jarvis.missions.ownership import current_process_identity, owner_is_alive
+from jarvis.missions.recovery import startup_recover
+from jarvis.missions.state_machine import MissionState
+
+
+@pytest_asyncio.fixture
+async def open_store(tmp_missions_db: Path):
+ m = MissionManager(tmp_missions_db)
+ await m.start(recover=False)
+ try:
+ yield m
+ finally:
+ await m.stop()
+
+
+async def _critiquing_mission_owned_by(
+ m: MissionManager, owner_pid: int, owner_start_ms: int
+) -> str:
+ """A mission mid-run whose heartbeat is 5 s old, stamped by ``owner_pid``."""
+ mid = await m.dispatch(prompt="artifact build")
+ await m.transition_state(mid, MissionState.RUNNING, reason="kontrollierer-start")
+ await m.transition_state(mid, MissionState.CRITIQUING, reason="iter-0-start")
+ await m.store.touch_heartbeat(mid, now_ms() - 5_000)
+ await m.store.conn.execute(
+ "UPDATE missions SET owner_pid = ?, owner_start_ms = ? WHERE id = ?",
+ (owner_pid, owner_start_ms, mid),
+ )
+ return mid
+
+
+def _exited_pid() -> int:
+ """PID of a real process that has already exited."""
+ proc = subprocess.run(
+ [sys.executable, "-c", "import os; print(os.getpid())"],
+ capture_output=True,
+ encoding="utf-8",
+ timeout=30,
+ check=True,
+ creationflags=NO_WINDOW_CREATIONFLAGS,
+ )
+ return int(proc.stdout.strip())
+
+
+def _dead_owner(pid: int, start_ms: int) -> bool | None:
+ return False
+
+
+def _alive_owner(pid: int, start_ms: int) -> bool | None:
+ return True
+
+
+def _unknown_owner(pid: int, start_ms: int) -> bool | None:
+ return None
+
+
+async def test_dead_owner_is_swept_despite_fresh_heartbeat(
+ open_store: MissionManager,
+) -> None:
+ """The 01a0fcc3 case: fresh heartbeat, but the process that wrote it died
+ in a restart. Recovery must fail the mission now, not 30+ minutes later."""
+ mid = await _critiquing_mission_owned_by(open_store, 424242, 1_790_947_000_000)
+
+ recovered = await startup_recover(open_store.store, owner_alive_fn=_dead_owner)
+
+ assert recovered == [mid]
+ view = await open_store.mission(mid)
+ assert view is not None and view.state == MissionState.FAILED
+ events = await open_store.store.events_for_mission(mid)
+ failed = [e.payload for e in events if e.payload.event_type == "MissionFailed"]
+ assert len(failed) == 1
+ assert failed[0].reason == "crash_recovery"
+ assert failed[0].last_state == MissionState.CRITIQUING.value
+ assert failed[0].error_detail and "424242" in failed[0].error_detail
+
+
+async def test_alive_owner_keeps_the_freshness_guard(open_store: MissionManager) -> None:
+ """A second instance sharing missions.db must never sweep a mission a live
+ first instance is running (the 019e6fea false-FAILED bug stays fixed)."""
+ mid = await _critiquing_mission_owned_by(open_store, 424242, 1_790_947_000_000)
+
+ recovered = await startup_recover(open_store.store, owner_alive_fn=_alive_owner)
+
+ assert recovered == []
+ view = await open_store.mission(mid)
+ assert view is not None and view.state == MissionState.CRITIQUING
+
+
+async def test_unknown_owner_keeps_the_freshness_guard(open_store: MissionManager) -> None:
+ """No stamp, no psutil, access denied: never sweep on a guess."""
+ mid = await _critiquing_mission_owned_by(open_store, 0, 0)
+
+ recovered = await startup_recover(open_store.store, owner_alive_fn=_unknown_owner)
+
+ assert recovered == []
+ view = await open_store.mission(mid)
+ assert view is not None and view.state == MissionState.CRITIQUING
+
+
+async def test_heartbeat_stamps_this_process_as_owner(open_store: MissionManager) -> None:
+ mid = await open_store.dispatch(prompt="artifact build")
+
+ assert await open_store.store.get_owner(mid) == (0, 0)
+ await open_store.store.touch_heartbeat(mid, now_ms())
+
+ assert await open_store.store.get_owner(mid) == current_process_identity()
+ assert await open_store.store.get_owner("no-such-mission") == (0, 0)
+
+
+async def test_mission_heartbeat_by_this_live_process_is_not_swept(
+ open_store: MissionManager,
+) -> None:
+ """End to end with the real liveness probe: the periodic re-sweep runs in
+ the same process that owns the mission and must leave it alone."""
+ mid = await open_store.dispatch(prompt="artifact build")
+ await open_store.transition_state(mid, MissionState.RUNNING, reason="r")
+ await open_store.store.touch_heartbeat(mid, now_ms())
+
+ assert await startup_recover(open_store.store) == []
+
+
+async def test_mission_owned_by_an_exited_process_is_swept(
+ open_store: MissionManager,
+) -> None:
+ """End to end with the real liveness probe and a real process that has
+ exited — the restart that killed mission 01a0fcc3's owner."""
+ mid = await _critiquing_mission_owned_by(open_store, _exited_pid(), now_ms() - 60_000)
+
+ recovered = await startup_recover(open_store.store)
+
+ assert recovered == [mid]
+
+
+def test_owner_is_alive_probe() -> None:
+ pid, start_ms = current_process_identity()
+ assert pid == os.getpid()
+ assert owner_is_alive(pid, start_ms) is True
+ assert owner_is_alive(0, 0) is None
+ if start_ms:
+ # Same pid, different start time: the pid was reused by a new process.
+ assert owner_is_alive(pid, start_ms - 3_600_000) is False