Skip to content

The agent that grows with you

fix(kanban): release a dead attempt's claim when its owner receipt cannot be resolved

PRELVAS949dae8d87efe5
diff --git a/hermes_cli/kanban_db.py b/hermes_cli/kanban_db.py
index 0e2b4e51d1b..5db382dec89 100644
--- a/hermes_cli/kanban_db.py
+++ b/hermes_cli/kanban_db.py
@@ -2372,9 +2372,9 @@ def release_stale_claims(
"WHERE status = 'running' AND claim_expires IS NOT NULL "
" AND claim_expires < ?", (now,),
).fetchall()
- from hermes_cli.kanban_owner_recovery import owner_reclaim_paused
+ from hermes_cli.kanban_owner_recovery import owner_reclaim_blocked
for row in stale:
- if owner_reclaim_paused(conn, row["id"]):
+ if owner_reclaim_blocked(conn, row["id"]):
continue
host_local = (row["claim_lock"] or "").startswith(host_prefix)
hb = _to_epoch(row["last_heartbeat_at"])
diff --git a/hermes_cli/kanban_db_dispatch.py b/hermes_cli/kanban_db_dispatch.py
index 1e01999a2f1..674556cbcbd 100644
--- a/hermes_cli/kanban_db_dispatch.py
+++ b/hermes_cli/kanban_db_dispatch.py
@@ -668,9 +668,9 @@ def enforce_max_runtime(conn: sqlite3.Connection, *, signal_fn=None) -> list[str
" AND COALESCE(r.started_at, t.started_at) IS NOT NULL "
" AND t.worker_pid IS NOT NULL"
).fetchall()
- from hermes_cli.kanban_owner_recovery import owner_reclaim_paused
+ from hermes_cli.kanban_owner_recovery import owner_reclaim_blocked
for row in rows:
- if owner_reclaim_paused(conn, row["id"]):
+ if owner_reclaim_blocked(conn, row["id"]):
continue
lock = row["claim_lock"] or ""
if not lock.startswith(host_prefix):
@@ -779,9 +779,9 @@ def detect_stale_running(
"WHERE t.status = 'running'"
).fetchall()
- from hermes_cli.kanban_owner_recovery import owner_reclaim_paused
+ from hermes_cli.kanban_owner_recovery import owner_reclaim_blocked
for row in rows:
- if owner_reclaim_paused(conn, row["id"]):
+ if owner_reclaim_blocked(conn, row["id"]):
continue
if row["active_started_at"] is None:
continue
@@ -866,9 +866,9 @@ def reconcile_orphaned_running(conn: sqlite3.Connection) -> list[str]:
"WHERE status = 'running' "
" AND (claim_lock IS NULL OR claim_expires IS NULL)"
).fetchall()
- from hermes_cli.kanban_owner_recovery import owner_reclaim_paused
+ from hermes_cli.kanban_owner_recovery import owner_reclaim_blocked
for row in rows:
- if owner_reclaim_paused(conn, row["id"]):
+ if owner_reclaim_blocked(conn, row["id"]):
continue
tid = row["id"]
pid = row["worker_pid"]
@@ -1168,7 +1168,7 @@ def _reclaim_dead_workers(conn: sqlite3.Connection, board: Optional[str] = None)
"WHERE status = 'running' AND worker_pid IS NOT NULL"
).fetchall()
host_prefix = _kb._host_prefix()
- from hermes_cli.kanban_owner_recovery import bound_interpreter_gone, managed_run_exit_code, owner_reclaim_paused
+ from hermes_cli.kanban_owner_recovery import managed_run_exit_code, owner_reclaim_blocked
for row in rows:
lock = row["claim_lock"] or ""
if not lock.startswith(host_prefix):
@@ -1180,9 +1180,9 @@ def _reclaim_dead_workers(conn: sqlite3.Connection, board: Optional[str] = None)
continue
if _worker_alive(row["worker_pid"], _kb._row_get(row, "worker_started_at")):
continue
- # An ambiguous owner receipt holds the card while the attempt may still run; once the
- # interpreter the owner bound to this run is dead nothing can, and it is a crash.
- if owner_reclaim_paused(conn, row["id"]) and not bound_interpreter_gone(conn, row["id"]):
+ # An ambiguous receipt holds the card while the attempt may still run; once nothing
+ # can be executing it (a dead bound interpreter, or a run that never bound one): crash.
+ if owner_reclaim_blocked(conn, row["id"]):
continue
pid = int(row["worker_pid"])
diff --git a/hermes_cli/kanban_diagnostics.py b/hermes_cli/kanban_diagnostics.py
index b1941f7dd72..87a300ab902 100644
--- a/hermes_cli/kanban_diagnostics.py
+++ b/hermes_cli/kanban_diagnostics.py
@@ -736,6 +736,72 @@ def _rule_stranded_in_ready(task, events, runs, now, cfg) -> list[Diagnostic]:
)]
+def _rule_stale_running(task, events, runs, now, cfg) -> list[Diagnostic]:
+ """A ``running`` card whose claim expired >= cfg["stale_running_grace_seconds"]
+ (default 1h) ago with no live recorded worker.
+
+ Nothing in the dispatcher is going to free this card — its reclaim paths can decline a
+ card whose owner receipt never resolved, and until they could not, the failure was
+ invisible: an operator saw only the cards queued behind the profile's worker cap. So the
+ jam itself is the signal. Silent in every shape that can still progress: a live claim
+ (the owner is still working), a heartbeat inside the dispatcher's max-stale window, a
+ foreign host's lock (its pids say nothing here), or a live recorded worker. Read-only.
+ """
+ grace_seconds = float(cfg.get("stale_running_grace_seconds", 60 * 60))
+ if _task_field(task, "status") != "running":
+ return []
+ from hermes_cli import kanban_db as kb
+ expires = kb._to_epoch(_task_field(task, "claim_expires", None))
+ if expires is None or (now - expires) < grace_seconds:
+ return []
+ lock = str(_task_field(task, "claim_lock", "") or "")
+ if not lock.startswith(kb._host_prefix()):
+ return []
+ hb = kb._to_epoch(_task_field(task, "last_heartbeat_at", None))
+ if hb is not None and (now - hb) <= kb.DEFAULT_CLAIM_HEARTBEAT_MAX_STALE_SECONDS:
+ return []
+
+ from hermes_cli.kanban_db_dispatch import _worker_alive
+ recorded = [(_task_field(task, "worker_pid", None),
+ _task_field(task, "worker_started_at", None))]
+ for ev in events:
+ if _event_kind(ev) in {"spawned", "worker_bound"}:
+ payload = _parse_payload(ev)
+ recorded.append((payload.get("pid"), payload.get("started_at")))
+ pids = sorted({int(pid) for pid, _ in recorded if pid})
+ if any(_worker_alive(int(pid), started_at) for pid, started_at in recorded if pid):
+ return []
+
+ expired_seconds = now - expires
+ age_str = (f"{expired_seconds / 3600:.1f}h" if expired_seconds >= 3600
+ else f"{int(expired_seconds / 60)}m")
+ # Escalate with age: <2x grace warning, 2x-6x error, >6x critical.
+ if expired_seconds >= grace_seconds * 6:
+ severity = "critical"
+ elif expired_seconds >= grace_seconds * 2:
+ severity = "error"
+ else:
+ severity = "warning"
+
+ task_id = str(_task_field(task, "id") or "")
+ return [Diagnostic(
+ kind="stale_running", severity=severity,
+ title=f"Running {age_str} past its expired claim with no live worker",
+ detail=f"Nothing is executing this card and nothing has reclaimed it: the claim expired "
+ f"{age_str} ago, no heartbeat arrived inside the last hour, and no pid recorded for "
+ f"the run ({', '.join(str(p) for p in pids) or 'none'}) is alive. The dispatcher "
+ f"declines a reclaim when the run's owner receipt never resolved, so the card can "
+ f"hold its profile's worker slot indefinitely while every card queued behind it "
+ f"stays deferred. Check the card's log, then reclaim it.",
+ actions=[DiagnosticAction(kind="reclaim", label="Reclaim the dead claim", suggested=True),
+ _cli_hint("Reclaim from the CLI", f"hermes kanban reclaim {task_id}")],
+ first_seen_at=expires, last_seen_at=expires, count=1,
+ data={"claim_expires": expires, "expired_seconds": int(expired_seconds),
+ "recorded_pids": pids, "last_heartbeat_at": hb,
+ "grace_seconds": int(grace_seconds)},
+ )]
+
+
# Order matters: earlier rules render first on severity ties.
_RULES: list[RuleFn] = [
_rule_hallucinated_cards,
@@ -748,6 +814,7 @@ _RULES: list[RuleFn] = [
_rule_stuck_in_blocked,
_rule_block_unblock_cycling,
_rule_stranded_in_ready,
+ _rule_stale_running,
]
@@ -762,6 +829,10 @@ DEFAULT_CONFIG = {
# Below 30 min the signal is dominated by tasks about to be claimed on
# the next dispatcher tick.
"stranded_threshold_seconds": 30 * 60,
+ # The dispatcher reclaims an expired claim on its own next tick, so a card
+ # still `running` an hour past its expiry means every reclaim path declined
+ # it. Same hour the heartbeat backstop uses.
+ "stale_running_grace_seconds": 60 * 60,
}
diff --git a/hermes_cli/kanban_owner_recovery.py b/hermes_cli/kanban_owner_recovery.py
index 36703081880..112f472681d 100644
--- a/hermes_cli/kanban_owner_recovery.py
+++ b/hermes_cli/kanban_owner_recovery.py
@@ -3,6 +3,7 @@ from contextlib import closing
import json
from pathlib import Path
import sqlite3
+import time
import psutil
@@ -32,20 +33,137 @@ def owner_reclaim_paused(conn, task_id):
return True
+def _latest_run_event(conn, task_id, kind):
+ """The newest ``kind`` payload on the task's CURRENT run, or ``{}``.
-def bound_interpreter_gone(conn, task_id):
- """True when the owner bound an interpreter to the task's current run and it is dead."""
- from hermes_cli.kanban_db_dispatch import _worker_alive
+ A payload that cannot be read is not a payload that says the attempt is over, so
+ callers treat ``{}`` as "no evidence" rather than as an absence.
+ """
row = conn.execute(
"SELECT e.payload FROM task_events e JOIN tasks t ON t.current_run_id=e.run_id "
- "WHERE t.id=? AND e.task_id=t.id AND e.kind='worker_bound' ORDER BY e.id DESC LIMIT 1",
- (task_id,)).fetchone()
- if row is None:
+ "WHERE t.id=? AND e.task_id=t.id AND e.kind=? ORDER BY e.id DESC LIMIT 1",
+ (task_id, kind)).fetchone()
+ try:
+ payload = json.loads(row['payload']) if row else {}
+ except (ValueError, TypeError):
+ return {}
+ return payload if isinstance(payload, dict) else {}
+
+
+def _claim_lock(conn, task_id):
+ row = conn.execute("SELECT claim_lock FROM tasks WHERE id=?", (task_id,)).fetchone()
+ return row['claim_lock'] if row is not None else None
+
+
+def _host_local_lock(lock) -> bool:
+ """True when ``lock`` was issued by this host — the only pids we can judge."""
+ from hermes_cli import kanban_db as kb
+ return bool(lock) and str(lock).startswith(kb._host_prefix())
+
+
+def bound_interpreter_gone(conn, task_id):
+ """True when the owner bound an interpreter to the task's current run and it is dead.
+
+ Host-local only: ``worker_bound`` records a pid of THIS board's host, so a numeric
+ coincidence on another host must never read as a death and release a live claim.
+ """
+ if not _host_local_lock(_claim_lock(conn, task_id)):
+ return False
+ bound = _latest_run_event(conn, task_id, 'worker_bound')
+ if not bound:
return False
- bound = json.loads(row['payload'])
+ from hermes_cli.kanban_db_dispatch import _worker_alive
return not _worker_alive(bound.get('pid'), bound.get('started_at'))
+def _owner_process_gone(receipt) -> bool:
+ """True when the receipt's own pid is gone, or its number now belongs to another process.
+
+ The identity rule ``owner_reclaim_paused`` already applies: a live pid whose ``birth``
+ differs is a recycled number, not the owner. A live pid we cannot compare (no ``birth``
+ recorded) and an unreadable one are never proof of death.
+ """
+ try:
+ pid = int(receipt['pid'])
+ except (KeyError, TypeError, ValueError):
+ return False
+ try:
+ from hermes_cli.process_identity import process_birth
+ current = process_birth(psutil.Process(pid))
+ except psutil.NoSuchProcess:
+ return True
+ except psutil.Error:
+ return False
+ birth = receipt.get('birth')
+ return birth is not None and current != birth
+
+
+def _run_worker_pids(conn, task_row, task_id):
+ """Every worker pid recorded for the task's current run, with its spawn fingerprint."""
+ recorded = [(task_row['worker_pid'], task_row['worker_started_at'])]
+ for kind in ('spawned', 'worker_bound'):
+ payload = _latest_run_event(conn, task_id, kind)
+ recorded.append((payload.get('pid'), payload.get('started_at')))
+ return [(int(pid), fingerprint) for pid, fingerprint in recorded if pid]
+
+
+def owner_attempt_provably_over(conn, task_id) -> bool:
+ """True when an unresolved owner receipt cannot still be executing this attempt.
+
+ ``owner_reclaim_paused`` is about effects: an admission that may still be in flight is
+ never repeated. It must not also mean the run keeps its claim forever. The attempts it
+ pauses in practice are ones whose owner was killed (restart, OOM), and such a run may
+ never have written a ``worker_bound`` event at all, so ``bound_interpreter_gone`` cannot
+ speak for it — 76 ``owner_admitted`` against 74 ``worker_bound`` rows on one board means
+ runs in exactly that shape. This asks the question that matters — can anything still be
+ executing this attempt? — from evidence the board already keeps:
+
+ * the claim is host-local and EXPIRED (a live claim is the owner still working);
+ * no heartbeat inside ``DEFAULT_CLAIM_HEARTBEAT_MAX_STALE_SECONDS``;
+ * the receipt's own process is gone or recycled;
+ * every worker pid recorded for the run (``tasks.worker_pid``, its ``spawned`` and
+ ``worker_bound`` events) is dead or recycled.
+
+ Missing evidence is never proof: no recorded worker pid, a live pid whose identity
+ cannot be compared, or an unreadable clock all return False and the pause holds.
+ """
+ from hermes_cli import kanban_db as kb
+ from hermes_cli.kanban_db_dispatch import _worker_alive
+ row = conn.execute(
+ "SELECT claim_lock, claim_expires, worker_pid, worker_started_at, last_heartbeat_at "
+ "FROM tasks WHERE id=?", (task_id,)).fetchone()
+ if row is None or not _host_local_lock(row['claim_lock']):
+ return False
+ now = int(time.time())
+ expires = kb._to_epoch(row['claim_expires'])
+ if expires is None or expires >= now:
+ return False
+ heartbeat = kb._to_epoch(row['last_heartbeat_at'])
+ if heartbeat is not None and now - heartbeat <= kb.DEFAULT_CLAIM_HEARTBEAT_MAX_STALE_SECONDS:
+ return False
+ if not _owner_process_gone(_latest_run_event(conn, task_id, 'owner_admitted')):
+ return False
+ pids = _run_worker_pids(conn, row, task_id)
+ if not pids:
+ return False
+ return not any(_worker_alive(pid, fingerprint) for pid, fingerprint in pids)
+
+
+def owner_reclaim_blocked(conn, task_id) -> bool:
+ """True when a reclaim path must leave this task alone.
+
+ The pause protects the effects of an attempt that may still be in flight, and stops
+ holding the claim as soon as nothing can still be executing it: either the owner bound an
+ interpreter that is gone (``bound_interpreter_gone``), or the receipt is unresolved while
+ every piece of the run's liveness evidence is dead (``owner_attempt_provably_over``) —
+ the shape that used to be unreclaimable forever, because its run wrote no ``worker_bound``
+ event and that override was the only one anyone consulted.
+ """
+ if not owner_reclaim_paused(conn, task_id):
+ return False
+ return not (bound_interpreter_gone(conn, task_id) or owner_attempt_provably_over(conn, task_id))
+
+
def managed_run_exit_code(conn, task_id, worker_pid, claim_lock):
"""The exit code a managed run's authority recorded for the task's CURRENT run, else None.
diff --git a/tests/hermes_cli/test_kanban_owner_receipt_reclaim.py b/tests/hermes_cli/test_kanban_owner_receipt_reclaim.py
new file mode 100644
index 00000000000..10c403c93ef
--- /dev/null
+++ b/tests/hermes_cli/test_kanban_owner_receipt_reclaim.py
@@ -0,0 +1,348 @@
+"""A dead attempt behind an unresolved owner receipt must not hold its claim forever.
+
+Regression for #140858. Card t_5ea03658 sat ``running`` for 16 hours: claimed 07:48, worker
+killed by a restart at 08:59, claim expired 08:03, no heartbeat, and no ``worker_bound``
+event. ``owner_reclaim_paused`` was true — a killed owner leaves a ``terminal/interrupted``
+admission, or a pid that is gone — and every reclaim path read that as "leave it alone", so
+``release_stale_claims``, ``detect_crashed_workers`` and ``reconcile_orphaned_running`` all
+declined. ``bound_interpreter_gone``, the one override, needs the ``worker_bound`` event that
+run never wrote. The card held its profile's ``max_in_progress_per_profile`` slot while every
+card queued behind it was deferred, and no diagnostics rule could see a ``running`` card at
+all, so the alarms named the victims and never the cause.
+
+The pause is about effects that may still be in flight. These tests pin both sides of that
+line: a live worker, a fresh heartbeat, a live claim, a foreign host's lock or an identity
+that cannot be read all keep the card; a provably dead attempt is reclaimed by the platform's
+own path.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sqlite3
+import time
+from pathlib import Path
+
+import psutil
+import pytest
+
+from hermes_cli import kanban_db as kb
+from hermes_cli import kanban_db_connect as kbc
+from hermes_cli import kanban_db_dispatch as kbd
+from hermes_cli import kanban_diagnostics as kd
+from hermes_cli.kanban_owner_recovery import (
+ bound_interpreter_gone,
+ owner_attempt_provably_over,
+ owner_reclaim_blocked,
+ owner_reclaim_paused,
+)
+
+NOW = int(time.time())
+HOST = kb._claimer_id().split(":", 1)[0]
+FINGERPRINT = "6a232414-2dbf-40ab-a534-d9b71db05658:116|4162240"
+
+
+def _absent_pid(base: int = 3_500_000) -> int:
+ """A pid that provably does not exist on this host, so a fixture can never hit a process."""
+ pid = base
+ while psutil.pid_exists(pid):
+ pid += 1
+ return pid
+
+
+class _Board:
+ """A scratch board plus the scratch admission ledger an owner receipt points at."""
+
+ def __init__(self, root: Path):
+ self.path = root / "kanban.db"
+ kbc.init_db(self.path)
+ self.conn = kbc.connect(self.path)
+ self.ledger = root / "state.db"
+ with sqlite3.connect(self.ledger) as store:
+ store.execute(
+ "CREATE TABLE session_admissions "
+ "(target_session_id TEXT, request_id TEXT, status TEXT, outcome TEXT)"
+ )
+
+ def close(self) -> None:
+ self.conn.close()
+
+ # --- seeding --------------------------------------------------------------------------
+ def dead_attempt(
+ self, tid: str = "t_ghost", *, worker_pid=None, worker_fingerprint=FINGERPRINT,
+ worker_bound: bool = False, record_worker_pid: bool = True, owner_pid=None,
+ owner_birth=None, admission=("terminal", "interrupted"), dead_hours: int = 15,
+ expires_age_seconds=None, lock_host=None, heartbeat_age=None,
+ ) -> int:
+ """A ``running`` card whose worker died the way t_5ea03658's did."""
+ worker_pid = _absent_pid() if worker_pid is None else worker_pid
+ owner_pid = _absent_pid(base=3_600_000) if owner_pid is None else owner_pid
+ lock = f"{lock_host or HOST}:{owner_pid}"
+ expires = NOW - (expires_age_seconds if expires_age_seconds is not None
+ else dead_hours * 3600)
+ started = NOW - dead_hours * 3600
+ run_id = self.conn.execute(
+ "INSERT INTO task_runs (task_id, profile, status, claim_lock, claim_expires, "
+ "worker_pid, started_at) VALUES (?, 'default', 'running', ?, ?, ?, ?)",
+ (tid, lock, expires, worker_pid if record_worker_pid else None, started),
+ ).lastrowid
+ assert run_id is not None
+ run = int(run_id)
+ self.conn.execute(
+ "INSERT INTO tasks (id, title, assignee, status, created_at, claim_lock, claim_expires, "
+ "worker_pid, worker_started_at, last_heartbeat_at, current_run_id, started_at, priority) "
+ "VALUES (?, ?, 'default', 'running', ?, ?, ?, ?, ?, ?, ?, ?, 5)",
+ (tid, tid, NOW - 86400, lock, expires,
+ worker_pid if record_worker_pid else None,
+ worker_fingerprint if record_worker_pid else None,
+ (NOW - heartbeat_age) if heartbeat_age is not None else None, run, started),
+ )
+ self.event(tid, run, "claimed", {"source_status": "ready"}, started)
+ if record_worker_pid:
+ spawned = {"pid": worker_pid, "started_at": worker_fingerprint}
+ else:
+ spawned = {}
+ self.event(tid, run, "spawned", spawned, started)
+ payload = {"db": str(self.ledger), "session_id": "sess-1", "pid": owner_pid,
+ "request_id": f'kanban:["default","{tid}",1]'}
+ if owner_birth is not None:
+ payload["birth"] = owner_birth
+ self.event(tid, run, "owner_admitted", payload, started + 30)
+ if worker_bound:
+ self.event(tid, run, "worker_bound",
+ {"pid": worker_pid, "claim_lock": lock, "started_at": worker_fingerprint},
+ started + 60)
+ if admission:
+ with sqlite3.connect(self.ledger) as store:
+ store.execute("INSERT INTO session_admissions VALUES ('sess-1', ?, ?, ?)",
+ (payload["request_id"], admission[0], admission[1]))
+ self.conn.commit()
+ return run
+
+ def event(self, tid: str, run: int, kind: str, payload: dict, ts: int) -> None:
+ self.conn.execute(
+ "INSERT INTO task_events (task_id, run_id, kind, payload, created_at) "
+ "VALUES (?, ?, ?, ?, ?)", (tid, run, kind, json.dumps(payload), ts))
+
+ def ready(self, tid: str) -> None:
+ self.conn.execute(
+ "INSERT INTO tasks (id, title, assignee, status, created_at, priority) "
+ "VALUES (?, ?, 'default', 'ready', ?, 5)", (tid, tid, NOW - 86400))
+ self.conn.commit()
+
+ # --- reading --------------------------------------------------------------------------
+ def state(self, tid: str) -> dict:
+ return dict(self.conn.execute("SELECT * FROM tasks WHERE id = ?", (tid,)).fetchone())
+
+ def events(self, tid: str) -> list:
+ return self.conn.execute(
+ "SELECT kind, payload, created_at FROM task_events WHERE task_id = ? ORDER BY id",
+ (tid,)).fetchall()
+
+ def diagnostics(self, tid: str, *, config: dict | None = None) -> list:
+ return kd.compute_task_diagnostics(
+ self.state(tid), self.events(tid), [], now=NOW, config=config or {})
+
+
+def board(tmp_path, monkeypatch):
+ monkeypatch.setenv("HERMES_KANBAN_CRASH_GRACE_SECONDS", "0")
+ scratch = _Board(tmp_path)
+ yield scratch
+ scratch.close()
+
+
+# --- the incident shape is reclaimed by the platform's own paths ---------------------------
+
+def test_dead_attempt_behind_an_unresolved_receipt_is_reclaimed(board):
+ """Every reclaim path declined this card; the stale-claim path must now take it."""
+ board.dead_attempt("t_ghost")
+ assert owner_reclaim_paused(board.conn, "t_ghost") is True # the pause still applies
+ assert bound_interpreter_gone(board.conn, "t_ghost") is False # nothing bound to read
+ assert owner_attempt_provably_over(board.conn, "t_ghost") is True
+ assert owner_reclaim_blocked(board.conn, "t_ghost") is False
+
+ assert kb.release_stale_claims(board.conn) == 1
+
+ row = board.state("t_ghost")
+ assert row["status"] == "ready"
+ assert (row["claim_lock"], row["claim_expires"], row["worker_pid"]) == (None, None, None)
+ assert row["current_run_id"] is None
+ run = board.conn.execute(
+ "SELECT status, outcome FROM task_runs WHERE task_id = 't_ghost'").fetchone()
+ assert tuple(run) == ("reclaimed", "reclaimed")
+
+
+def test_dead_attempt_behind_an_unresolved_receipt_is_reclaimed_by_the_crash_sweep(board):
+ """The dead-pid path reclaims the same shape instead of deferring to the receipt."""
+ board.dead_attempt("t_ghost")
+ assert kbd.detect_crashed_workers(board.conn) == ["t_ghost"]
+ assert board.state("t_ghost")["status"] == "ready"
+
+
+def test_reclaiming_the_dead_attempt_frees_its_profile_lane(board):
+ """The point of the fix: the card stops holding its profile's worker slot."""
+ board.dead_attempt("t_ghost")
+ board.ready("t_ready0")
+ counted = "SELECT COUNT(*) FROM tasks WHERE status = 'running' AND assignee = 'default'"
+ assert board.conn.execute(counted).fetchone()[0] == 1 # the jam: a dead card holds the slot
+
+ kbd.dispatch_once(board.conn, dry_run=True, max_in_progress=2,
+ max_in_progress_per_profile=1)
+ assert board.state("t_ghost")["status"] == "ready"
+ assert board.conn.execute(counted).fetchone()[0] == 0 # the slot is free again
+
+ # And the next tick spends it instead of deferring the queue forever.
+ lane = kbd.dispatch_once(board.conn, dry_run=True, max_in_progress=2,
+ max_in_progress_per_profile=1)
+ assert len(lane.spawned) == 1
+
+
+# --- the pause still holds every attempt that may be alive ---------------------------------
+
+def test_a_live_worker_holds_the_card_even_with_the_pause(board):
+ """Real pid, real fingerprint: reclaiming beside it would spawn a duplicate."""
+ board.dead_attempt("t_ghost", worker_pid=os.getpid(),
+ worker_fingerprint=kbd._process_fingerprint(os.getpid()))
+ assert owner_reclaim_paused(board.conn, "t_ghost") is True
+ assert owner_attempt_provably_over(board.conn, "t_ghost") is False
+ assert owner_reclaim_blocked(board.conn, "t_ghost") is True
+ assert kb.release_stale_claims(board.conn) == 0
+ assert board.state("t_ghost")["status"] == "running"
+ assert board.state("t_ghost")["worker_pid"] == os.getpid()
+
+
+def test_a_live_bound_interpreter_holds_the_card(board):
+ """A ``worker_bound`` interpreter that is still running is the same refusal."""
+ board.dead_attempt("t_ghost", worker_bound=True, worker_pid=os.getpid(),
+ worker_fingerprint=kbd._process_fingerprint(os.getpid()))
+ assert bound_interpreter_gone(board.conn, "t_ghost") is False
+ assert owner_reclaim_blocked(board.conn, "t_ghost") is True
+ assert kb.release_stale_claims(board.conn) == 0
+ assert board.state("t_ghost")["status"] == "running"
+
+
+def test_a_fresh_heartbeat_holds_the_card(board):
+ """A slow worker inside one tool-free call keeps its claim (#23025)."""
+ board.dead_attempt("t_ghost", heartbeat_age=300)
+ assert owner_attempt_provably_over(board.conn, "t_ghost") is False
+ assert kb.release_stale_claims(board.conn) == 0
+ assert board.state("t_ghost")["status"] == "running"
+
+
+def test_a_live_claim_is_never_overridden(board):
+ """An unexpired claim is the owner still working, whatever the receipt says."""
+ board.dead_attempt("t_ghost", expires_age_seconds=-3600)
+ assert owner_attempt_provably_over(board.conn, "t_ghost") is False
+ assert kb.release_stale_claims(board.conn) == 0
+ assert board.state("t_ghost")["status"] == "running"
+
+
+def test_another_hosts_lock_is_never_overridden(board):
+ """A foreign lock's pids say nothing here, and a numeric coincidence is not a death."""
+ board.dead_attempt("t_ghost", lock_host="some-other-host")
+ assert bound_interpreter_gone(board.conn, "t_ghost") is False
+ assert owner_attempt_provably_over(board.conn, "t_ghost") is False
+ assert kb.release_stale_claims(board.conn) == 0
+ assert board.state("t_ghost")["status"] == "running"
+
+
+def test_a_live_owner_without_a_birth_is_never_read_as_dead(board):
+ """The receipt names a live pid but no identity to compare: unprovable, so it holds."""
+ board.dead_attempt("t_ghost", owner_pid=os.getpid(), owner_birth=None)
+ assert owner_reclaim_paused(board.conn, "t_ghost") is True
+ assert owner_attempt_provably_over(board.conn, "t_ghost") is False
+ assert kb.release_stale_claims(board.conn) == 0
+ assert board.state("t_ghost")["status"] == "running"
+
+
+def test_a_recycled_owner_pid_is_dead_evidence(board):
+ """A live number whose birth moved on is a stranger, exactly as the pause reads it."""
+ board.dead_attempt("t_ghost", owner_pid=os.getpid(), owner_birth=1.0)
+ assert owner_reclaim_paused(board.conn, "t_ghost") is True
+ assert owner_attempt_provably_over(board.conn, "t_ghost") is True
+ assert kb.release_stale_claims(board.conn) == 1
+
+
+def test_owner_identity_reads_birth_through_the_shared_helper(board, monkeypatch):
+ """A live owner's birth must be read the way the pause reads it, or a clock step looks like death.
+
+ ``create_time()`` shifts on macOS after a boot-time change while the kernel's own start does
+ not (#140580), so comparing the fence's reading against the receipt's would call a live owner
+ recycled and release a claim that is still being worked. The fence defers to
+ ``process_identity.process_birth``; this pins that it does, whatever the platform.
+ """
+ from hermes_cli import process_identity
+ seen: list[int] = []
+
+ def fake_birth(proc):
+ seen.append(proc.pid)
+ return 1234.0
+
+ monkeypatch.setattr(process_identity, "process_birth", fake_birth)
+ board.dead_attempt("t_ghost", owner_pid=os.getpid(), owner_birth=1234.0)
+ assert owner_reclaim_paused(board.conn, "t_ghost") is True
+ assert owner_attempt_provably_over(board.conn, "t_ghost") is False
+ assert seen == [os.getpid()]
+ assert kb.release_stale_claims(board.conn) == 0
+ assert board.state("t_ghost")["status"] == "running"
+
+
+def test_no_recorded_worker_pid_is_never_proof(board):
+ """A run that recorded no worker pid cannot be shown to be dead — it holds."""
+ board.dead_attempt("t_ghost", record_worker_pid=False)
+ assert owner_attempt_provably_over(board.conn, "t_ghost") is False
+ assert kb.release_stale_claims(board.conn) == 0
+ assert board.state("t_ghost")["status"] == "running"
+
+
+# --- the jam itself becomes visible to `kanban diagnostics` --------------------------------
+
+def test_stale_running_diagnostic_names_the_jam(board):
+ board.dead_attempt("t_ghost")
+ hits = [d for d in board.diagnostics("t_ghost") if d.kind == "stale_running"]
+ assert len(hits) == 1
+ hit = hits[0]
+ assert hit.severity == "critical" # 15x the 1h grace
+ assert hit.data["expired_seconds"] >= 3600
+ assert hit.data["recorded_pids"] and str(hit.data["recorded_pids"][0]) in hit.detail
+ assert [a.kind for a in hit.actions] == ["reclaim", "cli_hint"]
+ assert hit.actions[0].suggested is True
+
+
[email protected]("expired_for,severity", [
+ (30 * 60, ""), # inside the grace: the dispatcher has not had its tick yet
+ (3 * 3600, "error"), # 3x grace
+ (7 * 3600, "critical"), # 7x grace
+])
+def test_stale_running_diagnostic_escalates_with_age(board, expired_for, severity):
+ board.dead_attempt("t_ghost", expires_age_seconds=expired_for)
+ hits = [d for d in board.diagnostics("t_ghost") if d.kind == "stale_running"]
+ assert [d.severity for d in hits] == ([severity] if severity else [])
+
+
+def test_stale_running_diagnostic_is_silent_for_a_live_worker(board):
+ board.dead_attempt("t_ghost", worker_pid=os.getpid(),
+ worker_fingerprint=kbd._process_fingerprint(os.getpid()))
+ assert not [d for d in board.diagnostics("t_ghost") if d.kind == "stale_running"]
+
+
+def test_stale_running_diagnostic_is_silent_for_a_fresh_heartbeat(board):
+ board.dead_attempt("t_ghost", heartbeat_age=300)
+ assert not [d for d in board.diagnostics("t_ghost") if d.kind == "stale_running"]
+
+
+def test_stale_running_diagnostic_is_silent_for_a_live_claim(board):
+ board.dead_attempt("t_ghost", expires_age_seconds=-60)
+ assert not [d for d in board.diagnostics("t_ghost") if d.kind == "stale_running"]
+
+
+def test_stale_running_diagnostic_is_silent_for_another_hosts_lock(board):
+ board.dead_attempt("t_ghost", lock_host="some-other-host")
+ assert not [d for d in board.diagnostics("t_ghost") if d.kind == "stale_running"]
+
+
+def test_stale_running_diagnostic_is_silent_for_a_ready_card(board):
+ board.ready("t_ready0")
+ assert not [d for d in board.diagnostics("t_ready0") if d.kind == "stale_running"]
diff --git a/website/docs/user-guide/features/kanban-worker-lanes.md b/website/docs/user-guide/features/kanban-worker-lanes.md
index 6507462f9e2..65606de0c8b 100644
--- a/website/docs/user-guide/features/kanban-worker-lanes.md
+++ b/website/docs/user-guide/features/kanban-worker-lanes.md
@@ -133,6 +133,7 @@ So lane authors don't have to reimplement these:
- **Run-level retry** — when a task is retried (post-block, post-crash, post-reclaim), the worker can use the `expected_run_id` parameter on terminating tools to fail fast if its own run was already superseded.
- **Per-task max runtime** — `task.max_runtime_seconds` hard-caps wall-clock time per run, regardless of PID liveness. Catches genuinely-deadlocked workers that the live-PID extension would otherwise keep running.
- **Stranded-task detection** — a ready task whose assignee never produces a claim within `kanban.stranded_threshold_seconds` (default 30 min) shows up in `hermes kanban diagnostics` as a `stranded_in_ready` warning. Severity escalates to error at 2x the threshold and critical at 6x. Catches typo'd assignees, deleted profiles, and down external worker pools in one signal — identity-agnostic, no per-board allowlist to curate.
+- **Dead attempt behind an unresolved owner receipt** — a worker killed by a restart can leave an `owner_admitted` receipt the dispatcher cannot resolve, and reclaims used to be refused on that ground alone, forever: the card stayed `running`, held its profile's `kanban.max_in_progress_per_profile` slot, and the alarms named the cards queued behind it rather than the cause. The refusal now lapses as soon as nothing can be executing the attempt — the owner's process is gone or recycled, no heartbeat inside `DEFAULT_CLAIM_HEARTBEAT_MAX_STALE_SECONDS`, and every pid the run recorded (`worker_pid`, `spawned`, `worker_bound`) is dead — including runs that never wrote a `worker_bound` event. Anything unprovable (a live pid, a missing fingerprint, no recorded pid, another host's lock) still holds the claim. A card that survives even that shows up in `hermes kanban diagnostics` as `stale_running`, escalated by age past `kanban.diagnostics.stale_running_grace_seconds` (default 1h).
- **Running with open parents** — a `running` card whose direct parent is not `done`/`archived` (the parent reopened mid-run, or the link predates the running-child refusal) shows a `running_with_open_parents` warning: the dependency gate is not serialising the two runs and `kanban_complete` will be refused until the parent finishes. Read-only; suggests `hermes kanban unlink`.
- **Legacy review dependency deadlock** — a parent sticky-blocked with `review-required:` while one or more direct children remain dependency-gated in `todo` produces an immediate `review_dependency_deadlock` error. The diagnostic is read-only: it suggests completing the finished phase or unlinking the incorrect edge but never removes a user block automatically.