The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(kanban): release a dead attempt's claim when its owner receipt cannot be resolved
PRELVAS949dae8d87efe5
diff --git a/hermes_cli/kanban_db.py b/hermes_cli/kanban_db.pyindex 0e2b4e51d1b..5db382dec89 100644--- a/hermes_cli/kanban_db.py+++ b/hermes_cli/kanban_db.py@@ -2372,9 +2372,9 @@ def release_stale_claims("WHERE status = 'running' AND claim_expires IS NOT NULL "" AND claim_expires < ?", (now,),).fetchall()- from hermes_cli.kanban_owner_recovery import owner_reclaim_paused+ from hermes_cli.kanban_owner_recovery import owner_reclaim_blockedfor row in stale:- if owner_reclaim_paused(conn, row["id"]):+ if owner_reclaim_blocked(conn, row["id"]):continuehost_local = (row["claim_lock"] or "").startswith(host_prefix)hb = _to_epoch(row["last_heartbeat_at"])diff --git a/hermes_cli/kanban_db_dispatch.py b/hermes_cli/kanban_db_dispatch.pyindex 1e01999a2f1..674556cbcbd 100644--- a/hermes_cli/kanban_db_dispatch.py+++ b/hermes_cli/kanban_db_dispatch.py@@ -668,9 +668,9 @@ def enforce_max_runtime(conn: sqlite3.Connection, *, signal_fn=None) -> list[str" AND COALESCE(r.started_at, t.started_at) IS NOT NULL "" AND t.worker_pid IS NOT NULL").fetchall()- from hermes_cli.kanban_owner_recovery import owner_reclaim_paused+ from hermes_cli.kanban_owner_recovery import owner_reclaim_blockedfor row in rows:- if owner_reclaim_paused(conn, row["id"]):+ if owner_reclaim_blocked(conn, row["id"]):continuelock = row["claim_lock"] or ""if not lock.startswith(host_prefix):@@ -779,9 +779,9 @@ def detect_stale_running("WHERE t.status = 'running'").fetchall()- from hermes_cli.kanban_owner_recovery import owner_reclaim_paused+ from hermes_cli.kanban_owner_recovery import owner_reclaim_blockedfor row in rows:- if owner_reclaim_paused(conn, row["id"]):+ if owner_reclaim_blocked(conn, row["id"]):continueif row["active_started_at"] is None:continue@@ -866,9 +866,9 @@ def reconcile_orphaned_running(conn: sqlite3.Connection) -> list[str]:"WHERE status = 'running' "" AND (claim_lock IS NULL OR claim_expires IS NULL)").fetchall()- from hermes_cli.kanban_owner_recovery import owner_reclaim_paused+ from hermes_cli.kanban_owner_recovery import owner_reclaim_blockedfor row in rows:- if owner_reclaim_paused(conn, row["id"]):+ if owner_reclaim_blocked(conn, row["id"]):continuetid = row["id"]pid = row["worker_pid"]@@ -1168,7 +1168,7 @@ def _reclaim_dead_workers(conn: sqlite3.Connection, board: Optional[str] = None)"WHERE status = 'running' AND worker_pid IS NOT NULL").fetchall()host_prefix = _kb._host_prefix()- from hermes_cli.kanban_owner_recovery import bound_interpreter_gone, managed_run_exit_code, owner_reclaim_paused+ from hermes_cli.kanban_owner_recovery import managed_run_exit_code, owner_reclaim_blockedfor row in rows:lock = row["claim_lock"] or ""if not lock.startswith(host_prefix):@@ -1180,9 +1180,9 @@ def _reclaim_dead_workers(conn: sqlite3.Connection, board: Optional[str] = None)continueif _worker_alive(row["worker_pid"], _kb._row_get(row, "worker_started_at")):continue- # An ambiguous owner receipt holds the card while the attempt may still run; once the- # interpreter the owner bound to this run is dead nothing can, and it is a crash.- if owner_reclaim_paused(conn, row["id"]) and not bound_interpreter_gone(conn, row["id"]):+ # An ambiguous receipt holds the card while the attempt may still run; once nothing+ # can be executing it (a dead bound interpreter, or a run that never bound one): crash.+ if owner_reclaim_blocked(conn, row["id"]):continuepid = int(row["worker_pid"])diff --git a/hermes_cli/kanban_diagnostics.py b/hermes_cli/kanban_diagnostics.pyindex b1941f7dd72..87a300ab902 100644--- a/hermes_cli/kanban_diagnostics.py+++ b/hermes_cli/kanban_diagnostics.py@@ -736,6 +736,72 @@ def _rule_stranded_in_ready(task, events, runs, now, cfg) -> list[Diagnostic]:)]+def _rule_stale_running(task, events, runs, now, cfg) -> list[Diagnostic]:+ """A ``running`` card whose claim expired >= cfg["stale_running_grace_seconds"]+ (default 1h) ago with no live recorded worker.++ Nothing in the dispatcher is going to free this card — its reclaim paths can decline a+ card whose owner receipt never resolved, and until they could not, the failure was+ invisible: an operator saw only the cards queued behind the profile's worker cap. So the+ jam itself is the signal. Silent in every shape that can still progress: a live claim+ (the owner is still working), a heartbeat inside the dispatcher's max-stale window, a+ foreign host's lock (its pids say nothing here), or a live recorded worker. Read-only.+ """+ grace_seconds = float(cfg.get("stale_running_grace_seconds", 60 * 60))+ if _task_field(task, "status") != "running":+ return []+ from hermes_cli import kanban_db as kb+ expires = kb._to_epoch(_task_field(task, "claim_expires", None))+ if expires is None or (now - expires) < grace_seconds:+ return []+ lock = str(_task_field(task, "claim_lock", "") or "")+ if not lock.startswith(kb._host_prefix()):+ return []+ hb = kb._to_epoch(_task_field(task, "last_heartbeat_at", None))+ if hb is not None and (now - hb) <= kb.DEFAULT_CLAIM_HEARTBEAT_MAX_STALE_SECONDS:+ return []++ from hermes_cli.kanban_db_dispatch import _worker_alive+ recorded = [(_task_field(task, "worker_pid", None),+ _task_field(task, "worker_started_at", None))]+ for ev in events:+ if _event_kind(ev) in {"spawned", "worker_bound"}:+ payload = _parse_payload(ev)+ recorded.append((payload.get("pid"), payload.get("started_at")))+ pids = sorted({int(pid) for pid, _ in recorded if pid})+ if any(_worker_alive(int(pid), started_at) for pid, started_at in recorded if pid):+ return []++ expired_seconds = now - expires+ age_str = (f"{expired_seconds / 3600:.1f}h" if expired_seconds >= 3600+ else f"{int(expired_seconds / 60)}m")+ # Escalate with age: <2x grace warning, 2x-6x error, >6x critical.+ if expired_seconds >= grace_seconds * 6:+ severity = "critical"+ elif expired_seconds >= grace_seconds * 2:+ severity = "error"+ else:+ severity = "warning"++ task_id = str(_task_field(task, "id") or "")+ return [Diagnostic(+ kind="stale_running", severity=severity,+ title=f"Running {age_str} past its expired claim with no live worker",+ detail=f"Nothing is executing this card and nothing has reclaimed it: the claim expired "+ f"{age_str} ago, no heartbeat arrived inside the last hour, and no pid recorded for "+ f"the run ({', '.join(str(p) for p in pids) or 'none'}) is alive. The dispatcher "+ f"declines a reclaim when the run's owner receipt never resolved, so the card can "+ f"hold its profile's worker slot indefinitely while every card queued behind it "+ f"stays deferred. Check the card's log, then reclaim it.",+ actions=[DiagnosticAction(kind="reclaim", label="Reclaim the dead claim", suggested=True),+ _cli_hint("Reclaim from the CLI", f"hermes kanban reclaim {task_id}")],+ first_seen_at=expires, last_seen_at=expires, count=1,+ data={"claim_expires": expires, "expired_seconds": int(expired_seconds),+ "recorded_pids": pids, "last_heartbeat_at": hb,+ "grace_seconds": int(grace_seconds)},+ )]++# Order matters: earlier rules render first on severity ties._RULES: list[RuleFn] = [_rule_hallucinated_cards,@@ -748,6 +814,7 @@ _RULES: list[RuleFn] = [_rule_stuck_in_blocked,_rule_block_unblock_cycling,_rule_stranded_in_ready,+ _rule_stale_running,]@@ -762,6 +829,10 @@ DEFAULT_CONFIG = {# Below 30 min the signal is dominated by tasks about to be claimed on# the next dispatcher tick."stranded_threshold_seconds": 30 * 60,+ # The dispatcher reclaims an expired claim on its own next tick, so a card+ # still `running` an hour past its expiry means every reclaim path declined+ # it. Same hour the heartbeat backstop uses.+ "stale_running_grace_seconds": 60 * 60,}diff --git a/hermes_cli/kanban_owner_recovery.py b/hermes_cli/kanban_owner_recovery.pyindex 36703081880..112f472681d 100644--- a/hermes_cli/kanban_owner_recovery.py+++ b/hermes_cli/kanban_owner_recovery.py@@ -3,6 +3,7 @@ from contextlib import closingimport jsonfrom pathlib import Pathimport sqlite3+import timeimport psutil@@ -32,20 +33,137 @@ def owner_reclaim_paused(conn, task_id):return True+def _latest_run_event(conn, task_id, kind):+ """The newest ``kind`` payload on the task's CURRENT run, or ``{}``.-def bound_interpreter_gone(conn, task_id):- """True when the owner bound an interpreter to the task's current run and it is dead."""- from hermes_cli.kanban_db_dispatch import _worker_alive+ A payload that cannot be read is not a payload that says the attempt is over, so+ callers treat ``{}`` as "no evidence" rather than as an absence.+ """row = conn.execute("SELECT e.payload FROM task_events e JOIN tasks t ON t.current_run_id=e.run_id "- "WHERE t.id=? AND e.task_id=t.id AND e.kind='worker_bound' ORDER BY e.id DESC LIMIT 1",- (task_id,)).fetchone()- if row is None:+ "WHERE t.id=? AND e.task_id=t.id AND e.kind=? ORDER BY e.id DESC LIMIT 1",+ (task_id, kind)).fetchone()+ try:+ payload = json.loads(row['payload']) if row else {}+ except (ValueError, TypeError):+ return {}+ return payload if isinstance(payload, dict) else {}+++def _claim_lock(conn, task_id):+ row = conn.execute("SELECT claim_lock FROM tasks WHERE id=?", (task_id,)).fetchone()+ return row['claim_lock'] if row is not None else None+++def _host_local_lock(lock) -> bool:+ """True when ``lock`` was issued by this host — the only pids we can judge."""+ from hermes_cli import kanban_db as kb+ return bool(lock) and str(lock).startswith(kb._host_prefix())+++def bound_interpreter_gone(conn, task_id):+ """True when the owner bound an interpreter to the task's current run and it is dead.++ Host-local only: ``worker_bound`` records a pid of THIS board's host, so a numeric+ coincidence on another host must never read as a death and release a live claim.+ """+ if not _host_local_lock(_claim_lock(conn, task_id)):+ return False+ bound = _latest_run_event(conn, task_id, 'worker_bound')+ if not bound:return False- bound = json.loads(row['payload'])+ from hermes_cli.kanban_db_dispatch import _worker_alivereturn not _worker_alive(bound.get('pid'), bound.get('started_at'))+def _owner_process_gone(receipt) -> bool:+ """True when the receipt's own pid is gone, or its number now belongs to another process.++ The identity rule ``owner_reclaim_paused`` already applies: a live pid whose ``birth``+ differs is a recycled number, not the owner. A live pid we cannot compare (no ``birth``+ recorded) and an unreadable one are never proof of death.+ """+ try:+ pid = int(receipt['pid'])+ except (KeyError, TypeError, ValueError):+ return False+ try:+ from hermes_cli.process_identity import process_birth+ current = process_birth(psutil.Process(pid))+ except psutil.NoSuchProcess:+ return True+ except psutil.Error:+ return False+ birth = receipt.get('birth')+ return birth is not None and current != birth+++def _run_worker_pids(conn, task_row, task_id):+ """Every worker pid recorded for the task's current run, with its spawn fingerprint."""+ recorded = [(task_row['worker_pid'], task_row['worker_started_at'])]+ for kind in ('spawned', 'worker_bound'):+ payload = _latest_run_event(conn, task_id, kind)+ recorded.append((payload.get('pid'), payload.get('started_at')))+ return [(int(pid), fingerprint) for pid, fingerprint in recorded if pid]+++def owner_attempt_provably_over(conn, task_id) -> bool:+ """True when an unresolved owner receipt cannot still be executing this attempt.++ ``owner_reclaim_paused`` is about effects: an admission that may still be in flight is+ never repeated. It must not also mean the run keeps its claim forever. The attempts it+ pauses in practice are ones whose owner was killed (restart, OOM), and such a run may+ never have written a ``worker_bound`` event at all, so ``bound_interpreter_gone`` cannot+ speak for it — 76 ``owner_admitted`` against 74 ``worker_bound`` rows on one board means+ runs in exactly that shape. This asks the question that matters — can anything still be+ executing this attempt? — from evidence the board already keeps:++ * the claim is host-local and EXPIRED (a live claim is the owner still working);+ * no heartbeat inside ``DEFAULT_CLAIM_HEARTBEAT_MAX_STALE_SECONDS``;+ * the receipt's own process is gone or recycled;+ * every worker pid recorded for the run (``tasks.worker_pid``, its ``spawned`` and+ ``worker_bound`` events) is dead or recycled.++ Missing evidence is never proof: no recorded worker pid, a live pid whose identity+ cannot be compared, or an unreadable clock all return False and the pause holds.+ """+ from hermes_cli import kanban_db as kb+ from hermes_cli.kanban_db_dispatch import _worker_alive+ row = conn.execute(+ "SELECT claim_lock, claim_expires, worker_pid, worker_started_at, last_heartbeat_at "+ "FROM tasks WHERE id=?", (task_id,)).fetchone()+ if row is None or not _host_local_lock(row['claim_lock']):+ return False+ now = int(time.time())+ expires = kb._to_epoch(row['claim_expires'])+ if expires is None or expires >= now:+ return False+ heartbeat = kb._to_epoch(row['last_heartbeat_at'])+ if heartbeat is not None and now - heartbeat <= kb.DEFAULT_CLAIM_HEARTBEAT_MAX_STALE_SECONDS:+ return False+ if not _owner_process_gone(_latest_run_event(conn, task_id, 'owner_admitted')):+ return False+ pids = _run_worker_pids(conn, row, task_id)+ if not pids:+ return False+ return not any(_worker_alive(pid, fingerprint) for pid, fingerprint in pids)+++def owner_reclaim_blocked(conn, task_id) -> bool:+ """True when a reclaim path must leave this task alone.++ The pause protects the effects of an attempt that may still be in flight, and stops+ holding the claim as soon as nothing can still be executing it: either the owner bound an+ interpreter that is gone (``bound_interpreter_gone``), or the receipt is unresolved while+ every piece of the run's liveness evidence is dead (``owner_attempt_provably_over``) —+ the shape that used to be unreclaimable forever, because its run wrote no ``worker_bound``+ event and that override was the only one anyone consulted.+ """+ if not owner_reclaim_paused(conn, task_id):+ return False+ return not (bound_interpreter_gone(conn, task_id) or owner_attempt_provably_over(conn, task_id))++def managed_run_exit_code(conn, task_id, worker_pid, claim_lock):"""The exit code a managed run's authority recorded for the task's CURRENT run, else None.diff --git a/tests/hermes_cli/test_kanban_owner_receipt_reclaim.py b/tests/hermes_cli/test_kanban_owner_receipt_reclaim.pynew file mode 100644index 00000000000..10c403c93ef--- /dev/null+++ b/tests/hermes_cli/test_kanban_owner_receipt_reclaim.py@@ -0,0 +1,348 @@+"""A dead attempt behind an unresolved owner receipt must not hold its claim forever.++Regression for #140858. Card t_5ea03658 sat ``running`` for 16 hours: claimed 07:48, worker+killed by a restart at 08:59, claim expired 08:03, no heartbeat, and no ``worker_bound``+event. ``owner_reclaim_paused`` was true — a killed owner leaves a ``terminal/interrupted``+admission, or a pid that is gone — and every reclaim path read that as "leave it alone", so+``release_stale_claims``, ``detect_crashed_workers`` and ``reconcile_orphaned_running`` all+declined. ``bound_interpreter_gone``, the one override, needs the ``worker_bound`` event that+run never wrote. The card held its profile's ``max_in_progress_per_profile`` slot while every+card queued behind it was deferred, and no diagnostics rule could see a ``running`` card at+all, so the alarms named the victims and never the cause.++The pause is about effects that may still be in flight. These tests pin both sides of that+line: a live worker, a fresh heartbeat, a live claim, a foreign host's lock or an identity+that cannot be read all keep the card; a provably dead attempt is reclaimed by the platform's+own path.+"""++from __future__ import annotations++import json+import os+import sqlite3+import time+from pathlib import Path++import psutil+import pytest++from hermes_cli import kanban_db as kb+from hermes_cli import kanban_db_connect as kbc+from hermes_cli import kanban_db_dispatch as kbd+from hermes_cli import kanban_diagnostics as kd+from hermes_cli.kanban_owner_recovery import (+ bound_interpreter_gone,+ owner_attempt_provably_over,+ owner_reclaim_blocked,+ owner_reclaim_paused,+)++NOW = int(time.time())+HOST = kb._claimer_id().split(":", 1)[0]+FINGERPRINT = "6a232414-2dbf-40ab-a534-d9b71db05658:116|4162240"+++def _absent_pid(base: int = 3_500_000) -> int:+ """A pid that provably does not exist on this host, so a fixture can never hit a process."""+ pid = base+ while psutil.pid_exists(pid):+ pid += 1+ return pid+++class _Board:+ """A scratch board plus the scratch admission ledger an owner receipt points at."""++ def __init__(self, root: Path):+ self.path = root / "kanban.db"+ kbc.init_db(self.path)+ self.conn = kbc.connect(self.path)+ self.ledger = root / "state.db"+ with sqlite3.connect(self.ledger) as store:+ store.execute(+ "CREATE TABLE session_admissions "+ "(target_session_id TEXT, request_id TEXT, status TEXT, outcome TEXT)"+ )++ def close(self) -> None:+ self.conn.close()++ # --- seeding --------------------------------------------------------------------------+ def dead_attempt(+ self, tid: str = "t_ghost", *, worker_pid=None, worker_fingerprint=FINGERPRINT,+ worker_bound: bool = False, record_worker_pid: bool = True, owner_pid=None,+ owner_birth=None, admission=("terminal", "interrupted"), dead_hours: int = 15,+ expires_age_seconds=None, lock_host=None, heartbeat_age=None,+ ) -> int:+ """A ``running`` card whose worker died the way t_5ea03658's did."""+ worker_pid = _absent_pid() if worker_pid is None else worker_pid+ owner_pid = _absent_pid(base=3_600_000) if owner_pid is None else owner_pid+ lock = f"{lock_host or HOST}:{owner_pid}"+ expires = NOW - (expires_age_seconds if expires_age_seconds is not None+ else dead_hours * 3600)+ started = NOW - dead_hours * 3600+ run_id = self.conn.execute(+ "INSERT INTO task_runs (task_id, profile, status, claim_lock, claim_expires, "+ "worker_pid, started_at) VALUES (?, 'default', 'running', ?, ?, ?, ?)",+ (tid, lock, expires, worker_pid if record_worker_pid else None, started),+ ).lastrowid+ assert run_id is not None+ run = int(run_id)+ self.conn.execute(+ "INSERT INTO tasks (id, title, assignee, status, created_at, claim_lock, claim_expires, "+ "worker_pid, worker_started_at, last_heartbeat_at, current_run_id, started_at, priority) "+ "VALUES (?, ?, 'default', 'running', ?, ?, ?, ?, ?, ?, ?, ?, 5)",+ (tid, tid, NOW - 86400, lock, expires,+ worker_pid if record_worker_pid else None,+ worker_fingerprint if record_worker_pid else None,+ (NOW - heartbeat_age) if heartbeat_age is not None else None, run, started),+ )+ self.event(tid, run, "claimed", {"source_status": "ready"}, started)+ if record_worker_pid:+ spawned = {"pid": worker_pid, "started_at": worker_fingerprint}+ else:+ spawned = {}+ self.event(tid, run, "spawned", spawned, started)+ payload = {"db": str(self.ledger), "session_id": "sess-1", "pid": owner_pid,+ "request_id": f'kanban:["default","{tid}",1]'}+ if owner_birth is not None:+ payload["birth"] = owner_birth+ self.event(tid, run, "owner_admitted", payload, started + 30)+ if worker_bound:+ self.event(tid, run, "worker_bound",+ {"pid": worker_pid, "claim_lock": lock, "started_at": worker_fingerprint},+ started + 60)+ if admission:+ with sqlite3.connect(self.ledger) as store:+ store.execute("INSERT INTO session_admissions VALUES ('sess-1', ?, ?, ?)",+ (payload["request_id"], admission[0], admission[1]))+ self.conn.commit()+ return run++ def event(self, tid: str, run: int, kind: str, payload: dict, ts: int) -> None:+ self.conn.execute(+ "INSERT INTO task_events (task_id, run_id, kind, payload, created_at) "+ "VALUES (?, ?, ?, ?, ?)", (tid, run, kind, json.dumps(payload), ts))++ def ready(self, tid: str) -> None:+ self.conn.execute(+ "INSERT INTO tasks (id, title, assignee, status, created_at, priority) "+ "VALUES (?, ?, 'default', 'ready', ?, 5)", (tid, tid, NOW - 86400))+ self.conn.commit()++ # --- reading --------------------------------------------------------------------------+ def state(self, tid: str) -> dict:+ return dict(self.conn.execute("SELECT * FROM tasks WHERE id = ?", (tid,)).fetchone())++ def events(self, tid: str) -> list:+ return self.conn.execute(+ "SELECT kind, payload, created_at FROM task_events WHERE task_id = ? ORDER BY id",+ (tid,)).fetchall()++ def diagnostics(self, tid: str, *, config: dict | None = None) -> list:+ return kd.compute_task_diagnostics(+ self.state(tid), self.events(tid), [], now=NOW, config=config or {})+++def board(tmp_path, monkeypatch):+ monkeypatch.setenv("HERMES_KANBAN_CRASH_GRACE_SECONDS", "0")+ scratch = _Board(tmp_path)+ yield scratch+ scratch.close()+++# --- the incident shape is reclaimed by the platform's own paths ---------------------------++def test_dead_attempt_behind_an_unresolved_receipt_is_reclaimed(board):+ """Every reclaim path declined this card; the stale-claim path must now take it."""+ board.dead_attempt("t_ghost")+ assert owner_reclaim_paused(board.conn, "t_ghost") is True # the pause still applies+ assert bound_interpreter_gone(board.conn, "t_ghost") is False # nothing bound to read+ assert owner_attempt_provably_over(board.conn, "t_ghost") is True+ assert owner_reclaim_blocked(board.conn, "t_ghost") is False++ assert kb.release_stale_claims(board.conn) == 1++ row = board.state("t_ghost")+ assert row["status"] == "ready"+ assert (row["claim_lock"], row["claim_expires"], row["worker_pid"]) == (None, None, None)+ assert row["current_run_id"] is None+ run = board.conn.execute(+ "SELECT status, outcome FROM task_runs WHERE task_id = 't_ghost'").fetchone()+ assert tuple(run) == ("reclaimed", "reclaimed")+++def test_dead_attempt_behind_an_unresolved_receipt_is_reclaimed_by_the_crash_sweep(board):+ """The dead-pid path reclaims the same shape instead of deferring to the receipt."""+ board.dead_attempt("t_ghost")+ assert kbd.detect_crashed_workers(board.conn) == ["t_ghost"]+ assert board.state("t_ghost")["status"] == "ready"+++def test_reclaiming_the_dead_attempt_frees_its_profile_lane(board):+ """The point of the fix: the card stops holding its profile's worker slot."""+ board.dead_attempt("t_ghost")+ board.ready("t_ready0")+ counted = "SELECT COUNT(*) FROM tasks WHERE status = 'running' AND assignee = 'default'"+ assert board.conn.execute(counted).fetchone()[0] == 1 # the jam: a dead card holds the slot++ kbd.dispatch_once(board.conn, dry_run=True, max_in_progress=2,+ max_in_progress_per_profile=1)+ assert board.state("t_ghost")["status"] == "ready"+ assert board.conn.execute(counted).fetchone()[0] == 0 # the slot is free again++ # And the next tick spends it instead of deferring the queue forever.+ lane = kbd.dispatch_once(board.conn, dry_run=True, max_in_progress=2,+ max_in_progress_per_profile=1)+ assert len(lane.spawned) == 1+++# --- the pause still holds every attempt that may be alive ---------------------------------++def test_a_live_worker_holds_the_card_even_with_the_pause(board):+ """Real pid, real fingerprint: reclaiming beside it would spawn a duplicate."""+ board.dead_attempt("t_ghost", worker_pid=os.getpid(),+ worker_fingerprint=kbd._process_fingerprint(os.getpid()))+ assert owner_reclaim_paused(board.conn, "t_ghost") is True+ assert owner_attempt_provably_over(board.conn, "t_ghost") is False+ assert owner_reclaim_blocked(board.conn, "t_ghost") is True+ assert kb.release_stale_claims(board.conn) == 0+ assert board.state("t_ghost")["status"] == "running"+ assert board.state("t_ghost")["worker_pid"] == os.getpid()+++def test_a_live_bound_interpreter_holds_the_card(board):+ """A ``worker_bound`` interpreter that is still running is the same refusal."""+ board.dead_attempt("t_ghost", worker_bound=True, worker_pid=os.getpid(),+ worker_fingerprint=kbd._process_fingerprint(os.getpid()))+ assert bound_interpreter_gone(board.conn, "t_ghost") is False+ assert owner_reclaim_blocked(board.conn, "t_ghost") is True+ assert kb.release_stale_claims(board.conn) == 0+ assert board.state("t_ghost")["status"] == "running"+++def test_a_fresh_heartbeat_holds_the_card(board):+ """A slow worker inside one tool-free call keeps its claim (#23025)."""+ board.dead_attempt("t_ghost", heartbeat_age=300)+ assert owner_attempt_provably_over(board.conn, "t_ghost") is False+ assert kb.release_stale_claims(board.conn) == 0+ assert board.state("t_ghost")["status"] == "running"+++def test_a_live_claim_is_never_overridden(board):+ """An unexpired claim is the owner still working, whatever the receipt says."""+ board.dead_attempt("t_ghost", expires_age_seconds=-3600)+ assert owner_attempt_provably_over(board.conn, "t_ghost") is False+ assert kb.release_stale_claims(board.conn) == 0+ assert board.state("t_ghost")["status"] == "running"+++def test_another_hosts_lock_is_never_overridden(board):+ """A foreign lock's pids say nothing here, and a numeric coincidence is not a death."""+ board.dead_attempt("t_ghost", lock_host="some-other-host")+ assert bound_interpreter_gone(board.conn, "t_ghost") is False+ assert owner_attempt_provably_over(board.conn, "t_ghost") is False+ assert kb.release_stale_claims(board.conn) == 0+ assert board.state("t_ghost")["status"] == "running"+++def test_a_live_owner_without_a_birth_is_never_read_as_dead(board):+ """The receipt names a live pid but no identity to compare: unprovable, so it holds."""+ board.dead_attempt("t_ghost", owner_pid=os.getpid(), owner_birth=None)+ assert owner_reclaim_paused(board.conn, "t_ghost") is True+ assert owner_attempt_provably_over(board.conn, "t_ghost") is False+ assert kb.release_stale_claims(board.conn) == 0+ assert board.state("t_ghost")["status"] == "running"+++def test_a_recycled_owner_pid_is_dead_evidence(board):+ """A live number whose birth moved on is a stranger, exactly as the pause reads it."""+ board.dead_attempt("t_ghost", owner_pid=os.getpid(), owner_birth=1.0)+ assert owner_reclaim_paused(board.conn, "t_ghost") is True+ assert owner_attempt_provably_over(board.conn, "t_ghost") is True+ assert kb.release_stale_claims(board.conn) == 1+++def test_owner_identity_reads_birth_through_the_shared_helper(board, monkeypatch):+ """A live owner's birth must be read the way the pause reads it, or a clock step looks like death.++ ``create_time()`` shifts on macOS after a boot-time change while the kernel's own start does+ not (#140580), so comparing the fence's reading against the receipt's would call a live owner+ recycled and release a claim that is still being worked. The fence defers to+ ``process_identity.process_birth``; this pins that it does, whatever the platform.+ """+ from hermes_cli import process_identity+ seen: list[int] = []++ def fake_birth(proc):+ seen.append(proc.pid)+ return 1234.0++ monkeypatch.setattr(process_identity, "process_birth", fake_birth)+ board.dead_attempt("t_ghost", owner_pid=os.getpid(), owner_birth=1234.0)+ assert owner_reclaim_paused(board.conn, "t_ghost") is True+ assert owner_attempt_provably_over(board.conn, "t_ghost") is False+ assert seen == [os.getpid()]+ assert kb.release_stale_claims(board.conn) == 0+ assert board.state("t_ghost")["status"] == "running"+++def test_no_recorded_worker_pid_is_never_proof(board):+ """A run that recorded no worker pid cannot be shown to be dead — it holds."""+ board.dead_attempt("t_ghost", record_worker_pid=False)+ assert owner_attempt_provably_over(board.conn, "t_ghost") is False+ assert kb.release_stale_claims(board.conn) == 0+ assert board.state("t_ghost")["status"] == "running"+++# --- the jam itself becomes visible to `kanban diagnostics` --------------------------------++def test_stale_running_diagnostic_names_the_jam(board):+ board.dead_attempt("t_ghost")+ hits = [d for d in board.diagnostics("t_ghost") if d.kind == "stale_running"]+ assert len(hits) == 1+ hit = hits[0]+ assert hit.severity == "critical" # 15x the 1h grace+ assert hit.data["expired_seconds"] >= 3600+ assert hit.data["recorded_pids"] and str(hit.data["recorded_pids"][0]) in hit.detail+ assert [a.kind for a in hit.actions] == ["reclaim", "cli_hint"]+ assert hit.actions[0].suggested is True++[email protected]("expired_for,severity", [+ (30 * 60, ""), # inside the grace: the dispatcher has not had its tick yet+ (3 * 3600, "error"), # 3x grace+ (7 * 3600, "critical"), # 7x grace+])+def test_stale_running_diagnostic_escalates_with_age(board, expired_for, severity):+ board.dead_attempt("t_ghost", expires_age_seconds=expired_for)+ hits = [d for d in board.diagnostics("t_ghost") if d.kind == "stale_running"]+ assert [d.severity for d in hits] == ([severity] if severity else [])+++def test_stale_running_diagnostic_is_silent_for_a_live_worker(board):+ board.dead_attempt("t_ghost", worker_pid=os.getpid(),+ worker_fingerprint=kbd._process_fingerprint(os.getpid()))+ assert not [d for d in board.diagnostics("t_ghost") if d.kind == "stale_running"]+++def test_stale_running_diagnostic_is_silent_for_a_fresh_heartbeat(board):+ board.dead_attempt("t_ghost", heartbeat_age=300)+ assert not [d for d in board.diagnostics("t_ghost") if d.kind == "stale_running"]+++def test_stale_running_diagnostic_is_silent_for_a_live_claim(board):+ board.dead_attempt("t_ghost", expires_age_seconds=-60)+ assert not [d for d in board.diagnostics("t_ghost") if d.kind == "stale_running"]+++def test_stale_running_diagnostic_is_silent_for_another_hosts_lock(board):+ board.dead_attempt("t_ghost", lock_host="some-other-host")+ assert not [d for d in board.diagnostics("t_ghost") if d.kind == "stale_running"]+++def test_stale_running_diagnostic_is_silent_for_a_ready_card(board):+ board.ready("t_ready0")+ assert not [d for d in board.diagnostics("t_ready0") if d.kind == "stale_running"]diff --git a/website/docs/user-guide/features/kanban-worker-lanes.md b/website/docs/user-guide/features/kanban-worker-lanes.mdindex 6507462f9e2..65606de0c8b 100644--- a/website/docs/user-guide/features/kanban-worker-lanes.md+++ b/website/docs/user-guide/features/kanban-worker-lanes.md@@ -133,6 +133,7 @@ So lane authors don't have to reimplement these:- **Run-level retry** — when a task is retried (post-block, post-crash, post-reclaim), the worker can use the `expected_run_id` parameter on terminating tools to fail fast if its own run was already superseded.- **Per-task max runtime** — `task.max_runtime_seconds` hard-caps wall-clock time per run, regardless of PID liveness. Catches genuinely-deadlocked workers that the live-PID extension would otherwise keep running.- **Stranded-task detection** — a ready task whose assignee never produces a claim within `kanban.stranded_threshold_seconds` (default 30 min) shows up in `hermes kanban diagnostics` as a `stranded_in_ready` warning. Severity escalates to error at 2x the threshold and critical at 6x. Catches typo'd assignees, deleted profiles, and down external worker pools in one signal — identity-agnostic, no per-board allowlist to curate.+- **Dead attempt behind an unresolved owner receipt** — a worker killed by a restart can leave an `owner_admitted` receipt the dispatcher cannot resolve, and reclaims used to be refused on that ground alone, forever: the card stayed `running`, held its profile's `kanban.max_in_progress_per_profile` slot, and the alarms named the cards queued behind it rather than the cause. The refusal now lapses as soon as nothing can be executing the attempt — the owner's process is gone or recycled, no heartbeat inside `DEFAULT_CLAIM_HEARTBEAT_MAX_STALE_SECONDS`, and every pid the run recorded (`worker_pid`, `spawned`, `worker_bound`) is dead — including runs that never wrote a `worker_bound` event. Anything unprovable (a live pid, a missing fingerprint, no recorded pid, another host's lock) still holds the claim. A card that survives even that shows up in `hermes kanban diagnostics` as `stale_running`, escalated by age past `kanban.diagnostics.stale_running_grace_seconds` (default 1h).- **Running with open parents** — a `running` card whose direct parent is not `done`/`archived` (the parent reopened mid-run, or the link predates the running-child refusal) shows a `running_with_open_parents` warning: the dependency gate is not serialising the two runs and `kanban_complete` will be refused until the parent finishes. Read-only; suggests `hermes kanban unlink`.- **Legacy review dependency deadlock** — a parent sticky-blocked with `review-required:` while one or more direct children remain dependency-gated in `todo` produces an immediate `review_dependency_deadlock` error. The diagnostic is read-only: it suggests completing the finished phase or unlinking the incorrect edge but never removes a user block automatically.