Skip to content

The agent that grows with you

fix(kanban): a lost worker's clean exit is a crash, not also a protocol violation

kshitijk4poore62c15e667e4
diff --git a/hermes_cli/kanban_db_dispatch.py b/hermes_cli/kanban_db_dispatch.py
index 6f7babc2c3d..ebebe54c7e5 100644
--- a/hermes_cli/kanban_db_dispatch.py
+++ b/hermes_cli/kanban_db_dispatch.py
@@ -486,14 +486,10 @@ def _terminate_reclaimed_worker(
info["terminated"] = True
return info
except OSError:
- # Windows reports an ABSENT pid as a bare OSError (WinError 87 from OpenProcess),
- # not ProcessLookupError, so reading every OSError as "termination not proven" made
- # the reclaim hold the claim and defer every tick for a worker that was already gone:
- # 2026-10-22 t_f5fad63f kept its card ``running`` with ``claim_expires`` 2067 s past.
- # Ask liveness again: gone is the outcome the reclaim wanted, alive stays unproven.
+ # Windows reports an ABSENT pid as a bare OSError (WinError 87), not ProcessLookupError:
+ # ask liveness again, so a gone pid is terminated and only a live one stays unproven.
if not _kb._pid_alive(pid):
info["terminated"] = True
- info["already_gone"] = True
return info
if _poll_worker_exit(pid, started_at):
@@ -1175,11 +1171,10 @@ class _CrashSweep:
def _apply_attempt_note(conn, task_id, pid, dead: _DeadWorker) -> None:
"""Fold the owner's own record of how the attempt ended into ONE dead worker's booking.
- A managed worker that dies during ``worker.adopt`` writes no ``worker_result``, so the
- exit trailer alone can read as a clean protocol violation. The owner's ``worker_lost``
- evidence outranks it: the attempt was lost, and it must charge the breaker like any other
- crash instead of riding the protocol-violation budget (run 730 showed
- ``consecutive_failures = 0`` after exactly this loss).
+ A managed worker that dies in ``worker.adopt`` writes no ``worker_result``, so its rc=0
+ exit trailer reads as a protocol violation. The owner's ``worker_lost`` evidence outranks
+ it: the turn never finished, so it books as a crash and charges the breaker, and the
+ protocol-violation streak (which reads the persisted payload) does not count it.
"""
from hermes_cli.kanban_owner_recovery import _latest_run_event, owner_attempt_note
note = owner_attempt_note(conn, task_id)
@@ -1190,11 +1185,11 @@ def _apply_attempt_note(conn, task_id, pid, dead: _DeadWorker) -> None:
lost = _latest_run_event(conn, task_id, "worker_lost")
if lost:
dead.event_payload["worker_lost"] = lost
- dead.protocol_violation = False
- dead.rate_limited = False
- if dead.kind == "clean_exit":
- # A worker whose managed execution was lost never proved the turn finished.
- dead.kind = "worker_lost"
+ if dead.protocol_violation:
+ dead.protocol_violation = False
+ dead.event_payload.pop("protocol_violation", None)
+ dead.event_payload["exit_kind"] = dead.kind
+ dead.event_kind = "crashed"
dead.error_text = f"pid {pid} exited cleanly after its managed worker was lost [{note}]"
diff --git a/tests/hermes_cli/test_kanban_managed_worker_loss.py b/tests/hermes_cli/test_kanban_managed_worker_loss.py
index 6cd7df4b8f1..d93c37e7543 100644
--- a/tests/hermes_cli/test_kanban_managed_worker_loss.py
+++ b/tests/hermes_cli/test_kanban_managed_worker_loss.py
@@ -25,7 +25,6 @@ These tests pin the disposition the dispatcher owes every lost attempt:
from __future__ import annotations
import json
-import os
import sqlite3
import time
from pathlib import Path
@@ -189,6 +188,12 @@ def test_a_lost_worker_that_exited_cleanly_still_charges_the_breaker(board):
assert run["outcome"] == "crashed"
assert "managed worker lost" in run["error"]
assert board.state("t_clean")["consecutive_failures"] == 1
+ # Not a protocol violation as well: one loss is charged to one budget.
+ assert "protocol_violation" not in json.loads(run["metadata"] or "{}")
+ kinds = [r[0] for r in board.conn.execute(
+ "SELECT kind FROM task_events WHERE task_id='t_clean' AND run_id=?", (run_id,))]
+ assert "protocol_violation" not in kinds and "crashed" in kinds
+ assert kbd._protocol_violation_streak(board.conn, "t_clean") == 0
# --- A2: an elapsed claim lock is never "still running" after one tick ---------------------
@@ -235,25 +240,6 @@ def test_a_settled_receipt_with_a_live_gateway_owner_no_longer_holds_the_claim(b
# --- the reclaim's kill step: an absent pid is gone, not "survived" -----------------------
-def test_terminating_an_absent_pid_reports_terminated(board):
- """A pid that is already gone must not be reported as unproven termination.
-
- Windows raises a bare ``OSError`` (WinError 87) for an absent PID instead of
- ``ProcessLookupError``; reading that as "termination not proven" made the reclaim hold
- the claim and defer every tick — the elapsed lock stayed ``running`` forever.
- """
- info = kbd._terminate_reclaimed_worker(_absent_pid(), f"{HOST}:{_absent_pid(base=3_960_000)}")
- assert info["host_local"] is True
- assert info["termination_attempted"] is True
- assert info["terminated"] is True
- if os.name == "nt":
- # Only Windows reports an absent pid as a bare OSError, so only there does the kill
- # step take the re-ask branch that marks this verdict; POSIX reaches the same
- # ``terminated`` through ProcessLookupError (pinned on every platform by
- # test_a_signalled_absent_pid_is_terminated_not_unproven).
- assert info["already_gone"] is True
-
-
def test_a_signalled_absent_pid_is_terminated_not_unproven(board):
"""The same reading through a signal seam: an OSError on a dead pid is a success."""
gone = _absent_pid(base=3_970_000)
@@ -264,7 +250,6 @@ def test_a_signalled_absent_pid_is_terminated_not_unproven(board):
info = kbd._terminate_reclaimed_worker(gone, f"{HOST}:{_absent_pid(base=3_980_000)}",
signal_fn=windows_kill)
assert info["terminated"] is True
- assert info["already_gone"] is True
# --- D1: the OWNER writes the loss where the dispatcher can read it -----------------------