Skip to content

The agent that grows with you

fix(kanban): keep the worker's last output when a lost worker's clean exit is rebooked

kshitijk4poorc777ba429fc8
diff --git a/hermes_cli/kanban_db_dispatch.py b/hermes_cli/kanban_db_dispatch.py
index 5585ace432b..5e5d8ff47ba 100644
--- a/hermes_cli/kanban_db_dispatch.py
+++ b/hermes_cli/kanban_db_dispatch.py
@@ -482,7 +482,7 @@ def _terminate_reclaimed_worker(
except OSError:
# Already gone = successful termination (Windows reports an absent pid as a bare OSError, not
# ProcessLookupError). Leaving terminated=False would make the reclaim defer forever.
- info["terminated"] = not _kb._pid_alive(pid)
+ info["terminated"] = not _worker_alive(pid, started_at)
return info
if _poll_worker_exit(pid, started_at):
@@ -960,10 +960,9 @@ def _protocol_violation_streak(conn: sqlite3.Connection, task_id: str) -> int:
outcome = row["outcome"] or ""
if outcome == "rate_limited":
continue
- if outcome == "crashed" and (
- _kb._json_dict(row["metadata"]).get("protocol_violation")
- or "protocol violation" in (row["error"] or "")
- ):
+ metadata = _kb._json_dict(row["metadata"]) # a lost worker's run quotes its output: no text match
+ if outcome == "crashed" and not metadata.get("worker_lost") and (
+ metadata.get("protocol_violation") or "protocol violation" in (row["error"] or "")):
streak += 1
continue
break
diff --git a/hermes_cli/kanban_owner_recovery.py b/hermes_cli/kanban_owner_recovery.py
index 5687b95ca61..59e30a604ad 100644
--- a/hermes_cli/kanban_owner_recovery.py
+++ b/hermes_cli/kanban_owner_recovery.py
@@ -56,9 +56,7 @@ def owner_reclaim_paused(conn, task_id):
return True
from hermes_cli.process_identity import worker_birth
return worker_birth(psutil.Process(receipt['pid'])) != receipt['birth']
- except (OSError, ValueError, KeyError, sqlite3.Error):
- return True
- except psutil.Error:
+ except (OSError, ValueError, KeyError, TypeError, psutil.Error):
return True
@@ -186,16 +184,19 @@ def owner_attempt_provably_over(conn, task_id) -> bool:
return settled or (bool(pids) and _owner_process_gone(receipt))
-def owner_attempt_note(conn, task_id) -> str:
- """One clause naming WHY a provably-over attempt ended, or ``""``.
+def owner_attempt_note(conn, task_id, lost=None) -> str:
+ """One clause naming why the owner says the attempt ended, or ``""``.
- The reclaim paths record this beside the pid/claim evidence so the board says what
- happened instead of a bare ``stale_lock=``: a managed worker that died (its owner
- recorded ``worker_lost``) is a different fault from a worker that never started.
+ The reclaim paths put this in the run error so the board says what happened instead of
+ a bare ``stale_lock=``: a managed worker that died (its owner recorded ``worker_lost``,
+ passed as ``lost`` when the caller already read it) is a different fault from a worker
+ that never started.
"""
- lost = str(_latest_run_event(conn, task_id, 'worker_lost').get('reason') or '')
- if lost:
- return f"managed worker lost: {lost[:80]}"
+ if lost is None:
+ lost = _latest_run_event(conn, task_id, 'worker_lost')
+ reason = str(lost.get('reason') or '')
+ if reason:
+ return f"managed worker lost: {reason[:80]}"
receipt = _owner_receipt(conn, task_id)
if receipt is None:
return ""
@@ -214,25 +215,30 @@ def reclaim_error(conn, task_id, claim_lock) -> str:
def apply_attempt_note(conn, task_id, pid, dead) -> None:
"""Fold the owner's own record of how the attempt ended into ONE dead worker's booking.
- A managed worker that dies in ``worker.adopt`` writes no ``worker_result``, so its rc=0
- exit trailer reads as a protocol violation. The owner's ``worker_lost`` evidence outranks
- it: the turn never finished, so it books as a crash and charges the breaker, and the
+ A bound interpreter whose ``worker_result`` says exit 0 after its managed execution was
+ lost reads as a protocol violation. The owner's ``worker_lost`` evidence outranks it: the
+ turn never finished, so it books as a crash and charges the breaker, and the
protocol-violation streak (which reads the persisted payload) does not count it.
"""
- note = owner_attempt_note(conn, task_id)
+ lost = _latest_run_event(conn, task_id, "worker_lost")
+ note = owner_attempt_note(conn, task_id, lost)
if not note:
return
- dead.error_text = f"{dead.error_text} [{note}]"
dead.event_payload["attempt_note"] = note
- lost = _latest_run_event(conn, task_id, "worker_lost")
if lost:
dead.event_payload["worker_lost"] = lost
if dead.protocol_violation:
+ from hermes_cli.kanban_db_dispatch import _PROTOCOL_VIOLATION_ERROR
dead.protocol_violation = False
dead.event_payload.pop("protocol_violation", None)
dead.event_payload["exit_kind"] = dead.kind
dead.event_kind = "crashed"
- dead.error_text = f"pid {pid} exited cleanly after its managed worker was lost [{note}]"
+ # Keep the worker's last output that _classify_dead_worker appended.
+ dead.error_text = (f"pid {pid} exited cleanly after its managed worker was lost."
+ + dead.error_text.removeprefix(_PROTOCOL_VIOLATION_ERROR))
+ # Before the worker's own output: the stored error is capped, and the output may be long.
+ head, sep, output = dead.error_text.partition(" Worker's last output: ")
+ dead.error_text = f"{head} [{note}]{sep}{output}"
def owner_reclaim_blocked(conn, task_id, *, crash_sweep=False) -> bool:
diff --git a/tests/hermes_cli/test_kanban_managed_worker_loss.py b/tests/hermes_cli/test_kanban_managed_worker_loss.py
index ecf3578d231..de107e37ae9 100644
--- a/tests/hermes_cli/test_kanban_managed_worker_loss.py
+++ b/tests/hermes_cli/test_kanban_managed_worker_loss.py
@@ -1,9 +1,9 @@
"""A lost managed worker must be DISPOSED OF on the board, within one dispatcher tick.
-Regression for a card that stayed ``running`` for 49 minutes after its managed worker died in
-the ``worker.adopt`` handshake (claim long expired, every recorded pid dead, the owner gateway
-alive and its admission settled ``terminal/interrupted``). These tests pin the disposition the
-dispatcher owes every lost attempt:
+Regression for a card left ``running`` after its managed worker died in the ``worker.adopt``
+handshake (claim expired, every recorded pid dead, the owner gateway alive and its admission
+settled ``terminal/interrupted``). These tests pin the disposition the dispatcher owes every
+lost attempt:
A1 a killed worker under a live claim is booked ``crashed`` within one tick;
A2 an elapsed ``claim_expires`` never leaves the card ``running`` after one tick, even
@@ -30,6 +30,7 @@ from hermes_cli import kanban_owner_recovery
NOW = int(time.time())
HOST = kb._claimer_id().split(":", 1)[0]
FINGERPRINT = "6a232414-2dbf-40ab-a534-d9b71db05658:116|4162240"
+CLAIM_LAPSED = -1800 # claim_expires_in for a claim that expired half an hour ago
def _absent_pid(base: int = 3_800_000) -> int:
@@ -70,6 +71,7 @@ class _Board:
self, tid: str = "t_lost", *, worker_pid=None, record_worker_pid: bool = True,
claim_expires_in: int = 900, heartbeat_age=None, owner: bool = True,
admission=("terminal", "interrupted"), worker_lost=None, dead_seconds: int = 120,
+ owner_pid=None,
) -> int:
"""A ``running`` card whose managed worker was lost."""
worker_pid = _absent_pid() if worker_pid is None else int(worker_pid)
@@ -89,13 +91,13 @@ class _Board:
FINGERPRINT if record_worker_pid else None,
(NOW - heartbeat_age) if heartbeat_age is not None else None, run_id, started),
)
- self.event(tid, run_id, "claimed", {"lock": lock, "expires": NOW - claim_expires_in * 0}, started)
+ self.event(tid, run_id, "claimed", {"lock": lock, "expires": NOW + claim_expires_in}, started)
self.event(tid, run_id, "spawned",
{"pid": worker_pid, "started_at": FINGERPRINT} if record_worker_pid else {}, started)
if owner:
request_id = f'kanban:["default","{tid}",{run_id}]'
self.event(tid, run_id, "owner_admitted",
- {"db": str(self.ledger), "session_id": "sess-1", "pid": _absent_pid(base=3_950_000),
+ {"db": str(self.ledger), "session_id": "sess-1", "pid": owner_pid or _absent_pid(base=3_950_000),
"request_id": request_id}, started + 3)
if admission:
with sqlite3.connect(self.ledger) as store:
@@ -172,11 +174,16 @@ def test_a_lost_worker_that_exited_cleanly_still_charges_the_breaker(board):
log = kb.worker_log_path("t_clean")
log.parent.mkdir(parents=True, exist_ok=True)
from hermes_cli.quiet_single_query import KANBAN_WORKER_EXIT_TRAILER
- log.write_text(f"{KANBAN_WORKER_EXIT_TRAILER}0\n", encoding="utf-8")
+ log.write_text(f"I fixed the protocol violation from the last run.\n{KANBAN_WORKER_EXIT_TRAILER}0\n",
+ encoding="utf-8")
board.tick(failure_limit=2)
run = board.run(run_id)
assert run["outcome"] == "crashed"
assert "managed worker lost" in run["error"]
+ assert run["error"].startswith("pid ") and "exited cleanly after its managed worker was lost." in run["error"]
+ assert "I fixed the protocol violation" in run["error"] # the worker's own output is kept
+ assert run["error"].index("[managed worker lost") < run["error"].index("Worker's last output")
+ assert "without kanban_complete" not in run["error"]
assert board.state("t_clean")["consecutive_failures"] == 1
# Not a protocol violation as well: one loss is charged to one budget.
assert "protocol_violation" not in json.loads(run["metadata"] or "{}")
@@ -188,36 +195,27 @@ def test_a_lost_worker_that_exited_cleanly_still_charges_the_breaker(board):
# --- A2: an elapsed claim lock is never "still running" after one tick ---------------------
- dict(record_worker_pid=False, owner=False), # no pid, no owner at all
- dict(record_worker_pid=False, owner=True), # no pid, settled receipt
- dict(owner=False), # dead pid, no owner
- dict(owner=True, worker_lost="managed_worker_failed"), # the incident shape
[email protected]("shape, error", [
+ (dict(record_worker_pid=False, owner=False), "stale_lock="),
+ (dict(record_worker_pid=False, owner=True), "owner admission settled interrupted"),
+ (dict(owner=False), "stale_lock="),
+ (dict(owner=True, worker_lost="managed_worker_failed"), "managed worker lost: managed_worker_failed"),
], ids=["no-pid", "no-pid-receipt", "dead-pid", "dead-pid-lost"])
-def test_an_elapsed_claim_never_stays_running_after_one_tick(board, shape):
- run_id = board.running("t_lapsed", claim_expires_in=-2067, heartbeat_age=None, **shape)
+def test_an_elapsed_claim_never_stays_running_after_one_tick(board, shape, error):
+ run_id = board.running("t_lapsed", claim_expires_in=CLAIM_LAPSED, heartbeat_age=None, **shape)
board.tick(failure_limit=2)
row, run = board.state("t_lapsed"), board.run(run_id)
- assert row["status"] != "running"
assert row["status"] == "ready"
- assert run["outcome"] in {"crashed", "reclaimed"}
- assert run["error"]
- assert row["consecutive_failures"] == 1
-
-
-def test_the_incident_shape_names_the_loss_on_the_reclaim(board):
- """The incident shape behind a settled receipt: the reclaim says what happened."""
- run_id = board.running("t_730", claim_expires_in=-2067, owner=True,
- worker_lost="managed_worker_failed")
- board.tick(failure_limit=2)
- run = board.run(run_id)
assert run["outcome"] == "reclaimed"
- assert "managed worker lost: managed_worker_failed" in run["error"]
+ assert error in run["error"]
+ assert row["consecutive_failures"] == 1
def test_a_settled_receipt_with_a_live_gateway_owner_no_longer_holds_the_claim(board):
"""The fence itself: a live owner PROCESS is not a live ATTEMPT."""
- board.running("t_gateway", claim_expires_in=-2067, owner=True, admission=("terminal", "interrupted"))
+ import os
+ board.running("t_gateway", claim_expires_in=CLAIM_LAPSED, owner=True, owner_pid=os.getpid(),
+ admission=("terminal", "interrupted"))
assert kanban_owner_recovery.owner_reclaim_paused(board.conn, "t_gateway") is True
assert kanban_owner_recovery.owner_attempt_provably_over(board.conn, "t_gateway") is True
board.tick(failure_limit=2)
@@ -226,16 +224,20 @@ def test_a_settled_receipt_with_a_live_gateway_owner_no_longer_holds_the_claim(b
# --- the reclaim's kill step: an absent pid is gone, not "survived" -----------------------
-def test_a_signalled_absent_pid_is_terminated_not_unproven(board):
- """The same reading through a signal seam: an OSError on a dead pid is a success."""
- gone = _absent_pid(base=3_970_000)
[email protected]("live", [False, True], ids=["absent-pid", "live-pid"])
+def test_a_refused_signal_reads_the_pid_not_the_error(board, live):
+ """A bare OSError from the kill (Windows reports an absent pid as error 87) is a success only
+ when the pid is really gone; a live worker that refuses the signal is not terminated."""
+ import os
+ pid = os.getpid() if live else _absent_pid(base=3_970_000)
- def windows_kill(pid, sig):
+ def refusing_kill(pid, sig):
raise OSError(87, "The parameter is incorrect")
- info = kbd._terminate_reclaimed_worker(gone, f"{HOST}:{_absent_pid(base=3_980_000)}",
- signal_fn=windows_kill)
- assert info["terminated"] is True
+ info = kbd._terminate_reclaimed_worker(pid, f"{HOST}:{_absent_pid(base=3_980_000)}",
+ started_at=kbd._process_fingerprint(pid) if live else None,
+ signal_fn=refusing_kill)
+ assert info["terminated"] is (not live)
# --- D1: the OWNER writes the loss where the dispatcher can read it -----------------------