The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(kanban): keep the worker's last output when a lost worker's clean exit is rebooked
kshitijk4poorc777ba429fc8
diff --git a/hermes_cli/kanban_db_dispatch.py b/hermes_cli/kanban_db_dispatch.pyindex 5585ace432b..5e5d8ff47ba 100644--- a/hermes_cli/kanban_db_dispatch.py+++ b/hermes_cli/kanban_db_dispatch.py@@ -482,7 +482,7 @@ def _terminate_reclaimed_worker(except OSError:# Already gone = successful termination (Windows reports an absent pid as a bare OSError, not# ProcessLookupError). Leaving terminated=False would make the reclaim defer forever.- info["terminated"] = not _kb._pid_alive(pid)+ info["terminated"] = not _worker_alive(pid, started_at)return infoif _poll_worker_exit(pid, started_at):@@ -960,10 +960,9 @@ def _protocol_violation_streak(conn: sqlite3.Connection, task_id: str) -> int:outcome = row["outcome"] or ""if outcome == "rate_limited":continue- if outcome == "crashed" and (- _kb._json_dict(row["metadata"]).get("protocol_violation")- or "protocol violation" in (row["error"] or "")- ):+ metadata = _kb._json_dict(row["metadata"]) # a lost worker's run quotes its output: no text match+ if outcome == "crashed" and not metadata.get("worker_lost") and (+ metadata.get("protocol_violation") or "protocol violation" in (row["error"] or "")):streak += 1continuebreakdiff --git a/hermes_cli/kanban_owner_recovery.py b/hermes_cli/kanban_owner_recovery.pyindex 5687b95ca61..59e30a604ad 100644--- a/hermes_cli/kanban_owner_recovery.py+++ b/hermes_cli/kanban_owner_recovery.py@@ -56,9 +56,7 @@ def owner_reclaim_paused(conn, task_id):return Truefrom hermes_cli.process_identity import worker_birthreturn worker_birth(psutil.Process(receipt['pid'])) != receipt['birth']- except (OSError, ValueError, KeyError, sqlite3.Error):- return True- except psutil.Error:+ except (OSError, ValueError, KeyError, TypeError, psutil.Error):return True@@ -186,16 +184,19 @@ def owner_attempt_provably_over(conn, task_id) -> bool:return settled or (bool(pids) and _owner_process_gone(receipt))-def owner_attempt_note(conn, task_id) -> str:- """One clause naming WHY a provably-over attempt ended, or ``""``.+def owner_attempt_note(conn, task_id, lost=None) -> str:+ """One clause naming why the owner says the attempt ended, or ``""``.- The reclaim paths record this beside the pid/claim evidence so the board says what- happened instead of a bare ``stale_lock=``: a managed worker that died (its owner- recorded ``worker_lost``) is a different fault from a worker that never started.+ The reclaim paths put this in the run error so the board says what happened instead of+ a bare ``stale_lock=``: a managed worker that died (its owner recorded ``worker_lost``,+ passed as ``lost`` when the caller already read it) is a different fault from a worker+ that never started."""- lost = str(_latest_run_event(conn, task_id, 'worker_lost').get('reason') or '')- if lost:- return f"managed worker lost: {lost[:80]}"+ if lost is None:+ lost = _latest_run_event(conn, task_id, 'worker_lost')+ reason = str(lost.get('reason') or '')+ if reason:+ return f"managed worker lost: {reason[:80]}"receipt = _owner_receipt(conn, task_id)if receipt is None:return ""@@ -214,25 +215,30 @@ def reclaim_error(conn, task_id, claim_lock) -> str:def apply_attempt_note(conn, task_id, pid, dead) -> None:"""Fold the owner's own record of how the attempt ended into ONE dead worker's booking.- A managed worker that dies in ``worker.adopt`` writes no ``worker_result``, so its rc=0- exit trailer reads as a protocol violation. The owner's ``worker_lost`` evidence outranks- it: the turn never finished, so it books as a crash and charges the breaker, and the+ A bound interpreter whose ``worker_result`` says exit 0 after its managed execution was+ lost reads as a protocol violation. The owner's ``worker_lost`` evidence outranks it: the+ turn never finished, so it books as a crash and charges the breaker, and theprotocol-violation streak (which reads the persisted payload) does not count it."""- note = owner_attempt_note(conn, task_id)+ lost = _latest_run_event(conn, task_id, "worker_lost")+ note = owner_attempt_note(conn, task_id, lost)if not note:return- dead.error_text = f"{dead.error_text} [{note}]"dead.event_payload["attempt_note"] = note- lost = _latest_run_event(conn, task_id, "worker_lost")if lost:dead.event_payload["worker_lost"] = lostif dead.protocol_violation:+ from hermes_cli.kanban_db_dispatch import _PROTOCOL_VIOLATION_ERRORdead.protocol_violation = Falsedead.event_payload.pop("protocol_violation", None)dead.event_payload["exit_kind"] = dead.kinddead.event_kind = "crashed"- dead.error_text = f"pid {pid} exited cleanly after its managed worker was lost [{note}]"+ # Keep the worker's last output that _classify_dead_worker appended.+ dead.error_text = (f"pid {pid} exited cleanly after its managed worker was lost."+ + dead.error_text.removeprefix(_PROTOCOL_VIOLATION_ERROR))+ # Before the worker's own output: the stored error is capped, and the output may be long.+ head, sep, output = dead.error_text.partition(" Worker's last output: ")+ dead.error_text = f"{head} [{note}]{sep}{output}"def owner_reclaim_blocked(conn, task_id, *, crash_sweep=False) -> bool:diff --git a/tests/hermes_cli/test_kanban_managed_worker_loss.py b/tests/hermes_cli/test_kanban_managed_worker_loss.pyindex ecf3578d231..de107e37ae9 100644--- a/tests/hermes_cli/test_kanban_managed_worker_loss.py+++ b/tests/hermes_cli/test_kanban_managed_worker_loss.py@@ -1,9 +1,9 @@"""A lost managed worker must be DISPOSED OF on the board, within one dispatcher tick.-Regression for a card that stayed ``running`` for 49 minutes after its managed worker died in-the ``worker.adopt`` handshake (claim long expired, every recorded pid dead, the owner gateway-alive and its admission settled ``terminal/interrupted``). These tests pin the disposition the-dispatcher owes every lost attempt:+Regression for a card left ``running`` after its managed worker died in the ``worker.adopt``+handshake (claim expired, every recorded pid dead, the owner gateway alive and its admission+settled ``terminal/interrupted``). These tests pin the disposition the dispatcher owes every+lost attempt:A1 a killed worker under a live claim is booked ``crashed`` within one tick;A2 an elapsed ``claim_expires`` never leaves the card ``running`` after one tick, even@@ -30,6 +30,7 @@ from hermes_cli import kanban_owner_recoveryNOW = int(time.time())HOST = kb._claimer_id().split(":", 1)[0]FINGERPRINT = "6a232414-2dbf-40ab-a534-d9b71db05658:116|4162240"+CLAIM_LAPSED = -1800 # claim_expires_in for a claim that expired half an hour agodef _absent_pid(base: int = 3_800_000) -> int:@@ -70,6 +71,7 @@ class _Board:self, tid: str = "t_lost", *, worker_pid=None, record_worker_pid: bool = True,claim_expires_in: int = 900, heartbeat_age=None, owner: bool = True,admission=("terminal", "interrupted"), worker_lost=None, dead_seconds: int = 120,+ owner_pid=None,) -> int:"""A ``running`` card whose managed worker was lost."""worker_pid = _absent_pid() if worker_pid is None else int(worker_pid)@@ -89,13 +91,13 @@ class _Board:FINGERPRINT if record_worker_pid else None,(NOW - heartbeat_age) if heartbeat_age is not None else None, run_id, started),)- self.event(tid, run_id, "claimed", {"lock": lock, "expires": NOW - claim_expires_in * 0}, started)+ self.event(tid, run_id, "claimed", {"lock": lock, "expires": NOW + claim_expires_in}, started)self.event(tid, run_id, "spawned",{"pid": worker_pid, "started_at": FINGERPRINT} if record_worker_pid else {}, started)if owner:request_id = f'kanban:["default","{tid}",{run_id}]'self.event(tid, run_id, "owner_admitted",- {"db": str(self.ledger), "session_id": "sess-1", "pid": _absent_pid(base=3_950_000),+ {"db": str(self.ledger), "session_id": "sess-1", "pid": owner_pid or _absent_pid(base=3_950_000),"request_id": request_id}, started + 3)if admission:with sqlite3.connect(self.ledger) as store:@@ -172,11 +174,16 @@ def test_a_lost_worker_that_exited_cleanly_still_charges_the_breaker(board):log = kb.worker_log_path("t_clean")log.parent.mkdir(parents=True, exist_ok=True)from hermes_cli.quiet_single_query import KANBAN_WORKER_EXIT_TRAILER- log.write_text(f"{KANBAN_WORKER_EXIT_TRAILER}0\n", encoding="utf-8")+ log.write_text(f"I fixed the protocol violation from the last run.\n{KANBAN_WORKER_EXIT_TRAILER}0\n",+ encoding="utf-8")board.tick(failure_limit=2)run = board.run(run_id)assert run["outcome"] == "crashed"assert "managed worker lost" in run["error"]+ assert run["error"].startswith("pid ") and "exited cleanly after its managed worker was lost." in run["error"]+ assert "I fixed the protocol violation" in run["error"] # the worker's own output is kept+ assert run["error"].index("[managed worker lost") < run["error"].index("Worker's last output")+ assert "without kanban_complete" not in run["error"]assert board.state("t_clean")["consecutive_failures"] == 1# Not a protocol violation as well: one loss is charged to one budget.assert "protocol_violation" not in json.loads(run["metadata"] or "{}")@@ -188,36 +195,27 @@ def test_a_lost_worker_that_exited_cleanly_still_charges_the_breaker(board):# --- A2: an elapsed claim lock is never "still running" after one tick ---------------------[email protected]("shape", [- dict(record_worker_pid=False, owner=False), # no pid, no owner at all- dict(record_worker_pid=False, owner=True), # no pid, settled receipt- dict(owner=False), # dead pid, no owner- dict(owner=True, worker_lost="managed_worker_failed"), # the incident shape[email protected]("shape, error", [+ (dict(record_worker_pid=False, owner=False), "stale_lock="),+ (dict(record_worker_pid=False, owner=True), "owner admission settled interrupted"),+ (dict(owner=False), "stale_lock="),+ (dict(owner=True, worker_lost="managed_worker_failed"), "managed worker lost: managed_worker_failed"),], ids=["no-pid", "no-pid-receipt", "dead-pid", "dead-pid-lost"])-def test_an_elapsed_claim_never_stays_running_after_one_tick(board, shape):- run_id = board.running("t_lapsed", claim_expires_in=-2067, heartbeat_age=None, **shape)+def test_an_elapsed_claim_never_stays_running_after_one_tick(board, shape, error):+ run_id = board.running("t_lapsed", claim_expires_in=CLAIM_LAPSED, heartbeat_age=None, **shape)board.tick(failure_limit=2)row, run = board.state("t_lapsed"), board.run(run_id)- assert row["status"] != "running"assert row["status"] == "ready"- assert run["outcome"] in {"crashed", "reclaimed"}- assert run["error"]- assert row["consecutive_failures"] == 1---def test_the_incident_shape_names_the_loss_on_the_reclaim(board):- """The incident shape behind a settled receipt: the reclaim says what happened."""- run_id = board.running("t_730", claim_expires_in=-2067, owner=True,- worker_lost="managed_worker_failed")- board.tick(failure_limit=2)- run = board.run(run_id)assert run["outcome"] == "reclaimed"- assert "managed worker lost: managed_worker_failed" in run["error"]+ assert error in run["error"]+ assert row["consecutive_failures"] == 1def test_a_settled_receipt_with_a_live_gateway_owner_no_longer_holds_the_claim(board):"""The fence itself: a live owner PROCESS is not a live ATTEMPT."""- board.running("t_gateway", claim_expires_in=-2067, owner=True, admission=("terminal", "interrupted"))+ import os+ board.running("t_gateway", claim_expires_in=CLAIM_LAPSED, owner=True, owner_pid=os.getpid(),+ admission=("terminal", "interrupted"))assert kanban_owner_recovery.owner_reclaim_paused(board.conn, "t_gateway") is Trueassert kanban_owner_recovery.owner_attempt_provably_over(board.conn, "t_gateway") is Trueboard.tick(failure_limit=2)@@ -226,16 +224,20 @@ def test_a_settled_receipt_with_a_live_gateway_owner_no_longer_holds_the_claim(b# --- the reclaim's kill step: an absent pid is gone, not "survived" ------------------------def test_a_signalled_absent_pid_is_terminated_not_unproven(board):- """The same reading through a signal seam: an OSError on a dead pid is a success."""- gone = _absent_pid(base=3_970_000)[email protected]("live", [False, True], ids=["absent-pid", "live-pid"])+def test_a_refused_signal_reads_the_pid_not_the_error(board, live):+ """A bare OSError from the kill (Windows reports an absent pid as error 87) is a success only+ when the pid is really gone; a live worker that refuses the signal is not terminated."""+ import os+ pid = os.getpid() if live else _absent_pid(base=3_970_000)- def windows_kill(pid, sig):+ def refusing_kill(pid, sig):raise OSError(87, "The parameter is incorrect")- info = kbd._terminate_reclaimed_worker(gone, f"{HOST}:{_absent_pid(base=3_980_000)}",- signal_fn=windows_kill)- assert info["terminated"] is True+ info = kbd._terminate_reclaimed_worker(pid, f"{HOST}:{_absent_pid(base=3_980_000)}",+ started_at=kbd._process_fingerprint(pid) if live else None,+ signal_fn=refusing_kill)+ assert info["terminated"] is (not live)# --- D1: the OWNER writes the loss where the dispatcher can read it -----------------------