The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(kanban): re-check owner recovery inside the reclaim transaction
Tekniumbdbee23c1e99
diff --git a/hermes_cli/kanban_db.py b/hermes_cli/kanban_db.pyindex 5db382dec89..e924027baa1 100644--- a/hermes_cli/kanban_db.py+++ b/hermes_cli/kanban_db.py@@ -2398,6 +2398,10 @@ def release_stale_claims()continuewith write_txn(conn):+ # Again under the write lock: an interpreter that bound since the check above+ # changes neither claim_lock nor worker_pid, so the CAS below cannot see it.+ if owner_reclaim_blocked(conn, row["id"]):+ continueretry_status = _retry_status_for_run(conn, row["id"])cur = conn.execute("UPDATE tasks SET status = ?, claim_lock = NULL, "diff --git a/hermes_cli/kanban_db_dispatch.py b/hermes_cli/kanban_db_dispatch.pyindex 2508d33559d..d4be600af9c 100644--- a/hermes_cli/kanban_db_dispatch.py+++ b/hermes_cli/kanban_db_dispatch.py@@ -706,6 +706,10 @@ def enforce_max_runtime(conn: sqlite3.Connection, *, signal_fn=None) -> list[strerror = f"elapsed {int(elapsed)}s > limit {limit}s"with _kb.write_txn(conn):+ # Again under the write lock, after the kill wait: an interpreter that bound+ # meanwhile is invisible to the pid/claim CAS below.+ if owner_reclaim_blocked(conn, tid):+ continueretry_status = _kb._retry_status_for_run(conn, tid)cur = conn.execute("UPDATE tasks SET status = ?, claim_lock = NULL, "@@ -811,6 +815,9 @@ def detect_stale_running(continuewith _kb.write_txn(conn):+ # Again under the write lock, after the termination wait (see enforce_max_runtime).+ if owner_reclaim_blocked(conn, tid):+ continueretry_status = _kb._retry_status_for_run(conn, tid)cur = conn.execute("UPDATE tasks SET status = ?, claim_lock = NULL, "@@ -880,6 +887,8 @@ def reconcile_orphaned_running(conn: sqlite3.Connection) -> list[str]:)continuewith _kb.write_txn(conn):+ if owner_reclaim_blocked(conn, tid):+ continuecur = conn.execute("UPDATE tasks SET status = 'ready', claim_lock = NULL, ""claim_expires = NULL, worker_pid = NULL, worker_started_at = NULL, "diff --git a/tests/hermes_cli/test_kanban_owner_receipt_reclaim.py b/tests/hermes_cli/test_kanban_owner_receipt_reclaim.pyindex 5af9921823a..c252117cb5c 100644--- a/tests/hermes_cli/test_kanban_owner_receipt_reclaim.py+++ b/tests/hermes_cli/test_kanban_owner_receipt_reclaim.py@@ -31,6 +31,7 @@ from hermes_cli import kanban_db as kbfrom hermes_cli import kanban_db_connect as kbcfrom hermes_cli import kanban_db_dispatch as kbdfrom hermes_cli import kanban_diagnostics as kd+from hermes_cli import kanban_owner_recoveryfrom hermes_cli.kanban_owner_recovery import (bound_interpreter_gone,owner_attempt_provably_over,@@ -194,3 +195,38 @@ def test_the_pause_still_holds_every_attempt_that_may_be_alive(board, shape):assert kb.release_stale_claims(board.conn) == 0assert kbd.detect_crashed_workers(board.conn) == []assert board.state("t_ghost")["status"] == "running"+++# --- the gate is decided under the same write lock as the release ---------------------------++_RECLAIM_PATHS = {+ "ttl": lambda conn: kb.release_stale_claims(conn),+ "stale": lambda conn: kbd.detect_stale_running(conn, stale_timeout_seconds=1, signal_fn=lambda *_: None),+ "runtime": lambda conn: kbd.enforce_max_runtime(conn, signal_fn=lambda *_: None),+}++[email protected]("path", sorted(_RECLAIM_PATHS))+def test_an_interpreter_bound_after_the_gate_keeps_its_claim(board, monkeypatch, path):+ """Binding changes neither claim_lock nor worker_pid, so only a re-check inside the+ reclaim transaction can see an owner that recovered after the first look."""+ run = board.dead_attempt("t_ghost")+ board.conn.execute("UPDATE tasks SET max_runtime_seconds = 1 WHERE id = 't_ghost'")+ board.conn.commit()+ real_gate, calls = owner_reclaim_blocked, []++ def gate_then_bind(conn, task_id, **kwargs):+ verdict = real_gate(conn, task_id, **kwargs)+ if not calls: # the recovered owner binds a live interpreter right after the first look+ board.event(task_id, run, "worker_bound",+ {"pid": os.getpid(), "started_at": kbd._process_fingerprint(os.getpid())}, NOW)+ board.conn.commit()+ calls.append(verdict)+ return verdict++ monkeypatch.setattr(kanban_owner_recovery, "owner_reclaim_blocked", gate_then_bind)+ assert not _RECLAIM_PATHS[path](board.conn)+ assert calls[0] is False and calls[-1] is True+ row = board.state("t_ghost")+ assert (row["status"], row["current_run_id"]) == ("running", run)+