Skip to content

The agent that grows with you

fix(kanban): re-check owner recovery inside the reclaim transaction

Tekniumbdbee23c1e99
diff --git a/hermes_cli/kanban_db.py b/hermes_cli/kanban_db.py
index 5db382dec89..e924027baa1 100644
--- a/hermes_cli/kanban_db.py
+++ b/hermes_cli/kanban_db.py
@@ -2398,6 +2398,10 @@ def release_stale_claims(
)
continue
with write_txn(conn):
+ # Again under the write lock: an interpreter that bound since the check above
+ # changes neither claim_lock nor worker_pid, so the CAS below cannot see it.
+ if owner_reclaim_blocked(conn, row["id"]):
+ continue
retry_status = _retry_status_for_run(conn, row["id"])
cur = conn.execute(
"UPDATE tasks SET status = ?, claim_lock = NULL, "
diff --git a/hermes_cli/kanban_db_dispatch.py b/hermes_cli/kanban_db_dispatch.py
index 2508d33559d..d4be600af9c 100644
--- a/hermes_cli/kanban_db_dispatch.py
+++ b/hermes_cli/kanban_db_dispatch.py
@@ -706,6 +706,10 @@ def enforce_max_runtime(conn: sqlite3.Connection, *, signal_fn=None) -> list[str
error = f"elapsed {int(elapsed)}s > limit {limit}s"
with _kb.write_txn(conn):
+ # Again under the write lock, after the kill wait: an interpreter that bound
+ # meanwhile is invisible to the pid/claim CAS below.
+ if owner_reclaim_blocked(conn, tid):
+ continue
retry_status = _kb._retry_status_for_run(conn, tid)
cur = conn.execute(
"UPDATE tasks SET status = ?, claim_lock = NULL, "
@@ -811,6 +815,9 @@ def detect_stale_running(
continue
with _kb.write_txn(conn):
+ # Again under the write lock, after the termination wait (see enforce_max_runtime).
+ if owner_reclaim_blocked(conn, tid):
+ continue
retry_status = _kb._retry_status_for_run(conn, tid)
cur = conn.execute(
"UPDATE tasks SET status = ?, claim_lock = NULL, "
@@ -880,6 +887,8 @@ def reconcile_orphaned_running(conn: sqlite3.Connection) -> list[str]:
)
continue
with _kb.write_txn(conn):
+ if owner_reclaim_blocked(conn, tid):
+ continue
cur = conn.execute(
"UPDATE tasks SET status = 'ready', claim_lock = NULL, "
"claim_expires = NULL, worker_pid = NULL, worker_started_at = NULL, "
diff --git a/tests/hermes_cli/test_kanban_owner_receipt_reclaim.py b/tests/hermes_cli/test_kanban_owner_receipt_reclaim.py
index 5af9921823a..c252117cb5c 100644
--- a/tests/hermes_cli/test_kanban_owner_receipt_reclaim.py
+++ b/tests/hermes_cli/test_kanban_owner_receipt_reclaim.py
@@ -31,6 +31,7 @@ from hermes_cli import kanban_db as kb
from hermes_cli import kanban_db_connect as kbc
from hermes_cli import kanban_db_dispatch as kbd
from hermes_cli import kanban_diagnostics as kd
+from hermes_cli import kanban_owner_recovery
from hermes_cli.kanban_owner_recovery import (
bound_interpreter_gone,
owner_attempt_provably_over,
@@ -194,3 +195,38 @@ def test_the_pause_still_holds_every_attempt_that_may_be_alive(board, shape):
assert kb.release_stale_claims(board.conn) == 0
assert kbd.detect_crashed_workers(board.conn) == []
assert board.state("t_ghost")["status"] == "running"
+
+
+# --- the gate is decided under the same write lock as the release ---------------------------
+
+_RECLAIM_PATHS = {
+ "ttl": lambda conn: kb.release_stale_claims(conn),
+ "stale": lambda conn: kbd.detect_stale_running(conn, stale_timeout_seconds=1, signal_fn=lambda *_: None),
+ "runtime": lambda conn: kbd.enforce_max_runtime(conn, signal_fn=lambda *_: None),
+}
+
+
[email protected]("path", sorted(_RECLAIM_PATHS))
+def test_an_interpreter_bound_after_the_gate_keeps_its_claim(board, monkeypatch, path):
+ """Binding changes neither claim_lock nor worker_pid, so only a re-check inside the
+ reclaim transaction can see an owner that recovered after the first look."""
+ run = board.dead_attempt("t_ghost")
+ board.conn.execute("UPDATE tasks SET max_runtime_seconds = 1 WHERE id = 't_ghost'")
+ board.conn.commit()
+ real_gate, calls = owner_reclaim_blocked, []
+
+ def gate_then_bind(conn, task_id, **kwargs):
+ verdict = real_gate(conn, task_id, **kwargs)
+ if not calls: # the recovered owner binds a live interpreter right after the first look
+ board.event(task_id, run, "worker_bound",
+ {"pid": os.getpid(), "started_at": kbd._process_fingerprint(os.getpid())}, NOW)
+ board.conn.commit()
+ calls.append(verdict)
+ return verdict
+
+ monkeypatch.setattr(kanban_owner_recovery, "owner_reclaim_blocked", gate_then_bind)
+ assert not _RECLAIM_PATHS[path](board.conn)
+ assert calls[0] is False and calls[-1] is True
+ row = board.state("t_ghost")
+ assert (row["status"], row["current_run_id"]) == ("running", run)
+