Skip to content

The agent that grows with you

fix(cron): a cancel reaches a run hidden behind a refused Run-now, and a stopped agent run reports why

Teknium7de83afa763e
diff --git a/cron/cancellation.py b/cron/cancellation.py
index 4c23e644768..5ea941eaadb 100644
--- a/cron/cancellation.py
+++ b/cron/cancellation.py
@@ -22,7 +22,7 @@ from typing import Any, Dict, Optional, Protocol
from hermes_constants import get_hermes_home
-from cron.executions import live_inflight_execution
+from cron.executions import cancellable_execution
# A marker whose run was SIGKILLed before it could clean up is unreadable garbage. Canceling a
# job is rare, so the prune rides along with the write instead of needing its own sweep.
@@ -183,7 +183,7 @@ def run_job_unless_cancelled(run_job, job, run_kwargs, teardown, fence, executio
def cancel_run(job_id: str, *, reason: Optional[str] = None) -> dict[str, Any]:
"""Request cancellation of the job's live run. A job with nothing in flight is a no-op."""
- record = live_inflight_execution(job_id)
+ record = cancellable_execution(job_id)
if record is None:
return {
"success": True,
diff --git a/cron/executions.py b/cron/executions.py
index 34bbe6de0e1..ca60622cca9 100644
--- a/cron/executions.py
+++ b/cron/executions.py
@@ -690,6 +690,22 @@ def live_inflight_execution(job_id: str) -> Optional[dict[str, Any]]:
return record
+def cancellable_execution(job_id: str) -> Optional[dict[str, Any]]:
+ """The job's newest claimed/running attempt under a LIVE owner, even when it is not the latest row.
+
+ A Run-now refused while a run is live books its own newer ``failed`` row ("Fire claim was not
+ acquired"), so ``live_inflight_execution`` (latest row only) would call the live run idle and a
+ cancel would never reach it.
+ """
+ with _transaction() as conn:
+ rows = conn.execute(
+ """SELECT * FROM executions WHERE job_id=? AND status IN ('claimed', 'running')
+ ORDER BY julianday(claimed_at) DESC, claimed_at DESC, id DESC""",
+ (job_id,),
+ ).fetchall()
+ return next((dict(row) for row in rows if _row_owner_live(row)), None)
+
+
def latest_executions(job_ids: list[str]) -> dict[str, dict[str, Any]]:
"""Load latest execution for many jobs in one query."""
clean = [str(job_id) for job_id in dict.fromkeys(job_ids) if job_id]
diff --git a/gateway/session_cron.py b/gateway/session_cron.py
index 46f174a923b..9df823ec503 100644
--- a/gateway/session_cron.py
+++ b/gateway/session_cron.py
@@ -296,10 +296,15 @@ async def execute(authority, ref, row, policy):
job = dict(data['cron_job'], execution_id=execution_id)
token = bind_execution(authority, ref.session_id, job['id'], execution_id)
try:
+ from cron.cancellation import CancelRequest, _CombinedCancelEvent
+
db_path = await asyncio.to_thread(Path(authority.db.db_path).resolve)
with _profile_runtime_scope(db_path.parent):
+ # The forwarded ``cron.cancel`` carries no reason, so the operator's marker is read here
+ # first: a stopped agent run reports "cancelled on request", not a lost fire claim.
+ stop = _CombinedCancelEvent(CancelRequest(execution_id), cancel)
result = await asyncio.to_thread(_run_identified, run_job, job, execution_id,
- extra_prompt=data['extra_prompt'], cancel_event=cancel)
+ extra_prompt=data['extra_prompt'], cancel_event=stop)
# run_job stamps run-side verdicts on ITS job copy (quota hold, unreachable model); the
# firing scheduler's bookkeeping reads them off its own dict, so they ride the result.
flags = {key: job[key] for key in RUN_JOB_FLAGS if key in job}
diff --git a/hermes_cli/web_routers/cron.py b/hermes_cli/web_routers/cron.py
index eb49f5b9a82..b7315d3bb5d 100644
--- a/hermes_cli/web_routers/cron.py
+++ b/hermes_cli/web_routers/cron.py
@@ -692,7 +692,9 @@ def _cancel_cron_job_sync(job_id: str, profile: Optional[str] = None):
"""Stop the job's live run; ``cancelled: false`` when nothing is in flight. Schedule untouched."""
selected = _job_profile(job_id, profile)
job = _found(_call_cron_for_profile(selected, "resolve_job_ref", job_id))
- return {**_call_cron_for_profile(selected, "cancel_job_run", job["id"]), "job_id": job["id"]}
+ # The reason lands in the run history row, so it names the surface that asked.
+ result = _call_cron_for_profile(selected, "cancel_job_run", job["id"], reason="cancelled on request (Stop)")
+ return {**result, "job_id": job["id"]}
def _delete_cron_job_sync(job_id: str, profile: Optional[str] = None):
diff --git a/tests/cron/test_cron_cancel.py b/tests/cron/test_cron_cancel.py
index 2795cd2794b..381fb035ca4 100644
--- a/tests/cron/test_cron_cancel.py
+++ b/tests/cron/test_cron_cancel.py
@@ -62,6 +62,21 @@ def test_cancel_run_targets_the_live_execution():
assert CancelRequest(record["id"]).is_set()
+def test_cancel_reaches_a_live_run_behind_a_refused_trigger():
+ """A Run-now refused while a run is live books a newer failed row; the live run stays cancellable."""
+ from cron.cancellation import cancel_run
+ from cron.executions import _transaction, create_execution, finish_execution
+
+ live = _claim("job-busy")
+ refused = create_execution("job-busy", source="test")
+ finish_execution(refused["id"], success=False, error="Fire claim was not acquired")
+ with _transaction() as conn: # the refused row is the job's newest attempt
+ conn.execute("UPDATE executions SET claimed_at='2999-01-01T00:00:00+00:00' WHERE id=?", (refused["id"],))
+
+ result = cancel_run("job-busy")
+ assert result["cancelled"] and result["execution_id"] == live["id"]
+
+
def test_cancel_is_a_noop_when_nothing_is_in_flight():
from cron.cancellation import cancel_run
@@ -142,7 +157,7 @@ def test_cancelling_a_script_job_kills_it_and_names_the_reason(tmp_path, monkeyp
while not ready.exists() and thread.is_alive() and time.monotonic() < deadline:
time.sleep(0.01)
assert ready.exists(), f"script must start before it can be cancelled (results={results})"
- pid = int(ready.read_text(encoding="utf-8"))
+ pid = int(ready.read_text(encoding="utf-8-sig"))
request_cancel("exec-live-script", reason="operator asked")
thread.join(timeout=20)
@@ -156,7 +171,7 @@ def test_cancelling_a_script_job_kills_it_and_names_the_reason(tmp_path, monkeyp
assert not _alive(pid), "the cancelled script must not outlive the cancel"
finally:
if ready.exists():
- pid = int(ready.read_text(encoding="utf-8"))
+ pid = int(ready.read_text(encoding="utf-8-sig"))
if _alive(pid):
os.kill(pid, 9)
thread.join(timeout=20)