Skip to content

The agent that grows with you

feat(cron): a stopped or removed run books 'cancelled'; each job picks its missed-run policy

Tekniumee394bd9213b
diff --git a/cron/cancellation.py b/cron/cancellation.py
index c7478299e31..a15be3352e0 100644
--- a/cron/cancellation.py
+++ b/cron/cancellation.py
@@ -18,7 +18,7 @@ import contextlib
import os
import time
from pathlib import Path
-from typing import Any, Dict, Optional
+from typing import Any, Dict, Optional, Protocol
from hermes_constants import get_hermes_home
@@ -81,6 +81,38 @@ def clear_cancel(execution_id: str) -> None:
_marker_path(execution_id).unlink(missing_ok=True)
+class _CancelEventLike(Protocol):
+ """Structural type for cancellation sources (``threading.Event``, ``_CombinedCancelEvent``)."""
+
+ def is_set(self) -> bool: ...
+ def set(self) -> None: ...
+
+
+class _CombinedCancelEvent:
+ """Duck-typed ``threading.Event`` ORing several cancellation sources (fire-claim heartbeat
+ ``lost_ownership`` + per-transport events). Workers only call is_set()/set(), so no pump thread.
+ """
+
+ def __init__(self, *events: Optional[_CancelEventLike]) -> None:
+ self._events = [event for event in events if event is not None]
+
+ def is_set(self) -> bool:
+ return any(event.is_set() for event in self._events)
+
+ def set(self) -> None:
+ for event in self._events:
+ event.set()
+
+ @property
+ def reason(self) -> Optional[str]:
+ """Why the run is being torn down, from whichever source fired — so a script or agent
+ stop never reports "fire claim lost" for an operator cancel (or the reverse)."""
+ for event in self._events:
+ if event.is_set():
+ return getattr(event, "reason", None)
+ return None
+
+
class CancelRequest:
"""``_CancelEventLike`` view of one execution's cancel marker.
@@ -109,7 +141,26 @@ class CancelRequest:
return self._reason
-def cancel_run(job_id: str, *, reason: Optional[str] = None) -> Dict[str, Any]:
+def cancel_reason(cancel_event: Any) -> str:
+ """Human-facing cause for a cancelled run; the legacy wording when no source carries one."""
+ return getattr(cancel_event, "reason", None) or "cron fire ownership was lost"
+
+
+def finish_cancelled_run(job: dict[str, Any], execution_id: str, fire_owner: Optional[str]) -> bool:
+ """Book an operator-cancelled run: ledger row ``cancelled`` with the recorded reason, and the
+ job's ``last_status`` — without a failure alert or a failure-streak bump, since the operator
+ asked for it. A job removed mid-run has no record left to mark; its ledger row still lands."""
+ from cron import scheduler
+
+ reason = CancelRequest(execution_id).reason
+ if not scheduler.self_removal_delivery_allowed(job["id"]):
+ owner = {"expected_fire_owner": fire_owner} if fire_owner is not None else {}
+ scheduler.mark_job_run(job["id"], False, f"Cancelled: {reason}", status="cancelled", **owner)
+ scheduler.finish_execution(execution_id, success=False, error=reason, cancelled=True)
+ return True
+
+
+def cancel_run(job_id: str, *, reason: Optional[str] = None) -> dict[str, Any]:
"""Request cancellation of the job's live run. A job with nothing in flight is a no-op."""
record = live_inflight_execution(job_id)
if record is None:
diff --git a/cron/executions.py b/cron/executions.py
index a3dfc01f0dc..34bbe6de0e1 100644
--- a/cron/executions.py
+++ b/cron/executions.py
@@ -34,7 +34,10 @@ MAX_TERMINAL_EXECUTIONS = 1000
HANDOFF_ADOPTION_GRACE_SECONDS = 30.0
# Floor for the live-owner stale-claim bound (#115692); see _live_owner_stale_after_seconds.
LIVE_OWNER_STALE_CLAIM_FLOOR_SECONDS = 7200.0
-_TERMINAL_STATES = ("completed", "failed", "unknown")
+# ``cancelled``: an operator stopped the run (``hermes cron cancel``, removing the job mid-run).
+# ``missed``: never started — one row books every occurrence the scheduler skipped past.
+_TERMINAL_STATES = ("completed", "failed", "unknown", "cancelled", "missed")
+_STATUS_CHECK = "('claimed','running','completed','failed','unknown','cancelled','missed')"
_lock = threading.RLock()
_PROCESS_ID = uuid.uuid4().hex
@@ -57,7 +60,7 @@ def _initialize_schema(conn: sqlite3.Connection) -> None:
from hermes_cli.sqlite_util import add_column_if_missing
conn.execute(
- """CREATE TABLE IF NOT EXISTS executions (
+ f"""CREATE TABLE IF NOT EXISTS executions (
id TEXT PRIMARY KEY,
job_id TEXT NOT NULL,
source TEXT NOT NULL,
@@ -65,7 +68,7 @@ def _initialize_schema(conn: sqlite3.Connection) -> None:
pid INTEGER NOT NULL,
process_started_at INTEGER,
status TEXT NOT NULL CHECK(status IN
- ('claimed','running','completed','failed','unknown')),
+ {_STATUS_CHECK}),
handoff_pending INTEGER NOT NULL DEFAULT 0,
handoff_started_at REAL,
claimed_at TEXT NOT NULL,
@@ -74,6 +77,7 @@ def _initialize_schema(conn: sqlite3.Connection) -> None:
error TEXT
)"""
)
+ _widen_status_check(conn)
add_column_if_missing(
conn, "executions", "handoff_pending",
"handoff_pending INTEGER NOT NULL DEFAULT 0",
@@ -98,6 +102,34 @@ def _initialize_schema(conn: sqlite3.Connection) -> None:
)
+def _widen_status_check(conn: sqlite3.Connection) -> None:
+ """Rebuild a ledger created before ``cancelled``/``missed`` existed: SQLite cannot alter a
+ CHECK constraint in place. Copies every column the old table has; runs once per file, and
+ re-reads the schema under the write lock so two processes opening it together rebuild once."""
+ def current_sql() -> Optional[str]:
+ row = conn.execute(
+ "SELECT sql FROM sqlite_master WHERE type='table' AND name='executions'").fetchone()
+ return row[0] if row is not None else None
+
+ if "'missed'" in (current_sql() or "'missed'"):
+ return
+ conn.commit()
+ conn.execute("BEGIN IMMEDIATE")
+ try:
+ old_sql = current_sql() or "'missed'"
+ if "'missed'" not in old_sql:
+ columns = ", ".join(r[1] for r in conn.execute("PRAGMA table_info(executions)"))
+ conn.execute("ALTER TABLE executions RENAME TO executions_pre_widen")
+ conn.execute(old_sql.replace("'unknown')", "'unknown','cancelled','missed')", 1))
+ conn.execute(
+ f"INSERT INTO executions ({columns}) SELECT {columns} FROM executions_pre_widen")
+ conn.execute("DROP TABLE executions_pre_widen")
+ conn.commit()
+ except BaseException:
+ conn.rollback()
+ raise
+
+
@contextmanager
def _transaction() -> Iterator[sqlite3.Connection]:
from hermes_cli.sqlite_util import transaction
@@ -202,7 +234,7 @@ def _prune_unlocked(conn: sqlite3.Connection) -> None:
conn.execute(
"""DELETE FROM executions WHERE id IN (
SELECT id FROM executions
- WHERE status IN ('completed','failed','unknown')
+ WHERE status IN ('completed','failed','unknown','cancelled','missed')
ORDER BY julianday(finished_at) DESC, finished_at DESC,
julianday(claimed_at) DESC, claimed_at DESC, id DESC LIMIT -1 OFFSET ?
)""",
@@ -311,7 +343,7 @@ def mark_execution_running(execution_id: str) -> Optional[dict[str, Any]]:
def finish_execution(
execution_id: str, *, success: bool, error: Optional[str] = None,
- delivery_outcome: Optional[str] = None,
+ delivery_outcome: Optional[str] = None, cancelled: bool = False,
) -> Optional[dict[str, Any]]:
"""Write a terminal result once; terminal attempts cannot be rewritten.
@@ -319,7 +351,7 @@ def finish_execution(
settles through ``recover_receipted_execution``.
"""
now = _hermes_now().isoformat()
- status = "completed" if success else "failed"
+ status = "cancelled" if cancelled else ("completed" if success else "failed")
detail = None if success else (str(error) if error else "unknown failure")
with _transaction() as conn:
owner = (_PROCESS_ID, os.getpid())
@@ -340,6 +372,34 @@ def finish_execution(
return record
+def record_missed_occurrences(
+ job_id: str, *, count: int, first_instant: Optional[str], detail: str,
+) -> Optional[dict[str, Any]]:
+ """Book occurrences the scheduler passed over without running as ONE terminal ``missed`` row
+ (``missed ×N``), so run history shows the gap instead of nothing. Never started, so no
+ owner fencing applies; ``first_instant`` is the earliest skipped slot."""
+ from cron.occurrences import scheduled_instant as canonical_instant
+
+ if count < 1:
+ return None
+ now = _hermes_now().isoformat()
+ execution_id = uuid.uuid4().hex
+ pid = os.getpid()
+ with _transaction() as conn:
+ conn.execute(
+ """INSERT INTO executions
+ (id, job_id, source, process_id, pid, process_started_at,
+ status, claimed_at, finished_at, error, scheduled_instant)
+ VALUES (?, ?, 'scheduler', ?, ?, ?, 'missed', ?, ?, ?, ?)""",
+ (execution_id, str(job_id), _PROCESS_ID, pid, _process_start_time(pid), now, now,
+ f"missed \u00d7{count} ({detail})", canonical_instant(first_instant)),
+ )
+ _prune_unlocked(conn)
+ record = _fetch(conn, execution_id)
+ _emit_execution_state(record)
+ return record
+
+
def recover_receipted_execution(execution_id, job_id, *, success, error=None, delivery_outcome=None):
"""Settle a verified canonical receipt after owner loss, including its unknown audit row.
diff --git a/cron/jobs.py b/cron/jobs.py
index 0357623a42b..5d5e5d30f0d 100644
--- a/cron/jobs.py
+++ b/cron/jobs.py
@@ -1744,6 +1744,13 @@ def _normalize_reasoning_effort(value: Any) -> Optional[str]:
# Normalizers for create_job (all fields) / update_job (present fields). Invalid values raise BEFORE
# storing.
+def _normalize_catch_up(value: Any) -> Optional[bool]:
+ """``catch_up``: True runs one catch-up for missed slots, False skips them, None = default."""
+ if value is None or isinstance(value, bool):
+ return value
+ raise ValueError("catch_up must be true, false, or null (use the default).")
+
+
_CREATE_FIELD_NORMALIZERS: dict[str, Callable[[Any], Any]] = {
"model": _normalize_job_optional_text,
"provider": _normalize_job_optional_text,
@@ -1759,6 +1766,7 @@ _CREATE_FIELD_NORMALIZERS: dict[str, Callable[[Any], Any]] = {
"interpreter": _normalize_job_optional_text,
"timezone": jobs_time.normalize_timezone,
"grace_minutes": jobs_time.normalize_grace_minutes,
+ "catch_up": _normalize_catch_up,
}
_UPDATE_FIELD_NORMALIZERS: dict[str, Callable[[Any], Any]] = {
# [] is an explicit zero-tool allowlist and must survive the update path as [] too (#82010).
@@ -1772,6 +1780,7 @@ _UPDATE_FIELD_NORMALIZERS: dict[str, Callable[[Any], Any]] = {
# ``in {None, "", False}`` test — ``0 == False`` would silently clear it.
"timezone": jobs_time.normalize_timezone,
"grace_minutes": jobs_time.normalize_grace_minutes,
+ "catch_up": _normalize_catch_up,
}
@@ -1819,6 +1828,7 @@ def create_job(
monitor_url: Optional[str] = None,
reasoning_effort: Optional[str] = None,
failure_deliver: Optional[str] = None,
+ catch_up: Optional[bool] = None,
paused: bool = False,
paused_reason: Optional[str] = None,
pinned: bool = False,
@@ -1928,6 +1938,7 @@ def create_job(
("attach_to_session", normalized_attach), ("reasoning_effort", normalized_reasoning_effort),
("failure_deliver", f["failure_deliver"]), ("interpreter", f["interpreter"]),
("timezone", f["timezone"]), ("grace_minutes", f["grace_minutes"]),
+ ("catch_up", f["catch_up"]),
):
if value is not None:
job[key] = value
@@ -2309,6 +2320,8 @@ def remove_job(job_id: str) -> bool:
marker = _self_removal_delivery.get()
if marker is not None and marker.job_id == canonical_id:
marker.removed = True
+ else: # someone else removed it: its live run must stop, not burn tokens to a discard
+ cancel_job_run(canonical_id, reason="job removed while running")
if job_output_dir.exists():
shutil.rmtree(job_output_dir)
try:
@@ -2323,6 +2336,13 @@ def remove_job(job_id: str) -> bool:
return True
+def cancel_job_run(job_id: str, reason: Optional[str] = None) -> dict[str, Any]:
+ """Stop the job's live run (if any); its schedule is untouched. See ``cron.cancellation``."""
+ from cron.cancellation import cancel_run
+
+ return cancel_run(job_id, reason=reason)
+
+
def _set_alert_flag(job_id: str, field: str, value: bool) -> bool:
"""Set/clear a persisted alert-dedup marker (alert exactly once until the condition heals;
survives restarts) and return the PRIOR value. Field: ``preflight_alerted`` (blocked config).
@@ -2381,14 +2401,15 @@ def _record_run_outcome(
delivery_failed = isinstance(delivery_error, str) and bool(delivery_error.strip())
job["last_status"] = status or (
"error" if not success else ("delivery_failed" if delivery_failed else "ok"))
- job["last_error"] = None if success else error
+ cancelled = status == "cancelled" # an operator stop is not a failure
+ job["last_error"] = None if success or cancelled else error
if success:
# Healthy run: drop the alert-once dedup markers so a FUTURE break re-alerts, and clear
# the forward-failure stamp so it only describes CURRENT auto-fire health.
job.pop("preflight_alerted", None)
job.pop("last_fire_error", None)
job["failure_streak"] = 0
- else:
+ elif not cancelled:
# Consecutive agent-failure streak; delivery failures do NOT count
# (scheduler._failure_streak_nudge).
job["failure_streak"] = int(job.get("failure_streak") or 0) + 1
@@ -3085,64 +3106,6 @@ def _reanchor_stale_cron(d: _DueJob) -> bool:
return False
-def _fast_forward_missed_recurring(d: _DueJob, grace: int) -> bool:
- """Re-anchor accumulated misses; return whether catch-up was explicitly disabled.
-
- The fast-forward is persisted immediately — NOT redundant with advance_next_run/mark_job_run:
- it
- protects the crash window before mark_job_run and covers the external fire_due path, which never
- calls advance_next_run. mark_job_run re-anchors on completion, so the value is provisional.
- """
- if _elapsed_seconds(d.scan.now, d.next_run_dt) <= grace:
- return False
- new_next = d.recompute_next()
- if not new_next:
- return False
- d.scan.persist(d.job["id"], next_run_at=new_next)
- # An explicit per-job grace_minutes is the user's catch-up limit for THIS job: past it the
- # miss is skipped even while the global cron.catch_up_missed default still runs it once.
- if (_instant_after(_ensure_aware(datetime.fromisoformat(new_next)), d.scan.now)
- and (jobs_time.grace_seconds_override(d.job.get("grace_minutes")) is not None
- or not _cron_config_number("catch_up_missed", True, lambda value: value is not False))):
- logger.info(
- "Job '%s' missed its scheduled time (%s, grace=%ds). "
- "Skipping missed occurrence (grace_minutes or cron.catch_up_missed=false); next run: %s",
- d.label, d.next_run, grace, new_next)
- d.scan.on_saved.append(lambda job=d.job: record_cron_missed(job))
- return True
- logger.info(
- "Job '%s' missed its scheduled time (%s, grace=%ds). "
- "Running now; next run provisionally set to: %s (re-anchored on completion)",
- d.label, d.next_run, grace, new_next)
- from cron.occurrences import record_catch_up_occurrence
- d.scan.on_saved.append(record_catch_up_occurrence)
- return False
-
-
-def _retire_expired_oneshot(d: _DueJob) -> bool:
- """One-shot grace gate; True when the job must not fire this tick.
-
- A one-shot beyond the grace window must never fire (create/update/resume reject such schedules
- and recovery never revives them; only the due scan used to dispatch them hours late). With no
- claim stamped, retire it with a diagnostic (never silently delete). A claim may mean a run is
- still in flight elsewhere — skip but keep the record so its mark_job_run can land."""
- if _elapsed_seconds(d.scan.now, d.next_run_dt) <= ONESHOT_GRACE_SECONDS:
- return False
- # Due during an outage of its store: it was skipped, not missed; fire it once on recovery.
- if store_health.outage_covers(_current_cron_store().cron_dir, d.next_run_dt.timestamp(), ONESHOT_GRACE_SECONDS):
- return False
- claim = d.job.get("run_claim")
- # An outage-covered claim that never reached its fire claim never started: it stays covered.
- if isinstance(claim, dict) and claim.get("outage") and not d.job.get("fire_claim"):
- return False
- if not (claim or d.job.get("fire_claim")):
- from cron.jobs_oneshot_diagnostics import write_missed
- write_missed(d.job, d.next_run)
- d.scan.retire(d.job["id"])
- d.scan.on_saved.append(lambda job=d.job: record_cron_missed(job))
- return True
-
-
def _oneshot_dispatch_limit_reached(job: dict[str, Any], scan: _DueScan) -> bool:
"""One-shot dispatch-limit guard; True when the job must not fire this tick.
@@ -3261,10 +3224,12 @@ def _evaluate_due_job(job: dict[str, Any], scan: _DueScan, run_claim_ttl: float)
if not manual_run and kind == "cron" and _reanchor_stale_cron(d):
return False
grace = _compute_grace_seconds(d.schedule, job.get("grace_minutes"))
- if not manual_run and recurring and _fast_forward_missed_recurring(d, grace):
+ from cron.missed_runs import fast_forward_missed_recurring, retire_expired_oneshot
+
+ if not manual_run and recurring and fast_forward_missed_recurring(d, grace):
return False
if kind == "once":
- if _retire_expired_oneshot(d) or _oneshot_dispatch_limit_reached(job, scan):
+ if retire_expired_oneshot(d) or _oneshot_dispatch_limit_reached(job, scan):
return False
# Durably claim the one-shot for the DURATION of its run: a second scheduler process on the
# same HERMES_HOME must not re-dispatch it while in flight, and advancing next_run_at by a
diff --git a/cron/missed_runs.py b/cron/missed_runs.py
new file mode 100644
index 00000000000..471dd16e8b4
--- /dev/null
+++ b/cron/missed_runs.py
@@ -0,0 +1,142 @@
+"""What happens to occurrences a job could not fire on time (machine off, gateway stopped).
+
+Each job may carry ``catch_up``: ``true`` runs ONE catch-up for everything it missed, ``false``
+skips to the next slot. Absent, recurring jobs follow ``cron.catch_up_missed`` (default true)
+and one-shots keep the 120s grace rule (#93526: a stale reminder is skipped unless the job opts
+in). Either way the skipped or collapsed slots are booked as one ``missed ×N`` row in the
+execution ledger, so run history shows the gap instead of nothing.
+
+Called from the due scan in ``cron/jobs.py`` under the store lock; ledger writes ride the
+scan's ``on_saved`` hooks so an unwritable store cannot book the same miss every tick.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime
+from typing import Any, Optional
+
+logger = logging.getLogger("cron.jobs")
+
+# Counting walks the schedule slot by slot; a once-a-minute job off for a month stops here.
+_MAX_COUNTED_SLOTS = 1000
+
+
+def job_catches_up(job: dict[str, Any]) -> bool:
+ """The job's own ``catch_up`` wins; otherwise one-shots skip, and recurring jobs skip past an
+ explicit ``grace_minutes`` or else follow ``cron.catch_up_missed``."""
+ value = job.get("catch_up")
+ if isinstance(value, bool):
+ return value
+ if (job.get("schedule") or {}).get("kind") == "once":
+ return False
+ from cron import jobs_time
+ from cron.jobs import _cron_config_number
+
+ # An explicit grace_minutes is the user's catch-up limit for THIS job: past it the miss is
+ # skipped even while the global cron.catch_up_missed default still runs it once.
+ if jobs_time.grace_seconds_override(job.get("grace_minutes")) is not None:
+ return False
+
+ return bool(_cron_config_number("catch_up_missed", True, lambda raw: raw is not False))
+
+
+def count_missed_slots(schedule: dict[str, Any], first: str, now: datetime) -> int:
+ """Occurrences from ``first`` up to ``now`` inclusive, capped at ``_MAX_COUNTED_SLOTS``."""
+ from cron.jobs import _instant_after, _parse_aware, compute_next_run
+
+ count, cursor = 0, first
+ while cursor and count < _MAX_COUNTED_SLOTS:
+ at = _parse_aware(cursor)
+ if at is None or _instant_after(at, now):
+ break
+ count += 1
+ following = compute_next_run(schedule, cursor)
+ cursor = following if following != cursor else None
+ return count
+
+
+def _book_missed(job: dict[str, Any], count: int, first: Optional[str], detail: str) -> None:
+ from cron.executions import record_missed_occurrences
+
+ if count >= _MAX_COUNTED_SLOTS:
+ detail = f"{detail}; counting stopped at {_MAX_COUNTED_SLOTS}"
+ try:
+ record_missed_occurrences(str(job.get("id")), count=count, first_instant=first, detail=detail)
+ except Exception:
+ logger.warning("Job '%s': could not book its missed occurrences", job.get("id"), exc_info=True)
+
+
+def fast_forward_missed_recurring(d: Any, grace: int) -> bool:
+ """Re-anchor accumulated misses; return whether catch-up is off for this job (skip the fire).
+
+ The fast-forward is persisted immediately — NOT redundant with advance_next_run/mark_job_run:
+ it protects the crash window before mark_job_run and covers the external fire_due path, which
+ never calls advance_next_run. mark_job_run re-anchors on completion, so the value is
+ provisional.
+ """
+ from cron.jobs import _elapsed_seconds, _ensure_aware, _instant_after, record_cron_missed
+
+ if _elapsed_seconds(d.scan.now, d.next_run_dt) <= grace:
+ return False
+ new_next = d.recompute_next()
+ if not new_next:
+ return False
+ d.scan.persist(d.job["id"], next_run_at=new_next)
+ missed = count_missed_slots(d.schedule, d.next_run, d.scan.now)
+ if (_instant_after(_ensure_aware(datetime.fromisoformat(new_next)), d.scan.now)
+ and not job_catches_up(d.job)):
+ logger.info(
+ "Job '%s' missed its scheduled time (%s, grace=%ds). "
+ "Skipping %d missed occurrence(s) because catch-up is off for this job; next run: %s",
+ d.label, d.next_run, grace, missed, new_next)
+ d.scan.on_saved.append(lambda job=d.job: record_cron_missed(job))
+ d.scan.on_saved.append(lambda job=d.job, first=d.next_run: _book_missed(
+ job, missed, first, f"skipped, catch-up off; next run {new_next}"))
+ return True
+ logger.info(
+ "Job '%s' missed its scheduled time (%s, grace=%ds). "
+ "Running now; next run provisionally set to: %s (re-anchored on completion)",
+ d.label, d.next_run, grace, new_next)
+ from cron.occurrences import record_catch_up_occurrence
+ d.scan.on_saved.append(record_catch_up_occurrence)
+ if missed > 1:
+ d.scan.on_saved.append(lambda job=d.job, first=d.next_run: _book_missed(
+ job, missed - 1, first, "collapsed into one catch-up run"))
+ return False
+
+
+def retire_expired_oneshot(d: Any) -> bool:
+ """One-shot grace gate; True when the job must not fire this tick.
+
+ A one-shot beyond the grace window fires once late only when the job opted in
+ (``catch_up: true``); otherwise it never fires (create/update/resume reject such schedules and
+ recovery never revives them). With no claim stamped, retire it with a diagnostic and a
+ ``missed ×1`` ledger row (never silently delete). A claim may mean a run is still in flight
+ elsewhere — skip but keep the record so its mark_job_run can land."""
+ from cron import store_health
+ from cron.jobs import (
+ ONESHOT_GRACE_SECONDS, _current_cron_store, _elapsed_seconds, record_cron_missed)
+ from cron.jobs_oneshot_diagnostics import write_missed
+
+ if _elapsed_seconds(d.scan.now, d.next_run_dt) <= ONESHOT_GRACE_SECONDS:
+ return False
+ if job_catches_up(d.job):
+ logger.info("Job '%s' missed its one-shot time %s; firing it once now (catch_up on).",
+ d.label, d.next_run)
+ return False
+ # Due during an outage of its store: it was skipped, not missed; fire it once on recovery.
+ if store_health.outage_covers(
+ _current_cron_store().cron_dir, d.next_run_dt.timestamp(), ONESHOT_GRACE_SECONDS):
+ return False
+ claim = d.job.get("run_claim")
+ # An outage-covered claim that never reached its fire claim never started: it stays covered.
+ if isinstance(claim, dict) and claim.get("outage") and not d.job.get("fire_claim"):
+ return False
+ if not (claim or d.job.get("fire_claim")):
+ write_missed(d.job, d.next_run)
+ d.scan.retire(d.job["id"])
+ d.scan.on_saved.append(lambda job=d.job: record_cron_missed(job))
+ d.scan.on_saved.append(lambda job=d.job, first=d.next_run: _book_missed(
+ job, 1, first, "one-shot past its grace window; catch-up off"))
+ return True
diff --git a/cron/scheduler.py b/cron/scheduler.py
index afaf1494958..c618230fcfb 100644
--- a/cron/scheduler.py
+++ b/cron/scheduler.py
@@ -30,7 +30,7 @@ except ImportError:
except ImportError:
msvcrt = None
from pathlib import Path
-from typing import Any, Callable, Dict, List, Optional, Protocol, Union
+from typing import Any, Callable, Dict, List, Optional, Union
# Must precede repo-level imports: standalone invocations (e.g. module reload after
# `hermes update`) otherwise fail with ModuleNotFoundError for hermes_time et al.
@@ -531,7 +531,8 @@ from cron.executions import (
recover_interrupted_executions, settle_unstarted_execution, terminalize_dead_owner)
from cron.scheduler_liveness import ExecutionProgressStamper, _inactivity_watchdog_loop
from cron.scheduler_bookkeeping import _classify_delivery_outcome
-from cron.cancellation import CancelRequest, clear_cancel
+from cron.cancellation import (CancelRequest, _CancelEventLike, _CombinedCancelEvent, cancel_reason,
+ clear_cancel, finish_cancelled_run as _finish_cancelled_run)
# Response marker that suppresses delivery (output is still saved locally for audit).
SILENT_MARKER = "[SILENT]"
@@ -658,43 +659,6 @@ _INFLIGHT_MIN_ALLOWANCE_MINUTES = 30.0
_interrupted_job_ids: set = set()
-class _CancelEventLike(Protocol):
- """Structural type for cancellation sources (``threading.Event``, ``_CombinedCancelEvent``)."""
-
- def is_set(self) -> bool: ...
- def set(self) -> None: ...
-
-
-class _CombinedCancelEvent:
- """Duck-typed ``threading.Event`` ORing several cancellation sources (fire-claim heartbeat
- ``lost_ownership`` + per-transport events). Workers only call is_set()/set(), so no pump thread.
- """
-
- def __init__(self, *events: Optional[_CancelEventLike]) -> None:
- self._events = [event for event in events if event is not None]
-
- def is_set(self) -> bool:
- return any(event.is_set() for event in self._events)
-
- def set(self) -> None:
- for event in self._events:
- event.set()
-
- @property
- def reason(self) -> Optional[str]:
- """Why the run is being torn down, from whichever source fired — so a script or agent
- stop never reports "fire claim lost" for an operator cancel (or the reverse)."""
- for event in self._events:
- if event.is_set():
- return getattr(event, "reason", None)
- return None
-
-
-def cancel_reason(cancel_event: Optional[_CancelEventLike]) -> str:
- """Human-facing cause for a cancelled run; the legacy wording when no source carries one."""
- return getattr(cancel_event, "reason", None) or "cron fire ownership was lost"
-
-
def get_running_job_ids() -> frozenset[str]:
"""Thread-safe snapshot of executing job IDs (dispatch until ``_process_job`` returns). Read by
the gateway shutdown drain, otherwise blind to cron work (runs outside ``_running_agents``).
@@ -3026,6 +2990,7 @@ class _FireOwnership:
# ``set()`` would propagate into the transport event this run does not own (#105861).
self.claim_lost = claim_lost
self.transport_cancel = transport_cancel
+ self.operator_cancel = operator_cancel
# ``operator_cancel`` (``hermes cron cancel``) is a THIRD source, deliberately not folded
# into ``transport_cancel``: only a transport cancel counts as claim loss for bookkeeping,
# while an operator cancel is a real outcome this run records itself.
@@ -3040,6 +3005,9 @@ class _FireOwnership:
def transport_cancelled(self) -> bool:
return self.transport_cancel is not None and self.transport_cancel.is_set()
+ def operator_cancelled(self) -> bool:
+ return self.operator_cancel is not None and self.operator_cancel.is_set()
+
def side_effect_fence(self):
if self.owner is None:
return contextlib.nullcontext(True)
@@ -3384,11 +3352,18 @@ def _run_one_job_body(
_run_kwargs["cancel_event"] = fence.cancel_event
try:
success, output, final_response, error = run_job(job, **_run_kwargs)
- except BaseException:
+ except BaseException as exc:
# run_job hands back the agent even when raising; tear down so a failed run never leaks.
# BaseException so KeyboardInterrupt/SystemExit mid-run still trigger teardown.
_teardown_deferred()
- raise
+ if not (isinstance(exc, Exception) and fence.operator_cancelled()):
+ raise
+ success = None
+ # Before claim loss: removing a job mid-run cancels it AND drops its claim (#123045).
+ if fence.operator_cancelled():
+ if success is not None: # the except path above already tore down
+ _teardown_deferred()
+ return _finish_cancelled_run(job, execution_id, fire_owner)
if _fire_claim_ownership_lost():
_teardown_deferred()
diff --git a/cron/scheduler_script.py b/cron/scheduler_script.py
index bc9ecca2277..97807e3f124 100644
--- a/cron/scheduler_script.py
+++ b/cron/scheduler_script.py
@@ -27,7 +27,7 @@ from typing import Any, Callable, Optional, TYPE_CHECKING
from hermes_cli._subprocess_compat import windows_hide_flags
if TYPE_CHECKING:
- from cron.scheduler import _CancelEventLike
+ from cron.cancellation import _CancelEventLike
# Log-record parity with the origin module.
logger = logging.getLogger("cron.scheduler")
@@ -495,7 +495,8 @@ def _run_job_script(
if cancel_event is not None and cancel_event.is_set():
_terminate_cron_script_tree(proc)
_drain_script_pipes(proc)
- return False, f"Script cancelled: {_sched.cancel_reason(cancel_event)}"
+ from cron.cancellation import cancel_reason
+ return False, f"Script cancelled: {cancel_reason(cancel_event)}"
remaining = deadline - time.monotonic()
if remaining <= 0:
_terminate_cron_script_tree(proc)
diff --git a/hermes_cli/cron.py b/hermes_cli/cron.py
index 171702fdef8..4efcd4f2baf 100644
--- a/hermes_cli/cron.py
+++ b/hermes_cli/cron.py
@@ -779,7 +779,7 @@ _JOB_ARG_FIELDS = (("name", "name"), ("deliver", "deliver"), ("failure_deliver",
("monitor_script", "monitor_script"), ("monitor_url", "monitor_url"),
("continuity", "continuity"), ("reasoning_effort", "reasoning_effort"),
("interpreter", "interpreter"), ("timezone", "timezone"),
- ("grace_minutes", "grace_minutes"))
+ ("grace_minutes", "grace_minutes"), ("catch_up", "catch_up"))
def _job_api_kwargs(args) -> dict[str, Any]:
diff --git a/hermes_cli/subcommands/cron.py b/hermes_cli/subcommands/cron.py
index 53707af8e3a..629bb72e735 100644
--- a/hermes_cli/subcommands/cron.py
+++ b/hermes_cli/subcommands/cron.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import argparse
from typing import Callable
from hermes_cli.subcommands._shared import add_accept_hooks_flag
@@ -11,6 +12,20 @@ def _flag(parser, *names, help, **kw):
parser.add_argument(*names, action="store_true", help=help, **kw)
+def _catch_up_flag(value: str) -> bool:
+ if value not in ("on", "off"):
+ raise argparse.ArgumentTypeError("use 'on' (run missed runs once) or 'off' (skip them)")
+ return value == "on"
+
+
+def _add_catch_up(parser) -> None:
+ parser.add_argument(
+ "--catch-up", dest="catch_up", type=_catch_up_flag, metavar="{on,off}",
+ help="Runs missed while the machine was off: 'on' runs once on return, 'off' skips "
+ "them (each is booked as a 'missed' row in `hermes cron runs`). Default: "
+ "cron.catch_up_missed for recurring jobs; one-shots skip.")
+
+
def build_cron_parser(subparsers, *, cmd_cron: Callable) -> None:
"""Attach the ``cron`` subcommand (and its sub-actions) to ``subparsers``."""
cron_parser = subparsers.add_parser(
@@ -36,6 +51,7 @@ def build_cron_parser(subparsers, *, cmd_cron: Callable) -> None:
"state stays visible in `hermes cron list`. Omit = failures "
"follow --deliver.")
cron_create.add_argument("--repeat", type=int, help="Optional repeat count")
+ _add_catch_up(cron_create)
cron_create.add_argument("--skill", dest="skills", action="append",
help="Attach a skill. Repeat to add multiple skills.")
cron_create.add_argument("--script",
@@ -110,6 +126,7 @@ def build_cron_parser(subparsers, *, cmd_cron: Callable) -> None:
help="Override target for failure notices (same grammar as --deliver; "
"'local' suppresses; '' clears the override)")
cron_edit.add_argument("--repeat", type=int, help="New repeat count")
+ _add_catch_up(cron_edit)
cron_edit.add_argument("--skill", dest="skills", action="append",
help="Replace the job's skills with this set. Repeat to attach multiple skills.")
cron_edit.add_argument("--add-skill", dest="add_skills", action="append",
diff --git a/hermes_cli/web_routers/cron.py b/hermes_cli/web_routers/cron.py
index 7827b1e294d..eb49f5b9a82 100644
--- a/hermes_cli/web_routers/cron.py
+++ b/hermes_cli/web_routers/cron.py
@@ -298,14 +298,14 @@ def _owner_profile_executions(canonical_job_id: str) -> list[dict[str, Any]]:
the owner-home scope so it opens the OWNER's cron/executions.db.
"""
try:
- from cron.executions import list_executions
+ from cron.executions import _TERMINAL_STATES, list_executions
rows = list_executions(job_id=canonical_job_id, limit=100)
except Exception:
return []
terminal: list[dict[str, Any]] = []
for row in rows:
- if str(row.get("status") or "") not in ("completed", "failed", "unknown"):
+ if str(row.get("status") or "") not in _TERMINAL_STATES:
continue
terminal.append({
"id": str(row.get("id") or ""),
@@ -339,13 +339,20 @@ def _execution_contains(
return finish is None or started_at <= finish + grace_seconds
+# A cancelled or missed attempt's ``error`` is its reason ("job removed while running",
+# "missed ×3 (...)"), the row's whole point — show it like a failure's.
+_LABELS_WITH_REASON = frozenset({"FAILED", "CANCELLED", "MISSED"})
+# Terminal attempts that carry a real outcome (``unknown`` is an owner that vanished).
+_OUTCOME_STATES = ("completed", "failed", "cancelled", "missed")
+
+
def _execution_status_title(status: str, error: str, fallback: str) -> str:
label = (status or "").replace("_", " ").upper()
if label == "UNKNOWN":
return fallback
if not label:
return fallback
- return f"{label} · {error}" if (label == "FAILED" and error) else f"{label} · {fallback}"
+ return f"{label} · {error}" if (label in _LABELS_WITH_REASON and error) else f"{label} · {fallback}"
def _list_cron_output_runs(
@@ -393,7 +400,7 @@ def _list_cron_output_runs(
**_cron_output_run_row(started_at, preview or "Script-only run", preview),
})
- if docs or any(attempt["status"] in ("completed", "failed") for attempt in attempts):
+ if docs or any(attempt["status"] in _OUTCOME_STATES for attempt in attempts):
return docs, attempts
# No output docs survived (pruned or never written) but the job HAS run:
@@ -559,7 +566,7 @@ def _reconcile_cron_runs(
by_execution = {s["cron_execution_id"]: s for s in session_runs if s.get("cron_execution_id")}
legacy = [s for s in session_runs if not s.get("cron_execution_id")]
- terminal = [(index, a) for index, a in enumerate(attempts) if a["status"] in ("completed", "failed")]
+ terminal = [(index, a) for index, a in enumerate(attempts) if a["status"] in _OUTCOME_STATES]
represented = {index for index, a in terminal if a["id"] in by_execution}
paired: set = set()
merged = list(session_runs)
@@ -681,6 +688,13 @@ def _trigger_cron_job_sync(job_id: str, profile: Optional[str] = None):
return {**job, "enabled": False, "state": "completed"}
+def _cancel_cron_job_sync(job_id: str, profile: Optional[str] = None):
+ """Stop the job's live run; ``cancelled: false`` when nothing is in flight. Schedule untouched."""
+ selected = _job_profile(job_id, profile)
+ job = _found(_call_cron_for_profile(selected, "resolve_job_ref", job_id))
+ return {**_call_cron_for_profile(selected, "cancel_job_run", job["id"]), "job_id": job["id"]}
+
+
def _delete_cron_job_sync(job_id: str, profile: Optional[str] = None):
selected = _job_profile(job_id, profile)
try:
@@ -764,6 +778,11 @@ async def trigger_cron_job(job_id: str, profile: Optional[str] = None):
return await _run_cron_dashboard_io(_trigger_cron_job_sync, job_id, profile)
[email protected]("/api/cron/jobs/{job_id}/cancel")
+async def cancel_cron_job(job_id: str, profile: Optional[str] = None):
+ return await _run_cron_dashboard_io(_cancel_cron_job_sync, job_id, profile)
+
+
@router.delete("/api/cron/jobs/{job_id}")
async def delete_cron_job(job_id: str, profile: Optional[str] = None):
return await _run_cron_dashboard_io(_delete_cron_job_sync, job_id, profile)
diff --git a/tests/cron/test_cron_stop_and_missed_policy.py b/tests/cron/test_cron_stop_and_missed_policy.py
new file mode 100644
index 00000000000..015e4cd377c
--- /dev/null
+++ b/tests/cron/test_cron_stop_and_missed_policy.py
@@ -0,0 +1,163 @@
+"""Stopping a live cron run (cancel / remove) and the per-job missed-run policy.
+
+Real store, real execution ledger, real cancel markers under a temp HERMES_HOME.
+"""
+
+from __future__ import annotations
+
+import sqlite3
+from datetime import timedelta
+
+import pytest
+
+from cron import jobs
+
+
+def home(tmp_path, monkeypatch):
+ monkeypatch.setenv("HERMES_HOME", str(tmp_path))
+ with jobs.use_cron_store(tmp_path / "cron"):
+ yield tmp_path
+
+
+def _park(job_id: str, seconds_ago: float) -> None:
+ stored = jobs.load_jobs()
+ row = next(r for r in stored if r["id"] == job_id)
+ row["next_run_at"] = (jobs._hermes_now() - timedelta(seconds=seconds_ago)).isoformat()
+ jobs.save_jobs(stored)
+
+
+def _ledger(job_id: str) -> list[dict]:
+ from cron.executions import list_executions
+
+ return list_executions(job_id=job_id)
+
+
+def _live_run(job_id: str) -> str:
+ from cron.executions import create_execution, mark_execution_running
+
+ record = create_execution(job_id, source="test")
+ mark_execution_running(record["id"])
+ return record["id"]
+
+
+# --- missed-run policy ----------------------------------------------------------------------
+
+
+def test_catch_up_off_skips_and_books_one_missed_row_with_the_count(home):
+ job = jobs.create_job(prompt="brief", schedule="every 1h", deliver="local", catch_up=False)
+ _park(job["id"], 4 * 3600 + 60) # slots at -4h1m, -3h1m, -2h1m, -1h1m, -1m
+
+ assert jobs.get_due_jobs() == []
+ rows = _ledger(job["id"])
+ assert [r["status"] for r in rows] == ["missed"]
+ assert rows[0]["error"].startswith("missed \u00d75 (skipped")
+ assert rows[0]["started_at"] is None
+
+
+def test_job_catch_up_beats_the_global_setting(home):
+ (home / "config.yaml").write_text("cron:\n catch_up_missed: false\n", encoding="utf-8")
+ job = jobs.create_job(prompt="scan", schedule="every 1h", deliver="local", catch_up=True)
+ _park(job["id"], 3 * 3600 + 60)
+
+ assert [j["id"] for j in jobs.get_due_jobs()] == [job["id"]]
+ # Four slots passed; one runs now, three were collapsed into it.
+ assert [(r["status"], r["error"].split(" (")[0]) for r in _ledger(job["id"])] == [
+ ("missed", "missed \u00d73")]
+
+
+def test_global_setting_stays_the_default_for_jobs_without_their_own(home):
+ (home / "config.yaml").write_text("cron:\n catch_up_missed: false\n", encoding="utf-8")
+ job = jobs.create_job(prompt="brief", schedule="every 1h", deliver="local")
+ _park(job["id"], 2 * 3600)
+
+ assert jobs.get_due_jobs() == []
+ assert [r["status"] for r in _ledger(job["id"])] == ["missed"]
+
+
+def test_one_shot_past_grace_skips_by_default_and_fires_once_when_opted_in(home):
+ run_at = (jobs._hermes_now() + timedelta(hours=1)).isoformat()
+ skipped = jobs.create_job(prompt="stale reminder", schedule=run_at, deliver="local")
+ kept = jobs.create_job(prompt="late reminder", schedule=run_at, deliver="local", catch_up=True)
+ for job in (skipped, kept):
+ _park(job["id"], 3600)
+
+ assert [j["id"] for j in jobs.get_due_jobs()] == [kept["id"]]
+ assert jobs.get_job(skipped["id"]) is None
+ assert [r["status"] for r in _ledger(skipped["id"])] == ["missed"]
+ assert _ledger(kept["id"]) == []
+
+
+def test_catch_up_rejects_non_boolean(home):
+ with pytest.raises(ValueError, match="catch_up"):
+ jobs.create_job(prompt="x", schedule="every 1h", deliver="local", catch_up="sometimes")
+
+
+# --- ledger schema --------------------------------------------------------------------------
+
+
+def test_a_pre_existing_ledger_is_widened_without_losing_rows(home):
+ db = home / "cron" / "executions.db"
+ db.parent.mkdir(parents=True, exist_ok=True)
+ conn = sqlite3.connect(db)
+ conn.execute(
+ """CREATE TABLE executions (
+ id TEXT PRIMARY KEY, job_id TEXT NOT NULL, source TEXT NOT NULL,
+ process_id TEXT NOT NULL, pid INTEGER NOT NULL, process_started_at INTEGER,
+ status TEXT NOT NULL CHECK(status IN
+ ('claimed','running','completed','failed','unknown')),
+ claimed_at TEXT NOT NULL, started_at TEXT, finished_at TEXT, error TEXT)""")
+ conn.execute(
+ "INSERT INTO executions (id, job_id, source, process_id, pid, status, claimed_at) "
+ "VALUES ('old', 'j1', 'builtin', 'p', 1, 'completed', '2026-01-01T00:00:00+00:00')")
+ conn.commit()
+ conn.close()
+
+ from cron.executions import record_missed_occurrences
+
+ record_missed_occurrences("j1", count=2, first_instant=None, detail="test")
+ assert sorted(r["status"] for r in _ledger("j1")) == ["completed", "missed"]
+
+
+# --- stopping a live run --------------------------------------------------------------------
+
+
+def test_removing_a_job_stops_its_live_run(home):
+ from cron.cancellation import CancelRequest
+
+ job = jobs.create_job(prompt="long", schedule="every 1h", deliver="local")
+ execution_id = _live_run(job["id"])
+
+ assert jobs.remove_job(job["id"])
+ request = CancelRequest(execution_id)
+ assert request.is_set()
+ assert request.reason == "job removed while running"
+
+
+def test_a_cancelled_run_books_cancelled_without_counting_as_a_failure(home):
+ from cron import scheduler
+ from cron.cancellation import request_cancel
+
+ from cron.executions import create_execution
+
+ job = jobs.create_job(prompt="long", schedule="every 1h", deliver="local")
+ execution_id = create_execution(job["id"], source="test")["id"] # the body starts it
+ request_cancel(execution_id, reason="operator asked")
+ delivered: list = []
+
+ def run_job(job, *, cancel_event=None, **_kw):
+ assert cancel_event is not None and cancel_event.is_set()
+ raise RuntimeError(f"Cron job was stopped: {scheduler.cancel_reason(cancel_event)}")
+
+ with pytest.MonkeyPatch.context() as mp:
+ mp.setattr(scheduler, "run_job", run_job)
+ mp.setattr(scheduler, "_deliver_crash_failure",
+ lambda *a, **k: delivered.append(a) or (None, "delivered"))
+ assert scheduler._run_one_job_body(dict(job, execution_id=execution_id)) is True
+
+ row = _ledger(job["id"])[0]
+ assert (row["status"], row["error"]) == ("cancelled", "operator asked")
+ stored = jobs.get_job(job["id"])
+ assert stored["last_status"] == "cancelled"
+ assert not stored.get("failure_streak")
+ assert delivered == [] # the operator asked for it: no failure alert
diff --git a/tools/cronjob_tools.py b/tools/cronjob_tools.py
index 4814c3facf5..a60947331c3 100644
--- a/tools/cronjob_tools.py
+++ b/tools/cronjob_tools.py
@@ -734,6 +734,7 @@ def _action_create(a: dict[str, Any]) -> str:
grace_minutes=a["grace_minutes"],
pinned=bool(a["pinned"]),
failure_deliver=_resolve_cron_context_deliver(_normalize_deliver_param(a["failure_deliver"])),
+ catch_up=a["catch_up"],
**({"paused": a["paused"], "paused_reason": a["paused_reason"]}
if a["paused"] is not False or a["paused_reason"] is not None else {}))
except CronSchedulerRegistrationError as exc:
@@ -876,6 +877,8 @@ def _update_core_fields(job: dict[str, Any], a: dict[str, Any], updates: dict[st
return bot_chat_error
_norm_fd = _resolve_cron_context_deliver(_norm_fd)
updates["failure_deliver"] = _norm_fd
+ if a["catch_up"] is not None:
+ updates["catch_up"] = a["catch_up"]
if skills is not None or skill is not None:
canonical_skills = _canonical_skills(skill, skills)
updates["skills"] = canonical_skills
@@ -1005,7 +1008,7 @@ def _action_update(job: dict[str, Any], a: dict[str, Any]) -> str:
_JOBLESS_ACTIONS = {"create": _action_create, "list": _action_list}
-def _action_cancel(job: Dict[str, Any], a: Dict[str, Any]) -> str:
+def _action_cancel(job: dict[str, Any], a: dict[str, Any]) -> str:
"""Stop the job's LIVE run. Per-run, not per-schedule: the job keeps its next fire."""
from cron.cancellation import cancel_run
@@ -1068,6 +1071,7 @@ def cronjob(
monitor_url: Optional[str] = None,
reasoning_effort: Optional[str] = None,
failure_deliver: Optional[str | list[str]] = None,
+ catch_up: Optional[bool] = None,
task_id: str | None = None,
session_id: Optional[str] = None,
paused: bool = False,
@@ -1166,6 +1170,10 @@ Jobs run in a fresh session with no current-chat context, so prompts must be sel
"items": {"type": "string"},
"description": "Optional ordered skill names loaded before the cron prompt. On update, [] clears."
},
+ "catch_up": {
+ "type": "boolean",
+ "description": "Runs missed while the machine was off: true = run once on return, false = skip. Omit for the default (recurring run once, one-shots skip)."
+ },
"script": {
"type": "string",
"description": _script_description("the profile HERMES_HOME")
@@ -1233,7 +1241,7 @@ def check_cronjob_requirements() -> bool:
_HANDLER_FORWARDED_ARGS = (
"job_id", "prompt", "schedule", "name", "repeat", "deliver", "failure_deliver", "skill", "skills", "reason",
"script", "context_from", "continuity", "enabled_toolsets", "workdir", "no_agent", "attach_to_session",
- "paused_reason", "pinned", "timezone", "grace_minutes")
+ "paused_reason", "pinned", "timezone", "grace_minutes", "catch_up")
def _cronjob_handler(args, **kw):