The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
feat(cron): a stopped or removed run books 'cancelled'; each job picks its missed-run policy
Tekniumee394bd9213b
diff --git a/cron/cancellation.py b/cron/cancellation.pyindex c7478299e31..a15be3352e0 100644--- a/cron/cancellation.py+++ b/cron/cancellation.py@@ -18,7 +18,7 @@ import contextlibimport osimport timefrom pathlib import Path-from typing import Any, Dict, Optional+from typing import Any, Dict, Optional, Protocolfrom hermes_constants import get_hermes_home@@ -81,6 +81,38 @@ def clear_cancel(execution_id: str) -> None:_marker_path(execution_id).unlink(missing_ok=True)+class _CancelEventLike(Protocol):+ """Structural type for cancellation sources (``threading.Event``, ``_CombinedCancelEvent``)."""++ def is_set(self) -> bool: ...+ def set(self) -> None: ...+++class _CombinedCancelEvent:+ """Duck-typed ``threading.Event`` ORing several cancellation sources (fire-claim heartbeat+ ``lost_ownership`` + per-transport events). Workers only call is_set()/set(), so no pump thread.+ """++ def __init__(self, *events: Optional[_CancelEventLike]) -> None:+ self._events = [event for event in events if event is not None]++ def is_set(self) -> bool:+ return any(event.is_set() for event in self._events)++ def set(self) -> None:+ for event in self._events:+ event.set()++ @property+ def reason(self) -> Optional[str]:+ """Why the run is being torn down, from whichever source fired — so a script or agent+ stop never reports "fire claim lost" for an operator cancel (or the reverse)."""+ for event in self._events:+ if event.is_set():+ return getattr(event, "reason", None)+ return None++class CancelRequest:"""``_CancelEventLike`` view of one execution's cancel marker.@@ -109,7 +141,26 @@ class CancelRequest:return self._reason-def cancel_run(job_id: str, *, reason: Optional[str] = None) -> Dict[str, Any]:+def cancel_reason(cancel_event: Any) -> str:+ """Human-facing cause for a cancelled run; the legacy wording when no source carries one."""+ return getattr(cancel_event, "reason", None) or "cron fire ownership was lost"+++def finish_cancelled_run(job: dict[str, Any], execution_id: str, fire_owner: Optional[str]) -> bool:+ """Book an operator-cancelled run: ledger row ``cancelled`` with the recorded reason, and the+ job's ``last_status`` — without a failure alert or a failure-streak bump, since the operator+ asked for it. A job removed mid-run has no record left to mark; its ledger row still lands."""+ from cron import scheduler++ reason = CancelRequest(execution_id).reason+ if not scheduler.self_removal_delivery_allowed(job["id"]):+ owner = {"expected_fire_owner": fire_owner} if fire_owner is not None else {}+ scheduler.mark_job_run(job["id"], False, f"Cancelled: {reason}", status="cancelled", **owner)+ scheduler.finish_execution(execution_id, success=False, error=reason, cancelled=True)+ return True+++def cancel_run(job_id: str, *, reason: Optional[str] = None) -> dict[str, Any]:"""Request cancellation of the job's live run. A job with nothing in flight is a no-op."""record = live_inflight_execution(job_id)if record is None:diff --git a/cron/executions.py b/cron/executions.pyindex a3dfc01f0dc..34bbe6de0e1 100644--- a/cron/executions.py+++ b/cron/executions.py@@ -34,7 +34,10 @@ MAX_TERMINAL_EXECUTIONS = 1000HANDOFF_ADOPTION_GRACE_SECONDS = 30.0# Floor for the live-owner stale-claim bound (#115692); see _live_owner_stale_after_seconds.LIVE_OWNER_STALE_CLAIM_FLOOR_SECONDS = 7200.0-_TERMINAL_STATES = ("completed", "failed", "unknown")+# ``cancelled``: an operator stopped the run (``hermes cron cancel``, removing the job mid-run).+# ``missed``: never started — one row books every occurrence the scheduler skipped past.+_TERMINAL_STATES = ("completed", "failed", "unknown", "cancelled", "missed")+_STATUS_CHECK = "('claimed','running','completed','failed','unknown','cancelled','missed')"_lock = threading.RLock()_PROCESS_ID = uuid.uuid4().hex@@ -57,7 +60,7 @@ def _initialize_schema(conn: sqlite3.Connection) -> None:from hermes_cli.sqlite_util import add_column_if_missingconn.execute(- """CREATE TABLE IF NOT EXISTS executions (+ f"""CREATE TABLE IF NOT EXISTS executions (id TEXT PRIMARY KEY,job_id TEXT NOT NULL,source TEXT NOT NULL,@@ -65,7 +68,7 @@ def _initialize_schema(conn: sqlite3.Connection) -> None:pid INTEGER NOT NULL,process_started_at INTEGER,status TEXT NOT NULL CHECK(status IN- ('claimed','running','completed','failed','unknown')),+ {_STATUS_CHECK}),handoff_pending INTEGER NOT NULL DEFAULT 0,handoff_started_at REAL,claimed_at TEXT NOT NULL,@@ -74,6 +77,7 @@ def _initialize_schema(conn: sqlite3.Connection) -> None:error TEXT)""")+ _widen_status_check(conn)add_column_if_missing(conn, "executions", "handoff_pending","handoff_pending INTEGER NOT NULL DEFAULT 0",@@ -98,6 +102,34 @@ def _initialize_schema(conn: sqlite3.Connection) -> None:)+def _widen_status_check(conn: sqlite3.Connection) -> None:+ """Rebuild a ledger created before ``cancelled``/``missed`` existed: SQLite cannot alter a+ CHECK constraint in place. Copies every column the old table has; runs once per file, and+ re-reads the schema under the write lock so two processes opening it together rebuild once."""+ def current_sql() -> Optional[str]:+ row = conn.execute(+ "SELECT sql FROM sqlite_master WHERE type='table' AND name='executions'").fetchone()+ return row[0] if row is not None else None++ if "'missed'" in (current_sql() or "'missed'"):+ return+ conn.commit()+ conn.execute("BEGIN IMMEDIATE")+ try:+ old_sql = current_sql() or "'missed'"+ if "'missed'" not in old_sql:+ columns = ", ".join(r[1] for r in conn.execute("PRAGMA table_info(executions)"))+ conn.execute("ALTER TABLE executions RENAME TO executions_pre_widen")+ conn.execute(old_sql.replace("'unknown')", "'unknown','cancelled','missed')", 1))+ conn.execute(+ f"INSERT INTO executions ({columns}) SELECT {columns} FROM executions_pre_widen")+ conn.execute("DROP TABLE executions_pre_widen")+ conn.commit()+ except BaseException:+ conn.rollback()+ raise++@contextmanagerdef _transaction() -> Iterator[sqlite3.Connection]:from hermes_cli.sqlite_util import transaction@@ -202,7 +234,7 @@ def _prune_unlocked(conn: sqlite3.Connection) -> None:conn.execute("""DELETE FROM executions WHERE id IN (SELECT id FROM executions- WHERE status IN ('completed','failed','unknown')+ WHERE status IN ('completed','failed','unknown','cancelled','missed')ORDER BY julianday(finished_at) DESC, finished_at DESC,julianday(claimed_at) DESC, claimed_at DESC, id DESC LIMIT -1 OFFSET ?)""",@@ -311,7 +343,7 @@ def mark_execution_running(execution_id: str) -> Optional[dict[str, Any]]:def finish_execution(execution_id: str, *, success: bool, error: Optional[str] = None,- delivery_outcome: Optional[str] = None,+ delivery_outcome: Optional[str] = None, cancelled: bool = False,) -> Optional[dict[str, Any]]:"""Write a terminal result once; terminal attempts cannot be rewritten.@@ -319,7 +351,7 @@ def finish_execution(settles through ``recover_receipted_execution``."""now = _hermes_now().isoformat()- status = "completed" if success else "failed"+ status = "cancelled" if cancelled else ("completed" if success else "failed")detail = None if success else (str(error) if error else "unknown failure")with _transaction() as conn:owner = (_PROCESS_ID, os.getpid())@@ -340,6 +372,34 @@ def finish_execution(return record+def record_missed_occurrences(+ job_id: str, *, count: int, first_instant: Optional[str], detail: str,+) -> Optional[dict[str, Any]]:+ """Book occurrences the scheduler passed over without running as ONE terminal ``missed`` row+ (``missed ×N``), so run history shows the gap instead of nothing. Never started, so no+ owner fencing applies; ``first_instant`` is the earliest skipped slot."""+ from cron.occurrences import scheduled_instant as canonical_instant++ if count < 1:+ return None+ now = _hermes_now().isoformat()+ execution_id = uuid.uuid4().hex+ pid = os.getpid()+ with _transaction() as conn:+ conn.execute(+ """INSERT INTO executions+ (id, job_id, source, process_id, pid, process_started_at,+ status, claimed_at, finished_at, error, scheduled_instant)+ VALUES (?, ?, 'scheduler', ?, ?, ?, 'missed', ?, ?, ?, ?)""",+ (execution_id, str(job_id), _PROCESS_ID, pid, _process_start_time(pid), now, now,+ f"missed \u00d7{count} ({detail})", canonical_instant(first_instant)),+ )+ _prune_unlocked(conn)+ record = _fetch(conn, execution_id)+ _emit_execution_state(record)+ return record++def recover_receipted_execution(execution_id, job_id, *, success, error=None, delivery_outcome=None):"""Settle a verified canonical receipt after owner loss, including its unknown audit row.diff --git a/cron/jobs.py b/cron/jobs.pyindex 0357623a42b..5d5e5d30f0d 100644--- a/cron/jobs.py+++ b/cron/jobs.py@@ -1744,6 +1744,13 @@ def _normalize_reasoning_effort(value: Any) -> Optional[str]:# Normalizers for create_job (all fields) / update_job (present fields). Invalid values raise BEFORE# storing.+def _normalize_catch_up(value: Any) -> Optional[bool]:+ """``catch_up``: True runs one catch-up for missed slots, False skips them, None = default."""+ if value is None or isinstance(value, bool):+ return value+ raise ValueError("catch_up must be true, false, or null (use the default).")++_CREATE_FIELD_NORMALIZERS: dict[str, Callable[[Any], Any]] = {"model": _normalize_job_optional_text,"provider": _normalize_job_optional_text,@@ -1759,6 +1766,7 @@ _CREATE_FIELD_NORMALIZERS: dict[str, Callable[[Any], Any]] = {"interpreter": _normalize_job_optional_text,"timezone": jobs_time.normalize_timezone,"grace_minutes": jobs_time.normalize_grace_minutes,+ "catch_up": _normalize_catch_up,}_UPDATE_FIELD_NORMALIZERS: dict[str, Callable[[Any], Any]] = {# [] is an explicit zero-tool allowlist and must survive the update path as [] too (#82010).@@ -1772,6 +1780,7 @@ _UPDATE_FIELD_NORMALIZERS: dict[str, Callable[[Any], Any]] = {# ``in {None, "", False}`` test — ``0 == False`` would silently clear it."timezone": jobs_time.normalize_timezone,"grace_minutes": jobs_time.normalize_grace_minutes,+ "catch_up": _normalize_catch_up,}@@ -1819,6 +1828,7 @@ def create_job(monitor_url: Optional[str] = None,reasoning_effort: Optional[str] = None,failure_deliver: Optional[str] = None,+ catch_up: Optional[bool] = None,paused: bool = False,paused_reason: Optional[str] = None,pinned: bool = False,@@ -1928,6 +1938,7 @@ def create_job(("attach_to_session", normalized_attach), ("reasoning_effort", normalized_reasoning_effort),("failure_deliver", f["failure_deliver"]), ("interpreter", f["interpreter"]),("timezone", f["timezone"]), ("grace_minutes", f["grace_minutes"]),+ ("catch_up", f["catch_up"]),):if value is not None:job[key] = value@@ -2309,6 +2320,8 @@ def remove_job(job_id: str) -> bool:marker = _self_removal_delivery.get()if marker is not None and marker.job_id == canonical_id:marker.removed = True+ else: # someone else removed it: its live run must stop, not burn tokens to a discard+ cancel_job_run(canonical_id, reason="job removed while running")if job_output_dir.exists():shutil.rmtree(job_output_dir)try:@@ -2323,6 +2336,13 @@ def remove_job(job_id: str) -> bool:return True+def cancel_job_run(job_id: str, reason: Optional[str] = None) -> dict[str, Any]:+ """Stop the job's live run (if any); its schedule is untouched. See ``cron.cancellation``."""+ from cron.cancellation import cancel_run++ return cancel_run(job_id, reason=reason)++def _set_alert_flag(job_id: str, field: str, value: bool) -> bool:"""Set/clear a persisted alert-dedup marker (alert exactly once until the condition heals;survives restarts) and return the PRIOR value. Field: ``preflight_alerted`` (blocked config).@@ -2381,14 +2401,15 @@ def _record_run_outcome(delivery_failed = isinstance(delivery_error, str) and bool(delivery_error.strip())job["last_status"] = status or ("error" if not success else ("delivery_failed" if delivery_failed else "ok"))- job["last_error"] = None if success else error+ cancelled = status == "cancelled" # an operator stop is not a failure+ job["last_error"] = None if success or cancelled else errorif success:# Healthy run: drop the alert-once dedup markers so a FUTURE break re-alerts, and clear# the forward-failure stamp so it only describes CURRENT auto-fire health.job.pop("preflight_alerted", None)job.pop("last_fire_error", None)job["failure_streak"] = 0- else:+ elif not cancelled:# Consecutive agent-failure streak; delivery failures do NOT count# (scheduler._failure_streak_nudge).job["failure_streak"] = int(job.get("failure_streak") or 0) + 1@@ -3085,64 +3106,6 @@ def _reanchor_stale_cron(d: _DueJob) -> bool:return False-def _fast_forward_missed_recurring(d: _DueJob, grace: int) -> bool:- """Re-anchor accumulated misses; return whether catch-up was explicitly disabled.-- The fast-forward is persisted immediately — NOT redundant with advance_next_run/mark_job_run:- it- protects the crash window before mark_job_run and covers the external fire_due path, which never- calls advance_next_run. mark_job_run re-anchors on completion, so the value is provisional.- """- if _elapsed_seconds(d.scan.now, d.next_run_dt) <= grace:- return False- new_next = d.recompute_next()- if not new_next:- return False- d.scan.persist(d.job["id"], next_run_at=new_next)- # An explicit per-job grace_minutes is the user's catch-up limit for THIS job: past it the- # miss is skipped even while the global cron.catch_up_missed default still runs it once.- if (_instant_after(_ensure_aware(datetime.fromisoformat(new_next)), d.scan.now)- and (jobs_time.grace_seconds_override(d.job.get("grace_minutes")) is not None- or not _cron_config_number("catch_up_missed", True, lambda value: value is not False))):- logger.info(- "Job '%s' missed its scheduled time (%s, grace=%ds). "- "Skipping missed occurrence (grace_minutes or cron.catch_up_missed=false); next run: %s",- d.label, d.next_run, grace, new_next)- d.scan.on_saved.append(lambda job=d.job: record_cron_missed(job))- return True- logger.info(- "Job '%s' missed its scheduled time (%s, grace=%ds). "- "Running now; next run provisionally set to: %s (re-anchored on completion)",- d.label, d.next_run, grace, new_next)- from cron.occurrences import record_catch_up_occurrence- d.scan.on_saved.append(record_catch_up_occurrence)- return False---def _retire_expired_oneshot(d: _DueJob) -> bool:- """One-shot grace gate; True when the job must not fire this tick.-- A one-shot beyond the grace window must never fire (create/update/resume reject such schedules- and recovery never revives them; only the due scan used to dispatch them hours late). With no- claim stamped, retire it with a diagnostic (never silently delete). A claim may mean a run is- still in flight elsewhere — skip but keep the record so its mark_job_run can land."""- if _elapsed_seconds(d.scan.now, d.next_run_dt) <= ONESHOT_GRACE_SECONDS:- return False- # Due during an outage of its store: it was skipped, not missed; fire it once on recovery.- if store_health.outage_covers(_current_cron_store().cron_dir, d.next_run_dt.timestamp(), ONESHOT_GRACE_SECONDS):- return False- claim = d.job.get("run_claim")- # An outage-covered claim that never reached its fire claim never started: it stays covered.- if isinstance(claim, dict) and claim.get("outage") and not d.job.get("fire_claim"):- return False- if not (claim or d.job.get("fire_claim")):- from cron.jobs_oneshot_diagnostics import write_missed- write_missed(d.job, d.next_run)- d.scan.retire(d.job["id"])- d.scan.on_saved.append(lambda job=d.job: record_cron_missed(job))- return True--def _oneshot_dispatch_limit_reached(job: dict[str, Any], scan: _DueScan) -> bool:"""One-shot dispatch-limit guard; True when the job must not fire this tick.@@ -3261,10 +3224,12 @@ def _evaluate_due_job(job: dict[str, Any], scan: _DueScan, run_claim_ttl: float)if not manual_run and kind == "cron" and _reanchor_stale_cron(d):return Falsegrace = _compute_grace_seconds(d.schedule, job.get("grace_minutes"))- if not manual_run and recurring and _fast_forward_missed_recurring(d, grace):+ from cron.missed_runs import fast_forward_missed_recurring, retire_expired_oneshot++ if not manual_run and recurring and fast_forward_missed_recurring(d, grace):return Falseif kind == "once":- if _retire_expired_oneshot(d) or _oneshot_dispatch_limit_reached(job, scan):+ if retire_expired_oneshot(d) or _oneshot_dispatch_limit_reached(job, scan):return False# Durably claim the one-shot for the DURATION of its run: a second scheduler process on the# same HERMES_HOME must not re-dispatch it while in flight, and advancing next_run_at by adiff --git a/cron/missed_runs.py b/cron/missed_runs.pynew file mode 100644index 00000000000..471dd16e8b4--- /dev/null+++ b/cron/missed_runs.py@@ -0,0 +1,142 @@+"""What happens to occurrences a job could not fire on time (machine off, gateway stopped).++Each job may carry ``catch_up``: ``true`` runs ONE catch-up for everything it missed, ``false``+skips to the next slot. Absent, recurring jobs follow ``cron.catch_up_missed`` (default true)+and one-shots keep the 120s grace rule (#93526: a stale reminder is skipped unless the job opts+in). Either way the skipped or collapsed slots are booked as one ``missed ×N`` row in the+execution ledger, so run history shows the gap instead of nothing.++Called from the due scan in ``cron/jobs.py`` under the store lock; ledger writes ride the+scan's ``on_saved`` hooks so an unwritable store cannot book the same miss every tick.+"""++from __future__ import annotations++import logging+from datetime import datetime+from typing import Any, Optional++logger = logging.getLogger("cron.jobs")++# Counting walks the schedule slot by slot; a once-a-minute job off for a month stops here.+_MAX_COUNTED_SLOTS = 1000+++def job_catches_up(job: dict[str, Any]) -> bool:+ """The job's own ``catch_up`` wins; otherwise one-shots skip, and recurring jobs skip past an+ explicit ``grace_minutes`` or else follow ``cron.catch_up_missed``."""+ value = job.get("catch_up")+ if isinstance(value, bool):+ return value+ if (job.get("schedule") or {}).get("kind") == "once":+ return False+ from cron import jobs_time+ from cron.jobs import _cron_config_number++ # An explicit grace_minutes is the user's catch-up limit for THIS job: past it the miss is+ # skipped even while the global cron.catch_up_missed default still runs it once.+ if jobs_time.grace_seconds_override(job.get("grace_minutes")) is not None:+ return False++ return bool(_cron_config_number("catch_up_missed", True, lambda raw: raw is not False))+++def count_missed_slots(schedule: dict[str, Any], first: str, now: datetime) -> int:+ """Occurrences from ``first`` up to ``now`` inclusive, capped at ``_MAX_COUNTED_SLOTS``."""+ from cron.jobs import _instant_after, _parse_aware, compute_next_run++ count, cursor = 0, first+ while cursor and count < _MAX_COUNTED_SLOTS:+ at = _parse_aware(cursor)+ if at is None or _instant_after(at, now):+ break+ count += 1+ following = compute_next_run(schedule, cursor)+ cursor = following if following != cursor else None+ return count+++def _book_missed(job: dict[str, Any], count: int, first: Optional[str], detail: str) -> None:+ from cron.executions import record_missed_occurrences++ if count >= _MAX_COUNTED_SLOTS:+ detail = f"{detail}; counting stopped at {_MAX_COUNTED_SLOTS}"+ try:+ record_missed_occurrences(str(job.get("id")), count=count, first_instant=first, detail=detail)+ except Exception:+ logger.warning("Job '%s': could not book its missed occurrences", job.get("id"), exc_info=True)+++def fast_forward_missed_recurring(d: Any, grace: int) -> bool:+ """Re-anchor accumulated misses; return whether catch-up is off for this job (skip the fire).++ The fast-forward is persisted immediately — NOT redundant with advance_next_run/mark_job_run:+ it protects the crash window before mark_job_run and covers the external fire_due path, which+ never calls advance_next_run. mark_job_run re-anchors on completion, so the value is+ provisional.+ """+ from cron.jobs import _elapsed_seconds, _ensure_aware, _instant_after, record_cron_missed++ if _elapsed_seconds(d.scan.now, d.next_run_dt) <= grace:+ return False+ new_next = d.recompute_next()+ if not new_next:+ return False+ d.scan.persist(d.job["id"], next_run_at=new_next)+ missed = count_missed_slots(d.schedule, d.next_run, d.scan.now)+ if (_instant_after(_ensure_aware(datetime.fromisoformat(new_next)), d.scan.now)+ and not job_catches_up(d.job)):+ logger.info(+ "Job '%s' missed its scheduled time (%s, grace=%ds). "+ "Skipping %d missed occurrence(s) because catch-up is off for this job; next run: %s",+ d.label, d.next_run, grace, missed, new_next)+ d.scan.on_saved.append(lambda job=d.job: record_cron_missed(job))+ d.scan.on_saved.append(lambda job=d.job, first=d.next_run: _book_missed(+ job, missed, first, f"skipped, catch-up off; next run {new_next}"))+ return True+ logger.info(+ "Job '%s' missed its scheduled time (%s, grace=%ds). "+ "Running now; next run provisionally set to: %s (re-anchored on completion)",+ d.label, d.next_run, grace, new_next)+ from cron.occurrences import record_catch_up_occurrence+ d.scan.on_saved.append(record_catch_up_occurrence)+ if missed > 1:+ d.scan.on_saved.append(lambda job=d.job, first=d.next_run: _book_missed(+ job, missed - 1, first, "collapsed into one catch-up run"))+ return False+++def retire_expired_oneshot(d: Any) -> bool:+ """One-shot grace gate; True when the job must not fire this tick.++ A one-shot beyond the grace window fires once late only when the job opted in+ (``catch_up: true``); otherwise it never fires (create/update/resume reject such schedules and+ recovery never revives them). With no claim stamped, retire it with a diagnostic and a+ ``missed ×1`` ledger row (never silently delete). A claim may mean a run is still in flight+ elsewhere — skip but keep the record so its mark_job_run can land."""+ from cron import store_health+ from cron.jobs import (+ ONESHOT_GRACE_SECONDS, _current_cron_store, _elapsed_seconds, record_cron_missed)+ from cron.jobs_oneshot_diagnostics import write_missed++ if _elapsed_seconds(d.scan.now, d.next_run_dt) <= ONESHOT_GRACE_SECONDS:+ return False+ if job_catches_up(d.job):+ logger.info("Job '%s' missed its one-shot time %s; firing it once now (catch_up on).",+ d.label, d.next_run)+ return False+ # Due during an outage of its store: it was skipped, not missed; fire it once on recovery.+ if store_health.outage_covers(+ _current_cron_store().cron_dir, d.next_run_dt.timestamp(), ONESHOT_GRACE_SECONDS):+ return False+ claim = d.job.get("run_claim")+ # An outage-covered claim that never reached its fire claim never started: it stays covered.+ if isinstance(claim, dict) and claim.get("outage") and not d.job.get("fire_claim"):+ return False+ if not (claim or d.job.get("fire_claim")):+ write_missed(d.job, d.next_run)+ d.scan.retire(d.job["id"])+ d.scan.on_saved.append(lambda job=d.job: record_cron_missed(job))+ d.scan.on_saved.append(lambda job=d.job, first=d.next_run: _book_missed(+ job, 1, first, "one-shot past its grace window; catch-up off"))+ return Truediff --git a/cron/scheduler.py b/cron/scheduler.pyindex afaf1494958..c618230fcfb 100644--- a/cron/scheduler.py+++ b/cron/scheduler.py@@ -30,7 +30,7 @@ except ImportError:except ImportError:msvcrt = Nonefrom pathlib import Path-from typing import Any, Callable, Dict, List, Optional, Protocol, Union+from typing import Any, Callable, Dict, List, Optional, Union# Must precede repo-level imports: standalone invocations (e.g. module reload after# `hermes update`) otherwise fail with ModuleNotFoundError for hermes_time et al.@@ -531,7 +531,8 @@ from cron.executions import (recover_interrupted_executions, settle_unstarted_execution, terminalize_dead_owner)from cron.scheduler_liveness import ExecutionProgressStamper, _inactivity_watchdog_loopfrom cron.scheduler_bookkeeping import _classify_delivery_outcome-from cron.cancellation import CancelRequest, clear_cancel+from cron.cancellation import (CancelRequest, _CancelEventLike, _CombinedCancelEvent, cancel_reason,+ clear_cancel, finish_cancelled_run as _finish_cancelled_run)# Response marker that suppresses delivery (output is still saved locally for audit).SILENT_MARKER = "[SILENT]"@@ -658,43 +659,6 @@ _INFLIGHT_MIN_ALLOWANCE_MINUTES = 30.0_interrupted_job_ids: set = set()-class _CancelEventLike(Protocol):- """Structural type for cancellation sources (``threading.Event``, ``_CombinedCancelEvent``)."""-- def is_set(self) -> bool: ...- def set(self) -> None: ...---class _CombinedCancelEvent:- """Duck-typed ``threading.Event`` ORing several cancellation sources (fire-claim heartbeat- ``lost_ownership`` + per-transport events). Workers only call is_set()/set(), so no pump thread.- """-- def __init__(self, *events: Optional[_CancelEventLike]) -> None:- self._events = [event for event in events if event is not None]-- def is_set(self) -> bool:- return any(event.is_set() for event in self._events)-- def set(self) -> None:- for event in self._events:- event.set()-- @property- def reason(self) -> Optional[str]:- """Why the run is being torn down, from whichever source fired — so a script or agent- stop never reports "fire claim lost" for an operator cancel (or the reverse)."""- for event in self._events:- if event.is_set():- return getattr(event, "reason", None)- return None---def cancel_reason(cancel_event: Optional[_CancelEventLike]) -> str:- """Human-facing cause for a cancelled run; the legacy wording when no source carries one."""- return getattr(cancel_event, "reason", None) or "cron fire ownership was lost"--def get_running_job_ids() -> frozenset[str]:"""Thread-safe snapshot of executing job IDs (dispatch until ``_process_job`` returns). Read bythe gateway shutdown drain, otherwise blind to cron work (runs outside ``_running_agents``).@@ -3026,6 +2990,7 @@ class _FireOwnership:# ``set()`` would propagate into the transport event this run does not own (#105861).self.claim_lost = claim_lostself.transport_cancel = transport_cancel+ self.operator_cancel = operator_cancel# ``operator_cancel`` (``hermes cron cancel``) is a THIRD source, deliberately not folded# into ``transport_cancel``: only a transport cancel counts as claim loss for bookkeeping,# while an operator cancel is a real outcome this run records itself.@@ -3040,6 +3005,9 @@ class _FireOwnership:def transport_cancelled(self) -> bool:return self.transport_cancel is not None and self.transport_cancel.is_set()+ def operator_cancelled(self) -> bool:+ return self.operator_cancel is not None and self.operator_cancel.is_set()+def side_effect_fence(self):if self.owner is None:return contextlib.nullcontext(True)@@ -3384,11 +3352,18 @@ def _run_one_job_body(_run_kwargs["cancel_event"] = fence.cancel_eventtry:success, output, final_response, error = run_job(job, **_run_kwargs)- except BaseException:+ except BaseException as exc:# run_job hands back the agent even when raising; tear down so a failed run never leaks.# BaseException so KeyboardInterrupt/SystemExit mid-run still trigger teardown._teardown_deferred()- raise+ if not (isinstance(exc, Exception) and fence.operator_cancelled()):+ raise+ success = None+ # Before claim loss: removing a job mid-run cancels it AND drops its claim (#123045).+ if fence.operator_cancelled():+ if success is not None: # the except path above already tore down+ _teardown_deferred()+ return _finish_cancelled_run(job, execution_id, fire_owner)if _fire_claim_ownership_lost():_teardown_deferred()diff --git a/cron/scheduler_script.py b/cron/scheduler_script.pyindex bc9ecca2277..97807e3f124 100644--- a/cron/scheduler_script.py+++ b/cron/scheduler_script.py@@ -27,7 +27,7 @@ from typing import Any, Callable, Optional, TYPE_CHECKINGfrom hermes_cli._subprocess_compat import windows_hide_flagsif TYPE_CHECKING:- from cron.scheduler import _CancelEventLike+ from cron.cancellation import _CancelEventLike# Log-record parity with the origin module.logger = logging.getLogger("cron.scheduler")@@ -495,7 +495,8 @@ def _run_job_script(if cancel_event is not None and cancel_event.is_set():_terminate_cron_script_tree(proc)_drain_script_pipes(proc)- return False, f"Script cancelled: {_sched.cancel_reason(cancel_event)}"+ from cron.cancellation import cancel_reason+ return False, f"Script cancelled: {cancel_reason(cancel_event)}"remaining = deadline - time.monotonic()if remaining <= 0:_terminate_cron_script_tree(proc)diff --git a/hermes_cli/cron.py b/hermes_cli/cron.pyindex 171702fdef8..4efcd4f2baf 100644--- a/hermes_cli/cron.py+++ b/hermes_cli/cron.py@@ -779,7 +779,7 @@ _JOB_ARG_FIELDS = (("name", "name"), ("deliver", "deliver"), ("failure_deliver",("monitor_script", "monitor_script"), ("monitor_url", "monitor_url"),("continuity", "continuity"), ("reasoning_effort", "reasoning_effort"),("interpreter", "interpreter"), ("timezone", "timezone"),- ("grace_minutes", "grace_minutes"))+ ("grace_minutes", "grace_minutes"), ("catch_up", "catch_up"))def _job_api_kwargs(args) -> dict[str, Any]:diff --git a/hermes_cli/subcommands/cron.py b/hermes_cli/subcommands/cron.pyindex 53707af8e3a..629bb72e735 100644--- a/hermes_cli/subcommands/cron.py+++ b/hermes_cli/subcommands/cron.py@@ -2,6 +2,7 @@from __future__ import annotations+import argparsefrom typing import Callablefrom hermes_cli.subcommands._shared import add_accept_hooks_flag@@ -11,6 +12,20 @@ def _flag(parser, *names, help, **kw):parser.add_argument(*names, action="store_true", help=help, **kw)+def _catch_up_flag(value: str) -> bool:+ if value not in ("on", "off"):+ raise argparse.ArgumentTypeError("use 'on' (run missed runs once) or 'off' (skip them)")+ return value == "on"+++def _add_catch_up(parser) -> None:+ parser.add_argument(+ "--catch-up", dest="catch_up", type=_catch_up_flag, metavar="{on,off}",+ help="Runs missed while the machine was off: 'on' runs once on return, 'off' skips "+ "them (each is booked as a 'missed' row in `hermes cron runs`). Default: "+ "cron.catch_up_missed for recurring jobs; one-shots skip.")++def build_cron_parser(subparsers, *, cmd_cron: Callable) -> None:"""Attach the ``cron`` subcommand (and its sub-actions) to ``subparsers``."""cron_parser = subparsers.add_parser(@@ -36,6 +51,7 @@ def build_cron_parser(subparsers, *, cmd_cron: Callable) -> None:"state stays visible in `hermes cron list`. Omit = failures ""follow --deliver.")cron_create.add_argument("--repeat", type=int, help="Optional repeat count")+ _add_catch_up(cron_create)cron_create.add_argument("--skill", dest="skills", action="append",help="Attach a skill. Repeat to add multiple skills.")cron_create.add_argument("--script",@@ -110,6 +126,7 @@ def build_cron_parser(subparsers, *, cmd_cron: Callable) -> None:help="Override target for failure notices (same grammar as --deliver; ""'local' suppresses; '' clears the override)")cron_edit.add_argument("--repeat", type=int, help="New repeat count")+ _add_catch_up(cron_edit)cron_edit.add_argument("--skill", dest="skills", action="append",help="Replace the job's skills with this set. Repeat to attach multiple skills.")cron_edit.add_argument("--add-skill", dest="add_skills", action="append",diff --git a/hermes_cli/web_routers/cron.py b/hermes_cli/web_routers/cron.pyindex 7827b1e294d..eb49f5b9a82 100644--- a/hermes_cli/web_routers/cron.py+++ b/hermes_cli/web_routers/cron.py@@ -298,14 +298,14 @@ def _owner_profile_executions(canonical_job_id: str) -> list[dict[str, Any]]:the owner-home scope so it opens the OWNER's cron/executions.db."""try:- from cron.executions import list_executions+ from cron.executions import _TERMINAL_STATES, list_executionsrows = list_executions(job_id=canonical_job_id, limit=100)except Exception:return []terminal: list[dict[str, Any]] = []for row in rows:- if str(row.get("status") or "") not in ("completed", "failed", "unknown"):+ if str(row.get("status") or "") not in _TERMINAL_STATES:continueterminal.append({"id": str(row.get("id") or ""),@@ -339,13 +339,20 @@ def _execution_contains(return finish is None or started_at <= finish + grace_seconds+# A cancelled or missed attempt's ``error`` is its reason ("job removed while running",+# "missed ×3 (...)"), the row's whole point — show it like a failure's.+_LABELS_WITH_REASON = frozenset({"FAILED", "CANCELLED", "MISSED"})+# Terminal attempts that carry a real outcome (``unknown`` is an owner that vanished).+_OUTCOME_STATES = ("completed", "failed", "cancelled", "missed")++def _execution_status_title(status: str, error: str, fallback: str) -> str:label = (status or "").replace("_", " ").upper()if label == "UNKNOWN":return fallbackif not label:return fallback- return f"{label} · {error}" if (label == "FAILED" and error) else f"{label} · {fallback}"+ return f"{label} · {error}" if (label in _LABELS_WITH_REASON and error) else f"{label} · {fallback}"def _list_cron_output_runs(@@ -393,7 +400,7 @@ def _list_cron_output_runs(**_cron_output_run_row(started_at, preview or "Script-only run", preview),})- if docs or any(attempt["status"] in ("completed", "failed") for attempt in attempts):+ if docs or any(attempt["status"] in _OUTCOME_STATES for attempt in attempts):return docs, attempts# No output docs survived (pruned or never written) but the job HAS run:@@ -559,7 +566,7 @@ def _reconcile_cron_runs(by_execution = {s["cron_execution_id"]: s for s in session_runs if s.get("cron_execution_id")}legacy = [s for s in session_runs if not s.get("cron_execution_id")]- terminal = [(index, a) for index, a in enumerate(attempts) if a["status"] in ("completed", "failed")]+ terminal = [(index, a) for index, a in enumerate(attempts) if a["status"] in _OUTCOME_STATES]represented = {index for index, a in terminal if a["id"] in by_execution}paired: set = set()merged = list(session_runs)@@ -681,6 +688,13 @@ def _trigger_cron_job_sync(job_id: str, profile: Optional[str] = None):return {**job, "enabled": False, "state": "completed"}+def _cancel_cron_job_sync(job_id: str, profile: Optional[str] = None):+ """Stop the job's live run; ``cancelled: false`` when nothing is in flight. Schedule untouched."""+ selected = _job_profile(job_id, profile)+ job = _found(_call_cron_for_profile(selected, "resolve_job_ref", job_id))+ return {**_call_cron_for_profile(selected, "cancel_job_run", job["id"]), "job_id": job["id"]}++def _delete_cron_job_sync(job_id: str, profile: Optional[str] = None):selected = _job_profile(job_id, profile)try:@@ -764,6 +778,11 @@ async def trigger_cron_job(job_id: str, profile: Optional[str] = None):return await _run_cron_dashboard_io(_trigger_cron_job_sync, job_id, profile)[email protected]("/api/cron/jobs/{job_id}/cancel")+async def cancel_cron_job(job_id: str, profile: Optional[str] = None):+ return await _run_cron_dashboard_io(_cancel_cron_job_sync, job_id, profile)++@router.delete("/api/cron/jobs/{job_id}")async def delete_cron_job(job_id: str, profile: Optional[str] = None):return await _run_cron_dashboard_io(_delete_cron_job_sync, job_id, profile)diff --git a/tests/cron/test_cron_stop_and_missed_policy.py b/tests/cron/test_cron_stop_and_missed_policy.pynew file mode 100644index 00000000000..015e4cd377c--- /dev/null+++ b/tests/cron/test_cron_stop_and_missed_policy.py@@ -0,0 +1,163 @@+"""Stopping a live cron run (cancel / remove) and the per-job missed-run policy.++Real store, real execution ledger, real cancel markers under a temp HERMES_HOME.+"""++from __future__ import annotations++import sqlite3+from datetime import timedelta++import pytest++from cron import jobs+++def home(tmp_path, monkeypatch):+ monkeypatch.setenv("HERMES_HOME", str(tmp_path))+ with jobs.use_cron_store(tmp_path / "cron"):+ yield tmp_path+++def _park(job_id: str, seconds_ago: float) -> None:+ stored = jobs.load_jobs()+ row = next(r for r in stored if r["id"] == job_id)+ row["next_run_at"] = (jobs._hermes_now() - timedelta(seconds=seconds_ago)).isoformat()+ jobs.save_jobs(stored)+++def _ledger(job_id: str) -> list[dict]:+ from cron.executions import list_executions++ return list_executions(job_id=job_id)+++def _live_run(job_id: str) -> str:+ from cron.executions import create_execution, mark_execution_running++ record = create_execution(job_id, source="test")+ mark_execution_running(record["id"])+ return record["id"]+++# --- missed-run policy ----------------------------------------------------------------------+++def test_catch_up_off_skips_and_books_one_missed_row_with_the_count(home):+ job = jobs.create_job(prompt="brief", schedule="every 1h", deliver="local", catch_up=False)+ _park(job["id"], 4 * 3600 + 60) # slots at -4h1m, -3h1m, -2h1m, -1h1m, -1m++ assert jobs.get_due_jobs() == []+ rows = _ledger(job["id"])+ assert [r["status"] for r in rows] == ["missed"]+ assert rows[0]["error"].startswith("missed \u00d75 (skipped")+ assert rows[0]["started_at"] is None+++def test_job_catch_up_beats_the_global_setting(home):+ (home / "config.yaml").write_text("cron:\n catch_up_missed: false\n", encoding="utf-8")+ job = jobs.create_job(prompt="scan", schedule="every 1h", deliver="local", catch_up=True)+ _park(job["id"], 3 * 3600 + 60)++ assert [j["id"] for j in jobs.get_due_jobs()] == [job["id"]]+ # Four slots passed; one runs now, three were collapsed into it.+ assert [(r["status"], r["error"].split(" (")[0]) for r in _ledger(job["id"])] == [+ ("missed", "missed \u00d73")]+++def test_global_setting_stays_the_default_for_jobs_without_their_own(home):+ (home / "config.yaml").write_text("cron:\n catch_up_missed: false\n", encoding="utf-8")+ job = jobs.create_job(prompt="brief", schedule="every 1h", deliver="local")+ _park(job["id"], 2 * 3600)++ assert jobs.get_due_jobs() == []+ assert [r["status"] for r in _ledger(job["id"])] == ["missed"]+++def test_one_shot_past_grace_skips_by_default_and_fires_once_when_opted_in(home):+ run_at = (jobs._hermes_now() + timedelta(hours=1)).isoformat()+ skipped = jobs.create_job(prompt="stale reminder", schedule=run_at, deliver="local")+ kept = jobs.create_job(prompt="late reminder", schedule=run_at, deliver="local", catch_up=True)+ for job in (skipped, kept):+ _park(job["id"], 3600)++ assert [j["id"] for j in jobs.get_due_jobs()] == [kept["id"]]+ assert jobs.get_job(skipped["id"]) is None+ assert [r["status"] for r in _ledger(skipped["id"])] == ["missed"]+ assert _ledger(kept["id"]) == []+++def test_catch_up_rejects_non_boolean(home):+ with pytest.raises(ValueError, match="catch_up"):+ jobs.create_job(prompt="x", schedule="every 1h", deliver="local", catch_up="sometimes")+++# --- ledger schema --------------------------------------------------------------------------+++def test_a_pre_existing_ledger_is_widened_without_losing_rows(home):+ db = home / "cron" / "executions.db"+ db.parent.mkdir(parents=True, exist_ok=True)+ conn = sqlite3.connect(db)+ conn.execute(+ """CREATE TABLE executions (+ id TEXT PRIMARY KEY, job_id TEXT NOT NULL, source TEXT NOT NULL,+ process_id TEXT NOT NULL, pid INTEGER NOT NULL, process_started_at INTEGER,+ status TEXT NOT NULL CHECK(status IN+ ('claimed','running','completed','failed','unknown')),+ claimed_at TEXT NOT NULL, started_at TEXT, finished_at TEXT, error TEXT)""")+ conn.execute(+ "INSERT INTO executions (id, job_id, source, process_id, pid, status, claimed_at) "+ "VALUES ('old', 'j1', 'builtin', 'p', 1, 'completed', '2026-01-01T00:00:00+00:00')")+ conn.commit()+ conn.close()++ from cron.executions import record_missed_occurrences++ record_missed_occurrences("j1", count=2, first_instant=None, detail="test")+ assert sorted(r["status"] for r in _ledger("j1")) == ["completed", "missed"]+++# --- stopping a live run --------------------------------------------------------------------+++def test_removing_a_job_stops_its_live_run(home):+ from cron.cancellation import CancelRequest++ job = jobs.create_job(prompt="long", schedule="every 1h", deliver="local")+ execution_id = _live_run(job["id"])++ assert jobs.remove_job(job["id"])+ request = CancelRequest(execution_id)+ assert request.is_set()+ assert request.reason == "job removed while running"+++def test_a_cancelled_run_books_cancelled_without_counting_as_a_failure(home):+ from cron import scheduler+ from cron.cancellation import request_cancel++ from cron.executions import create_execution++ job = jobs.create_job(prompt="long", schedule="every 1h", deliver="local")+ execution_id = create_execution(job["id"], source="test")["id"] # the body starts it+ request_cancel(execution_id, reason="operator asked")+ delivered: list = []++ def run_job(job, *, cancel_event=None, **_kw):+ assert cancel_event is not None and cancel_event.is_set()+ raise RuntimeError(f"Cron job was stopped: {scheduler.cancel_reason(cancel_event)}")++ with pytest.MonkeyPatch.context() as mp:+ mp.setattr(scheduler, "run_job", run_job)+ mp.setattr(scheduler, "_deliver_crash_failure",+ lambda *a, **k: delivered.append(a) or (None, "delivered"))+ assert scheduler._run_one_job_body(dict(job, execution_id=execution_id)) is True++ row = _ledger(job["id"])[0]+ assert (row["status"], row["error"]) == ("cancelled", "operator asked")+ stored = jobs.get_job(job["id"])+ assert stored["last_status"] == "cancelled"+ assert not stored.get("failure_streak")+ assert delivered == [] # the operator asked for it: no failure alertdiff --git a/tools/cronjob_tools.py b/tools/cronjob_tools.pyindex 4814c3facf5..a60947331c3 100644--- a/tools/cronjob_tools.py+++ b/tools/cronjob_tools.py@@ -734,6 +734,7 @@ def _action_create(a: dict[str, Any]) -> str:grace_minutes=a["grace_minutes"],pinned=bool(a["pinned"]),failure_deliver=_resolve_cron_context_deliver(_normalize_deliver_param(a["failure_deliver"])),+ catch_up=a["catch_up"],**({"paused": a["paused"], "paused_reason": a["paused_reason"]}if a["paused"] is not False or a["paused_reason"] is not None else {}))except CronSchedulerRegistrationError as exc:@@ -876,6 +877,8 @@ def _update_core_fields(job: dict[str, Any], a: dict[str, Any], updates: dict[streturn bot_chat_error_norm_fd = _resolve_cron_context_deliver(_norm_fd)updates["failure_deliver"] = _norm_fd+ if a["catch_up"] is not None:+ updates["catch_up"] = a["catch_up"]if skills is not None or skill is not None:canonical_skills = _canonical_skills(skill, skills)updates["skills"] = canonical_skills@@ -1005,7 +1008,7 @@ def _action_update(job: dict[str, Any], a: dict[str, Any]) -> str:_JOBLESS_ACTIONS = {"create": _action_create, "list": _action_list}-def _action_cancel(job: Dict[str, Any], a: Dict[str, Any]) -> str:+def _action_cancel(job: dict[str, Any], a: dict[str, Any]) -> str:"""Stop the job's LIVE run. Per-run, not per-schedule: the job keeps its next fire."""from cron.cancellation import cancel_run@@ -1068,6 +1071,7 @@ def cronjob(monitor_url: Optional[str] = None,reasoning_effort: Optional[str] = None,failure_deliver: Optional[str | list[str]] = None,+ catch_up: Optional[bool] = None,task_id: str | None = None,session_id: Optional[str] = None,paused: bool = False,@@ -1166,6 +1170,10 @@ Jobs run in a fresh session with no current-chat context, so prompts must be sel"items": {"type": "string"},"description": "Optional ordered skill names loaded before the cron prompt. On update, [] clears."},+ "catch_up": {+ "type": "boolean",+ "description": "Runs missed while the machine was off: true = run once on return, false = skip. Omit for the default (recurring run once, one-shots skip)."+ },"script": {"type": "string","description": _script_description("the profile HERMES_HOME")@@ -1233,7 +1241,7 @@ def check_cronjob_requirements() -> bool:_HANDLER_FORWARDED_ARGS = ("job_id", "prompt", "schedule", "name", "repeat", "deliver", "failure_deliver", "skill", "skills", "reason","script", "context_from", "continuity", "enabled_toolsets", "workdir", "no_agent", "attach_to_session",- "paused_reason", "pinned", "timezone", "grace_minutes")+ "paused_reason", "pinned", "timezone", "grace_minutes", "catch_up")def _cronjob_handler(args, **kw):