Skip to content

The agent that grows with you

fix(cron): a manual run is an extra fire, never the scheduled one

Tekniumf656543e030e
diff --git a/cron/extra_fire.py b/cron/extra_fire.py
new file mode 100644
index 00000000000..44f731ca6c4
--- /dev/null
+++ b/cron/extra_fire.py
@@ -0,0 +1,61 @@
+"""Manual runs are EXTRA fires: a run-now (``cronjob run``, ``hermes cron run``, dashboard Run,
+gateway-forwarded ``trigger_job``) leaves the job's pending scheduled occurrence and its
+``repeat`` budget alone. Without this a test-fire of an ``in 30m`` reminder completed the
+one-shot and the real reminder never fired; a recurring ``repeat.times=N`` job lost one of N.
+
+The claim (``claim_job_for_fire``) decides once, under the jobs lock, and stamps the pending
+slot on ``fire_claim["extra_slot"]``; every later step (``claim_dispatch``, ``mark_job_run``)
+reads that stamp, so a crash between claim and completion leaves ``next_run_at`` on the slot.
+"""
+from datetime import datetime, timedelta
+from typing import Any, Optional
+
+RESTORE_KEY = "manual_run_restore_at" # trigger_job's overwritten next_run_at
+_CLAIM_KEY = "extra_slot"
+
+
+def _future_slot(value: Any, now: datetime) -> Optional[str]:
+ from cron.jobs import _parse_aware
+
+ parsed = _parse_aware(value) if isinstance(value, str) else None
+ return value if parsed is not None and parsed > now else None
+
+
+def pending_slot(job: dict[str, Any], now: datetime, manual_fire: bool, skew_seconds: float) -> Optional[str]:
+ """The still-future scheduled instant an off-schedule fire must leave in place, or None when
+ this fire IS the scheduled occurrence (slot due, or no pending slot: nothing to preserve)."""
+ stored = job.get("next_run_at")
+ if job.get("manual_run_at") == stored: # trigger_job replaced next_run_at with "now"
+ return _future_slot(job.get(RESTORE_KEY), now)
+ if manual_fire:
+ return _future_slot(stored, now)
+ # An unforced external fire before the stored slot (dashboard Run on an active job): a
+ # scheduled tick only fires once next_run_at has arrived, within provider clock skew.
+ return _future_slot(stored, now + timedelta(seconds=skew_seconds))
+
+
+def stamp_claim(job: dict[str, Any], slot: str) -> None:
+ """Mark the fresh ``fire_claim`` as an extra fire and put ``next_run_at`` back on *slot* in the
+ same save, so the scheduled occurrence survives a crash before ``mark_job_run``."""
+ job["fire_claim"][_CLAIM_KEY] = slot
+ job["next_run_at"] = slot
+ job.pop(RESTORE_KEY, None)
+
+
+def is_extra(job: dict[str, Any]) -> bool:
+ claim = job.get("fire_claim")
+ return isinstance(claim, dict) and bool(claim.get(_CLAIM_KEY))
+
+
+def settle(job: dict[str, Any], now: str) -> None:
+ """After an extra fire: ``repeat.completed`` untouched, the job keeps its pending slot. A slot
+ that passed while the run was in flight (the tick lost the claim race) is not dropped: a
+ one-shot fires at once, a recurring job takes its next natural occurrence."""
+ from cron.jobs import compute_next_run
+
+ if _future_slot(job.get("next_run_at"), datetime.fromisoformat(now)):
+ return
+ if job.get("schedule", {}).get("kind") == "once":
+ job["next_run_at"] = now
+ else:
+ job["next_run_at"] = compute_next_run(job["schedule"], now) or job.get("next_run_at")
diff --git a/cron/jobs.py b/cron/jobs.py
index 1226d06d30d..5db362c9820 100644
--- a/cron/jobs.py
+++ b/cron/jobs.py
@@ -35,7 +35,7 @@ from cron.env_settings import cron_env_setting
from cron.jobs_invariants import _validate_job_mode_invariants
from cron.jobs_model_pin import _main_model_pin
from cron.scheduler_ownership import _claim_owner_is_dead
-from cron import store_health
+from cron import extra_fire, store_health
from typing import Optional, Dict, List, Any, Callable, Set, Tuple, Union, Collection
logger = logging.getLogger(__name__)
@@ -2178,8 +2178,9 @@ def trigger_job(job_id: str, extra_prompt: Optional[str] = None) -> Optional[dic
"paused_at": None,
"paused_reason": None,
"next_run_at": manual_run_at,
- # Run-now intent, so cron expression/TZ repair guards don't treat it as stale state.
- "manual_run_at": manual_run_at,
+ # Run-now intent, so cron expression/TZ repair guards don't treat it as stale state; the
+ # overwritten slot comes back at claim time (an extra fire, cron/extra_fire.py).
+ "manual_run_at": manual_run_at, "manual_run_restore_at": job.get("next_run_at"),
"manual_run_prompt": (extra_prompt or None),
})
@@ -2322,6 +2323,7 @@ def _record_run_outcome(
"""Stamp one completed run onto *job*: status fields, failure streak, alert markers, claims."""
job["last_run_at"] = now
job.pop("manual_run_at", None)
+ job.pop("manual_run_restore_at", None)
# The transient manual-run context is single-fire: the run that just completed consumed it.
job.pop("manual_run_prompt", None)
delivery_failed = isinstance(delivery_error, str) and bool(delivery_error.strip())
@@ -2418,7 +2420,8 @@ def mark_job_run(
recover_consumed_fire: bool = False,
) -> bool:
"""Mark a job as run: update last_run_at/last_status, bump completed, recompute next_run_at,
- and retire the record as a terminal completion when the repeat limit is reached.
+ and retire the record as a terminal completion when the repeat limit is reached. A manual run
+ (an extra fire, ``cron/extra_fire.py``) bumps nothing and keeps the pending occurrence.
``delivery_error`` is separate from the agent error: agent succeeded but delivery failed records
``last_status = "delivery_failed"`` (never "ok") while ``failure_streak`` is left alone. An
@@ -2449,11 +2452,15 @@ def mark_job_run(
job_id)
return False
now = _hermes_now().isoformat()
+ extra = extra_fire.is_extra(job) # read before the outcome clears the claim
_record_run_outcome(job, success, error, delivery_error, status, now)
if execution_id is not None: # retire ids recovery can no longer replay; latest stays last
from cron.scheduler_authority import replayable_completions as live
job["canonical_completions"] = [*live(job_id, job.get("canonical_completions")), execution_id]
- _advance_after_run(job, now, ladder_rung=ladder_rung)
+ if extra: # a manual run: repeat budget and pending occurrence untouched
+ extra_fire.settle(job, now)
+ else:
+ _advance_after_run(job, now, ladder_rung=ladder_rung)
from cron import quota_hold
from cron.unreachable_retry import clear_state, plan_retry
@@ -2479,71 +2486,6 @@ def mark_job_run(
return _under_fire_fence(job_id, locked)
-def _write_oneshot_diagnostic(job: dict[str, Any], text: str, what: str) -> bool:
- """Best-effort operator-visible trace in the job's output dir; never breaks the caller."""
- try:
- save_job_output(job.get("id", ""), text)
- return True
- except Exception as e:
- logger.debug("Failed to write %s diagnostic for job %r: %s", what, job.get("id"), e)
- return False
-
-
-def _write_wedged_oneshot_diagnostic(job: dict[str, Any]) -> None:
- """Trace for a wedged one-shot removal: dispatch was claimed but mark_job_run never ran
- (interrupted mid-run); removing it silently would leave no output, error, or record.
-
- A finite one-shot whose dispatch was claimed (``repeat.completed`` >= ``repeat.times``) but which never
- reached ``mark_job_run`` (``last_run_at`` is null) was interrupted mid-run — scheduler restart, gateway
- kill, or a non-Exception escape (#73973). The recovery guards remove such jobs so they stop appearing
- due, but a silent removal leaves the user with no output, no error, and no job record. Write a small
- diagnostic file into the job's output directory so the removal is observable and debuggable.
- """
- if job.get("last_run_at") is not None:
- return # a prior run was recorded — normal completion race, not a wedge
- repeat = job.get("repeat") or {}
- claim = job.get("run_claim") or {}
- written = _write_oneshot_diagnostic(
- job,
- "# Cron job removed without producing output\n\n"
- f"- job id: {job.get('id')}\n"
- f"- name: {job.get('name')}\n"
- f"- dispatch claimed: {repeat.get('completed', '?')}/{repeat.get('times', '?')}\n"
- f"- run claimed at: {claim.get('at', 'unknown')} by {claim.get('by', 'unknown')}\n"
- f"- removed at: {_hermes_now().isoformat()}\n\n"
- "This one-shot job's dispatch was claimed, but the run never "
- "completed (`last_run_at` was never written) — the scheduler "
- "process was most likely killed or restarted mid-execution. The "
- "job has been removed to stop it re-firing; recreate it to run "
- "again.\n",
- "wedged-oneshot")
- if written:
- logger.warning(
- "Job '%s': removed without a completed run — diagnostic written to "
- "its output directory",
- job.get("name", job.get("id", "?")))
-
-
-def _write_missed_oneshot_diagnostic(job: dict[str, Any], next_run: str) -> None:
- """Trace for a never-ran one-shot retired outside the grace window (else it would just vanish).
- """
- _write_oneshot_diagnostic(
- job,
- "# Cron job removed before firing (run time outside grace window)\n\n"
- f"- job id: {job.get('id')}\n"
- f"- name: {job.get('name')}\n"
- f"- scheduled run time: {next_run}\n"
- f"- grace window: {ONESHOT_GRACE_SECONDS}s\n"
- f"- removed at: {_hermes_now().isoformat()}\n\n"
- "This one-shot's run time is more than the grace window in the "
- "past (scheduler down past the window, host asleep, or jobs.json "
- "edited), which is outside the 'will never fire' contract "
- "enforced at create/update/resume time. The job was removed "
- "without running; recreate it (or use the Run button) to "
- "schedule it again.\n",
- "missed-oneshot")
-
-
def claim_dispatch(job_id: str) -> bool:
"""Atomically claim a finite one-shot dispatch BEFORE execution: ``repeat.completed`` is bumped
and persisted under the jobs lock so a tick dying mid-execution cannot lose the dispatch
@@ -2561,6 +2503,8 @@ def claim_dispatch(job_id: str) -> bool:
# Recurring jobs use advance_next_run(); no/infinite repeat limit always dispatches.
if job.get("schedule", {}).get("kind") != "once" or times is None or times <= 0:
return True
+ if extra_fire.is_extra(job): # a manual run leaves the scheduled fire's budget alone
+ return True
completed = repeat.get("completed", 0)
label = job.get("name", job.get("id", "?"))
if completed >= times:
@@ -2579,7 +2523,8 @@ def claim_dispatch(job_id: str) -> bool:
jobs.pop(i)
# See #73973.
save_jobs(jobs, removed_ids={job_id})
- _write_wedged_oneshot_diagnostic(job)
+ from cron.jobs_oneshot_diagnostics import write_wedged
+ write_wedged(job)
logger.info(
"Job '%s': dispatch limit reached (%d/%d) — removing", label, completed, times)
return False
@@ -2754,7 +2699,9 @@ def claim_job_for_fire(
job["fire_claim"] = {"at": now.isoformat(), "by": f"{_machine_id()}:{uuid.uuid4().hex}"}
# Claimed: the occurrence is now owned by a run (its ledger row + fire claim carry it).
job.pop("pending_slot", None)
- if job.get("schedule", {}).get("kind") in {"cron", "interval"}:
+ if slot := extra_fire.pending_slot(job, now, manual_fire, FIRE_CLAIM_SKEW_SECONDS):
+ extra_fire.stamp_claim(job, slot) # off-schedule: the pending occurrence stays
+ elif job.get("schedule", {}).get("kind") in {"cron", "interval"}:
nxt = compute_next_run(job["schedule"], now.isoformat())
if nxt:
job["next_run_at"] = nxt
@@ -3124,7 +3071,8 @@ def _retire_expired_oneshot(d: _DueJob) -> bool:
if isinstance(claim, dict) and claim.get("outage") and not d.job.get("fire_claim"):
return False
if not (claim or d.job.get("fire_claim")):
- _write_missed_oneshot_diagnostic(d.job, d.next_run)
+ from cron.jobs_oneshot_diagnostics import write_missed
+ write_missed(d.job, d.next_run)
d.scan.retire(d.job["id"])
d.scan.on_saved.append(lambda job=d.job: record_cron_missed(job))
return True
@@ -3168,7 +3116,8 @@ def _oneshot_dispatch_limit_reached(job: dict[str, Any], scan: _DueScan) -> bool
name, completed, times)
scan.retire(job["id"])
# The claimed run never completed here by definition — leave an operator-visible diagnostic.
- _write_wedged_oneshot_diagnostic(job)
+ from cron.jobs_oneshot_diagnostics import write_wedged
+ write_wedged(job)
return True
diff --git a/cron/jobs_oneshot_diagnostics.py b/cron/jobs_oneshot_diagnostics.py
new file mode 100644
index 00000000000..037ca8f5bc8
--- /dev/null
+++ b/cron/jobs_oneshot_diagnostics.py
@@ -0,0 +1,77 @@
+"""Operator-visible traces left in a job's output dir when the due scan or the dispatch-limit
+guard removes a one-shot that never completed (``cron/jobs.py`` late-imports these)."""
+import logging
+from typing import Any
+
+logger = logging.getLogger("cron.jobs")
+
+
+def _write(job: dict[str, Any], text: str, what: str) -> bool:
+ """Best-effort operator-visible trace in the job's output dir; never breaks the caller."""
+ from cron.jobs import save_job_output
+
+ try:
+ save_job_output(job.get("id", ""), text)
+ return True
+ except Exception as e:
+ logger.debug("Failed to write %s diagnostic for job %r: %s", what, job.get("id"), e)
+ return False
+
+
+def write_wedged(job: dict[str, Any]) -> None:
+ """Trace for a wedged one-shot removal: dispatch was claimed but mark_job_run never ran
+ (interrupted mid-run); removing it silently would leave no output, error, or record.
+
+ A finite one-shot whose dispatch was claimed (``repeat.completed`` >= ``repeat.times``) but which never
+ reached ``mark_job_run`` (``last_run_at`` is null) was interrupted mid-run — scheduler restart, gateway
+ kill, or a non-Exception escape (#73973). The recovery guards remove such jobs so they stop appearing
+ due, but a silent removal leaves the user with no output, no error, and no job record. Write a small
+ diagnostic file into the job's output directory so the removal is observable and debuggable.
+ """
+ from cron.jobs import _hermes_now
+
+ if job.get("last_run_at") is not None:
+ return # a prior run was recorded — normal completion race, not a wedge
+ repeat = job.get("repeat") or {}
+ claim = job.get("run_claim") or {}
+ written = _write(
+ job,
+ "# Cron job removed without producing output\n\n"
+ f"- job id: {job.get('id')}\n"
+ f"- name: {job.get('name')}\n"
+ f"- dispatch claimed: {repeat.get('completed', '?')}/{repeat.get('times', '?')}\n"
+ f"- run claimed at: {claim.get('at', 'unknown')} by {claim.get('by', 'unknown')}\n"
+ f"- removed at: {_hermes_now().isoformat()}\n\n"
+ "This one-shot job's dispatch was claimed, but the run never "
+ "completed (`last_run_at` was never written) — the scheduler "
+ "process was most likely killed or restarted mid-execution. The "
+ "job has been removed to stop it re-firing; recreate it to run "
+ "again.\n",
+ "wedged-oneshot")
+ if written:
+ logger.warning(
+ "Job '%s': removed without a completed run — diagnostic written to "
+ "its output directory",
+ job.get("name", job.get("id", "?")))
+
+
+def write_missed(job: dict[str, Any], next_run: str) -> None:
+ """Trace for a never-ran one-shot retired outside the grace window (else it would just vanish).
+ """
+ from cron.jobs import ONESHOT_GRACE_SECONDS, _hermes_now
+
+ _write(
+ job,
+ "# Cron job removed before firing (run time outside grace window)\n\n"
+ f"- job id: {job.get('id')}\n"
+ f"- name: {job.get('name')}\n"
+ f"- scheduled run time: {next_run}\n"
+ f"- grace window: {ONESHOT_GRACE_SECONDS}s\n"
+ f"- removed at: {_hermes_now().isoformat()}\n\n"
+ "This one-shot's run time is more than the grace window in the "
+ "past (scheduler down past the window, host asleep, or jobs.json "
+ "edited), which is outside the 'will never fire' contract "
+ "enforced at create/update/resume time. The job was removed "
+ "without running; recreate it (or use the Run button) to "
+ "schedule it again.\n",
+ "missed-oneshot")
diff --git a/tests/cron/test_manual_run_extra_fire.py b/tests/cron/test_manual_run_extra_fire.py
new file mode 100644
index 00000000000..59789364893
--- /dev/null
+++ b/tests/cron/test_manual_run_extra_fire.py
@@ -0,0 +1,139 @@
+"""A manual run is an EXTRA fire: it never consumes the job's scheduled occurrence or repeat budget.
+
+Each test drives the real bookkeeping sequence a fire goes through (claim_job_for_fire ->
+claim_dispatch -> mark_job_run), the same calls the cronjob tool, the dashboard and the
+ticker make around the agent run.
+"""
+from datetime import datetime, timedelta
+
+import pytest
+
+from cron.jobs import (
+ claim_dispatch,
+ claim_job_for_fire,
+ create_job,
+ get_due_jobs,
+ get_job,
+ load_jobs,
+ mark_job_run,
+ save_jobs,
+ trigger_job,
+)
+
+
+def clock(tmp_path, monkeypatch):
+ monkeypatch.setattr("cron.jobs.CRON_DIR", tmp_path / "cron")
+ monkeypatch.setattr("cron.jobs.JOBS_FILE", tmp_path / "cron" / "jobs.json")
+ monkeypatch.setattr("cron.jobs.OUTPUT_DIR", tmp_path / "cron" / "output")
+ state = {"now": datetime.fromisoformat("2026-10-28T09:00:00-07:00")}
+ monkeypatch.setattr("cron.jobs._hermes_now", lambda: state["now"])
+ return state
+
+
+def _fire(job_id, **claim_kwargs):
+ claimed = claim_job_for_fire(job_id, return_job=True, **claim_kwargs)
+ assert isinstance(claimed, dict), "fire was not claimed"
+ assert claim_dispatch(job_id)
+ assert mark_job_run(job_id, success=True, expected_fire_owner=claimed["fire_claim"]["by"])
+
+
+def test_manual_run_keeps_the_scheduled_oneshot_and_it_still_fires(clock):
+ job = create_job("remind me", "in 30m", name="reminder")
+ slot = job["next_run_at"]
+
+ _fire(job["id"], manual=True) # cronjob(action="run") / hermes cron run
+
+ after = get_job(job["id"])
+ assert (after["state"], after["enabled"], after["next_run_at"]) == ("scheduled", True, slot)
+ assert after["repeat"] == {"times": 1, "completed": 0}
+ assert after["last_status"] == "ok"
+
+ clock["now"] += timedelta(minutes=30, seconds=5)
+ assert [d["id"] for d in get_due_jobs()] == [job["id"]]
+ _fire(job["id"]) # the scheduled tick
+
+ done = get_job(job["id"])
+ assert (done["state"], done["next_run_at"]) == ("completed", None)
+ assert done["repeat"] == {"times": 1, "completed": 1}
+
+
+def test_gateway_forwarded_trigger_restores_the_slot_at_claim_time(clock):
+ """trigger_job overwrites next_run_at with "now"; the claim puts the slot back in the same
+ save, so a crash before mark_job_run cannot lose the scheduled occurrence."""
+ job = create_job("remind me", "in 30m", name="reminder")
+ slot = job["next_run_at"]
+ trigger_job(job["id"])
+ assert [d["id"] for d in get_due_jobs()] == [job["id"]]
+
+ claimed = claim_job_for_fire(job["id"], return_job=True) # the ticker's claim
+ assert isinstance(claimed, dict)
+ assert get_job(job["id"])["next_run_at"] == slot # durable before the run starts
+
+ assert claim_dispatch(job["id"])
+ assert mark_job_run(job["id"], success=True, expected_fire_owner=claimed["fire_claim"]["by"])
+ after = get_job(job["id"])
+ assert (after["state"], after["next_run_at"]) == ("scheduled", slot)
+ assert after["repeat"]["completed"] == 0
+ assert "manual_run_restore_at" not in after
+
+
+def test_manual_run_does_not_spend_a_recurring_repeat_budget_or_shift_the_schedule(clock):
+ job = create_job("digest", "every 60m", name="digest", repeat=2)
+ slot = job["next_run_at"]
+ clock["now"] += timedelta(minutes=20)
+
+ _fire(job["id"], manual=True)
+
+ after = get_job(job["id"])
+ assert after["repeat"] == {"times": 2, "completed": 0}
+ assert after["next_run_at"] == slot
+
+
+def test_dashboard_run_before_the_slot_is_an_extra_fire(clock):
+ """The dashboard Run button claims without manual=True; firing ahead of the stored slot is
+ still off-schedule."""
+ job = create_job("remind me", "in 30m", name="reminder")
+
+ _fire(job["id"])
+
+ after = get_job(job["id"])
+ assert (after["state"], after["next_run_at"]) == ("scheduled", job["next_run_at"])
+ assert after["repeat"]["completed"] == 0
+
+
+def test_crash_mid_manual_run_leaves_the_scheduled_oneshot_due(clock):
+ job = create_job("remind me", "in 30m", name="reminder")
+ assert isinstance(claim_job_for_fire(job["id"], manual=True, return_job=True), dict)
+ assert claim_dispatch(job["id"])
+ # The runner dies here: no mark_job_run, and its claim's owner is gone.
+ record = load_jobs()[0]
+ record["fire_claim"]["by"] = "dead-host:0"
+ record["fire_claim"]["at"] = (clock["now"] - timedelta(hours=2)).isoformat()
+ save_jobs([record])
+
+ clock["now"] += timedelta(minutes=30, seconds=5)
+ assert [d["id"] for d in get_due_jobs()] == [job["id"]]
+ assert get_job(job["id"])["repeat"]["completed"] == 0
+
+
+def test_a_slot_that_passes_during_the_manual_run_still_fires(clock):
+ job = create_job("remind me", "in 2m", name="reminder")
+ claimed = claim_job_for_fire(job["id"], manual=True, return_job=True)
+ assert claim_dispatch(job["id"])
+ clock["now"] += timedelta(minutes=5) # long run: the tick lost the claim race at the slot
+
+ assert mark_job_run(job["id"], success=True, expected_fire_owner=claimed["fire_claim"]["by"])
+
+ assert [d["id"] for d in get_due_jobs()] == [job["id"]]
+
+
+def test_run_at_or_after_the_due_slot_is_the_scheduled_fire(clock):
+ """Control: a manual run once the one-shot is already due consumes it (no double fire)."""
+ job = create_job("remind me", "in 2m", name="reminder")
+ clock["now"] += timedelta(minutes=2, seconds=10)
+
+ _fire(job["id"], manual=True)
+
+ done = get_job(job["id"])
+ assert (done["state"], done["repeat"]["completed"]) == ("completed", 1)