The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(cron): a manual run is an extra fire, never the scheduled one
Tekniumf656543e030e
diff --git a/cron/extra_fire.py b/cron/extra_fire.pynew file mode 100644index 00000000000..44f731ca6c4--- /dev/null+++ b/cron/extra_fire.py@@ -0,0 +1,61 @@+"""Manual runs are EXTRA fires: a run-now (``cronjob run``, ``hermes cron run``, dashboard Run,+gateway-forwarded ``trigger_job``) leaves the job's pending scheduled occurrence and its+``repeat`` budget alone. Without this a test-fire of an ``in 30m`` reminder completed the+one-shot and the real reminder never fired; a recurring ``repeat.times=N`` job lost one of N.++The claim (``claim_job_for_fire``) decides once, under the jobs lock, and stamps the pending+slot on ``fire_claim["extra_slot"]``; every later step (``claim_dispatch``, ``mark_job_run``)+reads that stamp, so a crash between claim and completion leaves ``next_run_at`` on the slot.+"""+from datetime import datetime, timedelta+from typing import Any, Optional++RESTORE_KEY = "manual_run_restore_at" # trigger_job's overwritten next_run_at+_CLAIM_KEY = "extra_slot"+++def _future_slot(value: Any, now: datetime) -> Optional[str]:+ from cron.jobs import _parse_aware++ parsed = _parse_aware(value) if isinstance(value, str) else None+ return value if parsed is not None and parsed > now else None+++def pending_slot(job: dict[str, Any], now: datetime, manual_fire: bool, skew_seconds: float) -> Optional[str]:+ """The still-future scheduled instant an off-schedule fire must leave in place, or None when+ this fire IS the scheduled occurrence (slot due, or no pending slot: nothing to preserve)."""+ stored = job.get("next_run_at")+ if job.get("manual_run_at") == stored: # trigger_job replaced next_run_at with "now"+ return _future_slot(job.get(RESTORE_KEY), now)+ if manual_fire:+ return _future_slot(stored, now)+ # An unforced external fire before the stored slot (dashboard Run on an active job): a+ # scheduled tick only fires once next_run_at has arrived, within provider clock skew.+ return _future_slot(stored, now + timedelta(seconds=skew_seconds))+++def stamp_claim(job: dict[str, Any], slot: str) -> None:+ """Mark the fresh ``fire_claim`` as an extra fire and put ``next_run_at`` back on *slot* in the+ same save, so the scheduled occurrence survives a crash before ``mark_job_run``."""+ job["fire_claim"][_CLAIM_KEY] = slot+ job["next_run_at"] = slot+ job.pop(RESTORE_KEY, None)+++def is_extra(job: dict[str, Any]) -> bool:+ claim = job.get("fire_claim")+ return isinstance(claim, dict) and bool(claim.get(_CLAIM_KEY))+++def settle(job: dict[str, Any], now: str) -> None:+ """After an extra fire: ``repeat.completed`` untouched, the job keeps its pending slot. A slot+ that passed while the run was in flight (the tick lost the claim race) is not dropped: a+ one-shot fires at once, a recurring job takes its next natural occurrence."""+ from cron.jobs import compute_next_run++ if _future_slot(job.get("next_run_at"), datetime.fromisoformat(now)):+ return+ if job.get("schedule", {}).get("kind") == "once":+ job["next_run_at"] = now+ else:+ job["next_run_at"] = compute_next_run(job["schedule"], now) or job.get("next_run_at")diff --git a/cron/jobs.py b/cron/jobs.pyindex 1226d06d30d..5db362c9820 100644--- a/cron/jobs.py+++ b/cron/jobs.py@@ -35,7 +35,7 @@ from cron.env_settings import cron_env_settingfrom cron.jobs_invariants import _validate_job_mode_invariantsfrom cron.jobs_model_pin import _main_model_pinfrom cron.scheduler_ownership import _claim_owner_is_dead-from cron import store_health+from cron import extra_fire, store_healthfrom typing import Optional, Dict, List, Any, Callable, Set, Tuple, Union, Collectionlogger = logging.getLogger(__name__)@@ -2178,8 +2178,9 @@ def trigger_job(job_id: str, extra_prompt: Optional[str] = None) -> Optional[dic"paused_at": None,"paused_reason": None,"next_run_at": manual_run_at,- # Run-now intent, so cron expression/TZ repair guards don't treat it as stale state.- "manual_run_at": manual_run_at,+ # Run-now intent, so cron expression/TZ repair guards don't treat it as stale state; the+ # overwritten slot comes back at claim time (an extra fire, cron/extra_fire.py).+ "manual_run_at": manual_run_at, "manual_run_restore_at": job.get("next_run_at"),"manual_run_prompt": (extra_prompt or None),})@@ -2322,6 +2323,7 @@ def _record_run_outcome("""Stamp one completed run onto *job*: status fields, failure streak, alert markers, claims."""job["last_run_at"] = nowjob.pop("manual_run_at", None)+ job.pop("manual_run_restore_at", None)# The transient manual-run context is single-fire: the run that just completed consumed it.job.pop("manual_run_prompt", None)delivery_failed = isinstance(delivery_error, str) and bool(delivery_error.strip())@@ -2418,7 +2420,8 @@ def mark_job_run(recover_consumed_fire: bool = False,) -> bool:"""Mark a job as run: update last_run_at/last_status, bump completed, recompute next_run_at,- and retire the record as a terminal completion when the repeat limit is reached.+ and retire the record as a terminal completion when the repeat limit is reached. A manual run+ (an extra fire, ``cron/extra_fire.py``) bumps nothing and keeps the pending occurrence.``delivery_error`` is separate from the agent error: agent succeeded but delivery failed records``last_status = "delivery_failed"`` (never "ok") while ``failure_streak`` is left alone. An@@ -2449,11 +2452,15 @@ def mark_job_run(job_id)return Falsenow = _hermes_now().isoformat()+ extra = extra_fire.is_extra(job) # read before the outcome clears the claim_record_run_outcome(job, success, error, delivery_error, status, now)if execution_id is not None: # retire ids recovery can no longer replay; latest stays lastfrom cron.scheduler_authority import replayable_completions as livejob["canonical_completions"] = [*live(job_id, job.get("canonical_completions")), execution_id]- _advance_after_run(job, now, ladder_rung=ladder_rung)+ if extra: # a manual run: repeat budget and pending occurrence untouched+ extra_fire.settle(job, now)+ else:+ _advance_after_run(job, now, ladder_rung=ladder_rung)from cron import quota_holdfrom cron.unreachable_retry import clear_state, plan_retry@@ -2479,71 +2486,6 @@ def mark_job_run(return _under_fire_fence(job_id, locked)-def _write_oneshot_diagnostic(job: dict[str, Any], text: str, what: str) -> bool:- """Best-effort operator-visible trace in the job's output dir; never breaks the caller."""- try:- save_job_output(job.get("id", ""), text)- return True- except Exception as e:- logger.debug("Failed to write %s diagnostic for job %r: %s", what, job.get("id"), e)- return False---def _write_wedged_oneshot_diagnostic(job: dict[str, Any]) -> None:- """Trace for a wedged one-shot removal: dispatch was claimed but mark_job_run never ran- (interrupted mid-run); removing it silently would leave no output, error, or record.-- A finite one-shot whose dispatch was claimed (``repeat.completed`` >= ``repeat.times``) but which never- reached ``mark_job_run`` (``last_run_at`` is null) was interrupted mid-run — scheduler restart, gateway- kill, or a non-Exception escape (#73973). The recovery guards remove such jobs so they stop appearing- due, but a silent removal leaves the user with no output, no error, and no job record. Write a small- diagnostic file into the job's output directory so the removal is observable and debuggable.- """- if job.get("last_run_at") is not None:- return # a prior run was recorded — normal completion race, not a wedge- repeat = job.get("repeat") or {}- claim = job.get("run_claim") or {}- written = _write_oneshot_diagnostic(- job,- "# Cron job removed without producing output\n\n"- f"- job id: {job.get('id')}\n"- f"- name: {job.get('name')}\n"- f"- dispatch claimed: {repeat.get('completed', '?')}/{repeat.get('times', '?')}\n"- f"- run claimed at: {claim.get('at', 'unknown')} by {claim.get('by', 'unknown')}\n"- f"- removed at: {_hermes_now().isoformat()}\n\n"- "This one-shot job's dispatch was claimed, but the run never "- "completed (`last_run_at` was never written) — the scheduler "- "process was most likely killed or restarted mid-execution. The "- "job has been removed to stop it re-firing; recreate it to run "- "again.\n",- "wedged-oneshot")- if written:- logger.warning(- "Job '%s': removed without a completed run — diagnostic written to "- "its output directory",- job.get("name", job.get("id", "?")))---def _write_missed_oneshot_diagnostic(job: dict[str, Any], next_run: str) -> None:- """Trace for a never-ran one-shot retired outside the grace window (else it would just vanish).- """- _write_oneshot_diagnostic(- job,- "# Cron job removed before firing (run time outside grace window)\n\n"- f"- job id: {job.get('id')}\n"- f"- name: {job.get('name')}\n"- f"- scheduled run time: {next_run}\n"- f"- grace window: {ONESHOT_GRACE_SECONDS}s\n"- f"- removed at: {_hermes_now().isoformat()}\n\n"- "This one-shot's run time is more than the grace window in the "- "past (scheduler down past the window, host asleep, or jobs.json "- "edited), which is outside the 'will never fire' contract "- "enforced at create/update/resume time. The job was removed "- "without running; recreate it (or use the Run button) to "- "schedule it again.\n",- "missed-oneshot")--def claim_dispatch(job_id: str) -> bool:"""Atomically claim a finite one-shot dispatch BEFORE execution: ``repeat.completed`` is bumpedand persisted under the jobs lock so a tick dying mid-execution cannot lose the dispatch@@ -2561,6 +2503,8 @@ def claim_dispatch(job_id: str) -> bool:# Recurring jobs use advance_next_run(); no/infinite repeat limit always dispatches.if job.get("schedule", {}).get("kind") != "once" or times is None or times <= 0:return True+ if extra_fire.is_extra(job): # a manual run leaves the scheduled fire's budget alone+ return Truecompleted = repeat.get("completed", 0)label = job.get("name", job.get("id", "?"))if completed >= times:@@ -2579,7 +2523,8 @@ def claim_dispatch(job_id: str) -> bool:jobs.pop(i)# See #73973.save_jobs(jobs, removed_ids={job_id})- _write_wedged_oneshot_diagnostic(job)+ from cron.jobs_oneshot_diagnostics import write_wedged+ write_wedged(job)logger.info("Job '%s': dispatch limit reached (%d/%d) — removing", label, completed, times)return False@@ -2754,7 +2699,9 @@ def claim_job_for_fire(job["fire_claim"] = {"at": now.isoformat(), "by": f"{_machine_id()}:{uuid.uuid4().hex}"}# Claimed: the occurrence is now owned by a run (its ledger row + fire claim carry it).job.pop("pending_slot", None)- if job.get("schedule", {}).get("kind") in {"cron", "interval"}:+ if slot := extra_fire.pending_slot(job, now, manual_fire, FIRE_CLAIM_SKEW_SECONDS):+ extra_fire.stamp_claim(job, slot) # off-schedule: the pending occurrence stays+ elif job.get("schedule", {}).get("kind") in {"cron", "interval"}:nxt = compute_next_run(job["schedule"], now.isoformat())if nxt:job["next_run_at"] = nxt@@ -3124,7 +3071,8 @@ def _retire_expired_oneshot(d: _DueJob) -> bool:if isinstance(claim, dict) and claim.get("outage") and not d.job.get("fire_claim"):return Falseif not (claim or d.job.get("fire_claim")):- _write_missed_oneshot_diagnostic(d.job, d.next_run)+ from cron.jobs_oneshot_diagnostics import write_missed+ write_missed(d.job, d.next_run)d.scan.retire(d.job["id"])d.scan.on_saved.append(lambda job=d.job: record_cron_missed(job))return True@@ -3168,7 +3116,8 @@ def _oneshot_dispatch_limit_reached(job: dict[str, Any], scan: _DueScan) -> boolname, completed, times)scan.retire(job["id"])# The claimed run never completed here by definition — leave an operator-visible diagnostic.- _write_wedged_oneshot_diagnostic(job)+ from cron.jobs_oneshot_diagnostics import write_wedged+ write_wedged(job)return Truediff --git a/cron/jobs_oneshot_diagnostics.py b/cron/jobs_oneshot_diagnostics.pynew file mode 100644index 00000000000..037ca8f5bc8--- /dev/null+++ b/cron/jobs_oneshot_diagnostics.py@@ -0,0 +1,77 @@+"""Operator-visible traces left in a job's output dir when the due scan or the dispatch-limit+guard removes a one-shot that never completed (``cron/jobs.py`` late-imports these)."""+import logging+from typing import Any++logger = logging.getLogger("cron.jobs")+++def _write(job: dict[str, Any], text: str, what: str) -> bool:+ """Best-effort operator-visible trace in the job's output dir; never breaks the caller."""+ from cron.jobs import save_job_output++ try:+ save_job_output(job.get("id", ""), text)+ return True+ except Exception as e:+ logger.debug("Failed to write %s diagnostic for job %r: %s", what, job.get("id"), e)+ return False+++def write_wedged(job: dict[str, Any]) -> None:+ """Trace for a wedged one-shot removal: dispatch was claimed but mark_job_run never ran+ (interrupted mid-run); removing it silently would leave no output, error, or record.++ A finite one-shot whose dispatch was claimed (``repeat.completed`` >= ``repeat.times``) but which never+ reached ``mark_job_run`` (``last_run_at`` is null) was interrupted mid-run — scheduler restart, gateway+ kill, or a non-Exception escape (#73973). The recovery guards remove such jobs so they stop appearing+ due, but a silent removal leaves the user with no output, no error, and no job record. Write a small+ diagnostic file into the job's output directory so the removal is observable and debuggable.+ """+ from cron.jobs import _hermes_now++ if job.get("last_run_at") is not None:+ return # a prior run was recorded — normal completion race, not a wedge+ repeat = job.get("repeat") or {}+ claim = job.get("run_claim") or {}+ written = _write(+ job,+ "# Cron job removed without producing output\n\n"+ f"- job id: {job.get('id')}\n"+ f"- name: {job.get('name')}\n"+ f"- dispatch claimed: {repeat.get('completed', '?')}/{repeat.get('times', '?')}\n"+ f"- run claimed at: {claim.get('at', 'unknown')} by {claim.get('by', 'unknown')}\n"+ f"- removed at: {_hermes_now().isoformat()}\n\n"+ "This one-shot job's dispatch was claimed, but the run never "+ "completed (`last_run_at` was never written) — the scheduler "+ "process was most likely killed or restarted mid-execution. The "+ "job has been removed to stop it re-firing; recreate it to run "+ "again.\n",+ "wedged-oneshot")+ if written:+ logger.warning(+ "Job '%s': removed without a completed run — diagnostic written to "+ "its output directory",+ job.get("name", job.get("id", "?")))+++def write_missed(job: dict[str, Any], next_run: str) -> None:+ """Trace for a never-ran one-shot retired outside the grace window (else it would just vanish).+ """+ from cron.jobs import ONESHOT_GRACE_SECONDS, _hermes_now++ _write(+ job,+ "# Cron job removed before firing (run time outside grace window)\n\n"+ f"- job id: {job.get('id')}\n"+ f"- name: {job.get('name')}\n"+ f"- scheduled run time: {next_run}\n"+ f"- grace window: {ONESHOT_GRACE_SECONDS}s\n"+ f"- removed at: {_hermes_now().isoformat()}\n\n"+ "This one-shot's run time is more than the grace window in the "+ "past (scheduler down past the window, host asleep, or jobs.json "+ "edited), which is outside the 'will never fire' contract "+ "enforced at create/update/resume time. The job was removed "+ "without running; recreate it (or use the Run button) to "+ "schedule it again.\n",+ "missed-oneshot")diff --git a/tests/cron/test_manual_run_extra_fire.py b/tests/cron/test_manual_run_extra_fire.pynew file mode 100644index 00000000000..59789364893--- /dev/null+++ b/tests/cron/test_manual_run_extra_fire.py@@ -0,0 +1,139 @@+"""A manual run is an EXTRA fire: it never consumes the job's scheduled occurrence or repeat budget.++Each test drives the real bookkeeping sequence a fire goes through (claim_job_for_fire ->+claim_dispatch -> mark_job_run), the same calls the cronjob tool, the dashboard and the+ticker make around the agent run.+"""+from datetime import datetime, timedelta++import pytest++from cron.jobs import (+ claim_dispatch,+ claim_job_for_fire,+ create_job,+ get_due_jobs,+ get_job,+ load_jobs,+ mark_job_run,+ save_jobs,+ trigger_job,+)+++def clock(tmp_path, monkeypatch):+ monkeypatch.setattr("cron.jobs.CRON_DIR", tmp_path / "cron")+ monkeypatch.setattr("cron.jobs.JOBS_FILE", tmp_path / "cron" / "jobs.json")+ monkeypatch.setattr("cron.jobs.OUTPUT_DIR", tmp_path / "cron" / "output")+ state = {"now": datetime.fromisoformat("2026-10-28T09:00:00-07:00")}+ monkeypatch.setattr("cron.jobs._hermes_now", lambda: state["now"])+ return state+++def _fire(job_id, **claim_kwargs):+ claimed = claim_job_for_fire(job_id, return_job=True, **claim_kwargs)+ assert isinstance(claimed, dict), "fire was not claimed"+ assert claim_dispatch(job_id)+ assert mark_job_run(job_id, success=True, expected_fire_owner=claimed["fire_claim"]["by"])+++def test_manual_run_keeps_the_scheduled_oneshot_and_it_still_fires(clock):+ job = create_job("remind me", "in 30m", name="reminder")+ slot = job["next_run_at"]++ _fire(job["id"], manual=True) # cronjob(action="run") / hermes cron run++ after = get_job(job["id"])+ assert (after["state"], after["enabled"], after["next_run_at"]) == ("scheduled", True, slot)+ assert after["repeat"] == {"times": 1, "completed": 0}+ assert after["last_status"] == "ok"++ clock["now"] += timedelta(minutes=30, seconds=5)+ assert [d["id"] for d in get_due_jobs()] == [job["id"]]+ _fire(job["id"]) # the scheduled tick++ done = get_job(job["id"])+ assert (done["state"], done["next_run_at"]) == ("completed", None)+ assert done["repeat"] == {"times": 1, "completed": 1}+++def test_gateway_forwarded_trigger_restores_the_slot_at_claim_time(clock):+ """trigger_job overwrites next_run_at with "now"; the claim puts the slot back in the same+ save, so a crash before mark_job_run cannot lose the scheduled occurrence."""+ job = create_job("remind me", "in 30m", name="reminder")+ slot = job["next_run_at"]+ trigger_job(job["id"])+ assert [d["id"] for d in get_due_jobs()] == [job["id"]]++ claimed = claim_job_for_fire(job["id"], return_job=True) # the ticker's claim+ assert isinstance(claimed, dict)+ assert get_job(job["id"])["next_run_at"] == slot # durable before the run starts++ assert claim_dispatch(job["id"])+ assert mark_job_run(job["id"], success=True, expected_fire_owner=claimed["fire_claim"]["by"])+ after = get_job(job["id"])+ assert (after["state"], after["next_run_at"]) == ("scheduled", slot)+ assert after["repeat"]["completed"] == 0+ assert "manual_run_restore_at" not in after+++def test_manual_run_does_not_spend_a_recurring_repeat_budget_or_shift_the_schedule(clock):+ job = create_job("digest", "every 60m", name="digest", repeat=2)+ slot = job["next_run_at"]+ clock["now"] += timedelta(minutes=20)++ _fire(job["id"], manual=True)++ after = get_job(job["id"])+ assert after["repeat"] == {"times": 2, "completed": 0}+ assert after["next_run_at"] == slot+++def test_dashboard_run_before_the_slot_is_an_extra_fire(clock):+ """The dashboard Run button claims without manual=True; firing ahead of the stored slot is+ still off-schedule."""+ job = create_job("remind me", "in 30m", name="reminder")++ _fire(job["id"])++ after = get_job(job["id"])+ assert (after["state"], after["next_run_at"]) == ("scheduled", job["next_run_at"])+ assert after["repeat"]["completed"] == 0+++def test_crash_mid_manual_run_leaves_the_scheduled_oneshot_due(clock):+ job = create_job("remind me", "in 30m", name="reminder")+ assert isinstance(claim_job_for_fire(job["id"], manual=True, return_job=True), dict)+ assert claim_dispatch(job["id"])+ # The runner dies here: no mark_job_run, and its claim's owner is gone.+ record = load_jobs()[0]+ record["fire_claim"]["by"] = "dead-host:0"+ record["fire_claim"]["at"] = (clock["now"] - timedelta(hours=2)).isoformat()+ save_jobs([record])++ clock["now"] += timedelta(minutes=30, seconds=5)+ assert [d["id"] for d in get_due_jobs()] == [job["id"]]+ assert get_job(job["id"])["repeat"]["completed"] == 0+++def test_a_slot_that_passes_during_the_manual_run_still_fires(clock):+ job = create_job("remind me", "in 2m", name="reminder")+ claimed = claim_job_for_fire(job["id"], manual=True, return_job=True)+ assert claim_dispatch(job["id"])+ clock["now"] += timedelta(minutes=5) # long run: the tick lost the claim race at the slot++ assert mark_job_run(job["id"], success=True, expected_fire_owner=claimed["fire_claim"]["by"])++ assert [d["id"] for d in get_due_jobs()] == [job["id"]]+++def test_run_at_or_after_the_due_slot_is_the_scheduled_fire(clock):+ """Control: a manual run once the one-shot is already due consumes it (no double fire)."""+ job = create_job("remind me", "in 2m", name="reminder")+ clock["now"] += timedelta(minutes=2, seconds=10)++ _fire(job["id"], manual=True)++ done = get_job(job["id"])+ assert (done["state"], done["repeat"]["completed"]) == ("completed", 1)