Skip to content

The agent that grows with you

fix(cron): a quota hold on a manual run delays a queued run-now instead of desyncing it

Tekniumf101b0f7b7c0
diff --git a/cron/extra_fire.py b/cron/extra_fire.py
index 97ae3c1ddf0..97a0cac3896 100644
--- a/cron/extra_fire.py
+++ b/cron/extra_fire.py
@@ -69,6 +69,15 @@ def held_markers(job: dict[str, Any]) -> dict[str, Any]:
return {key: job[key] for key in _MARKERS if key in job} if queued(job) else {}
+def reschedule(job: dict[str, Any], at: str) -> None:
+ """Move ``next_run_at`` from a failure-path planner (the quota hold). A queued
+ run-now's marker moves with it: the run-now is delayed, not turned into a scheduled fire that
+ would spend a repeat run and drop the slot saved in RESTORE_KEY."""
+ if queued(job):
+ job["manual_run_at"] = at
+ job["next_run_at"] = at
+
+
def settle(job: dict[str, Any], now: str) -> None:
"""After an extra fire: ``repeat.completed`` untouched, the job keeps its pending slot, and a
run-now queued during the run (``queued``) is left for its own claim. A slot that passed while
diff --git a/cron/quota_hold.py b/cron/quota_hold.py
index 5edefc4cdf4..4b1ddaaa472 100644
--- a/cron/quota_hold.py
+++ b/cron/quota_hold.py
@@ -20,6 +20,7 @@ import re
from datetime import datetime
from typing import Any, Dict, Optional
+from cron import extra_fire
from hermes_time import now as _hermes_now, safe_strftime
logger = logging.getLogger("cron.scheduler")
@@ -131,8 +132,9 @@ def plan_hold(
if not blocked and not recover:
clear_state(job)
return False
- if kind == "cron" and blocked:
+ if kind == "cron" and blocked and not extra_fire.queued(job):
# Coalesce cron occurrences inside the closed window to the first legal instant after it.
+ # A queued run-now is not an occurrence: it waits for the boundary, not the next slot.
parked = compute_next_run(schedule, window_end.isoformat()) or window_end.isoformat()
else:
parked = window_end.isoformat()
@@ -141,7 +143,7 @@ def plan_hold(
job[SCHEDULE_EXPR_KEY] = schedule.get("expr")
else:
job.pop(SCHEDULE_EXPR_KEY, None)
- job["next_run_at"] = parked
+ extra_fire.reschedule(job, parked)
job[STATE_KEY] = parked
logger.warning(
"Job '%s': provider usage window closed for %.0fs — holding fires until %s instead of "
diff --git a/tests/cron/test_manual_run_extra_fire.py b/tests/cron/test_manual_run_extra_fire.py
index 4e69c91cb3c..1c0ace6ae19 100644
--- a/tests/cron/test_manual_run_extra_fire.py
+++ b/tests/cron/test_manual_run_extra_fire.py
@@ -173,3 +173,34 @@ def test_trigger_landing_during_a_manual_run_survives_its_completion(clock):
assert (after["state"], after["next_run_at"]) == ("scheduled", slot)
assert after["repeat"] == {"times": 1, "completed": 0}
+
+
+def test_quota_hold_on_a_manual_run_delays_a_queued_run_now_without_spending_the_schedule(
+ clock, monkeypatch,
+):
+ """A manual run hits a provider quota window while another run-now is queued: the hold delays
+ that queued run-now to the window boundary, where it still fires as an extra run. It spends no
+ repeat run and the scheduled slot after the window stays put."""
+ monkeypatch.setattr("cron.quota_hold._hermes_now", lambda: clock["now"])
+ job = create_job("digest", "every 60m", name="digest", repeat=3)
+ slot = job["next_run_at"] # 10:00
+ clock["now"] += timedelta(minutes=20)
+ claimed = claim_job_for_fire(job["id"], manual=True, return_job=True)
+ assert isinstance(claimed, dict) and claim_dispatch(job["id"])
+ clock["now"] += timedelta(seconds=10)
+ trigger_job(job["id"]) # queued while the manual run is in flight
+ clock["now"] += timedelta(seconds=10)
+ assert mark_job_run(job["id"], success=False, error="quota", quota_hold_seconds=600,
+ expected_fire_owner=claimed["fire_claim"]["by"])
+
+ held = get_job(job["id"])
+ assert held["next_run_at"] > clock["now"].isoformat() # parked through the closed window
+ assert get_due_jobs() == []
+ clock["now"] = datetime.fromisoformat(held["next_run_at"]) + timedelta(seconds=5) # ~09:31
+ due = get_due_jobs()
+ assert [d["id"] for d in due] == [job["id"]]
+ advance_next_runs([d["id"] for d in due])
+ _fire(job["id"]) # the delayed run-now
+
+ after = get_job(job["id"])
+ assert (after["next_run_at"], after["repeat"]) == (slot, {"times": 3, "completed": 0})