Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend/copilot): skip a scheduled copilot turn while its owner is paywalled or over a cost cap (#15348)

Reinier van der Leerb8e97afeeb03
diff --git a/autogpt_platform/backend/backend/copilot/turn_queue.py b/autogpt_platform/backend/backend/copilot/turn_queue.py
index 31ec7bcf06..fd0f361ddf 100644
--- a/autogpt_platform/backend/backend/copilot/turn_queue.py
+++ b/autogpt_platform/backend/backend/copilot/turn_queue.py
@@ -33,6 +33,7 @@ import uuid
from typing import Any, Mapping
from prisma.errors import UniqueViolationError
+from pydantic import BaseModel
from backend.copilot.active_turns import TurnSlot, count_running_turns
from backend.copilot.config import ChatConfig, CopilotLlmAuthProvider
@@ -270,52 +271,14 @@ async def dispatch_next_for_user(user_id: str) -> bool:
head = queued[0]
route_provider = head.metadata.llm_auth_provider
- if route_provider == "codex":
- if not await has_codex_access(user_id):
- logger.info(
- "dispatch_next_for_user: user=%s lacks Codex entitlement, "
- "leaving session=%s queued",
- user_id,
- head.session_id,
- )
- return False
- elif route_provider == "platform":
- if await is_user_paywalled(user_id):
- logger.info(
- "dispatch_next_for_user: user=%s paywalled, leaving session=%s queued",
- user_id,
- head.session_id,
- )
- return False
-
- cfg = ChatConfig()
- try:
- daily_limit, weekly_limit, _ = await get_global_rate_limits(
- user_id,
- cfg.daily_cost_limit_microdollars,
- cfg.weekly_cost_limit_microdollars,
- )
- await check_rate_limit(
- user_id=user_id,
- daily_cost_limit=daily_limit,
- weekly_cost_limit=weekly_limit,
- )
- except RateLimitExceeded as exc:
- logger.info(
- "dispatch_next_for_user: user=%s rate-limited (%s), "
- "leaving session=%s queued",
- user_id,
- exc,
- head.session_id,
- )
- return False
- except RateLimitUnavailable:
- logger.warning(
- "dispatch_next_for_user: rate-limit service degraded for user=%s; "
- "leaving queue intact for the next tick",
- user_id,
- )
- return False
+ refusal = await turn_refusal(user_id, route_provider)
+ if refusal is not None:
+ logger.log(
+ logging.WARNING if refusal.transient else logging.INFO,
+ f"dispatch_next_for_user: user={user_id} {refusal.reason}, "
+ f"leaving session={head.session_id} queued",
+ )
+ return False
# Claim by transitioning the session ``queued`` → ``running``. A
# parallel cancel between validation and claim rejects this
@@ -445,3 +408,45 @@ async def dispatch_next_for_user(user_id: str) -> bool:
await invalidate_session_cache(head.session_id)
return True
+
+
+class TurnRefusal(BaseModel):
+ reason: str
+ # The usage store was unreadable: an outage, not the user's state.
+ transient: bool = False
+
+
+async def turn_refusal(
+ user_id: str, llm_auth_provider: CopilotLlmAuthProvider
+) -> TurnRefusal | None:
+ """Why a turn billed to this route may not start for this user, if so.
+
+ Access first (Codex entitlement, the platform paywall), then the platform
+ route's usage: its daily and weekly windows and a trial's total budget.
+ Lookup errors propagate, so each caller decides what an unknown means.
+ """
+ if llm_auth_provider == "codex":
+ if await has_codex_access(user_id):
+ return None
+ return TurnRefusal(reason="lacks Codex entitlement")
+ if llm_auth_provider != "platform":
+ return None
+ if await is_user_paywalled(user_id):
+ return TurnRefusal(reason="paywalled")
+ cfg = ChatConfig()
+ try:
+ daily_limit, weekly_limit, _ = await get_global_rate_limits(
+ user_id,
+ cfg.daily_cost_limit_microdollars,
+ cfg.weekly_cost_limit_microdollars,
+ )
+ await check_rate_limit(
+ user_id=user_id,
+ daily_cost_limit=daily_limit,
+ weekly_cost_limit=weekly_limit,
+ )
+ except RateLimitExceeded as exc:
+ return TurnRefusal(reason=f"rate-limited ({exc})")
+ except RateLimitUnavailable:
+ return TurnRefusal(reason="has unreadable usage limits", transient=True)
+ return None
diff --git a/autogpt_platform/backend/backend/executor/scheduler.py b/autogpt_platform/backend/backend/executor/scheduler.py
index 9717b3ce92..42280d5432 100644
--- a/autogpt_platform/backend/backend/executor/scheduler.py
+++ b/autogpt_platform/backend/backend/executor/scheduler.py
@@ -31,6 +31,7 @@ from sqlalchemy import MetaData, create_engine
from backend.api.features.experts.models import ExpertRoutine
from backend.copilot.active_turns import ConcurrentTurnLimitError
+from backend.copilot.config import CopilotLlmAuthProvider
from backend.copilot.credential_selection import CredentialPins
from backend.copilot.dream.scheduling import (
COMMUNITY_REBUILD_REGISTRATION_PREFIX,
@@ -43,6 +44,7 @@ from backend.copilot.model import create_chat_session, get_chat_session
from backend.copilot.optimize_blocks import optimize_block_descriptions
from backend.copilot.permissions import CopilotPermissions, routine_disabled_tools
from backend.copilot.transports import resolve_default_chat_route
+from backend.copilot.turn_queue import TurnRefusal, turn_refusal
from backend.data.db_accessors import experts_db
from backend.data.execution import ExecutionTrigger, GraphExecutionWithNodes
from backend.data.model import CredentialsMetaInput, GraphInput
@@ -462,6 +464,8 @@ async def _execute_copilot_turn(**kwargs):
llm_auth_provider, llm_credential_id = await resolve_default_chat_route(
args.user_id
)
+ if await _refuse_if_turn_may_not_start(args, llm_auth_provider):
+ return
new_session = await create_chat_session(
args.user_id,
dry_run=False,
@@ -550,6 +554,10 @@ async def _execute_copilot_turn(**kwargs):
if expert_status != "active":
await _skip_inactive_expert_scope(args, expert_status)
return
+ if await _refuse_if_turn_may_not_start(
+ args, session.metadata.llm_auth_provider
+ ):
+ return
target_session_id = args.session_id
target_session = session
# The target may be the user's own interactive Otto chat,
@@ -658,6 +666,50 @@ async def _execute_copilot_turn(**kwargs):
)
+async def _refuse_if_turn_may_not_start(
+ job_args: "CopilotTurnJobArgs", llm_auth_provider: CopilotLlmAuthProvider
+) -> bool:
+ """Refuse a fire that may not start, as ``dispatch_next_for_user`` refuses
+ a queued turn, and say whether it did: the owner has no access to the route
+ it bills to, or has used up their AutoPilot usage for the window.
+
+ A cron schedule stays registered and resumes once the owner has access or
+ the window resets. APScheduler drops a one-shot once it fires, so one
+ refused by an outage (unreadable usage limits or plan) is retried, and any
+ other is dropped with its routine switched off rather than left pending for
+ a time that has passed.
+ """
+ retry = None
+ try:
+ refusal = await turn_refusal(job_args.user_id, llm_auth_provider)
+ if refusal is not None and refusal.transient:
+ retry = _reschedule_one_shot_after_limits_unreadable
+ except Exception as exc:
+ # Unknown access is no access: skip the fire, and retry a one-shot.
+ refusal = TurnRefusal(
+ reason=f"has an unreadable plan ({type(exc).__name__}: {exc})",
+ transient=True,
+ )
+ retry = _reschedule_one_shot_after_plan_unreadable
+ if refusal is None:
+ return False
+ if job_args.run_at is None:
+ outcome = "the schedule stays registered"
+ elif retry is None:
+ await _drop_job_from_routine(job_args)
+ outcome = "the one-shot is dropped"
+ elif await retry(job_args):
+ outcome = "the one-shot is retried"
+ else:
+ outcome = "the one-shot is dropped"
+ logger.log(
+ logging.WARNING if refusal.transient else logging.INFO,
+ f"Skipping scheduled copilot turn {job_args.schedule_id}: owner "
+ f"{refusal.reason}; {outcome}",
+ )
+ return True
+
+
def _credential_pins_for_turn(
args: "CopilotTurnJobArgs", routine: ExpertRoutine | None
) -> CredentialPins:
@@ -690,6 +742,15 @@ def _session_id_label(args: "CopilotTurnJobArgs") -> str:
_CONCURRENCY_RETRY_DELAY_SECONDS = 300
_MAX_CAP_RETRIES = 1
_MAX_EXPERT_LOOKUP_RETRIES = 1
+_MAX_PLAN_LOOKUP_RETRIES = 1
+_MAX_LIMITS_RETRIES = 1
+_RetryKind = Literal["cap", "expert_lookup", "plan_lookup", "limits"]
+_MAX_RETRIES: dict[_RetryKind, int] = {
+ "cap": _MAX_CAP_RETRIES,
+ "expert_lookup": _MAX_EXPERT_LOOKUP_RETRIES,
+ "plan_lookup": _MAX_PLAN_LOOKUP_RETRIES,
+ "limits": _MAX_LIMITS_RETRIES,
+}
async def _reschedule_one_shot_after_cap(args: "CopilotTurnJobArgs") -> None:
@@ -701,6 +762,28 @@ async def _reschedule_one_shot_after_cap(args: "CopilotTurnJobArgs") -> None:
)
+async def _reschedule_one_shot_after_plan_unreadable(
+ args: "CopilotTurnJobArgs",
+) -> bool:
+ return await _reschedule_one_shot(
+ args,
+ reason="unreadable owner plan",
+ name_suffix="plan-retry",
+ retry_kind="plan_lookup",
+ )
+
+
+async def _reschedule_one_shot_after_limits_unreadable(
+ args: "CopilotTurnJobArgs",
+) -> bool:
+ return await _reschedule_one_shot(
+ args,
+ reason="unreadable usage limits",
+ name_suffix="limits-retry",
+ retry_kind="limits",
+ )
+
+
async def _reschedule_one_shot_after_expert_unavailable(
args: "CopilotTurnJobArgs",
) -> None:
@@ -717,25 +800,25 @@ async def _reschedule_one_shot(
*,
reason: str,
name_suffix: str,
- retry_kind: Literal["cap", "expert_lookup"],
-) -> None:
- """Re-create a one-shot copilot-turn schedule after a transient failure.
+ retry_kind: _RetryKind,
+) -> bool:
+ """Re-create a one-shot copilot-turn schedule after a transient failure,
+ and say whether it did.
Best-effort: failures are logged. Schedules that have already been
retried the limit for this failure kind are dropped to avoid loops. Retry
depths round-trip independently through APScheduler's persisted kwargs so
a transient expert lookup does not consume the concurrency-cap budget.
"""
- if retry_kind == "cap":
- retry_count = args.cap_retry_count
- max_retries = _MAX_CAP_RETRIES
- next_cap_retry_count = retry_count + 1
- next_expert_lookup_retry_count = args.expert_lookup_retry_count
- else:
- retry_count = args.expert_lookup_retry_count
- max_retries = _MAX_EXPERT_LOOKUP_RETRIES
- next_cap_retry_count = args.cap_retry_count
- next_expert_lookup_retry_count = retry_count + 1
+ counts: dict[_RetryKind, int] = {
+ "cap": args.cap_retry_count,
+ "expert_lookup": args.expert_lookup_retry_count,
+ "plan_lookup": args.plan_lookup_retry_count,
+ "limits": args.limits_retry_count,
+ }
+ retry_count = counts[retry_kind]
+ max_retries = _MAX_RETRIES[retry_kind]
+ counts[retry_kind] = retry_count + 1
if retry_count >= max_retries:
logger.error(
@@ -743,7 +826,8 @@ async def _reschedule_one_shot(
f"{_session_id_label(args)} — exhausted {max_retries} "
f"retry/retries after {reason}"
)
- return
+ await _drop_job_from_routine(args)
+ return False
try:
new_run_at = datetime.now(tz=timezone.utc) + timedelta(
seconds=_CONCURRENCY_RETRY_DELAY_SECONDS
@@ -754,8 +838,10 @@ async def _reschedule_one_shot(
message=args.message,
run_at=new_run_at,
name=f"{args.schedule_id or 'copilot'}-{name_suffix}",
- cap_retry_count=next_cap_retry_count,
- expert_lookup_retry_count=next_expert_lookup_retry_count,
+ cap_retry_count=counts["cap"],
+ expert_lookup_retry_count=counts["expert_lookup"],
+ plan_lookup_retry_count=counts["plan_lookup"],
+ limits_retry_count=counts["limits"],
# Preserve the user's timezone across the reschedule so the new
# one-shot job's trigger/timezone matches the original request.
user_timezone=args.user_timezone,
@@ -773,6 +859,9 @@ async def _reschedule_one_shot(
# an ungranted routine that merely lost a race to the concurrency
# cap would come back with everything the mute exists to withhold.
routine_id=args.routine_id,
+ # The retry gets a new schedule_id; this is the one the routine's
+ # row holds, so the retry can still switch the routine off.
+ routine_schedule_id=args.routine_schedule_id or args.schedule_id,
# And the accounts it was set up to run on, or the retry would
# take the first saved one instead.
credential_pins=args.credential_pins,
@@ -782,12 +871,15 @@ async def _reschedule_one_shot(
f"{_session_id_label(args)} to {new_run_at.isoformat()} after "
f"{reason} (retry {retry_count + 1}/{max_retries})"
)
+ return True
except Exception:
logger.warning(
f"Failed to reschedule one-shot copilot turn for session "
f"{_session_id_label(args)} after {reason}",
exc_info=True,
)
+ await _drop_job_from_routine(args)
+ return False
async def _best_effort_unschedule(
@@ -838,21 +930,26 @@ async def _self_delete_copilot_turn_schedule(args: "CopilotTurnJobArgs") -> None
args.user_id,
reason="session unavailable or scope mismatch",
)
+ await _drop_job_from_routine(args)
+
+
+async def _drop_job_from_routine(args: "CopilotTurnJobArgs") -> None:
# A job with no schedule_id predates the field and cannot be matched
# against the ids a routine row holds, so there is nothing to drop.
- if args.routine_id is None or args.schedule_id is None:
+ schedule_id = args.routine_schedule_id or args.schedule_id
+ if args.routine_id is None or schedule_id is None:
return
# The row outlives the job it lost, and a routine still listed as switched
# on with nothing scheduled behind it is the one state the owner cannot act
# on: the UI offers to switch off something that is already not running.
- # Most often this is a PINNED routine whose chat the owner deleted.
+ # Most often this is a PINNED routine whose chat the owner deleted, or a
+ # one-shot that fired while its owner could not pay.
try:
- await experts_db().mark_routine_unscheduled(args.routine_id, args.schedule_id)
+ await experts_db().mark_routine_unscheduled(args.routine_id, schedule_id)
except Exception:
logger.warning(
- "Could not switch off routine %s after removing its schedule %s",
- args.routine_id[:12],
- args.schedule_id,
+ f"Could not switch off routine {args.routine_id[:12]} after removing "
+ f"its schedule {schedule_id}",
exc_info=True,
)
@@ -1720,6 +1817,8 @@ class CopilotTurnJobArgs(BaseModel):
# concurrency-cap retry (or vice versa).
cap_retry_count: int = 0
expert_lookup_retry_count: int = 0
+ plan_lookup_retry_count: int = 0
+ limits_retry_count: int = 0
# Persisted so ``_reschedule_one_shot_after_cap`` can preserve the user's
# timezone when re-creating a one-shot job after a concurrency-cap miss —
# otherwise the rescheduled job's trigger defaults to UTC and the timezone
@@ -1745,6 +1844,9 @@ class CopilotTurnJobArgs(BaseModel):
# decides whether the turn may touch a connected service at all. None keeps
# ordinary ``schedule_followup`` jobs on their existing path.
routine_id: str | None = None
+ # The routine row's id for this job when it is a one-shot's retry, which
+ # gets a schedule_id of its own that the row does not hold.
+ routine_schedule_id: str | None = None
# ``{provider: pin}``: the account the user chose for each provider when
# the follow-up was made, which every fire runs on (SECRT-2804). A routine
# keeps its pins on its row instead. Empty on rows persisted before pins.
@@ -2457,11 +2559,14 @@ class Scheduler(AppService):
user_timezone: str | None = None,
cap_retry_count: int = 0,
expert_lookup_retry_count: int = 0,
+ plan_lookup_retry_count: int = 0,
+ limits_retry_count: int = 0,
organization_id: str | None = None,
team_id: str | None = None,
expert_id: str | None = None,
routine_id: str | None = None,
credential_pins: CredentialPins | None = None,
+ routine_schedule_id: str | None = None,
) -> CopilotTurnJobInfo:
"""Schedule a copilot turn at a future time.
@@ -2470,9 +2575,8 @@ class Scheduler(AppService):
the turn into it. Otherwise the turn resumes the named (existing)
session with its full history, after re-validating that scope.
- *cap_retry_count* and *expert_lookup_retry_count* are set internally
- to bound their respective transient retry paths; normal callers should
- leave both at 0.
+ The ``*_retry_count`` arguments are set internally to bound their
+ respective transient retry paths; normal callers should leave them at 0.
"""
# Mirror add_graph_execution_schedule: validate the expert scope at
# creation (active, owned, PRIVATE) and pin the schedule to the
@@ -2494,12 +2598,15 @@ class Scheduler(AppService):
run_at=run_at,
cap_retry_count=cap_retry_count,
expert_lookup_retry_count=expert_lookup_retry_count,
+ plan_lookup_retry_count=plan_lookup_retry_count,
+ limits_retry_count=limits_retry_count,
user_timezone=user_timezone,
organization_id=organization_id,
team_id=team_id,
expert_id=expert_id,
routine_id=routine_id,
credential_pins=credential_pins or {},
+ routine_schedule_id=routine_schedule_id,
)
default_name = (
f"copilot turn (session {session_id[:8]})"
diff --git a/autogpt_platform/backend/backend/executor/scheduler_unit_test.py b/autogpt_platform/backend/backend/executor/scheduler_unit_test.py
index 31eea8a287..fdc8d58cd3 100644
--- a/autogpt_platform/backend/backend/executor/scheduler_unit_test.py
+++ b/autogpt_platform/backend/backend/executor/scheduler_unit_test.py
@@ -24,9 +24,12 @@ from backend.copilot.permissions import (
CAPABILITY_GATE_NAMES,
ROUTINE_SELF_ESCALATION_TOOLS,
)
+from backend.copilot.rate_limit import RateLimitExceeded, RateLimitUnavailable
from backend.executor.scheduler import (
_MAX_CAP_RETRIES,
_MAX_EXPERT_LOOKUP_RETRIES,
+ _MAX_LIMITS_RETRIES,
+ _MAX_PLAN_LOOKUP_RETRIES,
CopilotTurnJobArgs,
CopilotTurnJobInfo,
GraphExecutionJobArgs,
@@ -42,6 +45,8 @@ from backend.executor.scheduler import (
_next_run_time_iso,
_reschedule_one_shot_after_cap,
_reschedule_one_shot_after_expert_unavailable,
+ _reschedule_one_shot_after_limits_unreadable,
+ _reschedule_one_shot_after_plan_unreadable,
_routine_turn_permissions,
_self_delete_copilot_turn_schedule,
_self_delete_morning_briefing_schedule,
@@ -56,6 +61,7 @@ from backend.util.service import EXPOSED_FLAG
from backend.util.settings import Config
_SCHEDULER_PATH = "backend.executor.scheduler"
+_TURN_QUEUE_PATH = "backend.copilot.turn_queue"
@pytest.fixture(autouse=True)
@@ -69,6 +75,30 @@ def mock_external_services(monkeypatch: pytest.MonkeyPatch) -> None:
)
+class _OwnerGate(NamedTuple):
+ is_user_paywalled: AsyncMock
+ check_rate_limit: AsyncMock
+ has_codex_access: AsyncMock
+
+
[email protected](autouse=True)
+def owner_gate(monkeypatch: pytest.MonkeyPatch) -> _OwnerGate:
+ """An owner with a subscription, AutoPilot usage left and Codex access;
+ tests that need another owner set these mocks rather than patch again.
+ Patched where the shared gate resolves them."""
+ gate = _OwnerGate(
+ AsyncMock(return_value=False), AsyncMock(), AsyncMock(return_value=True)
+ )
+ monkeypatch.setattr(f"{_TURN_QUEUE_PATH}.is_user_paywalled", gate.is_user_paywalled)
+ monkeypatch.setattr(
+ f"{_TURN_QUEUE_PATH}.get_global_rate_limits",
+ AsyncMock(return_value=(1_000_000, 5_000_000, None)),
+ )
+ monkeypatch.setattr(f"{_TURN_QUEUE_PATH}.check_rate_limit", gate.check_rate_limit)
+ monkeypatch.setattr(f"{_TURN_QUEUE_PATH}.has_codex_access", gate.has_codex_access)
+ return gate
+
+
# ---------------------------------------------------------------------------
# _build_trigger
# ---------------------------------------------------------------------------
@@ -969,6 +999,17 @@ async def test_self_delete_copilot_turn_swallows_errors():
await _self_delete_copilot_turn_schedule(args)
+async def test_self_delete_switches_off_the_routine_that_lost_the_job():
+ store = MagicMock(mark_routine_unscheduled=AsyncMock())
+ with (
+ patch(f"{_SCHEDULER_PATH}.get_scheduler_client", return_value=AsyncMock()),
+ patch(f"{_SCHEDULER_PATH}.experts_db", return_value=store),
+ ):
+ await _self_delete_copilot_turn_schedule(_args(routine_id="routine-1"))
+ store.mark_routine_unscheduled.assert_awaited_once_with("routine-1", "sched-1")
+
+
# ---------------------------------------------------------------------------
# _best_effort_unschedule / _cleanup_old_schedules_without_id
# ---------------------------------------------------------------------------
@@ -1140,6 +1181,38 @@ async def test_expert_lookup_and_cap_retries_have_independent_budgets():
assert expert_kwargs["expert_lookup_retry_count"] == 1
+ ("retry", "kind", "max_retries"),
+ [
+ (
+ _reschedule_one_shot_after_plan_unreadable,
+ "plan_lookup",
+ _MAX_PLAN_LOOKUP_RETRIES,
+ ),
+ (_reschedule_one_shot_after_limits_unreadable, "limits", _MAX_LIMITS_RETRIES),
+ ],
+ ids=["plan-lookup", "limits"],
+)
+async def test_an_outage_retry_and_a_cap_retry_have_independent_budgets(
+ retry, kind, max_retries
+):
+ mock_client = AsyncMock()
+ with patch(f"{_SCHEDULER_PATH}.get_scheduler_client", return_value=mock_client):
+ await retry(_args(cap_retry_count=_MAX_CAP_RETRIES))
+ outage_kwargs = mock_client.add_copilot_turn_schedule.await_args.kwargs
+ mock_client.reset_mock()
+ await _reschedule_one_shot_after_cap(
+ _args(**{f"{kind}_retry_count": max_retries})
+ )
+ cap_kwargs = mock_client.add_copilot_turn_schedule.await_args.kwargs
+
+ assert outage_kwargs["cap_retry_count"] == _MAX_CAP_RETRIES
+ assert outage_kwargs[f"{kind}_retry_count"] == 1
+ assert cap_kwargs["cap_retry_count"] == 1
+ assert cap_kwargs[f"{kind}_retry_count"] == max_retries
+
+
# ---------------------------------------------------------------------------
# GraphExecutionJobArgs back-compat — legacy rows have no `kind`
# ---------------------------------------------------------------------------
@@ -2327,6 +2400,227 @@ async def test_a_switched_on_routine_still_fires():
self_delete.assert_not_awaited()
+def _paywalled(gate: _OwnerGate) -> None:
+ gate.is_user_paywalled.return_value = True
+
+
+def _lost_codex(gate: _OwnerGate) -> None:
+ gate.has_codex_access.return_value = False
+
+
+def _over_a_cap(gate: _OwnerGate) -> None:
+ resets_at = datetime.now(tz=timezone.utc) + timedelta(hours=3)
+ gate.check_rate_limit.side_effect = RateLimitExceeded("daily", resets_at)
+
+
+def _limits_unreadable(gate: _OwnerGate) -> None:
+ gate.check_rate_limit.side_effect = RateLimitUnavailable()
+
+
+def _lookup_fails(gate: _OwnerGate) -> None:
+ gate.is_user_paywalled.side_effect = RuntimeError("tier lookup down")
+
+
+async def _fire_hourly(session_id: str | None, provider: str) -> dict[str, AsyncMock]:
+ args = _args(session_id=session_id, run_at=None, cron="9 * * * *")
+ mocks = {
+ "schedule_turn": AsyncMock(),
+ "create_chat_session": AsyncMock(
+ return_value=MagicMock(session_id="new-session", expert_id=None)
+ ),
+ "_self_delete_copilot_turn_schedule": AsyncMock(),
+ }
+ session = MagicMock(session_id="session-1", expert_id=None)
+ session.metadata.llm_auth_provider = provider
+ with (
+ patch(
+ f"{_SCHEDULER_PATH}.resolve_default_chat_route",
+ AsyncMock(return_value=(provider, None)),
+ ),
+ patch(f"{_SCHEDULER_PATH}.get_chat_session", AsyncMock(return_value=session)),
+ patch(f"{_SCHEDULER_PATH}.schedule_turn", mocks["schedule_turn"]),
+ patch(f"{_SCHEDULER_PATH}.create_chat_session", mocks["create_chat_session"]),
+ patch(
+ f"{_SCHEDULER_PATH}._self_delete_copilot_turn_schedule",
+ mocks["_self_delete_copilot_turn_schedule"],
+ ),
+ ):
+ await _execute_copilot_turn(**args.model_dump(mode="json"))
+ return mocks
+
+
+ "session_id", [None, "session-1"], ids=["fresh-chat", "existing-chat"]
+)
+ ("refuse", "provider"),
+ [
+ (_paywalled, "platform"),
+ (_over_a_cap, "platform"),
+ (_limits_unreadable, "platform"),
+ (_lost_codex, "codex"),
+ (_lookup_fails, "platform"),
+ ],
+ ids=["paywalled", "over-a-cap", "limits-unreadable", "lost-codex", "lookup-fails"],
+)
+async def test_a_tick_that_may_not_start_is_skipped_and_the_schedule_kept(
+ owner_gate, refuse, provider, session_id, caplog
+):
+ """No turn and no empty chat per tick, and nothing deleted, so the schedule
+ resumes by itself once the owner has access or the usage window resets."""
+ refuse(owner_gate)
+ with caplog.at_level(logging.INFO, logger=_SCHEDULER_PATH):
+ mocks = await _fire_hourly(session_id, provider)
+
+ mocks["schedule_turn"].assert_not_awaited()
+ mocks["create_chat_session"].assert_not_awaited()
+ mocks["_self_delete_copilot_turn_schedule"].assert_not_awaited()
+ assert "Skipping scheduled copilot turn sched-1" in caplog.text
+ assert "the schedule stays registered" in caplog.text
+ assert not any(record.levelno >= logging.ERROR for record in caplog.records)
+
+
+ "session_id", [None, "session-1"], ids=["fresh-chat", "existing-chat"]
+)
+async def test_a_tick_that_may_start_fires(owner_gate, session_id):
+ mocks = await _fire_hourly(session_id, "platform")
+
+ mocks["schedule_turn"].assert_awaited_once()
+ owner_gate.is_user_paywalled.assert_awaited_once_with("user-1")
+ owner_gate.check_rate_limit.assert_awaited_once()
+
+
+async def test_a_codex_billed_tick_is_not_held_to_the_platform_paywall(owner_gate):
+ """A Codex-routed turn spends no platform dollars, as the chat route and
+ the queue dispatcher already treat it."""
+ _paywalled(owner_gate)
+ mocks = await _fire_hourly("session-1", "codex")
+
+ mocks["schedule_turn"].assert_awaited_once()
+ owner_gate.has_codex_access.assert_awaited_once_with("user-1")
+
+
+_ONE_SHOT = {"run_at": datetime.now(tz=timezone.utc) + timedelta(minutes=5)}
+_HOURLY = {"run_at": None, "cron": "9 * * * *"}
+
+
+ ("refuse", "timing", "dropped", "budget"),
+ [
+ (_paywalled, _ONE_SHOT, [("routine-1", "sched-1")], None),
+ (_over_a_cap, _ONE_SHOT, [("routine-1", "sched-1")], None),
+ (_limits_unreadable, _ONE_SHOT, [], "limits"),
+ (_lookup_fails, _ONE_SHOT, [], "plan_lookup"),
+ (_paywalled, _HOURLY, [], None),
+ (_limits_unreadable, _HOURLY, [], None),
+ (_lookup_fails, _HOURLY, [], None),
+ ],
+ ids=[
+ "paywalled-one-shot",
+ "over-a-cap-one-shot",
+ "brown-out-one-shot",
+ "lookup-fails-one-shot",
+ "paywalled-cron",
+ "brown-out-cron",
+ "lookup-fails-cron",
+ ],
+)
+async def test_a_refused_one_shot_routine_is_retried_or_switched_off(
+ owner_gate, refuse, timing, dropped, budget
+):
+ """APScheduler drops a one-shot once it fires. An outage (unreadable limits
+ or plan) is ours, so the one-shot comes back in a few minutes; otherwise,
+ left on, it would read as pending for a time that has passed."""
+ refuse(owner_gate)
+ scheduler_client = AsyncMock()
+ routine = ExpertRoutine(
+ id="routine-1", title="CI", prompt="Check CI.", enabled=True
+ )
+ store = MagicMock(
+ get_routine=AsyncMock(return_value=routine),
+ mark_routine_unscheduled=AsyncMock(),
+ )
+ session = MagicMock(session_id="session-1", expert_id=None)
+ session.metadata.llm_auth_provider = "platform"
+ schedule_turn = AsyncMock()
+ args = _args(routine_id="routine-1", **timing)
+ with (
+ patch(f"{_SCHEDULER_PATH}.experts_db", return_value=store),
+ patch(f"{_SCHEDULER_PATH}.get_chat_session", AsyncMock(return_value=session)),
+ patch(f"{_SCHEDULER_PATH}.schedule_turn", schedule_turn),
+ patch(f"{_SCHEDULER_PATH}.get_scheduler_client", return_value=scheduler_client),
+ ):
+ await _execute_copilot_turn(**args.model_dump(mode="json"))
+
+ schedule_turn.assert_not_awaited()
+ calls = store.mark_routine_unscheduled.await_args_list
+ assert [c.args for c in calls] == dropped
+ retry = scheduler_client.add_copilot_turn_schedule
+ assert retry.await_count == int(budget is not None)
+ if budget is not None:
+ assert retry.call_args.kwargs["routine_id"] == "routine-1"
+ assert retry.call_args.kwargs["routine_schedule_id"] == "sched-1"
+ assert retry.call_args.kwargs[f"{budget}_retry_count"] == 1
+
+
+ ("budget_left", "happened", "did_not"),
+ [
+ (True, "the one-shot is retried", "the one-shot is dropped"),
+ (False, "the one-shot is dropped", "the one-shot is retried"),
+ ],
+ ids=["retried", "budget-spent"],
+)
+async def test_a_skipped_one_shot_logs_what_became_of_it(
+ owner_gate, budget_left, happened, did_not, caplog
+):
+ _lookup_fails(owner_gate)
+ args = _args(plan_lookup_retry_count=0 if budget_left else _MAX_PLAN_LOOKUP_RETRIES)
+ session = MagicMock(session_id="session-1", expert_id=None)
+ session.metadata.llm_auth_provider = "platform"
+ with (
+ patch(f"{_SCHEDULER_PATH}.get_chat_session", AsyncMock(return_value=session)),
+ patch(f"{_SCHEDULER_PATH}.schedule_turn", AsyncMock()),
+ patch(f"{_SCHEDULER_PATH}.get_scheduler_client", return_value=AsyncMock()),
+ caplog.at_level(logging.INFO, logger=_SCHEDULER_PATH),
+ ):
+ await _execute_copilot_turn(**args.model_dump(mode="json"))
+
+ assert happened in caplog.text
+ assert did_not not in caplog.text
+
+
[email protected]("give_up", ["exhausted", "scheduler-down"])
+async def test_a_one_shot_retry_that_gives_up_switches_its_routine_off(give_up):
+ """The retry's own schedule_id is new to the routine's row, so it has to
+ carry the one the row holds, or the routine stays on with nothing behind it."""
+ args = _args(
+ schedule_id="sched-retry",
+ routine_id="routine-1",
+ routine_schedule_id="sched-0",
+ plan_lookup_retry_count=(
+ _MAX_PLAN_LOOKUP_RETRIES if give_up == "exhausted" else 0
+ ),
+ )
+ scheduler_client = AsyncMock()
+ scheduler_client.add_copilot_turn_schedule.side_effect = RuntimeError("down")
+ store = MagicMock(mark_routine_unscheduled=AsyncMock())
+ with (
+ patch(f"{_SCHEDULER_PATH}.get_scheduler_client", return_value=scheduler_client),
+ patch(f"{_SCHEDULER_PATH}.experts_db", return_value=store),
+ ):
+ await _reschedule_one_shot_after_plan_unreadable(args)
+
+ store.mark_routine_unscheduled.assert_awaited_once_with("routine-1", "sched-0")
+
+
@pytest.mark.parametrize("name", ["", " ", "\t\n", "\u2003"])
def test_graph_schedule_rejects_blank_names_before_validation_or_persistence(name):
scheduler = Scheduler(register_system_tasks=False)
diff --git a/autogpt_platform/frontend/src/app/api/openapi.json b/autogpt_platform/frontend/src/app/api/openapi.json
index 1899cd501b..2e197f6148 100644
--- a/autogpt_platform/frontend/src/app/api/openapi.json
+++ b/autogpt_platform/frontend/src/app/api/openapi.json
@@ -19922,6 +19922,16 @@
"title": "Expert Lookup Retry Count",
"default": 0
},
+ "plan_lookup_retry_count": {
+ "type": "integer",
+ "title": "Plan Lookup Retry Count",
+ "default": 0
+ },
+ "limits_retry_count": {
+ "type": "integer",
+ "title": "Limits Retry Count",
+ "default": 0
+ },
"user_timezone": {
"anyOf": [{ "type": "string" }, { "type": "null" }],
"title": "User Timezone"
@@ -19942,6 +19952,10 @@
"anyOf": [{ "type": "string" }, { "type": "null" }],
"title": "Routine Id"
},
+ "routine_schedule_id": {
+ "anyOf": [{ "type": "string" }, { "type": "null" }],
+ "title": "Routine Schedule Id"
+ },
"credential_pins": {
"additionalProperties": {
"$ref": "#/components/schemas/CredentialPin"