Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

feat(backend/copilot): resume or send a chat's turn when the budget resets (#15390)

Nicholas Tindle2fd8ee6c1544
diff --git a/autogpt_platform/backend/backend/api/features/chat/at_reset.py b/autogpt_platform/backend/backend/api/features/chat/at_reset.py
new file mode 100644
index 0000000000..7a9f5319aa
--- /dev/null
+++ b/autogpt_platform/backend/backend/api/features/chat/at_reset.py
@@ -0,0 +1,224 @@
+"""Resume a paused task, or send a written message, when the budget resets.
+
+Served under ``/api/chat`` next to the other session routes. When to fire, and
+whether to still fire then, live in ``backend.copilot.at_reset``; the job is a
+scheduled copilot turn, at most one of each kind per chat.
+"""
+
+from datetime import datetime
+from typing import Annotated
+
+from autogpt_libs import auth
+from fastapi import APIRouter, HTTPException, Response, Security
+from pydantic import BaseModel, Field, model_validator
+
+from backend.api.features.chat.routes import (
+ _validate_and_get_session,
+ _validate_and_get_writable_session,
+)
+from backend.copilot.at_reset import (
+ AtResetKind,
+ NoResetAhead,
+ at_reset_schedule_id,
+ next_reset_run_at,
+)
+from backend.copilot.continuation import RESUMED_AT_RESET_NOTE, as_model_prompt
+from backend.copilot.model import ChatSessionInfo
+from backend.copilot.rate_limit import RateLimitUnavailable
+from backend.executor.scheduler import CopilotTurnJobInfo, SchedulerClient
+from backend.util.clients import get_scheduler_client
+
+router = APIRouter(tags=["chat"])
+
+# The same cap as a message sent now (``StreamChatRequest``), for one
+# message and for messages joined while waiting for the reset.
+_MAX_MESSAGE_CHARS = 64_000
+
+# What a resume job keeps as its message, for the Scheduled page: the note it
+# fires carries a marker that page would show raw. It always fires the note.
+_RESUME_DESCRIPTION = as_model_prompt(RESUMED_AT_RESET_NOTE)
+
+
+class AtResetRequest(BaseModel):
+ """A turn to run when the budget resets."""
+
+ kind: AtResetKind = Field(
+ description='"resume" picks the paused task back up; "send" sends '
+ "``message`` as the user's.",
+ )
+ message: str | None = Field(
+ default=None,
+ max_length=_MAX_MESSAGE_CHARS,
+ description="What to send. Required for send, ignored for resume.",
+ )
+
+ @model_validator(mode="after")
+ def send_has_a_message(self) -> "AtResetRequest":
+ if self.kind == "send" and not (self.message and self.message.strip()):
+ raise ValueError("A send needs a message")
+ return self
+
+
+class AtResetTurn(BaseModel):
+ """A turn set to run when the budget resets."""
+
+ kind: AtResetKind
+ run_at: datetime
+ # What a send will send; None for a resume.
+ message: str | None = None
+
+
+class AtResetTurnsResponse(BaseModel):
+ """The chat's turns set for the budget reset, at most one of each kind."""
+
+ jobs: list[AtResetTurn]
+
+
+ "/sessions/{session_id}/at-reset",
+ summary="Schedule turn at budget reset",
+ dependencies=[Security(auth.requires_user)],
+ responses={
+ 404: {"description": "Session not found, or not writable"},
+ 409: {
+ "description": '"not_out_of_budget": nothing to wait for, so carry '
+ 'on now. "no_reset": the allowance never refills (a trial).'
+ },
+ 413: {
+ "description": '"message_too_long": the message waiting plus '
+ "this one would be longer than a single message may be"
+ },
+ 503: {"description": "Usage could not be read; retry shortly"},
+ },
+)
+async def schedule_at_reset(
+ session_id: str,
+ request: AtResetRequest,
+ user_id: Annotated[str, Security(auth.get_user_id)],
+ ctx: Annotated[auth.RequestContext, Security(auth.get_request_context)],
+) -> AtResetTurn:
+ """Resume the chat's paused task, or send a message, when the budget resets.
+
+ It fires just after the exhausted window that refills last. A chat has
+ at most one of each: a resume set again replaces the earlier one, and a
+ send joins the message already waiting, so nothing written meanwhile is
+ lost. Sending a message in the chat cancels a pending resume.
+ """
+ session = await _validate_and_get_writable_session(session_id, user_id)
+ run_at = await _run_at(user_id, session, request.kind)
+ message = (
+ await _joined_with_waiting_send(user_id, session_id, request.message)
+ if request.kind == "send" and request.message
+ else None
+ )
+ await _scheduler().add_copilot_turn_schedule(
+ user_id=user_id,
+ session_id=session_id,
+ message=message or _RESUME_DESCRIPTION,
+ run_at=run_at,
+ # The budget windows reset in UTC.
+ user_timezone="UTC",
+ # Anchored on the session, as for a message sent now.
+ organization_id=session.organization_id or ctx.org_id,
+ team_id=session.team_id if session.organization_id else ctx.team_id,
+ expert_id=session.expert_id,
+ purpose=request.kind,
+ schedule_id=at_reset_schedule_id(request.kind, session_id),
+ )
+ return AtResetTurn(kind=request.kind, run_at=run_at, message=message)
+
+
+ "/sessions/{session_id}/at-reset",
+ summary="List turns at budget reset",
+ dependencies=[Security(auth.requires_user)],
+ responses={404: {"description": "Session not found or access denied"}},
+)
+async def list_at_reset(
+ session_id: str,
+ user_id: Annotated[str, Security(auth.get_user_id)],
+) -> AtResetTurnsResponse:
+ """The chat's pending resume and send at the budget reset."""
+ await _validate_and_get_session(session_id, user_id)
+ turns = [_as_turn(job) for job in await _chat_turns(user_id, session_id)]
+ return AtResetTurnsResponse(jobs=[t for t in turns if t is not None])
+
+
+ "/sessions/{session_id}/at-reset/{kind}",
+ summary="Cancel turn at budget reset",
+ dependencies=[Security(auth.requires_user)],
+ status_code=204,
+ responses={404: {"description": "Session not found or access denied"}},
+)
+async def cancel_at_reset(
+ session_id: str,
+ kind: AtResetKind,
+ user_id: Annotated[str, Security(auth.get_user_id)],
+) -> Response:
+ """Cancel the chat's resume or send at the budget reset; one that isn't
+ set is already cancelled."""
+ await _validate_and_get_session(session_id, user_id)
+ await _scheduler().cancel_at_reset_turn(
+ user_id=user_id, session_id=session_id, kind=kind
+ )
+ return Response(status_code=204)
+
+
+async def _run_at(
+ user_id: str, session: ChatSessionInfo, kind: AtResetKind
+) -> datetime:
+ """When the turn fires, or the 409 or 503 that says why it can't be set."""
+ if session.metadata.llm_auth_provider != "platform":
+ # Only the platform route spends this budget, so the chat isn't
+ # waiting on it and can carry on now.
+ raise HTTPException(status_code=409, detail="not_out_of_budget")
+ try:
+ return await next_reset_run_at(user_id, session.session_id, kind)
+ except NoResetAhead as e:
+ raise HTTPException(status_code=409, detail=e.reason) from e
+ except RateLimitUnavailable as e:
+ raise HTTPException(
+ status_code=503,
+ detail="Rate limit service degraded, retry shortly",
+ headers={"Retry-After": "30"},
+ ) from e
+
+
+async def _joined_with_waiting_send(user_id: str, session_id: str, message: str) -> str:
+ """The message to schedule: this one after any already waiting."""
+ waiting = next(
+ (
+ job.message
+ for job in await _chat_turns(user_id, session_id)
+ if job.purpose == "send" and job.message
+ ),
+ None,
+ )
+ if waiting is None:
+ return message
+ joined = f"{waiting}\n\n{message}"
+ if len(joined) > _MAX_MESSAGE_CHARS:
+ raise HTTPException(status_code=413, detail="message_too_long")
+ return joined
+
+
+async def _chat_turns(user_id: str, session_id: str) -> list[CopilotTurnJobInfo]:
+ schedules = await _scheduler().get_execution_schedules(
+ user_id=user_id, session_id=session_id, kind="copilot_turn"
+ )
+ return [s for s in schedules if isinstance(s, CopilotTurnJobInfo)]
+
+
+def _scheduler() -> SchedulerClient:
+ # Every call here is safe to repeat: a read, a job set again under the
+ # chat's own id, a cancel. A chat lists its turns each time it opens, so a
+ # call lost on a connection the scheduler had just closed failed the load.
+ return get_scheduler_client(should_retry=True)
+
+
+def _as_turn(job: CopilotTurnJobInfo) -> AtResetTurn | None:
+ if job.purpose == "followup" or job.run_at is None:
+ return None
+ message = job.message if job.purpose == "send" else None
+ return AtResetTurn(kind=job.purpose, run_at=job.run_at, message=message)
diff --git a/autogpt_platform/backend/backend/api/features/chat/at_reset_test.py b/autogpt_platform/backend/backend/api/features/chat/at_reset_test.py
new file mode 100644
index 0000000000..fbc178e0c4
--- /dev/null
+++ b/autogpt_platform/backend/backend/api/features/chat/at_reset_test.py
@@ -0,0 +1,318 @@
+"""Route tests for /sessions/{session_id}/at-reset: setting, listing and
+cancelling a resume or send at the budget reset.
+
+The session checks, the budget reads and the scheduler are mocked at the
+boundary; ``backend/copilot/at_reset_test.py`` covers when the turn fires.
+"""
+
+from datetime import UTC, datetime
+from unittest.mock import AsyncMock, MagicMock
+
+import fastapi
+import fastapi.testclient
+import pytest
+import pytest_mock
+from autogpt_libs.auth.dependencies import get_request_context
+from autogpt_libs.auth.jwt_utils import get_jwt_payload
+
+from backend.api.features.chat import at_reset as at_reset_routes
+from backend.copilot.at_reset import NoResetAhead
+from backend.copilot.continuation import RESUMED_AT_RESET_NOTE, as_model_prompt
+from backend.copilot.rate_limit import RateLimitUnavailable
+from backend.executor.scheduler import CopilotTurnJobInfo
+from backend.util.exceptions import NotFoundError
+
+app = fastapi.FastAPI()
+app.include_router(at_reset_routes.router)
+
+
[email protected]_handler(NotFoundError)
+async def _not_found_handler(
+ request: fastapi.Request, exc: NotFoundError
+) -> fastapi.responses.JSONResponse:
+ """Mirror the production NotFoundError → 404 mapping from the REST app."""
+ return fastapi.responses.JSONResponse(status_code=404, content={"detail": str(exc)})
+
+
+client = fastapi.testclient.TestClient(app)
+
+SESSION = "sess-1"
+RUN_AT = datetime(2026, 10, 15, 0, 9, 14, tzinfo=UTC)
+
+
[email protected](autouse=True)
+def setup_app_auth(mock_jwt_user):
+ app.dependency_overrides[get_jwt_payload] = mock_jwt_user["get_jwt_payload"]
+ app.dependency_overrides[get_request_context] = mock_jwt_user["get_request_context"]
+ yield
+ app.dependency_overrides.clear()
+
+
+def scheduler(mocker: pytest_mock.MockerFixture) -> MagicMock:
+ scheduler_client = MagicMock(
+ add_copilot_turn_schedule=AsyncMock(),
+ get_execution_schedules=AsyncMock(return_value=[]),
+ cancel_at_reset_turn=AsyncMock(return_value=True),
+ )
+ mocker.patch.object(
+ at_reset_routes, "get_scheduler_client", return_value=scheduler_client
+ )
+ return scheduler_client
+
+
+def session(mocker: pytest_mock.MockerFixture) -> MagicMock:
+ chat = MagicMock(
+ session_id=SESSION, organization_id="org-1", team_id="team-1", expert_id=None
+ )
+ chat.metadata.llm_auth_provider = "platform"
+ for check in ("_validate_and_get_writable_session", "_validate_and_get_session"):
+ mocker.patch.object(at_reset_routes, check, AsyncMock(return_value=chat))
+ return chat
+
+
+def run_at(mocker: pytest_mock.MockerFixture) -> AsyncMock:
+ return mocker.patch.object(
+ at_reset_routes, "next_reset_run_at", AsyncMock(return_value=RUN_AT)
+ )
+
+
+def _when(body: dict) -> datetime:
+ return datetime.fromisoformat(body["run_at"])
+
+
+ "method, path, body",
+ [
+ ("GET", f"/sessions/{SESSION}/at-reset", None),
+ ("POST", f"/sessions/{SESSION}/at-reset", {"kind": "send", "message": "x"}),
+ ("DELETE", f"/sessions/{SESSION}/at-reset/send", None),
+ ],
+)
+def test_every_scheduler_call_is_repeated_if_lost_in_transit(
+ method, path, body, scheduler, session, run_at, mocker: pytest_mock.MockerFixture
+):
+ # A chat lists these turns each time it opens. In e2e, a pooled connection
+ # the scheduler had just closed failed that load with a 500.
+ factory = mocker.patch.object(
+ at_reset_routes, "get_scheduler_client", return_value=scheduler
+ )
+
+ response = client.request(method, path, json=body)
+
+ assert response.status_code < 300
+ assert factory.call_args_list
+ assert all(c.kwargs == {"should_retry": True} for c in factory.call_args_list)
+
+
+def test_resume_at_reset(scheduler, session, run_at, test_user_id):
+ response = client.post(
+ f"/sessions/{SESSION}/at-reset", json={"kind": "resume", "message": "x"}
+ )
+
+ assert response.status_code == 200
+ body = response.json()
+ assert (body["kind"], _when(body), body["message"]) == ("resume", RUN_AT, None)
+ run_at.assert_awaited_once_with(test_user_id, SESSION, "resume")
+ job = scheduler.add_copilot_turn_schedule.await_args.kwargs
+ assert job["purpose"] == "resume"
+ assert job["schedule_id"] == f"at-reset-resume-{SESSION}"
+ assert job["session_id"] == SESSION
+ assert job["run_at"] == RUN_AT
+ # The listing shows a readable line; what fires is the note itself.
+ assert job["message"] == as_model_prompt(RESUMED_AT_RESET_NOTE)
+ assert (job["organization_id"], job["team_id"]) == ("org-1", "team-1")
+
+
+def test_send_at_reset_keeps_the_users_message(scheduler, session, run_at):
+ response = client.post(
+ f"/sessions/{SESSION}/at-reset",
+ json={"kind": "send", "message": "also check the invoices"},
+ )
+
+ assert response.status_code == 200
+ assert response.json()["message"] == "also check the invoices"
+ job = scheduler.add_copilot_turn_schedule.await_args.kwargs
+ assert job["purpose"] == "send"
+ assert job["schedule_id"] == f"at-reset-send-{SESSION}"
+ assert job["message"] == "also check the invoices"
+
+
+def test_a_second_send_joins_the_message_already_waiting(scheduler, session, run_at):
+ # One send per chat: replacing the waiting one would lose what the user
+ # wrote first.
+ scheduler.get_execution_schedules.return_value = [
+ _job("send", "draft the launch post")
+ ]
+ response = client.post(
+ f"/sessions/{SESSION}/at-reset",
+ json={"kind": "send", "message": "also the changelog"},
+ )
+
+ assert response.status_code == 200
+ joined = "draft the launch post\n\nalso the changelog"
+ assert response.json()["message"] == joined
+ assert scheduler.add_copilot_turn_schedule.await_args.kwargs["message"] == joined
+
+
+def test_a_joined_send_longer_than_a_message_is_413(scheduler, session, run_at):
+ scheduler.get_execution_schedules.return_value = [_job("send", "x" * 60_000)]
+ response = client.post(
+ f"/sessions/{SESSION}/at-reset",
+ json={"kind": "send", "message": "y" * 5_000},
+ )
+
+ assert response.status_code == 413
+ assert response.json()["detail"] == "message_too_long"
+ scheduler.add_copilot_turn_schedule.assert_not_awaited()
+
+
+def test_a_resume_set_again_replaces_it(scheduler, session, run_at):
+ scheduler.get_execution_schedules.return_value = [
+ _job("resume", "Resumed after the budget reset.")
+ ]
+ response = client.post(f"/sessions/{SESSION}/at-reset", json={"kind": "resume"})
+
+ assert response.status_code == 200
+ scheduler.get_execution_schedules.assert_not_awaited()
+
+
+ "payload",
+ [
+ {"kind": "send"},
+ {"kind": "send", "message": " "},
+ {"kind": "send", "message": "x" * 64_001},
+ {"kind": "later"},
+ ],
+)
+def test_a_send_needs_a_message_and_a_known_kind(scheduler, session, run_at, payload):
+ response = client.post(f"/sessions/{SESSION}/at-reset", json=payload)
+
+ assert response.status_code == 422
+ scheduler.add_copilot_turn_schedule.assert_not_awaited()
+
+
+def test_an_unknown_session_is_404(
+ scheduler, run_at, mocker: pytest_mock.MockerFixture
+):
+ mocker.patch.object(
+ at_reset_routes,
+ "_validate_and_get_writable_session",
+ AsyncMock(side_effect=NotFoundError(f"Session {SESSION} not found.")),
+ )
+
+ response = client.post(f"/sessions/{SESSION}/at-reset", json={"kind": "resume"})
+
+ assert response.status_code == 404
+ run_at.assert_not_awaited()
+ scheduler.add_copilot_turn_schedule.assert_not_awaited()
+
+
[email protected]("reason", ["not_out_of_budget", "no_reset"])
+def test_nothing_to_wait_for_is_409(scheduler, session, run_at, reason):
+ run_at.side_effect = NoResetAhead(reason)
+
+ response = client.post(f"/sessions/{SESSION}/at-reset", json={"kind": "resume"})
+
+ assert response.status_code == 409
+ assert response.json()["detail"] == reason
+ scheduler.add_copilot_turn_schedule.assert_not_awaited()
+
+
+def test_a_chat_off_the_platform_route_has_nothing_to_wait_for(
+ scheduler, session, run_at
+):
+ session.metadata.llm_auth_provider = "codex"
+
+ response = client.post(
+ f"/sessions/{SESSION}/at-reset", json={"kind": "send", "message": "hi"}
+ )
+
+ assert response.status_code == 409
+ assert response.json()["detail"] == "not_out_of_budget"
+ run_at.assert_not_awaited()
+
+
+def test_unreadable_usage_is_503(scheduler, session, run_at):
+ run_at.side_effect = RateLimitUnavailable()
+
+ response = client.post(f"/sessions/{SESSION}/at-reset", json={"kind": "resume"})
+
+ assert response.status_code == 503
+ assert response.headers["Retry-After"] == "30"
+
+
+def _job(purpose: str, message: str) -> CopilotTurnJobInfo:
+ return CopilotTurnJobInfo.model_validate(
+ {
+ "schedule_id": f"{purpose}-job",
+ "user_id": "user-1",
+ "session_id": SESSION,
+ "message": message,
+ "run_at": RUN_AT,
+ "purpose": purpose,
+ "id": f"{purpose}-job",
+ "name": purpose,
+ "next_run_time": RUN_AT.isoformat(),
+ }
+ )
+
+
+def test_list_shows_the_chats_resume_and_send_only(scheduler, session, test_user_id):
+ scheduler.get_execution_schedules.return_value = [
+ _job("followup", "check CI"),
+ _job("resume", "Resumed after the budget reset."),
+ _job("send", "also check the invoices"),
+ ]
+
+ response = client.get(f"/sessions/{SESSION}/at-reset")
+
+ assert response.status_code == 200
+ jobs = response.json()["jobs"]
+ assert [(j["kind"], j["message"]) for j in jobs] == [
+ ("resume", None),
+ ("send", "also check the invoices"),
+ ]
+ assert all(_when(j) == RUN_AT for j in jobs)
+ scheduler.get_execution_schedules.assert_awaited_once_with(
+ user_id=test_user_id, session_id=SESSION, kind="copilot_turn"
+ )
+
+
[email protected]("removed", [True, False])
+def test_cancel_is_204_whether_or_not_one_was_set(
+ scheduler, session, test_user_id, removed
+):
+ scheduler.cancel_at_reset_turn.return_value = removed
+
+ response = client.delete(f"/sessions/{SESSION}/at-reset/send")
+
+ assert response.status_code == 204
+ scheduler.cancel_at_reset_turn.assert_awaited_once_with(
+ user_id=test_user_id, session_id=SESSION, kind="send"
+ )
+
+
+def test_cancel_an_unknown_kind_is_422(scheduler, session):
+ response = client.delete(f"/sessions/{SESSION}/at-reset/later")
+
+ assert response.status_code == 422
+ scheduler.cancel_at_reset_turn.assert_not_awaited()
+
+
+def test_cancel_on_an_unknown_session_is_404(
+ scheduler, mocker: pytest_mock.MockerFixture
+):
+ mocker.patch.object(
+ at_reset_routes,
+ "_validate_and_get_session",
+ AsyncMock(side_effect=NotFoundError(f"Session {SESSION} not found.")),
+ )
+
+ response = client.delete(f"/sessions/{SESSION}/at-reset/resume")
+
+ assert response.status_code == 404
+ scheduler.cancel_at_reset_turn.assert_not_awaited()
diff --git a/autogpt_platform/backend/backend/api/features/chat/continuation.py b/autogpt_platform/backend/backend/api/features/chat/continuation.py
index df52253649..b1a527de2b 100644
--- a/autogpt_platform/backend/backend/api/features/chat/continuation.py
+++ b/autogpt_platform/backend/backend/api/features/chat/continuation.py
@@ -21,6 +21,7 @@ from backend.api.features.chat.routes import (
)
from backend.copilot import stream_registry
from backend.copilot.active_turns import ConcurrentTurnLimitError
+from backend.copilot.at_reset import cancel_resume_at_reset_in_background
from backend.copilot.config import CopilotLLMModel
from backend.copilot.continuation import KEEP_GOING_NOTE
from backend.copilot.executor.utils import schedule_chat_turn
@@ -104,6 +105,8 @@ async def continue_session(
# Another turn took the chat after the busy check above: a message
# sent at the same moment, or the first click of a double click.
raise HTTPException(status_code=409, detail="turn_in_progress")
+ # Picked up now, so a pending "Resume at reset" would repeat it.
+ cancel_resume_at_reset_in_background(session_id, user_id)
await _announce(turn_id, session_id)
return ContinueSessionResponse(turn_id=turn_id)
diff --git a/autogpt_platform/backend/backend/api/features/chat/continuation_test.py b/autogpt_platform/backend/backend/api/features/chat/continuation_test.py
index 63bef1952f..c9beb45193 100644
--- a/autogpt_platform/backend/backend/api/features/chat/continuation_test.py
+++ b/autogpt_platform/backend/backend/api/features/chat/continuation_test.py
@@ -13,6 +13,7 @@ from backend.api.features.chat import continuation as continuation_routes
from backend.api.features.chat import routes as chat_routes
from backend.copilot import model as chat_model
from backend.copilot.active_turns import ConcurrentTurnLimitError
+from backend.copilot.at_reset import cancel_resume_at_reset_in_background
from backend.copilot.constants import COPILOT_SYSTEM_PREFIX
from backend.copilot.continuation import KEEP_GOING_NOTE
from backend.copilot.executor import utils as executor_utils
@@ -85,6 +86,9 @@ def chat(mocker: pytest_mock.MockerFixture):
mocks.publish = mocker.patch.object(
continuation_routes.stream_registry, "publish_chunk", new=AsyncMock()
)
+ mocks.cancel_resume = mocker.patch.object(
+ continuation_routes, "cancel_resume_at_reset_in_background"
+ )
return mocks
@@ -175,6 +179,43 @@ def test_a_chat_taken_after_the_busy_check_is_a_409(chat):
chat.publish.assert_not_awaited()
+def test_keep_going_cancels_a_pending_resume_at_reset(chat, test_user_id):
+ # Only the resume: ``cancel_resume_at_reset_in_background`` leaves a
+ # pending send alone (copilot/at_reset_test.py).
+ assert client.post(URL).status_code == 202
+ chat.cancel_resume.assert_called_once_with(SESSION_ID, test_user_id)
+
+
[email protected]("refusal", ["busy", "duplicate"])
+def test_a_keep_going_that_didnt_start_leaves_the_resume(chat, refusal):
+ if refusal == "busy":
+ chat.busy.return_value = True
+ else:
+ chat.schedule.return_value = None
+
+ assert client.post(URL).status_code == 409
+ chat.cancel_resume.assert_not_called()
+
+
+def test_a_failing_resume_cancel_never_fails_keep_going(
+ chat, mocker: pytest_mock.MockerFixture
+):
+ mocker.patch.object(
+ continuation_routes,
+ "cancel_resume_at_reset_in_background",
+ cancel_resume_at_reset_in_background,
+ )
+ scheduler_down = MagicMock(
+ cancel_at_reset_turn=AsyncMock(side_effect=RuntimeError("scheduler down"))
+ )
+ mocker.patch(
+ "backend.copilot.at_reset.get_scheduler_client", return_value=scheduler_down
+ )
+
+ assert client.post(URL).status_code == 202
+ chat.schedule.assert_awaited_once()
+
+
_PAUSE = f"{COPILOT_SYSTEM_PREFIX} Paused: checking in after a long run."
diff --git a/autogpt_platform/backend/backend/api/features/chat/routes.py b/autogpt_platform/backend/backend/api/features/chat/routes.py
index a55aa8cb59..61734cce39 100644
--- a/autogpt_platform/backend/backend/api/features/chat/routes.py
+++ b/autogpt_platform/backend/backend/api/features/chat/routes.py
@@ -22,6 +22,7 @@ from backend.copilot.active_turns import (
get_inflight_turn_limit,
inflight_turn_limit_message,
)
+from backend.copilot.at_reset import cancel_resume_at_reset_in_background
from backend.copilot.computer import (
ComputerInfo,
computer_owner,
@@ -2141,6 +2142,10 @@ async def stream_chat_post(
logger.warning(
"[STREAM] Failed to publish initial status chunk", exc_info=True
)
+ if request.is_user_message:
+ # The user carried the chat on, so a pending "Resume at reset"
+ # would repeat their turn.
+ cancel_resume_at_reset_in_background(session_id, user_id)
setup_time = (time.perf_counter() - stream_start_time) * 1000
logger.info(
diff --git a/autogpt_platform/backend/backend/api/features/chat/routes_test.py b/autogpt_platform/backend/backend/api/features/chat/routes_test.py
index 84653bbf14..9fa1b491a6 100644
--- a/autogpt_platform/backend/backend/api/features/chat/routes_test.py
+++ b/autogpt_platform/backend/backend/api/features/chat/routes_test.py
@@ -19,7 +19,9 @@ from backend.api.features.chat import routes as chat_routes
from backend.api.features.chat.routes import _strip_injected_context
from backend.copilot import pending_messages as pending_messages_module
from backend.copilot import transports as chat_transports
+from backend.copilot.at_reset import cancel_resume_at_reset_in_background
from backend.copilot.config import CopilotLlmAuthProvider
+from backend.copilot.continuation import KEEP_GOING_NOTE
from backend.copilot.model import ChatSession
from backend.copilot.offers import EntitlementUnavailable
from backend.copilot.rate_limit import SubscriptionTier
@@ -367,12 +369,18 @@ def _mock_stream_internals(
new_callable=AsyncMock,
return_value=None,
)
+ # A sent message cancels the chat's pending resume at the budget reset;
+ # keep that off the scheduler.
+ mock_cancel_resume = mocker.patch(
+ "backend.api.features.chat.routes.cancel_resume_at_reset_in_background"
+ )
claims = _mock_client_message_claims(mocker)
return types.SimpleNamespace(
enqueue=mock_schedule,
paywall=mock_paywall,
session=mock_session,
owns_active_expert=mock_owns_active_expert,
+ cancel_resume=mock_cancel_resume,
claims=claims,
)
@@ -573,6 +581,62 @@ def test_stream_chat_skips_the_expert_gate_for_a_non_expert_session(
mocks.enqueue.assert_awaited_once()
+# ─── A sent message cancels the pending resume at the budget reset ────
+
+
+def test_a_user_message_cancels_the_pending_resume(
+ mocker: pytest_mock.MockerFixture, test_user_id: str
+) -> None:
+ mocks = _mock_stream_internals(mocker)
+
+ response = client.post("/sessions/sess-1/stream", json={"message": "carry on"})
+
+ assert response.status_code == 200
+ mocks.cancel_resume.assert_called_once_with("sess-1", test_user_id)
+
+
+ "payload, turn_id",
+ [
+ ({"message": KEEP_GOING_NOTE, "is_user_message": False}, "turn-id-mock"),
+ ({"message": "carry on"}, None),
+ ],
+ ids=["not-a-user-message", "duplicate-not-scheduled"],
+)
+def test_the_resume_stays_unless_a_user_turn_was_scheduled(
+ mocker: pytest_mock.MockerFixture, payload: dict, turn_id: str | None
+) -> None:
+ mocks = _mock_stream_internals(mocker)
+ mocks.enqueue.return_value = turn_id
+
+ response = client.post("/sessions/sess-1/stream", json=payload)
+
+ assert response.status_code == 200
+ mocks.cancel_resume.assert_not_called()
+
+
+def test_a_failing_resume_cancel_never_fails_the_send(
+ mocker: pytest_mock.MockerFixture,
+) -> None:
+ mocks = _mock_stream_internals(mocker)
+ mocker.patch.object(
+ chat_routes,
+ "cancel_resume_at_reset_in_background",
+ cancel_resume_at_reset_in_background,
+ )
+ scheduler_down = MagicMock(
+ cancel_at_reset_turn=AsyncMock(side_effect=RuntimeError("scheduler down"))
+ )
+ mocker.patch(
+ "backend.copilot.at_reset.get_scheduler_client", return_value=scheduler_down
+ )
+
+ response = client.post("/sessions/sess-1/stream", json={"message": "carry on"})
+
+ assert response.status_code == 200
+ mocks.enqueue.assert_awaited_once()
+
+
# ─── Duplicate message dedup ──────────────────────────────────────────
diff --git a/autogpt_platform/backend/backend/api/rest_api.py b/autogpt_platform/backend/backend/api/rest_api.py
index ccbaeb5b71..0b8eebb4fb 100644
--- a/autogpt_platform/backend/backend/api/rest_api.py
+++ b/autogpt_platform/backend/backend/api/rest_api.py
@@ -36,6 +36,7 @@ import backend.api.features.blocks.routes as blocks_routes
import backend.api.features.briefings.routes
import backend.api.features.builder
import backend.api.features.builder.routes
+import backend.api.features.chat.at_reset as chat_at_reset
import backend.api.features.chat.continuation as chat_continuation
import backend.api.features.chat.feedback as chat_feedback
import backend.api.features.chat.routes as chat_routes
@@ -578,6 +579,11 @@ app.include_router(
tags=["v2", "chat"],
prefix="/api/chat",
)
+app.include_router(
+ chat_at_reset.router,
+ tags=["v2", "chat"],
+ prefix="/api/chat",
+)
app.include_router(
chat_continuation.router,
tags=["v2", "chat"],
diff --git a/autogpt_platform/backend/backend/copilot/at_reset.py b/autogpt_platform/backend/backend/copilot/at_reset.py
new file mode 100644
index 0000000000..62dbdba3ce
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/at_reset.py
@@ -0,0 +1,281 @@
+"""Picking a chat back up when the budget resets: when to, and whether to still.
+
+Out of budget, a user can have a paused task resume when the budget comes back
+("Resume at reset"), or write a message now that sends then ("Send at reset").
+Each is a scheduled copilot turn bound to the chat, at most one of each kind
+per chat (``at_reset_schedule_id``), and kept out of the agent's own schedule
+tools. It fires a little after the window that is actually blocking refills,
+offset per chat so paused chats don't all fire at midnight UTC, and looks
+again when it fires (``fire_verdict``): the user may have picked the chat up
+already, or still be out of budget.
+"""
+
+import asyncio
+import hashlib
+import logging
+from datetime import UTC, datetime, timedelta
+from typing import Literal
+
+from pydantic import BaseModel
+
+from backend.copilot.config import ChatConfig
+from backend.copilot.continuation import KEEP_GOING_NOTE, RESUMED_AT_RESET_NOTE
+from backend.copilot.db import PaginatedMessages
+from backend.copilot.model import ChatMessage
+from backend.copilot.pending_message_helpers import (
+ StreamRegistryUnavailable,
+ is_turn_in_flight,
+)
+from backend.copilot.rate_limit import (
+ RateLimitExceeded,
+ RateLimitUnavailable,
+ SubscriptionTier,
+ check_rate_limit,
+ get_global_rate_limits,
+ get_usage_status,
+)
+from backend.data.db_accessors import chat_db, credit_db
+from backend.util.background import spawn_background_task
+from backend.util.clients import get_scheduler_client
+
+logger = logging.getLogger(__name__)
+
+AtResetKind = Literal["resume", "send"]
+NoResetReason = Literal["not_out_of_budget", "no_reset"]
+
+# What the Scheduled page calls each job.
+AT_RESET_JOB_NAMES: dict[AtResetKind, str] = {
+ "resume": "Resume when the budget resets",
+ "send": "Send when the budget resets",
+}
+
+config = ChatConfig()
+
+# Fire a little after the window refills rather than on the dot.
+_AFTER_RESET = timedelta(seconds=120)
+# Spread over ten minutes, so paused chats don't all fire at 00:00 UTC.
+_JITTER_SECONDS = 600
+# A send and a resume on one chat would otherwise fire together. The send goes
+# first; the resume then finds the user's message newer than itself and stands
+# down, as it does for a message sent now.
+_RESUME_AFTER_SEND = timedelta(seconds=60)
+# How many of the chat's latest rows the fire-time check reads.
+_RECENT_ROWS = 50
+_PICKUP_NOTES = (KEEP_GOING_NOTE, RESUMED_AT_RESET_NOTE)
+
+
+class NoResetAhead(Exception):
+ """There is no budget reset for an at-reset turn to wait for."""
+
+ def __init__(self, reason: NoResetReason):
+ super().__init__(reason)
+ self.reason: NoResetReason = reason
+
+
+class FireVerdict(BaseModel):
+ """What a due at-reset turn does now."""
+
+ # fire: dispatch it. drop: let it go. retry: try again in a few minutes.
+ # wait: still out of budget, so move it to ``run_at``, the next reset.
+ action: Literal["fire", "drop", "retry", "wait"]
+ run_at: datetime | None = None
+ why: str = ""
+
+
+def at_reset_schedule_id(kind: AtResetKind, session_id: str) -> str:
+ """The job's id: one per kind per chat, so setting it again replaces it."""
+ return f"at-reset-{kind}-{session_id}"
+
+
+async def next_reset_run_at(
+ user_id: str, session_id: str, kind: AtResetKind
+) -> datetime:
+ """When this chat's at-reset turn should fire.
+
+ Just after the exhausted window that refills last: a user out of both
+ today's and this week's budget can't start until the week resets. Raises
+ ``NoResetAhead`` when nothing is exhausted, or when what is exhausted never
+ refills (a trial's lifetime allowance, or a window that allows nothing),
+ and ``RateLimitUnavailable`` when usage can't be read.
+ """
+ offset = timedelta(seconds=reset_jitter_seconds(session_id))
+ if kind == "resume":
+ offset += _RESUME_AFTER_SEND
+ return await _blocking_reset(user_id) + _AFTER_RESET + offset
+
+
+def reset_jitter_seconds(session_id: str) -> int:
+ """This chat's offset into the ten minutes after a reset, in seconds.
+
+ A hash of the chat id rather than a random draw, so moving the turn to a
+ later reset keeps it at the same point in that window.
+ """
+ digest = hashlib.sha256(session_id.encode()).digest()
+ return int.from_bytes(digest[:8], "big") % _JITTER_SECONDS
+
+
+async def load_at_reset_chat(session_id: str, user_id: str) -> PaginatedMessages | None:
+ """The chat an at-reset turn is bound to, with its latest rows.
+
+ Read from the database rather than the session cache: the fire-time check
+ compares when rows were written, and rows appended in-process and cached
+ don't say. None when the chat is gone or isn't the user's.
+ """
+ return await chat_db().get_chat_messages_paginated(
+ session_id, limit=_RECENT_ROWS, user_id=user_id
+ )
+
+
+async def fire_verdict(
+ kind: AtResetKind, chat: PaginatedMessages, created_at: datetime | None
+) -> FireVerdict:
+ """Whether a due at-reset turn fires now, and if not, what becomes of it."""
+ session = chat.session
+ if kind == "resume" and await _picked_up_since(chat, created_at):
+ return FireVerdict(action="drop", why="the chat was picked up after it")
+ if kind == "resume" and session.metadata.llm_auth_provider == (
+ "microsoft_365_copilot"
+ ):
+ # Switched since it was set. Microsoft 365 Copilot has none of the
+ # task or AutoPilot's tools, and the resume note would reach it as
+ # the user's message. A send is the user's own words, so it goes.
+ return FireVerdict(
+ action="drop", why="the chat now runs on Microsoft 365 Copilot"
+ )
+ try:
+ busy = await is_turn_in_flight(session.session_id)
+ except StreamRegistryUnavailable:
+ return FireVerdict(action="retry", why="could not tell if the chat is busy")
+ if busy:
+ return FireVerdict(action="retry", why="a turn is running in the chat")
+ if session.metadata.llm_auth_provider != "platform":
+ # Only the platform route spends this budget.
+ return FireVerdict(action="fire")
+ return await _budget_verdict(session.user_id, session.session_id, kind)
+
+
+def cancel_resume_at_reset_in_background(session_id: str, user_id: str) -> asyncio.Task:
+ """Drop the chat's pending "Resume at reset": the user carried it on now.
+
+ A pending send stays. Off the caller's path, so it never fails or slows a
+ request; best effort, since the resume also stands down when it fires.
+ """
+ return spawn_background_task(
+ _cancel_resume(session_id, user_id),
+ name=f"cancel-resume-at-reset-{session_id[:12]}",
+ )
+
+
+async def _cancel_resume(session_id: str, user_id: str) -> None:
+ try:
+ # A cancel is safe to repeat if a closed connection loses it.
+ await get_scheduler_client(should_retry=True).cancel_at_reset_turn(
+ user_id=user_id, session_id=session_id, kind="resume"
+ )
+ except Exception:
+ logger.warning(
+ f"Could not cancel the resume-at-reset of session {session_id[:12]}",
+ exc_info=True,
+ )
+
+
+async def _blocking_reset(user_id: str) -> datetime:
+ daily_limit, weekly_limit, tier = await get_global_rate_limits(
+ user_id,
+ config.daily_cost_limit_microdollars,
+ config.weekly_cost_limit_microdollars,
+ )
+ # A window that allows nothing (a lapsed trial or plan) never refills, and
+ # nor does a trial's lifetime allowance. A trial's own daily or weekly
+ # window does, at that window's reset.
+ if daily_limit == 0 or weekly_limit == 0:
+ raise NoResetAhead("no_reset")
+ if tier == SubscriptionTier.TRIAL and await _trial_allowance_spent(user_id):
+ raise NoResetAhead("no_reset")
+ status = await get_usage_status(user_id, daily_limit, weekly_limit, tier=tier)
+ # As in ``check_rate_limit``: a negative limit means no cap at all.
+ exhausted = [w for w in (status.daily, status.weekly) if 0 <= w.limit <= w.used]
+ if not exhausted:
+ raise NoResetAhead("not_out_of_budget")
+ return max(w.resets_at for w in exhausted)
+
+
+async def _trial_allowance_spent(user_id: str) -> bool:
+ """Whether the trial has nothing left to wait for: its lifetime allowance is
+ used up, or the trial itself is over (as ``check_rate_limit`` reads it)."""
+ trial = await credit_db().get_subscription_trial(user_id)
+ return (
+ trial is None
+ or not trial.active
+ or trial.cost_microdollars >= trial.offer.total_cost_limit
+ )
+
+
+async def _budget_verdict(
+ user_id: str, session_id: str, kind: AtResetKind
+) -> FireVerdict:
+ try:
+ daily_limit, weekly_limit, tier = await get_global_rate_limits(
+ user_id,
+ config.daily_cost_limit_microdollars,
+ config.weekly_cost_limit_microdollars,
+ )
+ if daily_limit == 0 and weekly_limit == 0:
+ # No plan, or the trial is over. Settled here: an ended
+ # trial's usage state still reads TRIAL, which the tier guard
+ # below reports as unavailable, so it would only be retried.
+ return FireVerdict(action="drop", why="its allowance never refills")
+ await check_rate_limit(
+ user_id=user_id,
+ daily_cost_limit=daily_limit,
+ weekly_cost_limit=weekly_limit,
+ expected_tier=tier,
+ )
+ except RateLimitUnavailable:
+ return FireVerdict(action="retry", why="the budget could not be read")
+ except RateLimitExceeded:
+ return await _next_reset_verdict(user_id, session_id, kind)
+ return FireVerdict(action="fire")
+
+
+async def _next_reset_verdict(
+ user_id: str, session_id: str, kind: AtResetKind
+) -> FireVerdict:
+ try:
+ run_at = await next_reset_run_at(user_id, session_id, kind)
+ except NoResetAhead as e:
+ if e.reason == "no_reset":
+ return FireVerdict(action="drop", why="its allowance never refills")
+ # Over the limit a moment ago and under it now: look again shortly.
+ return FireVerdict(action="retry", why="the budget moved while checking")
+ except RateLimitUnavailable:
+ return FireVerdict(action="retry", why="the budget could not be read")
+ return FireVerdict(action="wait", run_at=run_at, why="still out of budget")
+
+
+async def _picked_up_since(chat: PaginatedMessages, since: datetime | None) -> bool:
+ """Whether the user carried the chat on after the resume was set: with a
+ message of their own, or a Keep going (or an earlier resume) that ran."""
+ if since is None:
+ return False
+ rows = list(chat.messages)
+ # Their newest message can be further back than the latest rows.
+ latest_user = await chat_db().get_latest_user_message_in_session(
+ chat.session.session_id
+ )
+ if latest_user is not None:
+ rows.append(latest_user)
+ return any(_picks_up(row) and _written_after(row, since) for row in rows)
+
+
+def _picks_up(row: ChatMessage) -> bool:
+ return row.role == "user" or row.content in _PICKUP_NOTES
+
+
+def _written_after(row: ChatMessage, since: datetime) -> bool:
+ return row.created_at is not None and _as_utc(row.created_at) > _as_utc(since)
+
+
+def _as_utc(moment: datetime) -> datetime:
+ """A naive time is UTC, as the database writes it."""
+ return moment if moment.tzinfo else moment.replace(tzinfo=UTC)
diff --git a/autogpt_platform/backend/backend/copilot/at_reset_test.py b/autogpt_platform/backend/backend/copilot/at_reset_test.py
new file mode 100644
index 0000000000..028f365566
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/at_reset_test.py
@@ -0,0 +1,346 @@
+from datetime import UTC, datetime, timedelta
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from backend.copilot.at_reset import (
+ FireVerdict,
+ NoResetAhead,
+ at_reset_schedule_id,
+ cancel_resume_at_reset_in_background,
+ fire_verdict,
+ next_reset_run_at,
+ reset_jitter_seconds,
+)
+from backend.copilot.continuation import KEEP_GOING_NOTE, RESUMED_AT_RESET_NOTE
+from backend.copilot.db import PaginatedMessages
+from backend.copilot.model import ChatMessage, ChatSessionInfo, ChatSessionMetadata
+from backend.copilot.pending_message_helpers import StreamRegistryUnavailable
+from backend.copilot.rate_limit import (
+ CoPilotUsageStatus,
+ RateLimitExceeded,
+ RateLimitUnavailable,
+ SubscriptionTier,
+ UsageWindow,
+)
+
+_MOD = "backend.copilot.at_reset"
+SESSION = "5b0c2c0e-9b1f-4c6e-8f43-0d7e2a1b3c4d"
+DAILY_RESET = datetime(2026, 10, 15, tzinfo=UTC)
+WEEKLY_RESET = datetime(2026, 10, 19, tzinfo=UTC)
+SET_AT = datetime(2026, 10, 14, 15, 0, tzinfo=UTC)
+JITTER = timedelta(seconds=reset_jitter_seconds(SESSION))
+
+
+def _status(daily_used: int, weekly_used: int, *, daily_limit: int = 3_000_000):
+ return CoPilotUsageStatus(
+ daily=UsageWindow(used=daily_used, limit=daily_limit, resets_at=DAILY_RESET),
+ weekly=UsageWindow(used=weekly_used, limit=6_000_000, resets_at=WEEKLY_RESET),
+ tier=SubscriptionTier.PRO,
+ )
+
+
+def _row(role: str, content: str, written: datetime | None) -> ChatMessage:
+ return ChatMessage(role=role, content=content, created_at=written)
+
+
+def _chat(*rows: ChatMessage, provider: str = "platform") -> PaginatedMessages:
+ session = ChatSessionInfo(
+ session_id=SESSION,
+ user_id="user-1",
+ usage=[],
+ started_at=SET_AT - timedelta(hours=2),
+ updated_at=SET_AT,
+ metadata=ChatSessionMetadata.model_validate({"llm_auth_provider": provider}),
+ )
+ return PaginatedMessages(
+ messages=list(rows), has_more=False, oldest_sequence=0, session=session
+ )
+
+
+def limits():
+ """The budget reads; tests set the usage ``status`` and the gate."""
+ with (
+ patch(
+ f"{_MOD}.get_global_rate_limits",
+ AsyncMock(return_value=(3_000_000, 6_000_000, SubscriptionTier.PRO)),
+ ) as global_limits,
+ patch(f"{_MOD}.get_usage_status", AsyncMock()) as status,
+ patch(f"{_MOD}.check_rate_limit", AsyncMock()) as gate,
+ ):
+ yield global_limits, status, gate
+
+
+def chat_reads():
+ """Nothing running in the chat, and no user message beyond the window."""
+ store = MagicMock(get_latest_user_message_in_session=AsyncMock(return_value=None))
+ with (
+ patch(f"{_MOD}.chat_db", return_value=store),
+ patch(f"{_MOD}.is_turn_in_flight", AsyncMock(return_value=False)) as busy,
+ ):
+ yield store, busy
+
+
+def trial(limits):
+ """A trial user on a legacy offer: daily and weekly caps under a lifetime
+ allowance with most of it left. Tests spend it as they need."""
+ global_limits, _, _ = limits
+ global_limits.return_value = (3_000_000, 6_000_000, SubscriptionTier.TRIAL)
+ state = MagicMock(active=True, cost_microdollars=4_000_000)
+ state.offer.total_cost_limit = 10_000_000
+ store = MagicMock(get_subscription_trial=AsyncMock(return_value=state))
+ with patch(f"{_MOD}.credit_db", return_value=store):
+ yield state
+
+
+def test_one_id_per_kind_per_chat():
+ assert at_reset_schedule_id("resume", SESSION) == f"at-reset-resume-{SESSION}"
+ assert at_reset_schedule_id("send", SESSION) == f"at-reset-send-{SESSION}"
+
+
+def test_jitter_is_stable_per_chat_and_inside_ten_minutes():
+ offsets = {reset_jitter_seconds(f"session-{i}") for i in range(200)}
+ assert all(0 <= offset < 600 for offset in offsets)
+ # Spread out, not everyone on the same second.
+ assert len(offsets) > 100
+ assert reset_jitter_seconds(SESSION) == reset_jitter_seconds(SESSION)
+
+
+async def test_out_of_todays_budget_waits_for_tomorrow(limits):
+ _, status, _ = limits
+ status.return_value = _status(daily_used=3_000_000, weekly_used=1_000_000)
+ run_at = await next_reset_run_at("user-1", SESSION, "send")
+ assert run_at == DAILY_RESET + timedelta(seconds=120) + JITTER
+
+
+async def test_out_of_both_waits_for_the_one_that_resets_last(limits):
+ _, status, _ = limits
+ status.return_value = _status(daily_used=3_000_000, weekly_used=6_500_000)
+ run_at = await next_reset_run_at("user-1", SESSION, "send")
+ assert run_at == WEEKLY_RESET + timedelta(seconds=120) + JITTER
+ after_reset = run_at - WEEKLY_RESET - timedelta(seconds=120)
+ assert timedelta(0) <= after_reset < timedelta(seconds=600)
+
+
+async def test_a_resume_fires_a_minute_after_a_send_on_the_same_chat(limits):
+ _, status, _ = limits
+ status.return_value = _status(daily_used=3_000_000, weekly_used=0)
+ send = await next_reset_run_at("user-1", SESSION, "send")
+ resume = await next_reset_run_at("user-1", SESSION, "resume")
+ assert resume - send == timedelta(seconds=60)
+
+
+async def test_nothing_exhausted_means_nothing_to_wait_for(limits):
+ _, status, _ = limits
+ status.return_value = _status(daily_used=2_999_999, weekly_used=0)
+ with pytest.raises(NoResetAhead) as raised:
+ await next_reset_run_at("user-1", SESSION, "resume")
+ assert raised.value.reason == "not_out_of_budget"
+
+
+async def test_an_uncapped_window_is_never_exhausted(limits):
+ _, status, _ = limits
+ status.return_value = _status(daily_used=9_000_000, weekly_used=0, daily_limit=-1)
+ with pytest.raises(NoResetAhead) as raised:
+ await next_reset_run_at("user-1", SESSION, "send")
+ assert raised.value.reason == "not_out_of_budget"
+
+
+async def test_a_spent_trial_allowance_never_resets(limits, trial):
+ trial.cost_microdollars = 10_000_000
+ with pytest.raises(NoResetAhead) as raised:
+ await next_reset_run_at("user-1", SESSION, "resume")
+ assert raised.value.reason == "no_reset"
+ limits[1].assert_not_awaited()
+
+
+ "daily_used, weekly_used, reset",
+ [(3_000_000, 2_000_000, DAILY_RESET), (1_000_000, 6_000_000, WEEKLY_RESET)],
+ ids=["daily-window", "weekly-window"],
+)
+async def test_a_trials_own_window_resets_like_any_other(
+ limits, trial, daily_used, weekly_used, reset
+):
+ # The lifetime allowance has room; only the offer's tighter window is out.
+ limits[1].return_value = _status(daily_used=daily_used, weekly_used=weekly_used)
+ run_at = await next_reset_run_at("user-1", SESSION, "send")
+ assert run_at == reset + timedelta(seconds=120) + JITTER
+
+
+ "allowance",
+ [(0, 0, SubscriptionTier.NO_TIER), (0, 6_000_000, SubscriptionTier.PRO)],
+ ids=["lapsed-trial-or-plan", "a-window-that-allows-nothing"],
+)
+async def test_a_window_with_no_allowance_never_resets(limits, allowance):
+ global_limits, status, _ = limits
+ global_limits.return_value = allowance
+ with pytest.raises(NoResetAhead) as raised:
+ await next_reset_run_at("user-1", SESSION, "send")
+ assert raised.value.reason == "no_reset"
+ status.assert_not_awaited()
+
+
+ "row",
+ [
+ _row("user", "never mind, do this instead", SET_AT + timedelta(minutes=5)),
+ _row("assistant", KEEP_GOING_NOTE, SET_AT + timedelta(minutes=5)),
+ _row("assistant", RESUMED_AT_RESET_NOTE, SET_AT + timedelta(hours=9)),
+ ],
+)
+async def test_a_resume_stands_down_when_the_chat_was_picked_up(
+ limits, chat_reads, row
+):
+ verdict = await fire_verdict("resume", _chat(row), SET_AT)
+ assert verdict.action == "drop"
+ limits[2].assert_not_awaited()
+
+
+async def test_a_newer_user_message_further_back_still_counts(limits, chat_reads):
+ store, _ = chat_reads
+ store.get_latest_user_message_in_session.return_value = _row(
+ "user", "carry on", SET_AT + timedelta(minutes=1)
+ )
+ verdict = await fire_verdict("resume", _chat(), SET_AT)
+ assert verdict.action == "drop"
+
+
+async def test_older_rows_and_unstamped_rows_dont_supersede_a_resume(
+ limits, chat_reads
+):
+ chat = _chat(
+ _row("user", "build the report", SET_AT - timedelta(minutes=30)),
+ _row("assistant", KEEP_GOING_NOTE, SET_AT - timedelta(minutes=20)),
+ _row("assistant", "Here is where I got to.", SET_AT + timedelta(minutes=1)),
+ _row("user", "no timestamp", None),
+ )
+ verdict = await fire_verdict("resume", chat, SET_AT)
+ assert verdict == FireVerdict(action="fire")
+
+
+async def test_a_send_fires_whatever_the_user_said_since(limits, chat_reads):
+ chat = _chat(_row("user", "later message", SET_AT + timedelta(minutes=5)))
+ verdict = await fire_verdict("send", chat, SET_AT)
+ assert verdict.action == "fire"
+
+
+async def test_a_busy_chat_tries_again(limits, chat_reads):
+ _, busy = chat_reads
+ busy.return_value = True
+ verdict = await fire_verdict("send", _chat(), SET_AT)
+ assert verdict.action == "retry"
+ busy.side_effect = StreamRegistryUnavailable()
+ verdict = await fire_verdict("send", _chat(), SET_AT)
+ assert verdict.action == "retry"
+
+
+async def test_a_chat_off_the_platform_route_skips_the_budget(limits, chat_reads):
+ verdict = await fire_verdict("resume", _chat(provider="codex"), SET_AT)
+ assert verdict.action == "fire"
+ limits[2].assert_not_awaited()
+
+
+async def test_a_resume_on_a_chat_moved_to_microsoft_365_is_dropped(limits, chat_reads):
+ """It would hand Microsoft the resume note as the user's message."""
+ chat = _chat(provider="microsoft_365_copilot")
+ verdict = await fire_verdict("resume", chat, SET_AT)
+ assert verdict.action == "drop"
+ assert verdict.why == "the chat now runs on Microsoft 365 Copilot"
+
+
+async def test_a_send_on_a_microsoft_365_chat_still_goes(limits, chat_reads):
+ chat = _chat(provider="microsoft_365_copilot")
+ verdict = await fire_verdict("send", chat, SET_AT)
+ assert verdict.action == "fire"
+
+
+async def test_still_out_of_budget_waits_for_the_next_reset(limits, chat_reads):
+ _, status, gate = limits
+ gate.side_effect = RateLimitExceeded("daily", DAILY_RESET)
+ status.return_value = _status(daily_used=3_000_000, weekly_used=6_000_000)
+ verdict = await fire_verdict("resume", _chat(), SET_AT)
+ assert verdict.action == "wait"
+ assert verdict.run_at == WEEKLY_RESET + timedelta(seconds=180) + JITTER
+
+
+async def test_an_unreadable_budget_tries_again(limits, chat_reads):
+ _, _, gate = limits
+ gate.side_effect = RateLimitUnavailable()
+ verdict = await fire_verdict("send", _chat(), SET_AT)
+ assert verdict.action == "retry"
+
+
+async def test_an_account_with_no_allowance_drops_the_turn_when_it_fires(
+ limits, chat_reads
+):
+ # An ended trial reads as NO_TIER with no allowance while its usage
+ # state still says TRIAL: the tier guard would report that as
+ # unavailable, and the turn would be retried for an hour.
+ global_limits, _, gate = limits
+ global_limits.return_value = (0, 0, SubscriptionTier.NO_TIER)
+ gate.side_effect = RateLimitUnavailable("Subscription changed")
+ verdict = await fire_verdict("send", _chat(), SET_AT)
+ assert verdict.action == "drop"
+ gate.assert_not_awaited()
+
+
+async def test_a_spent_trial_drops_the_turn_when_it_fires(limits, chat_reads, trial):
+ limits[2].side_effect = RateLimitExceeded("trial", SET_AT)
+ trial.cost_microdollars = 10_000_000
+ verdict = await fire_verdict("send", _chat(), SET_AT)
+ assert verdict.action == "drop"
+
+
+async def test_a_trials_spent_day_waits_for_tomorrow(limits, chat_reads, trial):
+ limits[2].side_effect = RateLimitExceeded("daily", DAILY_RESET)
+ limits[1].return_value = _status(daily_used=3_000_000, weekly_used=2_000_000)
+ verdict = await fire_verdict("send", _chat(), SET_AT)
+ assert verdict.action == "wait"
+ assert verdict.run_at == DAILY_RESET + timedelta(seconds=120) + JITTER
+
+
+async def test_cancelling_a_resume_runs_in_the_background_and_spares_a_send():
+ client = MagicMock(cancel_at_reset_turn=AsyncMock(return_value=True))
+ with patch(f"{_MOD}.get_scheduler_client", return_value=client) as factory:
+ await cancel_resume_at_reset_in_background(SESSION, "user-1")
+ client.cancel_at_reset_turn.assert_awaited_once_with(
+ user_id="user-1", session_id=SESSION, kind="resume"
+ )
+ # A cancel lost on a connection the scheduler had just closed is repeated.
+ factory.assert_called_once_with(should_retry=True)
+
+
+async def test_a_failing_cancel_is_swallowed():
+ client = MagicMock(cancel_at_reset_turn=AsyncMock(side_effect=RuntimeError("down")))
+ with patch(f"{_MOD}.get_scheduler_client", return_value=client):
+ task = cancel_resume_at_reset_in_background(SESSION, "user-1")
+ await task
+ assert task.exception() is None
diff --git a/autogpt_platform/backend/backend/copilot/tools/manage_schedules.py b/autogpt_platform/backend/backend/copilot/tools/manage_schedules.py
index c8daa1d89c..548e86d79d 100644
--- a/autogpt_platform/backend/backend/copilot/tools/manage_schedules.py
+++ b/autogpt_platform/backend/backend/copilot/tools/manage_schedules.py
@@ -26,7 +26,10 @@ def _is_in_session_scope(
job: GraphExecutionJobInfo | CopilotTurnJobInfo, session: ChatSession
) -> bool:
"""Personal AutoPilot manages every schedule on the account; an expert
- only its own."""
+ only its own. Neither sees a chat's resume or send at the budget reset:
+ the user set those from the chat, and cancels them there."""
+ if isinstance(job, CopilotTurnJobInfo) and job.purpose != "followup":
+ return False
return session.expert_id is None or job.expert_id == session.expert_id
diff --git a/autogpt_platform/backend/backend/copilot/tools/manage_schedules_test.py b/autogpt_platform/backend/backend/copilot/tools/manage_schedules_test.py
index 15736f78f0..e11606b085 100644
--- a/autogpt_platform/backend/backend/copilot/tools/manage_schedules_test.py
+++ b/autogpt_platform/backend/backend/copilot/tools/manage_schedules_test.py
@@ -452,6 +452,53 @@ async def test_expert_cannot_pause_another_experts_schedule():
mock_client.pause_schedule.assert_not_awaited()
+def _at_reset_info(kind: str) -> CopilotTurnJobInfo:
+ """A resume or send the user set on a chat for the budget reset."""
+ info = _make_copilot_info(schedule_id=f"at-reset-{kind}-session-xyz")
+ return info.model_copy(update={"purpose": kind})
+
+
+async def test_list_schedules_leaves_out_turns_at_the_budget_reset(list_tool, session):
+ mock_client = AsyncMock()
+ mock_client.get_execution_schedules = AsyncMock(
+ return_value=[
+ _make_copilot_info(),
+ _at_reset_info("resume"),
+ _at_reset_info("send"),
+ ]
+ )
+
+ with patch(f"{_SCHEDULES_PATH}.get_scheduler_client", return_value=mock_client):
+ result = await list_tool._execute(user_id=_USER, session=session)
+
+ assert isinstance(result, ScheduleListResponse)
+ assert [schedule.schedule_id for schedule in result.schedules] == ["cop-1"]
+
+
+ ("tool", "method"),
+ [
+ (DeleteScheduleTool(), "delete_schedule"),
+ (PauseScheduleTool(), "pause_schedule"),
+ (ResumeScheduleTool(), "resume_schedule"),
+ ],
+)
+async def test_the_agent_cannot_touch_a_turn_at_the_budget_reset(tool, method, session):
+ """Not even personal AutoPilot, which manages every other schedule."""
+ job = _at_reset_info("resume")
+ mock_client = AsyncMock()
+ mock_client.get_execution_schedules = AsyncMock(return_value=[job])
+ setattr(mock_client, method, AsyncMock())
+
+ with patch(f"{_SCHEDULES_PATH}.get_scheduler_client", return_value=mock_client):
+ result = await tool._execute(user_id=_USER, session=session, schedule_id=job.id)
+
+ assert isinstance(result, ErrorResponse) and result.error == "schedule_not_found"
+ getattr(mock_client, method).assert_not_awaited()
+
+
@pytest.mark.asyncio
async def test_list_schedules_marks_paused_entries(list_tool, session):
paused = _make_graph_info(schedule_id="paused-job")
diff --git a/autogpt_platform/backend/backend/copilot/tools/session_context.py b/autogpt_platform/backend/backend/copilot/tools/session_context.py
index 810d1f29a8..02f0fcdf1f 100644
--- a/autogpt_platform/backend/backend/copilot/tools/session_context.py
+++ b/autogpt_platform/backend/backend/copilot/tools/session_context.py
@@ -129,7 +129,13 @@ async def build_session_context(session_id: str, user_id: str) -> str:
# The endpoint already narrows by ``kind`` server-side; the isinstance
# filter is a belt-and-braces guard against a legacy untyped row that
# might slip through (matches ``schedules.routes.list_copilot_turn_schedules``).
- jobs = [j for j in raw_jobs if isinstance(j, CopilotTurnJobInfo)]
+ # A resume or send at the budget reset is the user's, set and cancelled
+ # from the chat, so the model isn't told about it.
+ jobs = [
+ j
+ for j in raw_jobs
+ if isinstance(j, CopilotTurnJobInfo) and j.purpose == "followup"
+ ]
if not jobs:
# Zero-follow-up sessions are the common case — collapse to one
diff --git a/autogpt_platform/backend/backend/copilot/tools/session_context_test.py b/autogpt_platform/backend/backend/copilot/tools/session_context_test.py
index 1e4b248043..f9375c8174 100644
--- a/autogpt_platform/backend/backend/copilot/tools/session_context_test.py
+++ b/autogpt_platform/backend/backend/copilot/tools/session_context_test.py
@@ -277,6 +277,29 @@ async def test_graph_job_is_filtered_out():
assert "some agent" not in ctx
+async def test_turns_at_the_budget_reset_are_not_followups():
+ """A resume or send the user set for the budget reset is theirs to cancel
+ from the chat, so the model isn't told about it."""
+ at_reset = [
+ _one_shot(
+ schedule_id=f"at-reset-{kind}-{_SESSION}",
+ name=f"{kind} when the budget resets",
+ fires_at="2026-05-23T00:07:00+00:00",
+ ).model_copy(update={"purpose": kind})
+ for kind in ("resume", "send")
+ ]
+ followup = _one_shot(
+ schedule_id="cop-1", name="Check CI", fires_at="2026-05-22T13:50:00+00:00"
+ )
+ with _patch_scheduler([*at_reset, followup]):
+ ctx = await build_session_context(_SESSION, _USER)
+
+ assert "pending_followups: 1" in ctx
+ assert '"Check CI"' in ctx
+ assert "budget resets" not in ctx
+
+
# ---------------------------------------------------------------------------
# sanitize_user_supplied_context — anti-spoofing contract
# ---------------------------------------------------------------------------
diff --git a/autogpt_platform/backend/backend/executor/scheduler.py b/autogpt_platform/backend/backend/executor/scheduler.py
index 2ad503e364..f7b399a370 100644
--- a/autogpt_platform/backend/backend/executor/scheduler.py
+++ b/autogpt_platform/backend/backend/executor/scheduler.py
@@ -32,8 +32,17 @@ from sqlalchemy import MetaData, create_engine
from backend.api.features.experts.models import ExpertRoutine
from backend.copilot.active_turns import ConcurrentTurnLimitError
+from backend.copilot.at_reset import (
+ AT_RESET_JOB_NAMES,
+ AtResetKind,
+ at_reset_schedule_id,
+ fire_verdict,
+ load_at_reset_chat,
+)
from backend.copilot.config import CopilotLlmAuthProvider
+from backend.copilot.continuation import RESUMED_AT_RESET_NOTE
from backend.copilot.credential_selection import CredentialPins
+from backend.copilot.db import PaginatedMessages
from backend.copilot.dream.scheduling import (
COMMUNITY_REBUILD_REGISTRATION_PREFIX,
NIGHTLY_BATCH_REGISTRATION_PREFIX,
@@ -42,9 +51,10 @@ from backend.copilot.dream.scheduling import (
)
from backend.copilot.executor.utils import ScheduledTurnOrigin, schedule_turn
from backend.copilot.graphiti.communities import rebuild_communities_for_user
-from backend.copilot.model import create_chat_session, get_chat_session
+from backend.copilot.model import ChatSessionInfo, create_chat_session, get_chat_session
from backend.copilot.optimize_blocks import optimize_block_descriptions
from backend.copilot.permissions import CopilotPermissions, routine_disabled_tools
+from backend.copilot.session_permissions import resolve_session_permissions
from backend.copilot.transports import resolve_default_chat_route
from backend.copilot.turn_queue import TurnRefusal, turn_refusal
from backend.data.db_accessors import experts_db
@@ -399,6 +409,9 @@ def _routine_turn_permissions(routine: ExpertRoutine | None) -> CopilotPermissio
async def _execute_copilot_turn(**kwargs):
expert_scope_was_persisted = "expert_id" in kwargs
args = CopilotTurnJobArgs(**kwargs)
+ if args.purpose != "followup":
+ await _execute_at_reset_turn(args, args.purpose)
+ return
routine = await _routine_for_turn(args)
if routine is not None and not routine.enabled:
# Deleting a routine's jobs is best effort — the scheduler can refuse,
@@ -934,6 +947,216 @@ async def _drop_job_from_routine(args: "CopilotTurnJobArgs") -> None:
)
+# When a chat's resume or send at the reset was cancelled, by job id. A job
+# cancelled while it runs is already out of the job store, so the cancel
+# finds nothing to remove; before firing or putting itself back the job looks
+# here. In process, like the jobs themselves: one scheduler runs them.
+_at_reset_cancels: dict[str, datetime] = {}
+_at_reset_cancels_lock = threading.Lock()
+_AT_RESET_CANCEL_MEMORY = timedelta(hours=2)
+
+# An at-reset turn that can't fire yet (the chat is busy, the budget or the
+# expert can't be read, the queue turned it away) tries again five minutes
+# later, this many times: an hour, so a long turn already running in the chat
+# can finish. One still out of budget moves to the next reset instead, which
+# starts the count again.
+_MAX_AT_RESET_RETRIES = 12
+
+
+async def _execute_at_reset_turn(args: "CopilotTurnJobArgs", kind: AtResetKind) -> None:
+ """Fire a "Resume at reset" or "Send at reset" the user set on a chat.
+
+ ``fire_verdict`` decides whether it still should; this carries that out.
+ Whenever it can't fire yet it goes back under its own id, so the chat
+ keeps at most one of each kind.
+ """
+ label = _session_id_label(args)
+ try:
+ chat = await _at_reset_chat(args)
+ if chat is None:
+ return
+ verdict = await fire_verdict(kind, chat, args.created_at)
+ if _cancelled_after_set(args):
+ logger.info(
+ f"{kind}-at-reset turn for session {label} was cancelled "
+ "while it ran; dropping it"
+ )
+ return
+ if verdict.action == "fire":
+ await _dispatch_at_reset_turn(args, kind, chat.session)
+ elif verdict.action == "wait" and verdict.run_at is not None:
+ logger.info(
+ f"{kind}-at-reset turn for session {label} is still out of "
+ f"budget; moving it to {verdict.run_at.isoformat()}"
+ )
+ await _readd_at_reset_turn(args, run_at=verdict.run_at, retries=0)
+ elif verdict.action == "retry":
+ await _rearm_at_reset_turn(args, reason=verdict.why)
+ else:
+ logger.info(
+ f"Dropping {kind}-at-reset turn for session {label}: {verdict.why}"
+ )
+ except Exception as e:
+ logger.warning(
+ f"{kind}-at-reset turn for session {label} could not be dispatched",
+ exc_info=True,
+ )
+ await _rearm_at_reset_turn(args, reason=f"{type(e).__name__}: {e}")
+
+
+async def _at_reset_chat(args: "CopilotTurnJobArgs") -> PaginatedMessages | None:
+ """The chat an at-reset turn fires into, or None once that's been handled.
+
+ A chat that is gone, someone else's, or out of the job's scope removes the
+ job, as for a follow-up; an expert that can't be looked up tries again.
+ """
+ chat = (
+ await load_at_reset_chat(args.session_id, args.user_id)
+ if args.session_id
+ else None
+ )
+ if chat is None or chat.session.expert_id != args.expert_id:
+ logger.info(
+ f"At-reset turn {args.schedule_id} skipped — session "
+ f"{_session_id_label(args)} is gone or out of scope; removing it"
+ )
+ await _self_delete_copilot_turn_schedule(args)
+ return None
+ if args.expert_id is None:
+ return chat
+ status = await _expert_scope_status(args.user_id, args.expert_id)
+ if status == "unavailable":
+ await _rearm_at_reset_turn(args, reason="the expert could not be looked up")
+ return None
+ if status != "active":
+ await _skip_inactive_expert_scope(args, status)
+ return None
+ return chat
+
+
+async def _dispatch_at_reset_turn(
+ args: "CopilotTurnJobArgs", kind: AtResetKind, session: ChatSessionInfo
+) -> None:
+ """Start the turn the way a follow-up into an existing chat starts.
+
+ A resume is a system note: grey in the chat, an instruction to the model.
+ A send is the user's own words, written before the reset, so unlike a
+ model-authored follow-up it lands as their message.
+ """
+ is_send = kind == "send"
+ await schedule_turn(
+ session_id=session.session_id,
+ user_id=args.user_id,
+ turn_id=str(uuid.uuid4()),
+ message=args.message if is_send else RESUMED_AT_RESET_NOTE,
+ is_user_message=is_send,
+ tool_call_id="scheduled_followup",
+ tool_name="schedule_followup",
+ unattended=True,
+ credential_pins=args.credential_pins,
+ organization_id=args.organization_id,
+ team_id=args.team_id,
+ llm_auth_provider=session.metadata.llm_auth_provider,
+ llm_credential_id=session.metadata.llm_credential_id,
+ # It carries on the user's own chat, so under the chat's own
+ # limits, as a message sent there now would be (a builder chat's).
+ permissions=resolve_session_permissions(session),
+ scheduled=ScheduledTurnOrigin(schedule_id=args.schedule_id),
+ )
+ logger.info(
+ f"Dispatched {kind}-at-reset turn for session {session.session_id[:12]}"
+ )
+
+
+async def _rearm_at_reset_turn(args: "CopilotTurnJobArgs", *, reason: str) -> None:
+ """Try an at-reset turn again in five minutes, a bounded number of times."""
+ label = _session_id_label(args)
+ if args.cap_retry_count >= _MAX_AT_RESET_RETRIES:
+ logger.error(
+ f"Dropping {args.purpose}-at-reset turn for session {label} after "
+ f"{_MAX_AT_RESET_RETRIES} retries ({reason})"
+ )
+ return
+ run_at = datetime.now(tz=timezone.utc) + timedelta(
+ seconds=_CONCURRENCY_RETRY_DELAY_SECONDS
+ )
+ logger.info(
+ f"{args.purpose}-at-reset turn for session {label} can't fire yet "
+ f"({reason}); retry {args.cap_retry_count + 1}/{_MAX_AT_RESET_RETRIES} "
+ f"at {run_at.isoformat()}"
+ )
+ await _readd_at_reset_turn(args, run_at=run_at, retries=args.cap_retry_count + 1)
+
+
+async def _readd_at_reset_turn(
+ args: "CopilotTurnJobArgs", *, run_at: datetime, retries: int
+) -> None:
+ """Put an at-reset turn back under its own id, keeping when it was set."""
+ if _cancelled_after_set(args):
+ logger.info(
+ f"Not putting back the {args.purpose}-at-reset turn for session "
+ f"{_session_id_label(args)}: it was cancelled while it ran"
+ )
+ return
+ try:
+ # Under its own id this is safe to repeat, and a put-back lost on a
+ # connection the scheduler had just closed would drop the user's turn.
+ await get_scheduler_client(should_retry=True).add_copilot_turn_schedule(
+ user_id=args.user_id,
+ session_id=args.session_id,
+ message=args.message,
+ run_at=run_at,
+ user_timezone=args.user_timezone,
+ cap_retry_count=retries,
+ organization_id=args.organization_id,
+ team_id=args.team_id,
+ expert_id=args.expert_id,
+ credential_pins=args.credential_pins,
+ purpose=args.purpose,
+ schedule_id=args.schedule_id,
+ created_at=args.created_at,
+ )
+ except Exception:
+ logger.warning(
+ f"Failed to put back the {args.purpose}-at-reset turn for session "
+ f"{_session_id_label(args)}",
+ exc_info=True,
+ )
+
+
+def _put_back_onto(
+ newer: "CopilotTurnJobInfo", message: str
+) -> tuple[str, datetime | None]:
+ """A send put back after the user wrote another while it ran: both go,
+ this one first, as the newer job the user set."""
+ return f"{message}\n\n{newer.message}", newer.created_at
+
+
+def _remember_at_reset_cancel(job_id: str) -> None:
+ now = datetime.now(tz=timezone.utc)
+ with _at_reset_cancels_lock:
+ stale = [
+ key
+ for key, at in _at_reset_cancels.items()
+ if now - at > _AT_RESET_CANCEL_MEMORY
+ ]
+ for key in stale:
+ del _at_reset_cancels[key]
+ _at_reset_cancels[job_id] = now
+
+
+def _cancelled_after_set(args: "CopilotTurnJobArgs") -> bool:
+ """Whether the user cancelled this job after it was set: setting it
+ again after a cancel makes a new job, set later than the cancel."""
+ if args.schedule_id is None:
+ return False
+ with _at_reset_cancels_lock:
+ cancelled_at = _at_reset_cancels.get(args.schedule_id)
+ if cancelled_at is None:
+ return False
+ return args.created_at is None or args.created_at <= cancelled_at
+
+
async def _handle_graph_validation_error(args: "GraphExecutionJobArgs") -> None:
logger.warning(
f"Scheduled Graph {args.graph_id} failed validation. Unscheduling graph"
@@ -1885,6 +2108,14 @@ class CopilotTurnJobArgs(BaseModel):
# the follow-up was made, which every fire runs on (SECRT-2804). A routine
# keeps its pins on its row instead. Empty on rows persisted before pins.
credential_pins: CredentialPins = Field(default_factory=dict)
+ # Every job is a follow-up but the two a user sets on an out-of-budget
+ # chat: "Resume at reset" and "Send at reset" (``copilot/at_reset.py``).
+ # Those keep one id per chat, stay out of the agent's schedule tools, and
+ # check the budget when they fire. Rows from before this load as followups.
+ purpose: Literal["followup", "resume", "send"] = "followup"
+ # When the job was first set, kept when it is put back for a retry; a
+ # resume stands down if the user carried the chat on after it.
+ created_at: datetime | None = None
def _timezone_from_job(job_obj: JobObj) -> str:
@@ -2618,6 +2849,9 @@ class Scheduler(AppService):
routine_id: str | None = None,
credential_pins: CredentialPins | None = None,
routine_schedule_id: str | None = None,
+ purpose: Literal["followup", "resume", "send"] = "followup",
+ schedule_id: str | None = None,
+ created_at: datetime | None = None,
) -> CopilotTurnJobInfo:
"""Schedule a copilot turn at a future time.
@@ -2628,6 +2862,14 @@ class Scheduler(AppService):
The ``*_retry_count`` arguments are set internally to bound their
respective transient retry paths; normal callers should leave them at 0.
+
+ *purpose* ``"resume"`` / ``"send"`` marks a chat's turn at the budget
+ reset; it comes with the chat's own *schedule_id*
+ (``at_reset_schedule_id``), so adding it again replaces the earlier
+ job. *created_at* is passed only when putting a job back, to keep when
+ it was first set; a job put back after the user set a newer one
+ under the same id while it ran doesn't overwrite that one (see
+ ``_put_back_onto``).
"""
# Mirror add_graph_execution_schedule: validate the expert scope at
# creation (active, owned, PRIVATE) and pin the schedule to the
@@ -2638,10 +2880,18 @@ class Scheduler(AppService):
experts_db().resolve_private_expert_tenancy(user_id, expert_id)
)
+ if purpose != "followup" and schedule_id and created_at is not None:
+ newer = self._newer_at_reset_job(schedule_id, created_at)
+ if newer is not None and purpose == "resume":
+ # Set again while this one ran: the newer resume stands.
+ return newer
+ if newer is not None:
+ message, created_at = _put_back_onto(newer, message)
+
user_timezone = _resolve_timezone(user_timezone, user_id)
trigger = _build_trigger(cron=cron, run_at=run_at, user_timezone=user_timezone)
job_args = CopilotTurnJobArgs(
- schedule_id=str(uuid.uuid4()),
+ schedule_id=schedule_id or str(uuid.uuid4()),
user_id=user_id,
session_id=session_id,
message=message,
@@ -2658,12 +2908,15 @@ class Scheduler(AppService):
routine_id=routine_id,
credential_pins=credential_pins or {},
routine_schedule_id=routine_schedule_id,
+ purpose=purpose,
+ created_at=created_at or datetime.now(timezone.utc),
)
- default_name = (
- f"copilot turn (session {session_id[:8]})"
- if session_id
- else "copilot turn (new chat)"
- )
+ if purpose != "followup":
+ default_name = AT_RESET_JOB_NAMES[purpose]
+ elif session_id:
+ default_name = f"copilot turn (session {session_id[:8]})"
+ else:
+ default_name = "copilot turn (new chat)"
job = self._persist_schedule(
dispatch_func=execute_copilot_turn,
job_args=job_args,
@@ -2675,11 +2928,50 @@ class Scheduler(AppService):
f"Added copilot-turn job {job.id} ({trigger.__class__.__name__}) "
f"for session {session_label} in timezone {user_timezone}"
)
- _record_copilot_turn_schedule_created(
- job_args, job, title=name or message[:80] or "Follow-up"
- )
+ # An at-reset turn is put back under its own id every time it can't
+ # fire yet, so the feed and the schedule metrics would count it again
+ # each time; only follow-ups are recorded.
+ if purpose == "followup":
+ _record_copilot_turn_schedule_created(
+ job_args, job, title=name or message[:80] or "Follow-up"
+ )
return CopilotTurnJobInfo.from_db(job_args, job)
+ def _newer_at_reset_job(
+ self, schedule_id: str, than: datetime
+ ) -> CopilotTurnJobInfo | None:
+ """The job under *schedule_id*, if it was set after *than*."""
+ job = self.scheduler.get_job(schedule_id, jobstore=Jobstores.EXECUTION.value)
+ info = _job_to_info(job) if job else None
+ if not isinstance(info, CopilotTurnJobInfo) or info.created_at is None:
+ return None
+ return info if info.created_at > than else None
+
+ @expose
+ def cancel_at_reset_turn(
+ self, user_id: str, session_id: str, kind: AtResetKind
+ ) -> bool:
+ """Remove a chat's resume or send at the budget reset, if it has one.
+
+ Idempotent and quiet when there is nothing to remove: the chat route
+ calls this whenever the user sends a message, so a missing job is the
+ usual case rather than an error. Returns whether a job was removed.
+ """
+ job_id = at_reset_schedule_id(kind, session_id)
+ job = self.scheduler.get_job(job_id, jobstore=Jobstores.EXECUTION.value)
+ info = _job_to_info(job) if job else None
+ if job is not None and (info is None or info.user_id != user_id):
+ return False
+ # Remembered even with no job to remove: it may be running right
+ # now, about to fire or to put itself back.
+ _remember_at_reset_cancel(job_id)
+ if job is None:
+ return False
+ job.remove()
+ self._invalidate_jobs_cache()
+ logger.info(f"Removed {kind}-at-reset job for session {session_id[:12]}")
+ return True
+
@expose
def delete_graph_execution_schedule(
self, schedule_id: str, user_id: str
@@ -3840,6 +4132,7 @@ class SchedulerClient(AppServiceClient):
add_execution_schedule = endpoint_to_async(Scheduler.add_graph_execution_schedule)
add_copilot_turn_schedule = endpoint_to_async(Scheduler.add_copilot_turn_schedule)
+ cancel_at_reset_turn = endpoint_to_async(Scheduler.cancel_at_reset_turn)
delete_schedule = endpoint_to_async(Scheduler.delete_graph_execution_schedule)
update_execution_schedule = endpoint_to_async(
Scheduler.update_graph_execution_schedule
diff --git a/autogpt_platform/backend/backend/executor/scheduler_at_reset_test.py b/autogpt_platform/backend/backend/executor/scheduler_at_reset_test.py
new file mode 100644
index 0000000000..5d59f66aef
--- /dev/null
+++ b/autogpt_platform/backend/backend/executor/scheduler_at_reset_test.py
@@ -0,0 +1,417 @@
+"""The scheduler's side of resume and send at the budget reset: the job's
+id and shape, and what a due one does (``copilot/at_reset.py`` decides)."""
+
+from datetime import UTC, datetime, timedelta
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from apscheduler.jobstores.memory import MemoryJobStore
+from apscheduler.schedulers.background import BackgroundScheduler
+
+from backend.copilot.active_turns import ConcurrentTurnLimitError
+from backend.copilot.at_reset import at_reset_schedule_id, reset_jitter_seconds
+from backend.copilot.continuation import RESUMED_AT_RESET_NOTE
+from backend.copilot.db import PaginatedMessages
+from backend.copilot.executor.utils import ScheduledTurnOrigin
+from backend.copilot.model import ChatMessage, ChatSessionInfo, ChatSessionMetadata
+from backend.copilot.rate_limit import (
+ CoPilotUsageStatus,
+ RateLimitExceeded,
+ SubscriptionTier,
+ UsageWindow,
+)
+from backend.executor.scheduler import (
+ _MAX_AT_RESET_RETRIES,
+ CopilotTurnJobArgs,
+ Jobstores,
+ Scheduler,
+ _execute_copilot_turn,
+)
+
+_SCHEDULER = "backend.executor.scheduler"
+_AT_RESET = "backend.copilot.at_reset"
+USER = "user-1"
+SESSION = "0d3f7a52-61c4-4f0e-a8b2-9c1e5d7f3a64"
+SET_AT = datetime(2026, 10, 14, 15, 0, tzinfo=UTC)
+DAILY_RESET = datetime(2026, 10, 15, tzinfo=UTC)
+WEEKLY_RESET = datetime(2026, 10, 19, tzinfo=UTC)
+
+
+def _args(kind: str = "resume", **overrides) -> CopilotTurnJobArgs:
+ payload = {
+ "schedule_id": f"at-reset-{kind}-{SESSION}",
+ "user_id": USER,
+ "session_id": SESSION,
+ "message": "also check the invoices" if kind == "send" else "Resume.",
+ "run_at": SET_AT + timedelta(hours=9),
+ "user_timezone": "UTC",
+ "purpose": kind,
+ "created_at": SET_AT,
+ **overrides,
+ }
+ return CopilotTurnJobArgs.model_validate(payload)
+
+
+def _chat(*rows: ChatMessage, expert_id: str | None = None) -> PaginatedMessages:
+ session = ChatSessionInfo(
+ session_id=SESSION,
+ user_id=USER,
+ usage=[],
+ started_at=SET_AT - timedelta(hours=1),
+ updated_at=SET_AT,
+ expert_id=expert_id,
+ metadata=ChatSessionMetadata(llm_auth_provider="platform"),
+ )
+ return PaginatedMessages(
+ messages=list(rows), has_more=False, oldest_sequence=0, session=session
+ )
+
+
+async def _fire(args: CopilotTurnJobArgs) -> None:
+ await _execute_copilot_turn(**args.model_dump(mode="json"))
+
+
[email protected](autouse=True)
+def no_cancels():
+ """Cancels are remembered in process: no test sees another's."""
+ with patch.dict(f"{_SCHEDULER}._at_reset_cancels", clear=True):
+ yield
+
+
+def fire_env():
+ """A due at-reset turn on an idle, platform-route chat with budget left."""
+ store = MagicMock(get_latest_user_message_in_session=AsyncMock(return_value=None))
+ client = MagicMock(
+ add_copilot_turn_schedule=AsyncMock(), delete_schedule=AsyncMock()
+ )
+ limits = (3_000_000, 6_000_000, SubscriptionTier.PRO)
+ with (
+ patch(
+ f"{_SCHEDULER}.load_at_reset_chat", AsyncMock(return_value=_chat())
+ ) as load,
+ patch(f"{_SCHEDULER}.schedule_turn", AsyncMock()) as dispatch,
+ patch(f"{_SCHEDULER}.get_scheduler_client", return_value=client) as factory,
+ patch(f"{_AT_RESET}.chat_db", return_value=store),
+ patch(f"{_AT_RESET}.is_turn_in_flight", AsyncMock(return_value=False)) as busy,
+ patch(f"{_AT_RESET}.get_global_rate_limits", AsyncMock(return_value=limits)),
+ patch(f"{_AT_RESET}.check_rate_limit", AsyncMock()) as gate,
+ patch(f"{_AT_RESET}.get_usage_status", AsyncMock()) as usage,
+ ):
+ yield SimpleNamespace(
+ load=load,
+ dispatch=dispatch,
+ client=client,
+ factory=factory,
+ busy=busy,
+ gate=gate,
+ usage=usage,
+ )
+
+
+def live_scheduler():
+ """The service over an in-memory jobstore, paused so nothing fires."""
+ service = Scheduler(register_system_tasks=False)
+ service.scheduler = BackgroundScheduler(
+ jobstores={Jobstores.EXECUTION.value: MemoryJobStore()}
+ )
+ service.scheduler.start(paused=True)
+ yield service
+ service.scheduler.shutdown(wait=False)
+
+
+def test_a_row_from_before_the_field_loads_as_a_followup():
+ legacy = CopilotTurnJobArgs.model_validate(
+ {"user_id": USER, "session_id": SESSION, "message": "check CI"}
+ )
+ assert legacy.purpose == "followup"
+ assert legacy.created_at is None
+ roundtrip = CopilotTurnJobArgs.model_validate(_args("send").model_dump(mode="json"))
+ assert roundtrip.purpose == "send"
+ assert roundtrip.created_at == SET_AT
+
+
+def test_an_at_reset_job_is_named_for_what_it_does_and_not_recorded():
+ service = Scheduler(register_system_tasks=False)
+ job = MagicMock(id="job", next_run_time=None)
+ job.name = "job"
+ with (
+ patch.object(service, "_persist_schedule", return_value=job) as persist,
+ patch("backend.executor.schedule_events.record_schedule_created") as record,
+ ):
+ service.add_copilot_turn_schedule(
+ user_id=USER,
+ session_id=SESSION,
+ message="Resume.",
+ run_at=SET_AT,
+ user_timezone="UTC",
+ purpose="resume",
+ schedule_id=at_reset_schedule_id("resume", SESSION),
+ )
+ record.assert_not_called()
+ service.add_copilot_turn_schedule(
+ user_id=USER, session_id=SESSION, message="check CI", run_at=SET_AT
+ )
+ record.assert_called_once()
+ at_reset, followup = (c.kwargs for c in persist.call_args_list)
+ assert at_reset["job_args"].schedule_id == f"at-reset-resume-{SESSION}"
+ assert at_reset["job_args"].purpose == "resume"
+ assert at_reset["job_args"].created_at is not None
+ assert at_reset["name"] == "Resume when the budget resets"
+ assert followup["job_args"].purpose == "followup"
+ assert followup["job_args"].schedule_id != at_reset["job_args"].schedule_id
+
+
+def test_setting_it_again_replaces_the_earlier_job(live_scheduler):
+ job_id = at_reset_schedule_id("send", SESSION)
+ first = datetime.now(UTC) + timedelta(hours=3)
+ for run_at, message in ((first, "draft"), (first + timedelta(days=1), "final")):
+ live_scheduler.add_copilot_turn_schedule(
+ user_id=USER,
+ session_id=SESSION,
+ message=message,
+ run_at=run_at,
+ user_timezone="UTC",
+ purpose="send",
+ schedule_id=job_id,
+ )
+ jobs = live_scheduler.scheduler.get_jobs(jobstore=Jobstores.EXECUTION.value)
+ assert [job.id for job in jobs] == [job_id]
+ assert jobs[0].kwargs["message"] == "final"
+ assert jobs[0].next_run_time == first + timedelta(days=1)
+
+
+def test_cancel_removes_only_the_owners_job_and_is_idempotent(live_scheduler):
+ live_scheduler.add_copilot_turn_schedule(
+ user_id=USER,
+ session_id=SESSION,
+ message="Resume.",
+ run_at=datetime.now(UTC) + timedelta(hours=3),
+ user_timezone="UTC",
+ purpose="resume",
+ schedule_id=at_reset_schedule_id("resume", SESSION),
+ )
+ assert (
+ live_scheduler.cancel_at_reset_turn("someone-else", SESSION, "resume") is False
+ )
+ assert live_scheduler.cancel_at_reset_turn(USER, SESSION, "send") is False
+ assert live_scheduler.cancel_at_reset_turn(USER, SESSION, "resume") is True
+ assert live_scheduler.scheduler.get_jobs(jobstore=Jobstores.EXECUTION.value) == []
+ assert live_scheduler.cancel_at_reset_turn(USER, SESSION, "resume") is False
+
+
+async def test_a_resume_the_user_already_picked_up_is_dropped(fire_env):
+ newer = ChatMessage(
+ role="user", content="do this instead", created_at=SET_AT + timedelta(minutes=5)
+ )
+ fire_env.load.return_value = _chat(newer)
+ await _fire(_args("resume"))
+ fire_env.dispatch.assert_not_awaited()
+ fire_env.client.add_copilot_turn_schedule.assert_not_awaited()
+
+
+ "kind, message, as_user",
+ [
+ ("resume", RESUMED_AT_RESET_NOTE, False),
+ ("send", "also check the invoices", True),
+ ],
+ ids=["resume-fires-the-note-not-as-the-user", "send-fires-the-users-words"],
+)
+async def test_it_fires_into_its_chat(fire_env, kind, message, as_user):
+ await _fire(_args(kind))
+ sent = fire_env.dispatch.await_args.kwargs
+ assert (sent["message"], sent["is_user_message"]) == (message, as_user)
+ assert sent["session_id"] == SESSION
+ assert sent["unattended"] is True
+ assert sent["llm_auth_provider"] == "platform"
+ assert sent["scheduled"] == ScheduledTurnOrigin(
+ schedule_id=f"at-reset-{kind}-{SESSION}"
+ )
+
+
+async def test_a_busy_chat_puts_it_back_in_five_minutes_under_its_own_id(fire_env):
+ fire_env.busy.return_value = True
+ before = datetime.now(UTC)
+ await _fire(_args("send", cap_retry_count=2))
+ fire_env.dispatch.assert_not_awaited()
+ readded = fire_env.client.add_copilot_turn_schedule.await_args.kwargs
+ assert readded["schedule_id"] == f"at-reset-send-{SESSION}"
+ assert readded["purpose"] == "send"
+ assert readded["message"] == "also check the invoices"
+ assert readded["created_at"] == SET_AT
+ assert readded["cap_retry_count"] == 3
+ assert timedelta(minutes=5) <= readded["run_at"] - before < timedelta(minutes=6)
+
+
+async def test_a_put_back_lost_in_transit_is_repeated(fire_env):
+ # Dropped, it would lose the user's turn; under its own id it is safe to
+ # repeat on a connection the scheduler had just closed.
+ fire_env.busy.return_value = True
+ await _fire(_args("send"))
+ fire_env.client.add_copilot_turn_schedule.assert_awaited_once()
+ fire_env.factory.assert_called_once_with(should_retry=True)
+
+
+async def test_retries_are_bounded(fire_env):
+ fire_env.busy.return_value = True
+ await _fire(_args("send", cap_retry_count=_MAX_AT_RESET_RETRIES))
+ fire_env.client.add_copilot_turn_schedule.assert_not_awaited()
+
+
+async def test_still_out_of_budget_moves_it_to_the_next_reset(fire_env):
+ fire_env.gate.side_effect = RateLimitExceeded("daily", DAILY_RESET)
+ fire_env.usage.return_value = CoPilotUsageStatus(
+ daily=UsageWindow(used=3_000_000, limit=3_000_000, resets_at=DAILY_RESET),
+ weekly=UsageWindow(used=0, limit=6_000_000, resets_at=WEEKLY_RESET),
+ tier=SubscriptionTier.PRO,
+ )
+ await _fire(_args("resume", cap_retry_count=4))
+ fire_env.dispatch.assert_not_awaited()
+ moved = fire_env.client.add_copilot_turn_schedule.await_args.kwargs
+ jitter = timedelta(seconds=reset_jitter_seconds(SESSION))
+ assert moved["run_at"] == DAILY_RESET + timedelta(seconds=180) + jitter
+ assert moved["schedule_id"] == f"at-reset-resume-{SESSION}"
+ assert moved["cap_retry_count"] == 0
+
+
+async def test_a_refused_dispatch_puts_it_back(fire_env):
+ fire_env.dispatch.side_effect = ConcurrentTurnLimitError("cap")
+ await _fire(_args("send"))
+ readded = fire_env.client.add_copilot_turn_schedule.await_args.kwargs
+ assert readded["purpose"] == "send"
+ assert readded["cap_retry_count"] == 1
+
+
[email protected]("chat", [None, _chat(expert_id="expert-2")])
+async def test_a_gone_or_rescoped_chat_removes_the_job(fire_env, chat):
+ fire_env.load.return_value = chat
+ await _fire(_args("resume"))
+ fire_env.dispatch.assert_not_awaited()
+ fire_env.client.delete_schedule.assert_awaited_once_with(
+ schedule_id=f"at-reset-resume-{SESSION}", user_id=USER
+ )
+
+
+async def test_an_unreadable_expert_puts_it_back_as_itself(fire_env):
+ fire_env.load.return_value = _chat(expert_id="expert-1")
+ with patch(
+ f"{_SCHEDULER}._expert_scope_status", AsyncMock(return_value="unavailable")
+ ):
+ await _fire(_args("resume", expert_id="expert-1"))
+ readded = fire_env.client.add_copilot_turn_schedule.await_args.kwargs
+ assert readded["purpose"] == "resume"
+ assert readded["schedule_id"] == f"at-reset-resume-{SESSION}"
+ assert readded["expert_id"] == "expert-1"
+
+
+async def test_followups_keep_their_own_path(fire_env):
+ followup = CopilotTurnJobArgs(
+ schedule_id="sched-1", user_id=USER, session_id=SESSION, message="check CI"
+ )
+ with patch(
+ f"{_SCHEDULER}.get_chat_session",
+ AsyncMock(return_value=MagicMock(expert_id=None)),
+ ):
+ await _fire(followup)
+ fire_env.load.assert_not_awaited()
+ assert fire_env.dispatch.await_args.kwargs["message"] == "check CI"
+
+
+async def test_a_job_cancelled_while_it_runs_neither_fires_nor_comes_back(
+ fire_env, live_scheduler
+):
+ # The cancel finds no job (it is out of the store while it runs) but is
+ # remembered, so the run that was deciding stands down either way.
+ assert live_scheduler.cancel_at_reset_turn(USER, SESSION, "send") is False
+ await _fire(_args("send", created_at=datetime.now(UTC) - timedelta(minutes=1)))
+ fire_env.dispatch.assert_not_awaited()
+
+ fire_env.busy.return_value = True
+ await _fire(_args("send", created_at=datetime.now(UTC) - timedelta(minutes=1)))
+ fire_env.client.add_copilot_turn_schedule.assert_not_awaited()
+
+
+async def test_a_send_set_again_after_a_cancel_still_fires(fire_env, live_scheduler):
+ live_scheduler.cancel_at_reset_turn(USER, SESSION, "send")
+ await _fire(_args("send", created_at=datetime.now(UTC) + timedelta(seconds=1)))
+ fire_env.dispatch.assert_awaited_once()
+
+
+async def test_a_builder_chat_keeps_its_tool_limits(fire_env):
+ chat = _chat()
+ chat.session.metadata = ChatSessionMetadata(
+ llm_auth_provider="platform", builder_graph_id="graph-1"
+ )
+ fire_env.load.return_value = chat
+ await _fire(_args("resume"))
+ permissions = fire_env.dispatch.await_args.kwargs["permissions"]
+ assert permissions is not None and permissions.tools
+
+
+def _job_args(service: Scheduler, kind: str) -> CopilotTurnJobArgs:
+ job = service.scheduler.get_job(
+ at_reset_schedule_id(kind, SESSION), jobstore=Jobstores.EXECUTION.value
+ )
+ return CopilotTurnJobArgs(**job.kwargs)
+
+
+def _add(service: Scheduler, kind: str, message: str, created_at=None) -> None:
+ service.add_copilot_turn_schedule(
+ user_id=USER,
+ session_id=SESSION,
+ message=message,
+ run_at=datetime.now(UTC) + timedelta(hours=3),
+ user_timezone="UTC",
+ purpose=kind,
+ schedule_id=at_reset_schedule_id(kind, SESSION),
+ created_at=created_at,
+ )
+
+
+def test_a_send_put_back_joins_one_written_while_it_ran(live_scheduler):
+ # The user set a new send while the old one was out of the store,
+ # deciding; putting the old one back must not drop what they wrote.
+ _add(live_scheduler, "send", "also the changelog")
+ _add(
+ live_scheduler,
+ "send",
+ "draft the launch post",
+ created_at=datetime.now(UTC) - timedelta(minutes=1),
+ )
+ args = _job_args(live_scheduler, "send")
+ assert args.message == "draft the launch post\n\nalso the changelog"
+
+
+def test_a_resume_put_back_leaves_one_set_while_it_ran(live_scheduler):
+ _add(live_scheduler, "resume", "Resume.")
+ newer = _job_args(live_scheduler, "resume").created_at
+ _add(
+ live_scheduler,
+ "resume",
+ "Resume.",
+ created_at=datetime.now(UTC) - timedelta(minutes=1),
+ )
+ assert _job_args(live_scheduler, "resume").created_at == newer
+
+
+def test_a_put_back_with_nothing_newer_is_itself(live_scheduler):
+ set_at = datetime.now(UTC) - timedelta(minutes=1)
+ _add(live_scheduler, "send", "draft the launch post", created_at=set_at)
+ args = _job_args(live_scheduler, "send")
+ assert (args.message, args.created_at) == ("draft the launch post", set_at)
diff --git a/autogpt_platform/backend/backend/util/clients.py b/autogpt_platform/backend/backend/util/clients.py
index 37e7e6a6d6..6e5aac2b5e 100644
--- a/autogpt_platform/backend/backend/util/clients.py
+++ b/autogpt_platform/backend/backend/util/clients.py
@@ -52,12 +52,16 @@ def get_database_manager_async_client(
@thread_cached
-def get_scheduler_client() -> "SchedulerClient":
- """Get a thread-cached SchedulerClient."""
+def get_scheduler_client(should_retry: bool = False) -> "SchedulerClient":
+ """Get a thread-cached SchedulerClient.
+
+ ``should_retry`` repeats a call that failed in transit, such as on a pooled
+ connection the scheduler had just closed; only for calls safe to repeat.
+ """
from backend.executor.scheduler import SchedulerClient
from backend.util.service import get_service_client
- return get_service_client(SchedulerClient)
+ return get_service_client(SchedulerClient, request_retry=should_retry)
@thread_cached
diff --git a/autogpt_platform/backend/backend/util/clients_test.py b/autogpt_platform/backend/backend/util/clients_test.py
index 96f6a0a349..36c1b0d40a 100644
--- a/autogpt_platform/backend/backend/util/clients_test.py
+++ b/autogpt_platform/backend/backend/util/clients_test.py
@@ -1,12 +1,15 @@
-"""Tests for get_openai_client prefer_openrouter parameter."""
+"""Tests for get_openai_client prefer_openrouter parameter, and for the
+scheduler client repeating a call lost in transit."""
-from unittest.mock import MagicMock, patch
+from unittest.mock import AsyncMock, MagicMock, patch
+import httpx
import pytest
from backend.util.clients import (
_get_local_openai_client,
get_openai_client,
+ get_scheduler_client,
openrouter_helper_cost_provider,
)
@@ -139,3 +142,34 @@ class TestOpenrouterHelperCostProvider:
)
self._patch_transport(monkeypatch, cfg)
assert openrouter_helper_cost_provider() == "open_router"
+
+
+class TestGetSchedulerClientRetry:
+ """A pooled connection the scheduler closed just as a call went out fails
+ that call with ``httpx.ReadError`` before the scheduler sees it."""
+
+ @pytest.mark.asyncio
+ async def test_a_retrying_client_repeats_a_call_lost_in_transit(self):
+ post = _lost_then_answered()
+ with patch.object(httpx.AsyncClient, "post", post):
+ client = get_scheduler_client(should_retry=True)
+ assert await client.get_execution_schedules(user_id="user") == []
+ assert post.await_count == 2
+
+ @pytest.mark.asyncio
+ async def test_the_default_client_does_not_repeat_it(self):
+ post = _lost_then_answered()
+ with patch.object(httpx.AsyncClient, "post", post):
+ with pytest.raises(httpx.ReadError):
+ await get_scheduler_client().get_execution_schedules(user_id="user")
+ assert post.await_count == 1
+
+
+def _lost_then_answered() -> AsyncMock:
+ request = httpx.Request("POST", "http://scheduler/get_execution_schedules")
+ return AsyncMock(
+ side_effect=[
+ httpx.ReadError("connection closed", request=request),
+ httpx.Response(200, json=[], request=request),
+ ]
+ )
diff --git a/autogpt_platform/frontend/src/app/api/openapi.json b/autogpt_platform/frontend/src/app/api/openapi.json
index c5c9b8182e..99c5145fab 100644
--- a/autogpt_platform/frontend/src/app/api/openapi.json
+++ b/autogpt_platform/frontend/src/app/api/openapi.json
@@ -4198,6 +4198,141 @@
}
}
},
+ "/api/chat/sessions/{session_id}/at-reset": {
+ "get": {
+ "tags": ["v2", "chat", "chat"],
+ "summary": "List turns at budget reset",
+ "description": "The chat's pending resume and send at the budget reset.",
+ "operationId": "getV2List turns at budget reset",
+ "security": [{ "HTTPBearerJWT": [] }],
+ "parameters": [
+ {
+ "name": "session_id",
+ "in": "path",
+ "required": true,
+ "schema": { "type": "string", "title": "Session Id" }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/AtResetTurnsResponse"
+ }
+ }
+ }
+ },
+ "401": {
+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"
+ },
+ "404": { "description": "Session not found or access denied" },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": { "$ref": "#/components/schemas/HTTPValidationError" }
+ }
+ }
+ }
+ }
+ },
+ "post": {
+ "tags": ["v2", "chat", "chat"],
+ "summary": "Schedule turn at budget reset",
+ "description": "Resume the chat's paused task, or send a message, when the budget resets.\n\nIt fires just after the exhausted window that refills last. A chat has\nat most one of each: a resume set again replaces the earlier one, and a\nsend joins the message already waiting, so nothing written meanwhile is\nlost. Sending a message in the chat cancels a pending resume.",
+ "operationId": "postV2Schedule turn at budget reset",
+ "security": [{ "HTTPBearerJWT": [] }],
+ "parameters": [
+ {
+ "name": "session_id",
+ "in": "path",
+ "required": true,
+ "schema": { "type": "string", "title": "Session Id" }
+ }
+ ],
+ "requestBody": {
+ "required": true,
+ "content": {
+ "application/json": {
+ "schema": { "$ref": "#/components/schemas/AtResetRequest" }
+ }
+ }
+ },
+ "responses": {
+ "200": {
+ "description": "Successful Response",
+ "content": {
+ "application/json": {
+ "schema": { "$ref": "#/components/schemas/AtResetTurn" }
+ }
+ }
+ },
+ "401": {
+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"
+ },
+ "404": { "description": "Session not found, or not writable" },
+ "409": {
+ "description": "\"not_out_of_budget\": nothing to wait for, so carry on now. \"no_reset\": the allowance never refills (a trial)."
+ },
+ "413": {
+ "description": "\"message_too_long\": the message waiting plus this one would be longer than a single message may be"
+ },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": { "$ref": "#/components/schemas/HTTPValidationError" }
+ }
+ }
+ },
+ "503": { "description": "Usage could not be read; retry shortly" }
+ }
+ }
+ },
+ "/api/chat/sessions/{session_id}/at-reset/{kind}": {
+ "delete": {
+ "tags": ["v2", "chat", "chat"],
+ "summary": "Cancel turn at budget reset",
+ "description": "Cancel the chat's resume or send at the budget reset; one that isn't\nset is already cancelled.",
+ "operationId": "deleteV2Cancel turn at budget reset",
+ "security": [{ "HTTPBearerJWT": [] }],
+ "parameters": [
+ {
+ "name": "session_id",
+ "in": "path",
+ "required": true,
+ "schema": { "type": "string", "title": "Session Id" }
+ },
+ {
+ "name": "kind",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "enum": ["resume", "send"],
+ "type": "string",
+ "title": "Kind"
+ }
+ }
+ ],
+ "responses": {
+ "204": { "description": "Successful Response" },
+ "401": {
+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"
+ },
+ "404": { "description": "Session not found or access denied" },
+ "422": {
+ "description": "Validation Error",
+ "content": {
+ "application/json": {
+ "schema": { "$ref": "#/components/schemas/HTTPValidationError" }
+ }
+ }
+ }
+ }
+ }
+ },
"/api/chat/sessions/{session_id}/cancel": {
"post": {
"tags": ["v2", "chat", "chat"],
@@ -19228,6 +19363,63 @@
"required": ["pod_id"],
"title": "AssignPodRequest"
},
+ "AtResetRequest": {
+ "properties": {
+ "kind": {
+ "type": "string",
+ "enum": ["resume", "send"],
+ "title": "Kind",
+ "description": "\"resume\" picks the paused task back up; \"send\" sends ``message`` as the user's."
+ },
+ "message": {
+ "anyOf": [
+ { "type": "string", "maxLength": 64000 },
+ { "type": "null" }
+ ],
+ "title": "Message",
+ "description": "What to send. Required for send, ignored for resume."
+ }
+ },
+ "type": "object",
+ "required": ["kind"],
+ "title": "AtResetRequest",
+ "description": "A turn to run when the budget resets."
+ },
+ "AtResetTurn": {
+ "properties": {
+ "kind": {
+ "type": "string",
+ "enum": ["resume", "send"],
+ "title": "Kind"
+ },
+ "run_at": {
+ "type": "string",
+ "format": "date-time",
+ "title": "Run At"
+ },
+ "message": {
+ "anyOf": [{ "type": "string" }, { "type": "null" }],
+ "title": "Message"
+ }
+ },
+ "type": "object",
+ "required": ["kind", "run_at"],
+ "title": "AtResetTurn",
+ "description": "A turn set to run when the budget resets."
+ },
+ "AtResetTurnsResponse": {
+ "properties": {
+ "jobs": {
+ "items": { "$ref": "#/components/schemas/AtResetTurn" },
+ "type": "array",
+ "title": "Jobs"
+ }
+ },
+ "type": "object",
+ "required": ["jobs"],
+ "title": "AtResetTurnsResponse",
+ "description": "The chat's turns set for the budget reset, at most one of each kind."
+ },
"AuthEmailRequest": {
"properties": {
"type": {
@@ -21276,6 +21468,19 @@
"type": "object",
"title": "Credential Pins"
},
+ "purpose": {
+ "type": "string",
+ "enum": ["followup", "resume", "send"],
+ "title": "Purpose",
+ "default": "followup"
+ },
+ "created_at": {
+ "anyOf": [
+ { "type": "string", "format": "date-time" },
+ { "type": "null" }
+ ],
+ "title": "Created At"
+ },
"id": { "type": "string", "title": "Id" },
"name": { "type": "string", "title": "Name" },
"next_run_time": { "type": "string", "title": "Next Run Time" },