AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(backend/copilot): AutoPilot turns pause at their limits instead of dying, and Keep going picks them up (#15388)
Nicholas Tindlec350948affed
diff --git a/autogpt_platform/backend/backend/api/features/chat/continuation.py b/autogpt_platform/backend/backend/api/features/chat/continuation.pynew file mode 100644index 0000000000..df52253649--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/chat/continuation.py@@ -0,0 +1,147 @@+"""Keep going: pick a paused task back up in the same chat, now.++The Paused card posts here after a step check-in, or when a task reached its+size limit while the account still has budget. The continuation runs as a+turn whose message is a system note (see ``backend.copilot.continuation``),+so the chat shows what happened without a user bubble. It is admitted like+any message the user sends: the same paywall, the same allowance check, and+one turn per chat at a time.+"""++import logging++from autogpt_libs import auth+from fastapi import APIRouter, HTTPException, Security+from pydantic import BaseModel, Field++from backend.api.features.chat.routes import (+ _validate_and_get_writable_session,+ enforce_model_entitlement,+ enforce_platform_allowance,+)+from backend.copilot import stream_registry+from backend.copilot.active_turns import ConcurrentTurnLimitError+from backend.copilot.config import CopilotLLMModel+from backend.copilot.continuation import KEEP_GOING_NOTE+from backend.copilot.executor.utils import schedule_chat_turn+from backend.copilot.model import ChatSessionInfo+from backend.copilot.pending_message_helpers import (+ StreamRegistryUnavailable,+ is_turn_in_flight,+)+from backend.copilot.rate_limit import enforce_payment_paywall+from backend.copilot.response_model import StreamStatus+from backend.copilot.session_permissions import resolve_session_permissions+from backend.integrations.codex.access import enforce_codex_access_http++logger = logging.getLogger(__name__)++router = APIRouter()+++class ContinueSessionRequest(BaseModel):+ model: CopilotLLMModel | None = Field(+ default=None,+ description="Model tier the continuation runs on, as for a message.",+ )+++class ContinueSessionResponse(BaseModel):+ turn_id: str = Field(description="The continuation turn; attach to its stream.")+++ "/sessions/{session_id}/continue",+ status_code=202,+ responses={+ 402: {"description": "Subscription required (NO_TIER user, paywall on)"},+ 404: {"description": "Session not found or access denied"},+ 409: {+ "description": "A turn is already running in this chat "+ "(turn_in_progress), or the chat runs on a connection that can't "+ "continue a task (unsupported_connection)"+ },+ 429: {"description": "Usage limit or concurrent-turn limit reached"},+ 503: {"description": "Chat service degraded; retry after Retry-After"},+ },+)+async def continue_session(+ session_id: str,+ request: ContinueSessionRequest | None = None,+ user_id: str = Security(auth.get_user_id),+ ctx: auth.RequestContext = Security(auth.get_request_context),+) -> ContinueSessionResponse:+ """Continue the chat's task from the checkpoint it paused at."""+ session = await _validate_and_get_writable_session(session_id, user_id)+ model = request.model if request is not None else None+ await enforce_model_entitlement(model, session, user_id)+ await _admit(session, user_id)+ await _refuse_if_busy(session_id)+ try:+ # Every note is the same text, so the trailing-text dedupe would refuse+ # a Keep going after a continuation that wrote only assistant rows, or+ # nothing. The fresh slot is what stops a second turn on this chat.+ turn_id = await schedule_chat_turn(+ session_id=session_id,+ user_id=user_id,+ message=KEEP_GOING_NOTE,+ require_fresh_slot=True,+ dedupe_content=False,+ is_user_message=False,+ expert_id=session.expert_id,+ session_origin=session.metadata.origin,+ session_source_platform=session.metadata.source_platform,+ organization_id=session.organization_id or ctx.org_id,+ team_id=session.team_id if session.organization_id else ctx.team_id,+ model=model,+ llm_auth_provider=session.metadata.llm_auth_provider,+ llm_credential_id=session.metadata.llm_credential_id,+ permissions=resolve_session_permissions(session),+ )+ except ConcurrentTurnLimitError as exc:+ raise HTTPException(status_code=429, detail=str(exc)) from exc+ if turn_id is None:+ # Another turn took the chat after the busy check above: a message+ # sent at the same moment, or the first click of a double click.+ raise HTTPException(status_code=409, detail="turn_in_progress")+ await _announce(turn_id, session_id)+ return ContinueSessionResponse(turn_id=turn_id)+++async def _admit(session: ChatSessionInfo, user_id: str) -> None:+ route = session.metadata.llm_auth_provider+ if route == "platform":+ await enforce_payment_paywall(user_id)+ await enforce_platform_allowance(user_id)+ elif route == "codex":+ await enforce_codex_access_http(user_id)+ elif route == "microsoft_365_copilot":+ # Microsoft 365 Copilot answers plain messages: it has none of+ # AutoPilot's tools and never saw the task, so it has nothing to pick+ # up, and the note would reach Microsoft as the user's message.+ raise HTTPException(status_code=409, detail="unsupported_connection")+++async def _refuse_if_busy(session_id: str) -> None:+ try:+ busy = await is_turn_in_flight(session_id)+ except StreamRegistryUnavailable as exc:+ raise HTTPException(+ status_code=503,+ detail="Chat service degraded, retry shortly",+ headers={"Retry-After": "30"},+ ) from exc+ if busy:+ raise HTTPException(status_code=409, detail="turn_in_progress")+++async def _announce(turn_id: str, session_id: str) -> None:+ """Give a subscriber something to show while the turn waits for pickup."""+ try:+ await stream_registry.publish_chunk(+ turn_id,+ StreamStatus(message="Picking the task back up…"),+ session_id=session_id,+ )+ except Exception:+ logger.warning("[CONTINUE] Failed to publish initial status", exc_info=True)diff --git a/autogpt_platform/backend/backend/api/features/chat/continuation_test.py b/autogpt_platform/backend/backend/api/features/chat/continuation_test.pynew file mode 100644index 0000000000..63bef1952f--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/chat/continuation_test.py@@ -0,0 +1,271 @@+"""Route tests for POST /sessions/{session_id}/continue (the Paused card's Keep going)."""++from contextlib import asynccontextmanager+from datetime import UTC, datetime, timedelta+from unittest.mock import AsyncMock, MagicMock++import fastapi+import fastapi.testclient+import pytest+import pytest_mock++from backend.api.features.chat import continuation as continuation_routes+from backend.api.features.chat import routes as chat_routes+from backend.copilot import model as chat_model+from backend.copilot.active_turns import ConcurrentTurnLimitError+from backend.copilot.constants import COPILOT_SYSTEM_PREFIX+from backend.copilot.continuation import KEEP_GOING_NOTE+from backend.copilot.executor import utils as executor_utils+from backend.copilot.model import ChatMessage, ChatSession+from backend.copilot.rate_limit import RateLimitExceeded, RateLimitUnavailable++app = fastapi.FastAPI()+app.include_router(continuation_routes.router)+client = fastapi.testclient.TestClient(app)++SESSION_ID = "8d2f5c1e-2b7a-4c55-9d0e-3f6a1b2c4d5e"+URL = f"/sessions/{SESSION_ID}/continue"++[email protected](autouse=True)+def setup_app_auth(mock_jwt_user):+ from autogpt_libs.auth.dependencies import get_request_context+ from autogpt_libs.auth.jwt_utils import get_jwt_payload++ app.dependency_overrides[get_jwt_payload] = mock_jwt_user["get_jwt_payload"]+ app.dependency_overrides[get_request_context] = mock_jwt_user["get_request_context"]+ yield+ app.dependency_overrides.clear()+++def _session(route: str = "platform") -> MagicMock:+ session = MagicMock()+ session.expert_id = None+ session.organization_id = None+ session.team_id = None+ session.metadata.llm_auth_provider = route+ session.metadata.llm_credential_id = None+ session.metadata.origin = "interactive"+ session.metadata.source_platform = None+ return session+++def chat(mocker: pytest_mock.MockerFixture):+ """Patch everything the route touches; tests flip what they need."""+ mocks = MagicMock()+ mocks.session = mocker.patch.object(+ continuation_routes,+ "_validate_and_get_writable_session",+ new=AsyncMock(return_value=_session()),+ )+ mocks.paywall = mocker.patch.object(+ continuation_routes, "enforce_payment_paywall", new=AsyncMock()+ )+ mocks.codex_access = mocker.patch.object(+ continuation_routes, "enforce_codex_access_http", new=AsyncMock()+ )+ mocker.patch.object(+ chat_routes,+ "get_global_rate_limits",+ new=AsyncMock(return_value=(1_000_000, 5_000_000, "PRO")),+ )+ mocks.check_rate_limit = mocker.patch.object(+ chat_routes, "check_rate_limit", new=AsyncMock()+ )+ mocks.busy = mocker.patch.object(+ continuation_routes, "is_turn_in_flight", new=AsyncMock(return_value=False)+ )+ mocks.schedule = mocker.patch.object(+ continuation_routes, "schedule_chat_turn", new=AsyncMock(return_value="t-1")+ )+ mocker.patch.object(+ continuation_routes, "resolve_session_permissions", return_value=None+ )+ mocks.publish = mocker.patch.object(+ continuation_routes.stream_registry, "publish_chunk", new=AsyncMock()+ )+ return mocks+++def test_keep_going_starts_a_continuation_turn(chat):+ response = client.post(URL, json={"model": "standard"})++ assert response.status_code == 202+ assert response.json() == {"turn_id": "t-1"}+ kwargs = chat.schedule.await_args.kwargs+ assert kwargs["message"] == KEEP_GOING_NOTE+ assert kwargs["is_user_message"] is False+ assert kwargs["require_fresh_slot"] is True+ assert kwargs["dedupe_content"] is False+ assert kwargs["model"] == "standard"+ chat.paywall.assert_awaited_once()+ chat.check_rate_limit.assert_awaited_once()+ chat.publish.assert_awaited_once()+++def test_body_is_optional(chat):+ assert client.post(URL).status_code == 202+ assert chat.schedule.await_args.kwargs["model"] is None+++def test_a_running_turn_is_not_doubled(chat):+ chat.busy.return_value = True++ response = client.post(URL)++ assert response.status_code == 409+ chat.schedule.assert_not_awaited()+++def test_an_out_of_budget_user_gets_the_usage_limit_envelope(chat):+ resets = datetime.now(UTC) + timedelta(hours=4)+ chat.check_rate_limit.side_effect = RateLimitExceeded("daily", resets)++ response = client.post(URL)++ assert response.status_code == 429+ detail = response.json()["detail"]+ assert detail["kind"] == "usage_limit"+ assert detail["resetsAt"] == int(resets.timestamp())+ chat.schedule.assert_not_awaited()+++def test_an_unreadable_allowance_fails_closed(chat):+ chat.check_rate_limit.side_effect = RateLimitUnavailable()++ response = client.post(URL)++ assert response.status_code == 503+ chat.schedule.assert_not_awaited()+++def test_a_chatgpt_chat_skips_the_platform_allowance(chat):+ chat.session.return_value = _session(route="codex")++ assert client.post(URL).status_code == 202+ chat.codex_access.assert_awaited_once()+ chat.paywall.assert_not_awaited()+ chat.check_rate_limit.assert_not_awaited()+++def test_a_microsoft_365_chat_is_refused_and_nothing_is_sent(chat):+ """Keep going would hand Microsoft our system note as the user's message,+ on a connection with no tools and none of the task."""+ chat.session.return_value = _session(route="microsoft_365_copilot")++ response = client.post(URL)++ assert response.status_code == 409+ assert response.json()["detail"] == "unsupported_connection"+ chat.schedule.assert_not_awaited()+ chat.publish.assert_not_awaited()+++def test_a_full_concurrency_cap_is_a_429(chat):+ chat.schedule.side_effect = ConcurrentTurnLimitError("5 turns running")++ assert client.post(URL).status_code == 429+++def test_a_chat_taken_after_the_busy_check_is_a_409(chat):+ chat.schedule.return_value = None++ assert client.post(URL).status_code == 409+ chat.publish.assert_not_awaited()+++_PAUSE = f"{COPILOT_SYSTEM_PREFIX} Paused: checking in after a long run."+++def stored(mocker: pytest_mock.MockerFixture, chat: MagicMock) -> MagicMock:+ """The real scheduler and row store behind the route, over rows in memory.++ Mocked at the boundaries: the turn slot, the session lock, the queue and+ the database.+ """+ mocker.patch.object(+ continuation_routes,+ "schedule_chat_turn",+ new=executor_utils.schedule_chat_turn,+ )+ io = MagicMock()+ io.slot = MagicMock(admitted=True)+ io.session = ChatSession.new("test-user-id", dry_run=False)+ io.session.messages = [+ ChatMessage(role="user", content="Build the report"),+ ChatMessage(role="assistant", content=_PAUSE),+ ]++ @asynccontextmanager+ async def acquire(*_args, **_kwargs):+ yield io.slot++ @asynccontextmanager+ async def session_lock(_session_id: str):+ yield True++ mocker.patch.object(executor_utils, "acquire_turn_slot", new=acquire)+ io.dispatch = mocker.patch.object(executor_utils, "dispatch_turn", new=AsyncMock())+ mocker.patch.object(chat_model, "_get_session_lock", new=session_lock)+ mocker.patch.object(+ chat_model, "get_chat_session", new=AsyncMock(return_value=io.session)+ )+ io.save = mocker.patch.object(chat_model, "_save_session_to_db", new=AsyncMock())+ mocker.patch.object(+ chat_model,+ "chat_db",+ return_value=MagicMock(get_next_sequence=AsyncMock(return_value=2)),+ )+ mocker.patch.object(chat_model, "cache_chat_session", new=AsyncMock())+ return io+++def _notes(session: ChatSession) -> int:+ return [m.content for m in session.messages].count(KEEP_GOING_NOTE)+++def test_a_chat_another_turn_holds_is_a_409_and_saves_nothing(stored):+ # /stream and /continue both found the chat idle; /stream reserved it+ # first, so this request only refreshed the slot.+ stored.slot.admitted = False++ assert client.post(URL).status_code == 409+ assert _notes(stored.session) == 0+ stored.save.assert_not_awaited()+ stored.dispatch.assert_not_awaited()+++def test_a_double_click_starts_one_turn(stored):+ first = client.post(URL)+ stored.slot.admitted = False # the first click's turn holds the chat now+ second = client.post(URL)++ assert (first.status_code, second.status_code) == (202, 409)+ assert _notes(stored.session) == 1+ stored.dispatch.assert_awaited_once()+++ "rows_after_the_note",+ [+ pytest.param(["Two of three sections done.", _PAUSE], id="assistant-rows"),+ pytest.param([], id="nothing"),+ ],+)+def test_keep_going_again_after_a_continuation_left_no_user_row(+ stored, rows_after_the_note: list[str]+):+ # The earlier note is still in the chat's trailing assistant run.+ stored.session.messages += [+ ChatMessage(role="assistant", content=KEEP_GOING_NOTE),+ *(ChatMessage(role="assistant", content=row) for row in rows_after_the_note),+ ]++ response = client.post(URL)++ assert response.status_code == 202+ assert stored.session.messages[-1].content == KEEP_GOING_NOTE+ assert _notes(stored.session) == 2+ stored.dispatch.assert_awaited_once()diff --git a/autogpt_platform/backend/backend/api/features/chat/routes.py b/autogpt_platform/backend/backend/api/features/chat/routes.pyindex 4fec9bd00a..a55aa8cb59 100644--- a/autogpt_platform/backend/backend/api/features/chat/routes.py+++ b/autogpt_platform/backend/backend/api/features/chat/routes.py@@ -186,6 +186,72 @@ logger = logging.getLogger(__name__)config = ChatConfig()+async def enforce_model_entitlement(+ model: CopilotLLMModel | None, session: ChatSessionInfo, user_id: str+) -> None:+ """Refuse the Advanced tier to a user who doesn't hold it.++ Microsoft 365 Copilot owns its model choice and ignores AutoGPT's tier.+ Every other route can spend platform-gated premium inference, so a client+ that skips the picker still has to hold the Advanced entitlement.+ """+ if model != "advanced" or (+ session.metadata.llm_auth_provider == "microsoft_365_copilot"+ ):+ return+ try:+ entitled = await advanced_tier_entitled(user_id)+ except EntitlementUnavailable:+ raise HTTPException(+ status_code=503, detail="advanced_tier_unavailable"+ ) from None+ if not entitled:+ raise HTTPException(status_code=403, detail="advanced_tier_not_entitled")+++async def enforce_platform_allowance(user_id: str) -> None:+ """Refuse a platform-billed turn when the user's allowance is spent.++ Raises 429 with a structured ``ProviderFailure`` envelope, or 503 when the+ allowance can't be read. Shared by every route that starts a turn.+ """+ try:+ daily_limit, weekly_limit, usage_tier = await get_global_rate_limits(+ user_id,+ config.daily_cost_limit_microdollars,+ config.weekly_cost_limit_microdollars,+ )+ await check_rate_limit(+ user_id=user_id,+ daily_cost_limit=daily_limit,+ weekly_cost_limit=weekly_limit,+ expected_tier=usage_tier,+ )+ except RateLimitExceeded as e:+ # Structured envelope (not a bare string) so the frontend can+ # offer "switch to another connection" (e.g. a connected+ # BYOSUB/Codex credential) instead of only "upgrade your plan" --+ # the platform cap does not apply once the turn is billed to a+ # user-supplied credential instead of platform dollars.+ failure = ProviderFailure(+ kind=ProviderFailureKind.USAGE_LIMIT,+ message=str(e),+ auth_provider="platform",+ resets_at=int(e.resets_at.timestamp()),+ )+ raise HTTPException(status_code=429, detail=failure.as_part()) from e+ except RateLimitUnavailable as e:+ # Fail-closed on Redis brown-out: the user may already be at or+ # past their USD cap and we cannot prove otherwise. 503 + a short+ # Retry-After is the right UX (transient outage, retry shortly),+ # not 429 ("you hit your limit").+ raise HTTPException(+ status_code=503,+ detail="Rate limit service degraded, retry shortly",+ headers={"Retry-After": "30"},+ ) from e++async def _validate_and_get_session(session_id: str,user_id: str | None,@@ -1790,21 +1856,7 @@ async def stream_chat_post(session = await _validate_and_get_writable_session(session_id, user_id)await _apply_autopilot_mode(session, user_id, request.autopilot_mode)- # Microsoft 365 Copilot owns its model choice and ignores AutoGPT's tier.- # Every other route can spend platform-gated premium inference, so a client- # that skips the picker still has to hold the Advanced entitlement.- if (- request.model == "advanced"- and session.metadata.llm_auth_provider != "microsoft_365_copilot"- ):- try:- entitled = await advanced_tier_entitled(user_id)- except EntitlementUnavailable:- raise HTTPException(- status_code=503, detail="advanced_tier_unavailable"- ) from None- if not entitled:- raise HTTPException(status_code=403, detail="advanced_tier_not_entitled")+ await enforce_model_entitlement(request.model, session, user_id)# Fire-and-forget; per-user Redis dedup inside the helper provides# cross-process / cross-restart idempotency. Same pattern as@@ -1931,41 +1983,7 @@ async def stream_chat_post(# pending-message frequency, and concurrent-turn caps remain enforced by# the shared scheduling path below.if user_id and is_platform_route:- try:- daily_limit, weekly_limit, usage_tier = await get_global_rate_limits(- user_id,- config.daily_cost_limit_microdollars,- config.weekly_cost_limit_microdollars,- )- await check_rate_limit(- user_id=user_id,- daily_cost_limit=daily_limit,- weekly_cost_limit=weekly_limit,- expected_tier=usage_tier,- )- except RateLimitExceeded as e:- # Structured envelope (not a bare string) so the frontend can- # offer "switch to another connection" (e.g. a connected- # BYOSUB/Codex credential) instead of only "upgrade your plan" --- # the platform cap does not apply once the turn is billed to a- # user-supplied credential instead of platform dollars.- failure = ProviderFailure(- kind=ProviderFailureKind.USAGE_LIMIT,- message=str(e),- auth_provider="platform",- resets_at=int(e.resets_at.timestamp()),- )- raise HTTPException(status_code=429, detail=failure.as_part()) from e- except RateLimitUnavailable as e:- # Fail-closed on Redis brown-out: the user may already be at or- # past their USD cap and we cannot prove otherwise. 503 + a short- # Retry-After is the right UX (transient outage, retry shortly),- # not 429 ("you hit your limit").- raise HTTPException(- status_code=503,- detail="Rate limit service degraded, retry shortly",- headers={"Retry-After": "30"},- ) from e+ await enforce_platform_allowance(user_id)# Enrich message with file metadata if file_ids are provided.# Also sanitise file_ids so only validated, workspace-scoped IDs arediff --git a/autogpt_platform/backend/backend/api/rest_api.py b/autogpt_platform/backend/backend/api/rest_api.pyindex b75822b823..ccbaeb5b71 100644--- a/autogpt_platform/backend/backend/api/rest_api.py+++ b/autogpt_platform/backend/backend/api/rest_api.py@@ -36,6 +36,7 @@ import backend.api.features.blocks.routes as blocks_routesimport backend.api.features.briefings.routesimport backend.api.features.builderimport backend.api.features.builder.routes+import backend.api.features.chat.continuation as chat_continuationimport backend.api.features.chat.feedback as chat_feedbackimport backend.api.features.chat.routes as chat_routesimport backend.api.features.chat.share as chat_share@@ -577,6 +578,11 @@ app.include_router(tags=["v2", "chat"],prefix="/api/chat",)+app.include_router(+ chat_continuation.router,+ tags=["v2", "chat"],+ prefix="/api/chat",+)app.include_router(chat_speech.router,tags=["chat"],diff --git a/autogpt_platform/backend/backend/copilot/baseline/continuation_note_test.py b/autogpt_platform/backend/backend/copilot/baseline/continuation_note_test.pynew file mode 100644index 0000000000..5ed1e2516b--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/baseline/continuation_note_test.py@@ -0,0 +1,86 @@+"""A continuation note that ends the chat reaches the model as the user's turn.++The note is stored as an assistant row, so a request built from history would+end on an assistant message: a prefill, which Claude 4.6+ rejects with a 400.+"""++import asyncio+from typing import Any+from unittest.mock import AsyncMock, MagicMock++import pytest++from backend.copilot.baseline import service+from backend.copilot.baseline.service_cancellation_test import baseline_io+from backend.copilot.continuation import KEEP_GOING_NOTE+from backend.copilot.model import ChatMessage, ChatSession++__all__ = ["baseline_io"]++CHECKPOINT = "Checkpoint: two of three sections done."+++async def test_a_trailing_note_is_sent_as_the_user_turn_without_its_marker(+ monkeypatch: pytest.MonkeyPatch, baseline_io: list[ChatSession]+) -> None:+ session = _chat(+ ChatMessage(role="assistant", content=CHECKPOINT),+ ChatMessage(role="assistant", content=KEEP_GOING_NOTE),+ )++ sent = await _sent_messages(monkeypatch, session, message=KEEP_GOING_NOTE)++ assert sent[-2:] == [+ {"role": "assistant", "content": CHECKPOINT},+ {+ "role": "user",+ "content": "Keep going: pick the task up from the checkpoint above.",+ },+ ]+ [persisted] = baseline_io+ [note] = [m for m in persisted.messages if m.content == KEEP_GOING_NOTE]+ assert note.role == "assistant"+++async def test_other_trailing_assistant_text_is_sent_as_it_is(+ monkeypatch: pytest.MonkeyPatch, baseline_io: list[ChatSession]+) -> None:+ session = _chat(ChatMessage(role="assistant", content=CHECKPOINT))++ sent = await _sent_messages(monkeypatch, session, message=None)++ assert sent[-1] == {"role": "assistant", "content": CHECKPOINT}+++def _chat(*rows: ChatMessage) -> ChatSession:+ session = ChatSession.new("user-1", dry_run=False)+ session.title = "Quarterly report"+ session.messages = [ChatMessage(role="user", content="Build the report"), *rows]+ return session+++async def _sent_messages(+ monkeypatch: pytest.MonkeyPatch, session: ChatSession, *, message: str | None+) -> list[dict[str, Any]]:+ """Run a non-user turn and return the messages the provider was sent."""+ stream = MagicMock()+ stream.__aiter__.return_value = []+ stream.close = AsyncMock()+ provider = AsyncMock(return_value=stream)+ monkeypatch.setattr(service, "call_provider_stream", provider)+ monkeypatch.setattr(service, "resolve_answered", AsyncMock(return_value=[]))++ async with asyncio.timeout(5):+ async for _ in service.stream_chat_completion_baseline(+ session.session_id,+ message,+ is_user_message=False,+ user_id="user-1",+ session=session,+ ):+ pass++ assert provider.await_args is not None+ return provider.await_args.kwargs["messages"]diff --git a/autogpt_platform/backend/backend/copilot/baseline/service.py b/autogpt_platform/backend/backend/copilot/baseline/service.pyindex ee1354d75c..d425d50c0d 100644--- a/autogpt_platform/backend/backend/copilot/baseline/service.py+++ b/autogpt_platform/backend/backend/copilot/baseline/service.py@@ -48,6 +48,7 @@ from backend.copilot.builder_context import (from backend.copilot.capabilities.dispatch import resolve_tool_dispatchfrom backend.copilot.config import CopilotLlmAuthProvider, CopilotLLMModelfrom backend.copilot.context import get_workspace_manager, set_execution_context+from backend.copilot.continuation import send_trailing_note_as_userfrom backend.copilot.expert_context import build_expert_identity_suffixfrom backend.copilot.expert_kickoff import is_expert_kickoff_turnfrom backend.copilot.gate import active_mode@@ -2058,6 +2059,9 @@ async def stream_chat_completion_baseline()elif msg.role == "user" and msg.content:openai_messages.append({"role": msg.role, "content": msg.content})+ # Before anything is appended below: this reads the last entry, and the+ # per-turn blocks then land on the note as they would on a typed message.+ send_trailing_note_as_user(openai_messages)# Inject user context into the first user message on first turn.# Done before attachment/URL injection so the context prefix lands atdiff --git a/autogpt_platform/backend/backend/copilot/billing_e2e/audit.py b/autogpt_platform/backend/backend/copilot/billing_e2e/audit.pyindex 7d8e9a21d0..31e5eeb52f 100644--- a/autogpt_platform/backend/backend/copilot/billing_e2e/audit.py+++ b/autogpt_platform/backend/backend/copilot/billing_e2e/audit.py@@ -5,7 +5,7 @@ from pydantic import BaseModelfrom backend.copilot.model import ChatSession, Usagefrom backend.data.platform_cost import PlatformCostEntry, usd_to_microdollars-from .invoice import Call+from .invoice import ANTHROPIC_PRICES, KIMI, MOONSHOT_FALLBACK, Callfrom .ledger import PlatformLedgerfrom .provider import FakeProviderfrom .scenarios import Check, Scenario, expected_charge_usd@@ -26,18 +26,22 @@ def audit_thread(session: ChatSession,usage_by_turn: list[list[Usage]],cli_resumed: list[tuple[int, bool]] | None,+ metered: dict[int, float] | None,) -> list[Finding]:"""Every way the thread strayed from its invoice or its planned path.``cli_resumed`` says, for each CLI launch of an SDK thread, its turn andwhether it resumed the chat's session; None for engines without a CLI.+ ``metered`` is the live spend each SDK turn's governor reached; None for+ engines without one."""findings = _audit_path(provider, cli_resumed, turns=len(usage_by_turn))findings += _audit_attribution(session, ledger)for turn, session_usage in enumerate(usage_by_turn):- findings += _audit_turn(- scenario, turn, provider.calls_in(turn), ledger, session_usage- )+ calls = provider.calls_in(turn)+ findings += _audit_turn(scenario, turn, calls, ledger, session_usage)+ if metered is not None:+ findings += _audit_meter(turn, calls, metered.get(turn, 0.0))return findings@@ -214,3 +218,38 @@ def _audit_tokens(if recorded != [owed]:messages.append(f"turn {turn}: session usage {recorded}, the turn owes {owed}")return [Finding(check="tokens", message=message) for message in messages]+++def _audit_meter(turn: int, calls: list[Call], metered: float) -> list[Finding]:+ """The governor's live spend: every call the CLI made, failed attempts+ included, at list price.++ A task lands on this figure before its budget runs out. A model the rate+ card can't find is priced as Opus, several times Sonnet, which lands+ tasks early: OpenRouter's ``anthropic/`` slugs once did that.+ """+ ranges = [r for r in map(_list_price_range, calls) if r is not None]+ if len(ranges) != len(calls):+ # A route the suite has no list price for (Codex).+ return []+ low = sum(r[0] for r in ranges)+ high = sum(r[1] for r in ranges)+ if low - 1e-9 <= metered <= high + 1e-9:+ return []+ message = (+ f"turn {turn}: the governor metered ${metered:.6f} for "+ f"{len(calls)} call(s) listed at ${low:.6f}-${high:.6f}"+ )+ return [Finding(check="meter", message=message)]+++def _list_price_range(call: Call) -> tuple[float, float] | None:+ """A call at list price, its cache writes at their own rate up to the 1h+ rate: the CLI's usage doesn't say which a write was."""+ if call.model == KIMI:+ price = MOONSHOT_FALLBACK.of(call.usage, None)+ return price, price+ list_price = ANTHROPIC_PRICES.get(call.model.removeprefix("anthropic/"))+ if list_price is None:+ return None+ return list_price.of(call.usage, call.cache_ttl), list_price.of(call.usage, "1h")diff --git a/autogpt_platform/backend/backend/copilot/billing_e2e/audit_test.py b/autogpt_platform/backend/backend/copilot/billing_e2e/audit_test.pyindex 96d8331bf7..4a86f0f89e 100644--- a/autogpt_platform/backend/backend/copilot/billing_e2e/audit_test.py+++ b/autogpt_platform/backend/backend/copilot/billing_e2e/audit_test.py@@ -1,11 +1,15 @@"""The audit on its own: a relaunched turn owes only the attempt that-succeeded, and a turn the failure ended owes nothing."""+succeeded, a turn the failure ended owes nothing, and a governor meters each+call at list price."""++from claude_agent_sdk import AssistantMessage, TextBlockfrom backend.copilot.model import Usage+from backend.copilot.sdk.usage_meter import UsageMeterfrom backend.data.platform_cost import PlatformCostEntry, usd_to_microdollars-from .audit import _audit_turn-from .invoice import ANTHROPIC_PRICES, SONNET, Call, scripted_usage+from .audit import _audit_meter, _audit_turn+from .invoice import ANTHROPIC_PRICES, SONNET, Call, Price, scripted_usagefrom .ledger import CostLogRow, PlatformLedger, SpendCharge, TreeChargefrom .provider import PlannedFaultfrom .scenarios import SCENARIOS, _sdk_anthropic_failing@@ -73,6 +77,46 @@ def test_charging_a_turn_the_failure_ended_is_reported():assert "owes nothing" in findings[0].message+# Two calls on OpenRouter, which names the model ``anthropic/claude-sonnet-5-5``.+_OPENROUTER_CALLS = [+ call.model_copy(update={"route": "openrouter", "model": f"anthropic/{SONNET}"})+ for call in (_FAILED_ATTEMPT, _RETRY)+]+++def test_a_governor_metering_its_calls_at_list_price_passes():+ meter = UsageMeter()+ for call in _OPENROUTER_CALLS:+ meter.observe(_as_the_cli_reports(call))++ assert _audit_meter(1, _OPENROUTER_CALLS, meter.spent_usd) == []+++def test_a_governor_pricing_sonnet_as_opus_is_reported():+ opus = Price(+ input=15, output=75, cache_read=1.5, cache_write_5m=18.75, cache_write_1h=30+ )+ metered = sum(opus.of(call.usage, "1h") for call in _OPENROUTER_CALLS)++ findings = _audit_meter(1, _OPENROUTER_CALLS, metered)++ assert [finding.check for finding in findings] == ["meter"]+++def _as_the_cli_reports(call: Call) -> AssistantMessage:+ return AssistantMessage(+ content=[TextBlock(text="x")],+ model=call.model,+ usage={+ "input_tokens": call.usage.input_tokens,+ "output_tokens": call.usage.output_tokens,+ "cache_read_input_tokens": call.usage.cache_read_tokens,+ "cache_creation_input_tokens": call.usage.cache_write_tokens,+ },+ message_id=f"call-{call.request}",+ )++def _turn_charged_for(*calls: Call) -> tuple[PlatformLedger, list[Usage]]:"""Turn 1's ledger and session usage, as if the platform charged ``calls``."""microdollars = usd_to_microdollars(sum(call.invoice_usd for call in calls))diff --git a/autogpt_platform/backend/backend/copilot/billing_e2e/scenarios.py b/autogpt_platform/backend/backend/copilot/billing_e2e/scenarios.pyindex 19445d4fc6..289243bf78 100644--- a/autogpt_platform/backend/backend/copilot/billing_e2e/scenarios.py+++ b/autogpt_platform/backend/backend/copilot/billing_e2e/scenarios.py@@ -26,8 +26,9 @@ from .provider import PlannedFaultChargeBasis = Literal["invoice", "cli_estimate", "moonshot_estimate", "none"]# What a finding is about: whether the thread took its planned path, what the-# turn was charged, or the tokens recorded against it.-Check = Literal["path", "charge", "tokens"]+# turn was charged, the tokens recorded against it, or the live spend the SDK+# turn's governor lands tasks on.+Check = Literal["path", "charge", "tokens", "meter"]class KnownGap(BaseModel):diff --git a/autogpt_platform/backend/backend/copilot/billing_e2e/sdk_turns.py b/autogpt_platform/backend/backend/copilot/billing_e2e/sdk_turns.pyindex f152d1563f..1c1a965a3d 100644--- a/autogpt_platform/backend/backend/copilot/billing_e2e/sdk_turns.py+++ b/autogpt_platform/backend/backend/copilot/billing_e2e/sdk_turns.py@@ -26,8 +26,10 @@ from backend.copilot.model import ChatSessionfrom backend.copilot.sdk import env as sdk_envfrom backend.copilot.sdk import openrouter_costfrom backend.copilot.sdk import service as sdk_service+from backend.copilot.sdk.governor import TurnGovernorfrom backend.copilot.transcript import TranscriptDownloadfrom backend.copilot.tree import root_envelope+from backend.copilot.turn_budget import TurnBudgetfrom backend.data.llm_registry.llm_models import LLMModelfrom backend.data.model import OAuth2Credentialsfrom backend.integrations.codex.auth_bundle import CodexAuthBundleV1, CodexAuthTokensV1@@ -62,6 +64,7 @@ class SdkTurns:# The turn of each launch, and the CLI session it resumed (None for a# fresh one).self.resumed: list[tuple[int, str | None]] = []+ self._governors: list[tuple[int, TurnGovernor]] = []self._turn = -1# The CLI inherits these, and the platform reads the CLI's session# files from the same place.@@ -103,10 +106,19 @@ class SdkTurns:):pass+ @property+ def metered(self) -> dict[int, float]:+ """The live spend each turn's governor reached, which it lands on."""+ totals: dict[int, float] = {}+ for turn, governor in self._governors:+ totals[turn] = totals.get(turn, 0.0) + governor.meter.spent_usd+ return totals+def _stubs(self, session: ChatSession) -> dict[str, Any]:return {"config": self._config,"ClaudeSDKClient": MagicMock(side_effect=self._client),+ "TurnGovernor": MagicMock(side_effect=self._governor),"AsyncClusterLock": MagicMock(side_effect=_lock),"_make_sdk_cwd": MagicMock(return_value=str(self._cwd)),"get_chat_session": AsyncMock(return_value=session),@@ -128,7 +140,10 @@ class SdkTurns:"get_copilot_tool_names": MagicMock(return_value=[]),"get_sdk_disallowed_tools": MagicMock(return_value=[]),"get_user_tier": AsyncMock(return_value=None),- "_resolve_dynamic_max_budget_usd": AsyncMock(return_value=100.0),+ # Well above any turn here, so the governor never lands one.+ "resolve_turn_budget": AsyncMock(+ return_value=TurnBudget(usd=100.0, limiting="task")+ ),"build_turn_budget_block": AsyncMock(return_value=""),"drain_pending_safe": AsyncMock(return_value=[]),"drain_pending_for_persist": AsyncMock(return_value=[]),@@ -148,6 +163,11 @@ class SdkTurns:options.disallowed_tools = []return ClaudeSDKClient(*args, options=options, **kwargs)+ def _governor(self, **kwargs: Any) -> TurnGovernor:+ governor = TurnGovernor(**kwargs)+ self._governors.append((self._turn, governor))+ return governor+async def _upload(self,*,diff --git a/autogpt_platform/backend/backend/copilot/billing_e2e/thread_billing_test.py b/autogpt_platform/backend/backend/copilot/billing_e2e/thread_billing_test.pyindex ac41264d64..0b85bee35a 100644--- a/autogpt_platform/backend/backend/copilot/billing_e2e/thread_billing_test.py+++ b/autogpt_platform/backend/backend/copilot/billing_e2e/thread_billing_test.py@@ -69,8 +69,9 @@ async def test_each_turn_is_charged_what_its_provider_billed(if isinstance(run_turn, SdkTurns)else None)+ metered = run_turn.metered if isinstance(run_turn, SdkTurns) else Nonefindings = audit_thread(- scenario, provider, ledger, session, usage_by_turn, cli_resumed+ scenario, provider, ledger, session, usage_by_turn, cli_resumed, metered)gap = scenario.known_gapunexplained = [f.message for f in findings if gap is None or f.check != gap.check]diff --git a/autogpt_platform/backend/backend/copilot/config.py b/autogpt_platform/backend/backend/copilot/config.pyindex e6b9582b2a..cc9f7c8d96 100644--- a/autogpt_platform/backend/backend/copilot/config.py+++ b/autogpt_platform/backend/backend/copilot/config.py@@ -478,12 +478,21 @@ class ChatConfig(BaseSettings):"CHAT_AGENT_MAX_TURNS","CHAT_CLAUDE_AGENT_MAX_TURNS",),- description="Maximum number of tool-call rounds per turn — applies to "- "both the baseline and Claude Agent SDK paths, including Codex models "- "on the SDK path, where each Codex function call counts as one round. "- "Prevents runaway tool loops from burning budget. Override via "- "CHAT_AGENT_MAX_TURNS env var (legacy CHAT_CLAUDE_AGENT_MAX_TURNS "- "still accepted).",+ description="Maximum number of tool-call rounds per turn on the "+ "baseline path, which has no spend governor. The Claude Agent SDK path, "+ "Codex models included, uses ``sdk_step_checkin`` instead. Override via "+ "CHAT_AGENT_MAX_TURNS env var (legacy CHAT_CLAUDE_AGENT_MAX_TURNS still "+ "accepted).",+ )+ sdk_step_checkin: int = Field(+ default=450,+ ge=1,+ le=10000,+ description="Model calls an SDK turn may make before its tools are "+ "refused for a check-in: the agent writes a progress summary and the "+ "user can choose to keep going, which starts a fresh count. The CLI's "+ "own max_turns sits a little above this as a backstop. Spend, not "+ "steps, is what bounds cost: the governor lands a turn at its budget.",)claude_agent_max_budget_usd: float = Field(default=10.0,diff --git a/autogpt_platform/backend/backend/copilot/continuation.py b/autogpt_platform/backend/backend/copilot/continuation.pynew file mode 100644index 0000000000..371c970ecb--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/continuation.py@@ -0,0 +1,53 @@+"""Picking a paused task back up: now ("Keep going") or when the budget resets.++Both run as a turn whose message is a system note rather than a user message:+the chat shows what happened in grey, and the model gets the same instruction+with the note marker taken off (see ``as_model_prompt``). Engines dedupe the+incoming message against the row the route saved, so the stored row and the+dispatched message have to be the same string.+"""++from typing import Any++from backend.copilot.constants import COPILOT_SYSTEM_PREFIX++KEEP_GOING_NOTE = (+ f"{COPILOT_SYSTEM_PREFIX} Keep going: pick the task up from the checkpoint above."+)+RESUMED_AT_RESET_NOTE = (+ f"{COPILOT_SYSTEM_PREFIX} Resumed after the budget reset: pick the task up "+ "from the checkpoint above."+)+++# Rows that start a turn rather than end one: not markers, so an error or+# a pause after one still gets its own row.+KICKOFF_NOTES = frozenset({KEEP_GOING_NOTE, RESUMED_AT_RESET_NOTE})+++def as_model_prompt(message: str) -> str:+ """The instruction a continuation note gives the model.++ A no-op for any other message, so engines can apply it to every+ non-user message they receive.+ """+ return message.removeprefix(COPILOT_SYSTEM_PREFIX).strip()+++def send_trailing_note_as_user(messages: list[dict[str, Any]]) -> None:+ """Send a note that ends a chat-completions request as the user turn.++ The note is stored as the chat's last row, an assistant one, so a request+ built from history ends on it. Sent as is, that is a prefill: Claude 4.6++ rejects it with a 400, and other models read the note as their own words.+ Only the request changes; the stored row stays a note.+ """+ if not messages or not _is_note(messages[-1]):+ return+ messages[-1] = {"role": "user", "content": as_model_prompt(messages[-1]["content"])}+++def _is_note(entry: dict[str, Any]) -> bool:+ if entry.get("role") != "assistant" or entry.get("tool_calls"):+ return False+ return str(entry.get("content") or "").startswith(COPILOT_SYSTEM_PREFIX)diff --git a/autogpt_platform/backend/backend/copilot/continuation_test.py b/autogpt_platform/backend/backend/copilot/continuation_test.pynew file mode 100644index 0000000000..92056a9afe--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/continuation_test.py@@ -0,0 +1,67 @@+from typing import Any++import pytest++from backend.copilot.continuation import (+ KEEP_GOING_NOTE,+ RESUMED_AT_RESET_NOTE,+ send_trailing_note_as_user,+)++_TASK = {"role": "user", "content": "Build the report"}+_TOOL_CALL = {+ "id": "call-1",+ "type": "function",+ "function": {"name": "run_agent", "arguments": "{}"},+}+++ ("note", "instruction"),+ [+ (KEEP_GOING_NOTE, "Keep going: pick the task up from the checkpoint above."),+ (+ RESUMED_AT_RESET_NOTE,+ "Resumed after the budget reset: pick the task up from the checkpoint "+ "above.",+ ),+ ],+)+def test_a_trailing_note_is_sent_as_the_user_turn(note: str, instruction: str):+ messages = [_TASK, {"role": "assistant", "content": note}]++ send_trailing_note_as_user(messages)++ assert messages == [_TASK, {"role": "user", "content": instruction}]+++ "last",+ [+ pytest.param({"role": "assistant", "content": "Two done."}, id="plain-text"),+ pytest.param(+ {+ "role": "assistant",+ "content": KEEP_GOING_NOTE,+ "tool_calls": [_TOOL_CALL],+ },+ id="tool-calls",+ ),+ pytest.param({"role": "assistant", "tool_calls": [_TOOL_CALL]}, id="no-text"),+ pytest.param(_TASK, id="user"),+ ],+)+def test_any_other_last_entry_is_left_as_it_is(last: dict[str, Any]):+ messages = [_TASK, {**last}]++ send_trailing_note_as_user(messages)++ assert messages == [_TASK, last]+++def test_an_empty_request_is_left_empty():+ messages: list[dict[str, Any]] = []++ send_trailing_note_as_user(messages)++ assert messages == []diff --git a/autogpt_platform/backend/backend/copilot/executor/processor.py b/autogpt_platform/backend/backend/copilot/executor/processor.pyindex 45aaa8372c..8580c66316 100644--- a/autogpt_platform/backend/backend/copilot/executor/processor.py+++ b/autogpt_platform/backend/backend/copilot/executor/processor.py@@ -568,6 +568,8 @@ class CoPilotProcessor:last_refresh = time.monotonic()refresh_interval = 30.0 # Refresh lock every 30 secondserror_msg = None+ # Re-published with the error: the client keys its dialogs on the code.+ error_code: str | None = Nonecredential_lease = Nonecost_context_stack = AsyncExitStack()scheduled_watch = ScheduledTurnWatch.for_entry(entry)@@ -795,6 +797,7 @@ class CoPilotProcessor:# not publish StreamError — that's done by mark_session_completed).if isinstance(chunk, StreamError):error_msg = chunk.errorText+ error_code = chunk.codebreakif scheduled_watch is not None:scheduled_watch.observe(chunk)@@ -841,7 +844,10 @@ class CoPilotProcessor:scheduled_watch.report(error_msg)try:await stream_registry.mark_session_completed(- entry.session_id, error_message=error_msg, turn_id=entry.turn_id+ entry.session_id,+ error_message=error_msg,+ error_code=error_code,+ turn_id=entry.turn_id,)except Exception as mark_err:log.error(f"Failed to mark session completed: {mark_err}")diff --git a/autogpt_platform/backend/backend/copilot/executor/processor_test.py b/autogpt_platform/backend/backend/copilot/executor/processor_test.pyindex 4a9e1a57fa..ea06ea1b9f 100644--- a/autogpt_platform/backend/backend/copilot/executor/processor_test.py+++ b/autogpt_platform/backend/backend/copilot/executor/processor_test.py@@ -818,7 +818,10 @@ async def test_unowned_expert_session_fails_before_engine_work() -> None:sdk_engine.assert_not_called()upsert.assert_not_awaited()mark_completed.assert_awaited_once_with(- "sess-1", error_message="expert is not owned by user", turn_id="turn-1"+ "sess-1",+ error_message="expert is not owned by user",+ error_code=None,+ turn_id="turn-1",)@@ -892,7 +895,10 @@ async def test_expert_tenancy_errors_publish_actionable_copy(baseline_engine.assert_not_called()sdk_engine.assert_not_called()mark_completed.assert_awaited_once_with(- "sess-1", error_message=expected_message, turn_id="turn-1"+ "sess-1",+ error_message=expected_message,+ error_code=None,+ turn_id="turn-1",)@@ -1130,6 +1136,7 @@ async def test_codex_release_failure_does_not_fail_successful_turn():mark_completed.assert_awaited_once_with("sess-codex",error_message=None,+ error_code=None,turn_id="turn-codex",)@@ -1184,6 +1191,7 @@ async def test_codex_checkpoint_failure_fails_closed_after_successful_turn():mark_completed.assert_awaited_once_with("sess-codex",error_message="codex_credential_checkpoint_failed",+ error_code=None,turn_id="turn-codex",)@@ -1219,6 +1227,7 @@ async def test_codex_queue_route_mismatch_fails_before_credential_acquire():mark_completed.assert_awaited_once_with("sess-codex",error_message="codex_session_route_mismatch",+ error_code=None,turn_id="turn-codex",)@@ -1261,6 +1270,7 @@ async def test_codex_entitlement_is_checked_before_credential_acquire():mark_completed.assert_awaited_once_with("sess-codex",error_message="Max plan required",+ error_code=None,turn_id="turn-codex",)@@ -1309,6 +1319,7 @@ async def test_codex_busy_credential_fails_closed_without_platform_fallback():mark_completed.assert_awaited_once_with("sess-codex",error_message="codex_credential_busy",+ error_code=None,turn_id="turn-codex",)diff --git a/autogpt_platform/backend/backend/copilot/executor/scheduled_turn_alert_test.py b/autogpt_platform/backend/backend/copilot/executor/scheduled_turn_alert_test.pyindex 60024378bd..ea4c94f86a 100644--- a/autogpt_platform/backend/backend/copilot/executor/scheduled_turn_alert_test.py+++ b/autogpt_platform/backend/backend/copilot/executor/scheduled_turn_alert_test.py@@ -358,6 +358,7 @@ async def test_executor_alerts_when_the_codex_checkpoint_fails_after_a_scheduledmark_completed.assert_awaited_once_with(entry.session_id,error_message="codex_credential_checkpoint_failed",+ error_code=None,turn_id=entry.turn_id,)sentry.capture_message.assert_called_once_with(diff --git a/autogpt_platform/backend/backend/copilot/executor/utils.py b/autogpt_platform/backend/backend/copilot/executor/utils.pyindex fdf68922f9..c9b59ef822 100644--- a/autogpt_platform/backend/backend/copilot/executor/utils.py+++ b/autogpt_platform/backend/backend/copilot/executor/utils.py@@ -835,6 +835,8 @@ async def schedule_chat_turn(message_id: str | None = None,message_metadata: dict[str, Any] | None = None,message_already_persisted: bool = False,+ require_fresh_slot: bool = False,+ dedupe_content: bool = True,is_user_message: bool = True,expert_id: str | None = None,session_origin: str | None = None,@@ -860,6 +862,9 @@ async def schedule_chat_turn(subscribe to the existing in-flight turn's stream instead of openinga new one). ``message_already_persisted`` re-dispatches an orphanedkickoff row only when this caller atomically admits the idle session.+ ``require_fresh_slot`` applies the same rule to a new message: when+ another turn already holds the session, return ``None`` before anything+ is saved. ``dedupe_content`` is passed to :func:`append_and_save_message`.Raises :class:`backend.copilot.active_turns.ConcurrentTurnLimitError`when the user is at the configured cap. Caller maps that to HTTP 429.@@ -890,7 +895,7 @@ async def schedule_chat_turn(message = VOICE_TURN_PREFIX + strip_server_injected_tags(message)async with acquire_turn_slot(user_id, session_id) as slot:- if message_already_persisted and not slot.admitted:+ if (message_already_persisted or require_fresh_slot) and not slot.admitted:return Noneis_duplicate = Falseif message and not message_already_persisted:@@ -901,7 +906,9 @@ async def schedule_chat_turn(metadata=message_metadata,)is_duplicate = (- await append_and_save_message(session_id, chat_message)+ await append_and_save_message(+ session_id, chat_message, dedupe_content=dedupe_content+ )) is Noneif not is_duplicate and is_user_message:track_user_message(diff --git a/autogpt_platform/backend/backend/copilot/executor/utils_test.py b/autogpt_platform/backend/backend/copilot/executor/utils_test.pyindex 5758b00477..78242373b1 100644--- a/autogpt_platform/backend/backend/copilot/executor/utils_test.py+++ b/autogpt_platform/backend/backend/copilot/executor/utils_test.py@@ -1,11 +1,13 @@"""Tests for CoPilot executor utils (queue config, message models, logging)."""-from contextlib import asynccontextmanager+from collections.abc import Iterator+from contextlib import asynccontextmanager, contextmanagerfrom unittest.mock import AsyncMock, MagicMock, patchimport pytestfrom backend.copilot import usage_activation_fixtures+from backend.copilot.continuation import KEEP_GOING_NOTEfrom backend.copilot.credential_selection import CredentialPinfrom backend.copilot.executor import utilsfrom backend.copilot.executor.utils import (@@ -366,6 +368,97 @@ async def test_schedule_chat_turn_leaves_an_already_saved_message_alone() -> Nonassert dispatch.await_args.kwargs["message"] == "kick it off"+@contextmanager+def _scheduling(*, admitted: bool) -> Iterator[tuple[AsyncMock, AsyncMock]]:+ """``schedule_chat_turn`` over a slot this call did or didn't admit."""+ slot = MagicMock(admitted=admitted)++ @asynccontextmanager+ async def acquire(*_args, **_kwargs):+ yield slot++ append = AsyncMock()+ dispatch = AsyncMock()+ with (+ patch.object(utils, "acquire_turn_slot", new=acquire),+ patch("backend.copilot.model.append_and_save_message", new=append),+ patch("backend.copilot.tracking.track_user_message", new=MagicMock()),+ patch.object(utils, "dispatch_turn", new=dispatch),+ ):+ yield append, dispatch+++async def test_schedule_chat_turn_needing_a_fresh_slot_saves_nothing_on_a_held_chat():+ # Another turn reserved the chat first, so this call only refreshed its+ # slot: saving the note and dispatching would run two turns at once.+ with _scheduling(admitted=False) as (append, dispatch):+ turn_id = await utils.schedule_chat_turn(+ session_id="s1",+ user_id="u1",+ message=KEEP_GOING_NOTE,+ is_user_message=False,+ require_fresh_slot=True,+ )++ assert turn_id is None+ append.assert_not_awaited()+ dispatch.assert_not_awaited()+++async def test_schedule_chat_turn_needing_a_fresh_slot_dispatches_when_admitted():+ with _scheduling(admitted=True) as (append, dispatch):+ turn_id = await utils.schedule_chat_turn(+ session_id="s1",+ user_id="u1",+ message=KEEP_GOING_NOTE,+ is_user_message=False,+ require_fresh_slot=True,+ )++ assert turn_id is not None+ append.assert_awaited_once()+ dispatch.assert_awaited_once()+++async def test_schedule_chat_turn_still_saves_and_dispatches_on_a_refreshed_slot():+ # /stream doesn't ask for a fresh slot; its behaviour is unchanged.+ with _scheduling(admitted=False) as (append, dispatch):+ turn_id = await utils.schedule_chat_turn(+ session_id="s1", user_id="u1", message="and the totals"+ )++ assert turn_id is not None+ append.assert_awaited_once()+ dispatch.assert_awaited_once()+++async def test_schedule_chat_turn_dedupes_content_by_default():+ with _scheduling(admitted=True) as (append, _dispatch):+ await utils.schedule_chat_turn(+ session_id="s1", user_id="u1", message="and the totals"+ )++ assert append.await_args.kwargs["dedupe_content"] is True+++async def test_schedule_chat_turn_passes_content_dedupe_off_to_the_save():+ with _scheduling(admitted=True) as (append, _dispatch):+ await utils.schedule_chat_turn(+ session_id="s1",+ user_id="u1",+ message=KEEP_GOING_NOTE,+ is_user_message=False,+ dedupe_content=False,+ )++ assert append.await_args.kwargs["dedupe_content"] is False++class TestNarrowPermissions:"""The envelope's tool set becomes the turn's whitelist. The encoding of"no tools at all" is the trap: it is a blacklist of every tool, and adiff --git a/autogpt_platform/backend/backend/copilot/markers.py b/autogpt_platform/backend/backend/copilot/markers.pyindex f10c7bbb68..e39bab1f02 100644--- a/autogpt_platform/backend/backend/copilot/markers.py+++ b/autogpt_platform/backend/backend/copilot/markers.py@@ -21,6 +21,7 @@ from backend.copilot.constants import (STREAM_ERROR_MARKER,STREAM_INCOMPLETE_MARKER,)+from backend.copilot.continuation import KICKOFF_NOTESfrom backend.copilot.model import ChatMessage, ChatSession@@ -44,6 +45,8 @@ def has_trailing_marker(session: ChatSession | None) -> bool:last = session.messages[-1]if last.role != "assistant" or not last.content:return False+ if last.content in KICKOFF_NOTES:+ return Falsereturn (is_error_marker(last)or last.content.startswith(COPILOT_SYSTEM_PREFIX)diff --git a/autogpt_platform/backend/backend/copilot/markers_test.py b/autogpt_platform/backend/backend/copilot/markers_test.pyindex 068225c2bb..a3dbc10b6e 100644--- a/autogpt_platform/backend/backend/copilot/markers_test.py+++ b/autogpt_platform/backend/backend/copilot/markers_test.py@@ -9,6 +9,7 @@ from backend.copilot.constants import (COPILOT_RETRYABLE_ERROR_PREFIX,STREAM_ERROR_MARKER,)+from backend.copilot.continuation import KEEP_GOING_NOTE, RESUMED_AT_RESET_NOTEfrom backend.copilot.markers import (append_error_marker,has_trailing_marker,@@ -92,6 +93,14 @@ class TestOneFailureIsOneCard:session = _session([ChatMessage(role="user", content="hi")])assert has_trailing_marker(session) is False+ @pytest.mark.parametrize("note", [KEEP_GOING_NOTE, RESUMED_AT_RESET_NOTE])+ def test_a_kickoff_note_never_blocks_the_marker(self, note: str) -> None:+ # A continuation that fails before writing anything still leaves+ # its error card: the note it started from isn't a marker.+ session = _session([ChatMessage(role="assistant", content=note)])+ assert has_trailing_marker(session) is False+ assert append_error_marker(session, "boom", retryable=True) is True+class TestRecognisingMarkers:def test_both_error_prefixes_count(self) -> None:diff --git a/autogpt_platform/backend/backend/copilot/model.py b/autogpt_platform/backend/backend/copilot/model.pyindex 3a66d486ef..a792e99b9d 100644--- a/autogpt_platform/backend/backend/copilot/model.py+++ b/autogpt_platform/backend/backend/copilot/model.py@@ -1266,7 +1266,7 @@ async def _save_session_to_db(async def append_and_save_message(- session_id: str, message: ChatMessage+ session_id: str, message: ChatMessage, *, dedupe_content: bool = True) -> ChatSession | None:"""Atomically append a message to a session and persist it.@@ -1274,6 +1274,11 @@ async def append_and_save_message(duplicate (idempotency guard). Callers must check for None and skip anydownstream work (e.g. enqueuing a new LLM turn) when a duplicate is detected.+ ``dedupe_content=False`` skips the trailing same-content collapse (layer 2+ below) for a message that is the same text every time it is sent, such as+ a continuation note; that caller has to rule out double sends itself. The+ id check (layer 1) still applies.+Uses _get_session_lock (Redis NX) to serialise concurrent writers across replicas.The idempotency check below provides a last-resort guard when the lock degrades."""@@ -1317,8 +1322,10 @@ async def append_and_save_message(# permanently. The fix is to ensure failed turns always write# an error/timeout assistant message so the session always ends# on an assistant turn.- if message.content is not None and is_message_duplicate(- session.messages, message.role, message.content+ if (+ dedupe_content+ and message.content is not None+ and is_message_duplicate(session.messages, message.role, message.content)):return None # duplicate — caller should skip enqueuediff --git a/autogpt_platform/backend/backend/copilot/model_test.py b/autogpt_platform/backend/backend/copilot/model_test.pyindex 806b63561b..80874f1443 100644--- a/autogpt_platform/backend/backend/copilot/model_test.py+++ b/autogpt_platform/backend/backend/copilot/model_test.py@@ -15,6 +15,8 @@ from openai.types.chat.chat_completion_message_tool_call_param import (from prisma.models import Expertfrom pytest_mock import MockerFixture+from backend.copilot.constants import COPILOT_SYSTEM_PREFIX+from backend.copilot.continuation import KEEP_GOING_NOTEfrom backend.data.redis_client import get_redis_asyncfrom backend.util.exceptions import NotFoundError, RedisError@@ -870,6 +872,111 @@ async def test_append_and_save_message_returns_none_for_duplicate(assert result is None+_PAUSE = f"{COPILOT_SYSTEM_PREFIX} Paused: checking in after a long run."++# Keep going was pressed once already, and that continuation wrote only+# assistant rows, or nothing at all: the earlier note is still in the chat's+# trailing assistant run.+_KEPT_GOING_ONCE = pytest.mark.parametrize(+ "rows_after_the_note",+ [+ pytest.param(["Two of three sections done.", _PAUSE], id="assistant-rows"),+ pytest.param([], id="nothing"),+ ],+)+++def _kept_going_once(rows_after_the_note: list[str]) -> ChatSession:+ return _make_session_with_messages(+ ChatMessage(role="user", content="Build the report"),+ ChatMessage(role="assistant", content=_PAUSE),+ ChatMessage(role="assistant", content=KEEP_GOING_NOTE),+ *(ChatMessage(role="assistant", content=row) for row in rows_after_the_note),+ )+++def _mock_session_store(mocker: MockerFixture, session: ChatSession) -> None:+ """The lock is taken, the session comes from cache, and writes succeed."""+ lock = mocker.AsyncMock()+ lock.acquire = mocker.AsyncMock(return_value=True)+ lock.release = mocker.AsyncMock()+ redis = mocker.MagicMock()+ redis.lock = mocker.MagicMock(return_value=lock)+ mocker.patch(+ "backend.copilot.model.get_redis_async",+ new_callable=mocker.AsyncMock,+ return_value=redis,+ )+ mocker.patch(+ "backend.copilot.model.get_chat_session",+ new_callable=mocker.AsyncMock,+ return_value=session,+ )+ mocker.patch(+ "backend.copilot.model._save_session_to_db", new_callable=mocker.AsyncMock+ )+ mocker.patch(+ "backend.copilot.model.chat_db",+ return_value=mocker.MagicMock(+ get_next_sequence=mocker.AsyncMock(return_value=len(session.messages))+ ),+ )+ mocker.patch(+ "backend.copilot.model.cache_chat_session", new_callable=mocker.AsyncMock+ )++[email protected](loop_scope="session")+@_KEPT_GOING_ONCE+async def test_append_and_save_message_collapses_a_repeated_note_by_default(+ mocker: MockerFixture, rows_after_the_note: list[str]+) -> None:+ session = _kept_going_once(rows_after_the_note)+ _mock_session_store(mocker, session)++ result = await append_and_save_message(+ session.session_id, ChatMessage(role="assistant", content=KEEP_GOING_NOTE)+ )++ assert result is None++[email protected](loop_scope="session")+@_KEPT_GOING_ONCE+async def test_append_and_save_message_saves_a_repeated_note_with_content_dedupe_off(+ mocker: MockerFixture, rows_after_the_note: list[str]+) -> None:+ session = _kept_going_once(rows_after_the_note)+ _mock_session_store(mocker, session)++ result = await append_and_save_message(+ session.session_id,+ ChatMessage(role="assistant", content=KEEP_GOING_NOTE),+ dedupe_content=False,+ )++ assert result is not None+ assert result.messages[-1].content == KEEP_GOING_NOTE+ assert [m.content for m in result.messages].count(KEEP_GOING_NOTE) == 2++[email protected](loop_scope="session")+async def test_append_and_save_message_keeps_the_id_check_with_content_dedupe_off(+ mocker: MockerFixture,+) -> None:+ session = _kept_going_once([])+ session.messages[-1].id = "click-1"+ _mock_session_store(mocker, session)++ result = await append_and_save_message(+ session.session_id,+ ChatMessage(id="click-1", role="assistant", content=KEEP_GOING_NOTE),+ dedupe_content=False,+ )++ assert result is None++@pytest.mark.asyncio(loop_scope="session")async def test_append_and_save_message_appends_new_message(mocker: MockerFixture,diff --git a/autogpt_platform/backend/backend/copilot/response_model.py b/autogpt_platform/backend/backend/copilot/response_model.pyindex 60e12e8ba6..5c62fbf132 100644--- a/autogpt_platform/backend/backend/copilot/response_model.py+++ b/autogpt_platform/backend/backend/copilot/response_model.py@@ -71,6 +71,9 @@ class ResponseType(str, Enum):# ``context_compaction`` tool row's JSON output.COMPACTION = "data-compaction"PROVIDER_FAILURE = "data-provider-failure"+ # The turn stopped on purpose at a budget or step limit; the client shows+ # a Paused card with ways to pick the task back up.+ TURN_PAUSED = "data-turn-paused"CHECKPOINT = "data-checkpoint"@@ -404,6 +407,23 @@ class StreamProviderFailure(StreamBaseResponse):return f"data: {json_dumps(data)}\n\n"+class StreamTurnPaused(StreamBaseResponse):+ """The turn landed at a budget or step limit instead of failing.++ Sent in place of the error frame those limits used to produce, just+ before ``StreamFinish``. The same payload is stored on a marker row so+ the card survives the stream (see ``backend.copilot.turn_pause``).+ """++ type: ResponseType = ResponseType.TURN_PAUSED+ pause: dict[str, Any] = Field(..., description="TurnPause.as_part() payload")++ def to_sse(self) -> str:+ """Emit as an AI SDK v5 data part."""+ data = {"type": self.type.value, "data": self.pause}+ return f"data: {json_dumps(data)}\n\n"++class StreamStatus(StreamBaseResponse):"""Transient status notification shown to the user during long operations.diff --git a/autogpt_platform/backend/backend/copilot/sdk/expert_tool_gate_test.py b/autogpt_platform/backend/backend/copilot/sdk/expert_tool_gate_test.pyindex 7232d1ae2a..63bf8ac580 100644--- a/autogpt_platform/backend/backend/copilot/sdk/expert_tool_gate_test.py+++ b/autogpt_platform/backend/backend/copilot/sdk/expert_tool_gate_test.py@@ -34,6 +34,7 @@ from claude_agent_sdk import ResultMessagefrom backend.copilot.model import ChatMessage, ChatSession, ChatSessionMetadatafrom backend.copilot.response_model import StreamStart+from backend.copilot.turn_budget import TurnBudget_SVC = "backend.copilot.sdk.service"@@ -146,6 +147,7 @@ def _make_patches(*, hire_experts_enabled: bool):use_e2b_sandbox=False,claude_agent_max_transient_retries=1,agent_max_turns=1000,+ sdk_step_checkin=1000,claude_agent_max_budget_usd=100.0,claude_agent_max_thinking_tokens=0,claude_agent_thinking_effort=None,@@ -156,8 +158,11 @@ def _make_patches(*, hire_experts_enabled: bool):),(f"{_SVC}.get_user_tier", dict(new_callable=AsyncMock, return_value=None)),(- f"{_SVC}._resolve_dynamic_max_budget_usd",- dict(new_callable=AsyncMock, return_value=100.0),+ f"{_SVC}.resolve_turn_budget",+ dict(+ new_callable=AsyncMock,+ return_value=TurnBudget(usd=100.0, limiting="task"),+ ),),(f"{_SVC}.drain_pending_safe", dict(new_callable=AsyncMock, return_value=[])),]diff --git a/autogpt_platform/backend/backend/copilot/sdk/governor.py b/autogpt_platform/backend/backend/copilot/sdk/governor.pynew file mode 100644index 0000000000..375ba08b3f--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/sdk/governor.py@@ -0,0 +1,171 @@+"""Decides when a running turn has to land, and records why it paused.++Two limits now end a turn on purpose instead of by accident: the money it may+spend (``TurnBudget``) and the number of model calls before a check-in. Close+to either, the governor switches to *landing*: every further tool call is+refused with a reason that asks for a checkpoint, so the agent stops at a+clean point and says where it got to. A warning alone doesn't do that --+models carried on past one in every test -- but a refused tool does, and the+refusal reaches the model inside the tool result on every transport.++The CLI's own limits stay, sized as backstops that fire only if landing+didn't: ``max_budget_usd`` above the budget (in the CLI's price units) and+``max_turns`` a little past the check-in.+"""++from claude_agent_sdk import AssistantMessage, Message, StreamEvent++from backend.copilot.turn_budget import TurnBudget+from backend.copilot.turn_pause import PauseReason, TurnPause++from .usage_meter import UsageMeter, cli_price_ratio++# Model calls past the check-in before the CLI's own max_turns stops the turn.+STEP_BACKSTOP_MARGIN = 50+# The CLI refuses to start a query whose budget is very small.+_CLI_MIN_BUDGET_USD = 0.5+# Spend held back for the checkpoint: about two calls at the latest call's+# price (the round whose tools are refused, then the reply), within bounds.+_RESERVE_MIN_USD = 0.02+_RESERVE_MAX_SHARE = 0.25+# Backstop headroom past the budget, paid plans only: a trial's allowance is+# a hard cap, so its backstop sits exactly on it.+_BACKSTOP_MIN_USD = 0.25+_BACKSTOP_SHARE = 0.15++_BUDGET_REFUSAL = (+ "Tool calls are paused: this task has used the budget it has for now, so "+ "every further tool call will be refused. Don't call any more tools. Reply "+ "to the user now with a short checkpoint in this form:\n"+ "Where I got to: <what is done>\n"+ "What's left: <what remains>\n"+ "Next step: <the exact next action>\n"+ "The user can pick the task up from there. Don't quote costs or budget "+ "figures."+)++_STEPS_REFUSAL = (+ "Tool calls are paused for a check-in: this task has run for {steps} "+ "steps, so every further tool call will be refused. Don't call any more "+ "tools. Reply to the user now with a short update in this form:\n"+ "Progress: <what is done so far>\n"+ "Next step: <the exact next action>\n"+ "The user can tell you to keep going from there."+)+++class TurnGovernor:+ """Per-turn spend and step tracking, and the landing decision.++ Lives for the whole turn, across retries: money spent on an attempt that+ failed was still spent.+ """++ def __init__(self, *, budget: TurnBudget | None, step_checkin: int) -> None:+ # None on routes the account doesn't pay for (a ChatGPT subscription):+ # those only get the step check-in.+ self.budget = budget+ self.step_checkin = step_checkin+ self.meter = UsageMeter()+ self.landing: PauseReason | None = None+ # Set once a tool call was refused: from then on the turn is landing+ # for real. A threshold crossed by the final answer, with no tool+ # call after it, finished the task rather than pausing it.+ self.refused = False+ # Set once the turn has closed as a pause: queued follow-ups then wait+ # for the user to pick the task back up instead of auto-continuing.+ self.paused = False++ def observe(self, message: Message) -> None:+ if isinstance(message, AssistantMessage):+ self.meter.observe(message)+ elif isinstance(message, StreamEvent):+ self.meter.observe_stream(message)+ else:+ return+ self._update()++ def refusal(self) -> str | None:+ """Why the next tool call is refused, or None to let it run."""+ self._update()+ if self.landing is None:+ return None+ self.refused = True+ if self.landing == "steps":+ return _STEPS_REFUSAL.format(steps=self.meter.main_calls)+ return _BUDGET_REFUSAL++ def pause(self, result_subtype: str, *, wrote_checkpoint: bool) -> TurnPause | None:+ """The pause this turn ended in, or None if it finished or failed.++ A backstop subtype means the CLI stopped the turn itself, before (or+ while) the agent wrote its checkpoint, so none is claimed. A genuine+ error while landing stays an error, and a turn that ended normally+ without a refused tool simply finished: only a turn that ended+ normally after its tools were refused is a pause.+ """+ if result_subtype == "error_max_turns":+ return self._pause("steps", checkpoint_saved=False)+ if result_subtype == "error_max_budget_usd":+ reason = self.budget.limiting if self.budget is not None else "task"+ return self._pause(reason, checkpoint_saved=False)+ if self.landing is None or not self.refused:+ return None+ if result_subtype != "success":+ return None+ return self._pause(self.landing, checkpoint_saved=wrote_checkpoint)++ def hangup_pause(self) -> TurnPause | None:+ """The pause for a turn the CLI ended without a result while landing.++ A hangup is otherwise a crash and keeps its own notice; once the tools+ were being refused it is the landing that ended, so it still pauses.+ """+ if self.landing is None or not self.refused:+ return None+ return self._pause(self.landing, checkpoint_saved=False)++ @property+ def landed(self) -> bool:+ """True once a tool was refused, whether or not it closed as a pause."""+ return self.paused or self.refused++ @property+ def cli_max_turns(self) -> int:+ return self.step_checkin + STEP_BACKSTOP_MARGIN++ def cli_budget_usd(self, model: str | None) -> float | None:+ """The CLI's ``max_budget_usd``: the budget plus headroom, in CLI units."""+ if self.budget is None:+ return None+ if self.budget.hard_cap:+ return self.budget.usd * cli_price_ratio(model)+ headroom = max(_BACKSTOP_MIN_USD, _BACKSTOP_SHARE * self.budget.usd)+ cli_usd = (self.budget.usd + headroom) * cli_price_ratio(model)+ return max(_CLI_MIN_BUDGET_USD, cli_usd)++ def _update(self) -> None:+ if self.landing is not None:+ return+ if self.meter.main_calls >= self.step_checkin:+ self.landing = "steps"+ elif self.budget is not None and self.meter.spent_usd >= (+ self.budget.usd - self._reserve_usd(self.budget)+ ):+ self.landing = self.budget.limiting++ def _reserve_usd(self, budget: TurnBudget) -> float:+ latest = 2 * self.meter.last_main_call_usd+ return min(_RESERVE_MAX_SHARE * budget.usd, max(_RESERVE_MIN_USD, latest))++ def _pause(self, reason: PauseReason, *, checkpoint_saved: bool) -> TurnPause:+ self.paused = True+ resets_at = None+ if reason in ("daily", "weekly") and self.budget is not None:+ resets_at = self.budget.resets_at+ return TurnPause(+ reason=reason,+ resets_at=resets_at,+ steps=self.meter.main_calls if reason == "steps" else None,+ checkpoint_saved=checkpoint_saved,+ )diff --git a/autogpt_platform/backend/backend/copilot/sdk/governor_test.py b/autogpt_platform/backend/backend/copilot/sdk/governor_test.pynew file mode 100644index 0000000000..96ab4b3529--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/sdk/governor_test.py@@ -0,0 +1,298 @@+"""Landing decisions: when a turn's tools are refused, and how it pauses."""++from datetime import UTC, datetime++import pytest+from claude_agent_sdk import AssistantMessage, StreamEvent, TextBlock++from backend.copilot.turn_budget import TurnBudget+from backend.copilot.turn_pause import TurnPause++from .governor import STEP_BACKSTOP_MARGIN, TurnGovernor+from .security_hooks import create_security_hooks+from .usage_meter import UsageMeter, cli_price_ratio++RESETS = datetime(2026, 10, 15, tzinfo=UTC)+KIMI = "moonshotai/kimi-k2.6"+++class _Meter(UsageMeter):+ """A meter whose readings are set directly, so thresholds are exact."""++ def __init__(self, spent: float = 0.0, calls: int = 0, last: float = 0.0):+ super().__init__()+ self.spent = spent+ self.calls = calls+ self.last = last++ def observe(self, message: AssistantMessage) -> None:+ self.calls += 1++ @property+ def spent_usd(self) -> float:+ return self.spent++ @property+ def main_calls(self) -> int:+ return self.calls++ @property+ def last_main_call_usd(self) -> float:+ return self.last+++def _governor(+ budget: TurnBudget | None = TurnBudget(usd=1.0, limiting="daily", resets_at=RESETS),+ *,+ checkin: int = 450,+ meter: _Meter | None = None,+) -> TurnGovernor:+ governor = TurnGovernor(budget=budget, step_checkin=checkin)+ governor.meter = meter or _Meter()+ return governor+++def test_tools_run_while_the_budget_has_room():+ governor = _governor(meter=_Meter(spent=0.5, last=0.05))+ assert governor.refusal() is None+ assert governor.landing is None+++def test_lands_with_room_left_for_the_checkpoint():+ # Reserve is two calls at the latest call's price (2 x 0.1), so a $1.00+ # budget lands at $0.80 rather than letting the checkpoint overspend.+ meter = _Meter(spent=0.79, last=0.1)+ governor = _governor(meter=meter)+ assert governor.refusal() is None+ meter.spent = 0.8+ refusal = governor.refusal()+ assert refusal is not None+ assert "Where I got to:" in refusal+ assert governor.landing == "daily"+++def test_reserve_never_takes_more_than_a_quarter_of_the_budget():+ # One expensive call can't make a small budget land before it starts.+ # The reserve caps at 25% of $0.40, so this lands near $0.30, not at $0.+ meter = _Meter(spent=0.29, last=0.5)+ governor = _governor(TurnBudget(usd=0.4, limiting="weekly"), meter=meter)+ assert governor.refusal() is None+ meter.spent = 0.31+ assert governor.refusal() is not None+ assert governor.landing == "weekly"+++def test_landing_is_sticky():+ meter = _Meter(spent=0.99, last=0.01)+ governor = _governor(meter=meter)+ assert governor.refusal() is not None+ meter.spent = 0.0+ assert governor.refusal() is not None+++def test_step_checkin_lands_with_a_progress_request():+ governor = _governor(checkin=3, meter=_Meter(calls=2))+ assert governor.refusal() is None+ governor.observe(AssistantMessage(content=[TextBlock(text="x")], model="m"))+ refusal = governor.refusal()+ assert governor.landing == "steps"+ assert refusal is not None+ assert "3 steps" in refusal+ assert "Progress:" in refusal+++def test_chatgpt_turns_get_the_step_checkin_only():+ governor = _governor(budget=None, meter=_Meter(spent=999.0, calls=10))+ assert governor.refusal() is None+ assert governor.cli_budget_usd(KIMI) is None+++def test_a_turn_that_never_landed_does_not_pause():+ governor = _governor()+ assert governor.pause("success", wrote_checkpoint=True) is None+ assert governor.paused is False+++def test_a_landed_turn_pauses_with_its_window_and_reset():+ governor = _governor(meter=_Meter(spent=1.0))+ governor.refusal()+ pause = governor.pause("success", wrote_checkpoint=True)+ assert pause == TurnPause(reason="daily", resets_at=RESETS)+ assert governor.paused is True++[email protected]("subtype", ["error", "error_during_execution"])+def test_a_genuine_error_while_landing_stays_an_error(subtype):+ governor = _governor(meter=_Meter(spent=1.0))+ governor.refusal()+ assert governor.pause(subtype, wrote_checkpoint=True) is None+ assert governor.paused is False+++def test_a_landed_turn_without_a_reply_says_so():+ governor = _governor(meter=_Meter(spent=1.0))+ governor.refusal()+ pause = governor.pause("success", wrote_checkpoint=False)+ assert pause is not None+ assert pause.checkpoint_saved is False+++def test_cli_turn_backstop_is_a_step_pause_without_a_checkpoint():+ governor = _governor(checkin=450, meter=_Meter(calls=500))+ pause = governor.pause("error_max_turns", wrote_checkpoint=True)+ assert pause == TurnPause(reason="steps", steps=500, checkpoint_saved=False)+++ "budget,reason",+ [+ (TurnBudget(usd=2.0, limiting="weekly", resets_at=RESETS), "weekly"),+ (TurnBudget(usd=10.0, limiting="task"), "task"),+ (TurnBudget(usd=0.5, limiting="trial", hard_cap=True), "trial"),+ (None, "task"),+ ],+)+def test_cli_budget_backstop_names_the_limiting_window(budget, reason):+ governor = _governor(budget)+ pause = governor.pause("error_max_budget_usd", wrote_checkpoint=True)+ assert pause is not None+ assert pause.reason == reason+ assert pause.checkpoint_saved is False+ assert pause.resets_at == (RESETS if reason == "weekly" else None)+++def test_a_cli_hangup_only_pauses_a_turn_that_was_landing():+ running = _governor(meter=_Meter(spent=0.1, last=0.01))+ assert running.hangup_pause() is None+ assert running.landed is False++ landing = _governor(meter=_Meter(spent=1.0))+ landing.refusal()+ assert landing.landed is True+ pause = landing.hangup_pause()+ assert pause == TurnPause(reason="daily", resets_at=RESETS, checkpoint_saved=False)+ assert landing.paused is True+++def test_cli_max_turns_sits_past_the_checkin():+ assert _governor(checkin=450).cli_max_turns == 450 + STEP_BACKSTOP_MARGIN+++def test_paid_backstop_adds_headroom_above_the_budget():+ governor = _governor(TurnBudget(usd=2.0, limiting="daily"))+ # max($0.25, 15% of $2.00) of headroom on Anthropic, whose CLI units are ours.+ assert governor.cli_budget_usd("anthropic/claude-sonnet-5") == pytest.approx(2.3)+++def test_paid_backstop_never_drops_below_what_the_cli_accepts():+ governor = _governor(TurnBudget(usd=0.05, limiting="daily"))+ assert governor.cli_budget_usd("anthropic/claude-sonnet-5") == 0.5+++def test_trial_backstop_sits_exactly_on_the_allowance():+ governor = _governor(TurnBudget(usd=0.05, limiting="trial", hard_cap=True))+ assert governor.cli_budget_usd("anthropic/claude-sonnet-5") == pytest.approx(0.05)+++def test_backstop_is_scaled_into_cli_units_for_non_anthropic_models():+ # The CLI counts Kimi at Sonnet prices; an unscaled backstop would stop+ # the turn at a fraction of its real budget.+ governor = _governor(TurnBudget(usd=2.0, limiting="daily"))+ assert cli_price_ratio(KIMI) > 4+ assert governor.cli_budget_usd(KIMI) == pytest.approx(2.3 * cli_price_ratio(KIMI))+++def _deny_reason(result: dict) -> str | None:+ output = result.get("hookSpecificOutput", {})+ if output.get("permissionDecision") != "deny":+ return None+ return output.get("permissionDecisionReason")+++async def test_pre_tool_hook_refuses_every_tool_once_landing():+ governor = _governor(meter=_Meter(spent=1.0))+ hooks = create_security_hooks(user_id="u1", sdk_cwd="/tmp/w", governor=governor)+ pre = hooks["PreToolUse"][0].hooks[0]+ for tool in ("mcp__copilot__run_block", "Agent", "WebSearch"):+ result = await pre(+ {"tool_name": tool, "tool_input": {"prompt": "x"}},+ tool_use_id=f"tu-{tool}",+ context={},+ )+ reason = _deny_reason(dict(result))+ assert reason is not None and "Where I got to:" in reason+++async def test_pre_tool_hook_lets_tools_run_below_the_limits():+ governor = _governor(meter=_Meter(spent=0.1, last=0.01))+ hooks = create_security_hooks(user_id="u1", sdk_cwd="/tmp/w", governor=governor)+ pre = hooks["PreToolUse"][0].hooks[0]+ result = await pre(+ {"tool_name": "mcp__copilot__run_block", "tool_input": {}},+ tool_use_id="tu-1",+ context={},+ )+ assert _deny_reason(dict(result)) is None+++def test_a_trial_limited_by_the_task_cap_gets_no_headroom():+ # The task cap binds, but the money is the trial's: the backstop sits+ # exactly on the budget, with no headroom and no CLI floor.+ governor = _governor(TurnBudget(usd=10.0, limiting="task", hard_cap=True))+ assert governor.cli_budget_usd("anthropic/claude-sonnet-5") == pytest.approx(10.0)+ small = _governor(TurnBudget(usd=0.1, limiting="daily", hard_cap=True))+ assert small.cli_budget_usd("anthropic/claude-sonnet-5") == pytest.approx(0.1)+++def test_a_threshold_crossed_by_the_final_answer_finishes_the_task():+ # The answer that crosses the budget calls no tool after it: nothing was+ # refused, so the task finished rather than paused.+ governor = _governor(meter=_Meter(spent=1.0))+ governor.observe(AssistantMessage(content=[TextBlock(text="All done")], model="m"))+ assert governor.landing == "daily"+ assert governor.pause("success", wrote_checkpoint=True) is None+ assert governor.hangup_pause() is None+ assert governor.landed is False+++def test_a_refused_tool_makes_the_turn_a_landing_one():+ governor = _governor(meter=_Meter(spent=1.0))+ assert governor.refusal() is not None+ assert governor.landed is True+++def _stream(event: dict, parent: str | None = None) -> StreamEvent:+ return StreamEvent(uuid="u", session_id="s", event=event, parent_tool_use_id=parent)+++def test_the_real_output_count_from_the_stream_lands_the_turn():+ # The assistant message carries only the placeholder output count; the+ # response's message_delta carries the real one, and that is what lands.+ governor = TurnGovernor(+ budget=TurnBudget(usd=0.1, limiting="daily", resets_at=RESETS), step_checkin=450+ )+ usage = {"input_tokens": 100, "output_tokens": 1}+ governor.observe(+ _stream(+ {+ "type": "message_start",+ "message": {"id": "m1", "model": "claude-sonnet-4-5", "usage": usage},+ }+ )+ )+ governor.observe(+ AssistantMessage(+ content=[TextBlock(text="x")],+ model="claude-sonnet-4-5",+ usage=usage,+ message_id="m1",+ )+ )+ assert governor.refusal() is None+ governor.observe(+ _stream({"type": "message_delta", "usage": {"output_tokens": 10_000}})+ )+ assert governor.refusal() is not None+ assert governor.landing == "daily"diff --git a/autogpt_platform/backend/backend/copilot/sdk/p0_guardrails_test.py b/autogpt_platform/backend/backend/copilot/sdk/p0_guardrails_test.pyindex 1f29e56d9a..38fb2eecb3 100644--- a/autogpt_platform/backend/backend/copilot/sdk/p0_guardrails_test.py+++ b/autogpt_platform/backend/backend/copilot/sdk/p0_guardrails_test.py@@ -236,6 +236,12 @@ class TestConfigDefaults:cfg = _make_config()assert cfg.agent_max_turns == 100+ def test_sdk_step_checkin_default(self):+ # The SDK path checks in after a long run instead of stopping at+ # ``agent_max_turns``; "Keep going" starts a fresh count.+ cfg = _make_config()+ assert cfg.sdk_step_checkin == 450+def test_max_budget_usd_default(self):cfg = _make_config()assert cfg.claude_agent_max_budget_usd == 10.0diff --git a/autogpt_platform/backend/backend/copilot/sdk/response_adapter.py b/autogpt_platform/backend/backend/copilot/sdk/response_adapter.pyindex 070270af49..187c0ce70b 100644--- a/autogpt_platform/backend/backend/copilot/sdk/response_adapter.py+++ b/autogpt_platform/backend/backend/copilot/sdk/response_adapter.py@@ -44,8 +44,11 @@ from backend.copilot.response_model import (StreamToolInputAvailable,StreamToolInputStart,StreamToolOutputAvailable,+ StreamTurnPaused,)+from backend.copilot.turn_pause import TurnPause+from .governor import TurnGovernorfrom .tool_adapter import MCP_TOOL_PREFIX, pop_pending_tool_outputfrom .tool_display import strip_display_token@@ -95,6 +98,10 @@ class SDKResponseAdapter:# so the empty-completion guard doesn't false-fire on a benign empty# trailing ResultMessage when the prior attempt already streamed content.self.prior_attempt_emitted_visible_content = False+ # Set by the service on every adapter it builds. When the turn ends+ # landed -- tools refused at a budget or step limit, or a CLI backstop+ # firing -- the adapter closes it as a pause rather than an error.+ self.governor: TurnGovernor | None = None# Maps tool_use_id → {"name": <tool name>, "input": <call args>}.# ``input`` is carried so the response adapter can pop the matching# stashed output via ``_output_key`` when several same-name tools run@@ -489,8 +496,11 @@ class SDKResponseAdapter:# a marker so the chat history just stops mid-task).if self._should_surface_empty_completion(sdk_message, had_orphan_tool_use):self._finish_step(responses)+ landed = self._landed(sdk_message.subtype, wrote_checkpoint=False)responses.append(- StreamError(+ StreamTurnPaused(pause=landed.as_part())+ if landed is not None+ else StreamError(errorText="The model returned an empty response.",code="empty_completion",)@@ -520,6 +530,11 @@ class SDKResponseAdapter:# prompt rule (``_USER_FOLLOW_UP_NOTE``'s closing clause)# asks the model to always end with text, but we can't rely# on it for extended_thinking / edge cases.+ #+ # Only text the model wrote counts as a landing turn's checkpoint:+ # a closing line filled in here is not one, even promoted from+ # its thinking, so it is read before the fallback below.+ wrote_checkpoint = self._text_since_last_tool_resultif (self._any_tool_results_seenand not self._text_since_last_tool_result@@ -565,11 +580,18 @@ class SDKResponseAdapter:delta=fallback_text,))+ self._text_since_last_tool_result = Trueself.end_open_blocks(responses)# Close the step before finishing.self._finish_step(responses)- if sdk_message.subtype == "success":+ landed = self._landed(+ sdk_message.subtype, wrote_checkpoint=wrote_checkpoint+ )+ if landed is not None:+ responses.append(StreamTurnPaused(pause=landed.as_part()))+ responses.append(StreamFinish())+ elif sdk_message.subtype == "success":responses.append(StreamFinish())elif sdk_message.subtype in ("error",@@ -622,6 +644,14 @@ class SDKResponseAdapter:return responses+ def _landed(+ self, result_subtype: str, *, wrote_checkpoint: bool+ ) -> TurnPause | None:+ """The pause this turn ended in, or None when it didn't land."""+ if self.governor is None:+ return None+ return self.governor.pause(result_subtype, wrote_checkpoint=wrote_checkpoint)+def _should_surface_empty_completion(self,msg: ResultMessage,diff --git a/autogpt_platform/backend/backend/copilot/sdk/response_adapter_landing_test.py b/autogpt_platform/backend/backend/copilot/sdk/response_adapter_landing_test.pynew file mode 100644index 0000000000..58c2de8b59--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/sdk/response_adapter_landing_test.py@@ -0,0 +1,135 @@+"""A turn that lands at a limit closes as a pause, not as an error."""++from datetime import UTC, datetime++from claude_agent_sdk import (+ AssistantMessage,+ ResultMessage,+ SystemMessage,+ TextBlock,+ ToolUseBlock,+)++from backend.copilot.response_model import (+ StreamBaseResponse,+ StreamError,+ StreamFinish,+ StreamTextDelta,+ StreamTurnPaused,+)+from backend.copilot.turn_budget import TurnBudget++from .governor import TurnGovernor+from .response_adapter import SDKResponseAdapter+from .tool_adapter import MCP_TOOL_PREFIX++RESETS = datetime(2026, 10, 15, tzinfo=UTC)+++def _adapter(*, landed: bool) -> SDKResponseAdapter:+ # A $0 budget is spent before the first call, so one refusal lands it.+ budget = TurnBudget(usd=0.0 if landed else 5.0, limiting="daily", resets_at=RESETS)+ governor = TurnGovernor(budget=budget, step_checkin=450)+ governor.refusal()+ adapter = SDKResponseAdapter(message_id="m1", session_id="s1")+ adapter.governor = governor+ adapter.convert_message(SystemMessage(subtype="init", data={}))+ return adapter+++def _result(subtype: str, *, output_tokens: int = 10) -> ResultMessage:+ return ResultMessage(+ subtype=subtype,+ duration_ms=100,+ duration_api_ms=50,+ is_error=subtype != "success",+ num_turns=3,+ session_id="s1",+ result="",+ usage={"output_tokens": output_tokens},+ )+++def _paused(events: list[StreamBaseResponse]) -> dict:+ parts = [e for e in events if isinstance(e, StreamTurnPaused)]+ assert len(parts) == 1, events+ assert isinstance(events[-1], StreamFinish)+ assert parts[0] is events[-2]+ assert not any(isinstance(e, StreamError) for e in events)+ return parts[0].pause+++def test_landed_turn_with_a_checkpoint_pauses_after_the_reply():+ adapter = _adapter(landed=True)+ reply = adapter.convert_message(+ AssistantMessage(+ content=[TextBlock(text="Where I got to: sent 5 of 12.")], model="test"+ )+ )+ assert any(isinstance(e, StreamTextDelta) for e in reply)+ pause = _paused(adapter.convert_message(_result("success")))+ assert pause["reason"] == "daily"+ assert pause["checkpoint_saved"] is True+ assert pause["resets_at"].startswith("2026-10-15")+++def test_budget_backstop_pauses_instead_of_erroring():+ pause = _paused(+ _adapter(landed=False).convert_message(_result("error_max_budget_usd"))+ )+ assert pause["reason"] == "daily"+ assert pause["checkpoint_saved"] is False+++def test_turn_backstop_is_a_step_checkin():+ pause = _paused(_adapter(landed=False).convert_message(_result("error_max_turns")))+ assert pause["reason"] == "steps"+ assert pause["checkpoint_saved"] is False+++def test_landed_turn_with_no_reply_pauses_rather_than_reporting_empty():+ adapter = _adapter(landed=True)+ adapter.convert_message(+ AssistantMessage(+ content=[+ ToolUseBlock(id="t-1", name=f"{MCP_TOOL_PREFIX}run_block", input={})+ ],+ model="test",+ )+ )+ adapter.convert_message(AssistantMessage(content=[], model="test"))+ pause = _paused(adapter.convert_message(_result("success", output_tokens=0)))+ assert pause["checkpoint_saved"] is False+++def test_an_error_while_landing_is_still_reported_as_an_error():+ adapter = _adapter(landed=True)+ events = adapter.convert_message(_result("error"))+ assert any(isinstance(e, StreamError) for e in events)+ assert not any(isinstance(e, StreamTurnPaused) for e in events)+++def test_a_turn_that_did_not_land_finishes_normally():+ adapter = _adapter(landed=False)+ adapter.convert_message(+ AssistantMessage(content=[TextBlock(text="All done.")], model="test")+ )+ events = adapter.convert_message(_result("success"))+ assert isinstance(events[-1], StreamFinish)+ assert not any(isinstance(e, StreamTurnPaused) for e in events)+++def test_a_closing_line_filled_in_for_a_thinking_only_reply_is_not_a_checkpoint():+ # The model answered the refused tools with thinking only, twice; the+ # adapter shows a closing line (its thinking, or a placeholder) so the+ # turn visibly ends. The model wrote no checkpoint, so the pause says so.+ adapter = _adapter(landed=True)+ adapter._last_thinking_content = "Maybe I should stop here."+ adapter.thinking_only_reprompted = True+ adapter._any_tool_results_seen = True+ adapter._text_since_last_tool_result = False++ events = adapter.convert_message(_result("success"))++ assert any(isinstance(e, StreamTextDelta) for e in events)+ assert _paused(events)["checkpoint_saved"] is Falsediff --git a/autogpt_platform/backend/backend/copilot/sdk/retry_scenarios_test.py b/autogpt_platform/backend/backend/copilot/sdk/retry_scenarios_test.pyindex d69beeec5a..7eaf640307 100644--- a/autogpt_platform/backend/backend/copilot/sdk/retry_scenarios_test.py+++ b/autogpt_platform/backend/backend/copilot/sdk/retry_scenarios_test.py@@ -53,6 +53,7 @@ from backend.copilot.transcript import (_messages_to_transcript,_transcript_to_messages,)+from backend.copilot.turn_budget import TurnBudgetfrom backend.util import jsonfrom .conftest import build_test_transcript as _build_transcript@@ -1067,6 +1068,7 @@ def _make_sdk_patches(use_e2b_sandbox=False,claude_agent_max_transient_retries=1,agent_max_turns=1000,+ sdk_step_checkin=1000,claude_agent_max_budget_usd=100.0,claude_agent_max_thinking_tokens=0,claude_agent_autocompact_pct_override=0,@@ -1080,13 +1082,15 @@ def _make_sdk_patches(),),(f"{_SVC}.get_user_tier", dict(new_callable=AsyncMock, return_value=None)),- # Bypass dynamic budget resolution — the helper hits Redis via- # get_global_rate_limits / get_remaining_usd_budget, which the- # retry_scenarios mocks don't stand up. Static cap is fine for- # retry-flow assertions (covered directly in service_test).+ # Bypass budget resolution — it hits Redis via the rate-limit+ # helpers, which the retry_scenarios mocks don't stand up. The task+ # cap is fine for retry-flow assertions (covered in turn_budget_test).(- f"{_SVC}._resolve_dynamic_max_budget_usd",- dict(new_callable=AsyncMock, return_value=100.0),+ f"{_SVC}.resolve_turn_budget",+ dict(+ new_callable=AsyncMock,+ return_value=TurnBudget(usd=100.0, limiting="task"),+ ),),# Stub pending-message drain so retry tests don't hit Redis.# Returns an empty list → no mid-turn injection happens.diff --git a/autogpt_platform/backend/backend/copilot/sdk/security_hooks.py b/autogpt_platform/backend/backend/copilot/sdk/security_hooks.pyindex f9b6bec12f..9365059aad 100644--- a/autogpt_platform/backend/backend/copilot/sdk/security_hooks.py+++ b/autogpt_platform/backend/backend/copilot/sdk/security_hooks.py@@ -19,6 +19,7 @@ from backend.copilot.context import ()from backend.copilot.pending_messages import drain_and_format_for_injection+from .governor import TurnGovernorfrom .tool_adapter import (BLOCKED_TOOLS,DANGEROUS_PATTERNS,@@ -74,6 +75,21 @@ def _deny(reason: str) -> dict[str, Any]:}+def _landing_refusal(governor: TurnGovernor | None) -> str | None:+ """The governor's refusal for the next tool call, if it is landing the turn.++ A governor failure lets the call run: the CLI's own budget and turn limits+ still stop the turn, and failing every tool call would strand it instead.+ """+ if governor is None:+ return None+ try:+ return governor.refusal()+ except Exception:+ logger.warning("[SDK] Turn governor could not decide a refusal", exc_info=True)+ return None++def _validate_workspace_path(tool_name: str, tool_input: dict[str, Any], sdk_cwd: str | None) -> dict[str, Any]:@@ -195,6 +211,7 @@ def create_security_hooks(max_subtasks: int = 3,on_compact: Callable[[str], None] | None = None,tool_display_bridge: SDKToolDisplayBridge | None = None,+ governor: TurnGovernor | None = None,) -> dict[HookEvent, list[HookMatcher]]:"""Create the security hooks configuration for Claude Agent SDK.@@ -212,6 +229,9 @@ def create_security_hooks(max_subtasks: Maximum concurrent sub-agent spawns allowed per sessionon_compact: Callback invoked when SDK starts compacting context.Receives the transcript_path from the hook input.+ governor: The turn's spend and step governor. Once it is landing the+ turn, every tool call -- the main agent's and its sub-agents' --+ is refused with a reason that asks for a checkpoint.Returns:Hooks configuration dict for ClaudeAgentOptions@@ -238,6 +258,16 @@ def create_security_hooks(tool_name = cast(str, input_data.get("tool_name", ""))tool_input = cast(dict[str, Any], input_data.get("tool_input", {}))+ refusal = _landing_refusal(governor)+ if governor is not None and refusal is not None:+ logger.info(+ "[SDK] Refusing %s: turn is landing (%s), user=%s",+ tool_name,+ governor.landing,+ user_id,+ )+ return cast(SyncHookJSONOutput, _deny(refusal))+# Rate-limit sub-agent spawns per session.# The SDK CLI renamed "Task" → "Agent" in v2.x; handle both.if tool_name in _SUBAGENT_TOOLS:diff --git a/autogpt_platform/backend/backend/copilot/sdk/service.py b/autogpt_platform/backend/backend/copilot/sdk/service.pyindex 4bed142dab..1418ea2a77 100644--- a/autogpt_platform/backend/backend/copilot/sdk/service.py+++ b/autogpt_platform/backend/backend/copilot/sdk/service.py@@ -53,6 +53,7 @@ from backend.copilot.model_router import (resolve_model_route,)from backend.copilot.budget_signal import build_turn_budget_block+from backend.copilot.continuation import as_model_promptfrom backend.copilot.feedback_db import RATEABLE_ROLESfrom backend.copilot.graphiti.context import fetch_warm_contextfrom backend.copilot.markers import append_error_marker@@ -61,12 +62,13 @@ from backend.copilot.segments import Segment, stamp_segmentfrom backend.copilot.graphiti.ingest import enqueue_conversation_turnfrom backend.copilot.learning.capture import capture_chat_turn, find_initiating_user_rowfrom backend.copilot.sdk.codex_compat_gateway import CodexAnthropicGateway-from backend.copilot.sdk.trial_budget import resolve_trial_sdk_budgetfrom backend.copilot.sdk.cost_tracking import (TokenUsage as _TokenUsage,read_cli_session_usage,)from backend.copilot.tool_display import tool_calls_for_provider+from backend.copilot.turn_budget import resolve_turn_budget+from backend.copilot.turn_pause import TurnPause, append_pause_markerfrom backend.data.db_accessors import chat_dbfrom backend.data.redis_client import get_redis_asyncfrom backend.executor.cluster_lock import AsyncClusterLock@@ -146,11 +148,7 @@ from ..prompting import (get_graphiti_supplement,get_sdk_supplement,)-from ..rate_limit import (- get_global_rate_limits,- get_remaining_usd_budget,- get_user_tier,-)+from ..rate_limit import get_user_tierfrom ..response_model import (StreamBaseResponse,StreamCheckpoint,@@ -173,6 +171,7 @@ from ..response_model import (StreamToolDisplayAvailable,StreamToolInputAvailable,StreamToolOutputAvailable,+ StreamTurnPaused,StreamUsage,)from ..builder_context import (@@ -228,6 +227,7 @@ from .compaction import (transcript_stats,)from .env import build_sdk_env # noqa: F401 — re-export for backward compat+from .governor import TurnGovernorfrom .openrouter_cost import TurnGenerations, record_turn_cost_from_openrouterfrom .response_adapter import SDKResponseAdapterfrom .security_hooks import create_security_hooks@@ -290,53 +290,6 @@ _CIRCUIT_BREAKER_ERROR_MSG = (_IDLE_TIMEOUT_SECONDS = 30 * 60_HUNG_TOOL_CAP_SECONDS = HUNG_TOOL_CAP_SECONDS-# Floor on the per-query SDK budget — too small and the CLI refuses to-# start a turn at all. Caller (``_resolve_dynamic_max_budget_usd``) clamps-# to this so a user near (but not yet at) their cap still gets a chance to-# wrap up gracefully instead of hitting an immediate hard stop.-_MAX_BUDGET_USD_FLOOR = 0.5---async def _resolve_dynamic_max_budget_usd(user_id: str | None) -> float:- """Pick the per-query ``max_budget_usd`` for this user *now*.-- Returns the smaller of:- * ``config.claude_agent_max_budget_usd`` (the static per-query cap)- * the user's remaining daily/weekly USD cap (from Redis)-- Falls back to the static cap when no ``user_id`` is available (e.g. an- internal turn run without auth). Floored at ``_MAX_BUDGET_USD_FLOOR``- so a near-capped user still gets enough headroom to surface the- "wrap up" reminder rather than failing to dispatch.- """- static_cap = config.claude_agent_max_budget_usd- if not user_id:- return static_cap- daily_limit, weekly_limit, tier = await get_global_rate_limits(- user_id,- config.daily_cost_limit_microdollars,- config.weekly_cost_limit_microdollars,- )- # Sentinel ``-1.0`` from ``get_remaining_usd_budget`` means Redis was- # unavailable. In that case fall back to the static per-query cap —- # we don't actually know the user is near their limit, and clamping- # to the floor would shrink every turn to $0.50 while Redis is in a- # brown-out. The pre-turn gate already failed closed at 503 in this- # branch, so this defensive fallback only matters in edge paths that- # bypass the gate.- remaining = await get_remaining_usd_budget(- user_id=user_id,- daily_cost_limit=daily_limit,- weekly_cost_limit=weekly_limit,- floor_usd=-1.0,- expected_tier=tier,- )- if tier == "TRIAL":- return resolve_trial_sdk_budget(static_cap, remaining)- if remaining < 0 or remaining == float("inf"):- return static_cap- return max(_MAX_BUDGET_USD_FLOOR, min(static_cap, remaining))-@dataclassclass _SDKLoopState:@@ -530,6 +483,21 @@ async def _consume_sdk_until_done(if isinstance(observed, str) and observed:state.observed_model = observed+ # Live spend and step count: the governor lands the turn when either+ # nears its limit (see ``security_hooks`` PreToolUse). It reads the+ # assistant messages and the raw stream, where each response's real+ # output count arrives. Metering must never fail the turn it meters;+ # the CLI's own budget and turn limits still back it up.+ if state.governor is not None:+ try:+ state.governor.observe(sdk_msg)+ except Exception:+ logger.warning(+ "%s Turn governor could not meter a message",+ ctx.log_prefix,+ exc_info=True,+ )+if loop_state.provider_refused:if isinstance(sdk_msg, ResultMessage):_record_result_usage(sdk_msg, state, ctx.log_prefix)@@ -1412,6 +1380,16 @@ class _RetryState:# Moonshot cost override gates on this so a Moonshot-→-Anthropic# fallback doesn't get mis-billed at Moonshot rates, and vice versa.observed_model: str | None = None+ # The turn's spend and step governor. On ``_RetryState`` so its meter+ # keeps what earlier attempts spent, and so every rebuilt adapter can be+ # handed the same one (see ``_fresh_adapter``).+ governor: TurnGovernor | None = None+++def _fresh_adapter(state: _RetryState, message_id: str, session_id: str) -> None:+ """Replace the adapter for a new attempt, keeping the turn's governor."""+ state.adapter = SDKResponseAdapter(message_id=message_id, session_id=session_id)+ state.adapter.governor = state.governor@dataclass@@ -1832,10 +1810,7 @@ async def _apply_building_mode_restart(# the original message once more, which only surfaces in the# rare CLI-file-loss fallback path (cosmetic duplicate).prior_adapter = state.adapter- state.adapter = SDKResponseAdapter(- message_id=message_id,- session_id=session_id,- )+ _fresh_adapter(state, message_id, session_id)state.adapter.thinking_only_reprompted = state.thinking_only_repromptedif prior_adapter.emitted_real_content_to_wire:state.adapter.prior_attempt_emitted_visible_content = True@@ -2508,10 +2483,7 @@ async def _do_transient_backoff(delete_stale_cli_session_file(ctx.sdk_cwd, fresh_session, ctx.log_prefix)yield StreamStatus(message=f"Connection interrupted, retrying in {backoff}s…")await asyncio.sleep(backoff)- state.adapter = SDKResponseAdapter(- message_id=ctx.message_id,- session_id=ctx.session_id,- )+ _fresh_adapter(state, ctx.message_id, ctx.session_id)def _is_fallback_stderr(line: str) -> bool:@@ -3858,6 +3830,17 @@ def _dispatch_response(failure=failure.as_part() if failure else None,)+ # Persist the pause so the Paused card survives the end of the stream.+ if isinstance(response, StreamTurnPaused):+ pause = TurnPause.model_validate(response.pause)+ logger.info(+ "%s Turn paused (%s, checkpoint_saved=%s)",+ log_prefix,+ pause.reason,+ pause.checkpoint_saved,+ )+ append_pause_marker(ctx.session, pause)+if isinstance(response, StreamReasoningStart):acc.reasoning_response = ChatMessage(role="reasoning", content="")messages = ctx.session.messages@@ -4386,6 +4369,7 @@ async def _run_stream_attempt()yield response+ hangup_pause: TurnPause | None = Noneif not acc.stream_completed and not loop_state.ended_with_stream_error:snapshot = await asyncio.to_thread(read_cli_session_usage,@@ -4410,18 +4394,27 @@ async def _run_stream_attempt(state.adapter.end_open_blocks(closing_responses)for r in closing_responses:yield r- notice_block_id = str(uuid.uuid4())- yield StreamTextStart(id=notice_block_id)- yield StreamTextDelta(id=notice_block_id, delta=STREAM_INCOMPLETE_MARKER)- yield StreamTextEnd(id=notice_block_id)- ctx.session.messages.append(- ChatMessage(role="assistant", content=STREAM_INCOMPLETE_MARKER)+ # While landing, the hangup ended the landing: still a pause.+ hangup_pause = (+ state.governor.hangup_pause() if state.governor is not None else None)+ if hangup_pause is not None:+ yield StreamTurnPaused(pause=hangup_pause.as_part())+ else:+ notice_block_id = str(uuid.uuid4())+ yield StreamTextStart(id=notice_block_id)+ yield StreamTextDelta(id=notice_block_id, delta=STREAM_INCOMPLETE_MARKER)+ yield StreamTextEnd(id=notice_block_id)+ ctx.session.messages.append(+ ChatMessage(role="assistant", content=STREAM_INCOMPLETE_MARKER)+ )if (acc.assistant_response.content or acc.assistant_response.tool_calls) and not acc.has_appended_assistant:ctx.session.messages.append(acc.assistant_response)+ if hangup_pause is not None:+ append_pause_marker(ctx.session, hangup_pause)# SECRT-2333: persist a visible-marker ChatMessage when the turn ended# with ``ended_with_stream_error=True`` and no prior marker is on the@@ -4990,6 +4983,9 @@ async def stream_chat_completion_sdk( # pyright: ignore[reportGeneralTypeIssues# bound name even when the loop never enters its happy path.ended_with_stream_error = Falsefinal_checkpoint: StreamCheckpoint | None = None+ # Built with the CLI options below; read after the loop to keep a paused+ # turn from auto-continuing.+ governor: TurnGovernor | None = None# Make sure there is no more code between the lock acquisition and try-block.try:@@ -5288,12 +5284,21 @@ async def stream_chat_completion_sdk( # pyright: ignore[reportGeneralTypeIssues# Track SDK-internal compaction (PreCompact hook → start, next msg → end)compaction = CompactionTracker()+ # Lands the turn with a checkpoint near its budget or step check-in.+ # A ChatGPT subscription pays for its own turns, so it gets the+ # check-in only.+ governor = TurnGovernor(+ budget=None if is_codex_transport else await resolve_turn_budget(user_id),+ step_checkin=config.sdk_step_checkin,+ )+security_hooks = create_security_hooks(user_id,sdk_cwd=sdk_cwd,max_subtasks=config.claude_agent_max_subtasks,on_compact=compaction.on_compact,tool_display_bridge=tool_display_bridge,+ governor=governor,)if permissions is not None:@@ -5360,9 +5365,9 @@ async def stream_chat_completion_sdk( # pyright: ignore[reportGeneralTypeIssuescwd=sdk_cwd,max_buffer_size=config.claude_agent_max_buffer_size,stderr=_on_stderr,- # max_turns: hard cap on agentic tool-use loops per query to- # prevent runaway execution from burning budget.- max_turns=config.agent_max_turns,+ # max_turns: a backstop a little past the governor's step+ # check-in, which normally lands the turn first.+ max_turns=governor.cli_max_turns,# effort: applies to models with extended thinking (Sonnet,# Opus, Mythos) and Kimi K2.6 via OpenRouter's ``reasoning``# extension (#12871).@@ -5374,7 +5379,9 @@ async def stream_chat_completion_sdk( # pyright: ignore[reportGeneralTypeIssues# presenting a fabricated USD ceiling or provider fallback to the# CLI would produce incorrect policy and accounting behavior.sdk_options.fallback_model = fallback_model- sdk_options.max_budget_usd = await _resolve_dynamic_max_budget_usd(user_id)+ # A backstop above the turn's budget, in the CLI's price units:+ # the governor lands the turn before it fires.+ sdk_options.max_budget_usd = governor.cli_budget_usd(sdk_model)sdk_options.thinking = {"type": "adaptive"}if config.claude_agent_max_thinking_tokens > 0:sdk_options.max_thinking_tokens = (@@ -5426,6 +5433,7 @@ async def stream_chat_completion_sdk( # pyright: ignore[reportGeneralTypeIssuesmessage_id=message_id,session_id=session_id,)+ adapter.governor = governor# Propagate user_id/session_id as OTEL context attributes so the# langsmith tracing integration attaches them to every span. This@@ -5525,6 +5533,10 @@ async def stream_chat_completion_sdk( # pyright: ignore[reportGeneralTypeIssues# On --resume, current_message may come from session history which was# already sanitized on the original turn; strip again as defence-in-depth.current_message = strip_user_context_tags(current_message)+ if not is_user_message:+ # A continuation note is stored as a system row; the model gets+ # its instruction without the rendering marker.+ current_message = as_model_prompt(current_message)# On the first turn inject user context into the message before building# the query so that _build_query_message sees the full prefixed content.@@ -5764,6 +5776,7 @@ async def stream_chat_completion_sdk( # pyright: ignore[reportGeneralTypeIssuesadapter=adapter,transcript_builder=transcript_builder,usage=_TokenUsage(),+ governor=governor,# Subagent files are read only on a turn the reconcile bills.generations=TurnGenerations((@@ -5915,10 +5928,7 @@ async def stream_chat_completion_sdk( # pyright: ignore[reportGeneralTypeIssuessession, is_user_message, state.query_message)prior_adapter = state.adapter- state.adapter = SDKResponseAdapter(- message_id=message_id,- session_id=session_id,- )+ _fresh_adapter(state, message_id, session_id)# Carry the per-turn re-prompt cap forward so a transient# retry mid-turn does not unlock another re-prompt round.state.adapter.thinking_only_reprompted = state.thinking_only_reprompted@@ -6718,8 +6728,12 @@ async def stream_chat_completion_sdk( # pyright: ignore[reportGeneralTypeIssues# BaseException both re-raise inside their except blocks, so the generator# closes before reaching here — messages queued during a cancelled turn are# preserved in Redis for the next manual turn.+ #+ # A paused turn doesn't continue either: it stopped because it ran out of+ # budget or reached its check-in, and the user picks it back up from the+ # Paused card. Anything queued meanwhile stays queued for that turn.# -------------------------------------------------------------------------- if not ended_with_stream_error:+ if not ended_with_stream_error and not (governor is not None and governor.landed):_auto_pending_messages = await drain_pending_safe(session_id, log_prefix)if _auto_pending_messages:logger.info(diff --git a/autogpt_platform/backend/backend/copilot/sdk/service_test.py b/autogpt_platform/backend/backend/copilot/sdk/service_test.pyindex cf10a1ba85..3624c33ed5 100644--- a/autogpt_platform/backend/backend/copilot/sdk/service_test.py+++ b/autogpt_platform/backend/backend/copilot/sdk/service_test.py@@ -22,7 +22,6 @@ from .cost_tracking import TokenUsagefrom .service import (_HUNG_TOOL_CAP_SECONDS,_IDLE_TIMEOUT_SECONDS,- _MAX_BUDGET_USD_FLOOR,_THINKING_ONLY_REPROMPT,_build_system_prompt_value,_close_codex_gateway_for_finally,@@ -35,7 +34,6 @@ from .service import (_raise_deferred_codex_cleanup_error,_record_result_usage,_redact_cli_stderr,- _resolve_dynamic_max_budget_usd,_resolve_sdk_model,_resolve_sdk_model_for_request,_safe_close_sdk_client,@@ -1912,128 +1910,6 @@ class TestConsumeSdkUntilDone:assert loop_state.consecutive_empty_tool_calls >= 0-class TestResolveDynamicMaxBudgetUsd:- """The per-query SDK budget is sized to the smaller of the static- config cap and the user's *actual* remaining USD spend, so the- CLI's "wrap up gracefully" reminder fires on real headroom."""-- @pytest.mark.asyncio- async def test_returns_static_cap_without_user_id(self):- with patch(- "backend.copilot.sdk.service.config.claude_agent_max_budget_usd",- 7.5,- ):- result = await _resolve_dynamic_max_budget_usd(None)- assert result == 7.5-- @pytest.mark.asyncio- async def test_returns_static_cap_when_unlimited(self):- # When tier limits are 0/0 we treat as unlimited and fall back to the- # static config cap (the per-query soft ceiling still applies).- with (- patch(- "backend.copilot.sdk.service.get_global_rate_limits",- new=AsyncMock(return_value=(0, 0, "FREE")),- ),- patch(- "backend.copilot.sdk.service.get_remaining_usd_budget",- new=AsyncMock(return_value=float("inf")),- ),- patch(- "backend.copilot.sdk.service.config.claude_agent_max_budget_usd",- 10.0,- ),- ):- result = await _resolve_dynamic_max_budget_usd("u-1")- assert result == 10.0-- @pytest.mark.asyncio- async def test_uses_remaining_when_smaller_than_static(self):- with (- patch(- "backend.copilot.sdk.service.get_global_rate_limits",- new=AsyncMock(return_value=(10_000_000, 50_000_000, "FREE")),- ),- patch(- "backend.copilot.sdk.service.get_remaining_usd_budget",- new=AsyncMock(return_value=2.5),- ),- patch(- "backend.copilot.sdk.service.config.claude_agent_max_budget_usd",- 10.0,- ),- ):- result = await _resolve_dynamic_max_budget_usd("u-1")- assert result == 2.5-- @pytest.mark.asyncio- async def test_clamps_to_floor_when_remaining_is_below(self):- # A near-capped user still gets enough headroom to dispatch the- # turn (and surface the wrap-up reminder) instead of being- # blocked at the SDK level.- with (- patch(- "backend.copilot.sdk.service.get_global_rate_limits",- new=AsyncMock(return_value=(10_000_000, 0, "FREE")),- ),- patch(- "backend.copilot.sdk.service.get_remaining_usd_budget",- new=AsyncMock(return_value=0.05),- ),- patch(- "backend.copilot.sdk.service.config.claude_agent_max_budget_usd",- 10.0,- ),- ):- result = await _resolve_dynamic_max_budget_usd("u-1")- assert result == _MAX_BUDGET_USD_FLOOR-- @pytest.mark.asyncio- async def test_static_cap_wins_when_smaller_than_remaining(self):- # User has plenty of headroom — the per-query soft ceiling still- # applies so a single chat cannot blow the entire daily budget.- with (- patch(- "backend.copilot.sdk.service.get_global_rate_limits",- new=AsyncMock(return_value=(100_000_000, 0, "PRO")),- ),- patch(- "backend.copilot.sdk.service.get_remaining_usd_budget",- new=AsyncMock(return_value=80.0),- ),- patch(- "backend.copilot.sdk.service.config.claude_agent_max_budget_usd",- 10.0,- ),- ):- result = await _resolve_dynamic_max_budget_usd("u-1")- assert result == 10.0-- @pytest.mark.asyncio- async def test_redis_brownout_falls_back_to_static_cap(self):- """Redis brown-out → ``get_remaining_usd_budget`` returns the- ``floor_usd`` we passed. The resolver passes ``-1.0`` as a- sentinel and falls back to the static cap when it sees that —- otherwise every turn during a Redis outage would shrink to the- $0.50 floor instead of the configured per-query cap."""- with (- patch(- "backend.copilot.sdk.service.get_global_rate_limits",- new=AsyncMock(return_value=(10_000_000, 50_000_000, "FREE")),- ),- patch(- "backend.copilot.sdk.service.get_remaining_usd_budget",- new=AsyncMock(return_value=-1.0),- ),- patch(- "backend.copilot.sdk.service.config.claude_agent_max_budget_usd",- 10.0,- ),- ):- result = await _resolve_dynamic_max_budget_usd("u-1")- assert result == 10.0--class TestStreamEndedWithoutResultMessage:"""When the SDK CLI hangs up without a ResultMessage (per-query budgetexhausted, max_turns hit, OOM, crash) the post-stream branch must@@ -2262,6 +2138,55 @@ class TestStreamEndedWithoutResultMessage:contents = [m.content for m in ctx.session.messages]assert STREAM_ERROR_MARKER not in contents, contents+ @pytest.mark.asyncio+ async def test_a_hangup_while_landing_pauses_instead_of_the_notice(self):+ """Once the governor is refusing tools, a CLI hangup ended the landing:+ the turn pauses (no checkpoint claimed) rather than showing the+ incomplete-turn notice, and a paused turn never auto-continues."""+ from backend.copilot.constants import STREAM_INCOMPLETE_MARKER+ from backend.copilot.response_model import StreamTurnPaused+ from backend.copilot.sdk.governor import TurnGovernor+ from backend.copilot.sdk.service import _run_stream_attempt+ from backend.copilot.turn_budget import TurnBudget+ from backend.copilot.turn_pause import pause_of++ ctx = self._ctx()+ state = self._state()+ governor = TurnGovernor(+ budget=TurnBudget(usd=0.0, limiting="weekly"), step_checkin=450+ )+ assert governor.refusal() is not None+ state.governor = governor+ state.adapter.governor = governor++ async def empty_iter(_client, wake=None, tool_display_wake=None):+ if False:+ yield None # pragma: no cover++ fake_client = MagicMock()+ fake_client.query = AsyncMock()+ fake_sdk_client = MagicMock()+ fake_sdk_client.__aenter__ = AsyncMock(return_value=fake_client)+ fake_sdk_client.__aexit__ = AsyncMock(return_value=False)++ with (+ patch(+ "backend.copilot.sdk.service.ClaudeSDKClient",+ return_value=fake_sdk_client,+ ),+ patch("backend.copilot.sdk.service._iter_sdk_messages", new=empty_iter),+ ):+ events = [ev async for ev in _run_stream_attempt(ctx, state)]++ paused = [e for e in events if isinstance(e, StreamTurnPaused)]+ assert len(paused) == 1+ assert paused[0].pause["reason"] == "weekly"+ assert paused[0].pause["checkpoint_saved"] is False+ contents = [m.content for m in ctx.session.messages]+ assert STREAM_INCOMPLETE_MARKER not in contents, contents+ assert pause_of(ctx.session.messages[-1]) is not None+ assert governor.landed is True+class TestHiddenShortNamesForPermissions:"""``_hidden_short_names_for_permissions`` is the pure function thatdiff --git a/autogpt_platform/backend/backend/copilot/sdk/session_persistence_test.py b/autogpt_platform/backend/backend/copilot/sdk/session_persistence_test.pyindex 6d09df3fc6..1e7cf57f9b 100644--- a/autogpt_platform/backend/backend/copilot/sdk/session_persistence_test.py+++ b/autogpt_platform/backend/backend/copilot/sdk/session_persistence_test.py@@ -33,6 +33,7 @@ from backend.copilot.response_model import (StreamTextDelta,StreamToolInputAvailable,StreamToolOutputAvailable,+ StreamTurnPaused,)from backend.copilot.sdk.service import (_dispatch_response,@@ -41,6 +42,7 @@ from backend.copilot.sdk.service import (_StreamAccumulator,)from backend.copilot.session_cleanup import prune_orphan_tool_calls+from backend.copilot.turn_pause import TurnPause, pause_of_NOW = datetime(2024, 1, 1, tzinfo=timezone.utc)@@ -727,3 +729,26 @@ class TestLateToolCallPersistence:with caplog.at_level(logging.WARNING):assert _resolve_tool_result_name(session, "missing", None) is Noneassert any("nameless tool-result" in r.message for r in caplog.records)+++class TestPausePersistence:+ """A paused turn leaves a row the Paused card is rebuilt from on reload."""++ def test_pause_is_stored_after_the_checkpoint(self) -> None:+ session = _make_session()+ ctx = _make_ctx(session)+ state = _make_state()+ acc = _StreamAccumulator(+ assistant_response=ChatMessage(role="assistant", content=""),+ accumulated_tool_calls=[],+ has_appended_assistant=False,+ has_tool_results=False,+ )+ checkpoint = StreamTextDelta(id="t1", delta="Where I got to: 5 of 12.")+ _dispatch_response(checkpoint, acc, ctx, state, False, "[test]")+ pause = TurnPause(reason="steps", steps=450)+ paused = StreamTurnPaused(pause=pause.as_part())++ assert _dispatch_response(paused, acc, ctx, state, False, "[test]") is paused+ assert [m.content for m in session.messages][0] == "Where I got to: 5 of 12."+ assert pause_of(session.messages[-1]) == pausediff --git a/autogpt_platform/backend/backend/copilot/sdk/trial_budget_test.py b/autogpt_platform/backend/backend/copilot/sdk/trial_budget_test.pyindex 6a2a6c1d4f..d9a1a20305 100644--- a/autogpt_platform/backend/backend/copilot/sdk/trial_budget_test.py+++ b/autogpt_platform/backend/backend/copilot/sdk/trial_budget_test.py@@ -1,40 +1,17 @@-from unittest.mock import AsyncMock, patch-import pytest-from backend.copilot.sdk.service import _resolve_dynamic_max_budget_usd---def trial_budget():- with (- patch(- "backend.copilot.sdk.service.get_global_rate_limits",- AsyncMock(return_value=(250_000, 1_000_000, "TRIAL")),- ),- patch(- "backend.copilot.sdk.service.get_remaining_usd_budget", AsyncMock()- ) as remaining,- patch("backend.copilot.sdk.service.config.claude_agent_max_budget_usd", 10.0),- ):- yield remaining+from backend.copilot.sdk.trial_budget import resolve_trial_sdk_budget-async def test_trial_budget_is_never_rounded_up_to_paid_floor(trial_budget):- trial_budget.return_value = 0.05- assert await _resolve_dynamic_max_budget_usd("trial-user") == 0.05+def test_trial_budget_is_never_rounded_up_to_paid_floor():+ assert resolve_trial_sdk_budget(10.0, 0.05) == 0.05@pytest.mark.parametrize("remaining", [0.0, -1.0, float("inf"), float("nan")])-async def test_unavailable_trial_budget_refuses_sdk_dispatch(trial_budget, remaining):- trial_budget.return_value = remaining+def test_unavailable_trial_budget_refuses_sdk_dispatch(remaining):with pytest.raises(ValueError, match="Trial budget"):- await _resolve_dynamic_max_budget_usd("trial-user")+ resolve_trial_sdk_budget(10.0, remaining)-async def test_trial_still_respects_static_per_turn_cap(trial_budget):- trial_budget.return_value = 20.0- assert await _resolve_dynamic_max_budget_usd("trial-user") == 10.0+def test_trial_still_respects_static_per_turn_cap():+ assert resolve_trial_sdk_budget(10.0, 20.0) == 10.0diff --git a/autogpt_platform/backend/backend/copilot/sdk/usage_meter.py b/autogpt_platform/backend/backend/copilot/sdk/usage_meter.pynew file mode 100644index 0000000000..21859c7677--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/sdk/usage_meter.py@@ -0,0 +1,266 @@+"""Live spend for a running turn, priced call by call.++The CLI reports what a turn cost only once it ends. Landing a task before+the money runs out needs a running figure, so each model call the CLI+streams is priced as it arrives, with the same rate cards the post-turn+estimate uses.++Two sources, keyed by the call's id with each count kept at its highest+value, since a call is reported several times:++* ``AssistantMessage`` (once per content block, for the main agent and for+ foreground sub-agents alike) has the id, the model and the input and cache+ counts, but its ``output_tokens`` is only the placeholder from the start+ of the response.+* The raw stream (``StreamEvent``, with partial messages on, the default)+ carries the real output count in each response's ``message_delta``, which+ names no call: it belongs to the last ``message_start`` on the same+ stream, the main agent's or a sub-agent's (by its parent tool call).++With partial messages off, output reads as the placeholder and the figure+runs low; the CLI backstop still bounds the turn. Compaction calls aren't+reported either. Billing still comes from the post-turn reconcile; this+only decides when to land.+"""++import re+from functools import cache+from typing import Any++from claude_agent_sdk import AssistantMessage, StreamEvent+from pydantic import BaseModel++from backend.copilot.anthropic_rate_card import compute_anthropic_cost_usd+from backend.copilot.moonshot import rate_card_usd+from backend.data.llm_registry.catalog import get_catalog++# The CLI's price table only knows Anthropic models and prices anything else+# like Sonnet. Also our price for a model no rate card names: overestimating+# lands a task early, underestimating overspends.+REFERENCE_MODEL = "claude-sonnet-4-5"++_PER_MTOK = 1_000_000++# A dated snapshot (``claude-sonnet-4-5-20250929``) costs what its alias does.+_SNAPSHOT_DATE = re.compile(r"-\d{8}$")+++class CallUsage(BaseModel):+ """One model call's token counts, as far as the stream has reported."""++ model: str | None = None+ main: bool = True+ input_tokens: int = 0+ output_tokens: int = 0+ cache_read_tokens: int = 0+ cache_creation_tokens: int = 0++ def merge(self, usage: dict[str, Any]) -> None:+ self.input_tokens = max(self.input_tokens, _count(usage, "input_tokens"))+ self.output_tokens = max(self.output_tokens, _count(usage, "output_tokens"))+ self.cache_read_tokens = max(+ self.cache_read_tokens, _count(usage, "cache_read_input_tokens")+ )+ self.cache_creation_tokens = max(+ self.cache_creation_tokens, _count(usage, "cache_creation_input_tokens")+ )++ @property+ def prompt_tokens(self) -> int:+ return self.input_tokens + self.cache_read_tokens + self.cache_creation_tokens+++class UsageMeter:+ """Spend and model-call count for one turn, across its retries."""++ def __init__(self) -> None:+ self._calls: dict[str, CallUsage] = {}+ self._priced: dict[str, float] = {}+ self._spent = 0.0+ self._last_main_id: str | None = None+ # The call each stream is generating, by parent tool call (None for+ # the main agent): a ``message_delta`` doesn't say which it belongs to.+ self._streaming: dict[str | None, str] = {}++ def observe(self, message: AssistantMessage) -> None:+ if not message.message_id or not message.usage:+ return+ self._record(+ message.message_id,+ message.usage,+ model=message.model,+ main=message.parent_tool_use_id is None,+ )++ def observe_stream(self, event: StreamEvent) -> None:+ data = event.event+ stream = event.parent_tool_use_id+ if data.get("type") == "message_start":+ message = data.get("message")+ if not isinstance(message, dict) or not message.get("id"):+ return+ self._streaming[stream] = message["id"]+ self._record(+ message["id"],+ message.get("usage") or {},+ model=message.get("model"),+ main=stream is None,+ )+ elif data.get("type") == "message_delta":+ call_id = self._streaming.get(stream)+ usage = data.get("usage")+ if call_id is not None and isinstance(usage, dict):+ self._record(call_id, usage)++ def _record(+ self,+ call_id: str,+ usage: dict[str, Any],+ *,+ model: str | None = None,+ main: bool = True,+ ) -> None:+ call = self._calls.get(call_id)+ if call is None:+ call = CallUsage(model=model or None, main=main)+ self._calls[call_id] = call+ call.merge(usage)+ priced = price_call_usd(call)+ self._spent += priced - self._priced.get(call_id, 0.0)+ self._priced[call_id] = priced+ if call.main:+ self._last_main_id = call_id++ @property+ def spent_usd(self) -> float:+ return self._spent++ @property+ def main_calls(self) -> int:+ """Model calls made by the main agent: the CLI's ``max_turns`` unit."""+ return sum(1 for call in self._calls.values() if call.main)++ @property+ def last_main_call_usd(self) -> float:+ if self._last_main_id is None:+ return 0.0+ return self._priced[self._last_main_id]+++def price_call_usd(call: CallUsage) -> float:+ """USD for one call at the model's list price."""+ anthropic = compute_anthropic_cost_usd(+ model=_rate_card_model(call.model),+ prompt_tokens=call.prompt_tokens,+ completion_tokens=call.output_tokens,+ cache_read_tokens=call.cache_read_tokens,+ cache_creation_tokens=call.cache_creation_tokens,+ )+ if anthropic is not None:+ return anthropic+ rates = _usd_per_mtok(call.model)+ if rates is None:+ return _reference_cost_usd(call)+ input_rate, output_rate = rates+ return (call.prompt_tokens * input_rate + call.output_tokens * output_rate) / (+ _PER_MTOK+ )+++def cli_price_ratio(model: str | None) -> float:+ """How many of the CLI's budget dollars one real dollar on *model* is.++ The CLI prices non-Anthropic models like Sonnet, so its budget counter+ runs several times faster than the real spend on, say, Kimi (~5x). A+ backstop handed to the CLI has to be scaled up by this much or it stops+ the turn before our own landing does. The larger of the input and+ output ratios is used so the backstop never fires early.+ """+ if _is_anthropic(model):+ return 1.0+ rates = _usd_per_mtok(model)+ if rates is None:+ return 1.0+ reference_in, reference_out = _reference_rates()+ real_in, real_out = rates+ ratios = [+ reference / real+ for reference, real in ((reference_in, real_in), (reference_out, real_out))+ if real > 0+ ]+ return max([1.0, *ratios])+++def _is_anthropic(model: str | None) -> bool:+ # The rate card prices Anthropic slugs and returns None for anything else.+ return (+ compute_anthropic_cost_usd(+ model=_rate_card_model(model), prompt_tokens=0, completion_tokens=0+ )+ is not None+ )+++def _rate_card_model(model: str | None) -> str:+ """The Anthropic rate card's name for *model*.++ The CLI reports each call's model as its route names it: OpenRouter's+ ``anthropic/claude-sonnet-5-5``, a dotted ``claude-sonnet-4.5`` or the+ API's dated snapshot. The card keys the bare alias, and prices a model it+ can't find as Opus 4.1, several times Sonnet. Other vendors' slugs pass+ through to their own rates.+ """+ if not model:+ return REFERENCE_MODEL+ vendor, slash, name = model.partition("/")+ if not slash:+ name = model+ elif vendor.lower() != "anthropic":+ return model+ if not name.lower().startswith("claude-"):+ return model+ return _SNAPSHOT_DATE.sub("", name.lower().replace(".", "-"))+++def _usd_per_mtok(model: str | None) -> tuple[float, float] | None:+ """(input, output) list price for a model priced outside Anthropic's card."""+ if not model:+ return None+ moonshot = rate_card_usd(model)+ if moonshot is not None:+ return moonshot+ return _catalog_rates().get(model)+++@cache+def _catalog_rates() -> dict[str, tuple[float, float]]:+ return {+ m.slug: (m.cost.provider_input_usd_per_1m, m.cost.provider_output_usd_per_1m)+ for m in get_catalog().models+ if m.cost is not None+ and m.cost.provider_input_usd_per_1m is not None+ and m.cost.provider_output_usd_per_1m is not None+ }+++@cache+def _reference_rates() -> tuple[float, float]:+ per_input = compute_anthropic_cost_usd(+ model=REFERENCE_MODEL, prompt_tokens=_PER_MTOK, completion_tokens=0+ )+ per_output = compute_anthropic_cost_usd(+ model=REFERENCE_MODEL, prompt_tokens=0, completion_tokens=_PER_MTOK+ )+ return per_input or 0.0, per_output or 0.0+++def _reference_cost_usd(call: CallUsage) -> float:+ reference_in, reference_out = _reference_rates()+ return (+ call.prompt_tokens * reference_in + call.output_tokens * reference_out+ ) / _PER_MTOK+++def _count(usage: dict[str, Any], key: str) -> int:+ value = usage.get(key)+ return value if isinstance(value, int) and value > 0 else 0diff --git a/autogpt_platform/backend/backend/copilot/sdk/usage_meter_test.py b/autogpt_platform/backend/backend/copilot/sdk/usage_meter_test.pynew file mode 100644index 0000000000..d650e60865--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/sdk/usage_meter_test.py@@ -0,0 +1,194 @@+"""The live spend figure the governor lands turns on."""++import pytest+from claude_agent_sdk import AssistantMessage, StreamEvent, TextBlock++from backend.copilot.moonshot import rate_card_usd++from .usage_meter import (+ REFERENCE_MODEL,+ CallUsage,+ UsageMeter,+ cli_price_ratio,+ price_call_usd,+)++KIMI = "moonshotai/kimi-k2.6"+++def _message(+ call_id: str | None,+ *,+ model: str = KIMI,+ output: int = 0,+ prompt: int = 0,+ parent: str | None = None,+ usage: bool = True,+) -> AssistantMessage:+ return AssistantMessage(+ content=[TextBlock(text="x")],+ model=model,+ parent_tool_use_id=parent,+ usage={"input_tokens": prompt, "output_tokens": output} if usage else None,+ message_id=call_id,+ )+++def _kimi_usd(prompt: int, output: int) -> float:+ rates = rate_card_usd(KIMI)+ assert rates is not None+ return (prompt * rates[0] + output * rates[1]) / 1_000_000+++def test_a_call_reported_once_per_block_is_counted_once():+ # The CLI repeats a call's AssistantMessage per content block; the last+ # block carries the latest output count.+ meter = UsageMeter()+ meter.observe(_message("gen-1", prompt=10_000, output=200))+ meter.observe(_message("gen-1", prompt=10_000, output=900))+ assert meter.main_calls == 1+ assert meter.spent_usd == pytest.approx(_kimi_usd(10_000, 900))+ assert meter.last_main_call_usd == pytest.approx(_kimi_usd(10_000, 900))+++def test_counts_never_go_backwards_within_a_call():+ meter = UsageMeter()+ meter.observe(_message("gen-1", prompt=10_000, output=900))+ meter.observe(_message("gen-1", prompt=10_000, output=100))+ assert meter.spent_usd == pytest.approx(_kimi_usd(10_000, 900))+++def test_sub_agent_calls_cost_money_but_are_not_main_steps():+ meter = UsageMeter()+ meter.observe(_message("gen-main", prompt=1_000, output=100))+ meter.observe(_message("gen-sub", prompt=50_000, output=2_000, parent="tu-1"))+ assert meter.main_calls == 1+ assert meter.spent_usd == pytest.approx(+ _kimi_usd(1_000, 100) + _kimi_usd(50_000, 2_000)+ )+ assert meter.last_main_call_usd == pytest.approx(_kimi_usd(1_000, 100))+++ "message", [_message(None, output=100), _message("gen-1", usage=False)]+)+def test_messages_without_an_id_or_usage_are_ignored(message):+ meter = UsageMeter()+ meter.observe(message)+ assert meter.main_calls == 0+ assert meter.spent_usd == 0.0+++def test_anthropic_calls_use_the_rate_card_with_cache_discounts():+ cached = CallUsage(+ model="anthropic/claude-sonnet-5", input_tokens=1_000, cache_read_tokens=99_000+ )+ uncached = CallUsage(model="anthropic/claude-sonnet-5", input_tokens=100_000)+ assert 0 < price_call_usd(cached) < price_call_usd(uncached) / 5+++def test_an_unknown_model_is_priced_like_the_reference():+ unknown = CallUsage(+ model="someone/new-model", input_tokens=10_000, output_tokens=500+ )+ reference = CallUsage(model=REFERENCE_MODEL, input_tokens=10_000, output_tokens=500)+ assert price_call_usd(unknown) == pytest.approx(price_call_usd(reference))+++ "reported, alias",+ [+ # OpenRouter's slug, as the CLI reports a call on that route.+ ("anthropic/claude-sonnet-5-5", "claude-sonnet-5-5"),+ ("anthropic/claude-opus-4.6", "claude-opus-4-6"),+ # The API's dated snapshot of an alias the card keys undated.+ ("claude-sonnet-5-5-20260930", "claude-sonnet-5-5"),+ ],+)+def test_a_call_is_priced_at_its_models_rate_however_its_route_names_it(+ reported: str, alias: str, caplog: pytest.LogCaptureFixture+):+ # Missing the card priced these as Opus 4.1: about 7x a Sonnet 5.5 turn.+ usage = {"input_tokens": 2_600, "cache_read_tokens": 24_000, "output_tokens": 30}+ as_reported = price_call_usd(CallUsage(model=reported, **usage))+ assert as_reported == pytest.approx(price_call_usd(CallUsage(model=alias, **usage)))+ assert "no entry for model" not in caplog.text+++def test_cli_price_ratio_is_one_for_anthropic_and_unknown_models():+ assert cli_price_ratio("anthropic/claude-sonnet-5") == 1.0+ assert cli_price_ratio("claude-sonnet-4-5") == 1.0+ assert cli_price_ratio("someone/new-model") == 1.0+ assert cli_price_ratio(None) == 1.0+++def test_cli_price_ratio_covers_the_cli_pricing_kimi_like_sonnet():+ # Sonnet is $3/$15 to Kimi K2.x's $0.60/$2.80: the larger ratio, output,+ # is what keeps the backstop from firing before the real budget is spent.+ assert cli_price_ratio(KIMI) == pytest.approx(15 / 2.80, rel=0.05)+++def _start(call_id: str, *, prompt: int = 0, parent: str | None = None) -> StreamEvent:+ return StreamEvent(+ uuid="u",+ session_id="s",+ event={+ "type": "message_start",+ "message": {+ "id": call_id,+ "model": KIMI,+ "usage": {"input_tokens": prompt, "output_tokens": 1},+ },+ },+ parent_tool_use_id=parent,+ )+++def _delta(output: int, *, parent: str | None = None) -> StreamEvent:+ return StreamEvent(+ uuid="u",+ session_id="s",+ event={"type": "message_delta", "usage": {"output_tokens": output}},+ parent_tool_use_id=parent,+ )+++def test_output_comes_from_the_stream_not_the_placeholder():+ # As the CLI streams it: message_start and every assistant message carry+ # the placeholder output count; message_delta carries the real one.+ meter = UsageMeter()+ meter.observe_stream(_start("m1", prompt=100))+ meter.observe(_message("m1", prompt=100, output=1))+ meter.observe(_message("m1", prompt=100, output=1))+ meter.observe_stream(_delta(10_000))+ assert meter.spent_usd == pytest.approx(_kimi_usd(100, 10_000))+ assert meter.main_calls == 1+++def test_a_sub_agents_stream_is_kept_apart_from_the_main_agents():+ meter = UsageMeter()+ meter.observe_stream(_start("main-1", prompt=100))+ meter.observe_stream(_start("sub-1", prompt=50, parent="tu-1"))+ meter.observe_stream(_delta(2_000, parent="tu-1"))+ meter.observe_stream(_delta(500))+ assert meter.spent_usd == pytest.approx(_kimi_usd(100, 500) + _kimi_usd(50, 2_000))+ assert meter.main_calls == 1+ assert meter.last_main_call_usd == pytest.approx(_kimi_usd(100, 500))+++def test_a_delta_with_no_call_started_on_its_stream_is_ignored():+ meter = UsageMeter()+ meter.observe_stream(_delta(10_000))+ meter.observe_stream(_start("main-1"))+ meter.observe_stream(_delta(10_000, parent="tu-unknown"))+ assert meter.spent_usd == pytest.approx(_kimi_usd(0, 1))+++def test_the_next_response_on_a_stream_gets_its_own_delta():+ meter = UsageMeter()+ meter.observe_stream(_start("m1"))+ meter.observe_stream(_delta(300))+ meter.observe_stream(_start("m2"))+ meter.observe_stream(_delta(700))+ assert meter.spent_usd == pytest.approx(_kimi_usd(0, 300) + _kimi_usd(0, 700))+ assert meter.main_calls == 2diff --git a/autogpt_platform/backend/backend/copilot/stream_drift/scripted.py b/autogpt_platform/backend/backend/copilot/stream_drift/scripted.pyindex c7134da954..df023edee7 100644--- a/autogpt_platform/backend/backend/copilot/stream_drift/scripted.py+++ b/autogpt_platform/backend/backend/copilot/stream_drift/scripted.py@@ -20,6 +20,7 @@ from backend.copilot.sdk.compaction import CompactionTrackerfrom backend.copilot.sdk.response_adapter import SDKResponseAdapterfrom backend.copilot.stream_checkpoint import turn_checkpointfrom backend.copilot.tree import root_envelope+from backend.copilot.turn_budget import TurnBudget# Its presence alone picks the OpenRouter transport the fixtures were recorded# under; without a key (fork PRs have no secrets) the turns record differently.@@ -193,7 +194,9 @@ async def sdk_service_turn("_resolve_sdk_model": MagicMock(return_value=None),"set_execution_context": MagicMock(),"get_user_tier": AsyncMock(return_value=None),- "_resolve_dynamic_max_budget_usd": AsyncMock(return_value=100.0),+ "resolve_turn_budget": AsyncMock(+ return_value=TurnBudget(usd=100.0, limiting="task")+ ),"drain_pending_safe": AsyncMock(side_effect=[[], queued_after_first] + [[]] * len(queries)),diff --git a/autogpt_platform/backend/backend/copilot/stream_registry.py b/autogpt_platform/backend/backend/copilot/stream_registry.pyindex 236c7b0b80..b471afaf40 100644--- a/autogpt_platform/backend/backend/copilot/stream_registry.py+++ b/autogpt_platform/backend/backend/copilot/stream_registry.py@@ -68,6 +68,7 @@ from .response_model import (StreamToolInputAvailable,StreamToolInputStart,StreamToolOutputAvailable,+ StreamTurnPaused,StreamUsage,)@@ -981,6 +982,7 @@ async def mark_session_completed(session_id: str,error_message: str | None = None,*,+ error_code: str | None = None,skip_error_publish: bool = False,turn_id: str = "",) -> bool:@@ -998,6 +1000,9 @@ async def mark_session_completed(session_id: Session ID to mark as completederror_message: If provided, marks as "failed" and publishes aStreamError before StreamFinish. Otherwise marks as "completed".+ error_code: The machine-readable code of that error. Republished+ with it so the client can tell, say, a usage limit from a crash+ without matching on the message text.skip_error_publish: If True, still marks the session as "failed" butdoes NOT publish a StreamError event. Use this for a user-initiatedcancel, which the frontend would otherwise render as "the assistant@@ -1086,7 +1091,9 @@ async def mark_session_completed(if error_message and not skip_error_publish:try:- await publish_chunk(turn_id, StreamError(errorText=error_message))+ await publish_chunk(+ turn_id, StreamError(errorText=error_message, code=error_code)+ )except Exception as e:logger.warning(f"Failed to publish error event for session {session_id}: {e}"@@ -1349,6 +1356,7 @@ CHUNK_TYPE_TO_CLASS: dict[str, type[StreamBaseResponse]] = {ResponseType.FOLLOW_UP_QUEUED.value: StreamFollowUpQueued,ResponseType.MODE_CHANGED.value: StreamModeChanged,ResponseType.PROVIDER_FAILURE.value: StreamProviderFailure,+ ResponseType.TURN_PAUSED.value: StreamTurnPaused,ResponseType.COMPACTION.value: StreamCompactionProgress,ResponseType.CHECKPOINT.value: StreamCheckpoint,}diff --git a/autogpt_platform/backend/backend/copilot/stream_registry_test.py b/autogpt_platform/backend/backend/copilot/stream_registry_test.pyindex 5afeefcec4..e5bb23d8f9 100644--- a/autogpt_platform/backend/backend/copilot/stream_registry_test.py+++ b/autogpt_platform/backend/backend/copilot/stream_registry_test.py@@ -415,6 +415,36 @@ async def test_mark_session_completed_skip_error_publish_emits_no_stream_error()assert any(isinstance(c, stream_registry.StreamError) for c in published)+async def test_mark_session_completed_republishes_the_error_code():+ """The executor re-publishes a turn's error here; without its code the+ client can only match the message text, which is how a budget stop came+ to open the "Daily usage limit reached" dialog."""+ publish_mock = AsyncMock()+ with (+ patch.object(+ stream_registry,+ "get_redis_async",+ new=AsyncMock(return_value=_FakeRedis({"status": "running"})),+ ),+ patch.object(+ stream_registry, "hash_compare_and_set", new=AsyncMock(return_value=True)+ ),+ patch.object(stream_registry, "publish_chunk", new=publish_mock),+ ):+ await stream_registry.mark_session_completed(+ "sess-1", error_message="The response stopped.", error_code="idle_timeout"+ )+ errors = [+ call.args[1]+ for call in publish_mock.call_args_list+ if isinstance(call.args[1], stream_registry.StreamError)+ ]+ assert len(errors) == 1+ assert errors[0].code == "idle_timeout"+ assert errors[0].to_sse().count("[code:idle_timeout]") == 1++@pytest.mark.asyncioasync def test_mark_session_completed_skips_lock_release_when_already_completed():"""CAS failure = someone else completed the session first; we must notdiff --git a/autogpt_platform/backend/backend/copilot/turn_budget.py b/autogpt_platform/backend/backend/copilot/turn_budget.pynew file mode 100644index 0000000000..05227669c9--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/turn_budget.py@@ -0,0 +1,120 @@+"""How much a turn may spend, and which limit that figure comes from.++A turn's budget is the smaller of a per-task cap (a runaway guard set well+above a normal task) and what the account has left in its tightest window.+Which one binds decides what the user is told when the turn pauses -- out of+today's budget, out of this week's, trial allowance used, or this task+reached its size limit -- and when the first two can pick back up.++Unlike the old per-query budget there is no floor: a user with a cent left+gets a cent, not a $0.50 top-up. The turn lands with a checkpoint instead of+overspending, and the CLI's own budget flag is sized separately as a loose+backstop (see ``governor.cli_budget_usd``).+"""++import math+from datetime import datetime+from typing import Literal++from pydantic import BaseModel++from backend.copilot.config import ChatConfig+from backend.copilot.rate_limit import (+ RateLimitUnavailable,+ SubscriptionTier,+ UsageWindow,+ get_global_rate_limits,+ get_remaining_usd_budget,+ get_usage_status,+)+from backend.copilot.sdk.trial_budget import resolve_trial_sdk_budget++BudgetWindow = Literal["daily", "weekly", "trial", "task"]++config = ChatConfig()++# Remaining amounts are microdollar integers converted to USD; anything this+# close counts as the same figure.+_SAME_USD = 1e-6+++class TurnBudget(BaseModel):+ """The money one turn may spend and the limit that sets it."""++ usd: float+ limiting: BudgetWindow+ # When the limiting window refills: daily and weekly only. A trial's+ # allowance never refills, and the task cap applies afresh per turn.+ resets_at: datetime | None = None+ # Spend past ``usd`` is never allowed, whichever limit binds: a trial's+ # allowance is real money with no plan behind it, so the CLI backstop+ # sits exactly on the budget instead of above it.+ hard_cap: bool = False+++async def resolve_turn_budget(user_id: str | None) -> TurnBudget:+ """Budget for a platform-billed turn starting now."""+ task_cap = config.claude_agent_max_budget_usd+ whole_task = TurnBudget(usd=task_cap, limiting="task")+ if not user_id:+ return whole_task+ daily_limit, weekly_limit, tier = await get_global_rate_limits(+ user_id,+ config.daily_cost_limit_microdollars,+ config.weekly_cost_limit_microdollars,+ )+ # A negative floor turns a Redis brown-out into -1 rather than a+ # made-up figure; the pre-turn gate has already failed closed on it.+ remaining = await get_remaining_usd_budget(+ user_id=user_id,+ daily_cost_limit=daily_limit,+ weekly_cost_limit=weekly_limit,+ floor_usd=-1.0,+ expected_tier=tier,+ )+ trial = tier == SubscriptionTier.TRIAL+ if trial:+ # Raises when the trial has nothing left: no trial turn may start.+ remaining = resolve_trial_sdk_budget(task_cap, remaining)+ if remaining < 0 or not math.isfinite(remaining) or task_cap <= remaining:+ return whole_task.model_copy(update={"hard_cap": trial})+ window, resets_at = await _limiting_window(+ user_id, daily_limit, weekly_limit, tier, remaining+ )+ return TurnBudget(+ usd=remaining, limiting=window, resets_at=resets_at, hard_cap=trial+ )+++async def _limiting_window(+ user_id: str,+ daily_limit: int,+ weekly_limit: int,+ tier: SubscriptionTier,+ remaining_usd: float,+) -> tuple[BudgetWindow, datetime | None]:+ """Name the window whose remainder is the turn's budget.++ When several are equally tight -- both empty, typically -- the one that+ refills last is the one that decides when work can resume.+ """+ unknown: BudgetWindow = "trial" if tier == SubscriptionTier.TRIAL else "daily"+ try:+ status = await get_usage_status(user_id, daily_limit, weekly_limit, tier=tier)+ except RateLimitUnavailable:+ return unknown, None+ binding = [+ (name, window)+ for name, window in (("daily", status.daily), ("weekly", status.weekly))+ if window.limit >= 0 and _left_usd(window) <= remaining_usd + _SAME_USD+ ]+ if not binding:+ return unknown, None+ name, window = max(binding, key=lambda pair: pair[1].resets_at)+ if tier == SubscriptionTier.TRIAL and name != "daily":+ return "trial", None+ return ("daily" if name == "daily" else "weekly"), window.resets_at+++def _left_usd(window: UsageWindow) -> float:+ return max(0, window.limit - window.used) / 1_000_000diff --git a/autogpt_platform/backend/backend/copilot/turn_budget_test.py b/autogpt_platform/backend/backend/copilot/turn_budget_test.pynew file mode 100644index 0000000000..3b42ec2c59--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/turn_budget_test.py@@ -0,0 +1,194 @@+from datetime import UTC, datetime, timedelta+from unittest.mock import AsyncMock, patch++import pytest++from backend.copilot.rate_limit import (+ CoPilotUsageStatus,+ RateLimitUnavailable,+ SubscriptionTier,+ UsageWindow,+)+from backend.copilot.turn_budget import TurnBudget, resolve_turn_budget++_MOD = "backend.copilot.turn_budget"+NOW = datetime(2026, 10, 14, 15, 0, tzinfo=UTC)+DAILY_RESET = NOW.replace(hour=0) + timedelta(days=1)+WEEKLY_RESET = datetime(2026, 10, 19, tzinfo=UTC)+++def _status(+ daily_used: int,+ weekly_used: int,+ *,+ tier: SubscriptionTier = SubscriptionTier.PRO,+ daily_limit: int = 3_000_000,+ weekly_limit: int = 6_000_000,+) -> CoPilotUsageStatus:+ return CoPilotUsageStatus(+ daily=UsageWindow(used=daily_used, limit=daily_limit, resets_at=DAILY_RESET),+ weekly=UsageWindow(+ used=weekly_used, limit=weekly_limit, resets_at=WEEKLY_RESET+ ),+ tier=tier,+ )+++def limits():+ """Patch the rate-limit reads; tests set ``remaining`` and ``status``."""+ with (+ patch(+ f"{_MOD}.get_global_rate_limits",+ AsyncMock(return_value=(3_000_000, 6_000_000, SubscriptionTier.PRO)),+ ) as global_limits,+ patch(f"{_MOD}.get_remaining_usd_budget", AsyncMock()) as remaining,+ patch(f"{_MOD}.get_usage_status", AsyncMock()) as status,+ patch(f"{_MOD}.config.claude_agent_max_budget_usd", 10.0),+ ):+ yield global_limits, remaining, status+++async def test_no_user_gets_the_task_cap():+ with patch(f"{_MOD}.config.claude_agent_max_budget_usd", 7.5):+ budget = await resolve_turn_budget(None)+ assert budget == TurnBudget(usd=7.5, limiting="task")++[email protected]("remaining", [float("inf"), -1.0])+async def test_uncapped_or_unreadable_allowance_gets_the_task_cap(limits, remaining):+ # inf: no cap configured. -1: Redis brown-out, which the pre-turn gate has+ # already failed closed on -- don't shrink the turn on a guess.+ _, remaining_mock, _ = limits+ remaining_mock.return_value = remaining+ budget = await resolve_turn_budget("u-1")+ assert budget == TurnBudget(usd=10.0, limiting="task")+++async def test_task_cap_binds_when_the_account_has_more_left(limits):+ _, remaining_mock, status_mock = limits+ remaining_mock.return_value = 80.0+ budget = await resolve_turn_budget("u-1")+ assert budget == TurnBudget(usd=10.0, limiting="task")+ status_mock.assert_not_awaited()+++async def test_daily_window_binds_with_its_reset(limits):+ _, remaining_mock, status_mock = limits+ remaining_mock.return_value = 0.5+ status_mock.return_value = _status(daily_used=2_500_000, weekly_used=1_000_000)+ budget = await resolve_turn_budget("u-1")+ assert budget == TurnBudget(usd=0.5, limiting="daily", resets_at=DAILY_RESET)+++async def test_no_floor_a_cent_left_is_a_cent(limits):+ # The old resolver topped a near-empty allowance up to $0.50. The turn now+ # gets what is left and lands with a checkpoint instead.+ _, remaining_mock, status_mock = limits+ remaining_mock.return_value = 0.01+ status_mock.return_value = _status(daily_used=2_990_000, weekly_used=1_000_000)+ budget = await resolve_turn_budget("u-1")+ assert budget.usd == 0.01+ assert budget.limiting == "daily"+++async def test_weekly_window_binds_when_it_is_tighter(limits):+ _, remaining_mock, status_mock = limits+ remaining_mock.return_value = 0.25+ status_mock.return_value = _status(daily_used=1_000_000, weekly_used=5_750_000)+ budget = await resolve_turn_budget("u-1")+ assert budget == TurnBudget(usd=0.25, limiting="weekly", resets_at=WEEKLY_RESET)+++async def test_both_windows_empty_names_the_one_that_resets_last(limits):+ # Telling the user "midnight" while the weekly cap holds them until Monday+ # was the old bug: the later reset is when work can actually resume.+ _, remaining_mock, status_mock = limits+ remaining_mock.return_value = 0.0+ status_mock.return_value = _status(daily_used=3_000_000, weekly_used=6_000_000)+ budget = await resolve_turn_budget("u-1")+ assert budget == TurnBudget(usd=0.0, limiting="weekly", resets_at=WEEKLY_RESET)+++async def test_unreadable_windows_still_name_a_window(limits):+ _, remaining_mock, status_mock = limits+ remaining_mock.return_value = 0.5+ status_mock.side_effect = RateLimitUnavailable("brown-out")+ budget = await resolve_turn_budget("u-1")+ assert budget == TurnBudget(usd=0.5, limiting="daily")+++def trial_limits(limits):+ global_limits, remaining_mock, status_mock = limits+ global_limits.return_value = (20_000_000, 20_000_000, SubscriptionTier.TRIAL)+ yield remaining_mock, status_mock+++async def test_trial_allowance_binds_without_a_reset(trial_limits):+ # A lifetime trial folds its allowance into the weekly gauge; it never+ # refills, so there is no reset to offer.+ remaining_mock, status_mock = trial_limits+ remaining_mock.return_value = 0.05+ status_mock.return_value = _status(+ daily_used=1_000_000,+ weekly_used=19_950_000,+ tier=SubscriptionTier.TRIAL,+ daily_limit=20_000_000,+ weekly_limit=20_000_000,+ )+ budget = await resolve_turn_budget("trial-user")+ assert budget == TurnBudget(usd=0.05, limiting="trial", hard_cap=True)+++async def test_trial_with_a_tighter_daily_window_resets_daily(trial_limits):+ remaining_mock, status_mock = trial_limits+ remaining_mock.return_value = 0.1+ status_mock.return_value = _status(+ daily_used=2_400_000,+ weekly_used=5_000_000,+ tier=SubscriptionTier.TRIAL,+ daily_limit=2_500_000,+ weekly_limit=20_000_000,+ )+ budget = await resolve_turn_budget("trial-user")+ assert budget == TurnBudget(+ usd=0.1, limiting="daily", resets_at=DAILY_RESET, hard_cap=True+ )++[email protected]("remaining", [0.0, -1.0, float("inf"), float("nan")])+async def test_trial_with_nothing_left_starts_no_turn(trial_limits, remaining):+ remaining_mock, _ = trial_limits+ remaining_mock.return_value = remaining+ with pytest.raises(ValueError, match="Trial budget"):+ await resolve_turn_budget("trial-user")+++async def test_a_trial_limited_by_the_task_cap_is_still_a_hard_cap(trial_limits):+ # More left than one task may spend: the task cap binds, but the money is+ # still the trial's, so nothing may be spent past it.+ remaining_mock, _ = trial_limits+ remaining_mock.return_value = 15.0+ budget = await resolve_turn_budget("trial-user")+ assert budget.limiting == "task"+ assert budget.hard_cap is True+++async def test_a_paid_budget_is_not_a_hard_cap(limits):+ _, remaining_mock, _ = limits+ remaining_mock.return_value = 15.0+ assert (await resolve_turn_budget("u-1")).hard_cap is Falsediff --git a/autogpt_platform/backend/backend/copilot/turn_pause.py b/autogpt_platform/backend/backend/copilot/turn_pause.pynew file mode 100644index 0000000000..8215c3f4f5--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/turn_pause.py@@ -0,0 +1,92 @@+"""Why a turn paused, and the chat row that remembers it.++A turn that runs out of budget or steps now stops on purpose: its tools are+refused, the agent writes a checkpoint, and the chat shows a Paused card with+the ways to pick the task back up. The live ``data-turn-paused`` part is gone+once the stream ends (the chat re-reads stored rows), so the pause is also+written as a marker row whose metadata rebuilds the card on reload. Older+clients that don't know the card still render the row as a system note.+"""++from datetime import datetime+from typing import Any, Literal++from pydantic import BaseModel, ValidationError++from backend.copilot.constants import COPILOT_SYSTEM_PREFIX+from backend.copilot.markers import has_trailing_marker+from backend.copilot.model import ChatMessage, ChatSession++PauseReason = Literal["daily", "weekly", "trial", "task", "steps"]++# Metadata key on the marker row. Namespaced for the same reason as+# ``PROVIDER_FAILURE_KEY``: the bag is shared with other writers.+PAUSE_KEY = "turn_paused"++_SUMMARIES: dict[PauseReason, str] = {+ "daily": "Paused: this task is out of today's budget.",+ "weekly": "Paused: this task is out of this week's budget.",+ "trial": "Paused: the trial allowance is used up.",+ "task": "Paused: this task reached its size limit.",+ "steps": "Paused: checking in after a long run.",+}++_NOT_STARTED: dict[PauseReason, str] = {+ "daily": "Not started: out of today's budget.",+ "weekly": "Not started: out of this week's budget.",+ "trial": "Not started: the trial allowance is used up.",+ "task": "Not started.",+ "steps": "Not started.",+}+++class TurnPause(BaseModel):+ """What the Paused card needs to say and offer."""++ reason: PauseReason+ # When the limiting window refills; set for daily and weekly pauses only.+ resets_at: datetime | None = None+ # Model calls the turn made; set for step check-ins.+ steps: int | None = None+ # False when a backstop stopped the turn before the agent could write+ # its own checkpoint, so the card mustn't claim it saved its progress.+ checkpoint_saved: bool = True+ # False when the turn was refused before it ran at all (out of budget at+ # the moment it was picked up), so there is nothing to pick back up from.+ started: bool = True++ def as_part(self) -> dict[str, Any]:+ return self.model_dump(mode="json")++ @property+ def summary(self) -> str:+ if not self.started:+ return _NOT_STARTED[self.reason]+ return _SUMMARIES[self.reason]+++def append_pause_marker(session: ChatSession | None, pause: TurnPause) -> bool:+ """Record the pause in the chat. Returns whether a row was added."""+ if session is None or has_trailing_marker(session):+ return False+ session.messages.append(+ ChatMessage(+ role="assistant",+ content=f"{COPILOT_SYSTEM_PREFIX} {pause.summary}",+ metadata={PAUSE_KEY: pause.as_part()},+ )+ )+ return True+++def pause_of(message: ChatMessage) -> TurnPause | None:+ """The pause recorded on a marker row, if this row is one."""+ if message.role != "assistant" or not message.metadata:+ return None+ raw = message.metadata.get(PAUSE_KEY)+ if not isinstance(raw, dict):+ return None+ try:+ return TurnPause.model_validate(raw)+ except ValidationError:+ return Nonediff --git a/autogpt_platform/backend/backend/copilot/turn_pause_test.py b/autogpt_platform/backend/backend/copilot/turn_pause_test.pynew file mode 100644index 0000000000..007dd34912--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/turn_pause_test.py@@ -0,0 +1,72 @@+from datetime import UTC, datetime++import pytest++from backend.copilot.constants import COPILOT_ERROR_PREFIX, COPILOT_SYSTEM_PREFIX+from backend.copilot.continuation import KEEP_GOING_NOTE, RESUMED_AT_RESET_NOTE+from backend.copilot.model import ChatMessage, ChatSession+from backend.copilot.turn_pause import (+ PAUSE_KEY,+ TurnPause,+ append_pause_marker,+ pause_of,+)+++def _session(*messages: ChatMessage) -> ChatSession:+ now = datetime(2026, 10, 14, tzinfo=UTC)+ return ChatSession(+ session_id="s1",+ user_id="u1",+ usage=[],+ started_at=now,+ updated_at=now,+ messages=list(messages),+ )+++def test_marker_row_carries_the_pause_and_a_readable_fallback():+ session = _session(ChatMessage(role="assistant", content="Where I got to: ..."))+ pause = TurnPause(reason="weekly", resets_at=datetime(2026, 10, 19, tzinfo=UTC))+ assert append_pause_marker(session, pause) is True+ row = session.messages[-1]+ assert row.role == "assistant"+ assert row.content is not None+ assert row.content.startswith(COPILOT_SYSTEM_PREFIX)+ assert "this week's budget" in row.content+ assert pause_of(row) == pause+++def test_one_pause_per_turn():+ session = _session(ChatMessage(role="assistant", content="done so far"))+ pause = TurnPause(reason="steps", steps=450)+ assert append_pause_marker(session, pause) is True+ assert append_pause_marker(session, pause) is False+ assert len(session.messages) == 2+++def test_no_pause_on_top_of_an_error_marker():+ session = _session(+ ChatMessage(role="assistant", content=f"{COPILOT_ERROR_PREFIX} broke")+ )+ assert append_pause_marker(session, TurnPause(reason="task")) is False+++def test_rows_without_a_valid_pause_read_as_none():+ assert pause_of(ChatMessage(role="assistant", content="hello")) is None+ assert pause_of(ChatMessage(role="user", content="x", metadata={})) is None+ broken = ChatMessage(+ role="assistant",+ content=f"{COPILOT_SYSTEM_PREFIX} Paused",+ metadata={PAUSE_KEY: {"reason": "not-a-reason"}},+ )+ assert pause_of(broken) is None++[email protected]("note", [KEEP_GOING_NOTE, RESUMED_AT_RESET_NOTE])+def test_a_kickoff_note_is_not_a_marker(note):+ # A continuation that pauses before writing anything still gets its+ # pause row: the note it started from begins a turn, it doesn't end one.+ session = _session(ChatMessage(role="assistant", content=note))+ assert append_pause_marker(session, TurnPause(reason="daily")) is True+ assert len(session.messages) == 2diff --git a/autogpt_platform/frontend/src/app/api/openapi.json b/autogpt_platform/frontend/src/app/api/openapi.jsonindex d56e69bf88..c5c9b8182e 100644--- a/autogpt_platform/frontend/src/app/api/openapi.json+++ b/autogpt_platform/frontend/src/app/api/openapi.json@@ -4330,6 +4330,72 @@}}},+ "/api/chat/sessions/{session_id}/continue": {+ "post": {+ "tags": ["v2", "chat"],+ "summary": "Continue Session",+ "description": "Continue the chat's task from the checkpoint it paused at.",+ "operationId": "postV2ContinueSession",+ "security": [{ "HTTPBearerJWT": [] }],+ "parameters": [+ {+ "name": "session_id",+ "in": "path",+ "required": true,+ "schema": { "type": "string", "title": "Session Id" }+ }+ ],+ "requestBody": {+ "content": {+ "application/json": {+ "schema": {+ "anyOf": [+ { "$ref": "#/components/schemas/ContinueSessionRequest" },+ { "type": "null" }+ ],+ "title": "Request"+ }+ }+ }+ },+ "responses": {+ "202": {+ "description": "Successful Response",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/ContinueSessionResponse"+ }+ }+ }+ },+ "401": {+ "$ref": "#/components/responses/HTTP401NotAuthenticatedError"+ },+ "402": {+ "description": "Subscription required (NO_TIER user, paywall on)"+ },+ "404": { "description": "Session not found or access denied" },+ "409": {+ "description": "A turn is already running in this chat (turn_in_progress), or the chat runs on a connection that can't continue a task (unsupported_connection)"+ },+ "422": {+ "description": "Validation Error",+ "content": {+ "application/json": {+ "schema": { "$ref": "#/components/schemas/HTTPValidationError" }+ }+ }+ },+ "429": {+ "description": "Usage limit or concurrent-turn limit reached"+ },+ "503": {+ "description": "Chat service degraded; retry after Retry-After"+ }+ }+ }+ },"/api/chat/sessions/{session_id}/credential-selection": {"put": {"tags": ["v2", "chat", "chat"],@@ -21033,6 +21099,32 @@],"title": "ContentType"},+ "ContinueSessionRequest": {+ "properties": {+ "model": {+ "anyOf": [+ { "type": "string", "enum": ["standard", "advanced"] },+ { "type": "null" }+ ],+ "title": "Model",+ "description": "Model tier the continuation runs on, as for a message."+ }+ },+ "type": "object",+ "title": "ContinueSessionRequest"+ },+ "ContinueSessionResponse": {+ "properties": {+ "turn_id": {+ "type": "string",+ "title": "Turn Id",+ "description": "The continuation turn; attach to its stream."+ }+ },+ "type": "object",+ "required": ["turn_id"],+ "title": "ContinueSessionResponse"+ },"CopilotSkillDetail": {"properties": {"name": { "type": "string", "title": "Name" },