AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend/copilot): cap concurrent AutoPilot turns per user at 15 (#13064)
Zamil Majdy7f3ec8e1244b
diff --git a/autogpt_platform/backend/backend/api/features/chat/routes.py b/autogpt_platform/backend/backend/api/features/chat/routes.pyindex 58be141819..95275f2a50 100644--- a/autogpt_platform/backend/backend/api/features/chat/routes.py+++ b/autogpt_platform/backend/backend/api/features/chat/routes.py@@ -13,15 +13,17 @@ from pydantic import BaseModel, ConfigDict, Field, field_validatorfrom backend.copilot import service as chat_servicefrom backend.copilot import stream_registry+from backend.copilot.active_turns import (+ ConcurrentTurnLimitError,+ concurrent_turn_limit_message,+)from backend.copilot.builder_context import resolve_session_permissionsfrom backend.copilot.config import ChatConfig, CopilotLlmModel, CopilotModefrom backend.copilot.db import get_chat_messages_paginated-from backend.copilot.executor.utils import enqueue_cancel_task, enqueue_copilot_turn+from backend.copilot.executor.utils import enqueue_cancel_task, schedule_chat_turnfrom backend.copilot.model import (- ChatMessage,ChatSession,ChatSessionMetadata,- append_and_save_message,create_chat_session,delete_chat_session,get_chat_session,@@ -87,7 +89,6 @@ from backend.copilot.tools.models import (TodoWriteResponse,UnderstandingUpdatedResponse,)-from backend.copilot.tracking import track_user_messagefrom backend.data.credit import UsageTransactionMetadata, get_user_credit_modelfrom backend.data.redis_client import get_redis_asyncfrom backend.data.understanding import get_business_understanding@@ -902,7 +903,10 @@ def _empty_ui_message_stream_response() -> StreamingResponse:"/sessions/{session_id}/stream",responses={404: {"description": "Session not found or access denied"},- 429: {"description": "Cost rate-limit or call-frequency cap exceeded"},+ 429: {+ "description": "Cost rate-limit, call-frequency cap, or "+ "per-user concurrent-turn limit exceeded"+ },503: {"description": "Chat service degraded (Redis unavailable for rate ""limit or stream registry); client should honour the Retry-After "@@ -1048,54 +1052,12 @@ async def stream_chat_post(# near the start) — that path returns early. Any request that# reaches this point is starting a fresh turn, so we always mint a# ``turn_id`` unless ``append_and_save_message`` reports a duplicate.- is_duplicate_message = False- if request.message:- message = ChatMessage(- id=request.message_id,- role="user" if request.is_user_message else "assistant",- content=request.message,- )- logger.info(f"[STREAM] Saving user message to session {session_id}")- is_duplicate_message = (- await append_and_save_message(session_id, message)- ) is None- logger.info(f"[STREAM] User message saved for session {session_id}")- if not is_duplicate_message and request.is_user_message:- track_user_message(- user_id=user_id,- session_id=session_id,- message_length=len(request.message),- )-- # Create a task in the stream registry for reconnection support.- # For duplicate messages, skip create_session entirely so the infra-retry- # client subscribes to the *existing* turn's Redis stream and receives the- # in-progress executor output rather than an empty stream.- turn_id = "" if is_duplicate_message else str(uuid4())- if turn_id:- log_meta["turn_id"] = turn_id- session_create_start = time.perf_counter()- await stream_registry.create_session(- session_id=session_id,- user_id=user_id,- tool_call_id="chat_stream",- tool_name="chat",- turn_id=turn_id,- )- logger.info(- f"[TIMING] create_session completed in {(time.perf_counter() - session_create_start) * 1000:.1f}ms",- extra={- "json_fields": {- **log_meta,- "duration_ms": (time.perf_counter() - session_create_start) * 1000,- }- },- )- await enqueue_copilot_turn(+ try:+ turn_id = await schedule_chat_turn(session_id=session_id,user_id=user_id,message=request.message,- turn_id=turn_id,+ message_id=request.message_id,is_user_message=request.is_user_message,context=request.context,file_ids=sanitized_file_ids,@@ -1104,10 +1066,17 @@ async def stream_chat_post(permissions=builder_permissions,request_arrival_at=request_arrival_at,)- else:+ except ConcurrentTurnLimitError as exc:+ raise HTTPException(+ status_code=429, detail=concurrent_turn_limit_message()+ ) from exc++ if turn_id is None:logger.info(f"[STREAM] Duplicate message detected for session {session_id}, skipping enqueue")+ else:+ log_meta["turn_id"] = turn_idsetup_time = (time.perf_counter() - stream_start_time) * 1000logger.info(diff --git a/autogpt_platform/backend/backend/api/features/chat/routes_test.py b/autogpt_platform/backend/backend/api/features/chat/routes_test.pyindex 51a59e9bce..64ebd98f4e 100644--- a/autogpt_platform/backend/backend/api/features/chat/routes_test.py+++ b/autogpt_platform/backend/backend/api/features/chat/routes_test.py@@ -162,27 +162,17 @@ def _mock_stream_internals(mocker: pytest_mock.MockerFixture):new_callable=AsyncMock,return_value=False,)- mock_save = mocker.patch(- "backend.api.features.chat.routes.append_and_save_message",- return_value=MagicMock(), # non-None = message was saved (not a duplicate)- )- mock_registry = mocker.MagicMock()- mock_registry.create_session = mocker.AsyncMock(return_value=None)- mocker.patch(- "backend.api.features.chat.routes.stream_registry",- mock_registry,- )- mock_enqueue = mocker.patch(- "backend.api.features.chat.routes.enqueue_copilot_turn",- return_value=None,- )- mocker.patch(- "backend.api.features.chat.routes.track_user_message",- return_value=None,- )- return types.SimpleNamespace(- save=mock_save, enqueue=mock_enqueue, registry=mock_registry+ # ``schedule_chat_turn`` owns acquire-slot + persist-message + dispatch+ # in one call. Patching it at the route boundary lets tests exercise+ # validation/enrichment (file_ids, message length, rate limits, etc.)+ # without touching Redis, RabbitMQ, or the chat DB. Returning a turn_id+ # mimics a fresh dispatch (vs ``None`` which would mean a duplicate).+ mock_schedule = mocker.patch(+ "backend.api.features.chat.routes.schedule_chat_turn",+ new_callable=AsyncMock,+ return_value="turn-id-mock",)+ return types.SimpleNamespace(enqueue=mock_schedule)def test_stream_chat_accepts_20_file_ids(mocker: pytest_mock.MockerFixture):@@ -217,21 +207,23 @@ def test_stream_chat_accepts_20_file_ids(mocker: pytest_mock.MockerFixture):def test_stream_chat_skips_enqueue_for_duplicate_message(mocker: pytest_mock.MockerFixture,):- """When append_and_save_message returns None (duplicate detected),- enqueue_copilot_turn and stream_registry.create_session must NOT be called- to avoid double-processing and to prevent overwriting the active stream's- turn_id in Redis (which would cause reconnecting clients to miss the response)."""+ """When ``schedule_chat_turn`` returns ``None`` (duplicate-message detected+ inside the slot context), the route must surface the empty UI message stream+ without scheduling another turn — otherwise reconnecting clients would miss+ the original turn's response."""mocks = _mock_stream_internals(mocker)- # Override save to return None — signalling a duplicate- mocks.save.return_value = None+ # Patched ``schedule_chat_turn`` returns None on duplicate.+ mocks.enqueue.return_value = Noneresponse = client.post("/sessions/sess-1/stream",json={"message": "hello"},)assert response.status_code == 200- mocks.enqueue.assert_not_called()- mocks.registry.create_session.assert_not_called()+ # The dup branch returns early; the helper was *called* once but it returned+ # None, so no further dispatch happened (verified by the helper's own+ # contract — see schedule_chat_turn tests).+ mocks.enqueue.assert_called_once()# ─── UUID format filtering ─────────────────────────────────────────────diff --git a/autogpt_platform/backend/backend/blocks/autopilot.py b/autogpt_platform/backend/backend/blocks/autopilot.pyindex ab044181d3..997d3b5693 100644--- a/autogpt_platform/backend/backend/blocks/autopilot.py+++ b/autogpt_platform/backend/backend/blocks/autopilot.py@@ -17,6 +17,10 @@ from backend.blocks._base import (BlockSchemaInput,BlockSchemaOutput,)+from backend.copilot.active_turns import (+ MAX_TURN_LIFETIME_SECONDS,+ concurrent_turn_limit_message,+)from backend.copilot.permissions import (DISABLED_LEGACY_TOOL_NAMES,CopilotPermissions,@@ -42,11 +46,10 @@ _AUTOPILOT_TOOL_CALL_ID = "autopilot_block"_AUTOPILOT_TOOL_NAME = "autopilot_block"# Ceiling on how long AutoPilotBlock.execute_copilot will wait for the-# enqueued turn's terminal event. Graph blocks run synchronously from-# the caller's perspective so we wait effectively as long as needed; 6h-# matches the previous abandoned-task cap and is much longer than any-# legitimate AutoPilot turn.-_AUTOPILOT_BLOCK_MAX_WAIT_SECONDS = 6 * 60 * 60 # 6 hours+# enqueued turn's terminal event. Sourced from the shared turn-lifetime+# constant so the block-level wait, the per-user slot stale-cutoff, and+# any future per-turn timer all advance together.+_AUTOPILOT_BLOCK_MAX_WAIT_SECONDS = MAX_TURN_LIFETIME_SECONDSclass SubAgentRecursionError(BlockExecutionError):@@ -353,6 +356,12 @@ class AutoPilotBlock(Block):tool_call_id=_AUTOPILOT_TOOL_CALL_ID,tool_name=_AUTOPILOT_TOOL_NAME,)+ if outcome == "rejected_concurrent_turn_cap":+ # No session record / transcript was created — the slot+ # cap rejected before ``create_session`` ran. Surface a+ # message that points at the actionable cause rather+ # than the empty transcript.+ raise RuntimeError(concurrent_turn_limit_message())if outcome == "failed":raise RuntimeError("AutoPilot turn failed — see the session's transcript"diff --git a/autogpt_platform/backend/backend/copilot/active_turns.py b/autogpt_platform/backend/backend/copilot/active_turns.pynew file mode 100644index 0000000000..f0a41177b0--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/active_turns.py@@ -0,0 +1,194 @@+"""Per-user concurrent AutoPilot turn tracking.++Caps how many copilot chat turns a single user can have running+concurrently so a single API key cannot spawn hundreds of simultaneous+turns and exhaust shared infrastructure.++This module is the **domain wrapper** over the generic+:func:`backend.data.redis_helpers.try_acquire_concurrency_slot` primitive+— it supplies the per-user pool keying, the cap-from-Settings lookup,+the user-facing error message, and the+:func:`acquire_turn_slot` context manager that drives the slot's+admit / release / refresh lifecycle.++Public API+----------++* :func:`acquire_turn_slot` — async context manager every entry point+ (HTTP route, ``run_sub_session`` tool, ``AutoPilotBlock``) wraps around+ the create-session + enqueue dance. Raises+ :class:`ConcurrentTurnLimitError` on rejection.+* :func:`release_turn_slot` — invoked by ``mark_session_completed``+ when a turn ends, freeing the slot for the next admission.+* :func:`get_concurrent_turn_limit` /+ :func:`concurrent_turn_limit_message` — operator-tunable cap and the+ matching user-facing 429 detail.+"""++import logging+import time+from contextlib import asynccontextmanager+from typing import AsyncIterator++from redis.exceptions import RedisClusterException, RedisError++from backend.data.redis_client import get_redis_async+from backend.data.redis_helpers import SlotAdmission, try_acquire_concurrency_slot+from backend.util.settings import Settings++logger = logging.getLogger(__name__)+++# Upper bound on a single AutoPilot turn's wall-clock duration. Beyond+# this we treat the turn as abandoned: the slot is reclaimed by the+# stale-cutoff sweep (so a crashed turn doesn't hold a slot forever) and+# the :class:`AutoPilotBlock` execution wait gives up. Far exceeds typical+# chat turn duration (seconds-minutes) so legitimate long-running tool+# calls (E2B sandbox, deep web crawls, etc.) aren't penalised. The normal+# release path is ``mark_session_completed``; this is the safety net.+MAX_TURN_LIFETIME_SECONDS = 6 * 60 * 60++_USER_ACTIVE_TURNS_KEY_PREFIX = "copilot:user_active_turns:"+++def get_concurrent_turn_limit() -> int:+ """Resolve the configured per-user concurrent-turn cap at call time.++ Reading at call time (rather than module load) lets operators retune+ the cap by editing the env-backed Settings without redeploying the+ code that imports this module.+ """+ return Settings().config.max_concurrent_copilot_turns_per_user+++def concurrent_turn_limit_message(limit: int | None = None) -> str:+ """User-facing 429 detail string. Pass ``limit`` if you already+ resolved it; otherwise we read the configured value."""+ resolved = get_concurrent_turn_limit() if limit is None else limit+ return (+ f"You've reached the limit of {resolved} active tasks. Please wait "+ f"for one of your current tasks to finish before starting a new one."+ )+++class ConcurrentTurnLimitError(Exception):+ """User has reached the configured concurrent in-flight AutoPilot+ turn cap. Maps to HTTP 429 in the API layer.+ """++ def __init__(self, message: str | None = None) -> None:+ super().__init__(message or concurrent_turn_limit_message())+++def _user_pool_key(user_id: str) -> str:+ # Hash-tag braces ensure all keys for a single user co-locate on the+ # same Redis Cluster slot — required for any future Lua that touches+ # multiple per-user keys atomically.+ return f"{_USER_ACTIVE_TURNS_KEY_PREFIX}{{{user_id}}}"+++async def _try_admit_user_turn(user_id: str, session_id: str) -> SlotAdmission:+ """Atomic admit/refresh against the user's active-turn pool.++ Fails open (returns ``ADMITTED``) on Redis errors so a brown-out+ doesn't 429 every user — the cap is a safeguard, not a budget.+ """+ try:+ redis = await get_redis_async()+ now = time.time()+ return await try_acquire_concurrency_slot(+ redis,+ pool_key=_user_pool_key(user_id),+ slot_id=session_id,+ capacity=get_concurrent_turn_limit(),+ score=now,+ stale_before_score=now - MAX_TURN_LIFETIME_SECONDS,+ ttl_seconds=MAX_TURN_LIFETIME_SECONDS,+ )+ except (RedisError, RedisClusterException, ConnectionError, OSError) as exc:+ logger.warning(+ "concurrent-turn cap: Redis unavailable for user=%s; failing open: %s",+ user_id,+ exc,+ )+ return SlotAdmission.ADMITTED+++async def release_turn_slot(user_id: str, session_id: str) -> None:+ """Free ``user_id``'s slot for ``session_id``. Idempotent.++ Best-effort — a Redis error only delays release until the next+ stale-cutoff sweep.+ """+ try:+ redis = await get_redis_async()+ await redis.zrem(_user_pool_key(user_id), session_id)+ except (RedisError, RedisClusterException, ConnectionError, OSError) as exc:+ logger.warning(+ "release_turn_slot: Redis unavailable for user=%s session=%s: %s",+ user_id,+ session_id,+ exc,+ )+++class TurnSlot:+ """Handle yielded by :func:`acquire_turn_slot`.++ Call :meth:`keep` once a turn has been successfully scheduled to+ transfer ownership to ``mark_session_completed`` (the release path).+ Without ``keep``, the context manager auto-releases on exit — but+ only when *this* caller admitted the slot. A re-entrant refresh+ leaves the slot alone, since some earlier caller still owns it.+ """++ __slots__ = ("user_id", "session_id", "admitted", "_kept")++ def __init__(self, user_id: str, session_id: str) -> None:+ self.user_id = user_id+ self.session_id = session_id+ self.admitted = False+ self._kept = False++ def keep(self) -> None:+ """Transfer slot ownership out of this context. Caller is now+ responsible for ensuring ``mark_session_completed`` releases the+ slot (or accepts the stale-cutoff fallback)."""+ self._kept = True+++@asynccontextmanager+async def acquire_turn_slot(+ user_id: str | None,+ session_id: str,+) -> AsyncIterator[TurnSlot]:+ """Reserve a turn slot for the duration of the ``async with`` block.++ Three branches on entry:++ * **Admitted** — fresh slot acquired; ``keep()`` transfers ownership+ to ``mark_session_completed``, otherwise the slot is released on+ exit.+ * **Refreshed** — same-``session_id`` re-entry (network retry,+ duplicate request); the existing slot's score is bumped but this+ caller does NOT own its release. Exiting without ``keep`` is a+ no-op.+ * **Rejected** — pool is at the configured cap; raises+ :class:`ConcurrentTurnLimitError` (caller maps to HTTP 429).++ Anonymous sessions (``user_id`` falsy) bypass the gate entirely and+ yield a no-op handle.+ """+ handle = TurnSlot(user_id or "", session_id)+ if user_id:+ outcome = await _try_admit_user_turn(user_id, session_id)+ if outcome is SlotAdmission.REJECTED:+ raise ConcurrentTurnLimitError()+ if outcome is SlotAdmission.ADMITTED:+ handle.admitted = True++ try:+ yield handle+ finally:+ if handle.admitted and not handle._kept:+ await release_turn_slot(handle.user_id, handle.session_id)diff --git a/autogpt_platform/backend/backend/copilot/active_turns_test.py b/autogpt_platform/backend/backend/copilot/active_turns_test.pynew file mode 100644index 0000000000..b86ad40173--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/active_turns_test.py@@ -0,0 +1,194 @@+"""Unit tests for active_turns: per-user concurrent AutoPilot turn tracking."""++from unittest.mock import AsyncMock, MagicMock++import pytest++from backend.copilot import active_turns as active_turns_module+from backend.copilot.active_turns import (+ ConcurrentTurnLimitError,+ acquire_turn_slot,+ release_turn_slot,+)+++def _patch_redis(monkeypatch: pytest.MonkeyPatch, redis_mock: MagicMock) -> None:+ monkeypatch.setattr(+ active_turns_module, "get_redis_async", AsyncMock(return_value=redis_mock)+ )+++def _redis_mock_eval(eval_return: int) -> MagicMock:+ """Build a mock Redis client whose ``eval`` returns the given Lua result.++ Lua return values match :class:`SlotAdmission`: 0=REJECTED, 1=ADMITTED, 2=REFRESHED.+ """+ redis_mock = MagicMock()+ redis_mock.eval = AsyncMock(return_value=eval_return)+ redis_mock.zrem = AsyncMock(return_value=1)+ return redis_mock+++# ── release_turn_slot ─────────────────────────────────────────────────+++async def test_release_zrems_session_from_user_pool(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ redis_mock = MagicMock()+ redis_mock.zrem = AsyncMock(return_value=1)+ _patch_redis(monkeypatch, redis_mock)++ await release_turn_slot("user-1", "session-a")+ redis_mock.zrem.assert_called_once()+ args, _ = redis_mock.zrem.call_args+ assert args[1] == "session-a"+ assert "user-1" in args[0]+++async def test_release_swallows_redis_errors(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Redis errors during release are logged, not raised — the slot+ will be swept on the next acquisition's stale-cutoff."""+ monkeypatch.setattr(+ active_turns_module,+ "get_redis_async",+ AsyncMock(side_effect=ConnectionError("down")),+ )+ await release_turn_slot("user-1", "session-a") # must not raise+++# ── acquire_turn_slot lifecycle ───────────────────────────────────────+++async def test_admitted_slot_releases_on_exit_without_keep(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Forgetting ``keep()`` on a clean exit releases the slot."""+ redis_mock = _redis_mock_eval(1) # ADMITTED+ _patch_redis(monkeypatch, redis_mock)++ async with acquire_turn_slot("user-1", "session-a"):+ pass++ redis_mock.zrem.assert_called_once()+++async def test_admitted_slot_releases_on_exception(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """An exception inside the with-block also releases the slot."""+ redis_mock = _redis_mock_eval(1) # ADMITTED+ _patch_redis(monkeypatch, redis_mock)++ with pytest.raises(RuntimeError, match="downstream blew up"):+ async with acquire_turn_slot("user-1", "session-a"):+ raise RuntimeError("downstream blew up")++ redis_mock.zrem.assert_called_once()+++async def test_kept_slot_is_not_released_on_exit(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """``keep()`` transfers ownership; the context manager leaves the slot held."""+ redis_mock = _redis_mock_eval(1) # ADMITTED+ _patch_redis(monkeypatch, redis_mock)++ async with acquire_turn_slot("user-1", "session-a") as slot:+ slot.keep()++ redis_mock.zrem.assert_not_called()+++async def test_rejection_raises_concurrent_turn_limit_error(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ redis_mock = _redis_mock_eval(0) # REJECTED+ _patch_redis(monkeypatch, redis_mock)++ with pytest.raises(ConcurrentTurnLimitError):+ async with acquire_turn_slot("user-1", "session-a"):+ pytest.fail("body should not run when acquire fails") # pragma: no cover++ # Reject path never admitted, so it must not release.+ redis_mock.zrem.assert_not_called()+++async def test_refreshed_slot_is_not_released_on_clean_exit(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Refreshed (re-entry) → caller doesn't own the slot, no release on exit."""+ redis_mock = _redis_mock_eval(2) # REFRESHED+ _patch_redis(monkeypatch, redis_mock)++ async with acquire_turn_slot("user-1", "session-a"):+ pass++ redis_mock.zrem.assert_not_called()+++async def test_refreshed_slot_is_not_released_on_exception(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Same-session retry's failure must NOT tear down the original turn's slot."""+ redis_mock = _redis_mock_eval(2) # REFRESHED+ _patch_redis(monkeypatch, redis_mock)++ with pytest.raises(RuntimeError, match="boom"):+ async with acquire_turn_slot("user-1", "session-a"):+ raise RuntimeError("boom")++ redis_mock.zrem.assert_not_called()+++async def test_anonymous_user_skips_gate(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """``user_id`` falsy → no slot acquired, no Redis touched, no exception."""+ redis_mock = _redis_mock_eval(0) # would reject if hit+ _patch_redis(monkeypatch, redis_mock)++ async with acquire_turn_slot(None, "session-a"):+ pass++ redis_mock.eval.assert_not_called()+ redis_mock.zrem.assert_not_called()+++async def test_redis_brownout_fails_open_admitting_the_turn(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Cap is a safeguard, not a budget — a Redis brown-out should not+ 429 every user. The context manager admits the turn and continues."""+ monkeypatch.setattr(+ active_turns_module,+ "get_redis_async",+ AsyncMock(side_effect=ConnectionError("down")),+ )++ async with acquire_turn_slot("user-1", "session-a") as slot:+ slot.keep() # body runs as if admitted; pretend a turn ran+++# ── default cap pinning ───────────────────────────────────────────────+++def test_schema_default_concurrent_turn_limit_is_15() -> None:+ """Pin the schema default so a config drift can't silently relax the+ abuse cap. Reads the field default directly so a local ``.env``+ override (e.g. lower cap for development) doesn't break the test."""+ from backend.util.settings import Config++ assert Config.model_fields["max_concurrent_copilot_turns_per_user"].default == 15diff --git a/autogpt_platform/backend/backend/copilot/executor/utils.py b/autogpt_platform/backend/backend/copilot/executor/utils.pyindex f386d90adf..71cae3eced 100644--- a/autogpt_platform/backend/backend/copilot/executor/utils.py+++ b/autogpt_platform/backend/backend/copilot/executor/utils.py@@ -9,6 +9,7 @@ import loggingfrom pydantic import BaseModel+from backend.copilot.active_turns import TurnSlot, acquire_turn_slotfrom backend.copilot.config import CopilotLlmModel, CopilotModefrom backend.copilot.permissions import CopilotPermissionsfrom backend.data.rabbitmq import Exchange, ExchangeType, Queue, RabbitMQConfig@@ -275,6 +276,204 @@ async def enqueue_copilot_turn()+async def schedule_turn(+ *,+ session_id: str,+ user_id: str | None,+ turn_id: str,+ message: str,+ tool_call_id: str = "chat_stream",+ tool_name: str = "chat",+ is_user_message: bool = True,+ context: dict[str, str] | None = None,+ file_ids: list[str] | None = None,+ mode: CopilotMode | None = None,+ model: CopilotLlmModel | None = None,+ permissions: CopilotPermissions | None = None,+ request_arrival_at: float = 0.0,+) -> None:+ """End-to-end "start a copilot turn": reserve a per-user concurrency+ slot, register the session in the stream registry, then publish the+ work to the executor queue.++ Convenience wrapper for callers that don't need to interleave any+ work between slot acquisition and dispatch (no message persistence,+ no duplicate-detection, etc.). Used by ``run_sub_session`` and+ ``AutoPilotBlock`` via ``session_waiter.run_copilot_turn_via_queue``.++ The HTTP chat route uses :func:`acquire_turn_slot` + :func:`dispatch_turn`+ directly because it must save the user message *inside* the slot+ context (so a 429 cannot leave a ghost message in chat history) and+ skip dispatch entirely when the message is a duplicate.++ Atomicity guarantees:++ * If the user is at the configured concurrent-turn cap,+ :class:`ConcurrentTurnLimitError` is raised before any side-effects+ (no stream registry write, no queue publish). Caller maps to HTTP 429.+ * If ``create_session`` or ``publish_message`` raises, the slot is+ auto-released by the :func:`acquire_turn_slot` context manager —+ a RabbitMQ blip cannot leak a slot until the stale-cutoff sweep.+ * On success the slot is *kept*: ownership transfers to+ ``mark_session_completed``, which releases it when the turn ends.+ """+ async with acquire_turn_slot(user_id, session_id) as slot:+ await dispatch_turn(+ slot,+ session_id=session_id,+ user_id=user_id,+ turn_id=turn_id,+ message=message,+ tool_call_id=tool_call_id,+ tool_name=tool_name,+ is_user_message=is_user_message,+ context=context,+ file_ids=file_ids,+ mode=mode,+ model=model,+ permissions=permissions,+ request_arrival_at=request_arrival_at,+ )+++async def dispatch_turn(+ slot: TurnSlot,+ *,+ session_id: str,+ user_id: str | None,+ turn_id: str,+ message: str,+ tool_call_id: str = "chat_stream",+ tool_name: str = "chat",+ is_user_message: bool = True,+ context: dict[str, str] | None = None,+ file_ids: list[str] | None = None,+ mode: CopilotMode | None = None,+ model: CopilotLlmModel | None = None,+ permissions: CopilotPermissions | None = None,+ request_arrival_at: float = 0.0,+) -> None:+ """Within an already-held turn slot, register the session in the+ stream registry, publish the work to the executor queue, and+ transfer slot ownership to ``mark_session_completed``.++ Caller is responsible for acquiring ``slot`` via+ :func:`acquire_turn_slot`. This function is the post-acquire dispatch+ glue — extracted so the HTTP chat route (which needs to save the+ user message between acquire and dispatch) can share the same+ create-session + enqueue + keep sequence as :func:`schedule_turn`'s+ callers without inlining it.++ Failure semantics: if ``create_session`` or the queue publish raises,+ ``slot.keep()`` is never reached, so the context manager that owns+ ``slot`` releases it on exit — no leak.+ """+ # Local import: stream_registry imports executor.utils (the+ # COPILOT_CONSUMER_TIMEOUT_SECONDS constant) → top-level circular.+ from backend.copilot import stream_registry++ await stream_registry.create_session(+ session_id=session_id,+ user_id=user_id,+ tool_call_id=tool_call_id,+ tool_name=tool_name,+ turn_id=turn_id,+ )+ await enqueue_copilot_turn(+ session_id=session_id,+ user_id=user_id,+ message=message,+ turn_id=turn_id,+ is_user_message=is_user_message,+ context=context,+ file_ids=file_ids,+ mode=mode,+ model=model,+ permissions=permissions,+ request_arrival_at=request_arrival_at,+ )+ slot.keep()+++async def schedule_chat_turn(+ *,+ session_id: str,+ user_id: str,+ message: str,+ message_id: str | None = None,+ is_user_message: bool = True,+ context: dict[str, str] | None = None,+ file_ids: list[str] | None = None,+ mode: CopilotMode | None = None,+ model: CopilotLlmModel | None = None,+ permissions: CopilotPermissions | None = None,+ request_arrival_at: float = 0.0,+) -> str | None:+ """HTTP-chat-flavoured :func:`schedule_turn`: acquires the user's+ concurrency slot, persists the user message *inside* the slot, then+ dispatches the turn — atomically, in that order.++ Returns the new ``turn_id`` on a fresh dispatch, or ``None`` if the+ inbound message was a duplicate of one already saved (caller should+ subscribe to the existing in-flight turn's stream instead of opening+ a new one).++ Raises :class:`backend.copilot.active_turns.ConcurrentTurnLimitError`+ when the user is at the configured cap. Caller maps that to HTTP 429.++ Acquire-before-persist is intentional: a 429 must not leave a ghost+ user-message in chat history with no answer ever scheduled. The+ re-acquire path of :func:`acquire_turn_slot` keeps same-session+ network retries safe — the existing slot's score is bumped, no new+ slot is admitted, and exiting without :meth:`TurnSlot.keep` does not+ release the slot held by the original turn.+ """+ # Deferred so the executor module stays a leaf for the queue dataclasses+ # (only the chat HTTP path persists user messages this way).+ from uuid import uuid4++ from backend.copilot.model import ChatMessage, append_and_save_message+ from backend.copilot.tracking import track_user_message++ async with acquire_turn_slot(user_id, session_id) as slot:+ is_duplicate = False+ if message:+ chat_message = ChatMessage(+ id=message_id,+ role="user" if is_user_message else "assistant",+ content=message,+ )+ is_duplicate = (+ await append_and_save_message(session_id, chat_message)+ ) is None+ if not is_duplicate and is_user_message:+ track_user_message(+ user_id=user_id,+ session_id=session_id,+ message_length=len(message),+ )++ if is_duplicate:+ return None++ turn_id = str(uuid4())+ await dispatch_turn(+ slot,+ session_id=session_id,+ user_id=user_id,+ turn_id=turn_id,+ message=message,+ is_user_message=is_user_message,+ context=context,+ file_ids=file_ids,+ mode=mode,+ model=model,+ permissions=permissions,+ request_arrival_at=request_arrival_at,+ )+ return turn_id++async def enqueue_cancel_task(session_id: str) -> None:"""Publish a cancel request for a running CoPilot session.diff --git a/autogpt_platform/backend/backend/copilot/sdk/session_waiter.py b/autogpt_platform/backend/backend/copilot/sdk/session_waiter.pyindex 69ec687c0b..f149990e60 100644--- a/autogpt_platform/backend/backend/copilot/sdk/session_waiter.py+++ b/autogpt_platform/backend/backend/copilot/sdk/session_waiter.py@@ -24,7 +24,8 @@ from dataclasses import dataclass, fieldfrom typing import TYPE_CHECKING, Literalfrom backend.copilot import stream_registry-from backend.copilot.executor.utils import enqueue_copilot_turn+from backend.copilot.active_turns import ConcurrentTurnLimitError+from backend.copilot.executor.utils import schedule_turnfrom backend.copilot.pending_message_helpers import (is_turn_in_flight,queue_user_message,@@ -39,7 +40,13 @@ if TYPE_CHECKING:logger = logging.getLogger(__name__)-SessionOutcome = Literal["completed", "failed", "running", "queued"]+SessionOutcome = Literal[+ "completed",+ "failed",+ "running",+ "queued",+ "rejected_concurrent_turn_cap",+]@dataclass@@ -195,21 +202,35 @@ async def run_copilot_turn_via_queue(observed.pending_buffer_length = state.buffer_lengthreturn outcome, observed+ # Same per-user concurrent-turn cap as the HTTP chat route via the+ # shared ``schedule_turn`` helper — without this a graph spawning N+ # AutoPilotBlocks or a tool-loop firing run_sub_session could fan+ # out unbounded copilot turns past the user's cap.turn_id = str(uuid.uuid4())- await stream_registry.create_session(- session_id=session_id,- user_id=user_id,- tool_call_id=tool_call_id,- tool_name=tool_name,- turn_id=turn_id,- )- await enqueue_copilot_turn(- session_id=session_id,- user_id=user_id,- message=message,- turn_id=turn_id,- permissions=permissions,- )+ try:+ await schedule_turn(+ session_id=session_id,+ user_id=user_id,+ turn_id=turn_id,+ message=message,+ tool_call_id=tool_call_id,+ tool_name=tool_name,+ permissions=permissions,+ )+ except ConcurrentTurnLimitError:+ # Sub-AutoPilot / run_sub_session caller is at the cap (this is+ # the graph-block / tool path, not the HTTP route). Use a+ # distinct ``rejected_concurrent_turn_cap`` outcome so callers+ # render an actionable "wait for an in-flight turn to finish"+ # error instead of a generic "see transcript" pointer (the+ # session was never created, so the transcript is empty).+ logger.warning(+ "[queue] session=%s user=%s rejected by concurrent-turn cap (tool=%s)",+ session_id[:12],+ user_id[:8] if user_id else "?",+ tool_name,+ )+ return "rejected_concurrent_turn_cap", SessionResult()return await wait_for_session_result(session_id=session_id,user_id=user_id,diff --git a/autogpt_platform/backend/backend/copilot/sdk/session_waiter_test.py b/autogpt_platform/backend/backend/copilot/sdk/session_waiter_test.pyindex dfdafa87ce..7d63181300 100644--- a/autogpt_platform/backend/backend/copilot/sdk/session_waiter_test.py+++ b/autogpt_platform/backend/backend/copilot/sdk/session_waiter_test.py@@ -46,7 +46,7 @@ async def test_queue_branch_timeout_zero_returns_immediately():new=create_session,),patch(- "backend.copilot.sdk.session_waiter.enqueue_copilot_turn",+ "backend.copilot.executor.utils.enqueue_copilot_turn",new=enqueue,),patch(@@ -98,7 +98,7 @@ async def test_queue_branch_positive_timeout_rides_inflight_turn():new=create_session,),patch(- "backend.copilot.sdk.session_waiter.enqueue_copilot_turn",+ "backend.copilot.executor.utils.enqueue_copilot_turn",new=enqueue,),patch(@@ -146,7 +146,7 @@ async def test_idle_session_enqueues_normally():new=create_session,),patch(- "backend.copilot.sdk.session_waiter.enqueue_copilot_turn",+ "backend.copilot.executor.utils.enqueue_copilot_turn",new=enqueue,),patch(@@ -167,3 +167,53 @@ async def test_idle_session_enqueues_normally():assert result.queued is Falsecreate_session.assert_awaited_once()enqueue.assert_awaited_once()+++async def test_idle_session_concurrent_turn_cap_returns_rejected_outcome():+ """Slot-cap rejection in ``schedule_turn`` surfaces as the dedicated+ ``rejected_concurrent_turn_cap`` outcome (not generic ``failed``) so+ callers can render an actionable message instead of pointing at an+ empty transcript."""+ from backend.copilot.active_turns import ConcurrentTurnLimitError++ create_session = AsyncMock()+ enqueue = AsyncMock()+ wait_result = AsyncMock()++ with (+ patch(+ "backend.copilot.sdk.session_waiter.is_turn_in_flight",+ new=AsyncMock(return_value=False),+ ),+ patch(+ "backend.copilot.executor.utils.acquire_turn_slot",+ side_effect=ConcurrentTurnLimitError(),+ ),+ patch(+ "backend.copilot.sdk.session_waiter.stream_registry.create_session",+ new=create_session,+ ),+ patch(+ "backend.copilot.executor.utils.enqueue_copilot_turn",+ new=enqueue,+ ),+ patch(+ "backend.copilot.sdk.session_waiter.wait_for_session_result",+ new=wait_result,+ ),+ ):+ outcome, result = await run_copilot_turn_via_queue(+ session_id="sess-idle",+ user_id="u1",+ message="kick off",+ timeout=0.1,+ tool_call_id="autopilot_block",+ tool_name="autopilot_block",+ )++ assert outcome == "rejected_concurrent_turn_cap"+ assert isinstance(result, SessionResult)+ create_session.assert_not_awaited()+ enqueue.assert_not_awaited()+ wait_result.assert_not_awaited()diff --git a/autogpt_platform/backend/backend/copilot/stream_registry.py b/autogpt_platform/backend/backend/copilot/stream_registry.pyindex 99b263069b..d3b9acfad0 100644--- a/autogpt_platform/backend/backend/copilot/stream_registry.py+++ b/autogpt_platform/backend/backend/copilot/stream_registry.py@@ -26,6 +26,7 @@ import orjsonfrom redis.exceptions import RedisErrorfrom backend.api.model import CopilotCompletionPayload+from backend.copilot.active_turns import release_turn_slotfrom backend.data.db_accessors import chat_dbfrom backend.data.notification_bus import (AsyncRedisNotificationEventBus,@@ -865,6 +866,16 @@ async def mark_session_completed(logger.debug(f"Session {session_id} already completed/failed, skipping")return False+ # Release the per-user concurrent-turn slot now that this turn is no+ # longer in flight. ``meta`` may be an empty dict if the session's+ # Redis key expired between the ``hgetall`` above and the CAS that+ # just succeeded — read ``user_id`` directly so we don't skip the+ # release on an empty-but-not-None payload. ``user_id`` is empty for+ # anonymous sessions; ``release_turn_slot`` is a no-op then.+ user_id = meta.get("user_id") or ""+ if user_id:+ await release_turn_slot(user_id, session_id)+# Force-release the executor's cluster lock so the next enqueued turn can# acquire it immediately. The lock holder's on_run_done will also release# (idempotent delete); doing it here unblocks cases where the task hangsdiff --git a/autogpt_platform/backend/backend/copilot/tools/run_sub_session.py b/autogpt_platform/backend/backend/copilot/tools/run_sub_session.pyindex 82bbc5e53b..81c94fa3f7 100644--- a/autogpt_platform/backend/backend/copilot/tools/run_sub_session.py+++ b/autogpt_platform/backend/backend/copilot/tools/run_sub_session.py@@ -27,6 +27,7 @@ import loggingimport timefrom typing import Any+from backend.copilot.active_turns import concurrent_turn_limit_messagefrom backend.copilot.constants import MAX_TOOL_WAIT_SECONDSfrom backend.copilot.context import get_current_permissionsfrom backend.copilot.model import ChatSession, create_chat_session, get_chat_session@@ -233,6 +234,21 @@ def response_from_outcome(elapsed_seconds=round(elapsed, 2),)+ if outcome == "rejected_concurrent_turn_cap":+ # No sub-session record / transcript exists yet — the per-user+ # concurrent-turn cap rejected before ``create_session`` ran.+ # Render the actionable message instead of a "see transcript"+ # pointer to nothing.+ return SubSessionStatusResponse(+ message=concurrent_turn_limit_message(),+ session_id=parent_session_id,+ status="error",+ sub_session_id=inner_session_id,+ sub_autopilot_session_id=inner_session_id,+ sub_autopilot_session_link=link,+ elapsed_seconds=round(elapsed, 2),+ )+if outcome == "failed":return SubSessionStatusResponse(message="Sub-AutoPilot failed. See the sub's transcript for details.",diff --git a/autogpt_platform/backend/backend/copilot/tools/sub_session_test.py b/autogpt_platform/backend/backend/copilot/tools/sub_session_test.pyindex 8c67ca0008..4d62ad0c22 100644--- a/autogpt_platform/backend/backend/copilot/tools/sub_session_test.py+++ b/autogpt_platform/backend/backend/copilot/tools/sub_session_test.py@@ -42,10 +42,13 @@ def mock_queue(monkeypatch):enqueue_cancel = AsyncMock()create_session = AsyncMock()- # run_sub_session calls enqueue_copilot_turn via session_waiter's- # run_copilot_turn_via_queue helper — patch at the helper's source.+ # run_sub_session reaches enqueue_copilot_turn / create_session via+ # session_waiter.run_copilot_turn_via_queue → schedule_turn → dispatch_turn.+ # Patch at the source modules (executor.utils for the queue publish,+ # stream_registry for the session-meta write) since neither+ # session_waiter nor schedule_turn holds a re-bound reference anymore.monkeypatch.setattr(- "backend.copilot.sdk.session_waiter.enqueue_copilot_turn",+ "backend.copilot.executor.utils.enqueue_copilot_turn",enqueue_turn,)monkeypatch.setattr(@@ -53,7 +56,7 @@ def mock_queue(monkeypatch):enqueue_cancel,)monkeypatch.setattr(- "backend.copilot.sdk.session_waiter.stream_registry.create_session",+ "backend.copilot.stream_registry.create_session",create_session,)return {diff --git a/autogpt_platform/backend/backend/data/redis_helpers.py b/autogpt_platform/backend/backend/data/redis_helpers.pyindex b549804259..3ddad1b675 100644--- a/autogpt_platform/backend/backend/data/redis_helpers.py+++ b/autogpt_platform/backend/backend/data/redis_helpers.py@@ -20,6 +20,7 @@ application code, add a helper here first — callers should not touchthis module can cover."""+from enum import IntEnumfrom typing import Any, castfrom backend.data.redis_client import AsyncRedisClient, RedisClient@@ -67,6 +68,48 @@ redis.call('EXPIRE', KEYS[2], tonumber(ARGV[5]))return redis.call('LLEN', KEYS[2])"""+# Atomically: sweep stale slots, refresh an existing slot's claim or add+# a new slot iff the pool is under capacity, then set the pool key's TTL.+# Returns 1 when a NEW slot was admitted, 2 when an EXISTING slot's claim+# was refreshed (no change to slot count), 0 when reservation was refused+# (pool full and slot wasn't already held).+#+# The new-vs-refreshed distinction lets callers tell apart a "fresh+# admission" (caller now owns the slot's release) from a "re-entrant+# touch" (someone else owns the release, caller is just bumping the+# heartbeat). Without it, a refresh path would `release_turn_slot` on+# context-manager exit and prematurely free a slot held by another+# concurrent caller for the same session_id.+#+# Used for distributed per-actor concurrency caps. A "pool" is a Redis+# sorted set whose members are active slot ids and whose scores are the+# slot's reservation timestamp. Stale slots (owner crashed without+# release) are reclaimed by the sweep so a one-time leak cannot+# permanently consume capacity.+#+# KEYS[1] pool key (sorted set)+# ARGV[1] slot id (member)+# ARGV[2] reservation score (typically now)+# ARGV[3] stale-cutoff score (slots with score <= this are dropped)+# ARGV[4] capacity (max concurrent slots before reservation is refused)+# ARGV[5] TTL seconds applied to the pool key on every successful reserve+_TRY_ACQUIRE_CONCURRENCY_SLOT_LUA = """+redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', ARGV[3])+local existing = redis.call('ZSCORE', KEYS[1], ARGV[1])+if existing then+ redis.call('ZADD', KEYS[1], ARGV[2], ARGV[1])+ redis.call('EXPIRE', KEYS[1], ARGV[5])+ return 2+end+local count = redis.call('ZCARD', KEYS[1])+if count >= tonumber(ARGV[4]) then+ return 0+end+redis.call('ZADD', KEYS[1], ARGV[2], ARGV[1])+redis.call('EXPIRE', KEYS[1], ARGV[5])+return 1+"""+async def incr_with_ttl(redis: AsyncRedisClient,@@ -183,6 +226,75 @@ async def capped_rpush_if_hash_field(return None if length < 0 else length+class SlotAdmission(IntEnum):+ """Result of :func:`try_acquire_concurrency_slot`.++ Distinguishes a fresh admission from a re-entrant refresh so callers+ that automatically release on exit (e.g. context managers) only do+ so for slots they actually admitted, not slots they just touched.+ """++ REJECTED = 0 # pool was full and slot wasn't already held+ ADMITTED = 1 # slot newly added to the pool+ REFRESHED = 2 # slot was already in the pool; score bumped only+++async def try_acquire_concurrency_slot(+ redis: AsyncRedisClient,+ *,+ pool_key: str,+ slot_id: str,+ score: float,+ capacity: int,+ stale_before_score: float,+ ttl_seconds: int,+) -> SlotAdmission:+ """Atomically reserve one of *capacity* slots in a Redis-backed pool.++ Use this whenever you need a distributed concurrency cap — "this+ actor may have at most N of X in flight at the same time". The pool+ is a sorted set whose members are active slot ids and whose scores+ are reservation timestamps; on every call we:++ 1. Sweep slots with ``score <= stale_before_score`` — slots whose+ holder crashed without releasing don't permanently consume capacity.+ 2. If ``slot_id`` is already in the pool, refresh its score+ (re-reservation is idempotent — same holder, same logical slot)+ and return :attr:`SlotAdmission.REFRESHED`.+ 3. Otherwise, reserve only if the post-sweep slot count is below+ ``capacity`` and return :attr:`SlotAdmission.ADMITTED`.+ 4. On a successful reserve, set ``ttl_seconds`` on the pool key as a+ belt-and-braces TTL for the case where the sweep ever stops.++ Returns the :class:`SlotAdmission` outcome — ``ADMITTED`` (newly+ added), ``REFRESHED`` (already held, score bumped), or ``REJECTED``+ (pool full).++ Why a Lua script: the reservation is conditional on the post-sweep+ count, and ``MULTI/EXEC`` cannot branch on intermediate replies.+ Without atomicity, two concurrent callers both read ``count =+ capacity - 1`` and both add, ending up over capacity.++ Redis Cluster: only ``KEYS[1]`` is touched, so callers are free to+ hash-tag *pool_key* (e.g. ``foo:{user_id}``) to colocate the pool+ on one shard without CROSSSLOT issues.+ """+ result = await cast(+ "Any",+ redis.eval(+ _TRY_ACQUIRE_CONCURRENCY_SLOT_LUA,+ 1,+ pool_key,+ slot_id,+ str(score),+ str(stale_before_score),+ str(capacity),+ str(ttl_seconds),+ ),+ )+ return SlotAdmission(int(result))++async def hash_compare_and_set(redis: AsyncRedisClient,key: str,diff --git a/autogpt_platform/backend/backend/util/settings.py b/autogpt_platform/backend/backend/util/settings.pyindex 76633c8147..e59b6aee9c 100644--- a/autogpt_platform/backend/backend/util/settings.py+++ b/autogpt_platform/backend/backend/util/settings.py@@ -174,6 +174,17 @@ class Config(UpdateTrackingModel["Config"], BaseSettings):le=1000,description="Maximum number of concurrent graph executions allowed per user per graph.",)+ max_concurrent_copilot_turns_per_user: int = Field(+ default=15,+ ge=1,+ le=1000,+ description=(+ "Hard cap on concurrent in-flight AutoPilot/CoPilot chat turns "+ "per user. Reached via the chat HTTP route, the AutoPilotBlock, "+ "and run_sub_session — all funnel through schedule_turn() and "+ "share this cap. The user-facing 429 message renders this value."+ ),+ )block_error_rate_threshold: float = Field(default=0.5,diff --git a/autogpt_platform/frontend/src/app/api/openapi.json b/autogpt_platform/frontend/src/app/api/openapi.jsonindex 0935cd744e..d152d6e2e0 100644--- a/autogpt_platform/frontend/src/app/api/openapi.json+++ b/autogpt_platform/frontend/src/app/api/openapi.json@@ -2602,7 +2602,7 @@}},"429": {- "description": "Cost rate-limit or call-frequency cap exceeded"+ "description": "Cost rate-limit, call-frequency cap, or per-user concurrent-turn limit exceeded"},"503": {"description": "Chat service degraded (Redis unavailable for rate limit or stream registry); client should honour the Retry-After header before retrying."diff --git a/docs/integrations/block-integrations/misc.md b/docs/integrations/block-integrations/misc.mdindex 72ede9c537..f213b5ebfe 100644--- a/docs/integrations/block-integrations/misc.md+++ b/docs/integrations/block-integrations/misc.md@@ -58,7 +58,7 @@ Tool and block identifiers provided in `tools` and `blocks` are validated at run| system_context | Optional additional context prepended to the prompt. Use this to constrain autopilot behavior, provide domain context, or set output format requirements. | str | No || session_id | Session ID to continue an existing autopilot conversation. Leave empty to start a new session. Use the session_id output from a previous run to continue. | str | No || max_recursion_depth | Maximum nesting depth when the autopilot calls this block recursively (sub-agent pattern). Prevents infinite loops. | int | No |-| tools | Tool names to filter. Works with tools_exclude to form an allow-list or deny-list. Leave empty to apply no tool filter. | List["add_understanding" \| "bash_exec" \| "browser_act" \| "browser_navigate" \| "browser_screenshot" \| "connect_integration" \| "continue_run_block" \| "create_agent" \| "create_feature_request" \| "create_folder" \| "customize_agent" \| "delete_folder" \| "delete_workspace_file" \| "edit_agent" \| "find_agent" \| "find_block" \| "find_library_agent" \| "fix_agent_graph" \| "get_agent_building_guide" \| "get_doc_page" \| "get_mcp_guide" \| "get_sub_session_result" \| "list_folders" \| "list_workspace_files" \| "memory_forget_confirm" \| "memory_forget_search" \| "memory_search" \| "memory_store" \| "move_agents_to_folder" \| "move_folder" \| "read_workspace_file" \| "run_agent" \| "run_block" \| "run_mcp_tool" \| "run_sub_session" \| "search_docs" \| "search_feature_requests" \| "update_folder" \| "validate_agent_graph" \| "view_agent_output" \| "web_fetch" \| "web_search" \| "write_workspace_file" \| "Agent" \| "Edit" \| "Glob" \| "Grep" \| "Read" \| "Task" \| "TodoWrite" \| "WebSearch" \| "Write"] | No |+| tools | Tool names to filter. Works with tools_exclude to form an allow-list or deny-list. Leave empty to apply no tool filter. | List["add_understanding" \| "bash_exec" \| "browser_act" \| "browser_navigate" \| "browser_screenshot" \| "connect_integration" \| "continue_run_block" \| "create_agent" \| "create_feature_request" \| "create_folder" \| "customize_agent" \| "delete_folder" \| "delete_workspace_file" \| "edit_agent" \| "find_agent" \| "find_block" \| "find_library_agent" \| "fix_agent_graph" \| "get_agent_building_guide" \| "get_doc_page" \| "get_mcp_guide" \| "get_platform_info" \| "get_sub_session_result" \| "list_folders" \| "list_workspace_files" \| "memory_forget_confirm" \| "memory_forget_search" \| "memory_search" \| "memory_store" \| "move_agents_to_folder" \| "move_folder" \| "read_workspace_file" \| "run_agent" \| "run_block" \| "run_mcp_tool" \| "run_sub_session" \| "search_docs" \| "search_feature_requests" \| "update_folder" \| "validate_agent_graph" \| "view_agent_output" \| "web_fetch" \| "web_search" \| "write_workspace_file" \| "Agent" \| "Edit" \| "Glob" \| "Grep" \| "Read" \| "Task" \| "TodoWrite" \| "WebSearch" \| "Write"] | No || tools_exclude | Controls how the 'tools' list is interpreted. True (default): 'tools' is a deny-list — listed tools are blocked, all others are allowed. An empty 'tools' list means allow everything. False: 'tools' is an allow-list — only listed tools are permitted. | bool | No || blocks | Block identifiers to filter when the copilot uses run_block. Each entry can be: a block name (e.g. 'HTTP Request'), a full block UUID, or the first 8 hex characters of the UUID (e.g. 'c069dc6b'). Works with blocks_exclude. Leave empty to apply no block filter. | List[str] | No || blocks_exclude | Controls how the 'blocks' list is interpreted. True (default): 'blocks' is a deny-list — listed blocks are blocked, all others are allowed. An empty 'blocks' list means allow everything. False: 'blocks' is an allow-list — only listed blocks are permitted. | bool | No |