AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(backend/copilot): a sub-session that finds the user's slots full waits for one (#15361)
Reinier van der Leer81d43f2982fe
diff --git a/autogpt_platform/backend/backend/copilot/active_turns.py b/autogpt_platform/backend/backend/copilot/active_turns.pyindex 8d6864646d..be3aa6dc3c 100644--- a/autogpt_platform/backend/backend/copilot/active_turns.py+++ b/autogpt_platform/backend/backend/copilot/active_turns.py@@ -86,13 +86,12 @@ def running_turn_limit_message(limit: int | None = None) -> str:)-def delegated_turn_limit_message() -> str:- """What a spawn tool tells the model when its turn hit the delegated cap."""+def spawn_limit_message() -> str:+ """What a spawn tool tells the model when its turn could not even queue."""return (- f"Sub-work may use {get_delegated_turn_limit()} of the user's "- f"{get_running_turn_limit()} task slots, and those are all taken (the "- "last slot is kept for the user's own messages). Wait for a running "- "task to finish before starting another."+ f"The user already has {get_inflight_turn_limit()} tasks running or "+ "waiting to start, the most they may have. Wait for one to finish "+ "before starting another.")diff --git a/autogpt_platform/backend/backend/copilot/active_turns_db_test.py b/autogpt_platform/backend/backend/copilot/active_turns_db_test.pyindex 5d8fd638f8..b3c919d3f9 100644--- a/autogpt_platform/backend/backend/copilot/active_turns_db_test.py+++ b/autogpt_platform/backend/backend/copilot/active_turns_db_test.py@@ -84,7 +84,7 @@ async def test_a_chat_fanning_out_six_sub_sessions_leaves_the_user_a_slot():async with acquire_turn_slot(user_id, chat.session_id) as slot:assert slot.admitted- assert outcomes == ["running"] * 3 + ["rejected_concurrent_turn_cap"] * 3+ assert outcomes == ["running"] * 3 + ["queued_for_slot"] * 3finally:await User.prisma().delete(where={"id": user_id})diff --git a/autogpt_platform/backend/backend/copilot/executor/utils.py b/autogpt_platform/backend/backend/copilot/executor/utils.pyindex 21c1b7989b..e52c90d627 100644--- a/autogpt_platform/backend/backend/copilot/executor/utils.py+++ b/autogpt_platform/backend/backend/copilot/executor/utils.py@@ -667,6 +667,56 @@ async def dispatch_turn()+async def queue_spawned_turn(+ *,+ session_id: str,+ user_id: str,+ message: str,+ tool_call_id: str,+ tool_name: str,+ llm_auth_provider: CopilotLlmAuthProvider,+ llm_credential_id: str | None,+ permissions: CopilotPermissions | None,+ spawn: SpawnRequest | None,+ message_metadata: dict[str, Any] | None,+) -> None:+ """Queue a turn another session started, to begin when a slot frees.++ Its envelope is derived and admitted now, inside the spawning turn it comes+ from; promotion re-checks it rather than deriving it from whichever turn+ frees the slot. Raises :class:`ConcurrentTurnLimitError` at the inflight+ cap, :class:`TreeRefusal` when the tree refuses it and+ :class:`SessionNotIdle` when the session already has a turn.+ """+ from backend.copilot.turn_queue import count_inflight_turns, enqueue_turn++ inflight_cap = get_inflight_turn_limit()+ if await count_inflight_turns(user_id) >= inflight_cap:+ raise ConcurrentTurnLimitError(inflight_turn_limit_message(inflight_cap))+ envelope = await _admitted_turn_envelope(+ str(uuid4()), session_id, user_id, permissions, spawn+ )+ try:+ await enqueue_turn(+ user_id=user_id,+ session_id=session_id,+ message=message,+ message_metadata=message_metadata,+ llm_auth_provider=llm_auth_provider,+ llm_credential_id=llm_credential_id,+ permissions=(+ permissions.model_dump(exclude_none=True) if permissions else None+ ),+ envelope=envelope,+ tool_call_id=tool_call_id,+ tool_name=tool_name,+ only_if_idle=True,+ )+ except BaseException:+ await release_turn(envelope)+ raise++async def _admitted_turn_envelope(turn_id: str,session_id: str,diff --git a/autogpt_platform/backend/backend/copilot/queued_spawns_db_test.py b/autogpt_platform/backend/backend/copilot/queued_spawns_db_test.pynew file mode 100644index 0000000000..67980df887--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/queued_spawns_db_test.py@@ -0,0 +1,408 @@+"""Sub-sessions that wait for a slot, against the database: the real spawn,+queue, promotion and completion path, with only the executor's queue publish+stubbed."""++import asyncio+import uuid+from collections.abc import AsyncIterator+from typing import Any+from unittest.mock import AsyncMock, patch++import pytest+import pytest_asyncio+from prisma.models import User+from pydantic import BaseModel, ConfigDict++from backend.api.features.chat.routes import cancel_session_task+from backend.copilot import active_turns, stream_registry, turn_queue+from backend.copilot.active_turns import acquire_turn_slot+from backend.copilot.context import set_execution_context+from backend.copilot.executor.utils import queue_spawned_turn+from backend.copilot.model import (+ CHAT_STATUS_IDLE,+ CHAT_STATUS_QUEUED,+ CHAT_STATUS_RUNNING,+ ChatSession,+ create_chat_session,+ get_chat_session,+)+from backend.copilot.permissions import ALL_TOOL_NAMES, CopilotPermissions+from backend.copilot.sdk.session_waiter import (+ SessionResult,+ run_copilot_turn_via_queue,+ wait_for_queued_session,+)+from backend.copilot.tools.get_sub_session_result import GetSubSessionResultTool+from backend.copilot.tree import (+ SpawnRequest,+ TurnEnvelope,+ get_tree_ledger,+ root_envelope,+)+from backend.data.db import prisma as db_client+from backend.data.db_accessors import chat_db+from backend.data.redis_client import get_redis_async+from backend.util.test import SpinTestServer++[email protected](loop_scope="session")+async def test_six_sub_sessions_from_one_turn_all_run_four_at_a_time(user: "_User"):+ """The 18:09Z shape: one turn starts six, three run and three wait; a+ message the user sends meanwhile runs at once and holds the fifth slot."""+ spawner = await user.running_chat()+ subs = await user.spawn(spawner, 6)+ assert [o for o, _ in subs] == ["running"] * 3 + ["queued_for_slot"] * 3++ typed = await user.chat()+ async with acquire_turn_slot(user.id, typed.session_id) as slot:+ assert slot.admitted+ await user.start(typed.session_id, slot)++ # The user's turn holds a slot, so the spawner's end frees none for sub-work.+ await user.end(spawner.session_id)+ assert user.dispatched_sessions() == [s for _, s in subs[:3]]+ await user.end(typed.session_id)+ for _, sub in subs:+ await user.end(sub)++ assert user.dispatched_sessions() == [s for _, s in subs]+ for _, sub in subs[3:]:+ promoted = user.dispatch_of(sub)+ assert promoted["envelope"].depth == 1+ assert promoted["envelope"].tree_id == spawner.tree_id+ meta = await stream_registry.get_session(sub)+ assert meta is not None+ assert (meta.tool_name, meta.tool_call_id) == (+ "run_sub_session",+ f"sub:{spawner.session_id}",+ )+ for _, sub in subs:+ assert await chat_db().get_chat_session_status(sub) == CHAT_STATUS_IDLE+ # The root plus one node per child: a promotion takes no second node.+ assert (await user.nodes(spawner.tree_id)) == 7++[email protected](loop_scope="session")+async def test_a_queued_sub_session_whose_tree_closed_is_refused_with_the_reason(+ user: "_User",+):+ spawner = await user.running_chat()+ subs = await user.spawn(spawner, 5)+ queued = [s for o, s in subs if o == "queued_for_slot"]+ assert len(queued) == 2+ await (await get_redis_async()).delete(+ (await get_tree_ledger()).key(spawner.tree_id)+ )++ await user.end(spawner.session_id)++ for sub in queued:+ assert await chat_db().get_chat_session_status(sub) == CHAT_STATUS_IDLE+ outcome, result = await wait_for_queued_session(+ session_id=sub, user_id=user.id, timeout=0+ )+ assert outcome == "refused"+ assert "tree has closed" in result.refusal+ assert sub not in user.dispatched_sessions()++[email protected](loop_scope="session")+async def test_cancelling_a_queued_sub_session_takes_it_out_and_returns_its_node(+ user: "_User",+):+ spawner = await user.running_chat()+ subs = await user.spawn(spawner, 4)+ (queued,) = [s for o, s in subs if o == "queued_for_slot"]+ tool = GetSubSessionResultTool()+ polled = await tool._execute(+ user.id, spawner.session, sub_session_id=queued, wait_if_running=0+ )+ assert polled.status == "queued"+ nodes = await user.nodes(spawner.tree_id)++ cancelled = await tool._execute(+ user.id, spawner.session, sub_session_id=queued, cancel=True+ )+ await user.end(spawner.session_id)++ assert cancelled.status == "cancelled"+ assert await chat_db().get_chat_session_status(queued) == CHAT_STATUS_IDLE+ # Never started, so a later poll reports the cancel, not a turn running.+ polled = await tool._execute(+ user.id, spawner.session, sub_session_id=queued, wait_if_running=0+ )+ assert (polled.status, polled.message) == ("error", turn_queue.TURN_CANCELLED)+ assert await user.nodes(spawner.tree_id) == nodes - 1+ assert queued not in user.dispatched_sessions()++[email protected](loop_scope="session")+async def test_cancelling_a_queued_sub_session_over_http_returns_its_node(+ user: "_User",+):+ """The chat page's Stop button dequeues through the same path."""+ spawner = await user.running_chat()+ subs = await user.spawn(spawner, 4)+ (queued,) = [s for o, s in subs if o == "queued_for_slot"]+ nodes = await user.nodes(spawner.tree_id)++ response = await cancel_session_task(session_id=queued, user_id=user.id)+ await user.end(spawner.session_id)++ assert response.reason == "dequeued"+ assert await chat_db().get_chat_session_status(queued) == CHAT_STATUS_IDLE+ assert await user.nodes(spawner.tree_id) == nodes - 1+ assert queued not in user.dispatched_sessions()++[email protected](loop_scope="session")+async def test_a_queued_sub_session_cancelled_while_awaited_reports_the_cancel(+ user: "_User",+):+ """Taken out of the queue while its spawner waits on it: it never started,+ so the wait ends with the cancel rather than reading a turn as running."""+ spawner = await user.running_chat()+ subs = await user.spawn(spawner, 4)+ (queued,) = [s for o, s in subs if o == "queued_for_slot"]++ waiting = asyncio.create_task(+ wait_for_queued_session(session_id=queued, user_id=user.id, timeout=10)+ )+ await asyncio.sleep(0.2)+ assert await turn_queue.cancel_queued_turn(user_id=user.id, session_id=queued)+ outcome, result = await asyncio.wait_for(waiting, timeout=5)+ await user.end(spawner.session_id)++ assert (outcome, result.refusal) == ("refused", turn_queue.TURN_CANCELLED)++[email protected](loop_scope="session")+async def test_two_children_queued_into_one_session_admit_one_node(user: "_User"):+ """The second finds the session already queued: it is refused and its node+ goes back, rather than waiting on a turn that replays the other's row."""+ spawner = await user.running_chat()+ await user.spawn(spawner, 1) # opens the spawner's tree ledger+ sub = await create_chat_session(+ user.id,+ dry_run=False,+ origin="automation",+ delegated_by_session_id=spawner.session_id,+ )+ nodes = await user.nodes(spawner.tree_id)++ async def in_the_spawners_turn() -> list[Any]:+ set_execution_context(user.id, spawner.session, envelope=spawner.envelope)+ return await asyncio.gather(+ *(+ queue_spawned_turn(+ session_id=sub.session_id,+ user_id=user.id,+ message=f"task {i}",+ tool_call_id=f"sub:{spawner.session_id}",+ tool_name="run_sub_session",+ llm_auth_provider="platform",+ llm_credential_id=None,+ permissions=None,+ spawn=SpawnRequest(may_spawn=True),+ message_metadata=None,+ )+ for i in range(2)+ ),+ return_exceptions=True,+ )++ results = await asyncio.create_task(in_the_spawners_turn())++ assert sorted(type(r).__name__ for r in results) == ["NoneType", "SessionNotIdle"]+ assert await user.nodes(spawner.tree_id) == nodes + 1+ # The loser wrote nothing: one row, the winner's, for promotion to replay.+ queued = await get_chat_session(sub.session_id, user.id)+ assert queued is not None+ assert [m.content for m in queued.messages if m.role == "user"] in (+ ["task 0"],+ ["task 1"],+ )+ assert await chat_db().get_chat_session_status(sub.session_id) == CHAT_STATUS_QUEUED+ assert await turn_queue.cancel_queued_turn(+ user_id=user.id, session_id=sub.session_id+ )++[email protected](loop_scope="session")+async def test_the_inflight_cap_still_refuses_a_spawn(user: "_User"):+ spawner = await user.running_chat()+ with patch(+ "backend.copilot.executor.utils.get_inflight_turn_limit", return_value=6+ ):+ subs = await user.spawn(spawner, 6)++ assert [o for o, _ in subs] == (+ ["running"] * 3 + ["queued_for_slot"] * 2 + ["rejected_concurrent_turn_cap"]+ )++[email protected](loop_scope="session")+async def test_a_tool_revoked_while_a_sub_session_waited_stays_revoked(+ user: "_User",+):+ spawner = await user.running_chat()+ subs = await user.spawn(spawner, 4)+ (queued,) = [s for o, s in subs if o == "queued_for_slot"]+ revoked = CopilotPermissions(tools=["web_fetch"], tools_exclude=True)++ with patch.object(turn_queue, "resolve_session_permissions", return_value=revoked):+ await user.end(spawner.session_id)++ promoted = user.dispatch_of(queued)+ granted = promoted["permissions"].effective_allowed_tools(ALL_TOOL_NAMES)+ assert "web_fetch" not in granted+ assert "web_search" in granted+ assert not promoted["envelope"].permits("web_fetch")+++class _Spawner(BaseModel):+ model_config = ConfigDict(arbitrary_types_allowed=True)++ session: ChatSession+ envelope: TurnEnvelope++ @property+ def session_id(self) -> str:+ return self.session.session_id++ @property+ def tree_id(self) -> str:+ return self.envelope.tree_id+++class _User:+ """A throwaway user whose turns publish to a stub instead of the executor."""++ def __init__(self, user_id: str, enqueued: AsyncMock) -> None:+ self.id = user_id+ self._enqueued = enqueued+ self._turns: dict[str, str] = {}++ async def chat(self) -> ChatSession:+ return await create_chat_session(self.id, dry_run=False)++ async def running_chat(self) -> _Spawner:+ session = await self.chat()+ turn_id = str(uuid.uuid4())+ assert await chat_db().update_chat_session_status(+ session_id=session.session_id,+ expect_status=CHAT_STATUS_IDLE,+ status=CHAT_STATUS_RUNNING,+ user_id=self.id,+ )+ await stream_registry.create_session(+ session.session_id, self.id, "chat_stream", "chat", turn_id+ )+ self._turns[session.session_id] = turn_id+ reloaded = await get_chat_session(session.session_id, self.id)+ assert reloaded is not None+ return _Spawner(+ session=reloaded,+ envelope=root_envelope(turn_id, session_id=session.session_id),+ )++ async def spawn(self, spawner: _Spawner, count: int) -> list[tuple[str, str]]:+ """``run_sub_session``'s spawn, ``count`` times, from inside the+ spawner's turn: each opens a delegated session and starts its turn."""++ async def in_the_spawners_turn() -> list[tuple[str, str]]:+ set_execution_context(self.id, spawner.session, envelope=spawner.envelope)+ spawned = []+ for _ in range(count):+ sub = await create_chat_session(+ self.id,+ dry_run=False,+ origin="automation",+ delegated_by_session_id=spawner.session_id,+ )+ outcome, _ = await run_copilot_turn_via_queue(+ session_id=sub.session_id,+ user_id=self.id,+ message="test a stream",+ timeout=0,+ tool_call_id=f"sub:{spawner.session_id}",+ tool_name="run_sub_session",+ spawn=SpawnRequest(may_spawn=True, shares_memory=True),+ allow_queue=False,+ )+ spawned.append((outcome, sub.session_id))+ return spawned++ return await asyncio.create_task(in_the_spawners_turn())++ async def start(self, session_id: str, slot) -> None:+ turn_id = str(uuid.uuid4())+ await stream_registry.create_session(+ session_id, self.id, "chat_stream", "chat", turn_id+ )+ self._turns[session_id] = turn_id+ slot.keep()++ async def end(self, session_id: str) -> None:+ """The executor's end of a turn, which promotes the next queued one."""+ turn_id = self._turns.get(session_id) or self.dispatch_of(session_id)["turn_id"]+ await stream_registry.mark_session_completed(session_id, turn_id=turn_id)++ def dispatched_sessions(self) -> list[str]:+ return [c.kwargs["session_id"] for c in self._enqueued.await_args_list]++ def dispatch_of(self, session_id: str) -> dict[str, Any]:+ (call,) = [+ c+ for c in self._enqueued.await_args_list+ if c.kwargs["session_id"] == session_id+ ]+ return call.kwargs++ async def nodes(self, tree_id: str) -> int:+ return (await (await get_tree_ledger()).snapshot(tree_id))["nodes"]+++@pytest_asyncio.fixture+async def user(server: SpinTestServer) -> AsyncIterator[_User]:+ user_id = str(uuid.uuid4())+ await User.prisma().create(+ data={"id": user_id, "email": f"queued-spawns-{user_id}@example.com"}+ )+ enqueued = AsyncMock()+ try:+ with (+ patch("backend.copilot.executor.utils.enqueue_copilot_turn", new=enqueued),+ patch.object(active_turns, "get_running_turn_limit", return_value=5),+ patch(+ "backend.copilot.executor.utils.get_inflight_turn_limit",+ return_value=15,+ ),+ patch.object(+ turn_queue, "is_user_paywalled", new=AsyncMock(return_value=False)+ ),+ patch.object(+ turn_queue,+ "get_global_rate_limits",+ new=AsyncMock(return_value=(1, 1, None)),+ ),+ patch.object(turn_queue, "check_rate_limit", new=AsyncMock()),+ patch(+ "backend.copilot.sdk.session_waiter.wait_for_session_result",+ new=AsyncMock(return_value=("running", SessionResult())),+ ),+ ):+ yield _User(user_id, enqueued)+ finally:+ await User.prisma().delete(where={"id": user_id})+++@pytest_asyncio.fixture(autouse=True)+async def absorb_a_stale_event_loop(server: SpinTestServer):+ """An earlier test can leave the shared Prisma client bound to a closed+ loop; only the first query on the new loop fails. Spend it here."""+ try:+ await db_client.execute_raw("SELECT 1")+ except RuntimeError as error:+ if "Event loop is closed" not in str(error):+ raisediff --git a/autogpt_platform/backend/backend/copilot/sdk/session_waiter.py b/autogpt_platform/backend/backend/copilot/sdk/session_waiter.pyindex 3aae035925..b1251f8b10 100644--- a/autogpt_platform/backend/backend/copilot/sdk/session_waiter.py+++ b/autogpt_platform/backend/backend/copilot/sdk/session_waiter.py@@ -19,20 +19,28 @@ from __future__ import annotationsimport asyncioimport logging+import timeimport uuidfrom dataclasses import dataclass, fieldfrom typing import TYPE_CHECKING, Any, Literalfrom backend.copilot import stream_registryfrom backend.copilot.active_turns import ConcurrentTurnLimitError-from backend.copilot.executor.utils import schedule_turn-from backend.copilot.model import get_chat_session+from backend.copilot.config import CopilotLlmAuthProvider+from backend.copilot.executor.utils import queue_spawned_turn, schedule_turn+from backend.copilot.model import (+ CHAT_STATUS_QUEUED,+ _get_session_lock,+ get_chat_session,+)from backend.copilot.pending_message_helpers import (is_turn_in_flight,queue_user_message,)from backend.copilot.response_model import StreamError, StreamFinishfrom backend.copilot.tree import SpawnRequest, TreeRefusal+from backend.copilot.turn_queue import SessionNotIdle, queued_turn_refusal+from backend.data.db_accessors import chat_dbfrom .stream_accumulator import EventAccumulator, ToolCallEntry, process_event@@ -47,10 +55,19 @@ SessionOutcome = Literal["failed","running","queued",+ # Waiting for one of the user's task slots; it starts on its own.+ "queued_for_slot","rejected_concurrent_turn_cap","refused",]+# How often a waiter checks whether a queued turn has started.+_QUEUE_POLL_SECONDS = 1.0+_ALREADY_WAITING = (+ "That session already has a task waiting to start, so this task cannot be "+ "handed to it right now. Wait for it, or start a fresh one."+)+@dataclassclass SessionResult:@@ -197,6 +214,10 @@ async def run_copilot_turn_via_queue(if session is None:raise RuntimeError("copilot_session_not_found")+ if not allow_queue and (+ await chat_db().get_chat_session_status(session_id) == CHAT_STATUS_QUEUED+ ):+ return "refused", SessionResult(refusal=_ALREADY_WAITING)if await is_turn_in_flight(session_id):if not allow_queue:return "refused", SessionResult(@@ -242,6 +263,7 @@ async def run_copilot_turn_via_queue(# AutoPilotBlocks or a tool-loop firing run_sub_session could fan# out unbounded copilot turns past the user's cap.turn_id = str(uuid.uuid4())+ delegated = session.metadata.delegated_by_session_id is not Nonetry:await schedule_turn(session_id=session_id,@@ -255,11 +277,25 @@ async def run_copilot_turn_via_queue(permissions=permissions,spawn=spawn,message_metadata=message_metadata,- delegated=session.metadata.delegated_by_session_id is not None,+ delegated=delegated,)except TreeRefusal as refused:return "refused", SessionResult(refusal=refused.message)except ConcurrentTurnLimitError:+ if delegated:+ return await _queue_for_slot(+ session_id=session_id,+ user_id=user_id,+ message=message,+ timeout=timeout,+ permissions=permissions,+ tool_call_id=tool_call_id,+ tool_name=tool_name,+ spawn=spawn,+ message_metadata=message_metadata,+ llm_auth_provider=session.metadata.llm_auth_provider,+ llm_credential_id=session.metadata.llm_credential_id,+ )# Sub-Otto / run_sub_session caller is at the cap (this is# the graph-block / tool path, not the HTTP route). Use a# distinct ``rejected_concurrent_turn_cap`` outcome so callers@@ -278,3 +314,72 @@ async def run_copilot_turn_via_queue(user_id=user_id,timeout=timeout,)+++async def wait_for_queued_session(+ *, session_id: str, user_id: str, timeout: float+) -> tuple[SessionOutcome, SessionResult]:+ """Wait for a queued turn to start, then for its result, all within+ ``timeout``. A turn closed at promotion, or cancelled before it started,+ comes back ``refused`` with the note that closed it."""+ deadline = time.monotonic() + timeout+ while await chat_db().get_chat_session_status(session_id) == CHAT_STATUS_QUEUED:+ remaining = deadline - time.monotonic()+ if remaining <= 0:+ return "queued_for_slot", SessionResult(queued=True)+ await asyncio.sleep(min(_QUEUE_POLL_SECONDS, remaining))+ # A cancel or a refusal writes its note under this lock with the flip seen.+ async with _get_session_lock(session_id):+ session = await get_chat_session(session_id, user_id)+ refusal = queued_turn_refusal(session) if session else None+ if refusal is not None:+ return "refused", SessionResult(refusal=refusal)+ return await wait_for_session_result(+ session_id=session_id,+ user_id=user_id,+ timeout=max(deadline - time.monotonic(), 0),+ )+++async def _queue_for_slot(+ *,+ session_id: str,+ user_id: str,+ message: str,+ timeout: float,+ permissions: "CopilotPermissions | None",+ tool_call_id: str,+ tool_name: str,+ spawn: SpawnRequest | None,+ message_metadata: dict[str, Any] | None,+ llm_auth_provider: CopilotLlmAuthProvider,+ llm_credential_id: str | None,+) -> tuple[SessionOutcome, SessionResult]:+ """A spawn the reserve refused waits in the queue instead, up to the+ inflight cap; past it the spawn is refused as before."""+ try:+ await queue_spawned_turn(+ session_id=session_id,+ user_id=user_id,+ message=message,+ tool_call_id=tool_call_id,+ tool_name=tool_name,+ llm_auth_provider=llm_auth_provider,+ llm_credential_id=llm_credential_id,+ permissions=permissions,+ spawn=spawn,+ message_metadata=message_metadata,+ )+ except TreeRefusal as refused:+ return "refused", SessionResult(refusal=refused.message)+ except SessionNotIdle:+ return "refused", SessionResult(refusal=_ALREADY_WAITING)+ except ConcurrentTurnLimitError:+ logger.warning(+ f"[queue] session={session_id[:12]} user={user_id[:8]} rejected at the "+ f"inflight cap (tool={tool_name})"+ )+ return "rejected_concurrent_turn_cap", SessionResult()+ return await wait_for_queued_session(+ session_id=session_id, user_id=user_id, timeout=timeout+ )diff --git a/autogpt_platform/backend/backend/copilot/sdk/session_waiter_test.py b/autogpt_platform/backend/backend/copilot/sdk/session_waiter_test.pyindex 6ca24f8841..d1018bb798 100644--- a/autogpt_platform/backend/backend/copilot/sdk/session_waiter_test.py+++ b/autogpt_platform/backend/backend/copilot/sdk/session_waiter_test.py@@ -235,8 +235,9 @@ async def test_idle_session_enqueues_normally():@pytest.mark.parametrize("delegated_by, inflight, expected",[- # A sub-session's turn with four running may not take the fifth slot.- ("parent-session", 4, "rejected_concurrent_turn_cap"),+ # A sub-session's turn with four running waits for a slot instead of+ # taking the fifth.+ ("parent-session", 4, "queued_for_slot"),# An AutoPilotBlock turn keeps the inflight cap...(None, 4, "completed"),# ...which still refuses at its number.@@ -254,6 +255,8 @@ async def test_a_delegated_turn_leaves_the_users_last_slot_free(side_effect=lambda **kw: "full" if 4 >= kw["capacity"] else "admitted")db.get_chat_session_status = AsyncMock(return_value="idle")+ queued = MagicMock()+ queued.get_chat_session_status = AsyncMock(return_value="queued")with (patch(@@ -275,6 +278,8 @@ async def test_a_delegated_turn_leaves_the_users_last_slot_free("backend.copilot.sdk.session_waiter.wait_for_session_result",new=AsyncMock(return_value=("completed", SessionResult())),),+ patch("backend.copilot.sdk.session_waiter.queue_spawned_turn"),+ patch("backend.copilot.sdk.session_waiter.chat_db", return_value=queued),):outcome, _ = await run_copilot_turn_via_queue(session_id="sess-sub",@@ -360,6 +365,10 @@ async def test_in_flight_session_with_allow_queue_false_is_refused():"backend.copilot.sdk.session_waiter.queue_user_message",new=queue_message,),+ patch(+ "backend.copilot.sdk.session_waiter.chat_db",+ return_value=_status_db("running"),+ ),):outcome, result = await run_copilot_turn_via_queue(session_id="sess-busy",@@ -376,6 +385,34 @@ async def test_in_flight_session_with_allow_queue_false_is_refused():queue_message.assert_not_awaited()+async def test_a_queued_session_with_allow_queue_false_is_refused():+ """A session that already has a task waiting for a slot takes no second+ one: its queued row is the one the promotion replays."""+ scheduled = AsyncMock()++ with (+ patch(+ "backend.copilot.sdk.session_waiter.chat_db",+ return_value=_status_db("queued"),+ ),+ patch("backend.copilot.sdk.session_waiter.schedule_turn", new=scheduled),+ ):+ outcome, result = await run_copilot_turn_via_queue(+ session_id="sess-waiting",+ user_id="u1",+ message="do the thing",+ timeout=5,+ tool_call_id="sub:parent",+ tool_name="run_sub_session",+ allow_queue=False,+ )++ assert outcome == "refused"+ assert result.refusal+ scheduled.assert_not_awaited()++@pytest.mark.asyncioasync def test_a_tree_refusal_becomes_the_refused_outcome():"""A ledger refusal (budget or node cap) reaches the caller as@@ -403,3 +440,9 @@ async def test_a_tree_refusal_becomes_the_refused_outcome():assert outcome == "refused"assert result.refusal == "This task's tree has closed."+++def _status_db(status: str) -> MagicMock:+ db = MagicMock()+ db.get_chat_session_status = AsyncMock(return_value=status)+ return dbdiff --git a/autogpt_platform/backend/backend/copilot/tools/delegate_to_expert.py b/autogpt_platform/backend/backend/copilot/tools/delegate_to_expert.pyindex a1c1bf48df..d6aaa466ae 100644--- a/autogpt_platform/backend/backend/copilot/tools/delegate_to_expert.py+++ b/autogpt_platform/backend/backend/copilot/tools/delegate_to_expert.py@@ -33,7 +33,7 @@ import timefrom typing import Anyfrom backend.api.features.experts.models import Expert-from backend.copilot.active_turns import delegated_turn_limit_message+from backend.copilot.active_turns import spawn_limit_messagefrom backend.copilot.budget_signal import build_spawn_state_notefrom backend.copilot.context import get_current_permissionsfrom backend.copilot.model import (@@ -220,7 +220,7 @@ class DelegateToExpertTool(BaseTool):# poll a delegation that no longer exists.return self._error((- delegated_turn_limit_message()+ spawn_limit_message()if outcome == "rejected_concurrent_turn_cap"else result.refusal or f"{target.name} could not start this task."),diff --git a/autogpt_platform/backend/backend/copilot/tools/delegate_to_expert_test.py b/autogpt_platform/backend/backend/copilot/tools/delegate_to_expert_test.pyindex 953a6d2c24..85bd61e350 100644--- a/autogpt_platform/backend/backend/copilot/tools/delegate_to_expert_test.py+++ b/autogpt_platform/backend/backend/copilot/tools/delegate_to_expert_test.py@@ -13,7 +13,7 @@ from unittest.mock import AsyncMock, MagicMockimport pytest-from backend.copilot.active_turns import delegated_turn_limit_message+from backend.copilot.active_turns import spawn_limit_messagefrom backend.copilot.sdk.session_waiter import SessionResultfrom .delegate_to_expert import DelegateToExpertTool@@ -925,11 +925,11 @@ class TestRefusedDelegationCleanup:assert isinstance(result, ErrorResponse)assert "inner-1" not in result.model_dump_json()- async def test_a_delegation_refused_at_the_reserve_says_to_wait(+ async def test_a_delegation_refused_at_the_inflight_cap_says_to_wait(self, roster, mock_turn, mock_sessions, monkeypatch):- """The thread is discarded, but the model still learns the last slot is- the user's and to wait for a running task, as the other spawn tools say."""+ """The thread is discarded, but the model still learns the user is at+ their task limit and to wait, as the other spawn tools say."""monkeypatch.setattr("backend.copilot.tools.run_sub_session.delete_chat_session", AsyncMock())@@ -943,7 +943,7 @@ class TestRefusedDelegationCleanup:)assert isinstance(result, ErrorResponse)- assert result.message == delegated_turn_limit_message()+ assert result.message == spawn_limit_message()async def test_a_refused_resume_keeps_the_prior_thread(self, roster, mock_turn, mock_sessions, monkeypatchdiff --git a/autogpt_platform/backend/backend/copilot/tools/get_sub_session_result.py b/autogpt_platform/backend/backend/copilot/tools/get_sub_session_result.pyindex 732c21f84c..778814c541 100644--- a/autogpt_platform/backend/backend/copilot/tools/get_sub_session_result.py+++ b/autogpt_platform/backend/backend/copilot/tools/get_sub_session_result.py@@ -24,13 +24,15 @@ from typing import Anyfrom backend.copilot import stream_registryfrom backend.copilot.executor.utils import enqueue_cancel_task-from backend.copilot.model import ChatSession, get_chat_session+from backend.copilot.model import CHAT_STATUS_QUEUED, ChatSession, get_chat_sessionfrom backend.copilot.sdk.session_waiter import (SessionOutcome,SessionResult,+ wait_for_queued_session,wait_for_session_result,)from backend.copilot.sdk.stream_accumulator import ToolCallEntry+from backend.copilot.turn_queue import cancel_queued_turn, queued_turn_refusalfrom backend.data.db_accessors import experts_dbfrom .base import BaseTool@@ -156,6 +158,7 @@ class GetSubSessionResultTool(BaseTool):delegate = await _delegated_expert_info(user_id, sub, session)actor = delegate.name if delegate is not None else "Subtask"borrowed = _is_borrowed_thread(sub, session)+ queued = sub.chat_status == CHAT_STATUS_QUEUEDif cancel:if borrowed:@@ -172,8 +175,15 @@ class GetSubSessionResultTool(BaseTool):# sub will break out of its stream and finalise the session# as failed. Return "cancelled" immediately; the sub may# still emit a little more output before the worker notices,- # but the agent doesn't need to wait for that.- await enqueue_cancel_task(inner_session_id)+ # but the agent doesn't need to wait for that. A queued one has+ # nothing running yet, so it simply leaves the queue.+ if not (+ queued+ and await cancel_queued_turn(+ user_id=user_id, session_id=inner_session_id+ )+ ):+ await enqueue_cancel_task(inner_session_id)return apply_delegated_expert(SubSessionStatusResponse(message=f"{actor} cancellation requested.",@@ -200,9 +210,16 @@ class GetSubSessionResultTool(BaseTool):getattr(registry_session, "status", "") == "running")terminal_result = None if turn_in_flight else _already_terminal_result(sub)+ refusal = None if turn_in_flight else queued_turn_refusal(sub)outcome: SessionOutcomeresult: SessionResult- if terminal_result is not None:+ if queued:+ outcome, result = await wait_for_queued_session(+ session_id=inner_session_id, user_id=user_id, timeout=effective_wait+ )+ elif refusal is not None:+ outcome, result = "refused", SessionResult(refusal=refusal)+ elif terminal_result is not None:outcome, result = "completed", terminal_resultelif effective_wait > 0:outcome, result = await wait_for_session_result(diff --git a/autogpt_platform/backend/backend/copilot/tools/handoff_to_expert.py b/autogpt_platform/backend/backend/copilot/tools/handoff_to_expert.pyindex ddb17ca6c6..c6d6b35fda 100644--- a/autogpt_platform/backend/backend/copilot/tools/handoff_to_expert.py+++ b/autogpt_platform/backend/backend/copilot/tools/handoff_to_expert.py@@ -25,7 +25,7 @@ import loggingfrom typing import Anyfrom backend.api.features.experts.models import Expert-from backend.copilot.active_turns import delegated_turn_limit_message+from backend.copilot.active_turns import spawn_limit_messagefrom backend.copilot.budget_signal import build_spawn_state_notefrom backend.copilot.context import get_current_permissionsfrom backend.copilot.model import (@@ -64,7 +64,7 @@ logger = logging.getLogger(__name__)# already picked up, or appended to a turn in flight there. Anything else (the# concurrent-turn cap, a failed dispatch) means nothing moved._OWNERSHIP_TAKEN: frozenset[SessionOutcome] = frozenset(- {"running", "queued", "completed"}+ {"running", "queued", "queued_for_slot", "completed"})@@ -207,6 +207,7 @@ class HandoffToExpertTool(BaseTool):inner_session_id=inner.session_id,parent_session_id=session.session_id,target_name=target.name,+ waiting_for_slot=outcome == "queued_for_slot",)transferred.message += await build_spawn_state_note()return apply_delegated_expert(@@ -318,6 +319,7 @@ def _transferred_response(inner_session_id: str,parent_session_id: str | None,target_name: str,+ waiting_for_slot: bool = False,) -> SubSessionStatusResponse:"""The terminal handoff contract: ownership moved, nothing to poll.@@ -330,8 +332,9 @@ def _transferred_response(link = _sub_session_link(inner_session_id)return SubSessionStatusResponse(message=(- f"{target_name} owns this now and will report to the user "- f"directly.{f' Follow along at {link}.' if link else ''}"+ f"{target_name} owns this now and will report to the user directly"+ f"{', starting when one of their task slots frees' if waiting_for_slot else ''}."+ f"{f' Follow along at {link}.' if link else ''}"),session_id=parent_session_id,status="transferred",@@ -349,7 +352,7 @@ def _refused_transfer_message(if outcome == "rejected_concurrent_turn_cap":return (f"The handoff to {target_name} did not happen — the task is still "- f"yours. {delegated_turn_limit_message()}"+ f"yours. {spawn_limit_message()}")if outcome == "refused" and refusal:return (diff --git a/autogpt_platform/backend/backend/copilot/tools/handoff_to_expert_test.py b/autogpt_platform/backend/backend/copilot/tools/handoff_to_expert_test.pyindex b33aa0b806..a83317bd79 100644--- a/autogpt_platform/backend/backend/copilot/tools/handoff_to_expert_test.py+++ b/autogpt_platform/backend/backend/copilot/tools/handoff_to_expert_test.py@@ -14,6 +14,7 @@ from unittest.mock import AsyncMock, MagicMock, patchimport pytest+from backend.copilot.active_turns import spawn_limit_messagefrom backend.copilot.sdk.session_waiter import SessionResultfrom backend.copilot.tools import (TOOL_GROUPS,@@ -626,7 +627,19 @@ class TestFailedTransfer:self, roster, mock_turn, mock_sessions):r = await self._handoff(mock_turn, "rejected_concurrent_turn_cap")- assert "kept for the user's own messages" in r.message+ assert spawn_limit_message() in r.message++ @pytest.mark.asyncio+ async def test_a_handoff_waiting_for_a_slot_has_still_moved(+ self, roster, mock_turn, mock_sessions, deleted_sessions+ ):+ """Its turn is queued in the target's own session, so the target owns+ it now and the thread stays."""+ r = await self._handoff(mock_turn, "queued_for_slot")+ assert isinstance(r, SubSessionStatusResponse)+ assert r.status == "transferred"+ assert "starting when one of their task slots frees" in r.message+ assert deleted_sessions == []@pytest.mark.parametrize("outcome", ["rejected_concurrent_turn_cap", "failed"])@pytest.mark.asynciodiff --git a/autogpt_platform/backend/backend/copilot/tools/run_sub_session.py b/autogpt_platform/backend/backend/copilot/tools/run_sub_session.pyindex 495d68fb4c..7f200852d5 100644--- a/autogpt_platform/backend/backend/copilot/tools/run_sub_session.py+++ b/autogpt_platform/backend/backend/copilot/tools/run_sub_session.py@@ -28,7 +28,7 @@ import loggingimport timefrom typing import Any-from backend.copilot.active_turns import delegated_turn_limit_message+from backend.copilot.active_turns import spawn_limit_messagefrom backend.copilot.budget_signal import build_spawn_state_notefrom backend.copilot.constants import MAX_TOOL_WAIT_SECONDSfrom backend.copilot.context import get_current_permissions, get_workspace_manager@@ -518,6 +518,22 @@ def response_from_outcome(elapsed_seconds=round(elapsed, 2),).from_outside()+ if outcome == "queued_for_slot":+ return SubSessionStatusResponse(+ message=(+ f"{actor} is queued: the user's task slots for sub-work are all "+ "taken, and it starts on its own when one frees. Call "+ "tool:get_sub_session_result to wait for it or check on it"+ f"{f', or watch at {link}' if link else ''}."+ ),+ session_id=parent_session_id,+ status="queued",+ sub_session_id=inner_session_id,+ sub_autopilot_session_id=inner_session_id,+ sub_autopilot_session_link=link,+ elapsed_seconds=round(elapsed, 2),+ ).from_outside()+if outcome == "running":return SubSessionStatusResponse(message=(@@ -552,7 +568,7 @@ def response_from_outcome(# Render the actionable message instead of a "see transcript"# pointer to nothing.return SubSessionStatusResponse(- message=delegated_turn_limit_message(),+ message=spawn_limit_message(),session_id=parent_session_id,status="error",sub_session_id=inner_session_id,diff --git a/autogpt_platform/backend/backend/copilot/tools/sub_session_test.py b/autogpt_platform/backend/backend/copilot/tools/sub_session_test.pyindex ba7fd7b8b5..5aa6c2137a 100644--- a/autogpt_platform/backend/backend/copilot/tools/sub_session_test.py+++ b/autogpt_platform/backend/backend/copilot/tools/sub_session_test.py@@ -11,11 +11,11 @@ tests patch the three integration seams — ``enqueue_copilot_turn``,from __future__ import annotationsimport json-from unittest.mock import AsyncMock, MagicMock+from unittest.mock import AsyncMock, MagicMock, patchimport pytest-from backend.copilot import active_turns+from backend.copilot import active_turns, turn_queuefrom backend.copilot.permissions import CopilotPermissionsfrom backend.copilot.sdk.session_waiter import SessionResultfrom backend.copilot.sdk.stream_accumulator import ToolCallEntry@@ -807,6 +807,7 @@ class TestGetSubSessionResult:sub = MagicMock(user_id="alice", expert_id=None)sub.metadata.delegated_by_session_id = Noneassistant = MagicMock()+ assistant.metadata = Noneassistant.role = "assistant"assistant.content = "already done"assistant.tool_calls = None@@ -838,16 +839,23 @@ class TestGetSubSessionResult:assert r.response == "already done"mock_waiter.result_mock.assert_not_awaited()+ @pytest.mark.parametrize(+ "prior_metadata",+ [None, {turn_queue._REFUSED_KEY: True}],+ ids=["stale-result", "stale-refusal"],+ )@pytest.mark.asyncioasync def test_resume_turn_in_flight_does_not_return_stale(- self, monkeypatch, mock_waiter+ self, monkeypatch, mock_waiter, prior_metadata):"""Regression for sentry r3105409601: on a resumed session whosestream_registry status is 'running' (new turn is mid-flight) the- tool must NOT short-circuit to the prior turn's terminal message.- It subscribes to the stream like a normal running-session poll."""- # DB state reflects the PREVIOUS turn's terminal assistant message.+ tool must NOT short-circuit to the prior turn's terminal message,+ nor to a refusal that closed an earlier queued turn. It subscribes+ to the stream like a normal running-session poll."""+ # DB state reflects the PREVIOUS turn's last assistant message.prior = MagicMock()+ prior.metadata = prior_metadataprior.role = "assistant"prior.content = "OLD stale result"prior.tool_calls = None@@ -885,6 +893,40 @@ class TestGetSubSessionResult:# And crucially NOT the stale content.assert r.response is None or r.response == ""+ @pytest.mark.asyncio+ async def test_a_turn_that_ended_without_a_reply_is_not_called_cancelled(+ self, mock_waiter+ ):+ """Idle with the user's message last and no stream left: only a cancel+ that wrote its note says "cancelled"; this one reaches the waiter."""+ asked = MagicMock(metadata=None, role="user", content="do it", tool_calls=None)+ sub = MagicMock(+ user_id="alice", expert_id=None, messages=[asked], chat_status="idle"+ )+ sub.metadata.delegated_by_session_id = None+ mock_waiter.result_mock.return_value = ("failed", SessionResult())++ with (+ patch(+ "backend.copilot.tools.get_sub_session_result.get_chat_session",+ AsyncMock(return_value=sub),+ ),+ patch(+ "backend.copilot.tools.get_sub_session_result.stream_registry.get_session",+ AsyncMock(return_value=None),+ ),+ ):+ r = await GetSubSessionResultTool()._execute(+ user_id="alice",+ session=_session("alice"),+ sub_session_id="inner-12",+ wait_if_running=30,+ )++ mock_waiter.result_mock.assert_awaited_once()+ assert isinstance(r, SubSessionStatusResponse)+ assert turn_queue.TURN_CANCELLED not in (r.message or "")+@pytest.mark.asyncioasync def test_cancel_publishes_cancel_event(self, monkeypatch, mock_queue, mock_waiter@@ -925,6 +967,7 @@ class TestGetSubSessionResult:sub = MagicMock(user_id="alice", expert_id=None)sub.metadata.delegated_by_session_id = Noneassistant = MagicMock()+ assistant.metadata = Noneassistant.role = "assistant"assistant.content = "done — see the docs I wrote"assistant.tool_calls = None # no write calls on the last message@@ -1108,8 +1151,8 @@ class TestHollowResponseRepro:# ----------------------------------------------------------------------------def test_a_sub_refused_at_the_cap_is_told_the_last_slot_is_the_users(monkeypatch):- monkeypatch.setattr(active_turns, "get_running_turn_limit", lambda: 5)+def test_a_sub_refused_at_the_inflight_cap_is_told_to_wait(monkeypatch):+ monkeypatch.setattr(active_turns, "get_inflight_turn_limit", lambda: 15)r = response_from_outcome(outcome="rejected_concurrent_turn_cap",result=SessionResult(),@@ -1118,7 +1161,19 @@ def test_a_sub_refused_at_the_cap_is_told_the_last_slot_is_the_users(monkeypatchelapsed=1.0,)assert r.message is not None- assert "Sub-work may use 4 of the user's 5 task slots" in r.message+ assert "already has 15 tasks running or waiting to start" in r.message+++def test_a_sub_waiting_for_a_slot_is_reported_queued_with_its_handle():+ r = response_from_outcome(+ outcome="queued_for_slot",+ result=SessionResult(queued=True),+ inner_session_id="inner-1",+ parent_session_id="parent-1",+ elapsed=1.0,+ )+ assert (r.status, r.sub_session_id) == ("queued", "inner-1")+ assert "starts on its own when one frees" in (r.message or "")class TestActorParameter:diff --git a/autogpt_platform/backend/backend/copilot/turn_queue.py b/autogpt_platform/backend/backend/copilot/turn_queue.pyindex d6ea807d8c..48768c3987 100644--- a/autogpt_platform/backend/backend/copilot/turn_queue.py+++ b/autogpt_platform/backend/backend/copilot/turn_queue.py@@ -65,7 +65,7 @@ from backend.copilot.rate_limit import ()from backend.copilot.session_permissions import resolve_session_permissionsfrom backend.copilot.tracking import track_user_message-from backend.copilot.tree import TreeRefusal, TurnEnvelope+from backend.copilot.tree import TreeRefusal, TurnEnvelope, release_turnfrom backend.data.db_accessors import chat_dbfrom backend.integrations.codex.access import has_codex_access@@ -73,9 +73,18 @@ logger = logging.getLogger(__name__)# Pending-row metadata: the envelope a queued approval wake starts under._ENVELOPE_KEY = "envelope"+# The admitted child's tree, kept apart from the versioned envelope so its node+# can still go back when a later deploy no longer parses that envelope.+_ENVELOPE_TREE_ID_KEY = "envelope_tree_id"# The assistant row that closes a queued turn the promotion could not start._REFUSED_KEY = "queued_turn_refused"WAKE_LATER = "Your answer is kept and reaches the assistant with your next message."+TURN_CANCELLED = "This task was cancelled before it started."+QUEUE_UNAVAILABLE = "Could not queue this task right now; try again shortly."+UNREADABLE_SPAWN = (+ "This task's limits could not be read back, so it was not started. "+ "Start it again."+)UNRECORDED_WAKE = ("The approved action could not be resumed: what it was allowed to do was "f"not recorded. {WAKE_LATER}"@@ -110,6 +119,11 @@ async def list_queued_sessions(user_id: str):)+class SessionNotIdle(Exception):+ """The session already has a turn queued or running, so a spawned child+ cannot be queued into it."""++class InflightCapExceeded(Exception):"""User's running + queued total has reached the configured hard cap.@@ -179,6 +193,9 @@ async def enqueue_turn(permissions: Mapping[str, Any] | None = None,request_arrival_at: float = 0.0,envelope: TurnEnvelope | None = None,+ tool_call_id: str | None = None,+ tool_name: str | None = None,+ only_if_idle: bool = False,) -> ChatMessage | None:"""Persist the user's pending message and flip the session to``"queued"``. Caller is responsible for the in-flight cap check@@ -203,16 +220,29 @@ async def enqueue_turn(metadata["permissions"] = dict(permissions)if request_arrival_at:metadata["request_arrival_at"] = request_arrival_at- # A wake starts under the envelope its call was held under, whichever turn- # frees the slot.+ # A wake starts under the envelope its call was held under, and a spawned+ # child under the one admitted for it, whichever turn frees the slot.if envelope is not None:metadata[_ENVELOPE_KEY] = envelope.model_dump(mode="json")+ metadata[_ENVELOPE_TREE_ID_KEY] = envelope.tree_id+ if tool_call_id is not None:+ metadata["tool_call_id"] = tool_call_id+ if tool_name is not None:+ metadata["tool_name"] = tool_name# The Redis NX session lock serialises with ``append_and_save_message``# so two concurrent submits to the same session can't pick the same# ``sequence`` and PK-collide on ``(sessionId, sequence)``.db = chat_db()- async with _get_session_lock(session_id):+ async with _get_session_lock(session_id) as locked:+ # A spawned child's row is the one promotion replays, so it claims the+ # idle session first and is written only if it won: a lost claim leaves+ # nothing behind. Promotion reads the row under this lock, so it never+ # sees the claim without it, and neither proceeds without the lock.+ if only_if_idle and not locked:+ raise TreeRefusal(QUEUE_UNAVAILABLE)+ if only_if_idle and not await _flip(db, session_id, user_id):+ raise SessionNotIdle(session_id)live_sequence = await db.get_next_sequence(session_id)try:row = await db.add_chat_message(@@ -223,23 +253,25 @@ async def enqueue_turn(sequence=live_sequence,metadata=metadata or None,)- except UniqueViolationError as exc:- if message_id and is_duplicate_chat_message_id_error(exc):+ except BaseException as exc:+ if only_if_idle:+ await _unclaim(db, session_id, user_id)+ if (+ isinstance(exc, UniqueViolationError)+ and message_id+ and is_duplicate_chat_message_id_error(exc)+ ):return Noneraise- # Flip the session to ``"queued"``. CAS-gated on ``"idle"`` so a- # double-submit (session already queued/running) leaves the state- # alone; the second pending message persists as a normal ChatMessage- # row. When the session eventually promotes, the dispatcher reads- # the most-recent user row via ``get_latest_user_message_in_session``;- # earlier pending rows aren't independently scheduled, they sit in- # the chat history and the model sees them as context.- await db.update_chat_session_status(- session_id=session_id,- expect_status=CHAT_STATUS_IDLE,- status=CHAT_STATUS_QUEUED,- user_id=user_id,- )+ # Flip the session to ``"queued"``. CAS-gated on ``"idle"`` so a+ # double-submit (session already queued/running) leaves the state+ # alone; the second pending message persists as a normal ChatMessage+ # row. When the session eventually promotes, the dispatcher reads+ # the most-recent user row via ``get_latest_user_message_in_session``;+ # earlier pending rows aren't independently scheduled, they sit in+ # the chat history and the model sees them as context.+ if not only_if_idle:+ await _flip(db, session_id, user_id)# Invalidate the session cache so the next /chat read picks up the# queued row + the session's new status (frontend renders the# 'Queued' badge from ``session.chat_status``).@@ -247,22 +279,78 @@ async def enqueue_turn(return row-async def cancel_queued_turn(*, user_id: str, session_id: str) -> bool:- """Flip the user's session from ``"queued"`` → ``"idle"``. Returns- True iff the CAS matched AND the session is owned by the user.- Cancel/dispatch races resolve in a single atomic update."""- ok = await chat_db().update_chat_session_status(+async def _unclaim(db: Any, session_id: str, user_id: str) -> None:+ """Undo a child's claim whose row was never written, without masking why."""+ try:+ await db.update_chat_session_status(+ session_id=session_id,+ expect_status=CHAT_STATUS_QUEUED,+ status=CHAT_STATUS_IDLE,+ user_id=user_id,+ )+ except Exception:+ logger.exception(+ f"enqueue_turn: could not release the claim on session={session_id}"+ )+++async def _flip(db: Any, session_id: str, user_id: str) -> bool:+ return await db.update_chat_session_status(session_id=session_id,- expect_status=CHAT_STATUS_QUEUED,- status=CHAT_STATUS_IDLE,+ expect_status=CHAT_STATUS_IDLE,+ status=CHAT_STATUS_QUEUED,user_id=user_id,)- if not ok:- return False- await invalidate_session_cache(session_id)+++async def cancel_queued_turn(*, user_id: str, session_id: str) -> bool:+ """Flip the user's session from ``"queued"`` → ``"idle"``. A queued child+ gets a closing row saying it was cancelled, which its waiter reads, and its+ node goes back to its tree. Returns True iff the CAS matched AND the+ session is owned by the user. Cancel/dispatch races resolve in a single+ atomic update."""+ db = chat_db()+ child: TurnEnvelope | None = None+ try:+ # Under the lock a waiter reads under: it never sees idle without the note.+ async with _get_session_lock(session_id):+ # Read before the flip: once idle, a new message can become the latest.+ pending = await db.get_latest_user_message_in_session(session_id)+ if not await db.update_chat_session_status(+ session_id=session_id,+ expect_status=CHAT_STATUS_QUEUED,+ status=CHAT_STATUS_IDLE,+ user_id=user_id,+ ):+ return False+ if pending is not None and _is_spawned(pending):+ child = _child_node(pending.metadata or {})+ await _note_cancel(db, session_id)+ await invalidate_session_cache(session_id)+ finally:+ # The node goes back whatever happened to the note.+ if child is not None:+ await release_turn(child)return True+async def _note_cancel(db: Any, session_id: str) -> None:+ """Close a cancelled child's thread with the note its waiter reads."""+ try:+ await db.add_chat_message(+ message_id=str(uuid.uuid4()),+ session_id=session_id,+ role="assistant",+ content=TURN_CANCELLED,+ sequence=await db.get_next_sequence(session_id),+ metadata={_REFUSED_KEY: True},+ )+ except Exception:+ logger.exception(+ f"cancel_queued_turn: could not note the cancel on session={session_id}"+ )++async def claim_queued_session(session: ChatSessionInfo, *, sub_work: bool) -> Literal["admitted", "full", "busy"]:@@ -337,12 +425,31 @@ async def _promote_head(user_id: str) -> bool | None:# Cancelled or claimed elsewhere since it was listed: try the next.return None+ # The admitted envelope of a queued child, whose node a refusal gives back.+ child: TurnEnvelope | None = None+ spawned = Falsetry:# Find the pending user message in this session (the most recent# user-role row with no following assistant rows — i.e. the one# that triggered the queue). Its ``metadata`` carries the# dispatcher payload.- pending = await chat_db().get_latest_user_message_in_session(head.session_id)+ # Under the session lock: a spawned child claims the session before it+ # writes its row, inside the same lock.+ async with _get_session_lock(head.session_id) as locked:+ pending = (+ await chat_db().get_latest_user_message_in_session(head.session_id)+ if locked+ else None+ )+ if not locked:+ # Unlocked, a child's claim may not have its row yet: leave it queued.+ await chat_db().update_chat_session_status(+ session_id=head.session_id,+ expect_status=CHAT_STATUS_RUNNING,+ status=CHAT_STATUS_QUEUED,+ )+ await invalidate_session_cache(head.session_id)+ return Falseif pending is None or pending.content is None:# Shouldn't happen — enqueue_turn always persists a row before# flipping the session to queued. If it does (corrupted@@ -359,6 +466,14 @@ async def _promote_head(user_id: str) -> bool | None:metadata = pending.metadata or {}queued_envelope = _stored_envelope(metadata)+ spawned = _is_spawned(pending)+ if spawned and queued_envelope is None:+ # Nothing but that envelope bounds a child its spawner admitted.+ await _refuse_queued_turn(+ head, UNREADABLE_SPAWN, release=_child_node(metadata)+ )+ return None+ child = queued_envelope if spawned else Noneif (is_answer_row(pending)and queued_envelope is None@@ -397,16 +512,22 @@ async def _promote_head(user_id: str) -> bool | None:model=metadata.get("model"),llm_auth_provider=head.metadata.llm_auth_provider,llm_credential_id=head.metadata.llm_credential_id,- permissions=_promotion_permissions(head, metadata),+ permissions=_promotion_permissions(head, metadata, spawned=spawned),request_arrival_at=float(metadata.get("request_arrival_at") or 0.0),+ tool_call_id=metadata.get("tool_call_id") or "chat_stream",+ tool_name=metadata.get("tool_name") or "chat",# A typed message is a root, as the chat route makes it, not the- # child of the turn this hook runs in; a wake brings its own.+ # child of the turn this hook runs in; a wake or a child brings its own.envelope=queued_envelope,root=queued_envelope is None,)except TreeRefusal as refused:# Only a stored envelope is re-checked here; a root is never refused.- await _refuse_queued_turn(head, f"{refused.message} {WAKE_LATER}")+ await _refuse_queued_turn(+ head,+ refused.message if spawned else f"{refused.message} {WAKE_LATER}",+ release=child,+ )return Noneexcept BaseException:# Roll the claim back so a missed-dispatch tick or the next@@ -434,7 +555,7 @@ async def _promote_head(user_id: str) -> bool | None:)raise- if pending.role == "user" and pending.content:+ if pending.role == "user" and pending.content and not spawned:try:track_user_message(user_id=user_id,@@ -452,15 +573,16 @@ async def _promote_head(user_id: str) -> bool | None:async def _is_sub_work(session: ChatSessionInfo) -> bool:- """A message the user typed is theirs whatever session it is in; what an- approval wakes there is sub-work if :func:`wakes_sub_work` says so."""+ """A message the user typed is theirs whatever session it is in; an+ approval wake or a spawned child there is sub-work if+ :func:`wakes_sub_work` says so."""if not wakes_sub_work(session):return False# Local: the gate imports this module back.from backend.copilot.gate.held import is_answer_rowwaiting = await chat_db().get_latest_user_message_in_session(session.session_id)- return waiting is not None and is_answer_row(waiting)+ return waiting is not None and (is_answer_row(waiting) or _is_spawned(waiting))async def _may_start(gates: "_UserGates", head: ChatSessionInfo) -> bool:@@ -540,14 +662,17 @@ def is_users_own_chat(session: ChatSessionInfo) -> bool:def wakes_sub_work(session: ChatSessionInfo) -> bool:- """An approval wake here is the sub-work of the session that opened this- one, so it is admitted within the user's reserve, direct or queued."""+ """An approval wake or a spawned child here is the sub-work of the session+ that opened this one, so it is admitted within the user's reserve."""return session.metadata.delegated_by_session_id is not None-async def _refuse_queued_turn(head: ChatSessionInfo, reason: str) -> None:- """Close a promoted turn that may not start: say why in its thread and- free its slot, which a failed post must not keep."""+async def _refuse_queued_turn(+ head: ChatSessionInfo, reason: str, *, release: TurnEnvelope | None = None+) -> None:+ """Close a promoted turn that may not start: say why in its thread, free+ its slot, which a failed post must not keep, and give back ``release``'s+ node: the node a queued child was admitted with."""try:await post_refusal(head.session_id, reason)except Exception:@@ -557,12 +682,16 @@ async def _refuse_queued_turn(head: ChatSessionInfo, reason: str) -> None:"was not started")finally:- await chat_db().update_chat_session_status(- session_id=head.session_id,- expect_status=CHAT_STATUS_RUNNING,- status=CHAT_STATUS_IDLE,- )- await invalidate_session_cache(head.session_id)+ try:+ await chat_db().update_chat_session_status(+ session_id=head.session_id,+ expect_status=CHAT_STATUS_RUNNING,+ status=CHAT_STATUS_IDLE,+ )+ await invalidate_session_cache(head.session_id)+ finally:+ if release is not None:+ await release_turn(release)async def post_refusal(@@ -588,9 +717,29 @@ def queued_turn_refusal(session: ChatSession) -> str | None:return last.content+def _is_spawned(pending: ChatMessage) -> bool:+ """A queued child: it carries the envelope admitted for it, where a typed+ message carries none and a wake carries its holding turn's."""+ # Local: the gate imports this module back.+ from backend.copilot.gate.held import is_answer_row++ return _ENVELOPE_KEY in (pending.metadata or {}) and not is_answer_row(pending)+++def _child_node(metadata: Mapping[str, Any]) -> TurnEnvelope | None:+ """The node a queued child holds on its tree, found by its stored tree id+ when the envelope itself no longer parses."""+ envelope = _stored_envelope(metadata)+ if envelope is not None:+ return envelope+ tree_id = metadata.get(_ENVELOPE_TREE_ID_KEY)+ # A child is never a root, so depth 1 is enough for release_turn to act.+ return TurnEnvelope(tree_id=tree_id, depth=1) if isinstance(tree_id, str) else None++def _stored_envelope(metadata: Mapping[str, Any]) -> TurnEnvelope | None:- """The envelope a wake was queued under. One that no longer parses (a schema- change between deploys) counts as unrecorded, so the wake is not started."""+ """The envelope a wake or a child was queued under. One that no longer+ parses (a schema change between deploys) counts as unrecorded."""stored = metadata.get(_ENVELOPE_KEY)if not stored:return None@@ -602,11 +751,11 @@ def _stored_envelope(metadata: Mapping[str, Any]) -> TurnEnvelope | None:def _promotion_permissions(- head: ChatSessionInfo, metadata: Mapping[str, Any]+ head: ChatSessionInfo, metadata: Mapping[str, Any], *, spawned: bool) -> CopilotPermissions | None:"""The session's permissions as they are now, never looser than the ones- stored at enqueue. Every writer stores this session's own resolved- permissions, so a stored value that no longer parses is read afresh."""+ stored at enqueue. Those are the session's own, so a value that no longer+ parses is read afresh; a child's are its spawner's, so it refuses instead."""current = resolve_session_permissions(head)stored = metadata.get("permissions")if not stored:@@ -614,6 +763,8 @@ def _promotion_permissions(try:queued = CopilotPermissions.model_validate(stored)except ValidationError:+ if spawned:+ raise TreeRefusal(UNREADABLE_SPAWN)logger.warning(f"dispatch_next_for_user: stored permissions on session={head.session_id} ""did not parse; using the session's current ones"diff --git a/autogpt_platform/backend/backend/copilot/turn_queue_test.py b/autogpt_platform/backend/backend/copilot/turn_queue_test.pyindex 92925e5590..d7b46b3530 100644--- a/autogpt_platform/backend/backend/copilot/turn_queue_test.py+++ b/autogpt_platform/backend/backend/copilot/turn_queue_test.py@@ -33,6 +33,13 @@ class _NoopAsyncCM:return None+class _UnheldLock(_NoopAsyncCM):+ """The session lock as Redis down or a timed-out acquire leaves it."""++ async def __aenter__(self):+ return False++def _pyd_message(**overrides) -> PydanticChatMessage:"""Build a Pydantic ChatMessage with sensible defaults."""base = {@@ -61,6 +68,17 @@ def _queued_row(session_id: str = "s1", title: str | None = "T") -> ChatSessionI)+# A queued child's stored envelope.+_CHILD = {"tree_id": "t1", "depth": 1}++[email protected](autouse=True)+def no_session_lock():+ """The Redis session lock is the DB tests' to exercise."""+ with patch.object(turn_queue, "_get_session_lock", return_value=_NoopAsyncCM()):+ yield++@pytest.fixture(autouse=True)def tracked_message(monkeypatch: pytest.MonkeyPatch) -> MagicMock:tracker = MagicMock()@@ -174,6 +192,7 @@ async def test_cancel_queued_turn_returns_true_and_invalidates_cache() -> None:invalidates the session cache so the frontend drops the badge."""db = MagicMock()db.update_chat_session_status = AsyncMock(return_value=True)+ db.get_latest_user_message_in_session = AsyncMock(return_value=None)invalidate = AsyncMock()with (patch.object(turn_queue, "chat_db", return_value=db),@@ -194,9 +213,208 @@ async def test_cancel_queued_turn_returns_true_and_invalidates_cache() -> None:async def test_cancel_queued_turn_returns_false_when_not_owned_or_not_queued() -> None:db = MagicMock()db.update_chat_session_status = AsyncMock(return_value=False)- with patch.object(turn_queue, "chat_db", return_value=db):+ db.get_latest_user_message_in_session = AsyncMock(+ return_value=_pyd_message(metadata={"envelope": _CHILD})+ )+ release = AsyncMock()+ with (+ patch.object(turn_queue, "chat_db", return_value=db),+ patch.object(turn_queue, "release_turn", new=release),+ ):ok = await turn_queue.cancel_queued_turn(user_id="u1", session_id="s1")assert ok is False+ release.assert_not_awaited()+++ "metadata, released",+ [+ ({"envelope": _CHILD}, True),+ ({"envelope": {"depth": "deep"}, "envelope_tree_id": "t1"}, True),+ # A wake carries the node of the turn that held its call, not its own.+ ({"envelope": _CHILD, held._WAKE_KEY: True}, False),+ (None, False),+ ],+ ids=["child", "child-unreadable", "wake", "typed"],+)+async def test_cancelling_a_queued_turn_returns_only_a_childs_node(+ metadata: dict | None, released: bool+) -> None:+ """A cancelled child also gets the note its waiter reads; nothing else does."""+ db = MagicMock()+ db.update_chat_session_status = AsyncMock(return_value=True)+ db.get_latest_user_message_in_session = AsyncMock(+ return_value=_pyd_message(metadata=metadata)+ )+ db.get_next_sequence = AsyncMock(return_value=2)+ db.add_chat_message = AsyncMock()+ release = AsyncMock()+ with (+ patch.object(turn_queue, "chat_db", return_value=db),+ patch.object(turn_queue, "invalidate_session_cache", new=AsyncMock()),+ patch.object(turn_queue, "release_turn", new=release),+ ):+ assert await turn_queue.cancel_queued_turn(user_id="u1", session_id="s1")+ if released:+ release.assert_awaited_once_with(TurnEnvelope.model_validate(_CHILD))+ note = db.add_chat_message.await_args.kwargs+ assert (note["role"], note["content"]) == (+ "assistant",+ turn_queue.TURN_CANCELLED,+ )+ assert note["metadata"] == {turn_queue._REFUSED_KEY: True}+ else:+ release.assert_not_awaited()+ db.add_chat_message.assert_not_awaited()+++async def test_a_queued_child_keeps_its_tree_apart_from_its_envelope() -> None:+ """So a deploy that stops parsing the envelope can still give its node back."""+ db = MagicMock()+ db.update_chat_session_status = AsyncMock(return_value=True)+ db.get_next_sequence = AsyncMock(return_value=1)+ db.add_chat_message = AsyncMock()++ with (+ patch.object(turn_queue, "chat_db", return_value=db),+ patch.object(turn_queue, "invalidate_session_cache", new=AsyncMock()),+ ):+ await turn_queue.enqueue_turn(+ user_id="u1",+ session_id="s1",+ message="task",+ envelope=TurnEnvelope.model_validate(_CHILD),+ only_if_idle=True,+ )++ assert db.add_chat_message.await_args.kwargs["metadata"]["envelope_tree_id"] == "t1"+++async def test_a_child_that_loses_the_claim_writes_no_row() -> None:+ """Claimed before it is written: when another turn took the session between+ a look and the flip, nothing is left for promotion to replay."""+ db = MagicMock()+ db.get_chat_session_status = AsyncMock(return_value="idle")+ db.update_chat_session_status = AsyncMock(return_value=False)+ db.get_next_sequence = AsyncMock(return_value=1)+ db.add_chat_message = AsyncMock()++ with (+ patch.object(turn_queue, "chat_db", return_value=db),+ patch.object(turn_queue, "invalidate_session_cache", new=AsyncMock()),+ pytest.raises(turn_queue.SessionNotIdle),+ ):+ await turn_queue.enqueue_turn(+ user_id="u1",+ session_id="s1",+ message="task",+ envelope=TurnEnvelope.model_validate(_CHILD),+ only_if_idle=True,+ )++ db.add_chat_message.assert_not_awaited()+++async def test_a_failed_unclaim_does_not_hide_why_the_child_was_not_written() -> None:+ db = MagicMock()+ db.update_chat_session_status = AsyncMock(+ side_effect=[True, RuntimeError("rollback failed")]+ )+ db.get_next_sequence = AsyncMock(return_value=1)+ db.add_chat_message = AsyncMock(side_effect=RuntimeError("write failed"))++ with (+ patch.object(turn_queue, "chat_db", return_value=db),+ pytest.raises(RuntimeError, match="write failed"),+ ):+ await turn_queue.enqueue_turn(+ user_id="u1",+ session_id="s1",+ message="task",+ envelope=TurnEnvelope.model_validate(_CHILD),+ only_if_idle=True,+ )++ assert db.update_chat_session_status.await_count == 2+++async def test_a_cancelled_child_returns_its_node_when_its_note_cannot_be_written() -> (+ None+):+ db = MagicMock()+ db.update_chat_session_status = AsyncMock(return_value=True)+ db.get_latest_user_message_in_session = AsyncMock(+ return_value=_pyd_message(metadata={"envelope": _CHILD})+ )+ db.get_next_sequence = AsyncMock(return_value=2)+ db.add_chat_message = AsyncMock(side_effect=RuntimeError("note failed"))+ release = AsyncMock()++ with (+ patch.object(turn_queue, "chat_db", return_value=db),+ patch.object(turn_queue, "invalidate_session_cache", new=AsyncMock()),+ patch.object(turn_queue, "release_turn", new=release),+ ):+ assert await turn_queue.cancel_queued_turn(user_id="u1", session_id="s1")++ release.assert_awaited_once_with(TurnEnvelope.model_validate(_CHILD))+++async def test_a_child_is_not_queued_without_the_session_lock() -> None:+ """Promotion reads its row under that lock, so unlocked it claims nothing."""+ db = MagicMock()+ db.update_chat_session_status = AsyncMock(return_value=True)+ db.add_chat_message = AsyncMock()++ with (+ patch.object(turn_queue, "chat_db", return_value=db),+ patch.object(turn_queue, "_get_session_lock", return_value=_UnheldLock()),+ pytest.raises(TreeRefusal, match="try again"),+ ):+ await turn_queue.enqueue_turn(+ user_id="u1",+ session_id="s1",+ message="task",+ envelope=TurnEnvelope.model_validate(_CHILD),+ only_if_idle=True,+ )++ db.update_chat_session_status.assert_not_awaited()+ db.add_chat_message.assert_not_awaited()+++async def test_promotion_without_the_session_lock_leaves_the_turn_queued() -> None:+ """Unlocked, a child's claim may not have its row yet: the next tick tries."""+ head = _queued_row()+ head.metadata.llm_auth_provider = "microsoft_365_copilot"+ db = MagicMock()+ db.update_chat_session_status = AsyncMock(return_value=True)+ db.get_latest_user_message_in_session = AsyncMock(return_value=_pyd_message())+ dispatched = AsyncMock()++ with (+ _patch_queued_list([head]),+ patch.object(turn_queue, "chat_db", return_value=db),+ patch.object(+ turn_queue, "claim_queued_session", new=AsyncMock(return_value="admitted")+ ),+ patch.object(turn_queue, "invalidate_session_cache", new=AsyncMock()),+ patch.object(turn_queue, "_get_session_lock", return_value=_UnheldLock()),+ patch("backend.copilot.executor.utils.dispatch_turn", new=dispatched),+ ):+ assert await turn_queue.dispatch_next_for_user("u1") is False++ dispatched.assert_not_awaited()+ db.update_chat_session_status.assert_awaited_once_with(+ session_id="s1", expect_status="running", status="queued"+ )# ── try_enqueue_turn ───────────────────────────────────────────────────@@ -432,6 +650,152 @@ async def test_a_turn_the_access_gate_refuses_does_not_hold_up_the_next() -> Non)+ "stored, refusal, reason, released",+ [+ # Nothing but its envelope bounds a child its spawner admitted.+ ({"envelope": {"depth": "deep"}}, None, turn_queue.UNREADABLE_SPAWN, False),+ # Its tree id is stored apart, so the node goes back.+ (+ {"envelope": {"depth": "deep"}, "envelope_tree_id": "t1"},+ None,+ turn_queue.UNREADABLE_SPAWN,+ True,+ ),+ # Its stored permissions are its spawner's, never re-derivable.+ (+ {"envelope": _CHILD, "permissions": {"tools": 5}},+ None,+ turn_queue.UNREADABLE_SPAWN,+ True,+ ),+ # A tree that closed while it waited: no "your answer is kept" note.+ ({"envelope": _CHILD}, "This tree has closed.", "This tree has closed.", True),+ ],+ ids=[+ "envelope-unreadable",+ "envelope-unreadable-tree-known",+ "permissions-unreadable",+ "tree-refused",+ ],+)+async def test_a_queued_child_that_may_not_start_is_closed_and_returns_its_node(+ stored: dict, refusal: str | None, reason: str, released: bool+) -> None:+ head = _queued_row()+ head.metadata.delegated_by_session_id = "parent"+ head.metadata.llm_auth_provider = "microsoft_365_copilot"+ db = MagicMock()+ db.update_chat_session_status = AsyncMock(return_value=True)+ db.get_latest_user_message_in_session = AsyncMock(+ return_value=_pyd_message(metadata=stored)+ )+ posted = AsyncMock()+ release = AsyncMock()+ dispatched = AsyncMock(side_effect=TreeRefusal(refusal) if refusal else None)++ with (+ patch.object(+ turn_queue, "list_queued_sessions", new=AsyncMock(side_effect=[[head], []])+ ),+ patch.object(turn_queue, "chat_db", return_value=db),+ patch.object(+ turn_queue, "claim_queued_session", new=AsyncMock(return_value="admitted")+ ),+ patch.object(turn_queue, "invalidate_session_cache", new=AsyncMock()),+ patch.object(turn_queue, "resolve_session_permissions", return_value=None),+ patch.object(turn_queue, "append_and_save_message", new=posted),+ patch.object(turn_queue, "release_turn", new=release),+ patch("backend.copilot.executor.utils.dispatch_turn", new=dispatched),+ ):+ assert await turn_queue.dispatch_next_for_user("u1") is False++ assert posted.await_args.args[1].content == reason+ db.update_chat_session_status.assert_awaited_once_with(+ session_id="s1", expect_status="running", status="idle"+ )+ if released:+ release.assert_awaited_once_with(TurnEnvelope.model_validate(_CHILD))+ else:+ release.assert_not_awaited()+++async def test_a_refused_child_returns_its_node_when_its_slot_cannot_be_freed() -> None:+ head = _queued_row()+ head.metadata.delegated_by_session_id = "parent"+ head.metadata.llm_auth_provider = "microsoft_365_copilot"+ db = MagicMock()+ db.update_chat_session_status = AsyncMock(side_effect=RuntimeError("db down"))+ db.get_latest_user_message_in_session = AsyncMock(+ return_value=_pyd_message(metadata={"envelope": _CHILD})+ )+ release = AsyncMock()++ with (+ _patch_queued_list([head]),+ patch.object(turn_queue, "chat_db", return_value=db),+ patch.object(+ turn_queue, "claim_queued_session", new=AsyncMock(return_value="admitted")+ ),+ patch.object(turn_queue, "invalidate_session_cache", new=AsyncMock()),+ patch.object(turn_queue, "resolve_session_permissions", return_value=None),+ patch.object(turn_queue, "append_and_save_message", new=AsyncMock()),+ patch.object(turn_queue, "release_turn", new=release),+ patch(+ "backend.copilot.executor.utils.dispatch_turn",+ new=AsyncMock(side_effect=TreeRefusal("This tree has closed.")),+ ),+ pytest.raises(RuntimeError, match="db down"),+ ):+ await turn_queue.dispatch_next_for_user("u1")++ release.assert_awaited_once_with(TurnEnvelope.model_validate(_CHILD))+++async def test_a_queued_child_is_sub_work_started_under_its_own_tool_call(+ tracked_message: MagicMock,+) -> None:+ """Claimed at the reserve, replayed under the tool call that spawned it, and+ not counted as a message the user sent."""+ head = _queued_row()+ head.metadata.delegated_by_session_id = "parent"+ head.metadata.llm_auth_provider = "microsoft_365_copilot"+ db = MagicMock()+ db.get_latest_user_message_in_session = AsyncMock(+ return_value=_pyd_message(+ metadata={+ "envelope": _CHILD,+ "tool_call_id": "sub:parent",+ "tool_name": "run_sub_session",+ }+ )+ )+ claim = AsyncMock(return_value="admitted")+ dispatched = AsyncMock()++ with (+ _patch_queued_list([head]),+ patch.object(turn_queue, "chat_db", return_value=db),+ patch.object(turn_queue, "claim_queued_session", new=claim),+ patch.object(turn_queue, "invalidate_session_cache", new=AsyncMock()),+ patch.object(turn_queue, "resolve_session_permissions", return_value=None),+ patch("backend.copilot.executor.utils.dispatch_turn", new=dispatched),+ ):+ assert await turn_queue.dispatch_next_for_user("u1") is True++ assert claim.await_args.kwargs["sub_work"] is True+ kwargs = dispatched.await_args.kwargs+ assert (kwargs["tool_call_id"], kwargs["tool_name"]) == (+ "sub:parent",+ "run_sub_session",+ )+ assert kwargs["envelope"] == TurnEnvelope.model_validate(_CHILD)+ tracked_message.assert_not_called()++@pytest.mark.asyncioasync def test_unparseable_stored_permissions_read_the_sessions_current_ones() -> None:"""Not left stuck at the head of the queue, and never read as none."""