Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend/copilot): queued messages and approval wakes start under the envelope they belong to (#15353)

Reinier van der Leer9baa345c3aa8
diff --git a/autogpt_platform/backend/backend/copilot/bot/card_buttons_db_test.py b/autogpt_platform/backend/backend/copilot/bot/card_buttons_db_test.py
index c830eea07e..719e271965 100644
--- a/autogpt_platform/backend/backend/copilot/bot/card_buttons_db_test.py
+++ b/autogpt_platform/backend/backend/copilot/bot/card_buttons_db_test.py
@@ -28,6 +28,7 @@ from backend.copilot.bot.bot_backend import BotBackend
from backend.copilot.bot.choices import CARD_KIND
from backend.copilot.bot.handler import MessageHandler
from backend.copilot.bot.text import format_batch
+from backend.copilot.context import set_execution_context
from backend.copilot.gate import channel as gate_channel
from backend.copilot.gate import chat_rules, check_action, held, resolve_mode
from backend.copilot.gate.classifier import Judgement
@@ -43,6 +44,7 @@ from backend.copilot.model import (
from backend.copilot.response_model import StreamCheckpoint, StreamTextDelta
from backend.copilot.tools.base import BaseTool
from backend.copilot.tools.models import ResponseType, ToolResponseBase
+from backend.copilot.tree import root_envelope
from backend.data.db_accessors import review_db
from backend.data.redis_client import get_redis_async
from backend.platform_linking import cards
@@ -409,12 +411,28 @@ async def _linked_session(user_id: str, platform: str) -> ChatSession:
)
+async def _hold(user_id: str, session: ChatSession, text: str, call_id: str):
+ """Hold a call as the gate does: inside a turn, whose envelope its wake
+ starts under."""
+
+ async def in_a_turn():
+ set_execution_context(
+ user_id,
+ session,
+ envelope=root_envelope(f"turn-{call_id}", session_id=session.session_id),
+ )
+ return await check_action(_POST, {"text": text}, user_id, session, call_id)
+
+ decision = await asyncio.create_task(in_a_turn())
+ assert not decision.allowed and decision.review_id
+ return decision
+
+
async def _held_card(
channel: _Channel, user_id: str, text: str = "hello"
) -> tuple[ChatSession, str, ChannelCard]:
session = await _linked_session(user_id, channel.platform)
- decision = await check_action(_POST, {"text": text}, user_id, session, "call-1")
- assert not decision.allowed and decision.review_id
+ decision = await _hold(user_id, session, text, "call-1")
# Opened from a member's message: the card is still the owner's to answer.
card = await cards.open_card(
Platform(channel.platform.upper()),
@@ -885,8 +903,7 @@ async def _bot_following(
async def _another_card(
channel: _Channel, user_id: str, session: ChatSession
) -> ChannelCard:
- decision = await check_action(_POST, {"text": "again"}, user_id, session, "call-2")
- assert not decision.allowed and decision.review_id
+ decision = await _hold(user_id, session, "again", "call-2")
card = await cards.open_card(
Platform(channel.platform.upper()),
channel.server_id,
diff --git a/autogpt_platform/backend/backend/copilot/executor/utils.py b/autogpt_platform/backend/backend/copilot/executor/utils.py
index a194d878b5..8b0d0b4bf5 100644
--- a/autogpt_platform/backend/backend/copilot/executor/utils.py
+++ b/autogpt_platform/backend/backend/copilot/executor/utils.py
@@ -29,6 +29,8 @@ from backend.copilot.tree import (
TurnEnvelope,
admit_turn,
derive_child_envelope,
+ narrowed_by,
+ readmit_turn,
release_turn,
root_envelope,
)
@@ -557,6 +559,8 @@ async def dispatch_turn(
unattended: bool = False,
credential_pins: CredentialPins | None = None,
scheduled: ScheduledTurnOrigin | None = None,
+ root: bool = False,
+ envelope: TurnEnvelope | None = None,
) -> None:
"""Within an already-held turn slot, register the session in the
stream registry, publish the work to the executor queue, and
@@ -577,14 +581,25 @@ async def dispatch_turn(
Failure semantics: if ``create_session`` or the queue publish raises,
``slot.keep()`` is never reached, so the context manager that owns
``slot`` releases it on exit — no leak.
+
+ ``root`` mints a root whatever turn's context the caller runs in: the
+ queue's slot-free hook runs inside the turn that just ended. An
+ ``envelope`` was admitted before (an approval wake carries the one its
+ call was held under): it is re-checked instead, narrowed by today's
+ ``permissions``, and its node is not released if dispatch fails.
"""
# Local import: stream_registry imports executor.utils (the
# COPILOT_CONSUMER_TIMEOUT_SECONDS constant) → top-level circular.
from backend.copilot import stream_registry
- envelope = await _admitted_turn_envelope(
- turn_id, session_id, user_id, permissions, spawn
- )
+ admitted_here = envelope is None
+ if envelope is None:
+ envelope = await _admitted_turn_envelope(
+ turn_id, session_id, user_id, permissions, spawn, root=root
+ )
+ else:
+ await readmit_turn(envelope, user_id=user_id)
+ envelope = narrowed_by(envelope, permissions)
# Everything after the admit above runs inside the try: the tree's node
# counter is already incremented, so an exception from ``create_session``
@@ -636,7 +651,8 @@ async def dispatch_turn(
committed = True
finally:
if not committed:
- await release_turn(envelope)
+ if admitted_here:
+ await release_turn(envelope)
try:
await stream_registry.delete_session_meta(session_id)
except BaseException:
@@ -654,6 +670,8 @@ async def _admitted_turn_envelope(
user_id: str | None,
permissions: CopilotPermissions | None,
spawn: SpawnRequest | None,
+ *,
+ root: bool = False,
) -> TurnEnvelope:
"""Derive this turn's envelope from the running turn's (a child) or mint
a root, then admit it against the tree ledger.
@@ -662,7 +680,7 @@ async def _admitted_turn_envelope(
outside any turn — the HTTP route, the scheduler, a graph block — is a
root by construction rather than by declaration.
"""
- spawner = get_current_envelope()
+ spawner = None if root else get_current_envelope()
if spawn is not None and spawner is None:
# A caller that passed a SpawnRequest is by construction a spawn tool
# running inside a turn, so a missing spawner envelope means the
diff --git a/autogpt_platform/backend/backend/copilot/gate/__init__.py b/autogpt_platform/backend/backend/copilot/gate/__init__.py
index 4f042f769c..964bf82441 100644
--- a/autogpt_platform/backend/backend/copilot/gate/__init__.py
+++ b/autogpt_platform/backend/backend/copilot/gate/__init__.py
@@ -19,6 +19,7 @@ from typing import Any, Awaitable, Callable
from prisma.enums import ReviewStatus
from pydantic import BaseModel, ConfigDict
+from backend.copilot.context import get_current_envelope
from backend.copilot.model import ChatSession
from backend.copilot.tree import raise_ceiling, spent_past_ceiling
from backend.platform_linking.models import Platform
@@ -227,6 +228,7 @@ async def check_action(
tool_call_id=tool_call_id,
args=args,
rule_key=rule_key,
+ envelope=get_current_envelope(),
)
return await _park(
call, user_id, session, reason, reason_kind, subject, decided_by, spend_shown
diff --git a/autogpt_platform/backend/backend/copilot/gate/held.py b/autogpt_platform/backend/backend/copilot/gate/held.py
index bbfa611fc9..53a9dd5864 100644
--- a/autogpt_platform/backend/backend/copilot/gate/held.py
+++ b/autogpt_platform/backend/backend/copilot/gate/held.py
@@ -22,6 +22,7 @@ from backend.copilot import woken_turns
from backend.copilot.constants import COPILOT_NODE_EXEC_ID_SEPARATOR
from backend.copilot.model import ChatSession
from backend.copilot.pending_messages import PendingMessage
+from backend.copilot.tree import TreeRefusal, TurnEnvelope
from backend.data.db_accessors import chat_db, review_db
from backend.data.redis_client import get_redis_async
from backend.util.encryption import JSONCryptor
@@ -83,6 +84,9 @@ class HeldCall(BaseModel):
held_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
# Rebuilt from a card whose stored copy of the arguments no longer binds.
lost: bool = False
+ # The turn that held it. Its wake starts under this, re-checked against the
+ # tree; a call rebuilt from its card has none and is not woken.
+ envelope: TurnEnvelope | None = None
async def remember(session_id: str, call: HeldCall) -> bool:
@@ -209,7 +213,14 @@ async def wake(
from backend.copilot.executor.utils import dispatch_turn
from backend.copilot.model import ChatMessage, append_and_save_message
from backend.copilot.session_permissions import resolve_session_permissions
- from backend.copilot.turn_queue import InflightCapExceeded, try_enqueue_turn
+ from backend.copilot.turn_queue import (
+ UNRECORDED_WAKE,
+ WAKE_LATER,
+ InflightCapExceeded,
+ is_users_own_chat,
+ post_refusal,
+ try_enqueue_turn,
+ )
try:
await _restore(user_id, session_id, answered_rows)
@@ -227,6 +238,13 @@ async def wake(
info = await chat_db().get_chat_session_metadata(session_id)
if info is None or info.user_id != user_id:
return
+ refusal_id = str(uuid.uuid5(uuid.NAMESPACE_URL, f"{wake_id}:refused"))
+ envelope = max(calls, key=lambda c: c.held_at).envelope
+ if envelope is None and not is_users_own_chat(info):
+ # Deriving one from the turn that just ended would run the approved
+ # action under that turn's limits, not the ones it was held under.
+ await post_refusal(session_id, UNRECORDED_WAKE, message_id=refusal_id)
+ return
permissions = resolve_session_permissions(info)
metadata = {_WAKE_KEY: True}
try:
@@ -260,6 +278,9 @@ async def wake(
llm_credential_id=info.metadata.llm_credential_id,
permissions=permissions,
message_metadata=metadata,
+ envelope=envelope,
+ # Lost in the user's own chat: a root, as their message would be.
+ root=envelope is None,
)
# A channel that answered one of these follows this turn.
await woken_turns.record(
@@ -278,6 +299,11 @@ async def wake(
permissions=(
permissions.model_dump(exclude_none=True) if permissions else None
),
+ envelope=envelope,
+ )
+ except TreeRefusal as refused:
+ await post_refusal(
+ session_id, f"{refused.message} {WAKE_LATER}", message_id=refusal_id
)
except InflightCapExceeded:
logger.info(f"Held calls in {session_id} wait for the user's next turn")
diff --git a/autogpt_platform/backend/backend/copilot/gate/held_db_test.py b/autogpt_platform/backend/backend/copilot/gate/held_db_test.py
index a906284456..1e16d32c84 100644
--- a/autogpt_platform/backend/backend/copilot/gate/held_db_test.py
+++ b/autogpt_platform/backend/backend/copilot/gate/held_db_test.py
@@ -15,6 +15,8 @@ import pytest
from prisma.enums import ReviewStatus
from prisma.models import PendingHumanReview
+from backend.copilot import stream_registry
+from backend.copilot.context import set_execution_context
from backend.copilot.gate import chat_rules, check_action, held
from backend.copilot.gate import review as review_store
from backend.copilot.gate.classifier import Judgement
@@ -28,8 +30,11 @@ from backend.copilot.model import (
upsert_chat_session,
)
from backend.copilot.pending_message_helpers import persist_pending_as_user_rows
+from backend.copilot.permissions import CopilotPermissions
from backend.copilot.tools.base import BaseTool
from backend.copilot.tools.models import ResponseType, ToolResponseBase
+from backend.copilot.tree import TurnEnvelope, get_tree_ledger, root_envelope
+from backend.copilot.turn_queue import UNRECORDED_WAKE, WAKE_LATER
from backend.data.db_accessors import review_db
from backend.data.redis_client import get_redis_async
@@ -72,22 +77,47 @@ def post_tool():
yield tool
-async def _new_session(user_id: str, mode: AutopilotMode = "auto") -> ChatSession:
- session = await upsert_chat_session(ChatSession.new(user_id=user_id, dry_run=False))
+async def _new_session(
+ user_id: str, mode: AutopilotMode = "auto", delegated_by: str | None = None
+) -> ChatSession:
+ session = await upsert_chat_session(
+ ChatSession.new(
+ user_id=user_id,
+ dry_run=False,
+ # A delegation from the user's chat inherits its origin.
+ delegated_by_session_id=delegated_by,
+ )
+ )
await update_session_autopilot_mode(session.session_id, user_id, mode)
reloaded = await get_chat_session(session.session_id, user_id)
assert reloaded is not None
return reloaded
-async def _hold(session: ChatSession, user_id: str, text: str) -> str:
- decision = await check_action(
- _TOOL, {"text": text}, user_id, session, tool_call_id="call-1"
- )
+async def _hold(
+ session: ChatSession,
+ user_id: str,
+ text: str,
+ envelope: TurnEnvelope | None = None,
+) -> str:
+ async def in_a_turn():
+ # The gate runs inside a turn, whose envelope the call's wake starts under.
+ set_execution_context(
+ user_id, session, envelope=envelope or _turn_envelope(session)
+ )
+ return await check_action(
+ _TOOL, {"text": text}, user_id, session, tool_call_id="call-1"
+ )
+
+ decision = await asyncio.create_task(in_a_turn())
assert not decision.allowed and decision.review_id
return decision.review_id
+def _turn_envelope(session: ChatSession) -> TurnEnvelope:
+ return root_envelope(f"turn-{session.session_id}", session_id=session.session_id)
+
+
async def _answer(review_id: str, status: ReviewStatus, age=timedelta(0)) -> None:
await PendingHumanReview.prisma().update(
where={"nodeExecId": review_id},
@@ -176,6 +206,127 @@ async def test_an_approval_runs_the_call_with_its_stored_arguments(
assert await _row(review_id, test_user_id) is None
[email protected](loop_scope="session")
+async def test_a_wake_starts_under_the_envelope_its_call_was_held_under(
+ setup_test_user, test_user_id, gate_on
+):
+ session = await _new_session(test_user_id)
+ review_id = await _hold(session, test_user_id, "under its own turn")
+ await _answer(review_id, ReviewStatus.APPROVED)
+ dispatch = AsyncMock()
+
+ with patch("backend.copilot.executor.utils.dispatch_turn", dispatch):
+ await held.wake(test_user_id, session.session_id)
+
+ assert dispatch.await_args.kwargs["envelope"] == _turn_envelope(session)
+
+
[email protected](loop_scope="session")
+async def test_a_call_rebuilt_from_its_card_wakes_as_a_root_in_the_users_own_chat(
+ setup_test_user, test_user_id, gate_on, post_tool
+):
+ """Its record was lost, but a turn in the user's own chat is a root anyway:
+ the wake starts as one, under the session's permissions as they are now."""
+ session = await _new_session(test_user_id)
+ review_id = await _hold(session, test_user_id, "lost in Redis")
+ assert await held._claim(session.session_id, review_id)
+ await _answer(review_id, ReviewStatus.APPROVED)
+ rows = await review_db().get_reviews_by_node_exec_ids([review_id], test_user_id)
+ today = CopilotPermissions(tools=["web_fetch"], tools_exclude=True)
+ dispatch = AsyncMock()
+
+ with (
+ patch("backend.copilot.executor.utils.dispatch_turn", dispatch),
+ patch(
+ "backend.copilot.session_permissions.resolve_session_permissions",
+ return_value=today,
+ ),
+ ):
+ await held.wake(test_user_id, session.session_id, rows.values())
+
+ woken = dispatch.await_args.kwargs
+ assert (woken["envelope"], woken["root"]) == (None, True)
+ assert woken["permissions"] == today
+
+
[email protected](loop_scope="session")
+async def test_a_call_rebuilt_from_its_card_in_a_sub_session_is_not_woken(
+ setup_test_user, test_user_id, gate_on, post_tool
+):
+ """Nothing on the row bounds what the call was held under there; the next
+ turn carries the answer."""
+ session = await _new_session(test_user_id, delegated_by="parent-session")
+ review_id = await _hold(session, test_user_id, "lost in Redis")
+ assert await held._claim(session.session_id, review_id)
+ await _answer(review_id, ReviewStatus.APPROVED)
+ rows = await review_db().get_reviews_by_node_exec_ids([review_id], test_user_id)
+ dispatch = AsyncMock()
+
+ with patch("backend.copilot.executor.utils.dispatch_turn", dispatch):
+ await held.wake(test_user_id, session.session_id, rows.values())
+ await held.wake(test_user_id, session.session_id, rows.values())
+
+ dispatch.assert_not_awaited()
+ reloaded = await get_chat_session(session.session_id, test_user_id)
+ assert reloaded is not None
+ refusals = [m for m in reloaded.messages if m.content == UNRECORDED_WAKE]
+ assert len(refusals) == 1
+ assert len(await held.resolve_answered(test_user_id, reloaded)) == 1
+
+
[email protected](loop_scope="session")
+async def test_a_wake_whose_tree_closed_is_not_started(
+ setup_test_user, test_user_id, gate_on
+):
+ """A sub-session's call answered after its tree expired: the reason goes in
+ the chat and the answer waits for the next turn."""
+ session = await _new_session(test_user_id)
+ sub_turn = TurnEnvelope(tree_id=f"tree-{session.session_id}", depth=1)
+ ledger = await get_tree_ledger()
+ await ledger.open(sub_turn.tree_id, ceiling_microdollars=1_000_000, max_nodes=10)
+ review_id = await _hold(session, test_user_id, "after expiry", envelope=sub_turn)
+ await _answer(review_id, ReviewStatus.APPROVED)
+ await (await get_redis_async()).delete(ledger.key(sub_turn.tree_id))
+ enqueued = AsyncMock()
+
+ with patch("backend.copilot.executor.utils.enqueue_copilot_turn", enqueued):
+ await held.wake(test_user_id, session.session_id)
+
+ enqueued.assert_not_awaited()
+ reloaded = await get_chat_session(session.session_id, test_user_id)
+ assert reloaded is not None
+ assert reloaded.messages[-1].content.startswith("This task's tree has closed")
+ assert reloaded.messages[-1].content.endswith(WAKE_LATER)
+
+
[email protected](loop_scope="session")
+async def test_a_tool_revoked_while_a_call_was_held_stays_revoked(
+ setup_test_user, test_user_id, gate_on
+):
+ session = await _new_session(test_user_id)
+ review_id = await _hold(session, test_user_id, "before the revoke")
+ await _answer(review_id, ReviewStatus.APPROVED)
+ revoked = CopilotPermissions(tools=["web_fetch"], tools_exclude=True)
+ enqueued = AsyncMock()
+
+ with (
+ patch(
+ "backend.copilot.session_permissions.resolve_session_permissions",
+ return_value=revoked,
+ ),
+ patch("backend.copilot.executor.utils.enqueue_copilot_turn", enqueued),
+ ):
+ await held.wake(test_user_id, session.session_id)
+ woken = enqueued.await_args.kwargs
+ # The woken turn holds one of the shared test user's slots until it ends.
+ await stream_registry.mark_session_completed(
+ session.session_id, turn_id=woken["turn_id"]
+ )
+
+ assert not woken["envelope"].permits("web_fetch")
+ assert woken["envelope"].tree_id == _turn_envelope(session).tree_id
+
+
async def _approve_after_losing_the_held_call(
session: ChatSession, user_id: str, review_id: str
) -> None:
diff --git a/autogpt_platform/backend/backend/copilot/gate/reads.py b/autogpt_platform/backend/backend/copilot/gate/reads.py
index dec1d03149..6d3d752d54 100644
--- a/autogpt_platform/backend/backend/copilot/gate/reads.py
+++ b/autogpt_platform/backend/backend/copilot/gate/reads.py
@@ -29,6 +29,7 @@ from pydantic import BaseModel, ConfigDict
from backend.api.features.graph_executions.review.model import PendingHumanReviewModel
from backend.copilot.constants import AUTOPILOT_NAME, COPILOT_NODE_PREFIX
+from backend.copilot.context import get_current_envelope
from backend.copilot.model import ChatSession
from backend.copilot.tools.models import ApprovalRequiredResponse, ResponseType
from backend.data.db_accessors import experts_db
@@ -250,6 +251,7 @@ async def screen_read(
tool_name=tool_name,
tool_call_id=tool_call_id,
args=args,
+ envelope=get_current_envelope(),
)
return await _hold(
call,
diff --git a/autogpt_platform/backend/backend/copilot/tree.py b/autogpt_platform/backend/backend/copilot/tree.py
index 6d3defe426..828db0df06 100644
--- a/autogpt_platform/backend/backend/copilot/tree.py
+++ b/autogpt_platform/backend/backend/copilot/tree.py
@@ -277,6 +277,18 @@ def derive_child_envelope(
)
+def narrowed_by(
+ envelope: TurnEnvelope, permissions: CopilotPermissions | None
+) -> TurnEnvelope:
+ """A stored envelope under today's permissions: a tool revoked since it was
+ stored stays out, though the envelope still lists it."""
+ if permissions is None:
+ return envelope
+ allowed = permissions.effective_allowed_tools(ALL_TOOL_NAMES)
+ tools = allowed if envelope.tools is None else envelope.tools & allowed
+ return envelope.model_copy(update={"tools": frozenset(tools)})
+
+
class TreeLedger:
"""One Redis hash per tree: ``ceiling``, ``spent``, ``nodes``, ``max_nodes``.
@@ -346,9 +358,28 @@ class TreeLedger:
async def admit(self, envelope: TurnEnvelope) -> None:
key = self.key(envelope.tree_id)
+ max_nodes = await self._admissible(envelope)
+ nodes = await self._hincrby(key, "nodes", 1)
+ if nodes > int(max_nodes):
+ await self._hincrby(key, "nodes", -1)
+ raise TreeRefusal(
+ f"This task has already used its {max_nodes} agents — that is a "
+ "lifetime budget for this request, not a concurrency limit, so "
+ "waiting will not free one. Finish the remaining work yourself."
+ )
+
+ async def recheck(self, envelope: TurnEnvelope) -> None:
+ """Refuse a turn admitted earlier whose tree has since closed or spent
+ its ceiling, without counting it as a node again."""
+ await self._admissible(envelope)
+
+ async def _admissible(self, envelope: TurnEnvelope) -> int:
+ """The tree's node cap, once its open state and spend allow the turn."""
ceiling, spent, max_nodes = await cast(
Awaitable[list[str | None]],
- self._redis.hmget(key, ["ceiling", "spent", "max_nodes"]),
+ self._redis.hmget(
+ self.key(envelope.tree_id), ["ceiling", "spent", "max_nodes"]
+ ),
)
if ceiling is None or spent is None or max_nodes is None:
raise TreeRefusal("This task's tree has closed; nothing more can start.")
@@ -358,14 +389,7 @@ class TreeLedger:
# the module docstring; the node cap is the bound that holds.
if envelope.depth > 0 and int(spent) >= int(ceiling):
raise TreeRefusal(_spend_refusal(int(spent), int(ceiling)))
- nodes = await self._hincrby(key, "nodes", 1)
- if nodes > int(max_nodes):
- await self._hincrby(key, "nodes", -1)
- raise TreeRefusal(
- f"This task has already used its {max_nodes} agents — that is a "
- "lifetime budget for this request, not a concurrency limit, so "
- "waiting will not free one. Finish the remaining work yourself."
- )
+ return int(max_nodes)
async def release(self, tree_id: str) -> None:
"""Undo an admit whose dispatch never happened."""
@@ -579,6 +603,25 @@ async def admit_turn(
) from e
+async def readmit_turn(envelope: TurnEnvelope, *, user_id: str | None) -> None:
+ """Re-check a turn admitted earlier — a queued child, an approval wake —
+ before it starts, without a second node. Raises :class:`TreeRefusal`."""
+ if envelope.depth == 0:
+ await admit_turn(envelope, user_id=user_id)
+ return
+ if envelope.deadline_at is not None and envelope.deadline_at <= datetime.now(UTC):
+ raise TreeRefusal("This task's deadline passed before it could start.")
+ try:
+ await (await get_tree_ledger()).recheck(envelope)
+ except TreeRefusal:
+ raise
+ except Exception as e:
+ logger.warning(f"Tree ledger unavailable; refusing queued turn: {e}")
+ raise TreeRefusal(
+ "Could not account for this work right now; try again shortly."
+ ) from e
+
+
async def _open_metered_root(
envelope: TurnEnvelope, user_id: str | None, ledger: TreeLedger | None
) -> None:
diff --git a/autogpt_platform/backend/backend/copilot/turn_queue.py b/autogpt_platform/backend/backend/copilot/turn_queue.py
index fd0f361ddf..47314792ca 100644
--- a/autogpt_platform/backend/backend/copilot/turn_queue.py
+++ b/autogpt_platform/backend/backend/copilot/turn_queue.py
@@ -33,7 +33,7 @@ import uuid
from typing import Any, Mapping
from prisma.errors import UniqueViolationError
-from pydantic import BaseModel
+from pydantic import BaseModel, ValidationError
from backend.copilot.active_turns import TurnSlot, count_running_turns
from backend.copilot.config import ChatConfig, CopilotLlmAuthProvider
@@ -43,10 +43,14 @@ from backend.copilot.model import (
CHAT_STATUS_QUEUED,
CHAT_STATUS_RUNNING,
ChatMessage,
+ ChatSession,
+ ChatSessionInfo,
_get_session_lock,
+ append_and_save_message,
invalidate_session_cache,
)
from backend.copilot.offers import EntitlementUnavailable, advanced_tier_entitled
+from backend.copilot.permissions import ALL_TOOL_NAMES, CopilotPermissions
from backend.copilot.rate_limit import (
RateLimitExceeded,
RateLimitUnavailable,
@@ -54,12 +58,24 @@ from backend.copilot.rate_limit import (
get_global_rate_limits,
is_user_paywalled,
)
+from backend.copilot.session_permissions import resolve_session_permissions
from backend.copilot.tracking import track_user_message
+from backend.copilot.tree import TreeRefusal, TurnEnvelope
from backend.data.db_accessors import chat_db
from backend.integrations.codex.access import has_codex_access
logger = logging.getLogger(__name__)
+# Pending-row metadata: the envelope a queued approval wake starts under.
+_ENVELOPE_KEY = "envelope"
+# The assistant row that closes a queued turn the promotion could not start.
+_REFUSED_KEY = "queued_turn_refused"
+WAKE_LATER = "Your answer is kept and reaches the assistant with your next message."
+UNRECORDED_WAKE = (
+ "The approved action could not be resumed: what it was allowed to do was "
+ f"not recorded. {WAKE_LATER}"
+)
+
async def count_queued_turns(user_id: str) -> int:
"""Number of ``chatStatus='queued'`` ChatSession rows for ``user_id``."""
@@ -112,6 +128,7 @@ async def try_enqueue_turn(
llm_credential_id: str | None = None,
permissions: Mapping[str, Any] | None = None,
request_arrival_at: float = 0.0,
+ envelope: TurnEnvelope | None = None,
) -> ChatMessage | None:
"""Admit a queued turn against the user's hard cap.
@@ -137,6 +154,7 @@ async def try_enqueue_turn(
llm_credential_id=llm_credential_id,
permissions=permissions,
request_arrival_at=request_arrival_at,
+ envelope=envelope,
)
@@ -155,6 +173,7 @@ async def enqueue_turn(
llm_credential_id: str | None = None,
permissions: Mapping[str, Any] | None = None,
request_arrival_at: float = 0.0,
+ envelope: TurnEnvelope | None = None,
) -> ChatMessage | None:
"""Persist the user's pending message and flip the session to
``"queued"``. Caller is responsible for the in-flight cap check
@@ -179,6 +198,10 @@ async def enqueue_turn(
metadata["permissions"] = dict(permissions)
if request_arrival_at:
metadata["request_arrival_at"] = request_arrival_at
+ # A wake starts under the envelope its call was held under, whichever turn
+ # frees the slot.
+ if envelope is not None:
+ metadata[_ENVELOPE_KEY] = envelope.model_dump(mode="json")
# The Redis NX session lock serialises with ``append_and_save_message``
# so two concurrent submits to the same session can't pick the same
@@ -260,11 +283,24 @@ async def dispatch_next_for_user(user_id: str) -> bool:
the next completion-driven tick once eligibility returns, or the
user cancels manually.
"""
+ # A head that may never start is closed and the next one tried, in a loop:
+ # recursing once per closed row overflows on a long run of them.
+ while (promoted := await _promote_head(user_id)) is None:
+ pass
+ return promoted
+
+
+async def _promote_head(user_id: str) -> bool | None:
+ """One promotion attempt: whether a session was promoted, or ``None`` when
+ the head was closed as one that may never start, so the next can be tried."""
# ``executor.utils`` stays a local import: it pulls
# ``turn_queue.count_inflight_turns`` lazily back through this module,
# so top-leveling it here would deadlock the import graph.
from backend.copilot.executor.utils import dispatch_turn
+ # Local for the same reason: the gate imports this module back.
+ from backend.copilot.gate.held import is_answer_row
+
queued = await list_queued_sessions(user_id)
if not queued:
return False
@@ -286,57 +322,67 @@ async def dispatch_next_for_user(user_id: str) -> bool:
if not await claim_queued_session(head.session_id):
return False
- # Find the pending user message in this session (the most recent
- # user-role row with no following assistant rows — i.e. the one
- # that triggered the queue). Its ``metadata`` carries the
- # dispatcher payload.
- pending = await chat_db().get_latest_user_message_in_session(head.session_id)
- if pending is None or pending.content is None:
- # Shouldn't happen — enqueue_turn always persists a row before
- # flipping the session to queued. If it does (corrupted
- # state), roll back to idle so the next tick doesn't loop.
- await chat_db().update_chat_session_status(
- session_id=head.session_id,
- expect_status=CHAT_STATUS_RUNNING,
- status=CHAT_STATUS_IDLE,
- )
- # Drop the cache so the sidebar doesn't keep showing the
- # stale ``running`` indicator after the rollback.
- await invalidate_session_cache(head.session_id)
- return False
-
- metadata = pending.metadata or {}
-
- # A turn can sit in the queue long enough for the plan that bought it to
- # lapse. The tier was checked when the turn was accepted, but promoting it
- # is a second, later decision to spend, so it gets its own check --
- # otherwise a downgrade between the two buys a free Advanced run. The turn
- # goes back to queued rather than quietly re-running on Standard: nothing
- # in this feature changes what a turn runs on without being asked. It
- # promotes itself once entitlement returns, and can be cancelled meanwhile.
- if route_provider == "platform" and metadata.get("model") == "advanced":
- try:
- entitled = await advanced_tier_entitled(user_id)
- except EntitlementUnavailable:
- entitled = False
- logger.warning(
- "dispatch_next_for_user: could not resolve the Advanced "
- "entitlement for user=%s; leaving session=%s queued",
- user_id,
- head.session_id,
- exc_info=True,
- )
- if not entitled:
+ try:
+ # Find the pending user message in this session (the most recent
+ # user-role row with no following assistant rows — i.e. the one
+ # that triggered the queue). Its ``metadata`` carries the
+ # dispatcher payload.
+ pending = await chat_db().get_latest_user_message_in_session(head.session_id)
+ if pending is None or pending.content is None:
+ # Shouldn't happen — enqueue_turn always persists a row before
+ # flipping the session to queued. If it does (corrupted
+ # state), roll back to idle so the next tick doesn't loop.
await chat_db().update_chat_session_status(
session_id=head.session_id,
expect_status=CHAT_STATUS_RUNNING,
- status=CHAT_STATUS_QUEUED,
+ status=CHAT_STATUS_IDLE,
)
+ # Drop the cache so the sidebar doesn't keep showing the
+ # stale ``running`` indicator after the rollback.
await invalidate_session_cache(head.session_id)
return False
- turn_id = str(uuid.uuid4())
- try:
+ metadata = pending.metadata or {}
+ queued_envelope = _stored_envelope(metadata)
+ if (
+ is_answer_row(pending)
+ and queued_envelope is None
+ and not is_users_own_chat(head)
+ ):
+ # Deriving one from the turn that just ended would run the approved
+ # action under that turn's limits; the answer reaches the next turn.
+ await _refuse_queued_turn(head, UNRECORDED_WAKE)
+ return None
+
+ # A turn can sit in the queue long enough for the plan that bought it to
+ # lapse. The tier was checked when the turn was accepted, but promoting it
+ # is a second, later decision to spend, so it gets its own check --
+ # otherwise a downgrade between the two buys a free Advanced run. The turn
+ # goes back to queued rather than quietly re-running on Standard: nothing
+ # in this feature changes what a turn runs on without being asked. It
+ # promotes itself once entitlement returns, and can be cancelled meanwhile.
+ if route_provider == "platform" and metadata.get("model") == "advanced":
+ try:
+ entitled = await advanced_tier_entitled(user_id)
+ except EntitlementUnavailable:
+ entitled = False
+ logger.warning(
+ "dispatch_next_for_user: could not resolve the Advanced "
+ "entitlement for user=%s; leaving session=%s queued",
+ user_id,
+ head.session_id,
+ exc_info=True,
+ )
+ if not entitled:
+ await chat_db().update_chat_session_status(
+ session_id=head.session_id,
+ expect_status=CHAT_STATUS_RUNNING,
+ status=CHAT_STATUS_QUEUED,
+ )
+ await invalidate_session_cache(head.session_id)
+ return False
+
+ turn_id = str(uuid.uuid4())
# The user's message is already persisted AND the session is
# already ``chatStatus='running'`` from claim_queued_session.
# Build a TurnSlot directly (no acquire) so we don't re-check
@@ -364,13 +410,21 @@ async def dispatch_next_for_user(user_id: str) -> bool:
model=metadata.get("model"),
llm_auth_provider=head.metadata.llm_auth_provider,
llm_credential_id=head.metadata.llm_credential_id,
- permissions=metadata.get("permissions"),
+ permissions=_promotion_permissions(head, metadata),
request_arrival_at=float(metadata.get("request_arrival_at") or 0.0),
+ # A typed message is a root, as the chat route makes it, not the
+ # child of the turn this hook runs in; a wake brings its own.
+ envelope=queued_envelope,
+ root=queued_envelope is None,
)
+ except TreeRefusal as refused:
+ # Only a stored envelope is re-checked here; a root is never refused.
+ await _refuse_queued_turn(head, f"{refused.message} {WAKE_LATER}")
+ return None
except BaseException:
# Roll the claim back so a missed-dispatch tick or the next
# slot-free event can retry. ``BaseException`` (not just
- # ``Exception``) so a task cancellation that lands mid-dispatch
+ # ``Exception``) so a task cancellation that lands after the claim
# still leaves the session in a recoverable ``queued`` state
# rather than a stuck ``running``. Redis-side cleanup of the
# meta that ``dispatch_turn``'s ``create_session`` wrote is
@@ -410,6 +464,97 @@ async def dispatch_next_for_user(user_id: str) -> bool:
return True
+def is_users_own_chat(session: ChatSessionInfo) -> bool:
+ """A chat the user opened themselves, not one another session or a graph
+ started. Its turns are roots, so a wake there whose envelope was lost can
+ start as one; elsewhere nothing on the row bounds what the call held."""
+ return (
+ session.metadata.origin == "interactive"
+ and session.metadata.delegated_by_session_id is None
+ )
+
+
+async def _refuse_queued_turn(head: ChatSessionInfo, reason: str) -> None:
+ """Close a promoted turn that may not start: say why in its thread and
+ free its slot, which a failed post must not keep."""
+ try:
+ await post_refusal(head.session_id, reason)
+ except Exception:
+ # Not raised: the drain goes on, and the next queued turn takes the slot.
+ logger.exception(
+ f"dispatch_next_for_user: could not post why session={head.session_id} "
+ "was not started"
+ )
+ finally:
+ await chat_db().update_chat_session_status(
+ session_id=head.session_id,
+ expect_status=CHAT_STATUS_RUNNING,
+ status=CHAT_STATUS_IDLE,
+ )
+ await invalidate_session_cache(head.session_id)
+
+
+async def post_refusal(
+ session_id: str, reason: str, *, message_id: str | None = None
+) -> None:
+ """Say in the thread why a turn that was waiting will not start."""
+ await append_and_save_message(
+ session_id,
+ ChatMessage(
+ id=message_id,
+ role="assistant",
+ content=reason,
+ metadata={_REFUSED_KEY: True},
+ ),
+ )
+
+
+def queued_turn_refusal(session: ChatSession) -> str | None:
+ """Why a waiting turn was not started, if its thread ends that way."""
+ last = session.messages[-1] if session.messages else None
+ if last is None or not (last.metadata or {}).get(_REFUSED_KEY):
+ return None
+ return last.content
+
+
+def _stored_envelope(metadata: Mapping[str, Any]) -> TurnEnvelope | None:
+ """The envelope a wake was queued under. One that no longer parses (a schema
+ change between deploys) counts as unrecorded, so the wake is not started."""
+ stored = metadata.get(_ENVELOPE_KEY)
+ if not stored:
+ return None
+ try:
+ return TurnEnvelope.model_validate(stored)
+ except ValidationError:
+ logger.warning("dispatch_next_for_user: a stored envelope did not parse")
+ return None
+
+
+def _promotion_permissions(
+ head: ChatSessionInfo, metadata: Mapping[str, Any]
+) -> CopilotPermissions | None:
+ """The session's permissions as they are now, never looser than the ones
+ stored at enqueue. Every writer stores this session's own resolved
+ permissions, so a stored value that no longer parses is read afresh."""
+ current = resolve_session_permissions(head)
+ stored = metadata.get("permissions")
+ if not stored:
+ return current
+ try:
+ queued = CopilotPermissions.model_validate(stored)
+ except ValidationError:
+ logger.warning(
+ f"dispatch_next_for_user: stored permissions on session={head.session_id} "
+ "did not parse; using the session's current ones"
+ )
+ return current
+ return (
+ queued
+ if current is None
+ else queued.merged_with_parent(current, ALL_TOOL_NAMES)
+ )
+
+
class TurnRefusal(BaseModel):
reason: str
# The usage store was unreadable: an outage, not the user's state.
diff --git a/autogpt_platform/backend/backend/copilot/turn_queue_db_test.py b/autogpt_platform/backend/backend/copilot/turn_queue_db_test.py
new file mode 100644
index 0000000000..9b217da7d3
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/turn_queue_db_test.py
@@ -0,0 +1,221 @@
+"""Queue promotion against the database: a queued turn starts from the
+slot-free hook, which runs inside the turn that just ended."""
+
+import asyncio
+import uuid
+from typing import Any
+from unittest.mock import AsyncMock, patch
+
+import pytest
+import pytest_asyncio
+from prisma.models import User
+
+from backend.copilot import stream_registry, turn_queue
+from backend.copilot.context import set_execution_context
+from backend.copilot.gate import held
+from backend.copilot.model import (
+ CHAT_STATUS_IDLE,
+ CHAT_STATUS_QUEUED,
+ CHAT_STATUS_RUNNING,
+ ChatSession,
+ create_chat_session,
+ get_chat_session,
+)
+from backend.copilot.permissions import CopilotPermissions
+from backend.copilot.tree import TurnEnvelope, admit_turn, get_tree_ledger
+from backend.data.db import prisma as db_client
+from backend.data.db_accessors import chat_db
+from backend.util.test import SpinTestServer
+
+
[email protected]("finished_depth", [1, 3])
[email protected](loop_scope="session")
+async def test_a_promoted_message_is_a_root_not_the_finished_turns_child(
+ finished_depth: int,
+):
+ finished = _envelope(finished_depth)
+ promoted, nodes_before, nodes_after = await _promote_after(
+ finished, message="the message that waited", message_metadata=None
+ )
+
+ assert promoted is not None, "the queued message was never dispatched"
+ assert promoted.depth == 0
+ assert promoted.tools is None
+ assert promoted.tree_id != finished.tree_id
+ assert nodes_after == nodes_before
+
+
[email protected](loop_scope="session")
+async def test_a_queued_wake_starts_under_the_envelope_its_call_was_held_under():
+ held_under = _envelope(1)
+ ledger = await get_tree_ledger()
+ await ledger.open(held_under.tree_id, ceiling_microdollars=1_000_000, max_nodes=10)
+ revoked = CopilotPermissions(tools=["web_fetch"], tools_exclude=True)
+
+ # A tool revoked while the wake waited stays revoked.
+ with patch.object(turn_queue, "resolve_session_permissions", return_value=revoked):
+ promoted, _, _ = await _promote_after(
+ _envelope(2),
+ message=held.WAKE_MESSAGE,
+ message_metadata={held._WAKE_KEY: True},
+ envelope=held_under,
+ )
+
+ assert promoted is not None
+ assert (promoted.tree_id, promoted.depth) == (held_under.tree_id, 1)
+ assert not promoted.permits("web_fetch")
+ # The call's turn was counted when it ran; its wake takes no second node.
+ assert (await ledger.snapshot(held_under.tree_id))["nodes"] == 0
+
+
[email protected](loop_scope="session")
+async def test_a_queued_wake_with_no_recorded_envelope_starts_as_a_root():
+ """In the user's own chat, not derived from the turn that freed the slot."""
+ promoted, _, _ = await _promote_after(
+ _envelope(1),
+ message=held.WAKE_MESSAGE,
+ message_metadata={held._WAKE_KEY: True},
+ )
+
+ assert promoted is not None
+ assert (promoted.depth, promoted.tools) == (0, None)
+
+
[email protected](loop_scope="session")
+async def test_a_queued_wake_with_no_recorded_envelope_in_a_sub_session_is_not_started():
+ promoted, _, _ = await _promote_after(
+ _envelope(1),
+ message=held.WAKE_MESSAGE,
+ message_metadata={held._WAKE_KEY: True},
+ expect_refusal=turn_queue.UNRECORDED_WAKE,
+ sub_work=True,
+ )
+
+ assert promoted is None
+
+
[email protected](loop_scope="session")
+async def test_a_queued_wake_whose_envelope_no_longer_parses_is_not_started():
+ """A stored record from before a schema change is as good as none."""
+ promoted, _, _ = await _promote_after(
+ _envelope(1),
+ message=held.WAKE_MESSAGE,
+ message_metadata={held._WAKE_KEY: True, "envelope": {"depth": "deep"}},
+ expect_refusal=turn_queue.UNRECORDED_WAKE,
+ sub_work=True,
+ )
+
+ assert promoted is None
+
+
+async def _promote_after(
+ finished: TurnEnvelope,
+ *,
+ message: str,
+ message_metadata: dict[str, Any] | None,
+ envelope: TurnEnvelope | None = None,
+ expect_refusal: str | None = None,
+ sub_work: bool = False,
+) -> tuple[TurnEnvelope | None, int, int]:
+ """Queue a turn behind a full cap, then end a turn carrying ``finished``.
+
+ Returns the promoted turn's envelope and the finished tree's node count
+ before and after."""
+ user_id = str(uuid.uuid4())
+ await User.prisma().create(
+ data={"id": user_id, "email": f"turn-queue-{user_id}@example.com"}
+ )
+ try:
+ sessions = [await create_chat_session(user_id, dry_run=False) for _ in range(5)]
+ for session in sessions:
+ assert await chat_db().update_chat_session_status(
+ session_id=session.session_id,
+ expect_status=CHAT_STATUS_IDLE,
+ status=CHAT_STATUS_RUNNING,
+ user_id=user_id,
+ )
+ ending, ending_turn = sessions[0], str(uuid.uuid4())
+ await stream_registry.create_session(
+ ending.session_id, user_id, "chat_stream", "chat", ending_turn
+ )
+ waiting = await create_chat_session(
+ user_id,
+ dry_run=False,
+ delegated_by_session_id=sessions[1].session_id if sub_work else None,
+ )
+ await turn_queue.try_enqueue_turn(
+ user_id=user_id,
+ inflight_cap=15,
+ session_id=waiting.session_id,
+ message=message,
+ message_metadata=message_metadata,
+ envelope=envelope,
+ )
+ assert (
+ await chat_db().get_chat_session_status(waiting.session_id)
+ ) == CHAT_STATUS_QUEUED
+
+ await admit_turn(finished, user_id=user_id)
+ ledger = await get_tree_ledger()
+ nodes_before = (await ledger.snapshot(finished.tree_id))["nodes"]
+
+ enqueued = AsyncMock()
+ with (
+ patch("backend.copilot.executor.utils.enqueue_copilot_turn", new=enqueued),
+ patch.object(
+ turn_queue, "is_user_paywalled", new=AsyncMock(return_value=False)
+ ),
+ patch.object(
+ turn_queue,
+ "get_global_rate_limits",
+ new=AsyncMock(return_value=(1, 1, None)),
+ ),
+ patch.object(turn_queue, "check_rate_limit", new=AsyncMock()),
+ ):
+ # The executor sets the envelope inside the engine's generator and
+ # completes the turn in the same task, so the hook inherits it.
+ await asyncio.create_task(_end_turn(user_id, finished, ending, ending_turn))
+
+ promoted = [
+ call.kwargs["envelope"]
+ for call in enqueued.await_args_list
+ if call.kwargs["session_id"] == waiting.session_id
+ ]
+ nodes_after = (await ledger.snapshot(finished.tree_id))["nodes"]
+ if expect_refusal is not None:
+ assert (
+ await chat_db().get_chat_session_status(waiting.session_id)
+ ) == CHAT_STATUS_IDLE
+ closed = await get_chat_session(waiting.session_id, user_id)
+ assert closed is not None
+ assert turn_queue.queued_turn_refusal(closed) == expect_refusal
+ return (promoted[0] if promoted else None), nodes_before, nodes_after
+ finally:
+ await User.prisma().delete(where={"id": user_id})
+
+
+def _envelope(depth: int) -> TurnEnvelope:
+ """A sub-session's turn, deep in a tree of its own."""
+ return TurnEnvelope(
+ tree_id=str(uuid.uuid4()),
+ depth=depth,
+ tools=frozenset({"run_sub_session", "connect_integration", "web_fetch"}),
+ )
+
+
+async def _end_turn(
+ user_id: str, envelope: TurnEnvelope, session: ChatSession, turn_id: str
+) -> None:
+ set_execution_context(user_id, session, envelope=envelope)
+ await stream_registry.mark_session_completed(session.session_id, turn_id=turn_id)
+
+
+@pytest_asyncio.fixture(autouse=True)
+async def absorb_a_stale_event_loop(server: SpinTestServer):
+ """An earlier test can leave the shared Prisma client bound to a closed
+ loop; only the first query on the new loop fails. Spend it here."""
+ try:
+ await db_client.execute_raw("SELECT 1")
+ except RuntimeError as error:
+ if "Event loop is closed" not in str(error):
+ raise
diff --git a/autogpt_platform/backend/backend/copilot/turn_queue_test.py b/autogpt_platform/backend/backend/copilot/turn_queue_test.py
index 7c04a1d1dd..29d67bb2f9 100644
--- a/autogpt_platform/backend/backend/copilot/turn_queue_test.py
+++ b/autogpt_platform/backend/backend/copilot/turn_queue_test.py
@@ -7,6 +7,8 @@ RPC into ``DatabaseManager``. Patching the accessor avoids reaching
for Prisma directly while still exercising the queue's branching.
"""
+import asyncio
+import sys
from datetime import datetime, timezone
from unittest.mock import AsyncMock, MagicMock, patch
@@ -14,7 +16,10 @@ import pytest
from prisma.errors import UniqueViolationError
from backend.copilot import turn_queue
+from backend.copilot.gate import held
from backend.copilot.model import ChatMessage as PydanticChatMessage
+from backend.copilot.permissions import CopilotPermissions
+from backend.copilot.tree import TreeRefusal
class _NoopAsyncCM:
@@ -304,6 +309,184 @@ def _patch_queued_list(rows):
)
+async def test_a_long_run_of_wakes_that_may_not_start_drains_without_recursing() -> (
+ None
+):
+ """Each is closed and the next tried; more of them than the recursion limit
+ must still drain rather than overflow inside the completion hook."""
+ queue = [_mock_session(f"s{i}") for i in range(sys.getrecursionlimit() + 100)]
+ for row in queue:
+ row.metadata.llm_auth_provider = "codex"
+ db = MagicMock()
+ db.get_latest_user_message_in_session = AsyncMock(
+ return_value=_pyd_message(metadata={held._WAKE_KEY: True})
+ )
+
+ async def closed(head, reason):
+ queue.remove(head)
+
+ with (
+ patch.object(
+ turn_queue,
+ "list_queued_sessions",
+ new=AsyncMock(side_effect=lambda _user: list(queue)),
+ ),
+ patch.object(turn_queue, "chat_db", return_value=db),
+ patch.object(turn_queue, "has_codex_access", new=AsyncMock(return_value=True)),
+ patch.object(
+ turn_queue, "claim_queued_session", new=AsyncMock(return_value=True)
+ ),
+ patch.object(turn_queue, "_refuse_queued_turn", new=closed),
+ ):
+ assert await turn_queue.dispatch_next_for_user("u1") is False
+
+ assert queue == []
+
+
+async def test_the_drain_stops_at_a_head_the_access_gate_refuses() -> None:
+ """A wake that may never start is closed and the next head tried; one the
+ shared access gate refuses stays queued, unclaimed, for a later tick."""
+ closed, refused = _mock_session("closed"), _mock_session("refused")
+ closed.metadata.llm_auth_provider = "microsoft_365_copilot"
+ refused.metadata.llm_auth_provider = "codex"
+ db = MagicMock()
+ db.update_chat_session_status = AsyncMock(return_value=True)
+ db.get_latest_user_message_in_session = AsyncMock(
+ return_value=_pyd_message(metadata={held._WAKE_KEY: True})
+ )
+ claim = AsyncMock(return_value=True)
+
+ with (
+ patch.object(
+ turn_queue,
+ "list_queued_sessions",
+ new=AsyncMock(side_effect=[[closed, refused], [refused]]),
+ ),
+ patch.object(turn_queue, "chat_db", return_value=db),
+ patch.object(turn_queue, "has_codex_access", new=AsyncMock(return_value=False)),
+ patch.object(turn_queue, "claim_queued_session", new=claim),
+ patch.object(turn_queue, "invalidate_session_cache", new=AsyncMock()),
+ patch.object(turn_queue, "append_and_save_message", new=AsyncMock()),
+ ):
+ assert await turn_queue.dispatch_next_for_user("u1") is False
+
+ claim.assert_awaited_once_with("closed")
+ db.update_chat_session_status.assert_awaited_once_with(
+ session_id="closed", expect_status="running", status="idle"
+ )
+
+
+async def test_unparseable_stored_permissions_read_the_sessions_current_ones() -> None:
+ """Not left stuck at the head of the queue, and never read as none."""
+ head = _mock_session()
+ head.metadata.llm_auth_provider = "codex"
+ db = MagicMock()
+ db.update_chat_session_status = AsyncMock(return_value=True)
+ db.get_latest_user_message_in_session = AsyncMock(
+ return_value=_pyd_message(metadata={"permissions": {"tools": 5}})
+ )
+ today = CopilotPermissions(tools=["web_fetch"], tools_exclude=True)
+ dispatched = AsyncMock()
+
+ with (
+ _patch_queued_list([head]),
+ patch.object(turn_queue, "chat_db", return_value=db),
+ patch.object(turn_queue, "has_codex_access", new=AsyncMock(return_value=True)),
+ patch.object(
+ turn_queue, "claim_queued_session", new=AsyncMock(return_value=True)
+ ),
+ patch.object(turn_queue, "invalidate_session_cache", new=AsyncMock()),
+ patch.object(turn_queue, "resolve_session_permissions", return_value=today),
+ patch("backend.copilot.executor.utils.dispatch_turn", new=dispatched),
+ ):
+ assert await turn_queue.dispatch_next_for_user("u1") is True
+
+ assert dispatched.await_args.kwargs["permissions"] == today
+
+
+ "stored_envelope",
+ [None, {"tree_id": "t1", "depth": 1}],
+ ids=["unrecorded-wake", "tree-refusal"],
+)
+async def test_a_refusal_that_cannot_be_posted_still_frees_the_slot(
+ stored_envelope: dict | None,
+) -> None:
+ """The session leaves ``running`` and the next queued one is tried."""
+ head = _mock_session()
+ head.metadata.llm_auth_provider = "codex"
+ metadata: dict = {held._WAKE_KEY: True}
+ if stored_envelope:
+ metadata["envelope"] = stored_envelope
+ db = MagicMock()
+ db.update_chat_session_status = AsyncMock(return_value=True)
+ db.get_latest_user_message_in_session = AsyncMock(
+ return_value=_pyd_message(metadata=metadata)
+ )
+
+ with (
+ patch.object(
+ turn_queue, "list_queued_sessions", new=AsyncMock(side_effect=[[head], []])
+ ),
+ patch.object(turn_queue, "chat_db", return_value=db),
+ patch.object(turn_queue, "has_codex_access", new=AsyncMock(return_value=True)),
+ patch.object(
+ turn_queue, "claim_queued_session", new=AsyncMock(return_value=True)
+ ),
+ patch.object(turn_queue, "invalidate_session_cache", new=AsyncMock()),
+ patch.object(turn_queue, "resolve_session_permissions", return_value=None),
+ patch.object(
+ turn_queue,
+ "append_and_save_message",
+ new=AsyncMock(side_effect=RuntimeError("db blip")),
+ ),
+ patch(
+ "backend.copilot.executor.utils.dispatch_turn",
+ new=AsyncMock(side_effect=TreeRefusal("This tree has closed.")),
+ ),
+ ):
+ assert await turn_queue.dispatch_next_for_user("u1") is False
+
+ db.update_chat_session_status.assert_awaited_once_with(
+ session_id="s1", expect_status="running", status="idle"
+ )
+
+
+ "failure", [RuntimeError("db blip"), asyncio.CancelledError()], ids=type
+)
+async def test_a_failure_between_the_claim_and_the_dispatch_requeues_the_turn(
+ failure: BaseException,
+) -> None:
+ """Not left ``running`` with no turn to end it: the next tick retries."""
+ head = _mock_session()
+ head.metadata.llm_auth_provider = "codex"
+ db = MagicMock()
+ db.update_chat_session_status = AsyncMock(return_value=True)
+ db.get_latest_user_message_in_session = AsyncMock(side_effect=failure)
+
+ with (
+ _patch_queued_list([head]),
+ patch.object(turn_queue, "chat_db", return_value=db),
+ patch.object(turn_queue, "has_codex_access", new=AsyncMock(return_value=True)),
+ patch.object(
+ turn_queue, "claim_queued_session", new=AsyncMock(return_value=True)
+ ),
+ patch.object(turn_queue, "invalidate_session_cache", new=AsyncMock()),
+ ):
+ with pytest.raises(type(failure)):
+ await turn_queue.dispatch_next_for_user("u1")
+
+ db.update_chat_session_status.assert_awaited_once_with(
+ session_id="s1", expect_status="running", status="queued"
+ )
+
+
@pytest.mark.asyncio
async def test_dispatch_returns_false_when_queue_empty() -> None:
with _patch_queued_list([]):