Skip to content

Build and run agents you can see, understand and trust.

feat(agent-team): tell the leader when a member's turn fails (#2386)

DavdGao13749ff66329
diff --git a/src/agentscope/app/_service/_chat.py b/src/agentscope/app/_service/_chat.py
index 310fb2322..f769aa533 100644
--- a/src/agentscope/app/_service/_chat.py
+++ b/src/agentscope/app/_service/_chat.py
@@ -13,6 +13,7 @@ that wants them subscribes through the
"""
import asyncio
import inspect
+import json
from dataclasses import dataclass
from typing import Literal, TYPE_CHECKING
@@ -20,6 +21,7 @@ from fastapi import HTTPException
from .._bus_ops import (
abandon_inbox_consumer,
+ deliver_to_inbox,
enqueue_channel_output,
enqueue_run_trigger,
has_pending_inbox_or_release,
@@ -67,7 +69,7 @@ from ...event import (
)
from ._errors import _classify_error, _classify_setup_error
from ..._utils._common import _generate_id
-from ...message import AssistantMsg, Msg, ToolCallState
+from ...message import AssistantMsg, HintBlock, Msg, ToolCallState
from ...permission import AdditionalWorkingDirectory
if TYPE_CHECKING:
@@ -85,6 +87,8 @@ class _LeaderContext:
class _WorkerContext:
"""This session is a worker; its leader is fully resolved."""
+ leader_session_id: str
+ leader_agent_id: str
leader_name: str
role: Literal["worker"] = "worker"
@@ -336,12 +340,102 @@ optional):
end_event.error.type if end_event.error else "error",
)
+ async def _notify_leader_of_failure(
+ self,
+ user_id: str,
+ team_ctx: "_TeamContext | None",
+ worker_name: str,
+ finished_reason: ReplyFinishedReason,
+ detail: str,
+ ) -> None:
+ """Tell a worker's team leader that the worker stopped early.
+
+ A member reports through ``TeamSay``; a member that failed or
+ was interrupted never got there, so without this the leader
+ waits forever for an answer that is not coming. Delivered as a
+ system reminder in the leader's inbox — not as a team message,
+ because the member did not say this, the framework did — waking
+ the leader when it is idle.
+
+ Best-effort: a failure to notify is logged and dropped rather
+ than replacing the failure being reported.
+
+ Args:
+ user_id (`str`):
+ The owner of both sessions.
+ team_ctx (`_TeamContext | None`):
+ The run's team identity. Anything but a
+ :class:`_WorkerContext` is a no-op — a leader or a
+ team-less session has nobody to report to.
+ worker_name (`str`):
+ The failing member's display name, as the leader knows
+ it from the roster.
+ finished_reason (`ReplyFinishedReason`):
+ How the reply ended, which decides what the leader is
+ asked to consider.
+ detail (`str`):
+ One-line, already-sanitized explanation of the error.
+ """
+ if not isinstance(team_ctx, _WorkerContext):
+ return
+
+ # Error messages come from several sources and only some end in
+ # punctuation; the reminder reads as prose either way.
+ detail = detail.strip()
+ if detail and detail[-1] not in ".!?":
+ detail += "."
+
+ if finished_reason == ReplyFinishedReason.INTERRUPTED:
+ reminder = (
+ f"Team member {worker_name!r} was interrupted mid-task and "
+ f"has stopped. Someone cancelled that run deliberately — "
+ f"possibly the user, who may be working with this member "
+ f"directly. Do not silently re-dispatch it: ask the user "
+ f"what should happen to the task, or ask the member how "
+ f"far it got. Left unresolved, the member is stranded with "
+ f"work nobody is tracking."
+ )
+ else:
+ reminder = (
+ f"Team member {worker_name!r} hit an error while running, "
+ f"so it never called TeamSay to report. Error: {detail} "
+ f"Judge from the error type whether to retry — with this "
+ f"member or a fresh one — or to raise it with the user: "
+ f"invalid credentials or an exhausted quota will fail the "
+ f"same way again. Assume no usable output unless the "
+ f"member reported partial results earlier."
+ )
+
+ try:
+ hint = HintBlock(
+ hint=f"<system-reminder>{reminder}</system-reminder>",
+ source=json.dumps(
+ {"label": "System", "sublabel": "Reminder"},
+ ensure_ascii=False,
+ ),
+ )
+ await deliver_to_inbox(
+ self._message_bus,
+ user_id=user_id,
+ session_id=team_ctx.leader_session_id,
+ agent_id=team_ctx.leader_agent_id,
+ payload=hint.model_dump(mode="json"),
+ )
+ except Exception: # pylint: disable=broad-except
+ logger.exception(
+ "Failed to notify team leader %r that member %r stopped.",
+ team_ctx.leader_name,
+ worker_name,
+ )
+
async def _report_failure(
self,
user_id: str,
session_id: str,
agent_id: str,
error: Exception,
+ team_ctx: "_TeamContext | None" = None,
+ worker_name: str | None = None,
) -> None:
"""Tell the client about a failure that reached no reply.
@@ -370,6 +464,12 @@ optional):
The agent that was being assembled.
error (`Exception`):
What went wrong while setting the run up.
+ team_ctx (`_TeamContext | None`, optional):
+ The run's team identity; a worker's leader is told the
+ run never happened.
+ worker_name (`str | None`, optional):
+ Display name used in that notification. Defaults to
+ ``agent_id`` when the agent record never loaded.
"""
try:
reply_id = _generate_id()
@@ -409,6 +509,14 @@ optional):
session_id,
)
+ await self._notify_leader_of_failure(
+ user_id,
+ team_ctx,
+ worker_name or agent_id,
+ ReplyFinishedReason.ERROR,
+ _classify_setup_error(error).message,
+ )
+
async def interrupt(
self,
user_id: str,
@@ -550,6 +658,11 @@ optional):
MessageBusKeys.session_lock(session_id),
ttl_secs=MessageBusKeys.SESSION_RUN_TTL_SECS,
):
+ # Bound before the try: an assembly failure reports through
+ # them, and may happen before either is resolved.
+ team_ctx: _TeamContext | None = None
+ worker_name: str = agent_id
+
# Steps 1-6 assemble the run; step 7 performs it. A failure
# here has no reply to attach to, so one is synthesized —
# otherwise the client sees a stream that simply stops and is
@@ -588,30 +701,13 @@ optional):
f"agent {agent_id!r}."
),
)
- workspace = await self._workspace_manager.get_workspace(
- user_id,
- agent_id,
- session_id,
- session_record.config.workspace_id,
- )
-
- # Add workspace working directory to the permission context
- working_dirs = (
- session_record.state.permission_context.working_directories
- )
- if workspace.workdir not in working_dirs:
- working_dirs[
- workspace.workdir
- ] = AdditionalWorkingDirectory(
- path=workspace.workdir,
- source="session",
- )
+ worker_name = agent_record.data.name
# -------------------------------------------------------------
- # 1b. Resolve team identity + channel binding ONCE; all
- # downstream consumers reuse these instead of re-fetching.
+ # 1b. Resolve the team identity ONCE, before anything that
+ # can fail: a worker whose assembly dies still has to reach
+ # its leader. Downstream consumers reuse this.
# -------------------------------------------------------------
- team_ctx: _TeamContext | None = None
if session_record.team_id is not None:
team = await self._storage.get_team(
user_id,
@@ -645,8 +741,35 @@ optional):
f"{session_id!r} cannot run."
),
)
- team_ctx = _WorkerContext(leader_name=leader.name)
+ team_ctx = _WorkerContext(
+ leader_session_id=leader.session_id,
+ leader_agent_id=leader.agent.id,
+ leader_name=leader.name,
+ )
+
+ workspace = await self._workspace_manager.get_workspace(
+ user_id,
+ agent_id,
+ session_id,
+ session_record.config.workspace_id,
+ )
+
+ # Add workspace working directory to the permission context
+ working_dirs = (
+ session_record.state.permission_context.working_directories
+ )
+ if workspace.workdir not in working_dirs:
+ working_dirs[
+ workspace.workdir
+ ] = AdditionalWorkingDirectory(
+ path=workspace.workdir,
+ source="session",
+ )
+ # -------------------------------------------------------------
+ # 1c. Resolve the channel binding ONCE; the toolkit and the
+ # system-prompt attachment share it.
+ # -------------------------------------------------------------
channel = (
self._channel_dispatcher.get_local_channel(
session_record.source_channel_id,
@@ -874,7 +997,14 @@ optional):
# to make: these events share a channel with a live reply's,
# so publishing them unserialised would drop a "reply failed"
# into the middle of an answer another run is streaming.
- await self._report_failure(user_id, session_id, agent_id, e)
+ await self._report_failure(
+ user_id,
+ session_id,
+ agent_id,
+ e,
+ team_ctx,
+ worker_name,
+ )
return
# ----------------------------------------------------------------
@@ -1060,6 +1190,8 @@ optional):
session_id,
agent_id,
e,
+ team_ctx,
+ worker_name,
)
else:
await self._close_failed_reply(
@@ -1119,19 +1251,45 @@ optional):
# acquire the lock and load a stale state from storage
# before this write lands.
async def _persist() -> None:
- for msg in reply_msgs:
- await self._storage.upsert_message(
- user_id,
- session_id,
- msg,
+ try:
+ for msg in reply_msgs:
+ await self._storage.upsert_message(
+ user_id,
+ session_id,
+ msg,
+ )
+ await self._storage.update_session_state(
+ user_id=user_id,
+ agent_id=agent_id,
+ session_id=session_id,
+ state=agent.state,
)
- await self._storage.update_session_state(
- user_id=user_id,
- agent_id=agent_id,
- session_id=session_id,
- state=agent.state,
- )
- await self._message_bus.log_trim(events_key)
+ await self._message_bus.log_trim(events_key)
+ finally:
+ # A worker whose turn died never reached
+ # ``TeamSay``. Sent from inside the shielded
+ # write so an interrupt cannot drop it, and from
+ # a ``finally`` so a storage failure cannot
+ # either — the leader has to learn about the
+ # dead turn even when recording it went wrong.
+ # COMPLETED / EXCEED_MAX_ITERS stay silent:
+ # those only escape the member middleware after
+ # a successful report.
+ for msg in reply_msgs:
+ if msg.finished_reason in (
+ ReplyFinishedReason.ERROR,
+ ReplyFinishedReason.INTERRUPTED,
+ ):
+ await self._notify_leader_of_failure(
+ user_id,
+ team_ctx,
+ worker_name,
+ msg.finished_reason,
+ msg.error.message
+ if msg.error
+ else "The turn stopped before it "
+ "finished.",
+ )
persist_task = asyncio.create_task(_persist())
try:
diff --git a/tests/service_team_failure_report_test.py b/tests/service_team_failure_report_test.py
new file mode 100644
index 000000000..d12cff67d
--- /dev/null
+++ b/tests/service_team_failure_report_test.py
@@ -0,0 +1,426 @@
+# -*- coding: utf-8 -*-
+# pylint: disable=protected-access, using-constant-test
+"""A team member's failed turn is reported to its leader.
+
+A member reports through ``TeamSay``; a turn that errored, was
+interrupted, or never assembled never got there, so the chat service
+delivers the news into the leader's inbox instead of leaving it
+waiting.
+"""
+import json
+from types import SimpleNamespace
+from typing import AsyncGenerator
+from unittest import IsolatedAsyncioTestCase
+from unittest.mock import patch
+
+from utils import AnyString
+
+from agentscope.agent import ContextConfig, ReActConfig
+from agentscope.app._service import ChatService
+from agentscope.app.message_bus import InMemoryMessageBus, MessageBusKeys
+from agentscope.app.storage import (
+ AgentData,
+ AgentRecord,
+ ChatModelConfig,
+ SessionConfig,
+ SessionRecord,
+ TeamData,
+ TeamRecord,
+)
+from agentscope.event import ReplyEndEvent, ReplyStartEvent
+from agentscope.types import ErrorInfo, ErrorType, ReplyFinishedReason
+
+_USER = "user-1"
+
+
+class _Storage:
+ """Serve the team's records and swallow every write."""
+
+ def __init__(
+ self,
+ sessions: dict[str, SessionRecord],
+ agents: dict[str, AgentRecord],
+ team: TeamRecord,
+ ) -> None:
+ self.sessions = sessions
+ self.agents = agents
+ self.team = team
+ self.fail_writes = False
+
+ async def get_session(
+ self,
+ user_id: str,
+ agent_id: str,
+ session_id: str,
+ ) -> SessionRecord | None:
+ """Return a detached copy of the requested session."""
+ del user_id, agent_id
+ record = self.sessions.get(session_id)
+ return record.model_copy(deep=True) if record else None
+
+ async def get_agent(
+ self,
+ user_id: str,
+ agent_id: str,
+ ) -> AgentRecord | None:
+ """Return a detached copy of the requested agent."""
+ del user_id
+ record = self.agents.get(agent_id)
+ return record.model_copy(deep=True) if record else None
+
+ async def get_team(self, user_id: str, team_id: str) -> TeamRecord | None:
+ """Return the one team."""
+ del user_id
+ return self.team if team_id == self.team.id else None
+
+ async def update_session_state(self, *_: object, **__: object) -> None:
+ """Accept the post-run state persistence, or fail it on demand."""
+ if self.fail_writes:
+ raise RuntimeError("storage is down")
+
+ async def upsert_message(self, *_: object, **__: object) -> None:
+ """Accept persisted replies, or fail them on demand."""
+ if self.fail_writes:
+ raise RuntimeError("storage is down")
+
+
+class _WorkspaceManager:
+ """Return a minimal workspace handle."""
+
+ async def get_workspace(self, *_: object, **__: object) -> object:
+ """Return an inert workspace."""
+ return SimpleNamespace(workdir="/tmp/agentscope-team-failure-test")
+
+
+class _FailingWorkspaceManager:
+ """A backend that is down — assembly dies before the agent exists."""
+
+ async def get_workspace(self, *_: object, **__: object) -> object:
+ """Fail the way an unreachable sandbox does."""
+ raise RuntimeError("workspace backend unreachable")
+
+
+def _agent(agent_id: str, name: str, source: str = "user") -> AgentRecord:
+ """Build a minimal agent record."""
+ return AgentRecord(
+ id=agent_id,
+ user_id=_USER,
+ source=source,
+ data=AgentData(
+ name=name,
+ context_config=ContextConfig(),
+ react_config=ReActConfig(),
+ ),
+ )
+
+
+def _session(session_id: str, agent_id: str) -> SessionRecord:
+ """Build a team-bound session record."""
+ return SessionRecord(
+ id=session_id,
+ user_id=_USER,
+ agent_id=agent_id,
+ team_id="team-1",
+ config=SessionConfig(
+ workspace_id="ws-1",
+ chat_model_config=ChatModelConfig(
+ type="test",
+ credential_id="cred-1",
+ model="m",
+ parameters={},
+ ),
+ ),
+ )
+
+
+def _agent_cls(events: list) -> type:
+ """An Agent stand-in replaying a fixed event sequence."""
+
+ class _Agent:
+ """Replay ``events``; the state is only read back for persistence."""
+
+ def __init__(self, *, state: object = None, **_: object) -> None:
+ self.state = state or SimpleNamespace()
+
+ async def reply_stream(
+ self,
+ inputs: object,
+ ) -> AsyncGenerator[object, None]:
+ """Yield the configured events."""
+ del inputs
+ for event in events:
+ yield event
+
+ return _Agent
+
+
+class TeamFailureReportTest(IsolatedAsyncioTestCase):
+ """Which endings reach the leader's inbox, and which do not."""
+
+ async def asyncSetUp(self) -> None:
+ """Wire a two-member team: one leader, one worker."""
+ self.leader_agent = _agent("agent-l", "Leader")
+ self.worker_agent = _agent("agent-w", "worker", source="team")
+ self.leader_session = _session("session-l", self.leader_agent.id)
+ self.worker_session = _session("session-w", self.worker_agent.id)
+ self.team = TeamRecord(
+ id="team-1",
+ user_id=_USER,
+ session_id=self.leader_session.id,
+ leader_agent_id=self.leader_agent.id,
+ data=TeamData(name="team"),
+ )
+ self.storage = _Storage(
+ sessions={
+ s.id: s for s in (self.leader_session, self.worker_session)
+ },
+ agents={a.id: a for a in (self.leader_agent, self.worker_agent)},
+ team=self.team,
+ )
+ self.bus = InMemoryMessageBus()
+ self.workspace_manager: object = _WorkspaceManager()
+
+ async def _run(
+ self,
+ session: SessionRecord,
+ agent_id: str,
+ events: list,
+ *,
+ model_fails: bool = False,
+ workspace_fails: bool = False,
+ persist_fails: bool = False,
+ ) -> list[dict]:
+ """Run one turn and return whatever landed in the leader's inbox."""
+
+ async def _get_toolkit(**_: object) -> object:
+ return object()
+
+ async def _get_model(*_: object, **__: object) -> object:
+ if model_fails:
+ raise RuntimeError("no credential for the worker")
+ return object()
+
+ if workspace_fails:
+ self.workspace_manager = _FailingWorkspaceManager()
+ if persist_fails:
+ self.storage.fail_writes = True
+
+ class _Access:
+ """Resolve any agent of this user."""
+
+ def __init__(self, storage: _Storage) -> None:
+ self._storage = storage
+
+ async def resolve_agent(
+ self,
+ user_id: str,
+ agent_id: str,
+ ) -> AgentRecord:
+ """Return the requested agent record."""
+ return await self._storage.get_agent(user_id, agent_id)
+
+ service = ChatService(
+ storage=self.storage,
+ workspace_manager=self.workspace_manager,
+ scheduler_manager=object(),
+ background_task_manager=object(),
+ message_bus=self.bus,
+ resource_access_service=_Access(self.storage),
+ custom_agent_cls=_agent_cls(events),
+ )
+ with (
+ patch(
+ "agentscope.app._service._chat.get_toolkit",
+ new=_get_toolkit,
+ ),
+ patch("agentscope.app._service._chat.get_model", new=_get_model),
+ ):
+ await service._run_impl(_USER, session.id, agent_id, None)
+
+ entries = await self.bus.queue_drain(
+ MessageBusKeys.inbox(self.leader_session.id),
+ )
+ return [payload for _entry_id, payload in entries]
+
+ def _events(
+ self,
+ reason: ReplyFinishedReason,
+ error: ErrorInfo | None = None,
+ ) -> list:
+ """A start/end pair ending with ``reason``."""
+ return [
+ ReplyStartEvent(
+ session_id=self.worker_session.id,
+ reply_id="reply-1",
+ name="worker",
+ ),
+ ReplyEndEvent(
+ session_id=self.worker_session.id,
+ reply_id="reply-1",
+ finished_reason=reason,
+ error=error,
+ ),
+ ]
+
+ async def test_worker_error_reaches_the_leader(self) -> None:
+ """An errored worker turn lands as a hint in the leader's inbox."""
+ delivered = await self._run(
+ self.worker_session,
+ self.worker_agent.id,
+ self._events(
+ ReplyFinishedReason.ERROR,
+ ErrorInfo(type=ErrorType.INTERNAL, message="model exploded"),
+ ),
+ )
+ self.assertEqual(len(delivered), 1)
+ self.assertDictEqual(
+ delivered[0],
+ {
+ "type": "hint",
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": AnyString(),
+ "hint": (
+ "<system-reminder>Team member 'worker' hit an error "
+ "while running, so it never called TeamSay to "
+ "report. Error: model exploded. Judge from the error "
+ "type whether to retry — with this member or a fresh "
+ "one — or to raise it with the user: invalid "
+ "credentials or an exhausted quota will fail the "
+ "same way again. Assume no usable output unless the "
+ "member reported partial results earlier."
+ "</system-reminder>"
+ ),
+ "source": json.dumps(
+ {"label": "System", "sublabel": "Reminder"},
+ ensure_ascii=False,
+ ),
+ },
+ )
+
+ async def test_worker_interruption_reaches_the_leader(self) -> None:
+ """An interrupted worker turn is reported too."""
+ delivered = await self._run(
+ self.worker_session,
+ self.worker_agent.id,
+ self._events(ReplyFinishedReason.INTERRUPTED),
+ )
+ self.assertEqual(len(delivered), 1)
+ self.assertDictEqual(
+ delivered[0],
+ {
+ "type": "hint",
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": AnyString(),
+ "hint": (
+ "<system-reminder>Team member 'worker' was "
+ "interrupted mid-task and has stopped. Someone "
+ "cancelled that run deliberately — possibly the "
+ "user, who may be working with this member directly. "
+ "Do not silently re-dispatch it: ask the user what "
+ "should happen to the task, or ask the member how "
+ "far it got. Left unresolved, the member is stranded "
+ "with work nobody is tracking.</system-reminder>"
+ ),
+ "source": json.dumps(
+ {"label": "System", "sublabel": "Reminder"},
+ ensure_ascii=False,
+ ),
+ },
+ )
+
+ async def test_completed_worker_turn_is_silent(self) -> None:
+ """A COMPLETED ending already implies a successful TeamSay."""
+ delivered = await self._run(
+ self.worker_session,
+ self.worker_agent.id,
+ self._events(ReplyFinishedReason.COMPLETED),
+ )
+ self.assertListEqual(delivered, [])
+
+ async def test_leader_failure_notifies_nobody(self) -> None:
+ """The leader's own failed turn has no one to report to."""
+ delivered = await self._run(
+ self.leader_session,
+ self.leader_agent.id,
+ self._events(
+ ReplyFinishedReason.ERROR,
+ ErrorInfo(type=ErrorType.INTERNAL, message="boom"),
+ ),
+ )
+ self.assertListEqual(delivered, [])
+
+ async def test_assembly_failure_reaches_the_leader(self) -> None:
+ """A worker that never assembled is reported as well."""
+ delivered = await self._run(
+ self.worker_session,
+ self.worker_agent.id,
+ [],
+ model_fails=True,
+ )
+ self.assertEqual(len(delivered), 1)
+ self.assertDictEqual(
+ delivered[0],
+ {
+ "type": "hint",
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": AnyString(),
+ "hint": (
+ "<system-reminder>Team member 'worker' hit an error "
+ "while running, so it never called TeamSay to "
+ "report. Error: The session could not be prepared — "
+ "check the agent's model, tools and knowledge bases. "
+ "Judge from the error type whether to retry — with "
+ "this member or a fresh one — or to raise it with "
+ "the user: invalid credentials or an exhausted quota "
+ "will fail the same way again. Assume no usable "
+ "output unless the member reported partial results "
+ "earlier.</system-reminder>"
+ ),
+ "source": json.dumps(
+ {"label": "System", "sublabel": "Reminder"},
+ ensure_ascii=False,
+ ),
+ },
+ )
+
+ async def test_workspace_failure_still_reaches_the_leader(self) -> None:
+ """Assembly dying before the workspace resolves still reports.
+
+ The team identity has to be resolved ahead of every fallible
+ assembly step, or a worker whose sandbox is down goes silent.
+ """
+ delivered = await self._run(
+ self.worker_session,
+ self.worker_agent.id,
+ [],
+ workspace_fails=True,
+ )
+ self.assertEqual(len(delivered), 1)
+ self.assertIn(
+ "Team member 'worker' hit an error while running",
+ delivered[0]["hint"],
+ )
+
+ async def test_persistence_failure_still_reaches_the_leader(self) -> None:
+ """A storage failure must not swallow the notification."""
+ with self.assertRaises(RuntimeError):
+ await self._run(
+ self.worker_session,
+ self.worker_agent.id,
+ self._events(
+ ReplyFinishedReason.ERROR,
+ ErrorInfo(type=ErrorType.INTERNAL, message="boom"),
+ ),
+ persist_fails=True,
+ )
+ entries = await self.bus.queue_drain(
+ MessageBusKeys.inbox(self.leader_session.id),
+ )
+ self.assertEqual(len(entries), 1)
+ self.assertIn(
+ "Error: boom.",
+ entries[0][1]["hint"],
+ )