Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
feat(agent-team): tell the leader when a member's turn fails (#2386)
DavdGao13749ff66329
diff --git a/src/agentscope/app/_service/_chat.py b/src/agentscope/app/_service/_chat.pyindex 310fb2322..f769aa533 100644--- a/src/agentscope/app/_service/_chat.py+++ b/src/agentscope/app/_service/_chat.py@@ -13,6 +13,7 @@ that wants them subscribes through the"""import asyncioimport inspect+import jsonfrom dataclasses import dataclassfrom typing import Literal, TYPE_CHECKING@@ -20,6 +21,7 @@ from fastapi import HTTPExceptionfrom .._bus_ops import (abandon_inbox_consumer,+ deliver_to_inbox,enqueue_channel_output,enqueue_run_trigger,has_pending_inbox_or_release,@@ -67,7 +69,7 @@ from ...event import ()from ._errors import _classify_error, _classify_setup_errorfrom ..._utils._common import _generate_id-from ...message import AssistantMsg, Msg, ToolCallState+from ...message import AssistantMsg, HintBlock, Msg, ToolCallStatefrom ...permission import AdditionalWorkingDirectoryif TYPE_CHECKING:@@ -85,6 +87,8 @@ class _LeaderContext:class _WorkerContext:"""This session is a worker; its leader is fully resolved."""+ leader_session_id: str+ leader_agent_id: strleader_name: strrole: Literal["worker"] = "worker"@@ -336,12 +340,102 @@ optional):end_event.error.type if end_event.error else "error",)+ async def _notify_leader_of_failure(+ self,+ user_id: str,+ team_ctx: "_TeamContext | None",+ worker_name: str,+ finished_reason: ReplyFinishedReason,+ detail: str,+ ) -> None:+ """Tell a worker's team leader that the worker stopped early.++ A member reports through ``TeamSay``; a member that failed or+ was interrupted never got there, so without this the leader+ waits forever for an answer that is not coming. Delivered as a+ system reminder in the leader's inbox — not as a team message,+ because the member did not say this, the framework did — waking+ the leader when it is idle.++ Best-effort: a failure to notify is logged and dropped rather+ than replacing the failure being reported.++ Args:+ user_id (`str`):+ The owner of both sessions.+ team_ctx (`_TeamContext | None`):+ The run's team identity. Anything but a+ :class:`_WorkerContext` is a no-op — a leader or a+ team-less session has nobody to report to.+ worker_name (`str`):+ The failing member's display name, as the leader knows+ it from the roster.+ finished_reason (`ReplyFinishedReason`):+ How the reply ended, which decides what the leader is+ asked to consider.+ detail (`str`):+ One-line, already-sanitized explanation of the error.+ """+ if not isinstance(team_ctx, _WorkerContext):+ return++ # Error messages come from several sources and only some end in+ # punctuation; the reminder reads as prose either way.+ detail = detail.strip()+ if detail and detail[-1] not in ".!?":+ detail += "."++ if finished_reason == ReplyFinishedReason.INTERRUPTED:+ reminder = (+ f"Team member {worker_name!r} was interrupted mid-task and "+ f"has stopped. Someone cancelled that run deliberately — "+ f"possibly the user, who may be working with this member "+ f"directly. Do not silently re-dispatch it: ask the user "+ f"what should happen to the task, or ask the member how "+ f"far it got. Left unresolved, the member is stranded with "+ f"work nobody is tracking."+ )+ else:+ reminder = (+ f"Team member {worker_name!r} hit an error while running, "+ f"so it never called TeamSay to report. Error: {detail} "+ f"Judge from the error type whether to retry — with this "+ f"member or a fresh one — or to raise it with the user: "+ f"invalid credentials or an exhausted quota will fail the "+ f"same way again. Assume no usable output unless the "+ f"member reported partial results earlier."+ )++ try:+ hint = HintBlock(+ hint=f"<system-reminder>{reminder}</system-reminder>",+ source=json.dumps(+ {"label": "System", "sublabel": "Reminder"},+ ensure_ascii=False,+ ),+ )+ await deliver_to_inbox(+ self._message_bus,+ user_id=user_id,+ session_id=team_ctx.leader_session_id,+ agent_id=team_ctx.leader_agent_id,+ payload=hint.model_dump(mode="json"),+ )+ except Exception: # pylint: disable=broad-except+ logger.exception(+ "Failed to notify team leader %r that member %r stopped.",+ team_ctx.leader_name,+ worker_name,+ )+async def _report_failure(self,user_id: str,session_id: str,agent_id: str,error: Exception,+ team_ctx: "_TeamContext | None" = None,+ worker_name: str | None = None,) -> None:"""Tell the client about a failure that reached no reply.@@ -370,6 +464,12 @@ optional):The agent that was being assembled.error (`Exception`):What went wrong while setting the run up.+ team_ctx (`_TeamContext | None`, optional):+ The run's team identity; a worker's leader is told the+ run never happened.+ worker_name (`str | None`, optional):+ Display name used in that notification. Defaults to+ ``agent_id`` when the agent record never loaded."""try:reply_id = _generate_id()@@ -409,6 +509,14 @@ optional):session_id,)+ await self._notify_leader_of_failure(+ user_id,+ team_ctx,+ worker_name or agent_id,+ ReplyFinishedReason.ERROR,+ _classify_setup_error(error).message,+ )+async def interrupt(self,user_id: str,@@ -550,6 +658,11 @@ optional):MessageBusKeys.session_lock(session_id),ttl_secs=MessageBusKeys.SESSION_RUN_TTL_SECS,):+ # Bound before the try: an assembly failure reports through+ # them, and may happen before either is resolved.+ team_ctx: _TeamContext | None = None+ worker_name: str = agent_id+# Steps 1-6 assemble the run; step 7 performs it. A failure# here has no reply to attach to, so one is synthesized —# otherwise the client sees a stream that simply stops and is@@ -588,30 +701,13 @@ optional):f"agent {agent_id!r}."),)- workspace = await self._workspace_manager.get_workspace(- user_id,- agent_id,- session_id,- session_record.config.workspace_id,- )-- # Add workspace working directory to the permission context- working_dirs = (- session_record.state.permission_context.working_directories- )- if workspace.workdir not in working_dirs:- working_dirs[- workspace.workdir- ] = AdditionalWorkingDirectory(- path=workspace.workdir,- source="session",- )+ worker_name = agent_record.data.name# -------------------------------------------------------------- # 1b. Resolve team identity + channel binding ONCE; all- # downstream consumers reuse these instead of re-fetching.+ # 1b. Resolve the team identity ONCE, before anything that+ # can fail: a worker whose assembly dies still has to reach+ # its leader. Downstream consumers reuse this.# -------------------------------------------------------------- team_ctx: _TeamContext | None = Noneif session_record.team_id is not None:team = await self._storage.get_team(user_id,@@ -645,8 +741,35 @@ optional):f"{session_id!r} cannot run."),)- team_ctx = _WorkerContext(leader_name=leader.name)+ team_ctx = _WorkerContext(+ leader_session_id=leader.session_id,+ leader_agent_id=leader.agent.id,+ leader_name=leader.name,+ )++ workspace = await self._workspace_manager.get_workspace(+ user_id,+ agent_id,+ session_id,+ session_record.config.workspace_id,+ )++ # Add workspace working directory to the permission context+ working_dirs = (+ session_record.state.permission_context.working_directories+ )+ if workspace.workdir not in working_dirs:+ working_dirs[+ workspace.workdir+ ] = AdditionalWorkingDirectory(+ path=workspace.workdir,+ source="session",+ )+ # -------------------------------------------------------------+ # 1c. Resolve the channel binding ONCE; the toolkit and the+ # system-prompt attachment share it.+ # -------------------------------------------------------------channel = (self._channel_dispatcher.get_local_channel(session_record.source_channel_id,@@ -874,7 +997,14 @@ optional):# to make: these events share a channel with a live reply's,# so publishing them unserialised would drop a "reply failed"# into the middle of an answer another run is streaming.- await self._report_failure(user_id, session_id, agent_id, e)+ await self._report_failure(+ user_id,+ session_id,+ agent_id,+ e,+ team_ctx,+ worker_name,+ )return# ----------------------------------------------------------------@@ -1060,6 +1190,8 @@ optional):session_id,agent_id,e,+ team_ctx,+ worker_name,)else:await self._close_failed_reply(@@ -1119,19 +1251,45 @@ optional):# acquire the lock and load a stale state from storage# before this write lands.async def _persist() -> None:- for msg in reply_msgs:- await self._storage.upsert_message(- user_id,- session_id,- msg,+ try:+ for msg in reply_msgs:+ await self._storage.upsert_message(+ user_id,+ session_id,+ msg,+ )+ await self._storage.update_session_state(+ user_id=user_id,+ agent_id=agent_id,+ session_id=session_id,+ state=agent.state,)- await self._storage.update_session_state(- user_id=user_id,- agent_id=agent_id,- session_id=session_id,- state=agent.state,- )- await self._message_bus.log_trim(events_key)+ await self._message_bus.log_trim(events_key)+ finally:+ # A worker whose turn died never reached+ # ``TeamSay``. Sent from inside the shielded+ # write so an interrupt cannot drop it, and from+ # a ``finally`` so a storage failure cannot+ # either — the leader has to learn about the+ # dead turn even when recording it went wrong.+ # COMPLETED / EXCEED_MAX_ITERS stay silent:+ # those only escape the member middleware after+ # a successful report.+ for msg in reply_msgs:+ if msg.finished_reason in (+ ReplyFinishedReason.ERROR,+ ReplyFinishedReason.INTERRUPTED,+ ):+ await self._notify_leader_of_failure(+ user_id,+ team_ctx,+ worker_name,+ msg.finished_reason,+ msg.error.message+ if msg.error+ else "The turn stopped before it "+ "finished.",+ )persist_task = asyncio.create_task(_persist())try:diff --git a/tests/service_team_failure_report_test.py b/tests/service_team_failure_report_test.pynew file mode 100644index 000000000..d12cff67d--- /dev/null+++ b/tests/service_team_failure_report_test.py@@ -0,0 +1,426 @@+# -*- coding: utf-8 -*-+# pylint: disable=protected-access, using-constant-test+"""A team member's failed turn is reported to its leader.++A member reports through ``TeamSay``; a turn that errored, was+interrupted, or never assembled never got there, so the chat service+delivers the news into the leader's inbox instead of leaving it+waiting.+"""+import json+from types import SimpleNamespace+from typing import AsyncGenerator+from unittest import IsolatedAsyncioTestCase+from unittest.mock import patch++from utils import AnyString++from agentscope.agent import ContextConfig, ReActConfig+from agentscope.app._service import ChatService+from agentscope.app.message_bus import InMemoryMessageBus, MessageBusKeys+from agentscope.app.storage import (+ AgentData,+ AgentRecord,+ ChatModelConfig,+ SessionConfig,+ SessionRecord,+ TeamData,+ TeamRecord,+)+from agentscope.event import ReplyEndEvent, ReplyStartEvent+from agentscope.types import ErrorInfo, ErrorType, ReplyFinishedReason++_USER = "user-1"+++class _Storage:+ """Serve the team's records and swallow every write."""++ def __init__(+ self,+ sessions: dict[str, SessionRecord],+ agents: dict[str, AgentRecord],+ team: TeamRecord,+ ) -> None:+ self.sessions = sessions+ self.agents = agents+ self.team = team+ self.fail_writes = False++ async def get_session(+ self,+ user_id: str,+ agent_id: str,+ session_id: str,+ ) -> SessionRecord | None:+ """Return a detached copy of the requested session."""+ del user_id, agent_id+ record = self.sessions.get(session_id)+ return record.model_copy(deep=True) if record else None++ async def get_agent(+ self,+ user_id: str,+ agent_id: str,+ ) -> AgentRecord | None:+ """Return a detached copy of the requested agent."""+ del user_id+ record = self.agents.get(agent_id)+ return record.model_copy(deep=True) if record else None++ async def get_team(self, user_id: str, team_id: str) -> TeamRecord | None:+ """Return the one team."""+ del user_id+ return self.team if team_id == self.team.id else None++ async def update_session_state(self, *_: object, **__: object) -> None:+ """Accept the post-run state persistence, or fail it on demand."""+ if self.fail_writes:+ raise RuntimeError("storage is down")++ async def upsert_message(self, *_: object, **__: object) -> None:+ """Accept persisted replies, or fail them on demand."""+ if self.fail_writes:+ raise RuntimeError("storage is down")+++class _WorkspaceManager:+ """Return a minimal workspace handle."""++ async def get_workspace(self, *_: object, **__: object) -> object:+ """Return an inert workspace."""+ return SimpleNamespace(workdir="/tmp/agentscope-team-failure-test")+++class _FailingWorkspaceManager:+ """A backend that is down — assembly dies before the agent exists."""++ async def get_workspace(self, *_: object, **__: object) -> object:+ """Fail the way an unreachable sandbox does."""+ raise RuntimeError("workspace backend unreachable")+++def _agent(agent_id: str, name: str, source: str = "user") -> AgentRecord:+ """Build a minimal agent record."""+ return AgentRecord(+ id=agent_id,+ user_id=_USER,+ source=source,+ data=AgentData(+ name=name,+ context_config=ContextConfig(),+ react_config=ReActConfig(),+ ),+ )+++def _session(session_id: str, agent_id: str) -> SessionRecord:+ """Build a team-bound session record."""+ return SessionRecord(+ id=session_id,+ user_id=_USER,+ agent_id=agent_id,+ team_id="team-1",+ config=SessionConfig(+ workspace_id="ws-1",+ chat_model_config=ChatModelConfig(+ type="test",+ credential_id="cred-1",+ model="m",+ parameters={},+ ),+ ),+ )+++def _agent_cls(events: list) -> type:+ """An Agent stand-in replaying a fixed event sequence."""++ class _Agent:+ """Replay ``events``; the state is only read back for persistence."""++ def __init__(self, *, state: object = None, **_: object) -> None:+ self.state = state or SimpleNamespace()++ async def reply_stream(+ self,+ inputs: object,+ ) -> AsyncGenerator[object, None]:+ """Yield the configured events."""+ del inputs+ for event in events:+ yield event++ return _Agent+++class TeamFailureReportTest(IsolatedAsyncioTestCase):+ """Which endings reach the leader's inbox, and which do not."""++ async def asyncSetUp(self) -> None:+ """Wire a two-member team: one leader, one worker."""+ self.leader_agent = _agent("agent-l", "Leader")+ self.worker_agent = _agent("agent-w", "worker", source="team")+ self.leader_session = _session("session-l", self.leader_agent.id)+ self.worker_session = _session("session-w", self.worker_agent.id)+ self.team = TeamRecord(+ id="team-1",+ user_id=_USER,+ session_id=self.leader_session.id,+ leader_agent_id=self.leader_agent.id,+ data=TeamData(name="team"),+ )+ self.storage = _Storage(+ sessions={+ s.id: s for s in (self.leader_session, self.worker_session)+ },+ agents={a.id: a for a in (self.leader_agent, self.worker_agent)},+ team=self.team,+ )+ self.bus = InMemoryMessageBus()+ self.workspace_manager: object = _WorkspaceManager()++ async def _run(+ self,+ session: SessionRecord,+ agent_id: str,+ events: list,+ *,+ model_fails: bool = False,+ workspace_fails: bool = False,+ persist_fails: bool = False,+ ) -> list[dict]:+ """Run one turn and return whatever landed in the leader's inbox."""++ async def _get_toolkit(**_: object) -> object:+ return object()++ async def _get_model(*_: object, **__: object) -> object:+ if model_fails:+ raise RuntimeError("no credential for the worker")+ return object()++ if workspace_fails:+ self.workspace_manager = _FailingWorkspaceManager()+ if persist_fails:+ self.storage.fail_writes = True++ class _Access:+ """Resolve any agent of this user."""++ def __init__(self, storage: _Storage) -> None:+ self._storage = storage++ async def resolve_agent(+ self,+ user_id: str,+ agent_id: str,+ ) -> AgentRecord:+ """Return the requested agent record."""+ return await self._storage.get_agent(user_id, agent_id)++ service = ChatService(+ storage=self.storage,+ workspace_manager=self.workspace_manager,+ scheduler_manager=object(),+ background_task_manager=object(),+ message_bus=self.bus,+ resource_access_service=_Access(self.storage),+ custom_agent_cls=_agent_cls(events),+ )+ with (+ patch(+ "agentscope.app._service._chat.get_toolkit",+ new=_get_toolkit,+ ),+ patch("agentscope.app._service._chat.get_model", new=_get_model),+ ):+ await service._run_impl(_USER, session.id, agent_id, None)++ entries = await self.bus.queue_drain(+ MessageBusKeys.inbox(self.leader_session.id),+ )+ return [payload for _entry_id, payload in entries]++ def _events(+ self,+ reason: ReplyFinishedReason,+ error: ErrorInfo | None = None,+ ) -> list:+ """A start/end pair ending with ``reason``."""+ return [+ ReplyStartEvent(+ session_id=self.worker_session.id,+ reply_id="reply-1",+ name="worker",+ ),+ ReplyEndEvent(+ session_id=self.worker_session.id,+ reply_id="reply-1",+ finished_reason=reason,+ error=error,+ ),+ ]++ async def test_worker_error_reaches_the_leader(self) -> None:+ """An errored worker turn lands as a hint in the leader's inbox."""+ delivered = await self._run(+ self.worker_session,+ self.worker_agent.id,+ self._events(+ ReplyFinishedReason.ERROR,+ ErrorInfo(type=ErrorType.INTERNAL, message="model exploded"),+ ),+ )+ self.assertEqual(len(delivered), 1)+ self.assertDictEqual(+ delivered[0],+ {+ "type": "hint",+ "id": AnyString(),+ "created_at": AnyString(),+ "finished_at": AnyString(),+ "hint": (+ "<system-reminder>Team member 'worker' hit an error "+ "while running, so it never called TeamSay to "+ "report. Error: model exploded. Judge from the error "+ "type whether to retry — with this member or a fresh "+ "one — or to raise it with the user: invalid "+ "credentials or an exhausted quota will fail the "+ "same way again. Assume no usable output unless the "+ "member reported partial results earlier."+ "</system-reminder>"+ ),+ "source": json.dumps(+ {"label": "System", "sublabel": "Reminder"},+ ensure_ascii=False,+ ),+ },+ )++ async def test_worker_interruption_reaches_the_leader(self) -> None:+ """An interrupted worker turn is reported too."""+ delivered = await self._run(+ self.worker_session,+ self.worker_agent.id,+ self._events(ReplyFinishedReason.INTERRUPTED),+ )+ self.assertEqual(len(delivered), 1)+ self.assertDictEqual(+ delivered[0],+ {+ "type": "hint",+ "id": AnyString(),+ "created_at": AnyString(),+ "finished_at": AnyString(),+ "hint": (+ "<system-reminder>Team member 'worker' was "+ "interrupted mid-task and has stopped. Someone "+ "cancelled that run deliberately — possibly the "+ "user, who may be working with this member directly. "+ "Do not silently re-dispatch it: ask the user what "+ "should happen to the task, or ask the member how "+ "far it got. Left unresolved, the member is stranded "+ "with work nobody is tracking.</system-reminder>"+ ),+ "source": json.dumps(+ {"label": "System", "sublabel": "Reminder"},+ ensure_ascii=False,+ ),+ },+ )++ async def test_completed_worker_turn_is_silent(self) -> None:+ """A COMPLETED ending already implies a successful TeamSay."""+ delivered = await self._run(+ self.worker_session,+ self.worker_agent.id,+ self._events(ReplyFinishedReason.COMPLETED),+ )+ self.assertListEqual(delivered, [])++ async def test_leader_failure_notifies_nobody(self) -> None:+ """The leader's own failed turn has no one to report to."""+ delivered = await self._run(+ self.leader_session,+ self.leader_agent.id,+ self._events(+ ReplyFinishedReason.ERROR,+ ErrorInfo(type=ErrorType.INTERNAL, message="boom"),+ ),+ )+ self.assertListEqual(delivered, [])++ async def test_assembly_failure_reaches_the_leader(self) -> None:+ """A worker that never assembled is reported as well."""+ delivered = await self._run(+ self.worker_session,+ self.worker_agent.id,+ [],+ model_fails=True,+ )+ self.assertEqual(len(delivered), 1)+ self.assertDictEqual(+ delivered[0],+ {+ "type": "hint",+ "id": AnyString(),+ "created_at": AnyString(),+ "finished_at": AnyString(),+ "hint": (+ "<system-reminder>Team member 'worker' hit an error "+ "while running, so it never called TeamSay to "+ "report. Error: The session could not be prepared — "+ "check the agent's model, tools and knowledge bases. "+ "Judge from the error type whether to retry — with "+ "this member or a fresh one — or to raise it with "+ "the user: invalid credentials or an exhausted quota "+ "will fail the same way again. Assume no usable "+ "output unless the member reported partial results "+ "earlier.</system-reminder>"+ ),+ "source": json.dumps(+ {"label": "System", "sublabel": "Reminder"},+ ensure_ascii=False,+ ),+ },+ )++ async def test_workspace_failure_still_reaches_the_leader(self) -> None:+ """Assembly dying before the workspace resolves still reports.++ The team identity has to be resolved ahead of every fallible+ assembly step, or a worker whose sandbox is down goes silent.+ """+ delivered = await self._run(+ self.worker_session,+ self.worker_agent.id,+ [],+ workspace_fails=True,+ )+ self.assertEqual(len(delivered), 1)+ self.assertIn(+ "Team member 'worker' hit an error while running",+ delivered[0]["hint"],+ )++ async def test_persistence_failure_still_reaches_the_leader(self) -> None:+ """A storage failure must not swallow the notification."""+ with self.assertRaises(RuntimeError):+ await self._run(+ self.worker_session,+ self.worker_agent.id,+ self._events(+ ReplyFinishedReason.ERROR,+ ErrorInfo(type=ErrorType.INTERNAL, message="boom"),+ ),+ persist_fails=True,+ )+ entries = await self.bus.queue_drain(+ MessageBusKeys.inbox(self.leader_session.id),+ )+ self.assertEqual(len(entries), 1)+ self.assertIn(+ "Error: boom.",+ entries[0][1]["hint"],+ )