Skip to content

Build and run agents you can see, understand and trust.

feat(sop): add standard operating procedures module in agentscope (#2393)

DavdGaobb47edfb1968
diff --git a/src/agentscope/sop/__init__.py b/src/agentscope/sop/__init__.py
new file mode 100644
index 000000000..95179a32d
--- /dev/null
+++ b/src/agentscope/sop/__init__.py
@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+"""Standard operating procedures — a fixed sequence of milestones.
+
+A :class:`SOP` says which steps there are and what each must prove; how
+a step gets there is its own business. :class:`SOPEngine` walks them,
+routes answers to whichever one parked, and records verdicts in a
+:class:`SOPRunState` that outlives the process.
+
+A definition holds no run state, so one can drive any number of runs.
+"""
+from ._engine import SOPEngine
+from ._schema import AgentLike, SOP, SOPStep, SOPStepBase
+from ._state import (
+ SOPPhase,
+ SOPRunState,
+ SOPStepRunState,
+ VerificationResult,
+)
+
+__all__ = [
+ # definition
+ "SOP",
+ "SOPStep",
+ "SOPStepBase",
+ "AgentLike",
+ # run state
+ "SOPRunState",
+ "SOPStepRunState",
+ "SOPPhase",
+ "VerificationResult",
+ # running it
+ "SOPEngine",
+]
diff --git a/src/agentscope/sop/_engine.py b/src/agentscope/sop/_engine.py
new file mode 100644
index 000000000..c2116bd9d
--- /dev/null
+++ b/src/agentscope/sop/_engine.py
@@ -0,0 +1,169 @@
+# -*- coding: utf-8 -*-
+"""Running a SOP.
+
+The engine walks the steps in order, hands resumption events to
+whichever one parked, and spends the attempt budget. It decides from
+each step's :class:`~._state.SOPStepRunState` alone, never from how the
+step reached it.
+"""
+from itertools import zip_longest
+from typing import AsyncGenerator
+
+from ._schema import SOP
+from ._state import SOPPhase, SOPRunState
+from ..event import (
+ AgentEvent,
+ CustomEvent,
+ ExternalExecutionResultEvent,
+ UserConfirmResultEvent,
+ UserInterruptEvent,
+)
+from ..message import Msg, TextBlock, UserMsg
+
+
+class SOPEngine:
+ """One run of a :class:`~._schema.SOP`, shaped like an agent.
+
+ Feed it, watch the events, and when something needs a person the
+ stream simply ends — nothing stays suspended. Come back with the
+ answer and it picks up from the state.
+ """
+
+ def __init__(self, sop: SOP, state: SOPRunState | None = None) -> None:
+ """Initialize the engine.
+
+ Args:
+ sop (`SOP`):
+ The procedure to run. Never written to.
+ state (`SOPRunState | None`, optional):
+ A stored run to carry on from. Omit to start a new one.
+ Only the SOP's own state is restored: an executor that
+ keeps state of its own — an :class:`~..agent.Agent` does
+ — is restored by whoever built it, before the SOP is
+ handed here.
+
+ Raises:
+ `ValueError`:
+ If the state has a different number of steps than the
+ SOP, which means the procedure was edited since.
+ """
+ self.state = state or SOPRunState()
+ if self.state.steps and len(self.state.steps) != len(sop.steps):
+ raise ValueError(
+ f"State has {len(self.state.steps)} steps, but this SOP "
+ f"has {len(sop.steps)}.",
+ )
+ self.sop = sop
+ # A stored run was read back as the base record; a step that
+ # keeps more than that gets it back through its own type.
+ self.state.steps = [
+ step.state_type.model_validate(stored.model_dump())
+ if stored is not None
+ else step.state_type()
+ for step, stored in zip_longest(sop.steps, self.state.steps)
+ ]
+
+ @property
+ def phase(self) -> SOPPhase:
+ """Where the run stands overall."""
+ return self.state.phase
+
+ async def reply_stream(
+ self,
+ inputs: Msg
+ | list[Msg]
+ | UserConfirmResultEvent
+ | UserInterruptEvent
+ | ExternalExecutionResultEvent
+ | None = None,
+ ) -> AsyncGenerator[AgentEvent | Msg, None]:
+ """Run the procedure, streaming what happens.
+
+ Named after :meth:`~..agent.Agent.reply_stream` so a SOP goes
+ wherever an agent goes.
+
+ Args:
+ inputs:
+ What starts the run, or the answer a parked step was
+ waiting for.
+
+ Yields:
+ `AgentEvent | Msg`:
+ Everything its steps produced on the way.
+ """
+ interrupting = isinstance(inputs, UserInterruptEvent)
+ resuming = interrupting or isinstance(
+ inputs,
+ (UserConfirmResultEvent, ExternalExecutionResultEvent),
+ )
+ if not resuming and inputs is not None:
+ self.state.inputs = (
+ [inputs] if isinstance(inputs, Msg) else list(inputs)
+ )
+
+ for index, step in enumerate(self.sop.steps):
+ record = self.state.steps[index]
+ if record.phase is SOPPhase.COMPLETED:
+ continue
+ if record.phase is SOPPhase.FAILED:
+ return
+
+ while True:
+ # The answer goes to the step that parked; a fresh
+ # attempt gets what the run knows so far.
+ if not resuming:
+ record.given = self._handover(index)
+ yield CustomEvent(
+ name="SOP_STEP_STARTED",
+ value={
+ "step": step.subject,
+ "attempt": len(record.verifications) + 1,
+ },
+ )
+ async for event in step.reply_stream(
+ inputs if resuming else record.given,
+ record,
+ ):
+ yield event
+ yield CustomEvent(
+ name="SOP_STEP_ENDED",
+ value={
+ "step": step.subject,
+ "phase": record.phase.value,
+ },
+ )
+ inputs, resuming = None, False
+
+ if interrupting:
+ # The parked reply was closed; nothing is retried.
+ return
+ if record.phase is SOPPhase.AWAITING:
+ # Let go of the stream rather than hold a coroutine
+ # open; the caller comes back with an answer.
+ return
+ if record.phase is SOPPhase.COMPLETED:
+ break
+ if len(record.verifications) >= step.max_attempts:
+ record.phase = SOPPhase.FAILED
+ return
+
+ def _handover(self, index: int) -> list[Msg]:
+ """What a step is given to work from.
+
+ The run's own inputs for the first step, and what the one before
+ handed over for the rest. Nothing else crosses: a step reads its
+ predecessor's account, not its files or its conversation.
+ """
+ if index == 0:
+ return list(self.state.inputs)
+ previous = self.sop.steps[index - 1]
+ return [
+ UserMsg(
+ name="sop",
+ content=[
+ TextBlock(text=f'<handover from="{previous.subject}">'),
+ *(self.state.steps[index - 1].submission or []),
+ TextBlock(text="</handover>"),
+ ],
+ ),
+ ]
diff --git a/src/agentscope/sop/_schema.py b/src/agentscope/sop/_schema.py
new file mode 100644
index 000000000..f773d3da1
--- /dev/null
+++ b/src/agentscope/sop/_schema.py
@@ -0,0 +1,405 @@
+# -*- coding: utf-8 -*-
+"""The SOP definition — what a person writes and can read back.
+
+:class:`SOPStep` is the usual shape: an executor does the work and a
+verifier judges it. The engine never looks inside a step, so anything
+that fills in its :class:`~._state.SOPStepRunState` on time will do —
+subclass :class:`SOPStepBase` for the rest.
+"""
+from abc import ABC, abstractmethod
+from typing import AsyncGenerator, ClassVar, Protocol, Type
+
+from pydantic import BaseModel, Field
+
+from ._state import SOPPhase, SOPStepRunState, VerificationResult
+from ..event import (
+ AgentEvent,
+ ExternalExecutionResultEvent,
+ RequireExternalExecutionEvent,
+ RequireUserConfirmEvent,
+ UserConfirmResultEvent,
+ UserInterruptEvent,
+)
+from ..message import Msg, TextBlock, UserMsg
+from ..pipeline import PipelineProtocol
+from ..types import ReplyFinishedReason
+
+
+class AgentLike(PipelineProtocol, Protocol):
+ """What a step needs of whatever does its work or judges it.
+
+ :class:`~..pipeline.PipelineProtocol` plus a reply that can be asked
+ to end in structured output.
+ """
+
+ def reply_stream(
+ self,
+ inputs: Msg
+ | list[Msg]
+ | UserConfirmResultEvent
+ | UserInterruptEvent
+ | ExternalExecutionResultEvent
+ | None = None,
+ structured_schema: Type[BaseModel] | None = None,
+ yield_final_msg: bool = False,
+ ) -> AsyncGenerator[AgentEvent | Msg, None]:
+ """Reply to the given inputs and stream what happens."""
+
+
+class SOPStepBase(ABC):
+ """One milestone: a name, what it must prove, and how many tries.
+
+ Subclasses put whatever they like in :meth:`reply_stream` — the
+ contract is only that **one call is one attempt**, and that it either
+ parks or files a verdict on the state it was handed.
+ """
+
+ state_type: ClassVar[type[SOPStepRunState]] = SOPStepRunState
+ """What this step's run state looks like. Subclass
+ :class:`~._state.SOPStepRunState` and name it here when a step has
+ to remember more than the engine's three fields."""
+
+ def __init__(
+ self,
+ subject: str,
+ description: str,
+ max_attempts: int = 3,
+ ) -> None:
+ """Initialize the step.
+
+ Args:
+ subject (`str`):
+ A brief, actionable name.
+ description (`str`):
+ What this step must achieve — the destination, not the
+ route.
+ max_attempts (`int`, defaults to `3`):
+ How many refusals before the run gives up on it. Enforced
+ by the engine, not here.
+ """
+ self.subject = subject
+ self.description = description
+ self.max_attempts = max_attempts
+
+ @abstractmethod
+ def reply_stream(
+ self,
+ inputs: Msg
+ | list[Msg]
+ | UserConfirmResultEvent
+ | UserInterruptEvent
+ | ExternalExecutionResultEvent
+ | None,
+ state: SOPStepRunState,
+ ) -> AsyncGenerator[AgentEvent | Msg, None]:
+ """Make one attempt at this step, streaming what happens.
+
+ Args:
+ inputs:
+ What the engine handed over for a fresh attempt, or the
+ answer a parked attempt was waiting for.
+ state (`SOPStepRunState`):
+ This step's record in the run. Read it to pick up where
+ the last call stopped; write its phase, submission and
+ verdicts as the attempt goes.
+ """
+
+ def record(
+ self,
+ state: SOPStepRunState,
+ passed: bool,
+ message: str = "",
+ verifier: str = "",
+ ) -> None:
+ """File a verdict on the current attempt.
+
+ A refusal clears the submission, so the next attempt starts from
+ the work rather than from the judging. Whether there is a next
+ attempt is the engine's call.
+ """
+ state.verifications.append(
+ VerificationResult(
+ passed=passed,
+ message=message,
+ verifier=verifier,
+ ),
+ )
+ if passed:
+ state.phase = SOPPhase.COMPLETED
+ else:
+ state.submission = None
+ state.phase = SOPPhase.PENDING
+
+
+class _Handover(BaseModel):
+ """What an executor hands on when it is done."""
+
+ handover: str = Field(
+ description=(
+ "What you are handing to the following steps. They cannot see "
+ "your files, your tools' output or this conversation — only "
+ "this. Write it for someone who has seen none of your work."
+ ),
+ )
+
+
+class _Verdict(BaseModel):
+ """What a verifier answers."""
+
+ passed: bool = Field(
+ description="Whether the work meets what the step had to prove.",
+ )
+ message: str = Field(
+ default="",
+ description=(
+ "If it does not pass, exactly what is wrong and what to do "
+ "about it. This is handed to the executor verbatim, so name "
+ "the specific claims, files or values at fault."
+ ),
+ )
+
+
+class SOPStep(SOPStepBase):
+ """The usual shape: someone does the work, someone else judges it.
+
+ One call is one attempt — work, then judgement. Parking in either
+ half ends the call; the next one picks up where it stopped, told
+ apart by whether anything was handed over yet.
+ """
+
+ def __init__(
+ self,
+ subject: str,
+ description: str,
+ executor: AgentLike,
+ verifier: AgentLike | None = None,
+ max_attempts: int = 3,
+ ) -> None:
+ """Initialize the step.
+
+ Args:
+ subject (`str`):
+ A brief, actionable name.
+ description (`str`):
+ What this step must achieve.
+ executor (`AgentLike`):
+ Does the work. Reuse one across steps and they share its
+ context; give each its own and they do not.
+ verifier (`AgentLike | None`, optional):
+ Judges it. ``None`` accepts whatever comes back, which is
+ right for a step that only has to happen.
+ max_attempts (`int`, defaults to `3`):
+ How many refusals before the run gives up on it.
+ """
+ super().__init__(subject, description, max_attempts)
+ self.executor = executor
+ self.verifier = verifier
+
+ async def reply_stream( # pylint: disable=invalid-overridden-method
+ self,
+ inputs: Msg
+ | list[Msg]
+ | UserConfirmResultEvent
+ | UserInterruptEvent
+ | ExternalExecutionResultEvent
+ | None,
+ state: SOPStepRunState,
+ ) -> AsyncGenerator[AgentEvent | Msg, None]:
+ """Make one attempt: do the work, then have it judged."""
+ if isinstance(inputs, UserInterruptEvent):
+ # Whichever side is parked closes its reply; the attempt is
+ # abandoned rather than refused, so it costs nothing from the
+ # budget and the step waits at PENDING for a fresh start.
+ parked = (
+ self.executor if state.submission is None else self.verifier
+ )
+ async for event in parked.reply_stream(inputs=inputs):
+ yield event
+ state.submission = None
+ state.phase = SOPPhase.PENDING
+ return
+
+ state.phase = SOPPhase.RUNNING
+
+ if state.submission is None:
+ handover = None
+ async for event in self.executor.reply_stream(
+ inputs=self._brief(inputs, state),
+ structured_schema=_Handover,
+ yield_final_msg=True,
+ ):
+ if isinstance(event, Msg) and (
+ event.finished_reason == ReplyFinishedReason.COMPLETED
+ ):
+ # The handover is state, not conversation: yielding it
+ # would put a second copy of the reply on screen.
+ handover = (event.structured_output or {}).get("handover")
+ continue
+ yield event
+ if isinstance(
+ event,
+ (RequireUserConfirmEvent, RequireExternalExecutionEvent),
+ ):
+ state.phase = SOPPhase.AWAITING
+ return
+
+ if handover is None:
+ # Structured output makes this unlikely, but a model can
+ # still burn its turns without producing one.
+ self.record(
+ state,
+ False,
+ "Your turn ended without a structured output, so "
+ "nothing was handed on.",
+ "sop",
+ )
+ return
+ state.submission = [TextBlock(text=handover)]
+ inputs = None
+
+ if self.verifier is None:
+ self.record(state, True)
+ return
+
+ verdict = None
+ async for event in self.verifier.reply_stream(
+ inputs=self._question(inputs, state),
+ structured_schema=_Verdict,
+ yield_final_msg=True,
+ ):
+ if isinstance(event, Msg) and (
+ event.finished_reason == ReplyFinishedReason.COMPLETED
+ ):
+ # Likewise the verdict: it belongs in the run state, and
+ # its message carries no content to show.
+ verdict = event.structured_output
+ continue
+ yield event
+ if isinstance(
+ event,
+ (RequireUserConfirmEvent, RequireExternalExecutionEvent),
+ ):
+ state.phase = SOPPhase.AWAITING
+ return
+
+ if verdict is None:
+ self.record(
+ state,
+ False,
+ "The verifier reached no verdict.",
+ "sop",
+ )
+ return
+ self.record(
+ state,
+ verdict["passed"],
+ verdict.get("message", ""),
+ getattr(self.verifier, "name", "verifier"),
+ )
+
+ def _brief(
+ self,
+ inputs: Msg
+ | list[Msg]
+ | UserConfirmResultEvent
+ | UserInterruptEvent
+ | ExternalExecutionResultEvent
+ | None,
+ state: SOPStepRunState,
+ ) -> (
+ list[Msg]
+ | UserConfirmResultEvent
+ | UserInterruptEvent
+ | ExternalExecutionResultEvent
+ ):
+ """What the executor is asked, on top of whatever came in.
+
+ Resumption events are passed straight through — the executor is
+ mid-reply and expects the answer, not a fresh instruction.
+ """
+ if inputs is not None and not isinstance(inputs, (Msg, list)):
+ return inputs
+
+ text = (
+ f"<system-reminder>You are running one step of the SOP.\n\n"
+ f"## {self.subject}\n\n{self.description}\n"
+ )
+ if state.verifications:
+ last = state.verifications[-1]
+ text += (
+ f"\nYour last attempt was not accepted:\n{last.message}\n"
+ f"This is attempt {len(state.verifications) + 1} of "
+ f"{self.max_attempts}."
+ )
+ text += "</system-reminder>"
+
+ brief = UserMsg(name="sop", content=text)
+ if inputs is None:
+ return [brief]
+ return [brief, *(inputs if isinstance(inputs, list) else [inputs])]
+
+ def _question(
+ self,
+ inputs: Msg
+ | list[Msg]
+ | UserConfirmResultEvent
+ | UserInterruptEvent
+ | ExternalExecutionResultEvent
+ | None,
+ state: SOPStepRunState,
+ ) -> (
+ list[Msg]
+ | UserConfirmResultEvent
+ | UserInterruptEvent
+ | ExternalExecutionResultEvent
+ ):
+ """What the verifier is asked, on top of whatever came in.
+
+ It sees what the step was given as well as what came back — a
+ draft can only be judged against what was asked for.
+ """
+ if inputs is not None:
+ return inputs
+ return [
+ UserMsg(
+ name="sop",
+ content=(
+ "<system-reminder>Judge the work below against what "
+ f"this step had to prove.\n\n## {self.subject}\n\n"
+ f"{self.description}</system-reminder>"
+ ),
+ ),
+ *state.given,
+ UserMsg(
+ name="sop",
+ content=[
+ TextBlock(text="<submission>"),
+ *(state.submission or []),
+ TextBlock(text="</submission>"),
+ ],
+ ),
+ ]
+
+
+class SOP:
+ """A fixed sequence of milestones, each verified before the next."""
+
+ def __init__(
+ self,
+ name: str,
+ steps: list[SOPStepBase],
+ description: str = "",
+ ) -> None:
+ """Initialize the procedure.
+
+ Args:
+ name (`str`):
+ The SOP name.
+ steps (`list[SOPStepBase]`):
+ The steps, in the order they run.
+ description (`str`, optional):
+ What this procedure is for.
+ """
+ self.name = name
+ self.steps = steps
+ self.description = description
diff --git a/src/agentscope/sop/_state.py b/src/agentscope/sop/_state.py
new file mode 100644
index 000000000..4aaecdb72
--- /dev/null
+++ b/src/agentscope/sop/_state.py
@@ -0,0 +1,130 @@
+# -*- coding: utf-8 -*-
+"""The runtime state of a SOP run — the half worth persisting.
+
+A definition is code and can be run any number of times; a run is plain
+data and belongs to exactly one of those times. The engine owns the run
+and hands each step the slice that is its own, so nothing about a run
+ever lives on a step object.
+"""
+from enum import StrEnum
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from ..message import DataBlock, Msg, TextBlock
+from .._utils._common import _generate_id, _generate_timestamp
+
+
+class SOPPhase(StrEnum):
+ """Where a step, or a whole run, stands.
+
+ One enum for both: a run is only ever as far along as its steps let
+ it be.
+ """
+
+ PENDING = "pending"
+ """Not started, or sent back to try again."""
+
+ RUNNING = "running"
+ """In flight."""
+
+ AWAITING = "awaiting"
+ """Parked — someone outside has to answer before it can go on."""
+
+ COMPLETED = "completed"
+ """Accepted."""
+
+ FAILED = "failed"
+ """Refused until :attr:`~._schema.SOPStepBase.max_attempts` ran out."""
+
+
+class VerificationResult(BaseModel):
+ """One settled verdict on one attempt.
+
+ Only settled verdicts exist — a step with nothing to say yet records
+ nothing, because a verdict that has not happened is not a verdict.
+ """
+
+ passed: bool
+ """Whether the attempt was accepted."""
+
+ message: str = ""
+ """Why it was refused. This goes back to the executor verbatim on the
+ next attempt, so it has to say what is missing rather than that
+ something is."""
+
+ verifier: str = ""
+ """Who decided — a model, a person, an external system."""
+
+ created_at: str = Field(default_factory=_generate_timestamp)
+ """When the verdict was reached."""
+
+
+class SOPStepRunState(BaseModel):
+ """What one step did in one run.
+
+ The engine writes :attr:`given`; the step keeps the other three
+ honest. To remember more, subclass this and name the subclass in
+ :attr:`~._schema.SOPStepBase.state_type` — extra fields survive a
+ round trip through storage.
+ """
+
+ model_config = ConfigDict(extra="allow")
+
+ phase: SOPPhase = SOPPhase.PENDING
+ """Where the step stands."""
+
+ given: list[Msg] = Field(default_factory=list)
+ """What the engine dispatched this attempt with — the run's inputs
+ for the first step, the one before's handover for the rest. Written
+ by the engine, read by the step: a verifier judging a draft has to
+ see what was asked for, and after a park the answer that resumes the
+ attempt is all the step is handed."""
+
+ submission: list[TextBlock | DataBlock] | None = None
+ """What the current attempt handed over, once it has.
+
+ Blocks rather than text: a step that produces a chart, a file or an
+ image hands it on as readily as a sentence, and the next step reads
+ it the same way. ``None`` — as opposed to empty — means the attempt
+ has not produced anything yet, which is also how a step tells, on
+ resume, that it parked while working rather than while being
+ judged."""
+
+ verifications: list[VerificationResult] = Field(default_factory=list)
+ """Every settled verdict, oldest first. Its length is the attempt
+ count, and its last entry is why the executor is being asked again."""
+
+
+class SOPRunState(BaseModel):
+ """One execution of a SOP, and the whole of what is worth saving.
+
+ It covers the SOP's own state and nothing below it: an executor that
+ keeps state of its own (an :class:`~..agent.Agent` does) is persisted
+ by whoever built it, the same way it is built.
+ """
+
+ id: str = Field(default_factory=_generate_id)
+ """The run identifier."""
+
+ inputs: list[Msg] = Field(default_factory=list)
+ """What the run was started with, and what its first step reads."""
+
+ steps: list[SOPStepRunState] = Field(default_factory=list)
+ """How each step is going, in the order the SOP declares them."""
+
+ created_at: str = Field(default_factory=_generate_timestamp)
+ """When the run was created."""
+
+ @property
+ def phase(self) -> SOPPhase:
+ """Where the run stands, worked out from its steps."""
+ phases = [_.phase for _ in self.steps]
+ if not phases or all(_ is SOPPhase.PENDING for _ in phases):
+ return SOPPhase.PENDING
+ if any(_ is SOPPhase.FAILED for _ in phases):
+ return SOPPhase.FAILED
+ if all(_ is SOPPhase.COMPLETED for _ in phases):
+ return SOPPhase.COMPLETED
+ if any(_ is SOPPhase.AWAITING for _ in phases):
+ return SOPPhase.AWAITING
+ return SOPPhase.RUNNING
diff --git a/src/agentscope/tui/_messages.py b/src/agentscope/tui/_messages.py
index 13006e397..1e9a00be2 100644
--- a/src/agentscope/tui/_messages.py
+++ b/src/agentscope/tui/_messages.py
@@ -683,10 +683,15 @@ class MessageUI(Vertical):
if isinstance(widget, Widget) and widget.is_mounted:
await widget.remove()
self._block_uis = next_widgets
- previous_widget = self.query_one(".as-message-header", Static)
- for widget in ordered:
- self.move_child(widget, after=previous_widget)
- previous_widget = widget
+ if new_keys != old_keys or retained != old_widgets:
+ # Reordering detaches and reinserts every block, so only do it
+ # when the layout actually changed — not on every delta. A
+ # rebuilt widget (a tool group, say) is mounted at the end,
+ # so it counts even when the keys are the same.
+ previous_widget = self.query_one(".as-message-header", Static)
+ for widget in ordered:
+ self.move_child(widget, after=previous_widget)
+ previous_widget = widget
class MessagesUI(VerticalScroll):
diff --git a/tests/sop_engine_test.py b/tests/sop_engine_test.py
new file mode 100644
index 000000000..60554c4cb
--- /dev/null
+++ b/tests/sop_engine_test.py
@@ -0,0 +1,521 @@
+# -*- coding: utf-8 -*-
+"""Unittests for the SOP engine.
+
+Driven with fake executors rather than a model: what is under test is
+the engine's own reasoning — order, attempt budget, parking, and the
+run state it hands out and takes back.
+"""
+from typing import Any, AsyncGenerator
+from unittest import IsolatedAsyncioTestCase
+
+from utils import AnyString
+
+from agentscope.event import (
+ ConfirmResult,
+ CustomEvent,
+ UserInterruptEvent,
+ RequireUserConfirmEvent,
+ UserConfirmResultEvent,
+)
+from agentscope.message import AssistantMsg, ToolCallBlock, UserMsg
+from agentscope.sop import (
+ SOP,
+ SOPEngine,
+ SOPRunState,
+ SOPPhase,
+ SOPStep,
+ SOPStepRunState,
+)
+from agentscope.types import ReplyFinishedReason
+
+
+def _finished(name: str, output: dict) -> AssistantMsg:
+ """A reply that ended with structured output."""
+ return AssistantMsg(
+ name=name,
+ content="",
+ finished_reason=ReplyFinishedReason.COMPLETED,
+ structured_output=output,
+ )
+
+
+def _park() -> RequireUserConfirmEvent:
+ """A reply that stopped for a person."""
+ return RequireUserConfirmEvent(
+ reply_id="reply-1",
+ tool_calls=[
+ ToolCallBlock(
+ type="tool_call",
+ id="call-1",
+ name="shell",
+ input="{}",
+ ),
+ ],
+ )
+
+
+def _answer() -> UserConfirmResultEvent:
+ """What comes back when the person says yes."""
+ call = ToolCallBlock(
+ type="tool_call",
+ id="call-1",
+ name="shell",
+ input="{}",
+ )
+ return UserConfirmResultEvent(
+ reply_id="reply-1",
+ confirm_results=[ConfirmResult(confirmed=True, tool_call=call)],
+ )
+
+
+class _Scripted:
+ """An ``AgentLike`` that replays one scripted reply per call."""
+
+ def __init__(self, name: str, script: list[list[Any]]) -> None:
+ """Remember the script, and what it gets asked."""
+ self.name = name
+ self.script = list(script)
+ self.asked: list[Any] = []
+
+ async def reply_stream( # pylint: disable=unused-argument
+ self,
+ inputs: Any = None,
+ structured_schema: Any = None,
+ yield_final_msg: bool = False,
+ ) -> AsyncGenerator[Any, None]:
+ """Replay the next scripted reply."""
+ self.asked.append(inputs)
+ for event in self.script.pop(0):
+ yield event
+
+
+class SOPEngineTest(IsolatedAsyncioTestCase):
+ """Test the SOP engine."""
+
+ async def _drive(self, engine: SOPEngine, inputs: Any = None) -> list:
+ """Consume one ``reply_stream`` call, collecting its events."""
+ return [event async for event in engine.reply_stream(inputs)]
+
+ async def test_steps_run_in_order_and_hand_over(self) -> None:
+ """A step reads what the one before handed over, and nothing else."""
+ first = _Scripted("one", [[_finished("one", {"handover": "did A"})]])
+ second = _Scripted("two", [[_finished("two", {"handover": "did B"})]])
+ sop = SOP(
+ name="demo",
+ description="d",
+ steps=[
+ SOPStep("A", "do a", first),
+ SOPStep("B", "do b", second),
+ ],
+ )
+ engine = SOPEngine(sop)
+
+ await self._drive(engine, UserMsg(name="user", content="go"))
+
+ self.assertEqual(engine.phase, SOPPhase.COMPLETED)
+ self.assertIn("did A", str(second.asked[0]))
+ self.maxDiff = None
+ self.assertDictEqual(
+ engine.state.model_dump(exclude={"inputs"}),
+ {
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "steps": [
+ {
+ "phase": SOPPhase.COMPLETED,
+ "given": [
+ {
+ "name": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "go",
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ ],
+ "role": "user",
+ "id": AnyString(),
+ "metadata": {},
+ "created_at": AnyString(),
+ "usage": None,
+ "finished_at": AnyString(),
+ "finished_reason": None,
+ "structured_output": None,
+ "error": None,
+ },
+ ],
+ "submission": [
+ {
+ "type": "text",
+ "text": "did A",
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ ],
+ "verifications": [
+ {
+ "passed": True,
+ "message": "",
+ "verifier": "",
+ "created_at": AnyString(),
+ },
+ ],
+ },
+ {
+ "phase": SOPPhase.COMPLETED,
+ "given": [
+ {
+ "name": "sop",
+ "content": [
+ {
+ "type": "text",
+ "text": '<handover from="A">',
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ {
+ "type": "text",
+ "text": "did A",
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ {
+ "type": "text",
+ "text": "</handover>",
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ ],
+ "role": "user",
+ "id": AnyString(),
+ "metadata": {},
+ "created_at": AnyString(),
+ "usage": None,
+ "finished_at": AnyString(),
+ "finished_reason": None,
+ "structured_output": None,
+ "error": None,
+ },
+ ],
+ "submission": [
+ {
+ "type": "text",
+ "text": "did B",
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ ],
+ "verifications": [
+ {
+ "passed": True,
+ "message": "",
+ "verifier": "",
+ "created_at": AnyString(),
+ },
+ ],
+ },
+ ],
+ },
+ )
+
+ async def test_refusal_comes_back_as_a_critique(self) -> None:
+ """A refused attempt is retried, told what was wrong and which try."""
+ executor = _Scripted(
+ "ex",
+ [
+ [_finished("ex", {"handover": "first draft"})],
+ [_finished("ex", {"handover": "fixed draft"})],
+ ],
+ )
+ verifier = _Scripted(
+ "ve",
+ [
+ [
+ _finished(
+ "ve",
+ {"passed": False, "message": "amount wrong"},
+ ),
+ ],
+ [_finished("ve", {"passed": True})],
+ ],
+ )
+ sop = SOP(
+ name="demo",
+ description="d",
+ steps=[SOPStep("A", "do a", executor, verifier)],
+ )
+ engine = SOPEngine(sop)
+
+ await self._drive(engine, UserMsg(name="user", content="go"))
+
+ self.assertEqual(engine.phase, SOPPhase.COMPLETED)
+ self.assertIn("amount wrong", str(executor.asked[1]))
+ self.assertIn("attempt 2 of 3", str(executor.asked[1]))
+ self.assertListEqual(
+ [_.model_dump() for _ in engine.state.steps[0].verifications],
+ [
+ {
+ "passed": False,
+ "message": "amount wrong",
+ "verifier": "ve",
+ "created_at": AnyString(),
+ },
+ {
+ "passed": True,
+ "message": "",
+ "verifier": "ve",
+ "created_at": AnyString(),
+ },
+ ],
+ )
+
+ async def test_run_fails_when_the_attempts_run_out(self) -> None:
+ """Exhausting the budget fails the step, and the run with it."""
+ executor = _Scripted(
+ "ex",
+ [[_finished("ex", {"handover": "nope"})] for _ in range(2)],
+ )
+ verifier = _Scripted(
+ "ve",
+ [
+ [_finished("ve", {"passed": False, "message": "no"})]
+ for _ in range(2)
+ ],
+ )
+ sop = SOP(
+ name="demo",
+ description="d",
+ steps=[
+ SOPStep(
+ "A",
+ "do a",
+ executor,
+ verifier,
+ max_attempts=2,
+ ),
+ SOPStep("B", "do b", _Scripted("two", [])),
+ ],
+ )
+ engine = SOPEngine(sop)
+
+ await self._drive(engine, UserMsg(name="user", content="go"))
+
+ self.assertEqual(engine.phase, SOPPhase.FAILED)
+ # The step behind a failure was never reached.
+ self.assertListEqual(
+ [_.phase for _ in engine.state.steps],
+ [SOPPhase.FAILED, SOPPhase.PENDING],
+ )
+
+ async def test_parking_in_the_executor_ends_the_stream(self) -> None:
+ """A parked run lets go, and picks up where it stopped."""
+ executor = _Scripted(
+ "ex",
+ [[_park()], [_finished("ex", {"handover": "did A"})]],
+ )
+ verifier = _Scripted("ve", [[_finished("ve", {"passed": True})]])
+ sop = SOP(
+ name="demo",
+ description="d",
+ steps=[SOPStep("A", "do a", executor, verifier)],
+ )
+ engine = SOPEngine(sop)
+
+ events = await self._drive(engine, UserMsg(name="user", content="go"))
+
+ self.assertEqual(engine.phase, SOPPhase.AWAITING)
+ self.assertListEqual(
+ [
+ (_.name, _.value) if isinstance(_, CustomEvent) else type(_)
+ for _ in events
+ ],
+ [
+ ("SOP_STEP_STARTED", {"step": "A", "attempt": 1}),
+ RequireUserConfirmEvent,
+ ("SOP_STEP_ENDED", {"step": "A", "phase": "awaiting"}),
+ ],
+ )
+
+ await self._drive(engine, _answer())
+
+ self.assertEqual(engine.phase, SOPPhase.COMPLETED)
+ # The answer went straight to the executor, not wrapped in a brief.
+ self.assertIsInstance(executor.asked[1], UserConfirmResultEvent)
+
+ async def test_parking_in_the_verifier_does_not_redo_the_work(
+ self,
+ ) -> None:
+ """Resuming a judged step resumes the judging, not the work."""
+ executor = _Scripted("ex", [[_finished("ex", {"handover": "did A"})]])
+ verifier = _Scripted(
+ "ve",
+ [[_park()], [_finished("ve", {"passed": True})]],
+ )
+ sop = SOP(
+ name="demo",
+ description="d",
+ steps=[SOPStep("A", "do a", executor, verifier)],
+ )
+ engine = SOPEngine(sop)
+
+ await self._drive(engine, UserMsg(name="user", content="go"))
+ self.assertEqual(engine.phase, SOPPhase.AWAITING)
+
+ await self._drive(engine, _answer())
+
+ self.assertEqual(engine.phase, SOPPhase.COMPLETED)
+ self.assertEqual(len(executor.asked), 1)
+ self.assertEqual(len(verifier.asked), 2)
+
+ async def test_the_verifier_sees_what_the_step_was_given(self) -> None:
+ """Judging a submission means seeing what was asked for, too."""
+ executor = _Scripted(
+ "ex",
+ [[_finished("ex", {"handover": "a draft"})]],
+ )
+ verifier = _Scripted("ve", [[_finished("ve", {"passed": True})]])
+ sop = SOP(
+ name="demo",
+ description="d",
+ steps=[SOPStep("A", "do a", executor, verifier)],
+ )
+ engine = SOPEngine(sop)
+
+ await self._drive(engine, UserMsg(name="user", content="the ask"))
+
+ asked = str(verifier.asked[0])
+ self.assertIn("the ask", asked)
+ self.assertIn("a draft", asked)
+
+ async def test_a_run_survives_the_process_that_started_it(self) -> None:
+ """Export a parked run, rebuild the SOP, and carry on."""
+ sop = SOP(
+ name="demo",
+ description="d",
+ steps=[
+ SOPStep(
+ "A",
+ "do a",
+ _Scripted("ex", [[_park()]]),
+ _Scripted("ve", []),
+ ),
+ ],
+ )
+ engine = SOPEngine(sop)
+ await self._drive(engine, UserMsg(name="user", content="go"))
+ stored = engine.state.model_dump_json()
+
+ revived = SOP(
+ name="demo",
+ description="d",
+ steps=[
+ SOPStep(
+ "A",
+ "do a",
+ _Scripted(
+ "ex",
+ [[_finished("ex", {"handover": "did A"})]],
+ ),
+ _Scripted("ve", [[_finished("ve", {"passed": True})]]),
+ ),
+ ],
+ )
+ engine2 = SOPEngine(revived, SOPRunState.model_validate_json(stored))
+
+ self.assertEqual(engine2.phase, SOPPhase.AWAITING)
+ self.assertEqual(engine2.state.id, engine.state.id)
+
+ await self._drive(engine2, _answer())
+
+ self.assertEqual(engine2.phase, SOPPhase.COMPLETED)
+
+ async def test_a_run_from_an_edited_sop_is_refused(self) -> None:
+ """Editing the SOP retires the runs of the old one."""
+ sop = SOP(
+ name="demo",
+ description="d",
+ steps=[SOPStep("A", "do a", _Scripted("ex", []))],
+ )
+ with self.assertRaises(ValueError) as ctx:
+ SOPEngine(
+ sop,
+ SOPRunState(steps=[SOPStepRunState(), SOPStepRunState()]),
+ )
+ self.assertEqual(
+ str(ctx.exception),
+ "State has 2 steps, but this SOP has 1.",
+ )
+
+ async def test_an_interrupt_abandons_the_attempt_without_charging_it(
+ self,
+ ) -> None:
+ """Interrupting a parked step closes its reply and stops the run.
+
+ The attempt is abandoned, not refused: no verdict is filed, the
+ budget is untouched, and the step waits at PENDING for a fresh
+ start rather than being retried on the spot.
+ """
+ executor = _Scripted("ex", [[_park()], []])
+ sop = SOP(
+ name="demo",
+ description="d",
+ steps=[SOPStep("A", "do a", executor)],
+ )
+ engine = SOPEngine(sop)
+ await self._drive(engine, UserMsg(name="user", content="go"))
+ self.assertEqual(engine.phase, SOPPhase.AWAITING)
+
+ events = await self._drive(
+ engine,
+ UserInterruptEvent(reply_id="reply-1"),
+ )
+
+ self.assertIsInstance(executor.asked[1], UserInterruptEvent)
+ self.assertEqual(len(executor.asked), 2)
+ self.assertDictEqual(
+ engine.state.steps[0].model_dump(),
+ {
+ "phase": SOPPhase.PENDING,
+ # What the abandoned attempt was dispatched with stays on
+ # record; the next dispatch overwrites it.
+ "given": [
+ {
+ "name": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "go",
+ "id": AnyString(),
+ "created_at": AnyString(),
+ "finished_at": None,
+ },
+ ],
+ "role": "user",
+ "id": AnyString(),
+ "metadata": {},
+ "created_at": AnyString(),
+ "usage": None,
+ "finished_at": AnyString(),
+ "finished_reason": None,
+ "structured_output": None,
+ "error": None,
+ },
+ ],
+ "submission": None,
+ "verifications": [],
+ },
+ )
+ self.assertListEqual(
+ [(_.name, _.value) for _ in events if isinstance(_, CustomEvent)],
+ [
+ ("SOP_STEP_STARTED", {"step": "A", "attempt": 1}),
+ ("SOP_STEP_ENDED", {"step": "A", "phase": "pending"}),
+ ],
+ )
diff --git a/tests/tui_test.py b/tests/tui_test.py
index 97f58fe6e..0b464aeed 100644
--- a/tests/tui_test.py
+++ b/tests/tui_test.py
@@ -134,6 +134,83 @@ class MessagesUITest(unittest.IsolatedAsyncioTestCase):
"First and second",
)
+ async def test_streaming_delta_does_not_reorder_blocks(self) -> None:
+ message = AssistantMsg(
+ name="agent",
+ id="reply",
+ content=[
+ TextBlock(id="first", text="One"),
+ ThinkingBlock(id="thought", thinking="Hmm"),
+ TextBlock(id="second", text="Two"),
+ ],
+ )
+ app = _MessagesApp([message])
+ async with app.run_test() as pilot:
+ ui = app.query_one(MessagesUI)
+ widget = ui._message_uis["reply"]
+ moved = []
+
+ def move_child(child: object, **_: Any) -> None:
+ moved.append(child)
+
+ message.content[2].text = "Two and a half"
+ with patch.object(MessageUI, "move_child", move_child):
+ await ui.update_message(message)
+ await pilot.pause()
+
+ # Appending to a block leaves the order alone, and reordering
+ # detaches and reinserts every block — so nothing may move.
+ self.assertListEqual(moved, [])
+ self.assertEqual(
+ widget._block_uis["second"].source,
+ "Two and a half",
+ )
+
+ async def test_rebuilt_tool_group_keeps_its_place(self) -> None:
+ message = AssistantMsg(
+ name="agent",
+ id="reply",
+ content=[
+ TextBlock(id="before", text="Before"),
+ ToolCallBlock(
+ type="tool_call",
+ id="call",
+ name="Bash",
+ input="{}",
+ state=ToolCallState.ALLOWED,
+ ),
+ TextBlock(id="after", text="After"),
+ ],
+ )
+ app = _MessagesApp([message])
+ async with app.run_test() as pilot:
+ ui = app.query_one(MessagesUI)
+ widget = ui._message_uis["reply"]
+
+ # A finished tool rebuilds the group widget, which mounts at
+ # the end; it has to be moved back between the two texts.
+ message.content.append(
+ ToolResultBlock(
+ id="call",
+ name="Bash",
+ output="done",
+ state=ToolResultState.SUCCESS,
+ ),
+ )
+ await ui.update_message(message)
+ await pilot.pause()
+
+ self.assertListEqual(
+ [type(child).__name__ for child in widget.children],
+ [
+ "Static",
+ "TextBlockUI",
+ "ToolGroupUI",
+ "TextBlockUI",
+ "Static",
+ ],
+ )
+
async def test_mutated_snapshot_preserves_existing_widgets(self) -> None:
message = AssistantMsg(
name="agent",