Skip to content

Build and run agents you can see, understand and trust.

fix(pipeline): cap the GoalPipeline structured-output retries (#3078)

XuRuuuye9450d080dbc
diff --git a/src/agentscope/pipeline/_goal_pipeline.py b/src/agentscope/pipeline/_goal_pipeline.py
index 66f9d31c9..142e44cb1 100644
--- a/src/agentscope/pipeline/_goal_pipeline.py
+++ b/src/agentscope/pipeline/_goal_pipeline.py
@@ -88,8 +88,27 @@ class GoalPipeline:
# Rounds already judged. On the instance rather than in
# ``reply_stream``, so a HITL resume does not restart the budget.
self._iters = 0
+ # Preserve each participant's retry budget across HITL resumes.
+ self._executor_retries = 0
+ self._verifier_retries = 0
self._goal: None | list[TextBlock | DataBlock] = None
+ def _check_retry_budget(
+ self,
+ retries: int,
+ participant: Literal["executor", "verifier"],
+ ) -> None:
+ """Raise when a participant exhausts its structured-output retries."""
+ if retries > self.max_retries:
+ output_name = {
+ "executor": "execution report",
+ "verifier": "verification result",
+ }[participant]
+ raise RuntimeError(
+ f"The {participant} failed to generate a valid "
+ f"{output_name} after {self.max_retries} retries.",
+ )
+
async def reply_stream(
self,
inputs: Msg
@@ -123,6 +142,8 @@ class GoalPipeline:
executor_inputs = deepcopy(inputs)
# A fresh run, so the iteration budget starts over.
self._iters = 0
+ self._executor_retries = 0
+ self._verifier_retries = 0
hint = (
"<system-reminder>When you finish the goal, you should "
@@ -205,6 +226,11 @@ class GoalPipeline:
break
if execution_report is None:
+ self._executor_retries += 1
+ self._check_retry_budget(
+ self._executor_retries,
+ "executor",
+ )
# Update the instruction
executor_inputs = UserMsg(
name="system",
@@ -272,8 +298,19 @@ class GoalPipeline:
# Escape the loop for hitl events
break
if final_msg is None:
+ # Retry without changing the current instruction.
+ self._verifier_retries += 1
+ self._check_retry_budget(
+ self._verifier_retries,
+ "verifier",
+ )
continue
if final_msg.structured_output is None:
+ self._verifier_retries += 1
+ self._check_retry_budget(
+ self._verifier_retries,
+ "verifier",
+ )
# Update the instruction for valid verification result
final_msg = None
instruction = UserMsg(
diff --git a/tests/pipeline_goal_test.py b/tests/pipeline_goal_test.py
index e724277e8..b22c1321e 100644
--- a/tests/pipeline_goal_test.py
+++ b/tests/pipeline_goal_test.py
@@ -321,6 +321,105 @@ class GoalPipelineTest(IsolatedAsyncioTestCase):
)
self.assertEqual(len(verifier.received), 1)
+ async def test_gives_up_on_an_executor_that_never_reports(self) -> None:
+ """Stop after the initial attempt and the allowed retries."""
+ for max_retries in (0, 2):
+ with self.subTest(max_retries=max_retries):
+ missing = _no_output("executor")
+ executor = StubAgent("executor", [[missing]])
+ verifier = StubAgent("verifier", [[_verdict("pass")]])
+ pipe = GoalPipeline(
+ executor,
+ verifier,
+ max_retries=max_retries,
+ )
+ yielded = []
+
+ with self.assertRaises(RuntimeError) as raised:
+ async for chunk in pipe.reply_stream(
+ UserMsg("user", "Build a crawler"),
+ ):
+ yielded.append(chunk)
+
+ self.assertEqual(
+ str(raised.exception),
+ "The executor failed to generate a valid execution "
+ f"report after {max_retries} retries.",
+ )
+ self.assertEqual(yielded, [missing] * (max_retries + 1))
+ self.assertEqual(len(executor.received), max_retries + 1)
+ self.assertEqual(verifier.received, [])
+
+ async def test_gives_up_on_a_verifier_that_never_verdicts(self) -> None:
+ """Bound retries with a missing verdict or an empty stream."""
+ for batch in ([], [_no_output("verifier")]):
+ with self.subTest(empty_stream=not batch):
+ report = _report("Completed; see main.py")
+ executor = StubAgent("executor", [[report]])
+ verifier = StubAgent("verifier", [batch])
+ pipe = GoalPipeline(executor, verifier, max_retries=2)
+ yielded = []
+
+ with self.assertRaises(RuntimeError) as raised:
+ async for chunk in pipe.reply_stream(
+ UserMsg("user", "Build a crawler"),
+ ):
+ yielded.append(chunk)
+
+ self.assertEqual(
+ str(raised.exception),
+ "The verifier failed to generate a valid verification "
+ "result after 2 retries.",
+ )
+ self.assertEqual(yielded, [report])
+ self.assertEqual(
+ (len(executor.received), len(verifier.received)),
+ (1, 3),
+ )
+
+ async def test_a_fresh_run_restarts_the_retry_budget(self) -> None:
+ """Each participant can retry again after a previous run failed."""
+ for participant in ("executor", "verifier"):
+ with self.subTest(participant=participant):
+ report = _report("Completed; see main.py")
+ verdict = _verdict("pass")
+ executor = StubAgent("executor", [[report]])
+ verifier = StubAgent("verifier", [[verdict]])
+ missing = _no_output(participant)
+ if participant == "executor":
+ executor.script = [[missing]] * 3 + [[report]]
+ expected_yielded = [missing, report]
+ expected_calls = (4, 1)
+ output_name = "execution report"
+ else:
+ verifier.script = [[missing]] * 3 + [[verdict]]
+ expected_yielded = [report]
+ expected_calls = (2, 4)
+ output_name = "verification result"
+ pipe = GoalPipeline(executor, verifier, max_retries=1)
+
+ with self.assertRaises(RuntimeError) as raised:
+ await self._run(
+ pipe,
+ UserMsg("user", "Build a crawler"),
+ )
+ self.assertEqual(
+ str(raised.exception),
+ f"The {participant} failed to generate a valid "
+ f"{output_name} after 1 retries.",
+ )
+
+ # The new run must retry once before succeeding.
+ yielded = await self._run(
+ pipe,
+ UserMsg("user", "Build another crawler"),
+ )
+ self.assertEqual(yielded, expected_yielded)
+ self.assertEqual(
+ (len(executor.received), len(verifier.received)),
+ expected_calls,
+ )
+
async def test_a_parked_executor_is_not_verified(self) -> None:
"""The work is unfinished while the executor waits on a human, so
there is nothing for the verifier to judge yet."""