Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
fix(pipeline): cap the GoalPipeline structured-output retries (#3078)
XuRuuuye9450d080dbc
diff --git a/src/agentscope/pipeline/_goal_pipeline.py b/src/agentscope/pipeline/_goal_pipeline.pyindex 66f9d31c9..142e44cb1 100644--- a/src/agentscope/pipeline/_goal_pipeline.py+++ b/src/agentscope/pipeline/_goal_pipeline.py@@ -88,8 +88,27 @@ class GoalPipeline:# Rounds already judged. On the instance rather than in# ``reply_stream``, so a HITL resume does not restart the budget.self._iters = 0+ # Preserve each participant's retry budget across HITL resumes.+ self._executor_retries = 0+ self._verifier_retries = 0self._goal: None | list[TextBlock | DataBlock] = None+ def _check_retry_budget(+ self,+ retries: int,+ participant: Literal["executor", "verifier"],+ ) -> None:+ """Raise when a participant exhausts its structured-output retries."""+ if retries > self.max_retries:+ output_name = {+ "executor": "execution report",+ "verifier": "verification result",+ }[participant]+ raise RuntimeError(+ f"The {participant} failed to generate a valid "+ f"{output_name} after {self.max_retries} retries.",+ )+async def reply_stream(self,inputs: Msg@@ -123,6 +142,8 @@ class GoalPipeline:executor_inputs = deepcopy(inputs)# A fresh run, so the iteration budget starts over.self._iters = 0+ self._executor_retries = 0+ self._verifier_retries = 0hint = ("<system-reminder>When you finish the goal, you should "@@ -205,6 +226,11 @@ class GoalPipeline:breakif execution_report is None:+ self._executor_retries += 1+ self._check_retry_budget(+ self._executor_retries,+ "executor",+ )# Update the instructionexecutor_inputs = UserMsg(name="system",@@ -272,8 +298,19 @@ class GoalPipeline:# Escape the loop for hitl eventsbreakif final_msg is None:+ # Retry without changing the current instruction.+ self._verifier_retries += 1+ self._check_retry_budget(+ self._verifier_retries,+ "verifier",+ )continueif final_msg.structured_output is None:+ self._verifier_retries += 1+ self._check_retry_budget(+ self._verifier_retries,+ "verifier",+ )# Update the instruction for valid verification resultfinal_msg = Noneinstruction = UserMsg(diff --git a/tests/pipeline_goal_test.py b/tests/pipeline_goal_test.pyindex e724277e8..b22c1321e 100644--- a/tests/pipeline_goal_test.py+++ b/tests/pipeline_goal_test.py@@ -321,6 +321,105 @@ class GoalPipelineTest(IsolatedAsyncioTestCase):)self.assertEqual(len(verifier.received), 1)+ async def test_gives_up_on_an_executor_that_never_reports(self) -> None:+ """Stop after the initial attempt and the allowed retries."""+ for max_retries in (0, 2):+ with self.subTest(max_retries=max_retries):+ missing = _no_output("executor")+ executor = StubAgent("executor", [[missing]])+ verifier = StubAgent("verifier", [[_verdict("pass")]])+ pipe = GoalPipeline(+ executor,+ verifier,+ max_retries=max_retries,+ )+ yielded = []++ with self.assertRaises(RuntimeError) as raised:+ async for chunk in pipe.reply_stream(+ UserMsg("user", "Build a crawler"),+ ):+ yielded.append(chunk)++ self.assertEqual(+ str(raised.exception),+ "The executor failed to generate a valid execution "+ f"report after {max_retries} retries.",+ )+ self.assertEqual(yielded, [missing] * (max_retries + 1))+ self.assertEqual(len(executor.received), max_retries + 1)+ self.assertEqual(verifier.received, [])++ async def test_gives_up_on_a_verifier_that_never_verdicts(self) -> None:+ """Bound retries with a missing verdict or an empty stream."""+ for batch in ([], [_no_output("verifier")]):+ with self.subTest(empty_stream=not batch):+ report = _report("Completed; see main.py")+ executor = StubAgent("executor", [[report]])+ verifier = StubAgent("verifier", [batch])+ pipe = GoalPipeline(executor, verifier, max_retries=2)+ yielded = []++ with self.assertRaises(RuntimeError) as raised:+ async for chunk in pipe.reply_stream(+ UserMsg("user", "Build a crawler"),+ ):+ yielded.append(chunk)++ self.assertEqual(+ str(raised.exception),+ "The verifier failed to generate a valid verification "+ "result after 2 retries.",+ )+ self.assertEqual(yielded, [report])+ self.assertEqual(+ (len(executor.received), len(verifier.received)),+ (1, 3),+ )++ async def test_a_fresh_run_restarts_the_retry_budget(self) -> None:+ """Each participant can retry again after a previous run failed."""+ for participant in ("executor", "verifier"):+ with self.subTest(participant=participant):+ report = _report("Completed; see main.py")+ verdict = _verdict("pass")+ executor = StubAgent("executor", [[report]])+ verifier = StubAgent("verifier", [[verdict]])+ missing = _no_output(participant)+ if participant == "executor":+ executor.script = [[missing]] * 3 + [[report]]+ expected_yielded = [missing, report]+ expected_calls = (4, 1)+ output_name = "execution report"+ else:+ verifier.script = [[missing]] * 3 + [[verdict]]+ expected_yielded = [report]+ expected_calls = (2, 4)+ output_name = "verification result"+ pipe = GoalPipeline(executor, verifier, max_retries=1)++ with self.assertRaises(RuntimeError) as raised:+ await self._run(+ pipe,+ UserMsg("user", "Build a crawler"),+ )+ self.assertEqual(+ str(raised.exception),+ f"The {participant} failed to generate a valid "+ f"{output_name} after 1 retries.",+ )++ # The new run must retry once before succeeding.+ yielded = await self._run(+ pipe,+ UserMsg("user", "Build another crawler"),+ )+ self.assertEqual(yielded, expected_yielded)+ self.assertEqual(+ (len(executor.received), len(verifier.received)),+ expected_calls,+ )+async def test_a_parked_executor_is_not_verified(self) -> None:"""The work is unfinished while the executor waits on a human, sothere is nothing for the verifier to judge yet."""