DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(manim): retry reasoning-starved structured stages at low effort
Bingxi Zhao (Frank)6e3dfe59aba3
diff --git a/deeptutor/agents/math_animator/agents/code_generator_agent.py b/deeptutor/agents/math_animator/agents/code_generator_agent.pyindex b5fded355..1899fd17b 100644--- a/deeptutor/agents/math_animator/agents/code_generator_agent.py+++ b/deeptutor/agents/math_animator/agents/code_generator_agent.py@@ -9,6 +9,7 @@ from typing import Anyfrom deeptutor.agents.base_agent import BaseAgentfrom deeptutor.core.trace import build_trace_metadata, new_call_idfrom deeptutor.services.llm import StreamOutcome+from deeptutor.services.llm.reasoning_params import RETRY_REASONING_EFFORTfrom ..models import ConceptAnalysis, GeneratedCode, SceneDesignfrom ..utils import (@@ -162,8 +163,8 @@ class CodeGeneratorAgent(BaseAgent):A retry is not a repeat. The provider reports whether it stopped at theoutput cap, and a reasoning model that spent the whole budget onchain-of-thought fails that way every time on the same request (#1547),- so a truncated attempt is retried with a larger budget and a directive- to keep the reasoning short. Whichever way the attempt failed, the+ so a truncated attempt is retried with a larger budget and lower+ reasoning effort. Whichever way the attempt failed, thereason reaches both the log and the raised error (#1545)."""@@ -185,6 +186,7 @@ class CodeGeneratorAgent(BaseAgent):system_prompt=system_prompt,response_format={"type": "json_object"},max_tokens=max_tokens or None,+ reasoning_effort=(RETRY_REASONING_EFFORT if structured_attempt else None),stage=stage,trace_meta=build_trace_metadata(call_id=new_call_id(call_id_prefix),diff --git a/deeptutor/agents/math_animator/agents/concept_analysis_agent.py b/deeptutor/agents/math_animator/agents/concept_analysis_agent.pyindex c1deb6c80..998c17a67 100644--- a/deeptutor/agents/math_animator/agents/concept_analysis_agent.py+++ b/deeptutor/agents/math_animator/agents/concept_analysis_agent.py@@ -5,10 +5,10 @@ from __future__ import annotationsfrom deeptutor.agents.base_agent import BaseAgentfrom deeptutor.core.context import Attachmentfrom deeptutor.core.trace import build_trace_metadata, new_call_id+from deeptutor.services.llm.structured_retry import json_with_reasoning_retryfrom deeptutor.services.prompt import get_prompt_managerfrom ..models import ConceptAnalysis-from ..utils import extract_json_objectclass ConceptAnalysisAgent(BaseAgent):@@ -66,23 +66,32 @@ class ConceptAnalysisAgent(BaseAgent):{"role": "system", "content": system_prompt},{"role": "user", "content": user_prompt},]- _chunks: list[str] = []- async for _c in self.stream_llm(- user_prompt=user_prompt,- system_prompt=system_prompt,- messages=messages,- attachments=attachments,- response_format={"type": "json_object"},- stage="concept_analysis",- trace_meta=build_trace_metadata(- call_id=new_call_id("math-analysis"),- phase="concept_analysis",- label="Concept analysis",- call_kind="math_concept_analysis",- trace_role="analyze",- trace_kind="llm_output",- ),- ):- _chunks.append(_c)- response = "".join(_chunks)- return ConceptAnalysis.model_validate(extract_json_object(response))++ async def _run(reasoning_effort: str | None) -> str:+ chunks: list[str] = []+ async for chunk in self.stream_llm(+ user_prompt=user_prompt,+ system_prompt=system_prompt,+ messages=messages,+ attachments=attachments,+ response_format={"type": "json_object"},+ reasoning_effort=reasoning_effort,+ stage="concept_analysis",+ trace_meta=build_trace_metadata(+ call_id=new_call_id("math-analysis"),+ phase="concept_analysis",+ label="Concept analysis",+ call_kind="math_concept_analysis",+ trace_role="analyze",+ trace_kind="llm_output",+ ),+ ):+ chunks.append(chunk)+ return "".join(chunks)++ payload = await json_with_reasoning_retry(+ _run, expected_key="learning_goal", logger_instance=self.logger+ )+ if not payload.get("learning_goal"):+ raise ValueError("Math animator concept analysis returned no learning goal.")+ return ConceptAnalysis.model_validate(payload)diff --git a/deeptutor/agents/math_animator/agents/concept_design_agent.py b/deeptutor/agents/math_animator/agents/concept_design_agent.pyindex e16e82605..3201fb074 100644--- a/deeptutor/agents/math_animator/agents/concept_design_agent.py+++ b/deeptutor/agents/math_animator/agents/concept_design_agent.py@@ -6,9 +6,9 @@ import jsonfrom deeptutor.agents.base_agent import BaseAgentfrom deeptutor.core.trace import build_trace_metadata, new_call_id+from deeptutor.services.llm.structured_retry import json_with_reasoning_retryfrom ..models import ConceptAnalysis, SceneDesign-from ..utils import extract_json_objectclass ConceptDesignAgent(BaseAgent):@@ -47,21 +47,30 @@ class ConceptDesignAgent(BaseAgent):style_hint=style_hint.strip() or "(none)",analysis_json=json.dumps(analysis.model_dump(), ensure_ascii=False, indent=2),)- _chunks: list[str] = []- async for _c in self.stream_llm(- user_prompt=user_prompt,- system_prompt=system_prompt,- response_format={"type": "json_object"},- stage="concept_design",- trace_meta=build_trace_metadata(- call_id=new_call_id("math-design"),- phase="concept_design",- label="Concept design",- call_kind="math_concept_design",- trace_role="design",- trace_kind="llm_output",- ),- ):- _chunks.append(_c)- response = "".join(_chunks)- return SceneDesign.model_validate(extract_json_object(response))++ async def _run(reasoning_effort: str | None) -> str:+ chunks: list[str] = []+ async for chunk in self.stream_llm(+ user_prompt=user_prompt,+ system_prompt=system_prompt,+ response_format={"type": "json_object"},+ reasoning_effort=reasoning_effort,+ stage="concept_design",+ trace_meta=build_trace_metadata(+ call_id=new_call_id("math-design"),+ phase="concept_design",+ label="Concept design",+ call_kind="math_concept_design",+ trace_role="design",+ trace_kind="llm_output",+ ),+ ):+ chunks.append(chunk)+ return "".join(chunks)++ payload = await json_with_reasoning_retry(+ _run, expected_key="scene_outline", logger_instance=self.logger+ )+ if not payload.get("scene_outline"):+ raise ValueError("Math animator concept design returned no scene outline.")+ return SceneDesign.model_validate(payload)diff --git a/deeptutor/agents/math_animator/agents/summary_agent.py b/deeptutor/agents/math_animator/agents/summary_agent.pyindex fdfe34ae7..77c2d41ee 100644--- a/deeptutor/agents/math_animator/agents/summary_agent.py+++ b/deeptutor/agents/math_animator/agents/summary_agent.py@@ -6,9 +6,9 @@ import jsonfrom deeptutor.agents.base_agent import BaseAgentfrom deeptutor.core.trace import build_trace_metadata, new_call_id+from deeptutor.services.llm.structured_retry import json_with_reasoning_retryfrom ..models import ConceptAnalysis, RenderResult, SceneDesign, SummaryPayload-from ..utils import extract_json_objectclass SummaryAgent(BaseAgent):@@ -49,21 +49,30 @@ class SummaryAgent(BaseAgent):design_json=json.dumps(design.model_dump(), ensure_ascii=False, indent=2),render_json=json.dumps(render_result.model_dump(), ensure_ascii=False, indent=2),)- _chunks: list[str] = []- async for _c in self.stream_llm(- user_prompt=user_prompt,- system_prompt=system_prompt,- response_format={"type": "json_object"},- stage="summary",- trace_meta=build_trace_metadata(- call_id=new_call_id("math-summary"),- phase="summary",- label="Summarize result",- call_kind="math_summary",- trace_role="summarize",- trace_kind="llm_output",- ),- ):- _chunks.append(_c)- response = "".join(_chunks)- return SummaryPayload.model_validate(extract_json_object(response))++ async def _run(reasoning_effort: str | None) -> str:+ chunks: list[str] = []+ async for chunk in self.stream_llm(+ user_prompt=user_prompt,+ system_prompt=system_prompt,+ response_format={"type": "json_object"},+ reasoning_effort=reasoning_effort,+ stage="summary",+ trace_meta=build_trace_metadata(+ call_id=new_call_id("math-summary"),+ phase="summary",+ label="Summarize result",+ call_kind="math_summary",+ trace_role="summarize",+ trace_kind="llm_output",+ ),+ ):+ chunks.append(chunk)+ return "".join(chunks)++ payload = await json_with_reasoning_retry(+ _run, expected_key="summary_text", logger_instance=self.logger+ )+ # Rendering has already succeeded. Keep returning its artifacts if the+ # summary remains empty after the low-effort retry.+ return SummaryPayload.model_validate(payload)diff --git a/tests/agents/math_animator/test_code_generator_agent.py b/tests/agents/math_animator/test_code_generator_agent.pyindex 1257403e0..6e11434e0 100644--- a/tests/agents/math_animator/test_code_generator_agent.py+++ b/tests/agents/math_animator/test_code_generator_agent.py@@ -9,6 +9,7 @@ from deeptutor.agents.math_animator.agents.code_generator_agent import (GeneratedCodeOutputError,)from deeptutor.agents.math_animator.models import ConceptAnalysis, SceneDesign+from deeptutor.services.llm.reasoning_params import RETRY_REASONING_EFFORTdef _agent(monkeypatch: pytest.MonkeyPatch, responses: list[str]) -> CodeGeneratorAgent:@@ -124,6 +125,11 @@ async def test_truncated_generation_grows_the_budget_and_asks_for_less_reasoning# Each truncated attempt buys a larger budget, capped at twice the# configured one so the request stays inside the model's own output limit.assert [call["max_tokens"] for call in calls] == [8000, 12000, 16000]+ assert [call["reasoning_effort"] for call in calls] == [+ None,+ RETRY_REASONING_EFFORT,+ RETRY_REASONING_EFFORT,+ ]assert "token limit" in str(calls[1]["user_prompt"])assert "token limit" not in str(calls[0]["user_prompt"])# The failure names the cause instead of "no usable code after 3 attempts".@@ -133,6 +139,57 @@ async def test_truncated_generation_grows_the_budget_and_asks_for_less_reasoningassert "reasoning_tokens=7800" in message[email protected]("stage", ["generation", "repair"])+async def test_truncated_reasoning_only_response_recovers_with_less_reasoning(+ monkeypatch: pytest.MonkeyPatch,+ stage: str,+) -> None:+ agent = _agent(monkeypatch, [])+ monkeypatch.setattr(agent, "get_max_tokens", lambda: 8000)+ calls: list[dict[str, object]] = []++ async def fake_provider(**kwargs) -> AsyncIterator[str]:+ calls.append(kwargs)+ if len(calls) == 1:+ kwargs["outcome"].finish_reason = "length"+ kwargs["outcome"].usage = {"reasoning_tokens": 7900}+ yield "<think>I need to plan every scene first.</think>"+ else:+ kwargs["outcome"].finish_reason = "stop"+ yield '{"code":"from manim import Scene","rationale":"ok"}'++ monkeypatch.setattr(agent, "stream_llm", fake_provider)+ if stage == "generation":+ generated = await agent.generate(+ user_input="Animate a proof",+ output_mode="video",+ analysis=ConceptAnalysis(),+ design=SceneDesign(),+ )+ else:+ agent.prompts.update(+ {+ "retry_system": "Repair the script.",+ "retry_user_template": (+ "{user_input} {output_mode} {attempt} {duration_requirement} "+ "{error_message} {current_code}"+ ),+ }+ )+ generated = await agent.repair(+ user_input="Animate a proof",+ output_mode="video",+ current_code="broken code",+ error_message="SyntaxError",+ attempt=1,+ )++ assert generated.code == "from manim import Scene"+ assert [call["reasoning_effort"] for call in calls] == [None, RETRY_REASONING_EFFORT]+ assert [call["max_tokens"] for call in calls] == [8000, 12000]++@pytest.mark.asyncioasync def test_malformed_output_is_reported_as_malformed_not_as_a_budget_problem(monkeypatch: pytest.MonkeyPatch,diff --git a/tests/agents/math_animator/test_reasoning_retry.py b/tests/agents/math_animator/test_reasoning_retry.pynew file mode 100644index 000000000..6a4051228--- /dev/null+++ b/tests/agents/math_animator/test_reasoning_retry.py@@ -0,0 +1,143 @@+"""A reasoning-only structured stage must not silently become an empty plan."""++from __future__ import annotations++from collections.abc import AsyncIterator++import pytest++from deeptutor.agents.math_animator.agents.concept_analysis_agent import (+ ConceptAnalysisAgent,+)+from deeptutor.agents.math_animator.agents.concept_design_agent import ConceptDesignAgent+from deeptutor.agents.math_animator.agents.summary_agent import SummaryAgent+from deeptutor.agents.math_animator.models import ConceptAnalysis, RenderResult, SceneDesign+from deeptutor.services.llm.reasoning_params import RETRY_REASONING_EFFORT++[email protected]("stage", ["analysis", "design", "summary"])+async def test_structured_manim_stage_retries_reasoning_only_response(+ monkeypatch: pytest.MonkeyPatch, stage: str+) -> None:+ if stage == "analysis":+ agent = ConceptAnalysisAgent()+ agent.prompts = {+ "system": "Analyze the concept.",+ "user_template": (+ "{user_input} {history_context} {output_mode} {style_hint} {reference_count}"+ ),+ }+ call = agent.process(+ user_input="Explain a circle",+ history_context="",+ output_mode="video",+ style_hint="",+ attachments=[],+ )+ answer = '{"learning_goal":"understand a circle","visual_targets":["radius"]}'+ expected = ("learning_goal", "understand a circle")+ elif stage == "design":+ agent = ConceptDesignAgent()+ agent.prompts = {+ "system": "Design the scene.",+ "user_template": "{user_input} {output_mode} {style_hint} {analysis_json}",+ }+ call = agent.process(+ user_input="Explain a circle",+ output_mode="video",+ analysis=ConceptAnalysis(learning_goal="understand a circle"),+ style_hint="",+ )+ answer = '{"scene_outline":["draw a circle"],"title":"Circle"}'+ expected = ("scene_outline", ["draw a circle"])+ else:+ agent = SummaryAgent()+ agent.prompts = {+ "system": "Summarize the result.",+ "user_template": (+ "{user_input} {output_mode} {analysis_json} {design_json} {render_json}"+ ),+ }+ call = agent.process(+ user_input="Explain a circle",+ output_mode="video",+ analysis=ConceptAnalysis(learning_goal="understand a circle"),+ design=SceneDesign(scene_outline=["draw a circle"]),+ render_result=RenderResult(output_mode="video"),+ )+ answer = '{"summary_text":"A circle animation is ready."}'+ expected = ("summary_text", "A circle animation is ready.")++ calls: list[dict[str, object]] = []++ async def fake_provider(**kwargs) -> AsyncIterator[str]:+ calls.append(kwargs)+ yield "<think>I will plan the animation first.</think>" if len(calls) == 1 else answer++ monkeypatch.setattr(agent, "stream_llm", fake_provider)+ result = await call++ assert getattr(result, expected[0]) == expected[1]+ assert [entry["reasoning_effort"] for entry in calls] == [+ None,+ RETRY_REASONING_EFFORT,+ ]+ assert calls[0]["trace_meta"]["call_id"] != calls[1]["trace_meta"]["call_id"]+ if stage == "analysis":+ assert [message["role"] for message in calls[1]["messages"]] == ["system", "user"]+++async def test_analysis_does_not_accept_two_empty_reasoning_only_responses(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ agent = ConceptAnalysisAgent()+ agent.prompts = {+ "system": "Analyze the concept.",+ "user_template": (+ "{user_input} {history_context} {output_mode} {style_hint} {reference_count}"+ ),+ }++ async def fake_provider(**_kwargs) -> AsyncIterator[str]:+ yield "<think>Still thinking.</think>"++ monkeypatch.setattr(agent, "stream_llm", fake_provider)+ with pytest.raises(ValueError, match="concept analysis returned no learning goal"):+ await agent.process(+ user_input="Explain a circle",+ history_context="",+ output_mode="video",+ style_hint="",+ attachments=[],+ )+++async def test_empty_summary_after_retry_keeps_a_completed_render(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ agent = SummaryAgent()+ agent.prompts = {+ "system": "Summarize the result.",+ "user_template": ("{user_input} {output_mode} {analysis_json} {design_json} {render_json}"),+ }+ calls = 0++ async def fake_provider(**_kwargs) -> AsyncIterator[str]:+ nonlocal calls+ calls += 1+ yield "<think>Still thinking.</think>"++ monkeypatch.setattr(agent, "stream_llm", fake_provider)+ summary = await agent.process(+ user_input="Explain a circle",+ output_mode="video",+ analysis=ConceptAnalysis(learning_goal="understand a circle"),+ design=SceneDesign(scene_outline=["draw a circle"]),+ render_result=RenderResult(output_mode="video"),+ )++ assert calls == 2+ assert summary.summary_text == ""