Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(manim): retry reasoning-starved structured stages at low effort

Bingxi Zhao (Frank)6e3dfe59aba3
diff --git a/deeptutor/agents/math_animator/agents/code_generator_agent.py b/deeptutor/agents/math_animator/agents/code_generator_agent.py
index b5fded355..1899fd17b 100644
--- a/deeptutor/agents/math_animator/agents/code_generator_agent.py
+++ b/deeptutor/agents/math_animator/agents/code_generator_agent.py
@@ -9,6 +9,7 @@ from typing import Any
from deeptutor.agents.base_agent import BaseAgent
from deeptutor.core.trace import build_trace_metadata, new_call_id
from deeptutor.services.llm import StreamOutcome
+from deeptutor.services.llm.reasoning_params import RETRY_REASONING_EFFORT
from ..models import ConceptAnalysis, GeneratedCode, SceneDesign
from ..utils import (
@@ -162,8 +163,8 @@ class CodeGeneratorAgent(BaseAgent):
A retry is not a repeat. The provider reports whether it stopped at the
output cap, and a reasoning model that spent the whole budget on
chain-of-thought fails that way every time on the same request (#1547),
- so a truncated attempt is retried with a larger budget and a directive
- to keep the reasoning short. Whichever way the attempt failed, the
+ so a truncated attempt is retried with a larger budget and lower
+ reasoning effort. Whichever way the attempt failed, the
reason reaches both the log and the raised error (#1545).
"""
@@ -185,6 +186,7 @@ class CodeGeneratorAgent(BaseAgent):
system_prompt=system_prompt,
response_format={"type": "json_object"},
max_tokens=max_tokens or None,
+ reasoning_effort=(RETRY_REASONING_EFFORT if structured_attempt else None),
stage=stage,
trace_meta=build_trace_metadata(
call_id=new_call_id(call_id_prefix),
diff --git a/deeptutor/agents/math_animator/agents/concept_analysis_agent.py b/deeptutor/agents/math_animator/agents/concept_analysis_agent.py
index c1deb6c80..998c17a67 100644
--- a/deeptutor/agents/math_animator/agents/concept_analysis_agent.py
+++ b/deeptutor/agents/math_animator/agents/concept_analysis_agent.py
@@ -5,10 +5,10 @@ from __future__ import annotations
from deeptutor.agents.base_agent import BaseAgent
from deeptutor.core.context import Attachment
from deeptutor.core.trace import build_trace_metadata, new_call_id
+from deeptutor.services.llm.structured_retry import json_with_reasoning_retry
from deeptutor.services.prompt import get_prompt_manager
from ..models import ConceptAnalysis
-from ..utils import extract_json_object
class ConceptAnalysisAgent(BaseAgent):
@@ -66,23 +66,32 @@ class ConceptAnalysisAgent(BaseAgent):
{"role": "system", "content": system_prompt},
{"role": "user", "content": user_prompt},
]
- _chunks: list[str] = []
- async for _c in self.stream_llm(
- user_prompt=user_prompt,
- system_prompt=system_prompt,
- messages=messages,
- attachments=attachments,
- response_format={"type": "json_object"},
- stage="concept_analysis",
- trace_meta=build_trace_metadata(
- call_id=new_call_id("math-analysis"),
- phase="concept_analysis",
- label="Concept analysis",
- call_kind="math_concept_analysis",
- trace_role="analyze",
- trace_kind="llm_output",
- ),
- ):
- _chunks.append(_c)
- response = "".join(_chunks)
- return ConceptAnalysis.model_validate(extract_json_object(response))
+
+ async def _run(reasoning_effort: str | None) -> str:
+ chunks: list[str] = []
+ async for chunk in self.stream_llm(
+ user_prompt=user_prompt,
+ system_prompt=system_prompt,
+ messages=messages,
+ attachments=attachments,
+ response_format={"type": "json_object"},
+ reasoning_effort=reasoning_effort,
+ stage="concept_analysis",
+ trace_meta=build_trace_metadata(
+ call_id=new_call_id("math-analysis"),
+ phase="concept_analysis",
+ label="Concept analysis",
+ call_kind="math_concept_analysis",
+ trace_role="analyze",
+ trace_kind="llm_output",
+ ),
+ ):
+ chunks.append(chunk)
+ return "".join(chunks)
+
+ payload = await json_with_reasoning_retry(
+ _run, expected_key="learning_goal", logger_instance=self.logger
+ )
+ if not payload.get("learning_goal"):
+ raise ValueError("Math animator concept analysis returned no learning goal.")
+ return ConceptAnalysis.model_validate(payload)
diff --git a/deeptutor/agents/math_animator/agents/concept_design_agent.py b/deeptutor/agents/math_animator/agents/concept_design_agent.py
index e16e82605..3201fb074 100644
--- a/deeptutor/agents/math_animator/agents/concept_design_agent.py
+++ b/deeptutor/agents/math_animator/agents/concept_design_agent.py
@@ -6,9 +6,9 @@ import json
from deeptutor.agents.base_agent import BaseAgent
from deeptutor.core.trace import build_trace_metadata, new_call_id
+from deeptutor.services.llm.structured_retry import json_with_reasoning_retry
from ..models import ConceptAnalysis, SceneDesign
-from ..utils import extract_json_object
class ConceptDesignAgent(BaseAgent):
@@ -47,21 +47,30 @@ class ConceptDesignAgent(BaseAgent):
style_hint=style_hint.strip() or "(none)",
analysis_json=json.dumps(analysis.model_dump(), ensure_ascii=False, indent=2),
)
- _chunks: list[str] = []
- async for _c in self.stream_llm(
- user_prompt=user_prompt,
- system_prompt=system_prompt,
- response_format={"type": "json_object"},
- stage="concept_design",
- trace_meta=build_trace_metadata(
- call_id=new_call_id("math-design"),
- phase="concept_design",
- label="Concept design",
- call_kind="math_concept_design",
- trace_role="design",
- trace_kind="llm_output",
- ),
- ):
- _chunks.append(_c)
- response = "".join(_chunks)
- return SceneDesign.model_validate(extract_json_object(response))
+
+ async def _run(reasoning_effort: str | None) -> str:
+ chunks: list[str] = []
+ async for chunk in self.stream_llm(
+ user_prompt=user_prompt,
+ system_prompt=system_prompt,
+ response_format={"type": "json_object"},
+ reasoning_effort=reasoning_effort,
+ stage="concept_design",
+ trace_meta=build_trace_metadata(
+ call_id=new_call_id("math-design"),
+ phase="concept_design",
+ label="Concept design",
+ call_kind="math_concept_design",
+ trace_role="design",
+ trace_kind="llm_output",
+ ),
+ ):
+ chunks.append(chunk)
+ return "".join(chunks)
+
+ payload = await json_with_reasoning_retry(
+ _run, expected_key="scene_outline", logger_instance=self.logger
+ )
+ if not payload.get("scene_outline"):
+ raise ValueError("Math animator concept design returned no scene outline.")
+ return SceneDesign.model_validate(payload)
diff --git a/deeptutor/agents/math_animator/agents/summary_agent.py b/deeptutor/agents/math_animator/agents/summary_agent.py
index fdfe34ae7..77c2d41ee 100644
--- a/deeptutor/agents/math_animator/agents/summary_agent.py
+++ b/deeptutor/agents/math_animator/agents/summary_agent.py
@@ -6,9 +6,9 @@ import json
from deeptutor.agents.base_agent import BaseAgent
from deeptutor.core.trace import build_trace_metadata, new_call_id
+from deeptutor.services.llm.structured_retry import json_with_reasoning_retry
from ..models import ConceptAnalysis, RenderResult, SceneDesign, SummaryPayload
-from ..utils import extract_json_object
class SummaryAgent(BaseAgent):
@@ -49,21 +49,30 @@ class SummaryAgent(BaseAgent):
design_json=json.dumps(design.model_dump(), ensure_ascii=False, indent=2),
render_json=json.dumps(render_result.model_dump(), ensure_ascii=False, indent=2),
)
- _chunks: list[str] = []
- async for _c in self.stream_llm(
- user_prompt=user_prompt,
- system_prompt=system_prompt,
- response_format={"type": "json_object"},
- stage="summary",
- trace_meta=build_trace_metadata(
- call_id=new_call_id("math-summary"),
- phase="summary",
- label="Summarize result",
- call_kind="math_summary",
- trace_role="summarize",
- trace_kind="llm_output",
- ),
- ):
- _chunks.append(_c)
- response = "".join(_chunks)
- return SummaryPayload.model_validate(extract_json_object(response))
+
+ async def _run(reasoning_effort: str | None) -> str:
+ chunks: list[str] = []
+ async for chunk in self.stream_llm(
+ user_prompt=user_prompt,
+ system_prompt=system_prompt,
+ response_format={"type": "json_object"},
+ reasoning_effort=reasoning_effort,
+ stage="summary",
+ trace_meta=build_trace_metadata(
+ call_id=new_call_id("math-summary"),
+ phase="summary",
+ label="Summarize result",
+ call_kind="math_summary",
+ trace_role="summarize",
+ trace_kind="llm_output",
+ ),
+ ):
+ chunks.append(chunk)
+ return "".join(chunks)
+
+ payload = await json_with_reasoning_retry(
+ _run, expected_key="summary_text", logger_instance=self.logger
+ )
+ # Rendering has already succeeded. Keep returning its artifacts if the
+ # summary remains empty after the low-effort retry.
+ return SummaryPayload.model_validate(payload)
diff --git a/tests/agents/math_animator/test_code_generator_agent.py b/tests/agents/math_animator/test_code_generator_agent.py
index 1257403e0..6e11434e0 100644
--- a/tests/agents/math_animator/test_code_generator_agent.py
+++ b/tests/agents/math_animator/test_code_generator_agent.py
@@ -9,6 +9,7 @@ from deeptutor.agents.math_animator.agents.code_generator_agent import (
GeneratedCodeOutputError,
)
from deeptutor.agents.math_animator.models import ConceptAnalysis, SceneDesign
+from deeptutor.services.llm.reasoning_params import RETRY_REASONING_EFFORT
def _agent(monkeypatch: pytest.MonkeyPatch, responses: list[str]) -> CodeGeneratorAgent:
@@ -124,6 +125,11 @@ async def test_truncated_generation_grows_the_budget_and_asks_for_less_reasoning
# Each truncated attempt buys a larger budget, capped at twice the
# configured one so the request stays inside the model's own output limit.
assert [call["max_tokens"] for call in calls] == [8000, 12000, 16000]
+ assert [call["reasoning_effort"] for call in calls] == [
+ None,
+ RETRY_REASONING_EFFORT,
+ RETRY_REASONING_EFFORT,
+ ]
assert "token limit" in str(calls[1]["user_prompt"])
assert "token limit" not in str(calls[0]["user_prompt"])
# The failure names the cause instead of "no usable code after 3 attempts".
@@ -133,6 +139,57 @@ async def test_truncated_generation_grows_the_budget_and_asks_for_less_reasoning
assert "reasoning_tokens=7800" in message
[email protected]("stage", ["generation", "repair"])
+async def test_truncated_reasoning_only_response_recovers_with_less_reasoning(
+ monkeypatch: pytest.MonkeyPatch,
+ stage: str,
+) -> None:
+ agent = _agent(monkeypatch, [])
+ monkeypatch.setattr(agent, "get_max_tokens", lambda: 8000)
+ calls: list[dict[str, object]] = []
+
+ async def fake_provider(**kwargs) -> AsyncIterator[str]:
+ calls.append(kwargs)
+ if len(calls) == 1:
+ kwargs["outcome"].finish_reason = "length"
+ kwargs["outcome"].usage = {"reasoning_tokens": 7900}
+ yield "<think>I need to plan every scene first.</think>"
+ else:
+ kwargs["outcome"].finish_reason = "stop"
+ yield '{"code":"from manim import Scene","rationale":"ok"}'
+
+ monkeypatch.setattr(agent, "stream_llm", fake_provider)
+ if stage == "generation":
+ generated = await agent.generate(
+ user_input="Animate a proof",
+ output_mode="video",
+ analysis=ConceptAnalysis(),
+ design=SceneDesign(),
+ )
+ else:
+ agent.prompts.update(
+ {
+ "retry_system": "Repair the script.",
+ "retry_user_template": (
+ "{user_input} {output_mode} {attempt} {duration_requirement} "
+ "{error_message} {current_code}"
+ ),
+ }
+ )
+ generated = await agent.repair(
+ user_input="Animate a proof",
+ output_mode="video",
+ current_code="broken code",
+ error_message="SyntaxError",
+ attempt=1,
+ )
+
+ assert generated.code == "from manim import Scene"
+ assert [call["reasoning_effort"] for call in calls] == [None, RETRY_REASONING_EFFORT]
+ assert [call["max_tokens"] for call in calls] == [8000, 12000]
+
+
@pytest.mark.asyncio
async def test_malformed_output_is_reported_as_malformed_not_as_a_budget_problem(
monkeypatch: pytest.MonkeyPatch,
diff --git a/tests/agents/math_animator/test_reasoning_retry.py b/tests/agents/math_animator/test_reasoning_retry.py
new file mode 100644
index 000000000..6a4051228
--- /dev/null
+++ b/tests/agents/math_animator/test_reasoning_retry.py
@@ -0,0 +1,143 @@
+"""A reasoning-only structured stage must not silently become an empty plan."""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterator
+
+import pytest
+
+from deeptutor.agents.math_animator.agents.concept_analysis_agent import (
+ ConceptAnalysisAgent,
+)
+from deeptutor.agents.math_animator.agents.concept_design_agent import ConceptDesignAgent
+from deeptutor.agents.math_animator.agents.summary_agent import SummaryAgent
+from deeptutor.agents.math_animator.models import ConceptAnalysis, RenderResult, SceneDesign
+from deeptutor.services.llm.reasoning_params import RETRY_REASONING_EFFORT
+
+
[email protected]("stage", ["analysis", "design", "summary"])
+async def test_structured_manim_stage_retries_reasoning_only_response(
+ monkeypatch: pytest.MonkeyPatch, stage: str
+) -> None:
+ if stage == "analysis":
+ agent = ConceptAnalysisAgent()
+ agent.prompts = {
+ "system": "Analyze the concept.",
+ "user_template": (
+ "{user_input} {history_context} {output_mode} {style_hint} {reference_count}"
+ ),
+ }
+ call = agent.process(
+ user_input="Explain a circle",
+ history_context="",
+ output_mode="video",
+ style_hint="",
+ attachments=[],
+ )
+ answer = '{"learning_goal":"understand a circle","visual_targets":["radius"]}'
+ expected = ("learning_goal", "understand a circle")
+ elif stage == "design":
+ agent = ConceptDesignAgent()
+ agent.prompts = {
+ "system": "Design the scene.",
+ "user_template": "{user_input} {output_mode} {style_hint} {analysis_json}",
+ }
+ call = agent.process(
+ user_input="Explain a circle",
+ output_mode="video",
+ analysis=ConceptAnalysis(learning_goal="understand a circle"),
+ style_hint="",
+ )
+ answer = '{"scene_outline":["draw a circle"],"title":"Circle"}'
+ expected = ("scene_outline", ["draw a circle"])
+ else:
+ agent = SummaryAgent()
+ agent.prompts = {
+ "system": "Summarize the result.",
+ "user_template": (
+ "{user_input} {output_mode} {analysis_json} {design_json} {render_json}"
+ ),
+ }
+ call = agent.process(
+ user_input="Explain a circle",
+ output_mode="video",
+ analysis=ConceptAnalysis(learning_goal="understand a circle"),
+ design=SceneDesign(scene_outline=["draw a circle"]),
+ render_result=RenderResult(output_mode="video"),
+ )
+ answer = '{"summary_text":"A circle animation is ready."}'
+ expected = ("summary_text", "A circle animation is ready.")
+
+ calls: list[dict[str, object]] = []
+
+ async def fake_provider(**kwargs) -> AsyncIterator[str]:
+ calls.append(kwargs)
+ yield "<think>I will plan the animation first.</think>" if len(calls) == 1 else answer
+
+ monkeypatch.setattr(agent, "stream_llm", fake_provider)
+ result = await call
+
+ assert getattr(result, expected[0]) == expected[1]
+ assert [entry["reasoning_effort"] for entry in calls] == [
+ None,
+ RETRY_REASONING_EFFORT,
+ ]
+ assert calls[0]["trace_meta"]["call_id"] != calls[1]["trace_meta"]["call_id"]
+ if stage == "analysis":
+ assert [message["role"] for message in calls[1]["messages"]] == ["system", "user"]
+
+
+async def test_analysis_does_not_accept_two_empty_reasoning_only_responses(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ agent = ConceptAnalysisAgent()
+ agent.prompts = {
+ "system": "Analyze the concept.",
+ "user_template": (
+ "{user_input} {history_context} {output_mode} {style_hint} {reference_count}"
+ ),
+ }
+
+ async def fake_provider(**_kwargs) -> AsyncIterator[str]:
+ yield "<think>Still thinking.</think>"
+
+ monkeypatch.setattr(agent, "stream_llm", fake_provider)
+ with pytest.raises(ValueError, match="concept analysis returned no learning goal"):
+ await agent.process(
+ user_input="Explain a circle",
+ history_context="",
+ output_mode="video",
+ style_hint="",
+ attachments=[],
+ )
+
+
+async def test_empty_summary_after_retry_keeps_a_completed_render(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ agent = SummaryAgent()
+ agent.prompts = {
+ "system": "Summarize the result.",
+ "user_template": ("{user_input} {output_mode} {analysis_json} {design_json} {render_json}"),
+ }
+ calls = 0
+
+ async def fake_provider(**_kwargs) -> AsyncIterator[str]:
+ nonlocal calls
+ calls += 1
+ yield "<think>Still thinking.</think>"
+
+ monkeypatch.setattr(agent, "stream_llm", fake_provider)
+ summary = await agent.process(
+ user_input="Explain a circle",
+ output_mode="video",
+ analysis=ConceptAnalysis(learning_goal="understand a circle"),
+ design=SceneDesign(scene_outline=["draw a circle"]),
+ render_result=RenderResult(output_mode="video"),
+ )
+
+ assert calls == 2
+ assert summary.summary_text == ""