DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(book): retry output-capped structured responses
Bingxi Zhao (Frank)7f6c16ef09d5
diff --git a/deeptutor/book/agents/ideation_agent.py b/deeptutor/book/agents/ideation_agent.pyindex f646a3ca5..7f7bcc595 100644--- a/deeptutor/book/agents/ideation_agent.py+++ b/deeptutor/book/agents/ideation_agent.py@@ -12,6 +12,7 @@ from typing import Anyfrom deeptutor.agents.base_agent import BaseAgentfrom deeptutor.services.llm.structured_retry import json_with_reasoning_retry+from deeptutor.services.llm.types import StreamOutcomefrom ..inputs import IdeationContextfrom ..models import BookProposal@@ -56,14 +57,18 @@ class IdeationAgent(BaseAgent):async def _run(reasoning_effort: str | None) -> str:chunks: list[str] = []+ outcome = StreamOutcome()async for chunk in self.stream_llm(user_prompt=user_prompt,system_prompt=system_prompt,response_format={"type": "json_object"},stage="ideation",reasoning_effort=reasoning_effort,+ outcome=outcome,):chunks.append(chunk)+ if outcome.truncated:+ return ""return "".join(chunks)# ``title`` is the one field the proposal cannot be rebuilt without, sodiff --git a/deeptutor/book/agents/source_explorer.py b/deeptutor/book/agents/source_explorer.pyindex 2656c56ae..253c05184 100644--- a/deeptutor/book/agents/source_explorer.py+++ b/deeptutor/book/agents/source_explorer.py@@ -37,6 +37,7 @@ from deeptutor.agents.base_agent import BaseAgentfrom deeptutor.core.context import UnifiedContextfrom deeptutor.runtime.stream_bus import StreamBusfrom deeptutor.services.llm.structured_retry import json_with_reasoning_retry+from deeptutor.services.llm.types import StreamOutcomefrom ..models import (BookInputs,@@ -299,14 +300,18 @@ class SourceExplorer(BaseAgent):async def _run(reasoning_effort: str | None) -> str:chunks: list[str] = []+ outcome = StreamOutcome()async for piece in self.stream_llm(user_prompt=user_prompt,system_prompt=system_prompt,response_format={"type": "json_object"},stage="explore_queries",reasoning_effort=reasoning_effort,+ outcome=outcome,):chunks.append(piece)+ if outcome.truncated:+ return ""return "".join(chunks)try:@@ -701,14 +706,18 @@ class SourceExplorer(BaseAgent):async def _run(reasoning_effort: str | None) -> str:buf: list[str] = []+ outcome = StreamOutcome()async for piece in self.stream_llm(user_prompt=user_prompt,system_prompt=system_prompt,response_format={"type": "json_object"},stage="explore_summary",reasoning_effort=reasoning_effort,+ outcome=outcome,):buf.append(piece)+ if outcome.truncated:+ return ""return "".join(buf)try:diff --git a/deeptutor/book/agents/spine_synthesizer.py b/deeptutor/book/agents/spine_synthesizer.pyindex 4f83565da..4032b3d32 100644--- a/deeptutor/book/agents/spine_synthesizer.py+++ b/deeptutor/book/agents/spine_synthesizer.py@@ -36,6 +36,7 @@ from typing import Anyfrom deeptutor.agents.base_agent import BaseAgentfrom deeptutor.services.llm.structured_retry import json_with_reasoning_retry+from deeptutor.services.llm.types import StreamOutcomefrom ..models import (BookProposal,@@ -265,17 +266,23 @@ class SpineSynthesizer(BaseAgent):system_prompt = system_prompt.rstrip() + language_directive(self.language)async def _run(reasoning_effort: str | None) -> str:- # Blocking rather than streamed: nothing consumes the partial JSON,- # and a reasoning model's <think> prelude never reaches the parser- # this way, so a truncated spine cannot collapse to one placeholder- # chapter (#707).- return await self.call_llm(+ # Collect the whole stream before parsing. Its terminal reason+ # distinguishes a capped, repairable fragment from a finished JSON+ # object; accepting that fragment can silently lose chapters.+ outcome = StreamOutcome()+ chunks: list[str] = []+ async for chunk in self.stream_llm(user_prompt=user_prompt,system_prompt=system_prompt,response_format={"type": "json_object"},stage=stage,reasoning_effort=reasoning_effort,- )+ outcome=outcome,+ ):+ chunks.append(chunk)+ if outcome.truncated:+ return ""+ return "".join(chunks)try:# A reasoning model can spend the whole budget thinking and returndiff --git a/deeptutor/book/blocks/_llm_writer.py b/deeptutor/book/blocks/_llm_writer.pyindex 478896815..4ea787af8 100644--- a/deeptutor/book/blocks/_llm_writer.py+++ b/deeptutor/book/blocks/_llm_writer.py@@ -18,8 +18,12 @@ from deeptutor.services.llm import (from deeptutor.services.llm import (complete as llm_complete,)+from deeptutor.services.llm import (+ stream as llm_stream,+)from deeptutor.services.llm.reasoning_params import RETRY_REASONING_EFFORTfrom deeptutor.services.llm.structured_retry import json_payload_is_usable+from deeptutor.services.llm.types import StreamOutcomefrom deeptutor.services.prompt.language import append_language_directivefrom deeptutor.utils.json_parser import parse_json_response@@ -33,6 +37,7 @@ async def llm_text(response_format: dict[str, Any] | None = None,language: str | None = None,reasoning_effort: str | None = None,+ outcome: StreamOutcome | None = None,) -> str:"""Run an LLM completion for a Book block / agent.@@ -54,16 +59,21 @@ async def llm_text(kwargs["response_format"] = response_formatif reasoning_effort is not None:kwargs["reasoning_effort"] = reasoning_effort- response = await llm_complete(- prompt=user_prompt,- system_prompt=system_prompt,- model=model,- api_key=config.api_key,- base_url=config.base_url,- api_version=getattr(config, "api_version", None),- binding=binding,+ call_kwargs = {+ "prompt": user_prompt,+ "system_prompt": system_prompt,+ "model": model,+ "api_key": config.api_key,+ "base_url": config.base_url,+ "api_version": getattr(config, "api_version", None),+ "binding": binding,**kwargs,- )+ }+ if outcome is None:+ response = await llm_complete(**call_kwargs)+ else:+ chunks = [chunk async for chunk in llm_stream(**call_kwargs, outcome=outcome)]+ response = "".join(chunks)return clean_thinking_tags(response, binding, model).strip()@@ -139,7 +149,8 @@ async def llm_json(Reasoning models can spend the whole response budget on hidden/scratchpadtokens and leave the visible JSON object empty. For structured book blockswe first honor the configured reasoning mode, then retry once with low- reasoning effort if parsing fails or the expected top-level key is missing+ reasoning effort if the provider stops at the output cap, parsing fails,+ or the expected top-level key is missing— the same rule the Book pipeline agents apply via:func:`deeptutor.services.llm.structured_retry.json_with_reasoning_retry`.@@ -152,6 +163,7 @@ async def llm_json("""async def _once(reasoning_effort: str | None) -> dict[str, Any]:+ outcome = StreamOutcome()raw = await llm_text(user_prompt=user_prompt,system_prompt=system_prompt,@@ -160,7 +172,10 @@ async def llm_json(temperature=temperature,language=language,reasoning_effort=reasoning_effort,+ outcome=outcome,)+ if outcome.truncated:+ return {}# First pass: strip thinking preamble (Qwen outputs thinking text# even with json_object format), then let parse_json_response handle# what remains via its json-repair fallback.diff --git a/tests/book/test_reasoning_output_cap.py b/tests/book/test_reasoning_output_cap.pynew file mode 100644index 000000000..b7fc9ff03--- /dev/null+++ b/tests/book/test_reasoning_output_cap.py@@ -0,0 +1,204 @@+"""A capped reasoning response must not become a partial book plan."""++from __future__ import annotations++from types import SimpleNamespace+from typing import Any++import pytest++from deeptutor.agents.base_agent import BaseAgent+from deeptutor.book.agents.ideation_agent import IdeationAgent+from deeptutor.book.agents.source_explorer import SourceExplorer+from deeptutor.book.agents.spine_synthesizer import SpineSynthesizer+from deeptutor.book.blocks import _llm_writer+from deeptutor.book.inputs import IdeationContext+from deeptutor.book.models import BookInputs, BookProposal+++async def test_ideation_retries_capped_json_with_a_repairable_title(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ efforts: list[str | None] = []++ async def fake_stream(self: BaseAgent, **kwargs: Any):+ efforts.append(kwargs.get("reasoning_effort"))+ outcome = kwargs.get("outcome")+ if len(efforts) == 1:+ if outcome is not None:+ outcome.finish_reason = "length"+ yield '{"title":"Partial","estimated_chapters":'+ else:+ if outcome is not None:+ outcome.finish_reason = "stop"+ yield '{"title":"Complete","estimated_chapters":4}'++ monkeypatch.setattr(BaseAgent, "stream_llm", fake_stream)++ proposal = await IdeationAgent().process(ideation_context=IdeationContext(user_intent="Math"))++ assert proposal.title == "Complete"+ assert efforts == [None, "low"]+++async def test_source_exploration_retries_capped_queries_before_summarising(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ calls: list[tuple[str, str | None]] = []++ async def fake_stream(self: BaseAgent, **kwargs: Any):+ stage = kwargs["stage"]+ effort = kwargs.get("reasoning_effort")+ calls.append((stage, effort))+ outcome = kwargs["outcome"]+ if stage == "explore_queries" and effort is None:+ outcome.finish_reason = "length"+ yield '{"queries":["partial"],"unfinished":'+ elif stage == "explore_queries":+ outcome.finish_reason = "stop"+ yield '{"queries":["complete first","complete second"]}'+ else:+ outcome.finish_reason = "stop"+ yield '{"summary":"Source summary","candidate_concepts":["math"]}'++ monkeypatch.setattr(BaseAgent, "stream_llm", fake_stream)++ report = await SourceExplorer().explore(+ book_id="book-1",+ proposal=BookProposal(title="Math"),+ inputs=BookInputs(user_intent="Math", source_context="A source about mathematics."),+ )++ assert report.queries == ["complete first", "complete second"]+ assert report.summary == "Source summary"+ assert calls == [+ ("explore_queries", None),+ ("explore_queries", "low"),+ ("explore_summary", None),+ ]+++async def test_spine_retries_capped_json_with_a_repairable_chapter(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ efforts: list[str | None] = []++ async def fake_stream(self: BaseAgent, **kwargs: Any):+ efforts.append(kwargs.get("reasoning_effort"))+ outcome = kwargs.get("outcome")+ if len(efforts) == 1:+ if outcome is not None:+ outcome.finish_reason = "max_tokens"+ yield '{"chapters":[{"title":"Partial"}],"concept_graph":'+ else:+ if outcome is not None:+ outcome.finish_reason = "stop"+ yield '{"chapters":[{"title":"First"},{"title":"Second"}]}'++ async def fake_call_llm(self: BaseAgent, **_kwargs: Any) -> str:+ return '{"chapters":[{"title":"Partial"}],"concept_graph":'++ monkeypatch.setattr(BaseAgent, "stream_llm", fake_stream)+ monkeypatch.setattr(BaseAgent, "call_llm", fake_call_llm)++ payload = await SpineSynthesizer()._call_json(+ system_prompt="Design a spine.",+ user_prompt="About mathematics.",+ stage="spine_draft",+ expected_key="chapters",+ )++ assert [chapter["title"] for chapter in payload["chapters"]] == ["First", "Second"]+ assert efforts == [None, "low"]+++async def test_spine_stream_does_not_parse_json_inside_hidden_reasoning(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def fake_stream(self: BaseAgent, **kwargs: Any):+ kwargs["outcome"].finish_reason = "stop"+ yield '<think>{"chapters":[{"title":"Internal draft"}]}</think>'+ yield '{"chapters":[{"title":"Reader draft"}]}'++ monkeypatch.setattr(BaseAgent, "stream_llm", fake_stream)++ payload = await SpineSynthesizer()._call_json(+ system_prompt="Design a spine.",+ user_prompt="About mathematics.",+ stage="spine_draft",+ expected_key="chapters",+ )++ assert payload["chapters"] == [{"title": "Reader draft"}]+++async def test_book_block_json_retries_capped_partial_list(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ efforts: list[str | None] = []++ async def fake_llm_text(**kwargs: Any) -> str:+ efforts.append(kwargs.get("reasoning_effort"))+ outcome = kwargs.get("outcome")+ if len(efforts) == 1:+ if outcome is not None:+ outcome.finish_reason = "max_output_tokens"+ return '{"events":[{"date":"2026"}],"notes":'+ if outcome is not None:+ outcome.finish_reason = "stop"+ return '{"events":[{"date":"2026"},{"date":"2027"}]}'++ monkeypatch.setattr(_llm_writer, "llm_text", fake_llm_text)++ payload = await _llm_writer.llm_json(+ user_prompt="Make a timeline.", system_prompt="system", expected_key="events"+ )++ assert len(payload["events"]) == 2+ assert efforts == [None, "low"]+++async def test_book_block_json_uses_stream_terminal_status(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ calls: list[dict[str, Any]] = []++ async def fake_stream(**kwargs: Any):+ calls.append(kwargs)+ kwargs["outcome"].finish_reason = "length"+ yield "<think>private reasoning</think>"+ yield '{"events":['++ async def unexpected_complete(**_kwargs: Any) -> str:+ raise AssertionError("structured block calls need the terminal stream status")++ monkeypatch.setattr(+ _llm_writer,+ "get_llm_config",+ lambda: SimpleNamespace(+ model="reasoning-model",+ binding="openai",+ api_key="test",+ base_url="https://test.invalid",+ api_version=None,+ ),+ )+ monkeypatch.setattr(_llm_writer, "llm_complete", unexpected_complete)+ monkeypatch.setattr(_llm_writer, "llm_stream", fake_stream, raising=False)++ from deeptutor.services.llm.types import StreamOutcome++ outcome = StreamOutcome()+ text = await _llm_writer.llm_text(+ user_prompt="Make a timeline.", system_prompt="system", outcome=outcome+ )++ assert text == '{"events":['+ assert outcome.truncated+ assert len(calls) == 1diff --git a/tests/book/test_spine_synthesizer_llm_call.py b/tests/book/test_spine_synthesizer_llm_call.pyindex b08d4a5c8..08aca2c33 100644--- a/tests/book/test_spine_synthesizer_llm_call.py+++ b/tests/book/test_spine_synthesizer_llm_call.py@@ -1,8 +1,8 @@"""The spine synthesizer must reach the LLM through its own agent seam.-``_call_json`` is blocking on purpose — nothing consumes a partial spine, and a-reasoning model's ``<think>`` prelude never reaches the parser this way (#707).-But it has to stay on ``BaseAgent.call_llm``: calling the factory directly drops+``_call_json`` collects the whole stream before parsing so a capped partial+spine cannot masquerade as a usable JSON payload. It stays on+``BaseAgent.stream_llm``: calling the factory directly dropsthe trace event the Book Activity panel renders and the per-agentapi_key / base_url / binding routing."""@@ -19,16 +19,16 @@ from deeptutor.book.agents.spine_synthesizer import SpineSynthesizer@pytest.mark.asyncio-async def test_call_json_goes_through_call_llm_with_its_stage(+async def test_call_json_goes_through_stream_llm_with_its_stage(monkeypatch: pytest.MonkeyPatch,) -> None:recorded: dict[str, Any] = {}- async def _call_llm(self: BaseAgent, **kwargs: Any) -> str:+ async def _stream_llm(self: BaseAgent, **kwargs: Any):recorded.update(kwargs)- return json.dumps({"chapters": [{"title": "Vectors"}]})+ yield json.dumps({"chapters": [{"title": "Vectors"}]})- monkeypatch.setattr(BaseAgent, "call_llm", _call_llm)+ monkeypatch.setattr(BaseAgent, "stream_llm", _stream_llm)payload = await SpineSynthesizer(language="ja")._call_json(system_prompt="Design a spine.",@@ -39,6 +39,7 @@ async def test_call_json_goes_through_call_llm_with_its_stage(assert payload == {"chapters": [{"title": "Vectors"}]}assert recorded["stage"] == "spine_draft"assert recorded["response_format"] == {"type": "json_object"}+ assert recorded["outcome"] is not None# The language directive rides on the system prompt, so a non-en/zh book# still tells the model which language to write in (#712).assert "日本語" in recorded["system_prompt"]@@ -48,10 +49,11 @@ async def test_call_json_goes_through_call_llm_with_its_stage(async def test_call_json_returns_an_empty_payload_when_the_call_fails(monkeypatch: pytest.MonkeyPatch,) -> None:- async def _boom(self: BaseAgent, **_kwargs: Any) -> str:+ async def _boom(self: BaseAgent, **_kwargs: Any):+ yield ""raise RuntimeError("provider down")- monkeypatch.setattr(BaseAgent, "call_llm", _boom)+ monkeypatch.setattr(BaseAgent, "stream_llm", _boom)payload = await SpineSynthesizer()._call_json(system_prompt="Design a spine.",@@ -76,13 +78,14 @@ async def test_call_json_retries_at_low_effort_when_reasoning_ate_the_budget("""efforts: list[str | None] = []- async def _call_llm(self: BaseAgent, **kwargs: Any) -> str:+ async def _stream_llm(self: BaseAgent, **kwargs: Any):efforts.append(kwargs.get("reasoning_effort"))if len(efforts) == 1:- return ""- return json.dumps({"chapters": [{"title": "Vectors"}, {"title": "Matrices"}]})+ yield ""+ else:+ yield json.dumps({"chapters": [{"title": "Vectors"}, {"title": "Matrices"}]})- monkeypatch.setattr(BaseAgent, "call_llm", _call_llm)+ monkeypatch.setattr(BaseAgent, "stream_llm", _stream_llm)payload = await SpineSynthesizer()._call_json(system_prompt="Design a spine.",@@ -108,13 +111,14 @@ async def test_call_json_retries_when_the_payload_lost_its_chapters("""calls: list[str | None] = []- async def _call_llm(self: BaseAgent, **kwargs: Any) -> str:+ async def _stream_llm(self: BaseAgent, **kwargs: Any):calls.append(kwargs.get("reasoning_effort"))if len(calls) == 1:- return json.dumps({"concept_graph": {"nodes": [], "edges": []}})- return json.dumps({"chapters": [{"title": "Vectors"}]})+ yield json.dumps({"concept_graph": {"nodes": [], "edges": []}})+ else:+ yield json.dumps({"chapters": [{"title": "Vectors"}]})- monkeypatch.setattr(BaseAgent, "call_llm", _call_llm)+ monkeypatch.setattr(BaseAgent, "stream_llm", _stream_llm)payload = await SpineSynthesizer()._call_json(system_prompt="Design a spine.",@@ -133,11 +137,11 @@ async def test_call_json_does_not_retry_a_good_first_answer() -> None:calls: list[str | None] = []- async def _call_llm(self: BaseAgent, **kwargs: Any) -> str:+ async def _stream_llm(self: BaseAgent, **kwargs: Any):calls.append(kwargs.get("reasoning_effort"))- return json.dumps({"chapters": [{"title": "Vectors"}]})+ yield json.dumps({"chapters": [{"title": "Vectors"}]})- monkeypatch.setattr(BaseAgent, "call_llm", _call_llm)+ monkeypatch.setattr(BaseAgent, "stream_llm", _stream_llm)await SpineSynthesizer()._call_json(system_prompt="Design a spine.",