Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(book): retry output-capped structured responses

Bingxi Zhao (Frank)7f6c16ef09d5
diff --git a/deeptutor/book/agents/ideation_agent.py b/deeptutor/book/agents/ideation_agent.py
index f646a3ca5..7f7bcc595 100644
--- a/deeptutor/book/agents/ideation_agent.py
+++ b/deeptutor/book/agents/ideation_agent.py
@@ -12,6 +12,7 @@ from typing import Any
from deeptutor.agents.base_agent import BaseAgent
from deeptutor.services.llm.structured_retry import json_with_reasoning_retry
+from deeptutor.services.llm.types import StreamOutcome
from ..inputs import IdeationContext
from ..models import BookProposal
@@ -56,14 +57,18 @@ class IdeationAgent(BaseAgent):
async def _run(reasoning_effort: str | None) -> str:
chunks: list[str] = []
+ outcome = StreamOutcome()
async for chunk in self.stream_llm(
user_prompt=user_prompt,
system_prompt=system_prompt,
response_format={"type": "json_object"},
stage="ideation",
reasoning_effort=reasoning_effort,
+ outcome=outcome,
):
chunks.append(chunk)
+ if outcome.truncated:
+ return ""
return "".join(chunks)
# ``title`` is the one field the proposal cannot be rebuilt without, so
diff --git a/deeptutor/book/agents/source_explorer.py b/deeptutor/book/agents/source_explorer.py
index 2656c56ae..253c05184 100644
--- a/deeptutor/book/agents/source_explorer.py
+++ b/deeptutor/book/agents/source_explorer.py
@@ -37,6 +37,7 @@ from deeptutor.agents.base_agent import BaseAgent
from deeptutor.core.context import UnifiedContext
from deeptutor.runtime.stream_bus import StreamBus
from deeptutor.services.llm.structured_retry import json_with_reasoning_retry
+from deeptutor.services.llm.types import StreamOutcome
from ..models import (
BookInputs,
@@ -299,14 +300,18 @@ class SourceExplorer(BaseAgent):
async def _run(reasoning_effort: str | None) -> str:
chunks: list[str] = []
+ outcome = StreamOutcome()
async for piece in self.stream_llm(
user_prompt=user_prompt,
system_prompt=system_prompt,
response_format={"type": "json_object"},
stage="explore_queries",
reasoning_effort=reasoning_effort,
+ outcome=outcome,
):
chunks.append(piece)
+ if outcome.truncated:
+ return ""
return "".join(chunks)
try:
@@ -701,14 +706,18 @@ class SourceExplorer(BaseAgent):
async def _run(reasoning_effort: str | None) -> str:
buf: list[str] = []
+ outcome = StreamOutcome()
async for piece in self.stream_llm(
user_prompt=user_prompt,
system_prompt=system_prompt,
response_format={"type": "json_object"},
stage="explore_summary",
reasoning_effort=reasoning_effort,
+ outcome=outcome,
):
buf.append(piece)
+ if outcome.truncated:
+ return ""
return "".join(buf)
try:
diff --git a/deeptutor/book/agents/spine_synthesizer.py b/deeptutor/book/agents/spine_synthesizer.py
index 4f83565da..4032b3d32 100644
--- a/deeptutor/book/agents/spine_synthesizer.py
+++ b/deeptutor/book/agents/spine_synthesizer.py
@@ -36,6 +36,7 @@ from typing import Any
from deeptutor.agents.base_agent import BaseAgent
from deeptutor.services.llm.structured_retry import json_with_reasoning_retry
+from deeptutor.services.llm.types import StreamOutcome
from ..models import (
BookProposal,
@@ -265,17 +266,23 @@ class SpineSynthesizer(BaseAgent):
system_prompt = system_prompt.rstrip() + language_directive(self.language)
async def _run(reasoning_effort: str | None) -> str:
- # Blocking rather than streamed: nothing consumes the partial JSON,
- # and a reasoning model's <think> prelude never reaches the parser
- # this way, so a truncated spine cannot collapse to one placeholder
- # chapter (#707).
- return await self.call_llm(
+ # Collect the whole stream before parsing. Its terminal reason
+ # distinguishes a capped, repairable fragment from a finished JSON
+ # object; accepting that fragment can silently lose chapters.
+ outcome = StreamOutcome()
+ chunks: list[str] = []
+ async for chunk in self.stream_llm(
user_prompt=user_prompt,
system_prompt=system_prompt,
response_format={"type": "json_object"},
stage=stage,
reasoning_effort=reasoning_effort,
- )
+ outcome=outcome,
+ ):
+ chunks.append(chunk)
+ if outcome.truncated:
+ return ""
+ return "".join(chunks)
try:
# A reasoning model can spend the whole budget thinking and return
diff --git a/deeptutor/book/blocks/_llm_writer.py b/deeptutor/book/blocks/_llm_writer.py
index 478896815..4ea787af8 100644
--- a/deeptutor/book/blocks/_llm_writer.py
+++ b/deeptutor/book/blocks/_llm_writer.py
@@ -18,8 +18,12 @@ from deeptutor.services.llm import (
from deeptutor.services.llm import (
complete as llm_complete,
)
+from deeptutor.services.llm import (
+ stream as llm_stream,
+)
from deeptutor.services.llm.reasoning_params import RETRY_REASONING_EFFORT
from deeptutor.services.llm.structured_retry import json_payload_is_usable
+from deeptutor.services.llm.types import StreamOutcome
from deeptutor.services.prompt.language import append_language_directive
from deeptutor.utils.json_parser import parse_json_response
@@ -33,6 +37,7 @@ async def llm_text(
response_format: dict[str, Any] | None = None,
language: str | None = None,
reasoning_effort: str | None = None,
+ outcome: StreamOutcome | None = None,
) -> str:
"""Run an LLM completion for a Book block / agent.
@@ -54,16 +59,21 @@ async def llm_text(
kwargs["response_format"] = response_format
if reasoning_effort is not None:
kwargs["reasoning_effort"] = reasoning_effort
- response = await llm_complete(
- prompt=user_prompt,
- system_prompt=system_prompt,
- model=model,
- api_key=config.api_key,
- base_url=config.base_url,
- api_version=getattr(config, "api_version", None),
- binding=binding,
+ call_kwargs = {
+ "prompt": user_prompt,
+ "system_prompt": system_prompt,
+ "model": model,
+ "api_key": config.api_key,
+ "base_url": config.base_url,
+ "api_version": getattr(config, "api_version", None),
+ "binding": binding,
**kwargs,
- )
+ }
+ if outcome is None:
+ response = await llm_complete(**call_kwargs)
+ else:
+ chunks = [chunk async for chunk in llm_stream(**call_kwargs, outcome=outcome)]
+ response = "".join(chunks)
return clean_thinking_tags(response, binding, model).strip()
@@ -139,7 +149,8 @@ async def llm_json(
Reasoning models can spend the whole response budget on hidden/scratchpad
tokens and leave the visible JSON object empty. For structured book blocks
we first honor the configured reasoning mode, then retry once with low
- reasoning effort if parsing fails or the expected top-level key is missing
+ reasoning effort if the provider stops at the output cap, parsing fails,
+ or the expected top-level key is missing
— the same rule the Book pipeline agents apply via
:func:`deeptutor.services.llm.structured_retry.json_with_reasoning_retry`.
@@ -152,6 +163,7 @@ async def llm_json(
"""
async def _once(reasoning_effort: str | None) -> dict[str, Any]:
+ outcome = StreamOutcome()
raw = await llm_text(
user_prompt=user_prompt,
system_prompt=system_prompt,
@@ -160,7 +172,10 @@ async def llm_json(
temperature=temperature,
language=language,
reasoning_effort=reasoning_effort,
+ outcome=outcome,
)
+ if outcome.truncated:
+ return {}
# First pass: strip thinking preamble (Qwen outputs thinking text
# even with json_object format), then let parse_json_response handle
# what remains via its json-repair fallback.
diff --git a/tests/book/test_reasoning_output_cap.py b/tests/book/test_reasoning_output_cap.py
new file mode 100644
index 000000000..b7fc9ff03
--- /dev/null
+++ b/tests/book/test_reasoning_output_cap.py
@@ -0,0 +1,204 @@
+"""A capped reasoning response must not become a partial book plan."""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+from typing import Any
+
+import pytest
+
+from deeptutor.agents.base_agent import BaseAgent
+from deeptutor.book.agents.ideation_agent import IdeationAgent
+from deeptutor.book.agents.source_explorer import SourceExplorer
+from deeptutor.book.agents.spine_synthesizer import SpineSynthesizer
+from deeptutor.book.blocks import _llm_writer
+from deeptutor.book.inputs import IdeationContext
+from deeptutor.book.models import BookInputs, BookProposal
+
+
+async def test_ideation_retries_capped_json_with_a_repairable_title(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ efforts: list[str | None] = []
+
+ async def fake_stream(self: BaseAgent, **kwargs: Any):
+ efforts.append(kwargs.get("reasoning_effort"))
+ outcome = kwargs.get("outcome")
+ if len(efforts) == 1:
+ if outcome is not None:
+ outcome.finish_reason = "length"
+ yield '{"title":"Partial","estimated_chapters":'
+ else:
+ if outcome is not None:
+ outcome.finish_reason = "stop"
+ yield '{"title":"Complete","estimated_chapters":4}'
+
+ monkeypatch.setattr(BaseAgent, "stream_llm", fake_stream)
+
+ proposal = await IdeationAgent().process(ideation_context=IdeationContext(user_intent="Math"))
+
+ assert proposal.title == "Complete"
+ assert efforts == [None, "low"]
+
+
+async def test_source_exploration_retries_capped_queries_before_summarising(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ calls: list[tuple[str, str | None]] = []
+
+ async def fake_stream(self: BaseAgent, **kwargs: Any):
+ stage = kwargs["stage"]
+ effort = kwargs.get("reasoning_effort")
+ calls.append((stage, effort))
+ outcome = kwargs["outcome"]
+ if stage == "explore_queries" and effort is None:
+ outcome.finish_reason = "length"
+ yield '{"queries":["partial"],"unfinished":'
+ elif stage == "explore_queries":
+ outcome.finish_reason = "stop"
+ yield '{"queries":["complete first","complete second"]}'
+ else:
+ outcome.finish_reason = "stop"
+ yield '{"summary":"Source summary","candidate_concepts":["math"]}'
+
+ monkeypatch.setattr(BaseAgent, "stream_llm", fake_stream)
+
+ report = await SourceExplorer().explore(
+ book_id="book-1",
+ proposal=BookProposal(title="Math"),
+ inputs=BookInputs(user_intent="Math", source_context="A source about mathematics."),
+ )
+
+ assert report.queries == ["complete first", "complete second"]
+ assert report.summary == "Source summary"
+ assert calls == [
+ ("explore_queries", None),
+ ("explore_queries", "low"),
+ ("explore_summary", None),
+ ]
+
+
+async def test_spine_retries_capped_json_with_a_repairable_chapter(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ efforts: list[str | None] = []
+
+ async def fake_stream(self: BaseAgent, **kwargs: Any):
+ efforts.append(kwargs.get("reasoning_effort"))
+ outcome = kwargs.get("outcome")
+ if len(efforts) == 1:
+ if outcome is not None:
+ outcome.finish_reason = "max_tokens"
+ yield '{"chapters":[{"title":"Partial"}],"concept_graph":'
+ else:
+ if outcome is not None:
+ outcome.finish_reason = "stop"
+ yield '{"chapters":[{"title":"First"},{"title":"Second"}]}'
+
+ async def fake_call_llm(self: BaseAgent, **_kwargs: Any) -> str:
+ return '{"chapters":[{"title":"Partial"}],"concept_graph":'
+
+ monkeypatch.setattr(BaseAgent, "stream_llm", fake_stream)
+ monkeypatch.setattr(BaseAgent, "call_llm", fake_call_llm)
+
+ payload = await SpineSynthesizer()._call_json(
+ system_prompt="Design a spine.",
+ user_prompt="About mathematics.",
+ stage="spine_draft",
+ expected_key="chapters",
+ )
+
+ assert [chapter["title"] for chapter in payload["chapters"]] == ["First", "Second"]
+ assert efforts == [None, "low"]
+
+
+async def test_spine_stream_does_not_parse_json_inside_hidden_reasoning(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def fake_stream(self: BaseAgent, **kwargs: Any):
+ kwargs["outcome"].finish_reason = "stop"
+ yield '<think>{"chapters":[{"title":"Internal draft"}]}</think>'
+ yield '{"chapters":[{"title":"Reader draft"}]}'
+
+ monkeypatch.setattr(BaseAgent, "stream_llm", fake_stream)
+
+ payload = await SpineSynthesizer()._call_json(
+ system_prompt="Design a spine.",
+ user_prompt="About mathematics.",
+ stage="spine_draft",
+ expected_key="chapters",
+ )
+
+ assert payload["chapters"] == [{"title": "Reader draft"}]
+
+
+async def test_book_block_json_retries_capped_partial_list(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ efforts: list[str | None] = []
+
+ async def fake_llm_text(**kwargs: Any) -> str:
+ efforts.append(kwargs.get("reasoning_effort"))
+ outcome = kwargs.get("outcome")
+ if len(efforts) == 1:
+ if outcome is not None:
+ outcome.finish_reason = "max_output_tokens"
+ return '{"events":[{"date":"2026"}],"notes":'
+ if outcome is not None:
+ outcome.finish_reason = "stop"
+ return '{"events":[{"date":"2026"},{"date":"2027"}]}'
+
+ monkeypatch.setattr(_llm_writer, "llm_text", fake_llm_text)
+
+ payload = await _llm_writer.llm_json(
+ user_prompt="Make a timeline.", system_prompt="system", expected_key="events"
+ )
+
+ assert len(payload["events"]) == 2
+ assert efforts == [None, "low"]
+
+
+async def test_book_block_json_uses_stream_terminal_status(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ calls: list[dict[str, Any]] = []
+
+ async def fake_stream(**kwargs: Any):
+ calls.append(kwargs)
+ kwargs["outcome"].finish_reason = "length"
+ yield "<think>private reasoning</think>"
+ yield '{"events":['
+
+ async def unexpected_complete(**_kwargs: Any) -> str:
+ raise AssertionError("structured block calls need the terminal stream status")
+
+ monkeypatch.setattr(
+ _llm_writer,
+ "get_llm_config",
+ lambda: SimpleNamespace(
+ model="reasoning-model",
+ binding="openai",
+ api_key="test",
+ base_url="https://test.invalid",
+ api_version=None,
+ ),
+ )
+ monkeypatch.setattr(_llm_writer, "llm_complete", unexpected_complete)
+ monkeypatch.setattr(_llm_writer, "llm_stream", fake_stream, raising=False)
+
+ from deeptutor.services.llm.types import StreamOutcome
+
+ outcome = StreamOutcome()
+ text = await _llm_writer.llm_text(
+ user_prompt="Make a timeline.", system_prompt="system", outcome=outcome
+ )
+
+ assert text == '{"events":['
+ assert outcome.truncated
+ assert len(calls) == 1
diff --git a/tests/book/test_spine_synthesizer_llm_call.py b/tests/book/test_spine_synthesizer_llm_call.py
index b08d4a5c8..08aca2c33 100644
--- a/tests/book/test_spine_synthesizer_llm_call.py
+++ b/tests/book/test_spine_synthesizer_llm_call.py
@@ -1,8 +1,8 @@
"""The spine synthesizer must reach the LLM through its own agent seam.
-``_call_json`` is blocking on purpose — nothing consumes a partial spine, and a
-reasoning model's ``<think>`` prelude never reaches the parser this way (#707).
-But it has to stay on ``BaseAgent.call_llm``: calling the factory directly drops
+``_call_json`` collects the whole stream before parsing so a capped partial
+spine cannot masquerade as a usable JSON payload. It stays on
+``BaseAgent.stream_llm``: calling the factory directly drops
the trace event the Book Activity panel renders and the per-agent
api_key / base_url / binding routing.
"""
@@ -19,16 +19,16 @@ from deeptutor.book.agents.spine_synthesizer import SpineSynthesizer
@pytest.mark.asyncio
-async def test_call_json_goes_through_call_llm_with_its_stage(
+async def test_call_json_goes_through_stream_llm_with_its_stage(
monkeypatch: pytest.MonkeyPatch,
) -> None:
recorded: dict[str, Any] = {}
- async def _call_llm(self: BaseAgent, **kwargs: Any) -> str:
+ async def _stream_llm(self: BaseAgent, **kwargs: Any):
recorded.update(kwargs)
- return json.dumps({"chapters": [{"title": "Vectors"}]})
+ yield json.dumps({"chapters": [{"title": "Vectors"}]})
- monkeypatch.setattr(BaseAgent, "call_llm", _call_llm)
+ monkeypatch.setattr(BaseAgent, "stream_llm", _stream_llm)
payload = await SpineSynthesizer(language="ja")._call_json(
system_prompt="Design a spine.",
@@ -39,6 +39,7 @@ async def test_call_json_goes_through_call_llm_with_its_stage(
assert payload == {"chapters": [{"title": "Vectors"}]}
assert recorded["stage"] == "spine_draft"
assert recorded["response_format"] == {"type": "json_object"}
+ assert recorded["outcome"] is not None
# The language directive rides on the system prompt, so a non-en/zh book
# still tells the model which language to write in (#712).
assert "日本語" in recorded["system_prompt"]
@@ -48,10 +49,11 @@ async def test_call_json_goes_through_call_llm_with_its_stage(
async def test_call_json_returns_an_empty_payload_when_the_call_fails(
monkeypatch: pytest.MonkeyPatch,
) -> None:
- async def _boom(self: BaseAgent, **_kwargs: Any) -> str:
+ async def _boom(self: BaseAgent, **_kwargs: Any):
+ yield ""
raise RuntimeError("provider down")
- monkeypatch.setattr(BaseAgent, "call_llm", _boom)
+ monkeypatch.setattr(BaseAgent, "stream_llm", _boom)
payload = await SpineSynthesizer()._call_json(
system_prompt="Design a spine.",
@@ -76,13 +78,14 @@ async def test_call_json_retries_at_low_effort_when_reasoning_ate_the_budget(
"""
efforts: list[str | None] = []
- async def _call_llm(self: BaseAgent, **kwargs: Any) -> str:
+ async def _stream_llm(self: BaseAgent, **kwargs: Any):
efforts.append(kwargs.get("reasoning_effort"))
if len(efforts) == 1:
- return ""
- return json.dumps({"chapters": [{"title": "Vectors"}, {"title": "Matrices"}]})
+ yield ""
+ else:
+ yield json.dumps({"chapters": [{"title": "Vectors"}, {"title": "Matrices"}]})
- monkeypatch.setattr(BaseAgent, "call_llm", _call_llm)
+ monkeypatch.setattr(BaseAgent, "stream_llm", _stream_llm)
payload = await SpineSynthesizer()._call_json(
system_prompt="Design a spine.",
@@ -108,13 +111,14 @@ async def test_call_json_retries_when_the_payload_lost_its_chapters(
"""
calls: list[str | None] = []
- async def _call_llm(self: BaseAgent, **kwargs: Any) -> str:
+ async def _stream_llm(self: BaseAgent, **kwargs: Any):
calls.append(kwargs.get("reasoning_effort"))
if len(calls) == 1:
- return json.dumps({"concept_graph": {"nodes": [], "edges": []}})
- return json.dumps({"chapters": [{"title": "Vectors"}]})
+ yield json.dumps({"concept_graph": {"nodes": [], "edges": []}})
+ else:
+ yield json.dumps({"chapters": [{"title": "Vectors"}]})
- monkeypatch.setattr(BaseAgent, "call_llm", _call_llm)
+ monkeypatch.setattr(BaseAgent, "stream_llm", _stream_llm)
payload = await SpineSynthesizer()._call_json(
system_prompt="Design a spine.",
@@ -133,11 +137,11 @@ async def test_call_json_does_not_retry_a_good_first_answer(
) -> None:
calls: list[str | None] = []
- async def _call_llm(self: BaseAgent, **kwargs: Any) -> str:
+ async def _stream_llm(self: BaseAgent, **kwargs: Any):
calls.append(kwargs.get("reasoning_effort"))
- return json.dumps({"chapters": [{"title": "Vectors"}]})
+ yield json.dumps({"chapters": [{"title": "Vectors"}]})
- monkeypatch.setattr(BaseAgent, "call_llm", _call_llm)
+ monkeypatch.setattr(BaseAgent, "stream_llm", _stream_llm)
await SpineSynthesizer()._call_json(
system_prompt="Design a spine.",