Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(research): identify unwritten report parts on failure (#1514)

Bingxi Zhao (Frank)cf44400fc83f
diff --git a/deeptutor/agents/research/pipeline.py b/deeptutor/agents/research/pipeline.py
index 7527fdb38..51bd8af6d 100644
--- a/deeptutor/agents/research/pipeline.py
+++ b/deeptutor/agents/research/pipeline.py
@@ -303,6 +303,10 @@ class ReportOutline:
sections: tuple[ReportSectionPlan, ...]
+class IncompleteReportError(RuntimeError):
+ """A report stopped after some parts may already have reached the reader."""
+
+
# ---------------------------------------------------------------------------
# ResearchPipeline
# ---------------------------------------------------------------------------
@@ -718,25 +722,37 @@ class ResearchPipeline:
call_id = new_call_id("research-failure")
meta = build_trace_metadata(
call_id=call_id,
- phase="researching",
- label=self._t("labels.research_step", default="Research step"),
+ phase="reporting" if isinstance(exc, IncompleteReportError) else "researching",
+ label=(
+ self._t("labels.report_failure", default="Report incomplete")
+ if isinstance(exc, IncompleteReportError)
+ else self._t("labels.research_step", default="Research step")
+ ),
call_kind="llm_final_response",
trace_id=call_id,
trace_role="response",
trace_group="stage",
)
- message = f"{type(exc).__name__}: {exc}" if str(exc) else type(exc).__name__
+ if isinstance(exc, IncompleteReportError):
+ cause = exc.__cause__ or exc
+ error_message = f"{type(cause).__name__}: {cause}"
+ visible_message = str(exc)
+ stage = "reporting"
+ else:
+ error_message = f"{type(exc).__name__}: {exc}" if str(exc) else type(exc).__name__
+ visible_message = error_message
+ stage = "researching"
await stream.error(
- message,
+ error_message,
source=SOURCE,
- stage="researching",
+ stage=stage,
metadata=merge_trace_metadata(meta, {"trace_kind": "error"}),
)
prefix = self._t("system.warning_prefix", default="⚠ ")
await stream.content(
- f"{prefix}{message}",
+ f"{prefix}{visible_message}",
source=SOURCE,
- stage="researching",
+ stage=stage,
metadata=merge_trace_metadata(meta, {"trace_kind": "llm_output"}),
)
@@ -1287,6 +1303,23 @@ class ResearchPipeline:
# them into every report-writing step.
section_count = len(outline.sections)
conclusion_number = section_count + 2
+ report_parts = [
+ f"1. {self._t('labels.report_intro_part', default='Introduction')}",
+ *(
+ f"{index + 1}. {section.title}"
+ for index, section in enumerate(outline.sections, start=1)
+ ),
+ f"{conclusion_number}. {self._t('labels.report_conclusion_part', default='Conclusion')}",
+ ]
+
+ def incomplete_error(first_missing: int) -> IncompleteReportError:
+ notice_template = self._t(
+ "notices.report_incomplete",
+ default="The report stopped before these parts could be written: {parts}. Retry to generate the full report.",
+ )
+ return IncompleteReportError(
+ notice_template.format(parts=", ".join(report_parts[first_missing:]))
+ )
section_texts: list[str] = []
@@ -1310,7 +1343,12 @@ class ResearchPipeline:
# assembled response was normalized correctly.
if section_texts:
await self._stream_report_separator(stream)
- intro = await self._write_intro(topic=topic, outline=outline, stream=stream, client=client)
+ try:
+ intro = await self._write_intro(
+ topic=topic, outline=outline, stream=stream, client=client
+ )
+ except Exception as exc:
+ raise incomplete_error(0) from exc
if intro:
section_texts.append(intro)
@@ -1318,32 +1356,38 @@ class ResearchPipeline:
for section_index, section in enumerate(outline.sections, start=1):
if section_texts:
await self._stream_report_separator(stream)
- body = await self._write_section(
- section=section,
- section_index=section_index,
- section_count=section_count,
- section_number=section_index + 1,
- topic=topic,
- outline=outline,
- blocks=blocks,
- citations=citations,
- stream=stream,
- client=client,
- )
+ try:
+ body = await self._write_section(
+ section=section,
+ section_index=section_index,
+ section_count=section_count,
+ section_number=section_index + 1,
+ topic=topic,
+ outline=outline,
+ blocks=blocks,
+ citations=citations,
+ stream=stream,
+ client=client,
+ )
+ except Exception as exc:
+ raise incomplete_error(section_index) from exc
if body:
section_texts.append(body)
section_bodies.append(body)
if section_texts:
await self._stream_report_separator(stream)
- conclusion = await self._write_conclusion(
- topic=topic,
- outline=outline,
- section_bodies=section_bodies,
- section_number=conclusion_number,
- stream=stream,
- client=client,
- )
+ try:
+ conclusion = await self._write_conclusion(
+ topic=topic,
+ outline=outline,
+ section_bodies=section_bodies,
+ section_number=conclusion_number,
+ stream=stream,
+ client=client,
+ )
+ except Exception as exc:
+ raise incomplete_error(section_count + 1) from exc
if conclusion:
section_texts.append(conclusion)
@@ -1943,7 +1987,7 @@ class ResearchPipeline:
"content": self._t(
"report.retry_complete",
default=(
- "The previous report part was empty or truncated. "
+ "The previous report part was incomplete or invalid. "
"Regenerate the complete part from its ## heading, "
"follow the required label protocol, and finish every sentence."
),
diff --git a/deeptutor/agents/research/prompts/en/pipeline.yaml b/deeptutor/agents/research/prompts/en/pipeline.yaml
index 1e9add2a4..687d28ec6 100644
--- a/deeptutor/agents/research/prompts/en/pipeline.yaml
+++ b/deeptutor/agents/research/prompts/en/pipeline.yaml
@@ -19,6 +19,9 @@ labels:
report_section: "Write section"
report_conclusion: "Write conclusion"
report_retry: "Report section was incomplete; retrying."
+ report_failure: "Report incomplete"
+ report_intro_part: "Introduction"
+ report_conclusion_part: "Conclusion"
queue_append: "Sub-topic added"
references_heading: "References"
addendum_title: "Additional findings"
@@ -41,6 +44,7 @@ system:
# --------------------- Notices (stream.progress) -----------------------
notices:
+ report_incomplete: "The report stopped before these parts could be written: {parts}. Retry to generate the full report."
protocol_retry: "The model violated the action-label protocol; retrying."
empty_tool_result: "Tool returned no result."
start_retrieval: "Starting retrieval"
@@ -209,7 +213,7 @@ note:
# =======================================================================
report:
retry_complete: |
- The previous report part was empty or truncated. Regenerate the complete part from its ## heading, follow the required label protocol, and finish every sentence. Do not comment on the retry.
+ The previous report part was incomplete or invalid. Regenerate the complete part from its ## heading, follow the required label protocol, and finish every sentence. Do not comment on the retry.
outline:
system: |
You are planning the final report's structure from a pile of finished research blocks. Each block has an id, a sub-topic title, and a short knowledge preview.
diff --git a/deeptutor/agents/research/prompts/zh/pipeline.yaml b/deeptutor/agents/research/prompts/zh/pipeline.yaml
index e5f6c1a7a..26f90713b 100644
--- a/deeptutor/agents/research/prompts/zh/pipeline.yaml
+++ b/deeptutor/agents/research/prompts/zh/pipeline.yaml
@@ -14,6 +14,9 @@ labels:
report_section: "撰写章节"
report_conclusion: "撰写结论"
report_retry: "报告段落不完整,正在重试。"
+ report_failure: "报告未完成"
+ report_intro_part: "引言"
+ report_conclusion_part: "结论"
queue_append: "新增子主题"
references_heading: "参考资料"
addendum_title: "补充发现"
@@ -33,6 +36,7 @@ system:
obsidian_kb_system_note: "用户已挂载知识库 {kb_name},这是一个只读 Obsidian vault。请使用 obsidian_search、obsidian_list 和 obsidian_read 收集证据。不要调用 rag。"
notices:
+ report_incomplete: "报告未能完成,以下部分尚未写出:{parts}。请重试以生成完整报告。"
protocol_retry: "模型未遵循标签协议,正在重试。"
empty_tool_result: "工具未返回任何结果。"
start_retrieval: "开始检索"
@@ -200,7 +204,7 @@ note:
# =======================================================================
report:
retry_complete: |
- 上一次报告段落为空或被截断。请从本段的 ## 标题开始重新生成完整内容,严格遵守标签协议,并写完每个句子。不要解释重试过程。
+ 上一次报告段落不完整或格式无效。请从本段的 ## 标题开始重新生成完整内容,严格遵守标签协议,并写完每个句子。不要解释重试过程。
outline:
system: |
根据已完成的研究块规划最终报告的结构。每个块有 id、子主题标题,以及一段简短的知识预览。
diff --git a/tests/agents/research/test_report_stream_formatting.py b/tests/agents/research/test_report_stream_formatting.py
index 693124b3a..445b61ffd 100644
--- a/tests/agents/research/test_report_stream_formatting.py
+++ b/tests/agents/research/test_report_stream_formatting.py
@@ -9,7 +9,9 @@ import pytest
from deeptutor.agents.research.pipeline import (
_PROTOCOL_REPORT_SECTION,
LABEL_SECTION,
+ IncompleteReportError,
ReportOutline,
+ ReportSectionPlan,
ResearchPipeline,
)
from deeptutor.agents.research.utils.citation_manager import CitationManager
@@ -94,6 +96,58 @@ async def test_report_title_is_separated_before_the_streamed_introduction(
assert live_content.startswith("# Report title\n\n## 1. Introduction")
+async def test_failed_report_section_names_unwritten_parts_without_provider_error(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path,
+) -> None:
+ pipeline = _make_pipeline(monkeypatch)
+ stream = StreamBus()
+
+ async def fake_outline(self, **_kwargs):
+ return ReportOutline(
+ title="Report title",
+ sections=(
+ ReportSectionPlan("S1", "First topic", "", ()),
+ ReportSectionPlan("S2", "Second topic", "", ()),
+ ),
+ )
+
+ async def fake_intro(self, *, stream, **_kwargs):
+ await stream.content("## 1. Introduction", stage="reporting")
+ return "## 1. Introduction"
+
+ async def fake_section(self, *, section_index, stream, **_kwargs):
+ if section_index == 2:
+ raise RuntimeError("provider secret detail")
+ await stream.content("## 2. First topic", stage="reporting")
+ return "## 2. First topic"
+
+ pipeline._gen_report_outline = types.MethodType(fake_outline, pipeline)
+ pipeline._write_intro = types.MethodType(fake_intro, pipeline)
+ pipeline._write_section = types.MethodType(fake_section, pipeline)
+
+ with pytest.raises(IncompleteReportError) as failure:
+ await pipeline._write_report(
+ topic="topic",
+ blocks=[],
+ citations=CitationManager("test-report", cache_dir=tmp_path),
+ stream=stream,
+ client=None,
+ )
+ await pipeline._emit_visible_failure(stream, failure.value)
+
+ content = "".join(event.content for event in stream._history if event.type.value == "content")
+ errors = "".join(event.content for event in stream._history if event.type.value == "error")
+ assert "## 1. Introduction" in content
+ assert "## 2. First topic" in content
+ assert "3. Second topic" in content
+ assert "4. Conclusion" in content
+ assert "1. Introduction, 2. First topic" not in content
+ assert "provider secret detail" not in content
+ assert "RuntimeError: provider secret detail" in errors
+ assert isinstance(failure.value.__cause__, RuntimeError)
+
+
async def test_report_step_retries_an_idle_truncated_response_before_streaming(
monkeypatch: pytest.MonkeyPatch,
) -> None:
@@ -182,6 +236,7 @@ async def test_report_retry_replays_reasoning_from_incomplete_attempt(
assistant = next(message for message in requests[1] if message["role"] == "assistant")
assert assistant["content"] == "``SECTION``\n## 2. Partial section"
assert assistant["reasoning_content"] == "Plan the missing evidence."
+ assert "incomplete or invalid" in requests[1][-1]["content"]
async def test_report_step_rejects_empty_success_after_all_retries(