Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

125 KB
Raw Blame
"""ResearchPipeline — agentic-engine-based replacement for the legacy
multi-agent ``ResearchPipeline``.

Phase shape:

* **Phase 1 (Rephrase)** — a mini agentic loop over ``THINK`` / ``TOOL``
  / ``FINISH`` whose only available tool is ``ask_user``. Up to 3
  ask_user rounds (each with 1-4 questions on one card). FINISH text
  is the refined research topic. May FINISH early when the user is
  unambiguous.
* **Phase 2 (Decompose)** — one ``OUTLINE`` labeled step turning the
  refined topic into N sub-topics. When ``confirmed_outline=None``
  ResearchPipeline returns the outline and exits so the capability can
  surface it as a preview; the same pipeline is invoked again with
  ``confirmed_outline`` once the user confirms.
* **Phase 3 (Research blocks)** — for each ``TopicBlock`` in the
  dynamic queue, one ``run_agentic_loop`` over ``THINK`` / ``TOOL`` /
  ``APPEND`` / ``FINISH``. ``APPEND`` is an intermediate label that
  mutates the queue (via ``_BlockLoopHost.on_intermediate``). The outer
  scheduler drains the queue in series or parallel; APPEND-added
  blocks naturally get picked up by subsequent batches.
* **Phase 4 (Reporting)** — sequence of one-shot labeled steps:
  ``OUTLINE`` (report structure) → ``INTRO`` → for each section
  ``SECTION`` → ``CONCLUSION`` → assemble. Each emits its own trace
  card and pulls evidence from :class:`CitationManager` for anchor
  injection.

Citations and the dynamic topic queue — the two features that make
deep research distinct — are preserved.
"""

from __future__ import annotations

import asyncio
from collections.abc import Awaitable
from dataclasses import dataclass, replace
import html
import logging
from pathlib import Path
import re
from typing import Any

from deeptutor.agents._shared.capability_result import emit_capability_result
from deeptutor.agents._shared.tool_composition import (
    ToolMountFlags,
    compose_enabled_tools,
    default_optional_tools,
    user_has_memory,
    user_has_notebooks,
)
from deeptutor.agents._shared.tool_runtime import (
    bind_workspace_tool_runtime,
    fallback_task_dir_from_metadata,
)
from deeptutor.agents.research.data_structures import (
    DynamicTopicQueue,
    ToolTrace,
    TopicBlock,
    TopicStatus,
)
from deeptutor.agents.research.utils.citation_manager import CitationManager
from deeptutor.core.context import Attachment, UnifiedContext
from deeptutor.core.trace import (
    build_trace_metadata,
    derive_trace_metadata,
    merge_trace_metadata,
    new_call_id,
)
from deeptutor.runtime.agentic import (
    DispatchOutcome,
    LabeledStepResult,
    LabelProtocol,
    LLMClientConfig,
    UsageTracker,
    build_completion_kwargs,
    build_openai_client,
    can_use_native_tool_calling,
    classify_label,
    dispatch_tool_calls,
    recover_finished_label,
    run_agentic_loop,
    run_labeled_step,
)
from deeptutor.runtime.agentic.messages import assistant_message
from deeptutor.runtime.agentic.tool_dispatch import (
    MAX_PARALLEL_TOOL_CALLS,
    tool_error_message_factory,
)
from deeptutor.runtime.registry.tool_registry import get_tool_registry
from deeptutor.runtime.stream_bus import StreamBus
from deeptutor.services.config import parse_language
from deeptutor.services.config.loader import get_capability_params
from deeptutor.services.llm import (
    finish_was_truncated,
    get_llm_config,
    prepare_multimodal_messages,
)
from deeptutor.services.llm.structured_retry import payload_with_reasoning_retry
from deeptutor.services.prompt import get_prompt_manager
from deeptutor.services.prompt.language import append_language_directive
from deeptutor.services.sandbox import exec_capability_available
from deeptutor.utils.json_parser import parse_json_response

logger = logging.getLogger(__name__)


SOURCE = "deep_research"

# Per-block research uses the same shared composition policy as chat
# (``compose_enabled_tools``), then narrows the result to tools that can
# produce evidence for a block-level research summary.
RESEARCH_OPTIONAL_TOOLS: list[str] = default_optional_tools()

# Read-only Obsidian tools a research block may use when the selected KB is a
# connected Obsidian vault. Write tools stay out of research blocks — the
# block loop only retrieves evidence, never mutates the vault. The vault root
# is injected server-side as ``_vault_path`` (see ``_augment_tool_kwargs``).
RESEARCH_OBSIDIAN_READ_TOOLS: tuple[str, ...] = (
    "obsidian_search",
    "obsidian_read",
    "obsidian_list",
)
RESEARCH_BLOCK_TOOL_ALLOWLIST: frozenset[str] = frozenset(
    {
        "rag",
        "web_search",
        "paper_search",
        "preprint",
        "research_audit",
        "research_lit",
        "exec",
        "workspace_list",
        "workspace_read",
        "workspace_search",
        "workspace_present",
        *RESEARCH_OBSIDIAN_READ_TOOLS,
    }
)

# ---------------------------------------------------------------------------
# Label vocabulary
# ---------------------------------------------------------------------------
LABEL_THINK = "THINK"
LABEL_TOOL = "TOOL"
LABEL_APPEND = "APPEND"
LABEL_FINISH = "FINISH"
LABEL_OUTLINE = "OUTLINE"
LABEL_INTRO = "INTRO"
LABEL_SECTION = "SECTION"
LABEL_CONCLUSION = "CONCLUSION"

# Rephrase loop: agent talks to the user through ``ask_user`` and then
# FINISHes with a refined topic statement.
_PROTOCOL_REPHRASE = LabelProtocol(
    allowed=(LABEL_THINK, LABEL_TOOL, LABEL_FINISH),
    terminal=frozenset({LABEL_FINISH}),
    intermediate=frozenset({LABEL_THINK}),
    final=frozenset({LABEL_FINISH}),
    tool_label=LABEL_TOOL,
)

# Decompose: one-shot ``OUTLINE`` payload (JSON list of sub-topics).
_PROTOCOL_DECOMPOSE = LabelProtocol(
    allowed=(LABEL_OUTLINE,),
    terminal=frozenset({LABEL_OUTLINE}),
    intermediate=frozenset(),
    final=frozenset({LABEL_OUTLINE}),
    tool_label=None,
)

# Per-block research: the headline agentic loop. ``APPEND`` is the
# intermediate label that extends the dynamic queue (handled by the
# host's ``on_intermediate`` hook).
_PROTOCOL_BLOCK = LabelProtocol(
    allowed=(LABEL_THINK, LABEL_TOOL, LABEL_APPEND, LABEL_FINISH),
    terminal=frozenset({LABEL_FINISH}),
    intermediate=frozenset({LABEL_THINK, LABEL_APPEND}),
    final=frozenset({LABEL_FINISH}),
    tool_label=LABEL_TOOL,
)

# Reporting sub-phases — each is one labeled step with one terminal
# label. Distinct names so the LLM and the trace UI both recognise
# which phase the call belongs to.
_PROTOCOL_REPORT_OUTLINE = LabelProtocol(
    allowed=(LABEL_OUTLINE,),
    terminal=frozenset({LABEL_OUTLINE}),
    intermediate=frozenset(),
    final=frozenset({LABEL_OUTLINE}),
    tool_label=None,
)
_PROTOCOL_REPORT_INTRO = LabelProtocol(
    allowed=(LABEL_INTRO,),
    terminal=frozenset({LABEL_INTRO}),
    intermediate=frozenset(),
    final=frozenset({LABEL_INTRO}),
    tool_label=None,
)
_PROTOCOL_REPORT_SECTION = LabelProtocol(
    allowed=(LABEL_SECTION,),
    terminal=frozenset({LABEL_SECTION}),
    intermediate=frozenset(),
    final=frozenset({LABEL_SECTION}),
    tool_label=None,
)
_PROTOCOL_REPORT_CONCLUSION = LabelProtocol(
    allowed=(LABEL_CONCLUSION,),
    terminal=frozenset({LABEL_CONCLUSION}),
    intermediate=frozenset(),
    final=frozenset({LABEL_CONCLUSION}),
    tool_label=None,
)
_PROTOCOL_ANSWER_NOW = LabelProtocol(
    allowed=(LABEL_FINISH,),
    terminal=frozenset({LABEL_FINISH}),
    intermediate=frozenset(),
    final=frozenset({LABEL_FINISH}),
    tool_label=None,
)

# Note Agent — single ``FINISH`` labeled step that compresses a raw tool
# result into a short dense summary for the citation sidecar.
_PROTOCOL_NOTE = LabelProtocol(
    allowed=(LABEL_FINISH,),
    terminal=frozenset({LABEL_FINISH}),
    intermediate=frozenset(),
    final=frozenset({LABEL_FINISH}),
    tool_label=None,
)

# Tools whose results get summarised + recorded in the citation manager.
# Any tool whose results carry source documents / external evidence
# should be added here.
CITABLE_TOOLS: frozenset[str] = frozenset(
    {
        "rag",
        "web_search",
        "paper_search",
        "exec",
        *RESEARCH_OBSIDIAN_READ_TOOLS,
    }
)


def _is_citable_tool(name: str) -> bool:
    return name in CITABLE_TOOLS or name.startswith(("pageindex_cloud_", "pageindex_oss_"))


# Token budget for the note summarization sidecar.
# How much of the raw tool output we feed into the note summarizer.
NOTE_RAW_INPUT_TRUNCATE_CHARS = 8000

# ---------------------------------------------------------------------------
# Defaults — preserved from the legacy presets so refactor is config-stable.
# ---------------------------------------------------------------------------
DEFAULT_REPHRASE_MAX_ITERATIONS = (
    8  # Inner loop iter cap; ask_user round cap is enforced separately.
)
DEFAULT_REPHRASE_MAX_ROUNDS = 3
DEFAULT_REPHRASE_MAX_QUESTIONS_PER_ROUND = 3
# Per-stage token budgets moved to agents.yaml — see
# ``DEFAULT_RESEARCH_PARAMS`` in services.config.loader. agents.yaml owns
# every LLM budget; this module keeps only runtime behaviour (iteration
# caps, tool policy), which is main.yaml's half of the split.
DEFAULT_BLOCK_MAX_ITERATIONS = 5
DEFAULT_REPORT_STEP_MAX_ATTEMPTS = 3
DEFAULT_INITIAL_SUBTOPICS = 5
DEFAULT_MAX_PARALLEL_TOPICS = 3
DEFAULT_QUEUE_MAX_LENGTH = 8


# ---------------------------------------------------------------------------
# Data shapes
# ---------------------------------------------------------------------------


@dataclass(frozen=True)
class SubTopicItem:
    """One entry in the decompose / outline preview output."""

    title: str
    overview: str = ""


@dataclass
class ResearchedBlock:
    """A finished topic block ready for reporting."""

    block: TopicBlock
    knowledge: str  # accumulated FINISH text from the block's agentic loop


@dataclass(frozen=True)
class ReportSectionPlan:
    """Per-section instruction in the report outline."""

    id: str
    title: str
    intent: str  # what this section should cover
    block_ids: tuple[str, ...]  # which TopicBlocks supply evidence


@dataclass(frozen=True)
class ReportOutline:
    title: str
    sections: tuple[ReportSectionPlan, ...]


class IncompleteReportError(RuntimeError):
    """A report stopped after some parts may already have reached the reader."""


# ---------------------------------------------------------------------------
# ResearchPipeline
# ---------------------------------------------------------------------------


def _outline_payload_is_usable(payload: Any) -> bool:
    """Whether a decompose response actually carries sub-topics.

    The parser accepts a bare array as well as an object, so "usable" cannot
    be the generic "is a non-empty dict" the object-shaped callers use — an
    array answer would be thrown away and retried for nothing.
    """
    if isinstance(payload, list):
        return bool(payload)
    if isinstance(payload, dict):
        return bool(payload.get("sub_topics") or payload.get("subtopics"))
    return False


def _report_outline_payload_is_usable(payload: Any) -> bool:
    """Whether a report-outline response actually carries sections."""
    if isinstance(payload, list):
        return bool(payload)
    if isinstance(payload, dict):
        return bool(payload.get("sections"))
    return False


class ResearchPipeline:
    """One-shot orchestrator: instantiate per turn, call :meth:`run` once.

    The pipeline owns control flow and per-phase prompt assembly; every
    LLM call goes through :mod:`deeptutor.runtime.agentic` primitives. The
    legacy ``DynamicTopicQueue`` + :class:`CitationManager` are reused
    verbatim as the in-flight scratchpad and citation registry.
    """

    def __init__(
        self,
        *,
        language: str = "en",
        runtime_config: dict[str, Any] | None = None,
        kb_name: str | None = None,
        enabled_tools: list[str] | None = None,
    ) -> None:
        self.language = parse_language(language)
        self.kb_name = (kb_name or "").strip() or None
        self.enabled_tools = list(enabled_tools or [])
        self.runtime_config: dict[str, Any] = dict(runtime_config or {})

        # Resolve whether the attached KB is a connected Obsidian vault. The
        # metadata comes from the access-controlled resolver (never the model),
        # and the resolution is a pure read with no RAG usage audit. Unresolvable
        # references / ordinary KBs behave exactly as before (``rag``-style KB).
        self._is_obsidian_kb = False
        self._vault_path: str | None = None
        if self.kb_name:
            try:
                from deeptutor.knowledge.kb_types import OBSIDIAN_KB_TYPE
                from deeptutor.multi_user.knowledge_access import resolve_kb_metadata

                meta = resolve_kb_metadata(self.kb_name)
                if meta and meta.get("type") == OBSIDIAN_KB_TYPE:
                    self._is_obsidian_kb = True
                    vault_path = str(meta.get("vault_path") or "").strip()
                    if vault_path:
                        self._vault_path = vault_path
            except Exception:
                logger.warning(
                    "Failed to resolve KB metadata for %r; treating as non-Obsidian.",
                    self.kb_name,
                )

        # Read structured policy sub-dicts produced by
        # :func:`build_research_runtime_config`. All keys are best-effort —
        # missing values fall back to module-level defaults so the pipeline
        # also runs against a minimal direct-instantiation in unit tests.
        researching = (
            self.runtime_config.get("researching")
            if isinstance(self.runtime_config.get("researching"), dict)
            else {}
        )
        planning = (
            self.runtime_config.get("planning")
            if isinstance(self.runtime_config.get("planning"), dict)
            else {}
        )
        reporting = (
            self.runtime_config.get("reporting")
            if isinstance(self.runtime_config.get("reporting"), dict)
            else {}
        )
        queue_cfg = (
            self.runtime_config.get("queue")
            if isinstance(self.runtime_config.get("queue"), dict)
            else {}
        )

        self.rephrase_max_iterations = _read_int(
            planning.get("rephrase"),
            key="max_iterations",
            default=DEFAULT_REPHRASE_MAX_ITERATIONS,
        )
        self.rephrase_max_rounds = DEFAULT_REPHRASE_MAX_ROUNDS
        self.rephrase_max_questions_per_round = DEFAULT_REPHRASE_MAX_QUESTIONS_PER_ROUND
        self.rephrase_enabled = bool(
            planning.get("rephrase", {}).get("enabled", True)
            if isinstance(planning.get("rephrase"), dict)
            else True
        )
        self.initial_subtopics = _read_int(
            planning.get("decompose"),
            key="initial_subtopics",
            default=DEFAULT_INITIAL_SUBTOPICS,
        )

        self.block_max_iterations = _read_int(
            researching,
            key="max_iterations",
            default=DEFAULT_BLOCK_MAX_ITERATIONS,
        )
        # A ceiling on a stalled provider, not a service-level objective. A
        # research tool is not quick by nature: one ``rag`` call against a
        # LightRAG or GraphRAG index makes its own LLM calls before it returns
        # anything, and a search-then-fetch chain waits on someone else's site.
        # 60s would have cancelled work that was going to succeed, and each
        # retry then pays the full timeout again. 240s matches the ceiling
        # ``geogebra_analysis`` uses for the same reason, and retries are off by
        # default: only a caller who knows its tools are flaky (rather than
        # slow) should pay for a second attempt.
        self._budgets = get_capability_params("research")
        self.tool_timeout = max(
            1,
            _read_int(
                researching,
                key="tool_timeout",
                default=240,
            ),
        )
        self.tool_max_retries = max(
            0,
            _read_int(
                researching,
                key="tool_max_retries",
                default=0,
            ),
        )
        self.max_parallel_topics = max(
            1,
            _read_int(
                researching,
                key="max_parallel_topics",
                default=DEFAULT_MAX_PARALLEL_TOPICS,
            ),
        )
        self.execution_mode = str(researching.get("execution_mode", "parallel"))
        self.research_mode = str(reporting.get("mode", "report"))

        self.queue_max_length = _read_int(
            queue_cfg, key="max_length", default=DEFAULT_QUEUE_MAX_LENGTH
        )

        # Block-loop tool composition runs through the same shared policy
        # chat uses (``compose_enabled_tools``) — see :meth:`_block_tool_names`.
        # There is no per-source enable_* gating here: the per-block loop
        # sees exactly the user-toggled tools plus auto-mounts (``rag``
        # when a KB is attached). Citations, ``ask_user`` is excluded by
        # the block prompt itself.

        # LLM client setup mirrors solve's pattern so the same engine
        # primitives behave identically across capabilities.
        self.llm_config = get_llm_config()
        self.binding = getattr(self.llm_config, "binding", None) or "openai"
        self.model = getattr(self.llm_config, "model", None)
        self.api_key = getattr(self.llm_config, "api_key", None)
        self.base_url = getattr(self.llm_config, "base_url", None)
        self.api_version = getattr(self.llm_config, "api_version", None)
        self.extra_headers = getattr(self.llm_config, "extra_headers", None) or {}
        self.reasoning_effort = getattr(self.llm_config, "reasoning_effort", None)
        self.client_config = LLMClientConfig(
            binding=self.binding,
            model=self.model,
            api_key=self.api_key,
            base_url=self.base_url,
            api_version=self.api_version,
            extra_headers=self.extra_headers or None,
            reasoning_effort=self.reasoning_effort,
            wire_api=getattr(self.llm_config, "wire_api", None) or "auto",
            api_format=getattr(self.llm_config, "api_format", None) or "auto",
        )

        self.registry = get_tool_registry()
        self.usage = UsageTracker(model=self.model)

        # Default sampling temperature — slightly higher than solve so
        # the research loop can take initiative on APPEND decisions.
        self._temperature = 0.3

        try:
            self._prompts: dict[str, Any] = (
                get_prompt_manager().load_prompts(
                    module_name="research",
                    agent_name="pipeline",
                    language=self.language,
                )
                or {}
            )
        except Exception as exc:
            logger.warning("Failed to load research pipeline prompts: %s", exc)
            self._prompts = {}

    # ------------------------------------------------------------------
    # Public entry points
    # ------------------------------------------------------------------
    async def run(
        self,
        *,
        context: UnifiedContext,
        topic: str,
        confirmed_outline: list[SubTopicItem] | None = None,
        attachments: list[Attachment] | None = None,
        stream: StreamBus,
    ) -> dict[str, Any]:
        """Drive the four phases.

        When ``confirmed_outline`` is ``None`` and rephrase + decompose
        complete, the pipeline emits an ``outline_preview`` result and
        returns so the capability can surface the outline for the user
        to edit / confirm. A subsequent call with ``confirmed_outline``
        skips Phase 1+2 (the user already saw the questions and approved
        a structure) and runs Phase 3+4 directly.
        """
        attachments = list(attachments or [])
        image_attachments = [a for a in attachments if getattr(a, "type", "") == "image"]
        client = self._build_client()

        try:
            await self._prepare_pageindex_tools()
            await self._prepare_research_providers(context)
            return await self._run_inner(
                context=context,
                topic=topic,
                image_attachments=image_attachments,
                confirmed_outline=confirmed_outline,
                stream=stream,
                client=client,
            )
        except Exception as exc:
            logger.exception("ResearchPipeline.run failed: %s", exc)
            await self._emit_visible_failure(stream, exc)
            raise

    async def _run_inner(
        self,
        *,
        context: UnifiedContext,
        topic: str,
        image_attachments: list[Attachment],
        confirmed_outline: list[SubTopicItem] | None,
        stream: StreamBus,
        client: Any,
    ) -> dict[str, Any]:
        logger.info(
            "ResearchPipeline.run start: lang=%s kb=%s requested_tools=%s "
            "max_iter/block=%d max_parallel=%d",
            self.language,
            self.kb_name,
            self.enabled_tools,
            self.block_max_iterations,
            self.max_parallel_topics,
        )

        # ----- Phase 1 + Phase 2 (planning) — skipped when an outline is
        # already confirmed so the user isn't asked clarifying questions a
        # second time on the same logical research task.
        if confirmed_outline is None:
            async with stream.stage("rephrasing", source=SOURCE):
                refined_topic = (
                    await self._rephrase(
                        topic=topic,
                        context=context,
                        image_attachments=image_attachments,
                        stream=stream,
                        client=client,
                    )
                    if self.rephrase_enabled
                    else topic.strip()
                )

            async with stream.stage(
                "decomposing",
                source=SOURCE,
                metadata={"research_status_key": "decompose_target"},
            ):
                outline = await self._decompose(
                    topic=refined_topic,
                    context=context,
                    image_attachments=image_attachments,
                    stream=stream,
                    client=client,
                )

            # Flat top-level keys: ``stream.result(data)`` merges ``data``
            # into the event's ``metadata`` field, and the frontend reads
            # ``event.metadata.outline_preview`` (not ``…metadata.metadata.…``).
            #
            # No ``stream.result`` here — the capability emits a single
            # result event after augmenting this payload with
            # ``research_config`` (which the frontend needs to send back
            # on confirm). Emitting here too would land first and the
            # frontend's ``find(type="result")`` would pick this one,
            # leaving ``research_config`` undefined.
            preview_payload: dict[str, Any] = {
                "response": "",
                "output_dir": "",
                "outline_preview": True,
                "topic": refined_topic,
                "sub_topics": [{"title": st.title, "overview": st.overview} for st in outline],
            }
            return preview_payload

        # ----- Phase 3 (research blocks) -----
        refined_topic = topic.strip()
        queue = DynamicTopicQueue(
            f"research_{context.session_id or 'adhoc'}",
            max_length=self.queue_max_length,
        )
        research_cache = (
            Path(context.runtime.workspace.output_dir) / "research"
            if context.runtime.workspace is not None
            else None
        )
        citations = CitationManager(queue.research_id, cache_dir=research_cache)
        for sub in confirmed_outline:
            queue.add_block(sub.title, sub.overview)

        async with stream.stage("researching", source=SOURCE):
            researched = await self._drive_queue(
                queue=queue,
                citations=citations,
                topic=refined_topic,
                context=context,
                image_attachments=image_attachments,
                stream=stream,
                client=client,
            )

        # A planned block that didn't reach COMPLETED (raised, or exhausted its
        # iteration budget without a FINISH) is backfilled with empty knowledge
        # so the surviving blocks can still produce a useful report. But the run
        # must not then look like a clean success — surface the shortfall both
        # visibly (a warning notice) and in the result envelope so callers can
        # tell the report is partial (issue #595).
        incomplete = [rb for rb in researched if rb.block.status != TopicStatus.COMPLETED]
        failed_block_titles = [rb.block.sub_topic for rb in incomplete]
        if incomplete:
            logger.warning(
                "Deep Research partial: %d/%d blocks did not complete: %s",
                len(incomplete),
                len(researched),
                failed_block_titles,
            )
            await stream.progress(
                self._t(
                    "notices.partial_results",
                    default=(
                        "{failed} of {total} research subtopics could not be completed; "
                        "the report is based on the remaining evidence."
                    ),
                    failed=len(incomplete),
                    total=len(researched),
                ),
                source=SOURCE,
                stage="researching",
                metadata={"trace_kind": "warning"},
            )

        # ----- Phase 4 (iterative reporting) -----
        async with stream.stage(
            "reporting",
            source=SOURCE,
            metadata={
                "research_status_key": "report_outline",
                "report_part": "outline",
            },
        ):
            report_text = await self._write_report(
                topic=refined_topic,
                blocks=researched,
                citations=citations,
                stream=stream,
                client=client,
            )

        result_payload: dict[str, Any] = {
            "response": report_text,
            "output_dir": "",
            "metadata": {
                "mode": "agentic_research",
                "topic": refined_topic,
                "block_count": len(researched),
                "citation_count": len(citations.get_all_citations()),
                "partial": bool(incomplete),
                "failed_block_count": len(incomplete),
                "failed_block_titles": failed_block_titles,
            },
        }
        await emit_capability_result(stream, result_payload, source=SOURCE, usage=self.usage)
        return result_payload

    async def _emit_visible_failure(self, stream: StreamBus, exc: BaseException) -> None:
        """Surface a runtime exception as a labelled error trace card so the
        user sees what went wrong instead of an empty assistant message."""
        call_id = new_call_id("research-failure")
        meta = build_trace_metadata(
            call_id=call_id,
            phase="reporting" if isinstance(exc, IncompleteReportError) else "researching",
            label=(
                self._t("labels.report_failure", default="Report incomplete")
                if isinstance(exc, IncompleteReportError)
                else self._t("labels.research_step", default="Research step")
            ),
            call_kind="llm_final_response",
            trace_id=call_id,
            trace_role="response",
            trace_group="stage",
        )
        if isinstance(exc, IncompleteReportError):
            cause = exc.__cause__ or exc
            error_message = f"{type(cause).__name__}: {cause}"
            visible_message = str(exc)
            stage = "reporting"
        else:
            error_message = f"{type(exc).__name__}: {exc}" if str(exc) else type(exc).__name__
            visible_message = error_message
            stage = "researching"
        await stream.error(
            error_message,
            source=SOURCE,
            stage=stage,
            metadata=merge_trace_metadata(meta, {"trace_kind": "error"}),
        )
        prefix = self._t("system.warning_prefix", default="⚠ ")
        await stream.content(
            f"{prefix}{visible_message}",
            source=SOURCE,
            stage=stage,
            metadata=merge_trace_metadata(meta, {"trace_kind": "llm_output"}),
        )

    # ------------------------------------------------------------------
    # Phase 1: rephrase
    # ------------------------------------------------------------------
    async def _rephrase(
        self,
        *,
        topic: str,
        context: UnifiedContext,
        image_attachments: list[Attachment] | None = None,
        stream: StreamBus,
        client: Any,
    ) -> str:
        """Mini agentic loop over ``THINK / TOOL / FINISH`` with only
        ``ask_user`` exposed.

        The host (``_RephraseLoopHost``) enforces the round cap, rejects
        any non-``ask_user`` tool call inline, and routes pause/resume
        through the runtime-supplied ``wait_for_user_reply``. When the
        LLM emits ``FINISH`` the post-label text is the refined topic.
        If the loop exits without a valid FINISH (e.g. ``ask_user`` is
        not available, or the LLM bails early) the original topic is
        returned unchanged so the rest of the pipeline can proceed.
        """
        if not self._tool_in_registry("ask_user"):
            return topic.strip()

        system_prompt = self._t(
            "rephrase.system",
            max_rounds=self.rephrase_max_rounds,
            max_questions_per_round=self.rephrase_max_questions_per_round,
            topic=topic,
        )
        user_prompt = self._t("rephrase.user_template", topic=topic)
        messages = self._build_system_user_messages(
            system_prompt, user_prompt, image_attachments=image_attachments
        )

        tool_names = ["ask_user"]
        tool_schemas = (
            self.registry.build_openai_schemas(tool_names)
            if can_use_native_tool_calling(binding=self.binding, model=self.model)
            else None
        )

        host = _RephraseLoopHost(
            pipeline=self,
            stream=stream,
            context=context,
            client=client,
            max_rounds=self.rephrase_max_rounds,
        )
        try:
            outcome = await run_agentic_loop(
                initial_messages=messages,
                protocol=_PROTOCOL_REPHRASE,
                client=client,
                model=self.model,
                completion_kwargs=self._completion_kwargs(self._budgets["block"]["max_tokens"]),
                binding=self.binding,
                tool_schemas=tool_schemas,
                stream=stream,
                source=SOURCE,
                stage="rephrasing",
                max_iterations=self.rephrase_max_iterations,
                host=host,
                usage=self.usage,
                # FINISH text is the model's brief user-facing confirmation
                # of what it's about to research — stream it live to the
                # chat bubble body so the user sees the summary forming
                # before the outline editor appears. It also doubles as
                # the refined topic input for the decompose phase.
                stream_body_live=True,
                # Lazy sub-trace open so the FINISH iteration (which
                # streams to the chat body, not into a reasoning card)
                # doesn't leave a near-empty "Reasoning" card behind.
                # THINK iters still get a card the moment their first
                # thinking chunk arrives.
                eager_sub_trace=False,
            )
        except Exception as exc:
            logger.warning("Rephrase loop failed; falling back to raw topic: %s", exc)
            return topic.strip()

        refined = (outcome.final_text or "").strip()
        return refined or topic.strip()

    # ------------------------------------------------------------------
    # Phase 2: decompose
    # ------------------------------------------------------------------
    async def _decompose(
        self,
        *,
        topic: str,
        context: UnifiedContext,
        image_attachments: list[Attachment] | None = None,
        stream: StreamBus,
        client: Any,
    ) -> list[SubTopicItem]:
        """One-shot ``OUTLINE`` labeled step.

        The LLM is asked to emit a JSON array of ``{title, overview}``
        objects (any object with a ``title`` field is accepted). Parsing
        is lenient — if JSON parse fails, the topic itself becomes the
        single fallback sub-topic so the pipeline can still drive the
        outline-preview UX.
        """
        system_prompt = self._t("decompose.system")
        user_prompt = self._t(
            "decompose.user_template",
            topic=topic,
            num_subtopics=self.initial_subtopics,
        )
        messages = self._build_system_user_messages(
            system_prompt, user_prompt, image_attachments=image_attachments
        )
        iter_meta = self._build_simple_trace_meta(
            call_id_root="research-decompose",
            label=self._t("labels.decompose", default="Decompose"),
            stage="decomposing",
            call_kind="llm_planning",
            trace_role="plan",
            trace_group="plan",
            research_status_key="decompose_target",
        )

        async def _attempt(reasoning_effort: str | None) -> str:
            step = await self._run_labeled_step(
                client=client,
                messages=messages,
                tool_schemas=None,
                protocol=_PROTOCOL_DECOMPOSE,
                stream=stream,
                stage="decomposing",
                iter_meta=iter_meta,
                max_tokens=self._budgets["outline"]["max_tokens"],
                reasoning_effort=reasoning_effort,
                eager_sub_trace=False,
            )
            return step.text

        # Decompose asks for the whole outline in one shot, so a reasoning
        # model that spends the budget thinking returns nothing to parse and
        # the topic silently degrades to a single sub-topic — research's
        # version of the one-chapter book (#1316).
        data = await payload_with_reasoning_retry(
            _attempt, is_usable=_outline_payload_is_usable, logger_instance=logger
        )
        return self._parse_outline(topic, data)

    def _parse_outline(self, topic: str, data: Any) -> list[SubTopicItem]:
        if isinstance(data, list):
            iterable: list[Any] = data
        elif isinstance(data, dict):
            iterable = list(data.get("sub_topics") or data.get("subtopics") or [])
        else:
            iterable = []

        items: list[SubTopicItem] = []
        for entry in iterable:
            if isinstance(entry, dict):
                title = str(entry.get("title") or entry.get("topic") or "").strip()
                overview = str(entry.get("overview") or entry.get("description") or "").strip()
            elif isinstance(entry, str):
                title = entry.strip()
                overview = ""
            else:
                continue
            if title:
                items.append(SubTopicItem(title=title, overview=overview))
            if len(items) >= self.initial_subtopics:
                break
        if not items:
            items = [SubTopicItem(title=topic, overview="")]
        return items

    # ------------------------------------------------------------------
    # Phase 3: research one block
    # ------------------------------------------------------------------
    async def _research_block(
        self,
        *,
        block: TopicBlock,
        queue: DynamicTopicQueue,
        citations: CitationManager,
        topic: str,
        context: UnifiedContext,
        stream: StreamBus,
        client: Any,
    ) -> ResearchedBlock:
        """Run one block of the dynamic research queue through an agentic
        loop with the ``THINK`` / ``TOOL`` / ``APPEND`` / ``FINISH``
        protocol. ``FINISH``'s post-label text becomes the block's
        consolidated knowledge for the report layer; ``APPEND`` triggers
        a queue mutation via :meth:`_BlockLoopHost.on_intermediate`.
        """
        queue.mark_researching(block.block_id)

        block_tool_names = self._block_tool_names()
        native_block_tools = self._use_native_block_tools(block_tool_names)
        prompt_tool_names = block_tool_names if native_block_tools else []
        effective_max_iterations = (
            max(self.block_max_iterations, 4) if prompt_tool_names else self.block_max_iterations
        )
        tool_schemas = (
            self._build_block_tool_schemas(prompt_tool_names) if native_block_tools else None
        )
        tool_list = (
            self.registry.build_prompt_text(
                prompt_tool_names,
                format="list_with_usage",
                language=self.language,
            )
            or self._fallback_empty_tool_list()
        )
        kb_note = self._kb_system_note()

        system_prompt = self._t(
            "research_step.system",
            topic=topic,
            block_title=block.sub_topic,
            block_overview=block.overview or "(no overview)",
            mode=self.research_mode,
            max_iterations=effective_max_iterations,
            kb_note=kb_note,
            tool_list=tool_list,
        )
        from deeptutor.agents._shared.workspace_prompt import workspace_system_note

        workspace_note = workspace_system_note(context, language=self.language)
        if workspace_note:
            system_prompt = f"{system_prompt}\n\n{workspace_note}"
        system_prompt = append_language_directive(system_prompt, self.language)

        sibling_topics = self._render_sibling_topics(queue, block)
        user_prompt = self._t(
            "research_step.user_template",
            accumulated_knowledge=self._t(
                "empty.no_evidence", default="(no evidence collected yet)"
            ),
            sibling_topics=sibling_topics,
        )
        messages = self._build_system_user_messages(system_prompt, user_prompt)

        host = _BlockLoopHost(
            pipeline=self,
            block=block,
            queue=queue,
            citations=citations,
            topic=topic,
            stream=stream,
            context=context,
            client=client,
        )
        try:
            outcome = await run_agentic_loop(
                initial_messages=messages,
                protocol=_PROTOCOL_BLOCK,
                client=client,
                model=self.model,
                completion_kwargs=self._completion_kwargs(self._budgets["block"]["max_tokens"]),
                binding=self.binding,
                tool_schemas=tool_schemas,
                stream=stream,
                source=SOURCE,
                stage="researching",
                max_iterations=effective_max_iterations,
                host=host,
                usage=self.usage,
                stream_body_live=False,
                eager_sub_trace=True,
            )
        except Exception as exc:
            logger.exception("Research block %s failed: %s", block.block_id, exc)
            queue.mark_failed(block.block_id)
            return ResearchedBlock(block=block, knowledge="")

        knowledge = (outcome.final_text or "").strip()
        if outcome.completed:
            queue.mark_completed(block.block_id)
        else:
            queue.mark_failed(block.block_id)
        block.iteration_count = outcome.iterations
        return ResearchedBlock(block=block, knowledge=knowledge)

    async def _force_finish_block(
        self,
        *,
        client: Any,
        messages: list[dict[str, Any]],
        stream: StreamBus,
        start_iteration: int,
        block: TopicBlock,
    ) -> tuple[str, bool, int]:
        """Per-block ``LoopHost.force_finalize`` recovery: prompt the model
        to emit a final ``FINISH`` consolidating what was learned so far,
        with a small retry budget for protocol-repair attempts."""
        calls = 0
        messages.append({"role": "user", "content": self._t("protocol.force_finish")})
        await stream.progress(
            self._t("notices.max_iterations_reached", default="Max iterations reached."),
            source=SOURCE,
            stage="researching",
            metadata={"trace_kind": "warning"},
        )
        for attempt in range(3):
            iter_meta = self._build_simple_trace_meta(
                call_id_root=f"research-{block.block_id}-force-{start_iteration + attempt}",
                label=self._t("labels.reasoning", default="Reasoning"),
                stage="researching",
                block_id=block.block_id,
                **_research_topic_status_meta(block),
            )
            result = await self._run_labeled_step(
                client=client,
                messages=messages,
                tool_schemas=None,
                protocol=LabelProtocol(
                    allowed=(LABEL_FINISH,),
                    terminal=frozenset({LABEL_FINISH}),
                    intermediate=frozenset(),
                    final=frozenset({LABEL_FINISH}),
                    tool_label=None,
                ),
                stream=stream,
                stage="researching",
                iter_meta=iter_meta,
            )
            calls += 1
            if result.label == LABEL_FINISH and result.text.strip():
                return result.text, True, calls
            messages.append(
                assistant_message(
                    result.text[:500],
                    reasoning_content=result.reasoning_content or None,
                    thinking_blocks=list(result.thinking_blocks) or None,
                )
            )
            messages.append({"role": "user", "content": self._t("protocol.force_finish_repair")})
        return self._t("protocol.fallback_final"), False, calls

    # ------------------------------------------------------------------
    # Per-block rendering helpers
    # ------------------------------------------------------------------
    def _render_sibling_topics(self, queue: DynamicTopicQueue, current_block: TopicBlock) -> str:
        """Compact list of other topics in the queue so APPEND can dedup
        against them at prompt-time (in addition to the runtime check)."""
        siblings = [
            f"  - [{b.block_id}] {b.sub_topic}"
            for b in queue.blocks
            if b.block_id != current_block.block_id
        ]
        if not siblings:
            return self._t("empty.no_subtopics", default="(none)")
        return "\n".join(siblings)

    def _kb_system_note(self) -> str:
        if not self.kb_name:
            return ""
        if self._is_obsidian_kb:
            return self._t(
                "system.obsidian_kb_system_note",
                default=(
                    f"Attached knowledge base {self.kb_name!r} is a read-only "
                    f"Obsidian vault. Gather evidence with obsidian_search, "
                    f"obsidian_list and obsidian_read. Do not call rag."
                ),
                kb_name=self.kb_name,
            )
        tool_context = getattr(self, "_pageindex_tool_context", None)
        if tool_context is not None:
            tools = ", ".join(tool.name for tool in tool_context.tools)
            docs = (
                "; ".join(
                    f"{name} (doc_id: {doc_id})"
                    for name, doc_id in sorted(tool_context.documents.items())
                )
                or "(no indexed documents)"
            )
            instructions = tool_context.instructions.strip()
            return (
                f"Attached PageIndex knowledge base: {self.kb_name!r}. Read it with "
                f"these tools inside the research loop; do not call rag: {tools}. "
                f"Documents: {docs}."
                + (f"\nPageIndex SDK reading instructions:\n{instructions}" if instructions else "")
            )
        return self._t(
            "system.kb_system_note",
            default=(
                f"Attached knowledge bases: {self.kb_name}. When calling rag, "
                f"kb_name must be {self.kb_name!r}."
            ),
            kb_name=self.kb_name,
            kb_name_repr=repr(self.kb_name),
        )

    def _fallback_empty_tool_list(self) -> str:
        return self._t("empty.no_tools", default="- none")

    # ------------------------------------------------------------------
    # Phase 4: reporting
    # ------------------------------------------------------------------
    # ------------------------------------------------------------------
    # Queue scheduler — drains the dynamic topic queue. Series vs.
    # parallel selected from ``researching.execution_mode``.
    # ------------------------------------------------------------------
    async def _drive_queue(
        self,
        *,
        queue: DynamicTopicQueue,
        citations: CitationManager,
        topic: str,
        context: UnifiedContext,
        image_attachments: list[Attachment] | None,
        stream: StreamBus,
        client: Any,
    ) -> list[ResearchedBlock]:
        researched_by_id: dict[str, ResearchedBlock] = {}
        rounds = 0
        safety_cap = max(20, (self.queue_max_length or 0) * 4)

        while True:
            pending = queue.get_all_pending_blocks()
            if not pending:
                break
            batch_size = (
                1 if str(self.execution_mode).lower() == "series" else self.max_parallel_topics
            )
            batch = pending[:batch_size]
            results = await asyncio.gather(
                *[
                    self._research_block(
                        block=block,
                        queue=queue,
                        citations=citations,
                        topic=topic,
                        context=context,
                        stream=stream,
                        client=client,
                    )
                    for block in batch
                ],
                return_exceptions=True,
            )
            for block, result in zip(batch, results, strict=True):
                if isinstance(result, BaseException):
                    logger.exception("Block %s research failed: %s", block.block_id, result)
                    if block.status == TopicStatus.RESEARCHING:
                        queue.mark_failed(block.block_id)
                    researched_by_id[block.block_id] = ResearchedBlock(block=block, knowledge="")
                    continue
                researched_by_id[block.block_id] = result

            rounds += 1
            if rounds > safety_cap:
                logger.warning(
                    "Research scheduler aborted after %d rounds — queue size %d",
                    rounds,
                    len(queue.blocks),
                )
                break

        # Preserve queue order so parents come before APPENDed children.
        ordered: list[ResearchedBlock] = []
        for block in queue.blocks:
            ordered.append(
                researched_by_id.get(block.block_id) or ResearchedBlock(block=block, knowledge="")
            )
        return ordered

    # ------------------------------------------------------------------
    # Note-agent sidecar — summarise + record citation for one tool call.
    # ------------------------------------------------------------------
    async def _summarise_tool_result(
        self,
        *,
        tool_name: str,
        query: str,
        raw_answer: str,
        client: Any,
    ) -> str:
        """One LLM call per citable tool result: condense long retrievals
        into a short, citation-ready summary. Failures fall back to a
        leading slice of the raw answer so the loop never stalls on a
        flaky summariser call.
        """
        cleaned = (raw_answer or "").strip()
        if not cleaned:
            return ""
        system_prompt = self._t("note.system")
        user_prompt = self._t(
            "note.user_template",
            tool_name=tool_name,
            query=query,
            raw_answer=cleaned[:NOTE_RAW_INPUT_TRUNCATE_CHARS],
        )
        messages = self._build_system_user_messages(system_prompt, user_prompt)
        try:
            kwargs = self._completion_kwargs(self._budgets["note"]["max_tokens"])
            response = await client.chat.completions.create(
                model=self.model, messages=messages, stream=False, **kwargs
            )
            content = (response.choices[0].message.content if response.choices else "") or ""
            parsed = classify_label(
                content,
                allowed_labels=(LABEL_FINISH,),
                final=True,
            )
            if parsed is not None:
                _label, content = parsed
            return content.strip() or cleaned[:600]
        except Exception as exc:
            logger.warning("Note summarisation failed (%s): %s", tool_name, exc)
            return cleaned[:600]

    # ------------------------------------------------------------------
    # Phase 4: iterative reporting
    #
    # Each sub-phase is one labeled step:
    #   _gen_report_outline → _write_intro → for sec: _write_section →
    #   _write_conclusion. Bodies stream live (``stream_body_live``)
    #   chunk-by-chunk for the section-level steps so the user sees the
    #   report assemble in real time.
    # ------------------------------------------------------------------
    async def _write_report(
        self,
        *,
        topic: str,
        blocks: list[ResearchedBlock],
        citations: CitationManager,
        stream: StreamBus,
        client: Any,
    ) -> str:
        outline = await self._gen_report_outline(
            topic=topic,
            blocks=blocks,
            citations=citations,
            stream=stream,
            client=client,
        )

        # Global numbering: introduction is 1, sub-topic sections are 2..N+1,
        # conclusion is N+2 (where N == len(outline.sections)). Both the
        # rendered ``##`` heading and any ``### N.x`` subsection prefixes the
        # LLM emits reference these numbers, so we resolve them here and pass
        # them into every report-writing step.
        section_count = len(outline.sections)
        conclusion_number = section_count + 2
        report_parts = [
            f"1. {self._t('labels.report_intro_part', default='Introduction')}",
            *(
                f"{index + 1}. {section.title}"
                for index, section in enumerate(outline.sections, start=1)
            ),
            f"{conclusion_number}. {self._t('labels.report_conclusion_part', default='Conclusion')}",
        ]

        def incomplete_error(first_missing: int) -> IncompleteReportError:
            notice_template = self._t(
                "notices.report_incomplete",
                default="The report stopped before these parts could be written: {parts}. Retry to generate the full report.",
            )
            return IncompleteReportError(
                notice_template.format(parts=", ".join(report_parts[first_missing:]))
            )

        section_texts: list[str] = []

        # Emit the report's H1 title before the introduction so the rendered
        # markdown opens with the report name, then the numbered ``## 1.``
        # Introduction heading. The LLM is not asked to write the title — we
        # already have it on the outline.
        title_block = self._render_report_title_block(outline.title)
        if title_block:
            await stream.content(
                title_block,
                source=SOURCE,
                stage="reporting",
                metadata={"trace_kind": "report_title"},
            )
            section_texts.append(title_block)

        # The separator must be on the wire before ``_write_intro`` emits its
        # first heading. Emitting it afterwards produced the literal live
        # stream ``# Report title## 1. Introduction`` even though the final
        # assembled response was normalized correctly.
        if section_texts:
            await self._stream_report_separator(stream)
        try:
            intro = await self._write_intro(
                topic=topic, outline=outline, stream=stream, client=client
            )
        except Exception as exc:
            raise incomplete_error(0) from exc
        if intro:
            section_texts.append(intro)

        section_bodies: list[str] = []
        for section_index, section in enumerate(outline.sections, start=1):
            if section_texts:
                await self._stream_report_separator(stream)
            try:
                body = await self._write_section(
                    section=section,
                    section_index=section_index,
                    section_count=section_count,
                    section_number=section_index + 1,
                    topic=topic,
                    outline=outline,
                    blocks=blocks,
                    citations=citations,
                    stream=stream,
                    client=client,
                )
            except Exception as exc:
                raise incomplete_error(section_index) from exc
            if body:
                section_texts.append(body)
                section_bodies.append(body)

        if section_texts:
            await self._stream_report_separator(stream)
        try:
            conclusion = await self._write_conclusion(
                topic=topic,
                outline=outline,
                section_bodies=section_bodies,
                section_number=conclusion_number,
                stream=stream,
                client=client,
            )
        except Exception as exc:
            raise incomplete_error(section_count + 1) from exc
        if conclusion:
            section_texts.append(conclusion)

        body = self._normalise_report_markdown(
            "\n\n".join(part for part in section_texts if part.strip()),
            citations,
        )
        used_citation_ids = _citation_ids_in_first_appearance(body, citations)
        citation_numbers = {
            citation_id: index for index, citation_id in enumerate(used_citation_ids, start=1)
        }
        body = self._linkify_report_citations(body, citations, citation_numbers=citation_numbers)
        references = self._render_reference_list(
            citations,
            citation_ids=used_citation_ids,
            citation_numbers=citation_numbers,
        )
        if references:
            await self._stream_report_separator(stream)
            await stream.content(
                references,
                source=SOURCE,
                stage="reporting",
                metadata={"trace_kind": "reference_list"},
            )
        return "\n\n".join(part for part in (body, references) if part.strip())

    def _render_reference_list(
        self,
        citations: CitationManager,
        *,
        citation_ids: list[str] | None = None,
        citation_numbers: dict[str, int] | None = None,
    ) -> str:
        """Append a collapsible reference appendix with stable anchors.

        Numbering follows first appearance in the final report body. That
        keeps body links and the appendix compact even when research collected
        additional tool traces that did not survive synthesis.
        """
        entries = []
        ordered_ids = citation_ids if citation_ids is not None else _sorted_citation_ids(citations)
        numbers = citation_numbers or {
            citation_id: index for index, citation_id in enumerate(ordered_ids, start=1)
        }
        for citation_id in ordered_ids:
            formatted = self._format_reference_entry(citations, citation_id)
            if formatted:
                anchor = _citation_anchor_id(citation_id)
                ref_number = numbers.get(citation_id, len(entries) + 1)
                entries.append(
                    f'<li id="{anchor}" data-citation-id="{html.escape(citation_id)}">'
                    f'<span data-ref-number="{ref_number}">{formatted}</span>'
                    "</li>"
                )
        if not entries:
            return ""
        heading = self._t("labels.references_heading", default="References")
        return (
            '<details id="references" open>\n'
            f"<summary>{heading}</summary>\n"
            "<ol>\n" + "\n".join(entries) + "\n</ol>\n"
            "</details>"
        )

    def _linkify_report_citations(
        self,
        text: str,
        citations: CitationManager,
        *,
        citation_numbers: dict[str, int] | None = None,
    ) -> str:
        """Turn generated ``[CIT-...]`` markers into same-page anchor links.

        The section writer is intentionally instructed to copy plain markers,
        because that is easier for models to follow. This renderer pass makes
        the final persisted Markdown navigable without asking the model to
        manufacture link syntax.
        """
        numbers = citation_numbers or _citation_number_map(citations)
        if not text or not numbers:
            return text

        def _replace(match: re.Match[str]) -> str:
            citation_id = match.group("id")
            ref_number = numbers.get(citation_id)
            if ref_number is None:
                return ""
            return _citation_markdown_link(citation_id, ref_number)

        return _REPORT_CITATION_MARKER_RE.sub(_replace, text)

    def _normalise_report_markdown(
        self,
        text: str,
        citations: CitationManager,
    ) -> str:
        """Clean model-authored report markdown before persistence.

        The report writer is instructed to copy raw ``[CIT-...]`` markers,
        but models sometimes pre-link them or repeat heading markers. This
        pass converts pre-linked markers back to canonical form, strips
        unknown citation ids, and normalises duplicate headings like
        ``## ## Title``.
        """
        if not text:
            return ""
        known = set(citations.get_all_citations())

        def _replace_link(match: re.Match[str]) -> str:
            citation_id = match.group("id")
            return f"[{citation_id}]" if citation_id in known else ""

        cleaned = _REPORT_CITATION_LINK_RE.sub(_replace_link, text)

        def _replace_bare(match: re.Match[str]) -> str:
            citation_id = match.group("id")
            return match.group(0) if citation_id in known else ""

        cleaned = _REPORT_CITATION_MARKER_RE.sub(_replace_bare, cleaned)
        return _normalise_markdown_headings(cleaned).strip()

    def _format_reference_entry(
        self,
        citations: CitationManager,
        citation_id: str,
    ) -> str | None:
        """Return a reference-list entry as HTML-safe HTML.

        ``CitationManager.format_citation_for_report`` already produces escaped
        HTML for known tool types; the fallback path below also escapes
        user-controlled fields so the caller can drop the result into the page
        without re-escaping.
        """
        formatted = citations.format_citation_for_report(citation_id)
        if formatted:
            return formatted
        citation = citations.get_citation(citation_id)
        if not citation:
            return None
        tool = str(citation.get("tool_type") or "tool")
        query = str(citation.get("query") or "").strip()
        summary = str(citation.get("summary") or "").strip()
        parts = [html.escape(tool.replace("_", " ").title())]
        if query:
            parts.append(f"query: {html.escape(query)}")
        if summary:
            parts.append(f"note: {html.escape(summary[:180])}")
        return " — ".join(parts)

    async def _stream_report_separator(self, stream: StreamBus) -> None:
        await stream.content("\n\n", source=SOURCE, stage="reporting")

    def _render_report_title_block(self, title: str) -> str:
        """Build the ``# <title>`` H1 block emitted before the introduction.

        Strips any leading ``#`` the model may have left on the outline title
        so the rendered output is always a single H1 line.
        """
        clean = _clean_report_heading_text(title or "").strip()
        if not clean:
            return ""
        return f"# {clean}"

    async def _gen_report_outline(
        self,
        *,
        topic: str,
        blocks: list[ResearchedBlock],
        citations: CitationManager,
        stream: StreamBus,
        client: Any,
    ) -> ReportOutline:
        """One ``OUTLINE`` labeled step that proposes the report sections.

        The model sees the topic + each researched block's title and a
        short knowledge snippet, then emits a JSON ``{title, sections:
        [{id, title, intent, block_ids}]}`` payload mapping each
        section to one or more blocks that supply its evidence.
        """
        block_summaries = []
        for rb in blocks:
            preview = (rb.knowledge or "").strip().split("\n\n")[0][:400]
            block_summaries.append(f"- [{rb.block.block_id}] {rb.block.sub_topic}\n  {preview}")
        system_prompt = self._t("report.outline.system")
        user_prompt = self._t(
            "report.outline.user_template",
            topic=topic,
            block_summaries="\n".join(block_summaries) or "(no researched blocks)",
        )
        messages = self._build_system_user_messages(system_prompt, user_prompt)
        iter_meta = self._build_simple_trace_meta(
            call_id_root="research-report-outline",
            label=self._t("labels.report_outline", default="Report outline"),
            stage="reporting",
            call_kind="llm_planning",
            trace_role="plan",
            trace_group="plan",
            research_status_key="report_outline",
            report_part="outline",
        )

        async def _attempt(reasoning_effort: str | None) -> str:
            step = await self._run_labeled_step(
                client=client,
                messages=messages,
                tool_schemas=None,
                protocol=_PROTOCOL_REPORT_OUTLINE,
                stream=stream,
                stage="reporting",
                iter_meta=iter_meta,
                max_tokens=self._budgets["report_outline"]["max_tokens"],
                reasoning_effort=reasoning_effort,
                eager_sub_trace=False,
            )
            return step.text

        # Same one-shot payload, same starvation: an empty outline here costs
        # the reader the whole report structure.
        data = await payload_with_reasoning_retry(
            _attempt, is_usable=_report_outline_payload_is_usable, logger_instance=logger
        )
        return self._parse_report_outline(topic, data, blocks)

    def _parse_report_outline(
        self, topic: str, data: Any, blocks: list[ResearchedBlock]
    ) -> ReportOutline:
        valid_ids = {rb.block.block_id for rb in blocks}
        title_to_id = {rb.block.sub_topic.lower(): rb.block.block_id for rb in blocks}

        if isinstance(data, dict):
            title = _clean_report_heading_text(str(data.get("title") or topic)) or topic
            raw_sections = data.get("sections") or []
        elif isinstance(data, list):
            title = _clean_report_heading_text(topic) or topic
            raw_sections = data
        else:
            title = _clean_report_heading_text(topic) or topic
            raw_sections = []

        sections: list[ReportSectionPlan] = []
        for i, item in enumerate(raw_sections if isinstance(raw_sections, list) else []):
            if not isinstance(item, dict):
                continue
            sec_title = _clean_report_heading_text(str(item.get("title") or ""))
            if not sec_title:
                continue
            sec_id = str(item.get("id") or f"R{i + 1}").strip() or f"R{i + 1}"
            intent = str(item.get("intent") or item.get("summary") or "").strip()
            raw_block_ids = item.get("block_ids") or item.get("blocks") or []
            resolved_ids: list[str] = []
            if isinstance(raw_block_ids, list):
                for entry in raw_block_ids:
                    if not isinstance(entry, str):
                        continue
                    if entry in valid_ids:
                        resolved_ids.append(entry)
                    else:
                        bid = title_to_id.get(entry.strip().lower())
                        if bid:
                            resolved_ids.append(bid)
            sections.append(
                ReportSectionPlan(
                    id=sec_id,
                    title=sec_title,
                    intent=intent,
                    block_ids=tuple(resolved_ids),
                )
            )
        if not sections:
            # Fallback: one section per researched block in queue order.
            for rb in blocks:
                sections.append(
                    ReportSectionPlan(
                        id=rb.block.block_id,
                        title=rb.block.sub_topic,
                        intent=rb.block.overview or "",
                        block_ids=(rb.block.block_id,),
                    )
                )
        sections = self._repair_report_section_coverage(sections, blocks)
        return ReportOutline(title=title, sections=tuple(sections))

    def _repair_report_section_coverage(
        self,
        sections: list[ReportSectionPlan],
        blocks: list[ResearchedBlock],
    ) -> list[ReportSectionPlan]:
        """Make report planning robust to partial / invalid block maps.

        The prompt asks the model to map every researched block into at
        least one report section, but outline JSON is still model output.
        This deterministic pass keeps the report from silently dropping a
        block when the LLM omits its id, misspells it, or leaves a section
        with an empty ``block_ids`` list.
        """
        if not sections or not blocks:
            return sections

        valid_ids = {rb.block.block_id for rb in blocks}
        covered: set[str] = set()
        repaired: list[ReportSectionPlan] = []

        for section in sections:
            ids = tuple(dict.fromkeys(bid for bid in section.block_ids if bid in valid_ids))
            if not ids:
                best = _best_block_for_section(section, blocks, exclude=covered)
                if best is not None:
                    ids = (best.block.block_id,)
            covered.update(ids)
            repaired.append(_section_with_block_ids(section, ids))

        missing = [rb for rb in blocks if rb.block.block_id not in covered]
        if not missing:
            return repaired

        addendum: list[str] = []
        for rb in missing:
            section_index = _best_section_for_block(rb, repaired)
            if section_index is None:
                addendum.append(rb.block.block_id)
                continue
            section = repaired[section_index]
            repaired[section_index] = _section_with_block_ids(
                section,
                tuple(dict.fromkeys((*section.block_ids, rb.block.block_id))),
            )

        if addendum:
            addendum_title = self._t("labels.addendum_title", default="Additional findings")
            addendum_intent = self._t(
                "labels.addendum_intent",
                default=(
                    "Covers researched blocks that did not fit cleanly into the earlier sections."
                ),
            )
            repaired.append(
                ReportSectionPlan(
                    id=f"S{len(repaired) + 1}",
                    title=addendum_title,
                    intent=addendum_intent,
                    block_ids=tuple(addendum),
                )
            )
        return repaired

    async def _write_intro(
        self,
        *,
        topic: str,
        outline: ReportOutline,
        stream: StreamBus,
        client: Any,
    ) -> str:
        system_prompt = self._t("report.intro.system", section_number=1)
        user_prompt = self._t(
            "report.intro.user_template",
            topic=topic,
            title=outline.title,
            section_number=1,
            sections_overview="\n".join(
                f"- {s.title}: {s.intent}".rstrip(": ") for s in outline.sections
            ),
        )
        return await self._stream_report_step(
            system_prompt=system_prompt,
            user_prompt=user_prompt,
            protocol=_PROTOCOL_REPORT_INTRO,
            stream=stream,
            client=client,
            label=self._t("labels.report_intro", default="Introduction"),
            call_id_root="research-report-intro",
            max_tokens=self._budgets["report_intro"]["max_tokens"],
            expected_section_number=1,
            extra_meta={
                "research_status_key": "report_intro",
                "report_part": "intro",
            },
        )

    async def _write_section(
        self,
        *,
        section: ReportSectionPlan,
        section_index: int,
        section_count: int,
        section_number: int,
        topic: str,
        outline: ReportOutline,
        blocks: list[ResearchedBlock],
        citations: CitationManager,
        stream: StreamBus,
        client: Any,
    ) -> str:
        evidence = self._render_section_evidence(
            section=section, blocks=blocks, citations=citations
        )
        system_prompt = self._t("report.section.system", section_number=section_number)
        user_prompt = self._t(
            "report.section.user_template",
            topic=topic,
            report_title=outline.title,
            section_id=section.id,
            section_title=section.title,
            section_intent=section.intent or "(no extra guidance)",
            section_number=section_number,
            evidence=evidence,
        )
        return await self._stream_report_step(
            system_prompt=system_prompt,
            user_prompt=user_prompt,
            protocol=_PROTOCOL_REPORT_SECTION,
            stream=stream,
            client=client,
            label=(f"{self._t('labels.report_section', default='Section')}: {section.title}"),
            call_id_root=f"research-report-section-{section.id}",
            max_tokens=self._budgets["report_section"]["max_tokens"],
            expected_section_number=section_number,
            extra_meta={
                "research_status_key": "report_section",
                "report_part": "section",
                "section_index": section_index,
                "section_count": section_count,
                "section_title": section.title,
            },
        )

    async def _write_conclusion(
        self,
        *,
        topic: str,
        outline: ReportOutline,
        section_bodies: list[str],
        section_number: int,
        stream: StreamBus,
        client: Any,
    ) -> str:
        # Recap: first paragraph of each rendered sub-topic section so the
        # conclusion writer can land the answer without re-reading the full
        # raw evidence. ``section_bodies`` contains only the middle sections
        # (title block and intro are intentionally excluded).
        recap_chunks: list[str] = []
        for sec, body in zip(outline.sections, section_bodies, strict=False):
            snippet = (body or "").strip().split("\n\n", 1)[0]
            recap_chunks.append(f"### {sec.title}\n{snippet[:300]}")
        system_prompt = self._t("report.conclusion.system", section_number=section_number)
        user_prompt = self._t(
            "report.conclusion.user_template",
            topic=topic,
            title=outline.title,
            section_number=section_number,
            sections_recap="\n\n".join(recap_chunks) or "(no section bodies)",
        )
        return await self._stream_report_step(
            system_prompt=system_prompt,
            user_prompt=user_prompt,
            protocol=_PROTOCOL_REPORT_CONCLUSION,
            stream=stream,
            client=client,
            label=self._t("labels.report_conclusion", default="Conclusion"),
            call_id_root="research-report-conclusion",
            max_tokens=self._budgets["report_conclusion"]["max_tokens"],
            expected_section_number=section_number,
            extra_meta={
                "research_status_key": "report_conclusion",
                "report_part": "conclusion",
            },
        )

    async def _stream_report_step(
        self,
        *,
        system_prompt: str,
        user_prompt: str,
        protocol: LabelProtocol,
        stream: StreamBus,
        client: Any,
        label: str,
        call_id_root: str,
        max_tokens: int,
        extra_meta: dict[str, Any] | None = None,
        expected_section_number: int | None = None,
    ) -> str:
        """Run and validate one report sub-phase, retrying partial streams.

        Report prose is buffered until the call is known to be complete. This
        prevents a timed-out first attempt from leaking half a section into
        the final bubble before a retry starts. Progress / reasoning traces
        still stream live, so long report calls remain observable.
        """
        messages = self._build_system_user_messages(system_prompt, user_prompt)
        trace_extra = dict(extra_meta or {})
        iter_meta = self._build_simple_trace_meta(
            call_id_root=call_id_root,
            label=label,
            stage="reporting",
            call_kind="llm_final_response",
            trace_role="response",
            trace_group="stage",
            **trace_extra,
        )
        final_call_id = new_call_id(f"{call_id_root}-final")
        final_meta = build_trace_metadata(
            call_id=final_call_id,
            phase="reporting",
            label=label,
            call_kind="llm_final_response",
            trace_id=final_call_id,
            trace_role="response",
            trace_group="stage",
            **trace_extra,
        )
        expected_label = protocol.allowed[0]
        last_reason = "empty response"
        for attempt in range(1, DEFAULT_REPORT_STEP_MAX_ATTEMPTS + 1):
            body = ""
            step: LabeledStepResult | None = None
            try:
                step = await self._run_labeled_step(
                    client=client,
                    messages=messages,
                    tool_schemas=None,
                    protocol=protocol,
                    stream=stream,
                    stage="reporting",
                    iter_meta=iter_meta,
                    max_tokens=max_tokens,
                    # Report prose is the reader-facing body; thinking on a
                    # reasoning model comes out of the same max_tokens budget
                    # and is the usual cause of first-attempt truncation.
                    reasoning_effort="none",
                    # Buffer body text until validation succeeds. Passing
                    # ``final_meta`` here would stream a failed attempt and
                    # make a clean retry impossible without duplicated prose.
                    final_meta=None,
                )
                resolved_label, body = _recover_report_step_reply(
                    step,
                    expected_label=expected_label,
                    expected_section_number=expected_section_number,
                )
                if resolved_label != step.label or body != (step.text or "").strip():
                    step = replace(step, label=resolved_label, text=body)
                last_reason = _report_step_incomplete_reason(
                    step,
                    expected_label=expected_label,
                    body=body,
                )
                if not last_reason:
                    await stream.content(
                        body,
                        source=SOURCE,
                        stage="reporting",
                        metadata=merge_trace_metadata(
                            final_meta,
                            {"trace_kind": "llm_chunk", "report_attempt": attempt},
                        ),
                    )
                    return body
            except Exception as exc:
                last_reason = f"provider error: {type(exc).__name__}"
                if attempt >= DEFAULT_REPORT_STEP_MAX_ATTEMPTS:
                    raise
                logger.warning(
                    "Report step %s attempt %d/%d failed: %s",
                    call_id_root,
                    attempt,
                    DEFAULT_REPORT_STEP_MAX_ATTEMPTS,
                    exc,
                )

            if attempt < DEFAULT_REPORT_STEP_MAX_ATTEMPTS:
                await stream.progress(
                    self._t(
                        "labels.report_retry",
                        default="Report section was incomplete; retrying.",
                    ),
                    source=SOURCE,
                    stage="reporting",
                    metadata={
                        "trace_kind": "warning",
                        "report_retry": True,
                        "report_attempt": attempt,
                        "report_retry_reason": last_reason,
                        **trace_extra,
                    },
                )
                messages.extend(
                    [
                        assistant_message(
                            f"``{expected_label}``\n{body}" if body else "",
                            reasoning_content=step.reasoning_content if step else None,
                            thinking_blocks=list(step.thinking_blocks) if step else None,
                        ),
                        {
                            "role": "user",
                            "content": self._t(
                                "report.retry_complete",
                                default=(
                                    "The previous report part was incomplete or invalid. "
                                    "Regenerate the complete part from its ## heading, "
                                    "follow the required label protocol, and finish every sentence."
                                ),
                            ),
                        },
                    ]
                )

        raise RuntimeError(
            f"Research report step {call_id_root!r} remained incomplete after "
            f"{DEFAULT_REPORT_STEP_MAX_ATTEMPTS} attempts ({last_reason})."
        )

    def _render_section_evidence(
        self,
        *,
        section: ReportSectionPlan,
        blocks: list[ResearchedBlock],
        citations: CitationManager,
    ) -> str:
        block_ids = section.block_ids or tuple(rb.block.block_id for rb in blocks)
        by_id = {rb.block.block_id: rb for rb in blocks}
        chunks: list[str] = []
        total = 0
        cap_per_block = 4000
        cap_total = 12000

        for bid in block_ids:
            rb = by_id.get(bid)
            if rb is None:
                continue
            title = _clean_report_heading_text(rb.block.sub_topic) or rb.block.sub_topic
            lines = [f"### Block [{rb.block.block_id}] {title}"]
            block_chars = 0
            for trace in rb.block.tool_traces:
                cid = trace.citation_id or trace.tool_id
                summary = (trace.summary or "").strip()
                if not summary:
                    continue
                citation = citations.get_citation(cid) or {}
                source_preview = _citation_source_preview(citation)
                line_parts = [
                    f"#### Evidence [{cid}]",
                    f"- tool: {trace.tool_type}",
                    f"- query: {trace.query}",
                ]
                if source_preview:
                    line_parts.append(f"- source hints: {source_preview}")
                line_parts.append(f"- note: {summary}")
                line = "\n".join(line_parts)
                line = line[: cap_per_block - block_chars]
                if not line:
                    break
                lines.append(line)
                block_chars += len(line)
                if block_chars >= cap_per_block:
                    break
            if rb.knowledge:
                lines.append(f"#### Block FINISH\n{rb.knowledge[:1500]}")
            chunk = "\n\n".join(lines)
            if total + len(chunk) > cap_total:
                chunk = chunk[: max(0, cap_total - total)]
            if chunk:
                chunks.append(chunk)
                total += len(chunk)
            if total >= cap_total:
                break
        return "\n\n".join(chunks) or "(no evidence available)"

    # ------------------------------------------------------------------
    # Tool composition for the block loop
    # ------------------------------------------------------------------
    async def _prepare_pageindex_tools(self) -> None:
        from deeptutor.modules.catalog import require_module

        self._pageindex_tool_context = None
        if not self.kb_name:
            return
        require_module("knowledge")
        from deeptutor.services.rag.pipelines.pageindex.tools import (
            build_pageindex_tool_context,
        )

        self._pageindex_tool_context = await build_pageindex_tool_context(
            self.kb_name,
            base_registry=self.registry,
        )
        if self._pageindex_tool_context is not None:
            self.registry = self._pageindex_tool_context.registry

    async def _prepare_research_providers(self, context):
        from deeptutor.multi_user.paths import current_owner_id
        from deeptutor.runtime.providers.scope import ToolScope
        from deeptutor.runtime.providers.view import build_tool_view
        from deeptutor.services.workspace.resources import current_resources
        from deeptutor.tools.research_tools import scientific_mcp_tool

        selected = current_resources().mcp
        raw_filter = (context.metadata or {}).get("mcp_tools_filter")
        scope = ToolScope(
            owner_id=current_owner_id(),
            session_id=context.session_id,
            workspace_mcp=frozenset(selected) if selected is not None else None,
            is_partner=(context.metadata or {}).get("source") == "partner",
            caller_whitelist=frozenset(str(name) for name in raw_filter)
            if isinstance(raw_filter, list)
            else None,
            exclusive_capability=bool(self._is_obsidian_kb),
        )
        view = await build_tool_view(
            base_registry=self.registry, scope=scope, language=self.language
        )
        self.registry = view.registry
        self._scientific_mcp_names = [tool.name for tool in view.pool if scientific_mcp_tool(tool)]

    def _pageindex_tool_names(self) -> list[str]:
        tool_context = getattr(self, "_pageindex_tool_context", None)
        return [tool.name for tool in tool_context.tools] if tool_context is not None else []

    def _block_tool_names(self) -> list[str]:
        """Tools available inside the per-block research loop.

        Uses the same shared composition policy chat uses
        (:func:`compose_enabled_tools`): user-toggled tools first, then
        the conditional and always-on auto-mounts. Two block-phase-only
        adjustments:

        * Only evidence-producing research tools are surfaced. Chat's
          always-on convenience tools (``write_memory``, ``web_fetch``,
          ``github``, ``ask_user``) are deliberately not part of the
          block loop because they do not provide broad citable retrieval
          for an arbitrary sub-topic.
        * Each name is filtered through the registry so an inactive tool
          (mis-configured backend, missing dep) doesn't end up in the
          prompt.
        """
        composed = compose_enabled_tools(
            registry=self.registry,
            requested_tools=self.enabled_tools,
            optional_whitelist=RESEARCH_OPTIONAL_TOOLS,
            mount_flags=ToolMountFlags(
                has_scientific_research=True,
                has_kb=bool(
                    self.kb_name
                    and not self._is_obsidian_kb
                    and not getattr(self, "_pageindex_tool_context", None)
                ),
                has_sources=False,
                has_memory=user_has_memory(),
                has_notebooks=user_has_notebooks(),
                has_exec=exec_capability_available(),
            ),
        )
        names = [
            name
            for name in composed
            if name in RESEARCH_BLOCK_TOOL_ALLOWLIST and self._tool_in_registry(name)
        ]
        if self._vault_path:
            names.extend(
                name
                for name in RESEARCH_OBSIDIAN_READ_TOOLS
                if name in RESEARCH_BLOCK_TOOL_ALLOWLIST and self._tool_in_registry(name)
            )
        names.extend(self._pageindex_tool_names())
        names.extend(getattr(self, "_scientific_mcp_names", []))
        return list(dict.fromkeys(names))

    def _build_block_tool_schemas(
        self,
        tool_names: list[str] | None = None,
    ) -> list[dict[str, Any]]:
        names = self._block_tool_names() if tool_names is None else tool_names
        schemas = self.registry.build_openai_schemas(names)
        kb_choices = [self.kb_name] if self.kb_name else []
        for schema in schemas:
            function = schema.get("function") if isinstance(schema, dict) else None
            if not isinstance(function, dict):
                continue
            parameters = function.get("parameters")
            if not isinstance(parameters, dict):
                continue
            properties = parameters.get("properties") or {}
            if function.get("name") == "rag" and isinstance(properties, dict):
                if isinstance(properties.get("query"), dict):
                    properties["query"].setdefault("minLength", 1)
                kb_schema = properties.get("kb_name")
                if isinstance(kb_schema, dict) and kb_choices:
                    kb_schema["enum"] = kb_choices
            parameters["additionalProperties"] = False
        return schemas

    def _augment_tool_kwargs(
        self,
        tool_name: str,
        args: dict[str, Any],
        context: UnifiedContext,
    ) -> dict[str, Any]:
        workspace = context.runtime.workspace
        task_dir = (
            Path(workspace.output_dir)
            if workspace is not None
            else fallback_task_dir_from_metadata(context, feature="deep_research")
        )
        kwargs = bind_workspace_tool_runtime(
            tool_name,
            args,
            context,
            fallback_task_dir=task_dir,
        )
        if tool_name == "preprint":
            from deeptutor.services.llm.capabilities import supports_vision

            kwargs["_vision_supported"] = supports_vision(self.binding, self.model)
            kwargs["_research_registry"] = self.registry
        if tool_name == "rag":
            kwargs.setdefault("mode", "hybrid")
            if self.kb_name:
                kwargs.setdefault("kb_name", self.kb_name)
        elif tool_name in RESEARCH_OBSIDIAN_READ_TOOLS:
            if self._vault_path:
                # Server-owned: overwrite any model-supplied value so the path
                # can't be forged to read outside the connected vault.
                kwargs["_vault_path"] = self._vault_path
        elif tool_name == "web_search":
            kwargs.setdefault("query", context.user_message)
            if task_dir is not None:
                kwargs.setdefault("output_dir", str(task_dir / "web_search"))
        return kwargs

    def _retrieve_trace_metadata(
        self,
        tool_meta: dict[str, Any],
        *,
        tool_name: str,
        tool_args: dict[str, Any],
    ) -> dict[str, Any] | None:
        if tool_name != "rag":
            return None
        return derive_trace_metadata(
            tool_meta,
            label=self._t("labels.retrieve", default="Retrieve"),
            call_kind="rag_retrieval",
            trace_role="retrieve",
            trace_group="retrieve",
            query=str(tool_args.get("query", "") or ""),
        )

    def _use_native_block_tools(self, tool_names: list[str] | None = None) -> bool:
        names = self._block_tool_names() if tool_names is None else tool_names
        return bool(names) and can_use_native_tool_calling(binding=self.binding, model=self.model)

    def _tool_in_registry(self, name: str) -> bool:
        try:
            return self.registry.get(name) is not None
        except Exception:
            return False

    # ------------------------------------------------------------------
    # LLM call helpers
    # ------------------------------------------------------------------
    def _build_client(self) -> Any:
        return build_openai_client(self.client_config)

    def _completion_kwargs(
        self, max_tokens: int, reasoning_effort: str | None = None
    ) -> dict[str, Any]:
        return build_completion_kwargs(
            temperature=self._temperature,
            model=self.model,
            max_tokens=max_tokens,
            binding=self.binding,
            # A step may turn thinking down for one call (a retry after the
            # model spent the whole budget on it); otherwise the configured
            # level stands.
            reasoning_effort=reasoning_effort or self.reasoning_effort,
        )

    async def _run_labeled_step(
        self,
        *,
        client: Any,
        messages: list[dict[str, Any]],
        tool_schemas: list[dict[str, Any]] | None,
        protocol: LabelProtocol,
        stream: StreamBus,
        stage: str,
        iter_meta: dict[str, Any],
        max_tokens: int | None = None,
        reasoning_effort: str | None = None,
        final_meta: dict[str, Any] | None = None,
        eager_sub_trace: bool = True,
    ) -> LabeledStepResult:
        """Research-flavoured thin wrapper over :func:`run_labeled_step`."""
        if max_tokens is None:
            max_tokens = int(self._budgets["block"]["max_tokens"])
        return await run_labeled_step(
            client=client,
            model=self.model,
            messages=messages,
            completion_kwargs=self._completion_kwargs(max_tokens, reasoning_effort),
            tool_schemas=tool_schemas,
            allowed_labels=protocol.allowed,
            final_labels=protocol.final,
            tool_label=protocol.tool_label,
            stream=stream,
            source=SOURCE,
            stage=stage,
            iter_meta=iter_meta,
            binding=self.binding,
            usage=self.usage,
            final_meta=final_meta,
            eager_sub_trace=eager_sub_trace,
        )

    # ------------------------------------------------------------------
    # Message + trace assembly
    # ------------------------------------------------------------------
    def _build_system_user_messages(
        self,
        system_prompt: str,
        user_prompt: str,
        *,
        image_attachments: list[Attachment] | None = None,
    ) -> list[dict[str, Any]]:
        messages: list[dict[str, Any]] = [
            {"role": "system", "content": system_prompt},
            {"role": "user", "content": user_prompt},
        ]
        if image_attachments:
            mm_result = prepare_multimodal_messages(
                messages, image_attachments, binding=self.binding, model=self.model
            )
            return mm_result.messages
        return messages

    def _build_simple_trace_meta(
        self,
        *,
        call_id_root: str,
        label: str,
        stage: str,
        call_kind: str = "llm_reasoning",
        trace_role: str = "thought",
        trace_group: str = "stage",
        **extra: Any,
    ) -> dict[str, Any]:
        call_id = new_call_id(call_id_root)
        return build_trace_metadata(
            call_id=call_id,
            phase=stage,
            label=label,
            call_kind=call_kind,
            trace_id=call_id,
            trace_role=trace_role,
            trace_group=trace_group,
            **extra,
        )

    # ------------------------------------------------------------------
    # YAML lookup
    # ------------------------------------------------------------------
    def _t(self, key: str, default: str = "", **kwargs: Any) -> str:
        value: Any = self._prompts
        for part in key.split("."):
            if not isinstance(value, dict) or part not in value:
                return default
            value = value[part]
        if not isinstance(value, str):
            return default
        if kwargs:
            try:
                return value.format(**kwargs)
            except (KeyError, IndexError, ValueError):
                return value
        return value


# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------

_REPORT_TOKEN_RE = re.compile(r"[^\W_]+", re.UNICODE)
_REPORT_SECTION_HEADING_RE = re.compile(r"\A##\s+(\d+)\.\s*\S")


def _report_heading_number(body: str) -> int | None:
    match = _REPORT_SECTION_HEADING_RE.match(body)
    if match is None:
        return None
    return int(match.group(1))


def _recover_report_step_reply(
    step: LabeledStepResult,
    *,
    expected_label: str,
    expected_section_number: int | None,
) -> tuple[str, str]:
    """Recover a protocol label from a finished report reply.

    The streaming 64-char probe stays strict. Once the full reply is in
    hand, accept an unclosed/over-wide fence around the expected label, or
    a missing label whose body opens with ``## {n}.`` for *this* section.
    """
    raw = step.text or ""
    body = raw.strip()
    if step.label == expected_label:
        return expected_label, body

    recovered = recover_finished_label(raw, allowed_labels=(expected_label,))
    if recovered is not None and recovered[0] == expected_label:
        return expected_label, recovered[1].strip()

    if (
        expected_section_number is not None
        and _report_heading_number(body) == expected_section_number
    ):
        return expected_label, body
    return step.label, body


def _report_step_incomplete_reason(
    step: LabeledStepResult,
    *,
    expected_label: str,
    body: str,
) -> str:
    """Return why a generated report part is unsafe to persist, or ``""``.

    Deep Research report calls are stricter than ordinary chat: an empty
    formal channel, a token-limit finish, or the labeled-step idle escape
    means the report is incomplete even though the provider call itself did
    not raise. Every report part must also contain its numbered H2 heading.
    """

    if step.label != expected_label:
        return f"expected {expected_label} label, got {step.label or 'none'}"
    if step.stream_idle_timeout:
        return "provider stream went idle before an explicit finish"
    finish_reason = (step.finish_reason or "").strip().lower()
    if finish_was_truncated(finish_reason):
        return f"provider stopped at its output limit ({finish_reason})"
    if len(body) < 80:
        return f"body is too short ({len(body)} characters)"
    if not _REPORT_SECTION_HEADING_RE.match(body):
        return "numbered report heading is missing"
    return ""


_REPORT_STOPWORDS = {
    "a",
    "an",
    "and",
    "as",
    "at",
    "by",
    "for",
    "from",
    "in",
    "into",
    "of",
    "on",
    "or",
    "the",
    "to",
    "vs",
    "with",
}


_REPORT_CITATION_MARKER_RE = re.compile(r"`?\[(?P<id>CIT-\d+-\d+|PLAN-\d+)\]`?(?!\s*\()")
_REPORT_CITATION_LINK_RE = re.compile(r"`?\[(?P<id>CIT-\d+-\d+|PLAN-\d+)\]`?\([^)]*\)")
_MARKDOWN_HEADING_RE = re.compile(r"^(?P<hashes>#{1,6})\s+(?P<title>.+?)\s*$")
_LEADING_HEADING_MARKERS_RE = re.compile(r"^(?:#{1,6}\s+)+")
_LEADING_SECTION_ID_RE = re.compile(r"^(?:[\[\(]?S\d+[\]\)]?\s*[::\-]\s*)+", re.I)


def _clean_report_heading_text(text: str) -> str:
    cleaned = _LEADING_HEADING_MARKERS_RE.sub("", (text or "").strip())
    cleaned = _LEADING_SECTION_ID_RE.sub("", cleaned).strip()
    return cleaned


def _normalise_markdown_headings(text: str) -> str:
    lines: list[str] = []
    for line in (text or "").splitlines():
        match = _MARKDOWN_HEADING_RE.match(line)
        if not match:
            lines.append(line.rstrip())
            continue
        title = _clean_report_heading_text(match.group("title"))
        hashes = match.group("hashes")
        lines.append(f"{hashes} {title}" if title else hashes)
    return "\n".join(lines)


def _citation_ids_in_first_appearance(
    text: str,
    citations: CitationManager,
) -> list[str]:
    known = set(citations.get_all_citations())
    ordered: list[str] = []
    seen: set[str] = set()
    for match in _REPORT_CITATION_MARKER_RE.finditer(text or ""):
        citation_id = match.group("id")
        if citation_id not in known or citation_id in seen:
            continue
        seen.add(citation_id)
        ordered.append(citation_id)
    return ordered


def _citation_source_preview(citation: dict[str, Any]) -> str:
    if not citation:
        return ""
    tool_type = str(citation.get("tool_type") or "").lower()
    if tool_type == "web_search":
        sources = citation.get("web_sources")
        if isinstance(sources, list):
            hints = []
            for source in sources[:3]:
                if not isinstance(source, dict):
                    continue
                title = str(source.get("title") or source.get("domain") or "").strip()
                url = str(source.get("url") or "").strip()
                if title and url:
                    hints.append(f"{title} <{url}>")
                elif url:
                    hints.append(url)
            return "; ".join(hints)
    if tool_type == "paper_search":
        papers = citation.get("papers")
        if isinstance(papers, list):
            hints = []
            for paper in papers[:3]:
                if not isinstance(paper, dict):
                    continue
                title = str(paper.get("title") or "").strip()
                year = str(paper.get("year") or "").strip()
                if title:
                    hints.append(f"{title} ({year})" if year else title)
            return "; ".join(hints)
    if tool_type in {"rag", "rag_naive", "rag_hybrid"}:
        sources = citation.get("sources")
        if isinstance(sources, list):
            hints = []
            for source in sources[:3]:
                if not isinstance(source, dict):
                    continue
                title = str(source.get("title") or source.get("source_file") or "").strip()
                page = str(source.get("page") or "").strip()
                if title:
                    hints.append(f"{title} p.{page}" if page else title)
            return "; ".join(hints)
    return ""


def _section_with_block_ids(
    section: ReportSectionPlan,
    block_ids: tuple[str, ...],
) -> ReportSectionPlan:
    return ReportSectionPlan(
        id=section.id,
        title=section.title,
        intent=section.intent,
        block_ids=block_ids,
    )


def _report_tokens(text: str) -> set[str]:
    tokens: set[str] = set()
    for raw in _REPORT_TOKEN_RE.findall((text or "").lower()):
        token = raw.strip()
        if not token or token in _REPORT_STOPWORDS:
            continue
        if len(token) > 4 and token.endswith("ies"):
            token = token[:-3] + "y"
        elif len(token) > 3 and token.endswith("s"):
            token = token[:-1]
        tokens.add(token)
    return tokens


def _report_overlap_score(left: str, right: str) -> float:
    left_tokens = _report_tokens(left)
    right_tokens = _report_tokens(right)
    if not left_tokens or not right_tokens:
        return 0.0
    overlap = left_tokens & right_tokens
    if not overlap:
        return 0.0
    return len(overlap) / max(1, min(len(left_tokens), len(right_tokens)))


def _best_block_for_section(
    section: ReportSectionPlan,
    blocks: list[ResearchedBlock],
    *,
    exclude: set[str],
) -> ResearchedBlock | None:
    candidates = [rb for rb in blocks if rb.block.block_id not in exclude] or blocks
    section_text = f"{section.title} {section.intent}"
    best: tuple[float, ResearchedBlock] | None = None
    for rb in candidates:
        block_text = f"{rb.block.sub_topic} {rb.block.overview} {rb.knowledge[:500]}"
        score = _report_overlap_score(section_text, block_text)
        if best is None or score > best[0]:
            best = (score, rb)
    return best[1] if best is not None else None


def _best_section_for_block(
    block: ResearchedBlock,
    sections: list[ReportSectionPlan],
) -> int | None:
    block_text = f"{block.block.sub_topic} {block.block.overview} {block.knowledge[:500]}"
    best: tuple[float, int] | None = None
    for idx, section in enumerate(sections):
        section_text = f"{section.title} {section.intent}"
        score = _report_overlap_score(block_text, section_text)
        if best is None or score > best[0]:
            best = (score, idx)
    if best is None or best[0] <= 0:
        return None
    return best[1]


def _citation_anchor_id(citation_id: str) -> str:
    return "ref-" + re.sub(r"[^a-z0-9_-]+", "-", citation_id.lower()).strip("-")


def _citation_markdown_link(citation_id: str, ref_number: int) -> str:
    return f'[{ref_number}](#{_citation_anchor_id(citation_id)} "citation")'


def _sorted_citation_ids(citations: CitationManager) -> list[str]:
    return sorted(citations.get_all_citations(), key=_citation_sort_key)


def _citation_number_map(citations: CitationManager) -> dict[str, int]:
    return {
        citation_id: index
        for index, citation_id in enumerate(_sorted_citation_ids(citations), start=1)
    }


def _citation_sort_key(citation_id: str) -> tuple[int, int, int]:
    """Sort PLAN and per-block CIT ids in the same order users see them."""
    try:
        if citation_id.startswith("PLAN-"):
            return (0, 0, int(citation_id.replace("PLAN-", "", 1)))
        if citation_id.startswith("CIT-"):
            block_num, seq_num = citation_id.replace("CIT-", "", 1).split("-", 1)
            return (1, int(block_num), int(seq_num))
    except (IndexError, ValueError):
        pass
    return (999, 999, 999)


def _topic_index_from_block_id(block_id: str) -> int | None:
    match = re.search(r"block_(\d+)", block_id or "")
    if match is None:
        return None
    try:
        return int(match.group(1))
    except ValueError:
        return None


def _research_topic_status_meta(block: TopicBlock) -> dict[str, Any]:
    return {
        "research_status_key": "research_topic",
        "topic_index": _topic_index_from_block_id(block.block_id),
        "topic_title": block.sub_topic,
    }


def _read_int(cfg: Any, *, key: str, default: int) -> int:
    if isinstance(cfg, dict):
        value = cfg.get(key, default)
    else:
        value = default
    try:
        return int(value)
    except (TypeError, ValueError):
        return default


# ---------------------------------------------------------------------------
# _BlockLoopHost — per-block research loop callbacks
# ---------------------------------------------------------------------------


class _BlockLoopHost:
    """Binds a ``ResearchPipeline`` + one ``TopicBlock`` + queue + citation
    store so the generic agentic loop can call back for per-block side
    effects.

    Three loop hooks are implemented beyond the default protocol:
    * :meth:`dispatch_tools` — wraps the standard parallel dispatch with
      a note-agent sidecar that summarises citable tool results and
      records them in :class:`CitationManager`. The summary then
      replaces the raw tool message content sent back to the model so
      long retrievals don't blow the context budget.
    * :meth:`on_intermediate` — handles the ``APPEND`` label. Parses
      ``<title>\n<overview?>`` out of the post-label text, runs dedup
      + capacity checks, appends a child block to the queue, and
      returns a confirmation / rejection message that the loop injects
      as the next iteration's user feedback.
    * :meth:`force_finalize` — drives the per-block force-finish
      recovery when the iteration budget is exhausted before a
      terminal label fires.
    """

    def __init__(
        self,
        *,
        pipeline: "ResearchPipeline",
        block: TopicBlock,
        queue: DynamicTopicQueue,
        citations: CitationManager,
        topic: str,
        stream: StreamBus,
        context: UnifiedContext,
        client: Any,
    ) -> None:
        self._pipeline = pipeline
        self._block = block
        self._queue = queue
        self._citations = citations
        self._topic = topic
        self._stream = stream
        self._context = context
        self._client = client
        self._tool_rounds_used = 0

    async def guard_context_window(self, messages: list[dict[str, Any]]) -> None:
        # Per-block budgets keep messages bounded; no trimming for v1.
        return None

    def build_iteration_trace_meta(self, iteration: int) -> tuple[dict[str, Any], dict[str, Any]]:
        status_meta = _research_topic_status_meta(self._block)
        iter_call_id = new_call_id(f"research-{self._block.block_id}-iter-{iteration}")
        iter_meta = build_trace_metadata(
            call_id=iter_call_id,
            phase="researching",
            label=self._pipeline._t("labels.reasoning", default="Reasoning"),
            call_kind="llm_reasoning",
            trace_id=iter_call_id,
            trace_role="thought",
            trace_group="stage",
            block_id=self._block.block_id,
            **status_meta,
        )
        final_call_id = new_call_id(f"research-{self._block.block_id}-final")
        final_meta = build_trace_metadata(
            call_id=final_call_id,
            phase="researching",
            label=(
                f"{self._pipeline._t('labels.research_step', default='Research step')}: "
                f"{self._block.sub_topic}"
            ),
            call_kind="llm_final_response",
            trace_id=final_call_id,
            trace_role="response",
            trace_group="stage",
            block_id=self._block.block_id,
            **status_meta,
        )
        return iter_meta, final_meta

    async def dispatch_tools(
        self,
        *,
        iteration: int,
        tool_calls: list[dict[str, Any]],
    ) -> DispatchOutcome:
        too_many = None
        if len(tool_calls) > MAX_PARALLEL_TOOL_CALLS:
            too_many = self._pipeline._t(
                "notices.too_many_tool_calls",
                requested=len(tool_calls),
                limit=MAX_PARALLEL_TOOL_CALLS,
            )
        outcome = await dispatch_tool_calls(
            tool_calls=tool_calls,
            context=self._context,
            stream=self._stream,
            source=SOURCE,
            stage="researching",
            iteration_index=iteration,
            registry=self._pipeline.registry,
            kwarg_augmenter=self._pipeline._augment_tool_kwargs,
            retrieve_meta_factory=lambda meta, tn, ta: self._pipeline._retrieve_trace_metadata(
                meta, tool_name=tn, tool_args=ta
            ),
            tool_call_label=self._pipeline._t("labels.tool_call", default="Tool call"),
            retrieve_label=self._pipeline._t("labels.retrieve", default="Retrieve"),
            empty_tool_result_message=self._pipeline._t("notices.empty_tool_result"),
            start_retrieval_message=self._pipeline._t(
                "notices.start_retrieval", default="Starting retrieval"
            ),
            too_many_tool_calls_message=too_many,
            tool_error_message_factory=tool_error_message_factory(self._pipeline._t),
            trace_id_prefix=f"research-{self._block.block_id}-iter",
            tool_timeout=self._pipeline.tool_timeout,
            tool_max_retries=self._pipeline.tool_max_retries,
        )
        pageindex_sources = [
            source for source in outcome.sources if source.get("type") == "pageindex"
        ]
        if pageindex_sources:
            await self._stream.sources(pageindex_sources, source=SOURCE, stage="researching")
        if tool_calls:
            self._tool_rounds_used += 1
        await self._summarise_and_record(tool_calls, outcome)
        return outcome

    async def validate_terminal(self, label: str, text: str) -> str | None:
        """Reject a first-turn FINISH when native evidence tools are callable.

        The prompt tells the model to call a tool before FINISH, but some
        models still jump straight to synthesis. This hook makes that rule
        executable instead of prompt-only.
        """
        if (
            label == LABEL_FINISH
            and self._tool_rounds_used <= 0
            and self._pipeline._use_native_block_tools()
        ):
            return "finish_without_tool"
        return None

    async def _summarise_and_record(
        self,
        tool_calls: list[dict[str, Any]],
        outcome: DispatchOutcome,
    ) -> None:
        """Walk every emitted tool message; for citable tools, summarise
        the raw answer through ``_summarise_tool_result``, register the
        result in :class:`CitationManager`, and substitute the summary
        back into the tool message body the LLM sees."""
        if not outcome.tool_messages:
            return
        call_meta_by_id: dict[str, tuple[str, dict[str, Any]]] = {}
        for tc in tool_calls:
            cid = tc.get("id") or ""
            name = str(tc.get("name") or "")
            raw_args = tc.get("arguments") or {}
            if isinstance(raw_args, str):
                parsed = parse_json_response(raw_args, fallback={})
                args = parsed if isinstance(parsed, dict) else {}
            elif isinstance(raw_args, dict):
                args = raw_args
            else:
                args = {}
            call_meta_by_id[cid] = (name, args)

        for tm in outcome.tool_messages:
            tool_call_id = str(tm.get("tool_call_id") or "")
            tool_name, tool_args = call_meta_by_id.get(tool_call_id, ("", {}))
            if not _is_citable_tool(tool_name):
                continue
            raw_answer = str(tm.get("content") or "")
            if not raw_answer.strip():
                continue
            try:
                query_key = {
                    "obsidian_read": "note",
                    "obsidian_list": "folder",
                }.get(tool_name, "query")
                query = str(tool_args.get(query_key) or "")
                if tool_name == "obsidian_list" and not query:
                    query = "/"
                summary = await self._pipeline._summarise_tool_result(
                    tool_name=tool_name,
                    query=query,
                    raw_answer=raw_answer,
                    client=self._client,
                )
                citation_id = await self._citations.generate_research_citation_id_async(
                    self._block.block_id
                )
                trace = ToolTrace.create_with_size_limit(
                    tool_id=f"{self._block.block_id}-tool-{citation_id}",
                    citation_id=citation_id,
                    tool_type=tool_name,
                    query=query,
                    raw_answer=raw_answer,
                    summary=summary,
                )
                self._block.add_tool_trace(trace)
                tool_metadata = outcome.tool_metadata_by_id.get(tool_call_id)
                await self._citations.add_citation_async(
                    citation_id, tool_name, trace, raw_answer, tool_metadata
                )
                tm["content"] = f"[{citation_id}] {summary}"
            except Exception:
                logger.exception(
                    "Failed to record research citation for %s in %s",
                    tool_name,
                    self._block.block_id,
                )

    async def resolve_pause(self, dispatch: DispatchOutcome) -> bool:
        # Per-block research never surfaces ask_user; clarification only
        # happens in the rephrase phase.
        return False

    async def emit_terminator(self, payload: dict[str, Any] | None) -> None:
        return None

    async def emit_final(self, text: str, final_meta: dict[str, Any]) -> None:
        # Block FINISH text is consumed by the reporting phase, not
        # streamed live as user-facing content — streaming it here would
        # dump raw per-block findings into the chat bubble before the
        # actual report begins.
        return None

    def protocol_retry_notice(self) -> str:
        return self._pipeline._t(
            "notices.protocol_retry",
            default="The model violated the action-label protocol; retrying.",
        )

    def protocol_repair_message(self, violation: str) -> str:
        return self._pipeline._t(
            f"protocol.{violation}",
            default=f"Protocol violation: {violation}.",
        )

    async def force_finalize(
        self,
        *,
        messages: list[dict[str, Any]],
        start_iteration: int,
    ) -> tuple[str, bool, int]:
        return await self._pipeline._force_finish_block(
            client=self._client,
            messages=messages,
            stream=self._stream,
            block=self._block,
            start_iteration=start_iteration,
        )

    async def on_intermediate(self, label: str, text: str) -> str | None:
        """Parse ``APPEND`` payload and extend the queue.

        Format: first line = title, remainder (optional) = overview.
        Empty / duplicate / over-capacity proposals get a rejection note
        the loop injects as the next iteration's user message so the LLM
        can adapt rather than spam the same proposal.
        """
        if label != LABEL_APPEND:
            return None
        first_line, _, rest = (text or "").strip().partition("\n")
        # Strip leading markdown heading markers (``#``, ``##``, …) so the
        # queue stores a clean title even if the LLM rendered the new
        # sub-topic as a markdown header.
        title = first_line.strip().lstrip("#").strip()
        overview = rest.strip()
        if not title:
            return self._pipeline._t(
                "notices.append_rejected_empty",
                default="APPEND rejected: missing title.",
            )

        if self._queue.is_full():
            await self._stream.progress(
                self._pipeline._t(
                    "notices.append_rejected_full_progress",
                    title=title,
                    default=f"Queue is full; rejected append: {title}",
                ),
                source=SOURCE,
                stage="researching",
                metadata={
                    "trace_kind": "queue_append_rejected",
                    "block_id": self._block.block_id,
                    "reason": "full",
                    "title": title,
                },
            )
            return self._pipeline._t(
                "notices.append_rejected_full",
                default=(
                    "APPEND rejected: the topic queue is at capacity. Continue "
                    "researching the current block and emit FINISH when done."
                ),
            )

        dup = self._queue.find_similar(title)
        if dup is not None:
            await self._stream.progress(
                self._pipeline._t(
                    "notices.append_rejected_dup_progress",
                    title=title,
                    existing=dup.block_id,
                    default=(
                        f"APPEND rejected: '{title}' is too similar to "
                        f"existing block {dup.block_id}"
                    ),
                ),
                source=SOURCE,
                stage="researching",
                metadata={
                    "trace_kind": "queue_append_rejected",
                    "block_id": self._block.block_id,
                    "reason": "duplicate",
                    "title": title,
                    "existing_id": dup.block_id,
                },
            )
            return self._pipeline._t(
                "notices.append_rejected_duplicate",
                existing_id=dup.block_id,
                existing_title=dup.sub_topic,
                default=(
                    f"APPEND rejected: too similar to existing block "
                    f"{dup.block_id} ({dup.sub_topic!r})."
                ),
            )

        new_block = self._queue.append_child(parent=self._block, sub_topic=title, overview=overview)
        if new_block is None:
            return self._pipeline._t(
                "notices.append_rejected_full",
                default="APPEND rejected: queue is full.",
            )

        await self._stream.progress(
            self._pipeline._t(
                "notices.append_accepted_progress",
                title=title,
                new_block_id=new_block.block_id,
                default=f"Sub-topic queued: {title}",
            ),
            source=SOURCE,
            stage="researching",
            metadata={
                "trace_kind": "queue_append",
                "block_id": self._block.block_id,
                "parent_block_id": self._block.block_id,
                "new_block_id": new_block.block_id,
                "title": title,
            },
        )
        return self._pipeline._t(
            "notices.append_accepted",
            new_block_id=new_block.block_id,
            title=title,
            default=f"Appended block {new_block.block_id}: {title}",
        )


# ---------------------------------------------------------------------------
# _RephraseLoopHost — ask_user-only mini loop
# ---------------------------------------------------------------------------


class _RephraseLoopHost:
    """Drives the rephrase mini-loop. Only ``ask_user`` is allowed as a
    tool; the round cap (``rephrase_max_rounds``) is enforced here. When
    exhausted, any further ``ask_user`` call is replied to with a tool
    message instructing the model to FINISH with the best refined topic
    it has."""

    def __init__(
        self,
        *,
        pipeline: "ResearchPipeline",
        stream: StreamBus,
        context: UnifiedContext,
        client: Any,
        max_rounds: int,
    ) -> None:
        self._pipeline = pipeline
        self._stream = stream
        self._context = context
        self._client = client
        self._max_rounds = max(0, int(max_rounds))
        self._rounds_used = 0
        # Reuse one call_id across all rephrase iterations so the FE
        # groups every THINK trace from before *and* after ``ask_user``
        # into a single "Rephrasing" reasoning card. Without this each
        # iter would get its own card and the post-answer reasoning
        # would appear as a brand-new box below the user's reply.
        self._shared_iter_call_id = new_call_id("research-rephrase-iter")

    async def guard_context_window(self, messages: list[dict[str, Any]]) -> None:
        return None

    def build_iteration_trace_meta(self, iteration: int) -> tuple[dict[str, Any], dict[str, Any]]:
        iter_meta = build_trace_metadata(
            call_id=self._shared_iter_call_id,
            phase="rephrasing",
            label=self._pipeline._t("labels.rephrase", default="Rephrase"),
            call_kind="llm_reasoning",
            trace_id=self._shared_iter_call_id,
            trace_role="thought",
            trace_group="stage",
        )
        final_call_id = new_call_id(f"research-rephrase-final-{iteration}")
        final_meta = build_trace_metadata(
            call_id=final_call_id,
            phase="rephrasing",
            label=self._pipeline._t("labels.rephrase", default="Rephrase"),
            call_kind="llm_final_response",
            trace_id=final_call_id,
            trace_role="response",
            trace_group="stage",
        )
        return iter_meta, final_meta

    async def dispatch_tools(
        self,
        *,
        iteration: int,
        tool_calls: list[dict[str, Any]],
    ) -> DispatchOutcome:
        allowed: list[dict[str, Any]] = []
        rejected: list[dict[str, Any]] = []
        for tc in tool_calls:
            if tc.get("name") == "ask_user":
                allowed.append(tc)
            else:
                rejected.append(
                    {
                        "role": "tool",
                        "tool_call_id": tc.get("id"),
                        "name": tc.get("name", ""),
                        "content": self._pipeline._t(
                            "notices.rephrase_only_ask_user",
                            tool=tc.get("name", ""),
                            default=(
                                "Only `ask_user` is available in this phase. "
                                "Use it or emit FINISH with the refined topic."
                            ),
                        ),
                    }
                )

        if self._rounds_used >= self._max_rounds and allowed:
            cap_messages = [
                {
                    "role": "tool",
                    "tool_call_id": tc.get("id"),
                    "name": "ask_user",
                    "content": self._pipeline._t(
                        "notices.rephrase_cap_reached",
                        max_rounds=self._max_rounds,
                        default=(
                            f"ask_user limit reached ({self._max_rounds} "
                            "rounds). Emit FINISH now with the best refined "
                            "topic you can produce from prior answers."
                        ),
                    ),
                }
                for tc in allowed
            ]
            return DispatchOutcome(sources=[], tool_messages=cap_messages + rejected)

        if not allowed:
            return DispatchOutcome(sources=[], tool_messages=rejected)

        too_many = None
        if len(allowed) > MAX_PARALLEL_TOOL_CALLS:
            too_many = self._pipeline._t(
                "notices.too_many_tool_calls",
                requested=len(allowed),
                limit=MAX_PARALLEL_TOOL_CALLS,
            )
        outcome = await dispatch_tool_calls(
            tool_calls=allowed,
            context=self._context,
            stream=self._stream,
            source=SOURCE,
            stage="rephrasing",
            iteration_index=iteration,
            registry=self._pipeline.registry,
            kwarg_augmenter=self._pipeline._augment_tool_kwargs,
            retrieve_meta_factory=lambda meta, tn, ta: None,
            tool_call_label=self._pipeline._t("labels.tool_call", default="Tool call"),
            retrieve_label=self._pipeline._t("labels.retrieve", default="Retrieve"),
            empty_tool_result_message=self._pipeline._t("notices.empty_tool_result"),
            start_retrieval_message=self._pipeline._t(
                "notices.start_retrieval", default="Starting retrieval"
            ),
            too_many_tool_calls_message=too_many,
            tool_error_message_factory=tool_error_message_factory(self._pipeline._t),
            trace_id_prefix="research-rephrase-iter",
        )
        if rejected:
            outcome.tool_messages.extend(rejected)
        self._rounds_used += 1
        return outcome

    async def resolve_pause(self, dispatch: DispatchOutcome) -> bool:
        from deeptutor.agents.loop.pipeline import (
            _format_user_reply_body,
            _normalise_user_reply,
        )

        ask_user = (dispatch.pause_payload or {}).get("ask_user") or {}
        waiter = self._context.runtime.wait_for_user_reply
        if not callable(waiter):
            return False
        raw_reply = await waiter()
        if raw_reply is None:
            return False
        reply_text, answers = _normalise_user_reply(raw_reply)
        body_text = _format_user_reply_body(reply_text, answers, ask_user)
        for tm in dispatch.tool_messages:
            if tm.get("tool_call_id") == dispatch.pause_tool_call_id:
                tm["content"] = (
                    f"{body_text}\n\n[ask_user resolved. Continue to FINISH "
                    "with the refined research topic when you have enough "
                    "information.]"
                )
                break
        progress_meta: dict[str, Any] = {
            "trace_kind": "user_reply",
            "ask_user_resolved": True,
            "ask_user_tool_call_id": dispatch.pause_tool_call_id,
            "reply_preview": (reply_text or "")[:200],
        }
        if answers:
            progress_meta["answers"] = list(answers)
        await self._stream.progress("", source=SOURCE, stage="rephrasing", metadata=progress_meta)
        return True

    async def emit_terminator(self, payload: dict[str, Any] | None) -> None:
        return None

    async def emit_final(self, text: str, final_meta: dict[str, Any]) -> None:
        # The refined topic is internal; not streamed as user content.
        return None

    def protocol_retry_notice(self) -> str:
        return self._pipeline._t(
            "notices.protocol_retry",
            default="The model violated the action-label protocol; retrying.",
        )

    def protocol_repair_message(self, violation: str) -> str:
        return self._pipeline._t(
            f"protocol.{violation}",
            default=f"Protocol violation: {violation}.",
        )

    async def force_finalize(
        self,
        *,
        messages: list[dict[str, Any]],
        start_iteration: int,
    ) -> tuple[str, bool, int]:
        # Rephrase exhaustion falls back to the raw topic (handled by
        # the caller); we report no extra calls and "not completed" so
        # the loop returns with an empty final_text.
        return ("", False, 0)


# Re-export ``Awaitable`` so host implementations can type their
# coroutine return values without an extra import (mirrors solve).
_ = Awaitable  # type: ignore[assignment]


__all__ = [
    "CITABLE_TOOLS",
    "LABEL_APPEND",
    "LABEL_CONCLUSION",
    "LABEL_FINISH",
    "LABEL_INTRO",
    "LABEL_OUTLINE",
    "LABEL_SECTION",
    "LABEL_THINK",
    "LABEL_TOOL",
    "RESEARCH_BLOCK_TOOL_ALLOWLIST",
    "RESEARCH_OBSIDIAN_READ_TOOLS",
    "ResearchPipeline",
    "ResearchedBlock",
    "ReportOutline",
    "ReportSectionPlan",
    "SOURCE",
    "SubTopicItem",
]