Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

73.9 KB
Raw Blame
import importlib
import re
from typing import Any, Dict, List, Optional
from datetime import datetime, UTC

from langchain_core.language_models import BaseChatModel
from loguru import logger

# Fix circular import by importing directly from source modules
from .config.llm_config import get_llm
from .config.thread_settings import get_setting_from_snapshot
from .search_system import AdvancedSearchSystem
from .text_optimization.citation_formatter import (
    LDR_APPENDED_SOURCES_SENTINEL,
)
from .utilities.json_utils import get_llm_response_text
from .utilities.llm_utils import invoke_llm_sync

# Default constants for context accumulation to avoid repetition
# These are used as fallbacks when settings are not available
DEFAULT_MAX_CONTEXT_SECTIONS = (
    3  # Number of previous sections to include as context
)
DEFAULT_MAX_CONTEXT_CHARS = (
    4000  # Max characters for context (safe for smaller local models)
)

# Spelled-out output rules appended to every per-subsection prompt. Small
# and/or quantized local models routinely ignore long, vague instructions,
# so this is intentionally short, numbered, and refers to the actual
# rendering consequences (a duplicate heading, a duplicate bibliography)
# rather than abstract guidance.
#
# The framework always (a) inserts `## i.j Name` itself before appending
# LLM content and (b) appends one consolidated `## Sources` block to the
# whole report at the end of _format_final_report. The LLM does not need
# to repeat either of those things, and historically has — producing
# visibly stacked duplicate headings and duplicate "## Sources" sections
# per subsection.
_SUBSECTION_OUTPUT_GUIDANCE = (
    "\n\nOUTPUT FORMAT RULES (FOLLOW EXACTLY):\n"
    "1. Do NOT start your output with a Markdown heading of any level "
    "(#, ##, ###, ####, etc.). The framework already inserts the "
    "subsection heading for you; starting with your own heading line "
    "creates a visible duplicate next to it in the final report.\n"
    "2. Do NOT end your output with a sources, references, bibliography, "
    "citations, key references, or selected bibliography section or list. "
    "Do NOT include any trailing heading or label (such as 'Sources:', "
    "'**Sources:**', or 'References:'), and do NOT append a list or block "
    "of citations/links at the end of your response. Citations must ONLY "
    "appear inline within sentences (e.g. 'According to [1], ...'). "
    "The framework automatically compiles and appends the consolidated "
    "source list after all subsections are completed; including your own "
    "trailing sources or bibliography duplicates it.\n"
    "3. Begin your output with prose (a paragraph or a table), not a "
    "heading and not an italic purpose statement. You may use '###' / "
    "'####' / deeper levels for internal sub-subheadings inside this "
    "subsection only."
)

# Pre-compiled normalisers used by _strip_subsection_boilerplate. These
# are best-effort cleanups that run on the LLM's raw `current_knowledge`
# before it is appended to the section — they do NOT touch the
# framework's own headings or its trailing '## Sources' block.
#
# Leading heading: any ATX heading (level 1-6) at the very start of the
# subsection content. Limited to a single match so we never strip a real
# sub-subheading used to organise the body. Real reports show the LLM
# almost always opens with a redundant ``### <subsection name>`` (or a
# sibling's name, or a Roman-numeral-prefixed restatement) immediately
# under the framework's own ``## i.j Name`` heading.
_LEADING_HEADING_RE = re.compile(
    r"\A[ \t\n]*[ \t]{0,3}#{1,6}[ \t]+[^\n]*",
)

# Leading italic purpose statement that mirrors the framework's
# ``_<purpose>_`` subtitle (e.g. ``_To summarise the section's scope..._``).
# Supports single underscores (_..._) or single asterisks (*...*).
# Limited to a single match at the start.
_LEADING_ITALIC_PURPOSE_RE = re.compile(
    r"\A[ \t\n]*(?:_[^_\n]+_|(?:\*(?!\*)[^*\n]+\*))[ \t]*",
)

_BIB_KEYWORD_PATTERN = (
    r"(?:"
    r"Sources?|References?|Bibliograph(?:y|ies)|Citations?|"
    r"Key[ \t]+References?|Selected[ \t]+Bibliograph(?:y|ies)|"
    r"Works?[ \t]+Cited|Cited[ \t]+Works?|"
    r"Additional[ \t]+Resources?|Further[ \t]+Reading"
    r")"
)

_BIB_HEADINGS_CHAIN = (
    rf"{_BIB_KEYWORD_PATTERN}"
    rf"(?:[ \t]*(?:,|and|&|/)[ \t]+{_BIB_KEYWORD_PATTERN})*"
)

# ATX-style heading (with #) allowing:
# - levels 1-6 with optional empty heading line preceding it (e.g. "###\n**Sources:**")
# - optional section numbers (e.g. "1." or "1.1")
# - optional bold/italic markers ("**Sources**", "*References:*")
# - optional trailing colons, dashes, periods, or qualifiers
_ATX_BIB_HEADING = (
    rf"[ \t]{{0,3}}#{{1,6}}[ \t]*(?:\n[ \t]*)?"
    rf"(?:\d+(?:\.\d+)*\.?[ \t]+)?"
    rf"(?:\*\*|\*|__|_)?{_BIB_HEADINGS_CHAIN}(?:\*\*|\*|__|_)?(?:"
    rf"[ \t]*[:\-\u2013\u2014.][ \t]*[^\n]*"
    rf"|[ \t]+for[ \t]+[^\n]+"
    rf"|[ \t]*\([^\n]+\)"
    rf")?[ \t]*$"
)

# Pseudo-heading (without #) where the keyword line acts as a section header:
# - bold or italic wrapped: "**Sources:**", "**Sources**", "*References:*"
# - or plain keyword ending with a colon or dash: "Sources:", "References:"
_PSEUDO_BIB_HEADING = (
    r"[ \t]{0,3}"
    rf"(?:"
    rf"(?:\*\*|\*|__|_){_BIB_HEADINGS_CHAIN}[:\-\u2013\u2014.]?(?:\*\*|\*|__|_)"
    rf"|"
    rf"{_BIB_HEADINGS_CHAIN}[ \t]*[:\-\u2013\u2014]"
    rf")[ \t]*$"
)

# Bibliography-style heading at levels 1-6 or pseudo-heading line.
_BIBLIOGRAPHY_HEADING_RE = re.compile(
    rf"(?m)^(?:{_ATX_BIB_HEADING}|{_PSEUDO_BIB_HEADING})",
    re.IGNORECASE,
)

# Next ATX heading at levels 1-6 — marks the end of a bibliography block
# when more subsection content follows it (common when the LLM appends a
# "Selected Bibliography" mid-stream and then continues writing).
_NEXT_HEADING_RE = re.compile(r"(?m)^[ \t]{0,3}#{1,6}[ \t]+")

# Decorative horizontal rules the LLM often wraps around bibliography
# blocks (``---``, ``***``, or ``___`` on their own line).
_HR_LINE_RE = re.compile(
    r"(?m)^[ \t]{0,3}(?:(?:-[ \t]*){3,}|(?:_[ \t]*){3,}|(?:\*[ \t]*){3,})[ \t]*\n?"
)

# Empty heading line (e.g. `###`) or decorative horizontal rule line (`---`, `***`, `___`)
_EMPTY_HEADING_OR_HR_RE = re.compile(
    r"^[ \t]*(?:#{1,6}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,}|(?:\*[ \t]*){3,})[ \t]*$"
)

# Reference-style link definitions in markdown: [1]: https://... or [name]: url
_MD_LINK_DEF_RE = re.compile(r"^[ \t]*\[[^\]]+\]:[ \t]*\S+")

# Ordinary ordered or bulleted list item prefix
_ORDERED_OR_BULLET_LIST_RE = re.compile(r"^[ \t]*(?:[-*•]|\d+[\.\)])\s+")

# Citation-list shape for ATX headings (## Sources, ### References):
# digit+dot, bullet, bracket citation start (single or double brackets),
# lenticular brackets, URL, DOI, or markdown link.
# Excludes markdown reference-style link definitions ("[1]: https://...").
_CITATION_LINE_RE = re.compile(
    r"^\s*(?:"
    r"\d+[\.\)]\s+"
    r"|[-*•]\s+"
    r"|\[\[\d+\]\](?:\([^)]*\))?"
    r"|\[\d+\](?!\:)"
    r"|【\d+】"
    r"|\[cite\d*\]"
    r"|\bhttps?://"
    r"|\bdoi:\s*\S+"
    r"|\[[^\]]+\]\((?:https?://|/library/)[^)]+\)"
    r")",
    re.IGNORECASE,
)

# Citation-list shape for pseudo-headings (Sources:, **Sources** without #):
# Must carry explicit citation markers (brackets, links, URLs, DOIs), NOT plain numbered/bullet lists.
_PSEUDO_BIB_CITATION_LINE_RE = re.compile(
    r"^\s*(?:(?:[-*•]|\d+[\.\)])\s+)?"
    r"(?:"
    r"\[\[\d+\]\](?:\([^)]*\))?"
    r"|\[\d+\](?!\:)"
    r"|【\d+】"
    r"|\[cite\d*\]"
    r"|\bhttps?://"
    r"|\bdoi:\s*\S+"
    r"|\[[^\]]+\]\((?:https?://|/library/)[^)]+\)"
    r")",
    re.IGNORECASE,
)

# Single citation token for concatenated line check:
# Supports single brackets [1], double brackets [[1]], optional markdown link target,
# [cite1], or Chinese/Japanese lenticular brackets 【1】
_CONCATENATED_CITATION_ITEM = (
    r"(?:(?:\[\[\d+\]\]|\[\d+\]|\[cite\d*\])(?:\([^)]*\))?|【\d+】)"
)

# Line composed purely of citation markers, brackets, links, and commas.
# Structured without overlapping leading/trailing whitespace quantifiers to prevent ReDoS.
_CONCATENATED_CITATIONS_LINE_RE = re.compile(
    rf"^(?:[,\s]*{_CONCATENATED_CITATION_ITEM})+[,\s]*$"
)

# Unheaded citation list item start (for trailing citation blocks lacking any heading).
# Excludes numbered/bullet lists, markdown link definitions, and bare URLs without citation markers.
# Requires double brackets [[...]], single brackets with link target/source, cite tag, or DOI.
_UNHEADED_CITATION_ITEM_RE = re.compile(
    r"^\s*(?:"
    r"\[\[\d+\]\](?:\([^)]*\))?"
    r"|\[\d+\](?!\:)(?:\((?:https?://|/library/)[^)]+\)|\s+(?:https?://|doi:))"
    r"|\[cite\d*\]"
    r"|doi:\s*\S+"
    r")",
    re.IGNORECASE,
)

# Note line matching LLM boilerplate (e.g. "*(Note: ... bibliography ...)*" or
# "[Note: The framework will append...]")
_BIB_NOTE_RE = re.compile(
    r"^\s*(?:"
    r"[\*_]\s*\(?Note\b[^\n]*?\b(?:bibliography|sources?|references?|citations?)\b"
    r"|\[\s*Note\b"
    r"(?=[^\n]*\b(?:consolidat\w*|append\w*|omit\w*|duplicat\w*|placeholder\w*)\b)"
    r"[^\n]*?\b(?:bibliography|sources?|references?|citations?)\b)",
    re.IGNORECASE,
)


def _get_code_fence_spans(text: str) -> List[tuple[int, int]]:
    """Return character index ranges (start, end) that are inside code blocks.

    Recognizes:
    - Backtick fenced blocks (``` or ```` etc.), paired by delimiter type and length.
    - Tilde fenced blocks (~~~ or ~~~~ etc.), paired by delimiter type and length.
    - Indented code blocks (lines indented by 4+ spaces or tabs, preceded by a blank line).
    """
    spans: List[tuple[int, int]] = []
    if not text:
        return spans

    lines = text.splitlines(keepends=True)
    in_fence = False
    fence_char = ""
    fence_len = 0
    fence_start = 0

    in_indented = False
    indented_start = 0
    last_indented_end = 0
    prev_blank = True

    line_offset = 0
    fence_open_re = re.compile(r"^[ \t]{0,3}(`{3,}|~{3,})")

    for line in lines:
        line_start = line_offset
        line_end = line_offset + len(line)
        line_offset = line_end

        stripped = line.rstrip("\r\n")

        if in_fence:
            close_re = (
                rf"^[ \t]{{0,3}}{re.escape(fence_char)}{{{fence_len},}}[ \t]*$"
            )
            if re.match(close_re, stripped):
                in_fence = False
                spans.append((fence_start, line_end))
            continue

        open_match = fence_open_re.match(stripped)
        if open_match:
            if in_indented:
                in_indented = False
                spans.append((indented_start, last_indented_end))

            in_fence = True
            delim = open_match.group(1)
            fence_char = delim[0]
            fence_len = len(delim)
            fence_start = line_start
            prev_blank = False
            continue

        is_indented = stripped.startswith("    ") or stripped.startswith("\t")
        is_blank = not stripped.strip()

        if in_indented:
            if is_indented:
                last_indented_end = line_end
            elif is_blank:
                pass
            else:
                in_indented = False
                spans.append((indented_start, last_indented_end))
        else:
            if is_indented and prev_blank:
                in_indented = True
                indented_start = line_start
                last_indented_end = line_end

        prev_blank = is_blank

    if in_fence:
        spans.append((fence_start, len(text)))
    elif in_indented:
        spans.append((indented_start, last_indented_end))

    return spans


def _is_in_spans(pos: int, spans: List[tuple[int, int]]) -> bool:
    """Return True if pos falls within any range in spans."""
    return any(start <= pos < end for start, end in spans)


def get_report_generator(search_system=None):
    """Return an instance of the report generator with default settings.

    Args:
        search_system: Optional existing AdvancedSearchSystem to use
    """
    return IntegratedReportGenerator(search_system=search_system)


class IntegratedReportGenerator:
    def __init__(
        self,
        searches_per_section: int = 2,
        search_system=None,
        llm: BaseChatModel | None = None,
        settings_snapshot: Optional[Dict] = None,
    ):
        """
        Args:
            searches_per_section: Number of searches to perform for each
                section in the report.
            search_system: Custom search system to use, otherwise just uses
                the default.
            llm: Custom LLM to use. Required if search_system is not provided.
            settings_snapshot: Optional settings snapshot for configurable values.

        """
        # If search_system is provided, use its LLM; otherwise use the provided LLM
        self._owns_llm = False
        if search_system:
            self.search_system = search_system
            self.model = llm or search_system.model
        elif llm:
            self.model = llm
            self.search_system = AdvancedSearchSystem(llm=self.model)  # type: ignore[call-arg]
        else:
            # Fallback for backwards compatibility - will only work with auth
            self._owns_llm = True
            self.model = get_llm()
            self.search_system = AdvancedSearchSystem(llm=self.model)  # type: ignore[call-arg]

        self.searches_per_section = (
            searches_per_section  # Control search depth per section
        )

        # Load context settings from snapshot or use defaults
        self.max_context_sections = get_setting_from_snapshot(
            "report.max_context_sections",
            default=DEFAULT_MAX_CONTEXT_SECTIONS,
            settings_snapshot=settings_snapshot,
        )
        self.max_context_chars = get_setting_from_snapshot(
            "report.max_context_chars",
            default=DEFAULT_MAX_CONTEXT_CHARS,
            settings_snapshot=settings_snapshot,
        )

    def close(self) -> None:
        """Close the LLM client if this instance created it."""
        from .utilities.resource_utils import safe_close

        if self._owns_llm:
            safe_close(self.model, "report generator LLM")

    def generate_report(
        self,
        initial_findings: Dict,
        query: str,
        progress_callback=None,
    ) -> Dict:
        """Generate a complete research report with section-specific research.

        Args:
            initial_findings: Results from initial research phase.
            query: Original user query.
            progress_callback: Optional callable(message, progress_percent, metadata)
                for reporting progress (0-100%) and checking cancellation.
        """

        # Step 1: Determine structure
        if progress_callback:
            progress_callback(
                "Determining report structure",
                0,
                {"phase": "report_structure"},
            )
        structure = self._determine_report_structure(initial_findings, query)

        # Step 2: Research and generate content for each section in one step
        sections = self._research_and_generate_sections(
            initial_findings,
            structure,
            query,
            progress_callback=progress_callback,
        )

        # Step 3: Format final report
        if progress_callback:
            progress_callback(
                "Formatting final report",
                90,
                {"phase": "report_formatting"},
            )
        report = self._format_final_report(sections, structure, query)

        if progress_callback:
            progress_callback(
                "Report complete", 100, {"phase": "report_complete"}
            )

        return report

    def _determine_report_structure(
        self, findings: Dict, query: str
    ) -> List[Dict]:
        """Analyze content and determine the optimal detailed report structure."""
        combined_content = findings["current_knowledge"]
        prompt = f"""
        Analyze this research content about: {query}

        Content Summary:
        {combined_content}

        Determine the most appropriate detailed report structure by:
        1. Analyzing the type of content (technical, business, academic, etc.)
        2. Identifying main themes and logical groupings
        3. Considering the depth and breadth of the research

        Return a table of contents structure in this exact format:
        STRUCTURE
        1. [Section Name]
           - [Subsection] | [purpose]
        2. [Section Name]
           - [Subsection] | [purpose]
        ...
        END_STRUCTURE

        Make the structure specific to the content, not generic.
        Each subsection must include its purpose after the | symbol.
        DO NOT include sections about sources, citations, references, or methodology.
        """

        response = get_llm_response_text(invoke_llm_sync(self.model, prompt))

        # Parse the structure
        structure: List[Dict[str, Any]] = []
        current_section: Optional[Dict[str, Any]] = None

        for line in response.split("\n"):
            if line.strip() in ["STRUCTURE", "END_STRUCTURE"]:
                continue

            if line.strip().startswith(tuple("123456789")):
                # Main section — require a dot-delimited name (e.g. "1. Intro").
                parts = line.split(".", 1)
                if len(parts) < 2 or not parts[1].strip():
                    continue
                section_name = parts[1].strip()
                current_section = {"name": section_name, "subsections": []}
                structure.append(current_section)
            elif line.strip().startswith("-") and current_section:
                # Subsection with or without purpose
                parts = line.strip("- ").split(
                    "|", 1
                )  # Only split on first pipe
                if len(parts) == 2:
                    current_section["subsections"].append(
                        {"name": parts[0].strip(), "purpose": parts[1].strip()}
                    )
                elif len(parts) == 1 and parts[0].strip():
                    # Subsection without purpose - add default
                    current_section["subsections"].append(
                        {
                            "name": parts[0].strip(),
                            "purpose": f"Provide detailed information about {parts[0].strip()}",
                        }
                    )

        # Check if the last section is source-related and remove it
        if structure:
            last_section = structure[-1]
            section_name_lower = last_section["name"].lower()
            source_keywords = [
                "source",
                "citation",
                "reference",
                "bibliography",
            ]

            # Only check the last section for source-related content
            if any(
                keyword in section_name_lower for keyword in source_keywords
            ):
                logger.info(
                    f"Removed source-related last section: {last_section['name']}"
                )
                structure = structure[:-1]

        return structure

    def _truncate_at_sentence_boundary(self, text: str, max_chars: int) -> str:
        """Truncate text at a sentence boundary to preserve readability.

        Attempts to cut at the last sentence-ending punctuation (.!?) before
        the limit. If no suitable boundary is found within 80% of the limit,
        falls back to hard truncation.

        Args:
            text: Text to truncate.
            max_chars: Maximum characters allowed.

        Returns:
            Truncated text with ``[...truncated]`` marker if truncation
            occurred, otherwise the original text unchanged.
        """
        if len(text) <= max_chars:
            return text

        truncated = text[:max_chars]

        # Look for sentence boundaries (. ! ?) followed by space or newline
        # Search backwards from the end for the last complete sentence
        last_sentence_end = -1
        for i in range(len(truncated) - 1, -1, -1):
            if truncated[i] in ".!?" and (
                i + 1 >= len(truncated) or truncated[i + 1] in " \n"
            ):
                last_sentence_end = i + 1
                break

        # Only use sentence boundary if it preserves at least 80% of content
        min_acceptable = int(max_chars * 0.8)
        if last_sentence_end > min_acceptable:
            return truncated[:last_sentence_end] + "\n[...truncated]"

        # Fall back to hard truncation
        return truncated + "\n[...truncated]"

    @staticmethod
    def _normalize_heading_text(text: str) -> str:
        """Normalize heading text for fuzzy comparison against subsection names.

        Strips markdown bold markers, leading Roman-numeral / alphabetic /
        numeric enumeration prefixes (``II.``, ``A.``, ``1.``), collapses
        whitespace, and lowercases. Used only for matching — never written
        back into the report.
        """
        cleaned = text.strip()
        # Drop surrounding bold/italic markers the LLM wraps titles in
        # (``**Title**``, ``*Title*``, ``__Title__``). Avoid regex here —
        # CodeQL flags ``^[*_]+|[*_]+$`` as polynomial on long ``*`` runs.
        cleaned = cleaned.strip("*_").strip()
        cleaned = re.sub(
            r"^(?:"
            r"[IVXLCDM]+\.|"  # Roman numerals: II. III. IV. (uppercase only to avoid "Mix." false strip)
            r"[A-Z]\.|"  # Single letters: A. B.
            r"\d+\."  # Arabic numerals: 1. 2.
            r")\s+",
            "",
            cleaned,
        )
        # Normalise curly/smart quotes and dashes so "Author's" (U+2019)
        # matches "Author's" (ASCII apostrophe) and en/em dashes collapse
        # to a plain hyphen for comparison.
        cleaned = cleaned.translate(
            str.maketrans(
                "\u2018\u2019\u201c\u201d\u2013\u2014\u2212", "''\"\"---"
            )
        )
        return re.sub(r"\s+", " ", cleaned).strip().lower()

    @classmethod
    def _heading_restates_name(cls, heading_text: str, name: str) -> bool:
        """Return True if *heading_text* is a restatement of *name*.

        Handles exact matches, prefix matches (heading extends the name
        with a colon/dash subtitle), and the reverse (name is a longer
        form of a short heading). Empty names never match.
        """
        if not name or not heading_text:
            return False
        norm_heading = cls._normalize_heading_text(heading_text)
        norm_name = cls._normalize_heading_text(name)
        if not norm_heading or not norm_name:
            return False
        if norm_heading == norm_name:
            return True
        # Heading extends the subsection name with a subtitle (e.g. "Name: subtitle")
        # Delimiters only: bare space prefixes (e.g. "Introduction to …")
        # are intentional non-matches. En/em dashes are already folded to
        # ASCII "-" by _normalize_heading_text.
        if norm_heading.startswith(norm_name):
            rest = norm_heading[len(norm_name) :].lstrip()
            if rest and rest[0] in (":", "-", "("):
                return True
        # Name extends a short heading with a subtitle.
        if norm_name.startswith(norm_heading):
            rest = norm_name[len(norm_heading) :].lstrip()
            if rest and rest[0] in (":", "-", "("):
                return True
        return False

    def _strip_leading_heading(
        self,
        content: str,
        subsection_name: str,
        sibling_subsection_names: Optional[List[str]] = None,
    ) -> str:
        """Drop a single leading heading line only when it restates subsection or sibling name."""
        leading = _LEADING_HEADING_RE.match(content)
        if not leading:
            return content

        fence_spans = _get_code_fence_spans(content)
        if _is_in_spans(leading.start(), fence_spans):
            return content

        heading_text = re.sub(r"^\s*#{1,6}[ \t]+", "", leading.group(0)).strip()
        # Subsection name: allow subtitle extensions (Name: subtitle)
        if self._heading_restates_name(heading_text, subsection_name):
            end_pos = leading.end()
            while end_pos < len(content) and content[end_pos] in "\r\n":
                end_pos += 1
            return content[end_pos:]
        # Siblings: require exact normalized equality (avoid over-strip of
        # "### <Sibling>: Subtitle" organizers)
        norm_heading = self._normalize_heading_text(heading_text)
        for sibling in sibling_subsection_names or []:
            if sibling and norm_heading == self._normalize_heading_text(
                sibling
            ):
                end_pos = leading.end()
                while end_pos < len(content) and content[end_pos] in "\r\n":
                    end_pos += 1
                return content[end_pos:]
        return content

    def _strip_leading_italic_purpose(
        self,
        content: str,
        purpose: Optional[str] = None,
    ) -> str:
        """Drop a single leading italic purpose statement if it mirrors purpose or boilerplate."""
        leading_italic = _LEADING_ITALIC_PURPOSE_RE.match(content)
        if not leading_italic:
            return content

        fence_spans = _get_code_fence_spans(content)
        if _is_in_spans(leading_italic.start(), fence_spans):
            return content

        italic_raw = leading_italic.group(0).strip()
        italic_text = italic_raw.strip("_*").strip()
        should_strip = False

        if purpose:
            norm_italic = self._normalize_heading_text(italic_text)
            norm_purpose = self._normalize_heading_text(purpose)
            # Guard against degenerate purpose that normalizes to "" (e.g. "***")
            # — startswith("") is always True and would strip any italic line.
            if norm_italic and norm_purpose:
                if (
                    norm_italic == norm_purpose
                    or norm_italic.startswith(norm_purpose)
                    or norm_purpose.startswith(norm_italic)
                    or self._heading_restates_name(italic_text, purpose)
                ):
                    should_strip = True

        if not should_strip:
            norm_italic = self._normalize_heading_text(italic_text)
            if norm_italic:
                # Narrow unconditional boilerplate markers — always safe to strip
                if norm_italic.startswith(
                    (
                        "purpose:",
                        "scope:",
                        "this subsection ",
                        "this section ",
                    )
                ):
                    should_strip = True
                # Verb-prefix boilerplate: check for relevance to purpose to avoid
                # eating legitimate epigraphs like "_To review the evidence is to understand the past._"
                # when purpose is unrelated (S2).
                elif purpose is not None and norm_italic.startswith(
                    (
                        "to summarize",
                        "to summarise",
                        "to clarify",
                        "to examine",
                        "to outline",
                        "to provide",
                        "to describe",
                        "to detail",
                        "to analyze",
                        "to analyse",
                        "to explore",
                        "to present",
                        "to discuss",
                        "to evaluate",
                        "to investigate",
                        "to review",
                        "to assess",
                        "aims to",
                        "designed to",
                    )
                ):
                    norm_purpose = self._normalize_heading_text(purpose)
                    if norm_purpose:
                        _stop_words = {
                            "to",
                            "the",
                            "and",
                            "for",
                            "that",
                            "this",
                            "with",
                            "from",
                            "summarize",
                            "summarise",
                            "clarify",
                            "examine",
                            "outline",
                            "provide",
                            "describe",
                            "detail",
                            "analyze",
                            "analyse",
                            "explore",
                            "present",
                            "discuss",
                            "evaluate",
                            "investigate",
                            "review",
                            "assess",
                            "aims",
                            "designed",
                            "section",
                            "subsection",
                        }
                        purpose_words = {
                            w
                            for w in re.findall(
                                r"\b[a-z0-9]{3,}\b", norm_purpose.lower()
                            )
                            if w not in _stop_words
                        }
                        italic_words = {
                            w
                            for w in re.findall(
                                r"\b[a-z0-9]{3,}\b", norm_italic.lower()
                            )
                            if w not in _stop_words
                        }
                        if (
                            self._heading_restates_name(italic_text, purpose)
                            or norm_italic.startswith(norm_purpose)
                            or norm_purpose.startswith(norm_italic)
                            or bool(purpose_words & italic_words)
                        ):
                            should_strip = True

        if should_strip:
            end_pos = leading_italic.end()
            while end_pos < len(content) and content[end_pos] in "\r\n":
                end_pos += 1
            return content[end_pos:]
        return content

    def _strip_embedded_bibliographies(self, content: str) -> str:
        """Remove every embedded bibliography block outside code blocks.

        Only removes a heading that looks like a bibliography *and* whose
        following block looks like a citation list — this prevents
        substantive headings such as "### Sources of Bias..." or
        "### Sources. Data Collection Methodology" from being deleted with
        their analysis body (R1). Consecutive bibliography headings no longer
        consume intervening prose (R8).
        """
        fence_spans = _get_code_fence_spans(content)
        pieces: List[str] = []
        cursor = 0
        bib_removed = False

        for match in _BIBLIOGRAPHY_HEADING_RE.finditer(content):
            if _is_in_spans(match.start(), fence_spans):
                continue

            matched_heading_text = match.group(0).lstrip()
            is_atx_heading = matched_heading_text.startswith("#")
            bib_line_re = (
                _CITATION_LINE_RE
                if is_atx_heading
                else _PSEUDO_BIB_CITATION_LINE_RE
            )

            after_heading = match.end()
            next_heading_start = None
            next_heading_is_bib = False
            for nh in _NEXT_HEADING_RE.finditer(content, after_heading):
                if not _is_in_spans(nh.start(), fence_spans):
                    next_heading_start = nh.start()
                    # Peek if the next heading itself is a bibliography heading
                    # — if so, we will trim the current block to its citation
                    # list end rather than to the heading (R8).
                    heading_line = content[nh.start() :].split("\n", 1)[0]
                    if _BIBLIOGRAPHY_HEADING_RE.match(heading_line):
                        next_heading_is_bib = True
                    break

            block_end = (
                next_heading_start
                if next_heading_start is not None
                else len(content)
            )
            block_text = content[after_heading:block_end]

            # If the block immediately begins with a code fence, the heading is
            # introducing fenced example/code content, not a bibliography.
            offset = 0
            first_non_empty_start = None
            for ln in block_text.splitlines(keepends=True):
                if ln.strip():
                    first_non_empty_start = after_heading + offset
                    break
                offset += len(ln)

            if first_non_empty_start is not None and _is_in_spans(
                first_non_empty_start, fence_spans
            ):
                continue

            # Body-shape safeguard: only treat as bibliography if the block
            # contains citation-like lines. Require at least one citation
            # marker, or >30% of non-empty lines look like citations, to
            # avoid deleting substantive analysis headings that happen to
            # match the label grammar (e.g. "### Sources for the Analysis"
            # with prose body).
            non_empty_lines = []
            line_offset = 0
            for ln in block_text.splitlines(keepends=True):
                line_start = after_heading + line_offset
                if ln.strip() and not _is_in_spans(line_start, fence_spans):
                    non_empty_lines.append(ln)
                line_offset += len(ln)
            if not non_empty_lines:
                # Empty block — treat as bibliography (heading with no body)
                is_bib_block = True
            else:
                # Intervening prose safeguard: a bibliography heading must introduce
                # citations directly (or via an immediate note / 1-line lead-in).
                # If substantive prose precedes citations, the heading is introducing
                # substantive analysis, not an embedded bibliography.
                first_line = non_empty_lines[0]
                first_is_bib = (
                    bib_line_re.match(first_line)
                    or _BIB_NOTE_RE.match(first_line)
                    or _HR_LINE_RE.match(first_line)
                )
                if not first_is_bib:
                    # Check if it's a short 1-line lead-in immediately followed by citations
                    second_is_citation = len(
                        non_empty_lines
                    ) > 1 and bib_line_re.match(non_empty_lines[1])
                    is_lead_in = (
                        len(first_line.strip()) < 120
                        and second_is_citation
                        and bool(
                            re.search(
                                r"\b(?:sources?|references?|consulted|cited|following|basis)\b",
                                first_line,
                                re.IGNORECASE,
                            )
                        )
                    )
                    has_early_note = any(
                        _BIB_NOTE_RE.match(ln) for ln in non_empty_lines[:3]
                    )
                    if not (is_lead_in or has_early_note):
                        continue

                citation_lines = sum(
                    1 for ln in non_empty_lines if bib_line_re.match(ln)
                )
                # Require explicit italic note boilerplate (e.g. "*(Note: ... bibliography ...)*")
                # rather than matching arbitrary prose containing "bibliography" (S1).
                has_bib_note = any(
                    _BIB_NOTE_RE.match(ln) for ln in non_empty_lines[:3]
                )
                if citation_lines == 0 and not has_bib_note:
                    # No citation shape and no italic bib note — preserve heading + body
                    continue
                # If citation lines present, require they are not a tiny minority
                # unless block is short (e.g. "- item" single line)
                if citation_lines == 0:
                    is_bib_block = has_bib_note
                elif len(non_empty_lines) <= 3:
                    is_bib_block = citation_lines >= 1
                else:
                    is_bib_block = (
                        citation_lines / len(non_empty_lines)
                    ) >= 0.3 or citation_lines >= 2

            if not is_bib_block:
                continue

            # Passed safeguard — this is a real bibliography block to remove
            bib_removed = True
            block_start = match.start()
            prefix = content[cursor:block_start]
            prefix_lines = prefix.splitlines(keepends=True)
            cut_idx = len(prefix)
            curr_offset = len(prefix)
            found_delimiter = False
            for pline in reversed(prefix_lines):
                line_start = curr_offset - len(pline)
                abs_pos = cursor + line_start
                if _is_in_spans(abs_pos, fence_spans):
                    break
                pstripped = pline.strip()
                if not pstripped:
                    curr_offset = line_start
                    continue
                if _EMPTY_HEADING_OR_HR_RE.match(pstripped):
                    found_delimiter = True
                    cut_idx = line_start
                    curr_offset = line_start
                else:
                    break
            if found_delimiter and cut_idx < len(prefix):
                block_start = cursor + cut_idx

            pieces.append(content[cursor:block_start])

            # R8: consecutive bib headings — end at citation list, not at next bib heading
            if next_heading_is_bib:
                # Find end of citation list within block_text
                lines = block_text.splitlines(keepends=True)
                offset = 0
                last_citation_end = 0
                for ln in lines:
                    line_start = after_heading + offset
                    if not _is_in_spans(line_start, fence_spans) and (
                        bib_line_re.match(ln)
                        or (_BIB_NOTE_RE.match(ln) and offset < 300)
                    ):
                        last_citation_end = offset + len(ln)
                    # Also keep consecutive citation lines; stop at first
                    # non-citation prose that is not a blank/bib note?
                    offset += len(ln)
                # Also include trailing blank lines after citations, but not
                # intervening prose before next bib heading
                if last_citation_end > 0:
                    # Advance cursor to after the citation list (preserve
                    # intervening prose between two bib headings)
                    cursor = after_heading + last_citation_end
                    # Skip following blank lines / HRs but not prose
                    while cursor < block_end and content[cursor] in " \t\r\n-":
                        # Only skip whitespace/HRs, stop at prose
                        # Peek next non-whitespace chunk
                        nxt = content[cursor:].lstrip(" \t\r\n")
                        if nxt.startswith("-") and nxt[1:2] in " \t":
                            # Another HR-like line, skip it
                            cursor += len(content[cursor:]) - len(nxt)
                            # consume the HR line
                            eol = content.find("\n", cursor)
                            cursor = eol + 1 if eol != -1 else len(content)
                        else:
                            break
                        if cursor >= block_end:
                            break
                    # Ensure we don't overshoot into next bib heading's prefix;
                    # if cursor is before next_heading_start, keep it, else
                    # fall back to next_heading_start
                    if cursor > block_end:
                        cursor = block_end
                else:
                    cursor = block_end
            else:
                # End the removal at the citation list rather than at the end
                # of the block, so a table, list or paragraph the model wrote
                # after its bibliography survives the strip.
                offset = 0
                citation_end = 0
                non_empty_seen = 0
                trailing_content = False
                for ln in block_text.splitlines(keepends=True):
                    line_start = after_heading + offset
                    substantive = bool(ln.strip())
                    outside_fence = not _is_in_spans(line_start, fence_spans)
                    non_empty_seen += substantive and outside_fence
                    # The note window is the classifier's first-three-non-empty-
                    # lines rule, so a block accepted as a bibliography always
                    # has a qualifying line here and cannot fall through.
                    is_bib_line = bool(
                        outside_fence
                        and (
                            bib_line_re.match(ln)
                            or (_BIB_NOTE_RE.match(ln) and non_empty_seen <= 3)
                            or _HR_LINE_RE.match(ln)
                        )
                    )
                    if citation_end and substantive and not is_bib_line:
                        trailing_content = True
                        break
                    offset += len(ln)
                    if substantive and is_bib_line:
                        citation_end = offset
                if not trailing_content:
                    # Nothing substantive follows, so the whole block is the
                    # bibliography and is removed as before.
                    citation_end = len(block_text)

                cursor = after_heading + citation_end

        if bib_removed:
            pieces.append(content[cursor:])
            content = "".join(pieces)
        return self._strip_trailing_citation_blocks(content)

    def _strip_trailing_citation_blocks(self, content: str) -> str:
        """Remove unheaded trailing citation blocks or concatenated citations.

        When the LLM is instructed not to include a sources section heading,
        some models omit the heading entirely but append a list or block of
        citations at the end of the response (e.g. consecutive lines of
        `[[111]](/library/...)` or a concatenated line of citations).

        This defensive helper scans backwards from the end of the content
        outside code fences and strips trailing citation blocks while
        preserving all substantive prose, tables, code fences, reference-style
        link definitions, further-reading URLs, quiz answer keys, and CJK notes.
        """
        if not content:
            return content

        fence_spans = _get_code_fence_spans(content)
        lines = content.splitlines(keepends=True)
        if not lines:
            return content

        line_offsets: List[int] = []
        curr = 0
        for ln in lines:
            line_offsets.append(curr)
            curr += len(ln)

        # Skip trailing empty / whitespace lines
        cut_idx = len(lines)
        while cut_idx > 0 and not lines[cut_idx - 1].strip():
            cut_idx -= 1

        if cut_idx == 0:
            return content

        # Never strip inside code fences
        if _is_in_spans(line_offsets[cut_idx - 1], fence_spans):
            return content

        citation_lines_count = 0
        concatenated_lines_count = 0
        bib_heading_preceded = False
        idx = cut_idx

        while idx > 0:
            offset = line_offsets[idx - 1]
            if _is_in_spans(offset, fence_spans):
                break

            line_str = lines[idx - 1].strip()
            if not line_str:
                idx -= 1
                continue

            # Reference-style link definitions ([1]: https://...) must NEVER be stripped
            if _MD_LINK_DEF_RE.match(line_str):
                break

            # Plain numbered or bulleted list items must NEVER be stripped as unheaded citations
            if _ORDERED_OR_BULLET_LIST_RE.match(line_str):
                break

            # Concatenated citation clusters (e.g. [[217]][[218]] or [[1]](/a), [2], [[3]])
            if _CONCATENATED_CITATIONS_LINE_RE.match(line_str):
                # Must contain double brackets [[...]], cite tags, or adjacent bracket tokens;
                # exclude plain numbered markdown links like [1](url) [2](url)
                if (
                    "[[" in line_str
                    or "[cite" in line_str.lower()
                    or bool(re.search(r"\]\s*\[|】\s*[【\[]|\]\s*【", line_str))
                ):
                    concatenated_lines_count += 1
                    idx -= 1
                    continue
                break

            if _UNHEADED_CITATION_ITEM_RE.match(line_str):
                citation_lines_count += 1
                idx -= 1
                continue

            # Support indented continuation lines within a citation entry.
            # When scanning backwards, find the preceding non-indented line.
            if lines[idx - 1].startswith(("  ", "\t")):
                p = idx - 2
                while p >= 0 and lines[p].startswith(("  ", "\t")):
                    p -= 1
                if p >= 0 and _UNHEADED_CITATION_ITEM_RE.match(
                    lines[p].strip()
                ):
                    idx -= 1
                    continue

            if _BIB_NOTE_RE.match(line_str):
                idx -= 1
                continue

            # Check if preceded by an explicit bibliography heading or label
            if citation_lines_count > 0 or concatenated_lines_count > 0:
                if _BIBLIOGRAPHY_HEADING_RE.match(line_str):
                    bib_heading_preceded = True
                    idx -= 1
                    continue
                if _EMPTY_HEADING_OR_HR_RE.match(line_str):
                    idx -= 1
                    continue

            break

        # Only strip if:
        # 1. Found a concatenated citation cluster
        # 2. Preceded by an explicit bibliography heading/label and found >= 1 citation items
        # 3. Found >= 2 citation items in an unheaded block (bare --- does NOT arm single items)
        should_strip = False
        if concatenated_lines_count > 0:
            should_strip = True
        elif bib_heading_preceded and citation_lines_count >= 1:
            should_strip = True
        elif citation_lines_count >= 2:
            should_strip = True

        if should_strip:
            # Also consume any preceding empty headings, labels, or HRs
            while idx > 0:
                offset = line_offsets[idx - 1]
                if _is_in_spans(offset, fence_spans):
                    break
                pstripped = lines[idx - 1].strip()
                if not pstripped:
                    idx -= 1
                    continue
                if _EMPTY_HEADING_OR_HR_RE.match(
                    pstripped
                ) or _BIBLIOGRAPHY_HEADING_RE.match(pstripped):
                    idx -= 1
                else:
                    break
            return "".join(lines[:idx])

        return content

    def _strip_subsection_boilerplate(
        self,
        content: str,
        subsection_name: str,
        section_name: str,
        sibling_subsection_names: Optional[List[str]] = None,
        purpose: Optional[str] = None,
    ) -> str:
        """Strip boilerplate the LLM tends to emit around subsection content.

        Small and/or quantized local models routinely produce three artefacts
        that pollute the rendered report even when the OUTPUT FORMAT RULES
        spelled out in the subsection prompt explicitly forbid them:

        1. A redundant leading heading (stripped by :meth:`_strip_leading_heading`)
        2. An italic purpose statement (stripped by :meth:`_strip_leading_italic_purpose`)
        3. An embedded bibliography block (stripped by :meth:`_strip_embedded_bibliographies`)

        This helper normalises the per-subsection content so the rendered
        report stays clean even when the model ignored the prompt. It is a
        defensive cleanup, not a substitute for the prompt rules.

        Args:
            content: Raw ``current_knowledge`` returned for one subsection.
            subsection_name: The subsection's name (used for matching and
                logging).
            section_name: The parent section's name (used for logging).
            sibling_subsection_names: Optional names of other subsections
                in the same section. When provided, a leading heading that
                restates a *sibling's* name (context-bleed) is also stripped.
            purpose: Optional purpose string for the subsection. Used to
                verify if a leading italic line mirrors the subsection purpose.

        Returns:
            ``content`` with a redundant leading heading removed when it
            restates this or a sibling subsection name, a leading
            italic-purpose statement removed, and any embedded bibliography
            blocks removed. Returns content unchanged if no artefacts are
            found, aside from collapsing 3+ blank lines to 2 and
            normalising trailing whitespace (collapsed outside code fences
            only, so fenced blocks are never rewritten).
        """
        if not content:
            return content

        original_len = len(content)
        new_content = self._strip_leading_heading(
            content, subsection_name, sibling_subsection_names
        )
        new_content = self._strip_leading_italic_purpose(new_content, purpose)
        pre_bib_len = len(new_content)
        new_content = self._strip_embedded_bibliographies(new_content)
        pre_trailing_len = len(new_content)
        new_content = self._strip_trailing_citation_blocks(new_content)
        bib_chars_removed = (pre_bib_len - pre_trailing_len) + (
            pre_trailing_len - len(new_content)
        )

        # R2: fence-aware whitespace collapse — never rewrite inside code fences
        fence_spans = _get_code_fence_spans(new_content)
        if not fence_spans:
            collapsed = re.sub(r"\n{3,}", "\n\n", new_content).strip()
        else:
            parts: List[str] = []
            last = 0
            for s, e in sorted(fence_spans):
                # collapse outside fence
                parts.append(re.sub(r"\n{3,}", "\n\n", new_content[last:s]))
                # keep fence content verbatim
                parts.append(new_content[s:e])
                last = e
            parts.append(re.sub(r"\n{3,}", "\n\n", new_content[last:]))
            collapsed = "".join(parts).strip()
        new_content = collapsed
        if new_content:
            new_content += "\n"

        if len(new_content) != original_len:
            # R5: make silent deletions observable
            if not new_content.strip() and content.strip():
                logger.warning(
                    "Stripped subsection boilerplate emptied non-trivial content for "
                    f"'{section_name} > {subsection_name}': "
                    f"{original_len} -> {len(new_content)} chars — "
                    "check for over-strip (R1/R3 regression)"
                )
            elif len(new_content) < 0.5 * original_len:
                # If the truncation below 50% was driven by stripping an
                # embedded bibliography (where pre-bibliography content was
                # >= 50% and bib removal did the rest), this is expected
                # cleanup rather than an unexpected over-strip.
                if (
                    bib_chars_removed > 0
                    and pre_bib_len >= 0.5 * original_len
                    and (original_len - pre_bib_len) < 0.25 * original_len
                ):
                    logger.info(
                        "Stripped embedded bibliography from subsection "
                        f"'{section_name} > {subsection_name}': "
                        f"{original_len} -> {len(new_content)} chars "
                        "(expected cleanup; warning demoted to INFO)"
                    )
                else:
                    logger.warning(
                        "Stripped subsection boilerplate heavily truncated content for "
                        f"'{section_name} > {subsection_name}': "
                        f"{original_len} -> {len(new_content)} chars"
                    )
            else:
                logger.debug(
                    "Stripped subsection boilerplate for "
                    f"'{section_name} > {subsection_name}': "
                    f"{original_len} -> {len(new_content)} chars"
                )
        return new_content

    def _build_previous_context(self, accumulated_findings: List[str]) -> str:
        """Build context block from previously generated sections.

        Creates a formatted context block containing content from the last
        N sections (defined by self.max_context_sections) with explicit instructions
        not to repeat this content. Context is truncated if it exceeds
        self.max_context_chars to stay safe for smaller local models.

        Args:
            accumulated_findings: List of previously generated section content,
                each formatted as "[Section > Subsection]\\n{content}"

        Returns:
            Formatted context block with delimiters, or empty string if no
            previous findings exist
        """
        if not accumulated_findings:
            return ""

        recent_findings = accumulated_findings[-self.max_context_sections :]
        previous_context = "\n\n---\n\n".join(recent_findings)

        # Truncate at sentence boundary if too long
        if len(previous_context) > self.max_context_chars:
            previous_context = self._truncate_at_sentence_boundary(
                previous_context, self.max_context_chars
            )

        return (
            f"\n\n=== CONTENT ALREADY WRITTEN (DO NOT REPEAT) ===\n"
            f"{previous_context}\n"
            f"=== END OF PREVIOUS CONTENT ===\n\n"
            f"CRITICAL: The above content has already been written. Do NOT repeat "
            f"these points, examples, or explanations. Focus on NEW information "
            f"not covered above.\n"
        )

    def _research_and_generate_sections(
        self,
        initial_findings: Dict,
        structure: List[Dict],
        query: str,
        progress_callback=None,
    ) -> Dict[str, str]:
        """Research and generate content for each section in one step.

        This method processes sections sequentially, accumulating generated
        content as it goes. For each new section/subsection, it passes context
        from the last few previously generated sections to help the LLM avoid
        repetition.

        The context accumulation mechanism:
        - Tracks all generated content in accumulated_findings list
        - Before generating each section, builds context from recent findings
        - Uses self.max_context_sections (configurable, default: 3) to limit context size
        - Truncates context to self.max_context_chars (configurable, default: 4000) for safety
        - Includes explicit "DO NOT REPEAT" instructions with actual content

        Args:
            initial_findings: Results from initial research phase, may contain
                questions_by_iteration to preserve search continuity
            structure: List of section definitions, each with name and subsections
            query: Original user query for context

        Returns:
            Dict mapping section names to their generated markdown content
        """
        sections = {}

        # Accumulate content from previous sections to avoid repetition
        accumulated_findings: List[str] = []

        # Count total subsections for progress tracking
        total_subsections = sum(
            max(len(section.get("subsections", [])), 1) for section in structure
        )
        completed_subsections = 0

        # Preserve questions from initial research to avoid repetition
        # This follows the same pattern as citation tracking (all_links_of_system)
        existing_questions = initial_findings.get("questions_by_iteration", {})
        if existing_questions:
            # Set questions on both search system and its strategy
            if hasattr(self.search_system, "questions_by_iteration"):
                self.search_system.questions_by_iteration = (
                    existing_questions.copy()
                )

            # More importantly, set it on the strategy which actually uses it
            if hasattr(self.search_system, "strategy") and hasattr(
                self.search_system.strategy, "questions_by_iteration"
            ):
                self.search_system.strategy.questions_by_iteration = (
                    existing_questions.copy()
                )
                logger.info(
                    f"Initialized strategy with {len(existing_questions)} iterations of previous questions"
                )

        for i, section in enumerate(structure, 1):
            logger.info(f"Processing section: {section['name']}")
            section_content = []

            section_content.append(f"# {i}. {section['name']}\n")

            # If section has no subsections, create one from the section itself
            if not section["subsections"]:
                # Parse section name for purpose
                if "|" in section["name"]:
                    parts = section["name"].split("|", 1)
                    section["subsections"] = [
                        {"name": parts[0].strip(), "purpose": parts[1].strip()}
                    ]
                else:
                    # No purpose provided - use section name as subsection
                    section["subsections"] = [
                        {
                            "name": section["name"],
                            "purpose": f"Provide comprehensive content for {section['name']}",
                        }
                    ]

            # Process each subsection by directly researching it
            for j, subsection in enumerate(section["subsections"], 1):
                # Only add subsection header if there are multiple subsections
                if len(section["subsections"]) > 1:
                    section_content.append(f"## {i}.{j} {subsection['name']}\n")
                    section_content.append(f"_{subsection['purpose']}_\n\n")

                # Get other subsections in this section for context
                other_subsections = [
                    f"- {s['name']}: {s['purpose']}"
                    for s in section["subsections"]
                    if s["name"] != subsection["name"]
                ]
                other_subsections_text = (
                    "\n".join(other_subsections)
                    if other_subsections
                    else "None"
                )

                # Get all other sections for broader context
                other_sections = [
                    f"- {s['name']}"
                    for s in structure
                    if s["name"] != section["name"]
                ]
                other_sections_text = (
                    "\n".join(other_sections) if other_sections else "None"
                )

                # Check if this is actually a section-level content (only one subsection, likely auto-created)
                is_section_level = len(section["subsections"]) == 1

                # Build context from previously generated sections to avoid repetition
                previous_context_section = self._build_previous_context(
                    accumulated_findings
                )

                # Generate appropriate search query
                if is_section_level:
                    # Section-level prompt - more comprehensive
                    subsection_query = (
                        f"Research task: Create comprehensive content for the '{subsection['name']}' section in a report about '{query}'. "
                        f"Section purpose: {subsection['purpose']} "
                        f"\n"
                        f"Other sections in the report:\n{other_sections_text}\n"
                        f"{previous_context_section}"
                        f"This is a standalone section requiring comprehensive coverage of its topic. "
                        f"Provide a thorough exploration that may include synthesis of information from previous sections where relevant. "
                        f"Include unique insights, specific examples, and concrete data. "
                        f"Use tables to organize information where applicable. "
                        f"For conclusion sections: synthesize key findings and provide forward-looking insights. "
                        f"Build upon the research findings from earlier sections to create a cohesive narrative."
                        f"{_SUBSECTION_OUTPUT_GUIDANCE}"
                    )
                else:
                    # Subsection-level prompt - more focused
                    subsection_query = (
                        f"Research task: Create content for subsection '{subsection['name']}' in a report about '{query}'. "
                        f"This subsection's purpose: {subsection['purpose']} "
                        f"Part of section: '{section['name']}' "
                        f"\n"
                        f"Other sections in the report:\n{other_sections_text}\n"
                        f"\n"
                        f"Other subsections in this section will cover:\n{other_subsections_text}\n"
                        f"{previous_context_section}"
                        f"Focus ONLY on information specific to your subsection's purpose. "
                        f"Include unique details, specific examples, and concrete data. "
                        f"Use tables to organize information where applicable. "
                        f"IMPORTANT: Avoid repeating information that would logically be covered in other sections - focus on what makes this subsection unique. "
                        f"Previous research exists - find specific angles for this subsection."
                        f"{_SUBSECTION_OUTPUT_GUIDANCE}"
                    )

                logger.info(
                    f"Researching subsection: {subsection['name']} with query: {subsection_query}"
                )

                # Report progress and check for cancellation
                if progress_callback:
                    pct = int(
                        10
                        + (completed_subsections / max(total_subsections, 1))
                        * 80
                    )
                    progress_callback(
                        f"Researching: {section['name']} > {subsection['name']}",
                        pct,
                        {
                            "phase": "report_section_research",
                            "subsection": subsection["name"],
                        },
                    )

                # Fix iteration override: modify strategy's settings_snapshot
                # which is read dynamically via get_setting()
                strategy = self.search_system.strategy
                original_iterations = strategy.settings_snapshot.get(
                    "search.iterations"
                )
                had_iterations_key = (
                    "search.iterations" in strategy.settings_snapshot
                )
                strategy.settings_snapshot["search.iterations"] = 1
                # Belt-and-suspenders: also override max_iterations for
                # strategies that cache it at __init__ time
                original_max_iter = getattr(strategy, "max_iterations", None)
                strategy.max_iterations = 1
                # If the strategy wraps a delegate (e.g.
                # EnhancedContextualFollowUpStrategy), the actual research
                # happens inside ``delegate_strategy.analyze_topic`` and the
                # delegate has its own cached ``max_iterations`` from
                # construction — without this extra override the per-subsection
                # cap never reaches the worker and each subsection runs the
                # full configured loop instead of one iteration.
                delegate = getattr(strategy, "delegate_strategy", None)
                original_delegate_max_iter = (
                    getattr(delegate, "max_iterations", None)
                    if delegate is not None
                    else None
                )
                if delegate is not None and hasattr(delegate, "max_iterations"):
                    delegate.max_iterations = 1

                try:
                    # Perform search for this subsection
                    subsection_results = self.search_system.analyze_topic(
                        subsection_query
                    )
                finally:
                    # Restore original iteration settings
                    if had_iterations_key:
                        strategy.settings_snapshot["search.iterations"] = (
                            original_iterations
                        )
                    else:
                        strategy.settings_snapshot.pop(
                            "search.iterations", None
                        )
                    if original_max_iter is not None:
                        strategy.max_iterations = original_max_iter
                    if delegate is not None and hasattr(
                        delegate, "max_iterations"
                    ):
                        if original_delegate_max_iter is not None:
                            delegate.max_iterations = original_delegate_max_iter
                        else:
                            delegate.max_iterations = None

                completed_subsections += 1

                # Add the researched content for this subsection
                if subsection_results.get("current_knowledge"):
                    generated_content = subsection_results["current_knowledge"]
                    # Strip the boilerplate the LLM tends to emit around
                    # subsections: a redundant leading heading that mirrors
                    # the framework's own `## i.j Name` heading, an italic
                    # purpose statement that mirrors the framework's
                    # `_<purpose>_` subtitle, and an embedded
                    # '## Sources' / '## References' bibliography block
                    # (the framework appends one master '## Sources' to
                    # the whole report). Small/quantized local models
                    # routinely ignore the OUTPUT FORMAT RULES in the
                    # prompt above; this normalisation step guarantees a
                    # clean rendered output regardless.
                    sibling_names = [
                        s["name"]
                        for s in section["subsections"]
                        if s["name"] != subsection["name"]
                    ]
                    try:
                        generated_content = self._strip_subsection_boilerplate(
                            generated_content,
                            subsection_name=subsection["name"],
                            section_name=section["name"],
                            sibling_subsection_names=sibling_names,
                            purpose=subsection.get("purpose"),
                        )
                    except Exception:
                        logger.exception(
                            "Boilerplate strip failed for "
                            f"'{section['name']} > {subsection['name']}' — "
                            "using raw LLM content"
                        )
                        # Fall back to raw content so a cosmetic step never
                        # aborts the entire multi-section report.
                        generated_content = (
                            subsection_results.get("current_knowledge", "")
                            or ""
                        )
                    if generated_content.strip():
                        section_content.append(generated_content)
                        # Accumulate for context in subsequent sections
                        accumulated_findings.append(
                            f"[{section['name']} > {subsection['name']}]\n{generated_content}"
                        )
                    else:
                        section_content.append(
                            "*Limited information was found for this subsection.*\n"
                        )
                else:
                    section_content.append(
                        "*Limited information was found for this subsection.*\n"
                    )

                section_content.append("\n\n")

            # Combine all content for this section
            sections[section["name"]] = "\n".join(section_content)

        return sections

    def _generate_sections(
        self,
        initial_findings: Dict,
        _section_research: Dict[str, List[Dict]],
        structure: List[Dict],
        query: str,
    ) -> Dict[str, str]:
        """
        This method is kept for compatibility but no longer used.
        The functionality has been moved to _research_and_generate_sections.
        """
        return {}

    def _format_final_report(
        self,
        sections: Dict[str, str],
        structure: List[Dict],
        query: str,
    ) -> Dict:
        """Format the final report with table of contents and sections."""
        # Generate TOC
        toc = ["# Table of Contents\n"]
        for i, section in enumerate(structure, 1):
            toc.append(f"{i}. **{section['name']}**")
            if len(section["subsections"]) > 1:
                for j, subsection in enumerate(section["subsections"], 1):
                    toc.append(
                        f"   {i}.{j} {subsection['name']} | _{subsection['purpose']}_"
                    )

        # Combine TOC and sections
        report_parts = ["\n".join(toc), ""]

        # Add a summary of the research
        report_parts.append("# Research Summary")
        report_parts.append(
            "This report was researched using an advanced search system."
        )
        report_parts.append(
            "Research included targeted searches for each section and subsection."
        )
        report_parts.append("\n---\n")

        # Add each section's content
        for section in structure:
            if section["name"] in sections:
                report_parts.append(sections[section["name"]])
                report_parts.append("")

        # Format links from search system
        # Get utilities module dynamically to avoid circular imports
        utilities = importlib.import_module("local_deep_research.utilities")
        # Imported here for the same reason, and directly rather than off
        # the module object above so the count is computed by the real
        # grouping helper.
        from .utilities.search_utilities import (
            count_distinct_sources,
            filter_cited_links,
            resolve_uncited_sources_mode,
        )

        # Body prose BEFORE the Sources tail: citation indices/URLs are
        # parsed from this, so uncited accumulated hits stay out of the
        # bibliography. Honors report.uncited_sources_mode.
        body_prose = "\n\n".join(report_parts)
        mode = resolve_uncited_sources_mode()
        visible_links = filter_cited_links(
            self.search_system.all_links_of_system,
            body_prose,
            mode=mode,
        )
        formatted_all_links = (
            utilities.search_utilities.format_links_to_markdown(
                all_links=visible_links,
            )
        )

        # Create final report with all parts. The Sources tail is
        # kept here so in-memory consumers (MCP `generate_report`,
        # programmatic API) get the full assembled blob unchanged.
        # The DB save site (research_service.py) strips this Sources
        # section via format_document_split before persisting, so the
        # answer-only invariant on report_content still holds.
        final_report_content = body_prose
        # Explicit "\n\n" separator: downstream regex consumers
        # (_SOURCES_SECTION_PATTERNS in text_optimization/citation_formatter.py
        # and _LEGACY_SOURCES_RE in web/services/report_assembly_service.py)
        # use line-anchored `re.MULTILINE` matching. Today the trailing
        # newlines produced by `"\n\n".join` happen to keep `## Sources`
        # at the start of a line, but that is incidental; an explicit
        # separator preserves the invariant against future section
        # template changes.
        #
        # The HTML-comment sentinel around the appended `## Sources`
        # block lets ``format_document_split`` locate this section
        # unambiguously, even when the LLM has emitted its own
        # `## Sources` header earlier in the prose. The legacy regex
        # patterns would otherwise match the first `## Sources` they
        # find and could over-strip a multi-section report where the
        # LLM happened to include an inline sources block. Without the
        # sentinel, a 1380-source run legitimately pushes the
        # answer/sources ratio below the 50% safety threshold and the
        # splitter logs a misleading "over-stripped" warning.
        # The sentinel is imported at module top; no circular-import
        # concern at use time.
        final_report_content += (
            f"\n\n{LDR_APPENDED_SOURCES_SENTINEL}\n\n"
            f"## Sources\n\n{formatted_all_links}"
        )

        # Create metadata dictionary
        metadata = {
            "generated_at": datetime.now(UTC).isoformat(),
            # Count the same visible, canonically grouped source set that
            # the Sources block above renders. Document-level: one document
            # cited via several chunk anchors still counts once even though
            # it renders several bibliography lines (distinct sources <=
            # bibliography lines <= citation indices) — so a count below
            # the rendered line total is expected, not a bug.
            "initial_sources": count_distinct_sources(visible_links),
            "sections_researched": len(structure),
            "searches_per_section": self.searches_per_section,
            "query": query,
        }

        # Return both content and metadata
        return {"content": final_report_content, "metadata": metadata}

    def _generate_error_report(self, query: str, error_msg: str) -> str:
        return f"=== ERROR REPORT ===\nQuery: {query}\nError: {error_msg}"