~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
import importlib
import re
from typing import Any, Dict, List, Optional
from datetime import datetime, UTC
from langchain_core.language_models import BaseChatModel
from loguru import logger
# Fix circular import by importing directly from source modules
from .config.llm_config import get_llm
from .config.thread_settings import get_setting_from_snapshot
from .search_system import AdvancedSearchSystem
from .text_optimization.citation_formatter import (
LDR_APPENDED_SOURCES_SENTINEL,
)
from .utilities.json_utils import get_llm_response_text
from .utilities.llm_utils import invoke_llm_sync
# Default constants for context accumulation to avoid repetition
# These are used as fallbacks when settings are not available
DEFAULT_MAX_CONTEXT_SECTIONS = (
3 # Number of previous sections to include as context
)
DEFAULT_MAX_CONTEXT_CHARS = (
4000 # Max characters for context (safe for smaller local models)
)
# Spelled-out output rules appended to every per-subsection prompt. Small
# and/or quantized local models routinely ignore long, vague instructions,
# so this is intentionally short, numbered, and refers to the actual
# rendering consequences (a duplicate heading, a duplicate bibliography)
# rather than abstract guidance.
#
# The framework always (a) inserts `## i.j Name` itself before appending
# LLM content and (b) appends one consolidated `## Sources` block to the
# whole report at the end of _format_final_report. The LLM does not need
# to repeat either of those things, and historically has — producing
# visibly stacked duplicate headings and duplicate "## Sources" sections
# per subsection.
_SUBSECTION_OUTPUT_GUIDANCE = (
"\n\nOUTPUT FORMAT RULES (FOLLOW EXACTLY):\n"
"1. Do NOT start your output with a Markdown heading of any level "
"(#, ##, ###, ####, etc.). The framework already inserts the "
"subsection heading for you; starting with your own heading line "
"creates a visible duplicate next to it in the final report.\n"
"2. Do NOT end your output with a sources, references, bibliography, "
"citations, key references, or selected bibliography section or list. "
"Do NOT include any trailing heading or label (such as 'Sources:', "
"'**Sources:**', or 'References:'), and do NOT append a list or block "
"of citations/links at the end of your response. Citations must ONLY "
"appear inline within sentences (e.g. 'According to [1], ...'). "
"The framework automatically compiles and appends the consolidated "
"source list after all subsections are completed; including your own "
"trailing sources or bibliography duplicates it.\n"
"3. Begin your output with prose (a paragraph or a table), not a "
"heading and not an italic purpose statement. You may use '###' / "
"'####' / deeper levels for internal sub-subheadings inside this "
"subsection only."
)
# Pre-compiled normalisers used by _strip_subsection_boilerplate. These
# are best-effort cleanups that run on the LLM's raw `current_knowledge`
# before it is appended to the section — they do NOT touch the
# framework's own headings or its trailing '## Sources' block.
#
# Leading heading: any ATX heading (level 1-6) at the very start of the
# subsection content. Limited to a single match so we never strip a real
# sub-subheading used to organise the body. Real reports show the LLM
# almost always opens with a redundant ``### <subsection name>`` (or a
# sibling's name, or a Roman-numeral-prefixed restatement) immediately
# under the framework's own ``## i.j Name`` heading.
_LEADING_HEADING_RE = re.compile(
r"\A[ \t\n]*[ \t]{0,3}#{1,6}[ \t]+[^\n]*",
)
# Leading italic purpose statement that mirrors the framework's
# ``_<purpose>_`` subtitle (e.g. ``_To summarise the section's scope..._``).
# Supports single underscores (_..._) or single asterisks (*...*).
# Limited to a single match at the start.
_LEADING_ITALIC_PURPOSE_RE = re.compile(
r"\A[ \t\n]*(?:_[^_\n]+_|(?:\*(?!\*)[^*\n]+\*))[ \t]*",
)
_BIB_KEYWORD_PATTERN = (
r"(?:"
r"Sources?|References?|Bibliograph(?:y|ies)|Citations?|"
r"Key[ \t]+References?|Selected[ \t]+Bibliograph(?:y|ies)|"
r"Works?[ \t]+Cited|Cited[ \t]+Works?|"
r"Additional[ \t]+Resources?|Further[ \t]+Reading"
r")"
)
_BIB_HEADINGS_CHAIN = (
rf"{_BIB_KEYWORD_PATTERN}"
rf"(?:[ \t]*(?:,|and|&|/)[ \t]+{_BIB_KEYWORD_PATTERN})*"
)
# ATX-style heading (with #) allowing:
# - levels 1-6 with optional empty heading line preceding it (e.g. "###\n**Sources:**")
# - optional section numbers (e.g. "1." or "1.1")
# - optional bold/italic markers ("**Sources**", "*References:*")
# - optional trailing colons, dashes, periods, or qualifiers
_ATX_BIB_HEADING = (
rf"[ \t]{{0,3}}#{{1,6}}[ \t]*(?:\n[ \t]*)?"
rf"(?:\d+(?:\.\d+)*\.?[ \t]+)?"
rf"(?:\*\*|\*|__|_)?{_BIB_HEADINGS_CHAIN}(?:\*\*|\*|__|_)?(?:"
rf"[ \t]*[:\-\u2013\u2014.][ \t]*[^\n]*"
rf"|[ \t]+for[ \t]+[^\n]+"
rf"|[ \t]*\([^\n]+\)"
rf")?[ \t]*$"
)
# Pseudo-heading (without #) where the keyword line acts as a section header:
# - bold or italic wrapped: "**Sources:**", "**Sources**", "*References:*"
# - or plain keyword ending with a colon or dash: "Sources:", "References:"
_PSEUDO_BIB_HEADING = (
r"[ \t]{0,3}"
rf"(?:"
rf"(?:\*\*|\*|__|_){_BIB_HEADINGS_CHAIN}[:\-\u2013\u2014.]?(?:\*\*|\*|__|_)"
rf"|"
rf"{_BIB_HEADINGS_CHAIN}[ \t]*[:\-\u2013\u2014]"
rf")[ \t]*$"
)
# Bibliography-style heading at levels 1-6 or pseudo-heading line.
_BIBLIOGRAPHY_HEADING_RE = re.compile(
rf"(?m)^(?:{_ATX_BIB_HEADING}|{_PSEUDO_BIB_HEADING})",
re.IGNORECASE,
)
# Next ATX heading at levels 1-6 — marks the end of a bibliography block
# when more subsection content follows it (common when the LLM appends a
# "Selected Bibliography" mid-stream and then continues writing).
_NEXT_HEADING_RE = re.compile(r"(?m)^[ \t]{0,3}#{1,6}[ \t]+")
# Decorative horizontal rules the LLM often wraps around bibliography
# blocks (``---``, ``***``, or ``___`` on their own line).
_HR_LINE_RE = re.compile(
r"(?m)^[ \t]{0,3}(?:(?:-[ \t]*){3,}|(?:_[ \t]*){3,}|(?:\*[ \t]*){3,})[ \t]*\n?"
)
# Empty heading line (e.g. `###`) or decorative horizontal rule line (`---`, `***`, `___`)
_EMPTY_HEADING_OR_HR_RE = re.compile(
r"^[ \t]*(?:#{1,6}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,}|(?:\*[ \t]*){3,})[ \t]*$"
)
# Reference-style link definitions in markdown: [1]: https://... or [name]: url
_MD_LINK_DEF_RE = re.compile(r"^[ \t]*\[[^\]]+\]:[ \t]*\S+")
# Ordinary ordered or bulleted list item prefix
_ORDERED_OR_BULLET_LIST_RE = re.compile(r"^[ \t]*(?:[-*•]|\d+[\.\)])\s+")
# Citation-list shape for ATX headings (## Sources, ### References):
# digit+dot, bullet, bracket citation start (single or double brackets),
# lenticular brackets, URL, DOI, or markdown link.
# Excludes markdown reference-style link definitions ("[1]: https://...").
_CITATION_LINE_RE = re.compile(
r"^\s*(?:"
r"\d+[\.\)]\s+"
r"|[-*•]\s+"
r"|\[\[\d+\]\](?:\([^)]*\))?"
r"|\[\d+\](?!\:)"
r"|【\d+】"
r"|\[cite\d*\]"
r"|\bhttps?://"
r"|\bdoi:\s*\S+"
r"|\[[^\]]+\]\((?:https?://|/library/)[^)]+\)"
r")",
re.IGNORECASE,
)
# Citation-list shape for pseudo-headings (Sources:, **Sources** without #):
# Must carry explicit citation markers (brackets, links, URLs, DOIs), NOT plain numbered/bullet lists.
_PSEUDO_BIB_CITATION_LINE_RE = re.compile(
r"^\s*(?:(?:[-*•]|\d+[\.\)])\s+)?"
r"(?:"
r"\[\[\d+\]\](?:\([^)]*\))?"
r"|\[\d+\](?!\:)"
r"|【\d+】"
r"|\[cite\d*\]"
r"|\bhttps?://"
r"|\bdoi:\s*\S+"
r"|\[[^\]]+\]\((?:https?://|/library/)[^)]+\)"
r")",
re.IGNORECASE,
)
# Single citation token for concatenated line check:
# Supports single brackets [1], double brackets [[1]], optional markdown link target,
# [cite1], or Chinese/Japanese lenticular brackets 【1】
_CONCATENATED_CITATION_ITEM = (
r"(?:(?:\[\[\d+\]\]|\[\d+\]|\[cite\d*\])(?:\([^)]*\))?|【\d+】)"
)
# Line composed purely of citation markers, brackets, links, and commas.
# Structured without overlapping leading/trailing whitespace quantifiers to prevent ReDoS.
_CONCATENATED_CITATIONS_LINE_RE = re.compile(
rf"^(?:[,\s]*{_CONCATENATED_CITATION_ITEM})+[,\s]*$"
)
# Unheaded citation list item start (for trailing citation blocks lacking any heading).
# Excludes numbered/bullet lists, markdown link definitions, and bare URLs without citation markers.
# Requires double brackets [[...]], single brackets with link target/source, cite tag, or DOI.
_UNHEADED_CITATION_ITEM_RE = re.compile(
r"^\s*(?:"
r"\[\[\d+\]\](?:\([^)]*\))?"
r"|\[\d+\](?!\:)(?:\((?:https?://|/library/)[^)]+\)|\s+(?:https?://|doi:))"
r"|\[cite\d*\]"
r"|doi:\s*\S+"
r")",
re.IGNORECASE,
)
# Note line matching LLM boilerplate (e.g. "*(Note: ... bibliography ...)*" or
# "[Note: The framework will append...]")
_BIB_NOTE_RE = re.compile(
r"^\s*(?:"
r"[\*_]\s*\(?Note\b[^\n]*?\b(?:bibliography|sources?|references?|citations?)\b"
r"|\[\s*Note\b"
r"(?=[^\n]*\b(?:consolidat\w*|append\w*|omit\w*|duplicat\w*|placeholder\w*)\b)"
r"[^\n]*?\b(?:bibliography|sources?|references?|citations?)\b)",
re.IGNORECASE,
)
def _get_code_fence_spans(text: str) -> List[tuple[int, int]]:
"""Return character index ranges (start, end) that are inside code blocks.
Recognizes:
- Backtick fenced blocks (``` or ```` etc.), paired by delimiter type and length.
- Tilde fenced blocks (~~~ or ~~~~ etc.), paired by delimiter type and length.
- Indented code blocks (lines indented by 4+ spaces or tabs, preceded by a blank line).
"""
spans: List[tuple[int, int]] = []
if not text:
return spans
lines = text.splitlines(keepends=True)
in_fence = False
fence_char = ""
fence_len = 0
fence_start = 0
in_indented = False
indented_start = 0
last_indented_end = 0
prev_blank = True
line_offset = 0
fence_open_re = re.compile(r"^[ \t]{0,3}(`{3,}|~{3,})")
for line in lines:
line_start = line_offset
line_end = line_offset + len(line)
line_offset = line_end
stripped = line.rstrip("\r\n")
if in_fence:
close_re = (
rf"^[ \t]{{0,3}}{re.escape(fence_char)}{{{fence_len},}}[ \t]*$"
)
if re.match(close_re, stripped):
in_fence = False
spans.append((fence_start, line_end))
continue
open_match = fence_open_re.match(stripped)
if open_match:
if in_indented:
in_indented = False
spans.append((indented_start, last_indented_end))
in_fence = True
delim = open_match.group(1)
fence_char = delim[0]
fence_len = len(delim)
fence_start = line_start
prev_blank = False
continue
is_indented = stripped.startswith(" ") or stripped.startswith("\t")
is_blank = not stripped.strip()
if in_indented:
if is_indented:
last_indented_end = line_end
elif is_blank:
pass
else:
in_indented = False
spans.append((indented_start, last_indented_end))
else:
if is_indented and prev_blank:
in_indented = True
indented_start = line_start
last_indented_end = line_end
prev_blank = is_blank
if in_fence:
spans.append((fence_start, len(text)))
elif in_indented:
spans.append((indented_start, last_indented_end))
return spans
def _is_in_spans(pos: int, spans: List[tuple[int, int]]) -> bool:
"""Return True if pos falls within any range in spans."""
return any(start <= pos < end for start, end in spans)
def get_report_generator(search_system=None):
"""Return an instance of the report generator with default settings.
Args:
search_system: Optional existing AdvancedSearchSystem to use
"""
return IntegratedReportGenerator(search_system=search_system)
class IntegratedReportGenerator:
def __init__(
self,
searches_per_section: int = 2,
search_system=None,
llm: BaseChatModel | None = None,
settings_snapshot: Optional[Dict] = None,
):
"""
Args:
searches_per_section: Number of searches to perform for each
section in the report.
search_system: Custom search system to use, otherwise just uses
the default.
llm: Custom LLM to use. Required if search_system is not provided.
settings_snapshot: Optional settings snapshot for configurable values.
"""
# If search_system is provided, use its LLM; otherwise use the provided LLM
self._owns_llm = False
if search_system:
self.search_system = search_system
self.model = llm or search_system.model
elif llm:
self.model = llm
self.search_system = AdvancedSearchSystem(llm=self.model) # type: ignore[call-arg]
else:
# Fallback for backwards compatibility - will only work with auth
self._owns_llm = True
self.model = get_llm()
self.search_system = AdvancedSearchSystem(llm=self.model) # type: ignore[call-arg]
self.searches_per_section = (
searches_per_section # Control search depth per section
)
# Load context settings from snapshot or use defaults
self.max_context_sections = get_setting_from_snapshot(
"report.max_context_sections",
default=DEFAULT_MAX_CONTEXT_SECTIONS,
settings_snapshot=settings_snapshot,
)
self.max_context_chars = get_setting_from_snapshot(
"report.max_context_chars",
default=DEFAULT_MAX_CONTEXT_CHARS,
settings_snapshot=settings_snapshot,
)
def close(self) -> None:
"""Close the LLM client if this instance created it."""
from .utilities.resource_utils import safe_close
if self._owns_llm:
safe_close(self.model, "report generator LLM")
def generate_report(
self,
initial_findings: Dict,
query: str,
progress_callback=None,
) -> Dict:
"""Generate a complete research report with section-specific research.
Args:
initial_findings: Results from initial research phase.
query: Original user query.
progress_callback: Optional callable(message, progress_percent, metadata)
for reporting progress (0-100%) and checking cancellation.
"""
# Step 1: Determine structure
if progress_callback:
progress_callback(
"Determining report structure",
0,
{"phase": "report_structure"},
)
structure = self._determine_report_structure(initial_findings, query)
# Step 2: Research and generate content for each section in one step
sections = self._research_and_generate_sections(
initial_findings,
structure,
query,
progress_callback=progress_callback,
)
# Step 3: Format final report
if progress_callback:
progress_callback(
"Formatting final report",
90,
{"phase": "report_formatting"},
)
report = self._format_final_report(sections, structure, query)
if progress_callback:
progress_callback(
"Report complete", 100, {"phase": "report_complete"}
)
return report
def _determine_report_structure(
self, findings: Dict, query: str
) -> List[Dict]:
"""Analyze content and determine the optimal detailed report structure."""
combined_content = findings["current_knowledge"]
prompt = f"""
Analyze this research content about: {query}
Content Summary:
{combined_content}
Determine the most appropriate detailed report structure by:
1. Analyzing the type of content (technical, business, academic, etc.)
2. Identifying main themes and logical groupings
3. Considering the depth and breadth of the research
Return a table of contents structure in this exact format:
STRUCTURE
1. [Section Name]
- [Subsection] | [purpose]
2. [Section Name]
- [Subsection] | [purpose]
...
END_STRUCTURE
Make the structure specific to the content, not generic.
Each subsection must include its purpose after the | symbol.
DO NOT include sections about sources, citations, references, or methodology.
"""
response = get_llm_response_text(invoke_llm_sync(self.model, prompt))
# Parse the structure
structure: List[Dict[str, Any]] = []
current_section: Optional[Dict[str, Any]] = None
for line in response.split("\n"):
if line.strip() in ["STRUCTURE", "END_STRUCTURE"]:
continue
if line.strip().startswith(tuple("123456789")):
# Main section — require a dot-delimited name (e.g. "1. Intro").
parts = line.split(".", 1)
if len(parts) < 2 or not parts[1].strip():
continue
section_name = parts[1].strip()
current_section = {"name": section_name, "subsections": []}
structure.append(current_section)
elif line.strip().startswith("-") and current_section:
# Subsection with or without purpose
parts = line.strip("- ").split(
"|", 1
) # Only split on first pipe
if len(parts) == 2:
current_section["subsections"].append(
{"name": parts[0].strip(), "purpose": parts[1].strip()}
)
elif len(parts) == 1 and parts[0].strip():
# Subsection without purpose - add default
current_section["subsections"].append(
{
"name": parts[0].strip(),
"purpose": f"Provide detailed information about {parts[0].strip()}",
}
)
# Check if the last section is source-related and remove it
if structure:
last_section = structure[-1]
section_name_lower = last_section["name"].lower()
source_keywords = [
"source",
"citation",
"reference",
"bibliography",
]
# Only check the last section for source-related content
if any(
keyword in section_name_lower for keyword in source_keywords
):
logger.info(
f"Removed source-related last section: {last_section['name']}"
)
structure = structure[:-1]
return structure
def _truncate_at_sentence_boundary(self, text: str, max_chars: int) -> str:
"""Truncate text at a sentence boundary to preserve readability.
Attempts to cut at the last sentence-ending punctuation (.!?) before
the limit. If no suitable boundary is found within 80% of the limit,
falls back to hard truncation.
Args:
text: Text to truncate.
max_chars: Maximum characters allowed.
Returns:
Truncated text with ``[...truncated]`` marker if truncation
occurred, otherwise the original text unchanged.
"""
if len(text) <= max_chars:
return text
truncated = text[:max_chars]
# Look for sentence boundaries (. ! ?) followed by space or newline
# Search backwards from the end for the last complete sentence
last_sentence_end = -1
for i in range(len(truncated) - 1, -1, -1):
if truncated[i] in ".!?" and (
i + 1 >= len(truncated) or truncated[i + 1] in " \n"
):
last_sentence_end = i + 1
break
# Only use sentence boundary if it preserves at least 80% of content
min_acceptable = int(max_chars * 0.8)
if last_sentence_end > min_acceptable:
return truncated[:last_sentence_end] + "\n[...truncated]"
# Fall back to hard truncation
return truncated + "\n[...truncated]"
@staticmethod
def _normalize_heading_text(text: str) -> str:
"""Normalize heading text for fuzzy comparison against subsection names.
Strips markdown bold markers, leading Roman-numeral / alphabetic /
numeric enumeration prefixes (``II.``, ``A.``, ``1.``), collapses
whitespace, and lowercases. Used only for matching — never written
back into the report.
"""
cleaned = text.strip()
# Drop surrounding bold/italic markers the LLM wraps titles in
# (``**Title**``, ``*Title*``, ``__Title__``). Avoid regex here —
# CodeQL flags ``^[*_]+|[*_]+$`` as polynomial on long ``*`` runs.
cleaned = cleaned.strip("*_").strip()
cleaned = re.sub(
r"^(?:"
r"[IVXLCDM]+\.|" # Roman numerals: II. III. IV. (uppercase only to avoid "Mix." false strip)
r"[A-Z]\.|" # Single letters: A. B.
r"\d+\." # Arabic numerals: 1. 2.
r")\s+",
"",
cleaned,
)
# Normalise curly/smart quotes and dashes so "Author's" (U+2019)
# matches "Author's" (ASCII apostrophe) and en/em dashes collapse
# to a plain hyphen for comparison.
cleaned = cleaned.translate(
str.maketrans(
"\u2018\u2019\u201c\u201d\u2013\u2014\u2212", "''\"\"---"
)
)
return re.sub(r"\s+", " ", cleaned).strip().lower()
@classmethod
def _heading_restates_name(cls, heading_text: str, name: str) -> bool:
"""Return True if *heading_text* is a restatement of *name*.
Handles exact matches, prefix matches (heading extends the name
with a colon/dash subtitle), and the reverse (name is a longer
form of a short heading). Empty names never match.
"""
if not name or not heading_text:
return False
norm_heading = cls._normalize_heading_text(heading_text)
norm_name = cls._normalize_heading_text(name)
if not norm_heading or not norm_name:
return False
if norm_heading == norm_name:
return True
# Heading extends the subsection name with a subtitle (e.g. "Name: subtitle")
# Delimiters only: bare space prefixes (e.g. "Introduction to …")
# are intentional non-matches. En/em dashes are already folded to
# ASCII "-" by _normalize_heading_text.
if norm_heading.startswith(norm_name):
rest = norm_heading[len(norm_name) :].lstrip()
if rest and rest[0] in (":", "-", "("):
return True
# Name extends a short heading with a subtitle.
if norm_name.startswith(norm_heading):
rest = norm_name[len(norm_heading) :].lstrip()
if rest and rest[0] in (":", "-", "("):
return True
return False
def _strip_leading_heading(
self,
content: str,
subsection_name: str,
sibling_subsection_names: Optional[List[str]] = None,
) -> str:
"""Drop a single leading heading line only when it restates subsection or sibling name."""
leading = _LEADING_HEADING_RE.match(content)
if not leading:
return content
fence_spans = _get_code_fence_spans(content)
if _is_in_spans(leading.start(), fence_spans):
return content
heading_text = re.sub(r"^\s*#{1,6}[ \t]+", "", leading.group(0)).strip()
# Subsection name: allow subtitle extensions (Name: subtitle)
if self._heading_restates_name(heading_text, subsection_name):
end_pos = leading.end()
while end_pos < len(content) and content[end_pos] in "\r\n":
end_pos += 1
return content[end_pos:]
# Siblings: require exact normalized equality (avoid over-strip of
# "### <Sibling>: Subtitle" organizers)
norm_heading = self._normalize_heading_text(heading_text)
for sibling in sibling_subsection_names or []:
if sibling and norm_heading == self._normalize_heading_text(
sibling
):
end_pos = leading.end()
while end_pos < len(content) and content[end_pos] in "\r\n":
end_pos += 1
return content[end_pos:]
return content
def _strip_leading_italic_purpose(
self,
content: str,
purpose: Optional[str] = None,
) -> str:
"""Drop a single leading italic purpose statement if it mirrors purpose or boilerplate."""
leading_italic = _LEADING_ITALIC_PURPOSE_RE.match(content)
if not leading_italic:
return content
fence_spans = _get_code_fence_spans(content)
if _is_in_spans(leading_italic.start(), fence_spans):
return content
italic_raw = leading_italic.group(0).strip()
italic_text = italic_raw.strip("_*").strip()
should_strip = False
if purpose:
norm_italic = self._normalize_heading_text(italic_text)
norm_purpose = self._normalize_heading_text(purpose)
# Guard against degenerate purpose that normalizes to "" (e.g. "***")
# — startswith("") is always True and would strip any italic line.
if norm_italic and norm_purpose:
if (
norm_italic == norm_purpose
or norm_italic.startswith(norm_purpose)
or norm_purpose.startswith(norm_italic)
or self._heading_restates_name(italic_text, purpose)
):
should_strip = True
if not should_strip:
norm_italic = self._normalize_heading_text(italic_text)
if norm_italic:
# Narrow unconditional boilerplate markers — always safe to strip
if norm_italic.startswith(
(
"purpose:",
"scope:",
"this subsection ",
"this section ",
)
):
should_strip = True
# Verb-prefix boilerplate: check for relevance to purpose to avoid
# eating legitimate epigraphs like "_To review the evidence is to understand the past._"
# when purpose is unrelated (S2).
elif purpose is not None and norm_italic.startswith(
(
"to summarize",
"to summarise",
"to clarify",
"to examine",
"to outline",
"to provide",
"to describe",
"to detail",
"to analyze",
"to analyse",
"to explore",
"to present",
"to discuss",
"to evaluate",
"to investigate",
"to review",
"to assess",
"aims to",
"designed to",
)
):
norm_purpose = self._normalize_heading_text(purpose)
if norm_purpose:
_stop_words = {
"to",
"the",
"and",
"for",
"that",
"this",
"with",
"from",
"summarize",
"summarise",
"clarify",
"examine",
"outline",
"provide",
"describe",
"detail",
"analyze",
"analyse",
"explore",
"present",
"discuss",
"evaluate",
"investigate",
"review",
"assess",
"aims",
"designed",
"section",
"subsection",
}
purpose_words = {
w
for w in re.findall(
r"\b[a-z0-9]{3,}\b", norm_purpose.lower()
)
if w not in _stop_words
}
italic_words = {
w
for w in re.findall(
r"\b[a-z0-9]{3,}\b", norm_italic.lower()
)
if w not in _stop_words
}
if (
self._heading_restates_name(italic_text, purpose)
or norm_italic.startswith(norm_purpose)
or norm_purpose.startswith(norm_italic)
or bool(purpose_words & italic_words)
):
should_strip = True
if should_strip:
end_pos = leading_italic.end()
while end_pos < len(content) and content[end_pos] in "\r\n":
end_pos += 1
return content[end_pos:]
return content
def _strip_embedded_bibliographies(self, content: str) -> str:
"""Remove every embedded bibliography block outside code blocks.
Only removes a heading that looks like a bibliography *and* whose
following block looks like a citation list — this prevents
substantive headings such as "### Sources of Bias..." or
"### Sources. Data Collection Methodology" from being deleted with
their analysis body (R1). Consecutive bibliography headings no longer
consume intervening prose (R8).
"""
fence_spans = _get_code_fence_spans(content)
pieces: List[str] = []
cursor = 0
bib_removed = False
for match in _BIBLIOGRAPHY_HEADING_RE.finditer(content):
if _is_in_spans(match.start(), fence_spans):
continue
matched_heading_text = match.group(0).lstrip()
is_atx_heading = matched_heading_text.startswith("#")
bib_line_re = (
_CITATION_LINE_RE
if is_atx_heading
else _PSEUDO_BIB_CITATION_LINE_RE
)
after_heading = match.end()
next_heading_start = None
next_heading_is_bib = False
for nh in _NEXT_HEADING_RE.finditer(content, after_heading):
if not _is_in_spans(nh.start(), fence_spans):
next_heading_start = nh.start()
# Peek if the next heading itself is a bibliography heading
# — if so, we will trim the current block to its citation
# list end rather than to the heading (R8).
heading_line = content[nh.start() :].split("\n", 1)[0]
if _BIBLIOGRAPHY_HEADING_RE.match(heading_line):
next_heading_is_bib = True
break
block_end = (
next_heading_start
if next_heading_start is not None
else len(content)
)
block_text = content[after_heading:block_end]
# If the block immediately begins with a code fence, the heading is
# introducing fenced example/code content, not a bibliography.
offset = 0
first_non_empty_start = None
for ln in block_text.splitlines(keepends=True):
if ln.strip():
first_non_empty_start = after_heading + offset
break
offset += len(ln)
if first_non_empty_start is not None and _is_in_spans(
first_non_empty_start, fence_spans
):
continue
# Body-shape safeguard: only treat as bibliography if the block
# contains citation-like lines. Require at least one citation
# marker, or >30% of non-empty lines look like citations, to
# avoid deleting substantive analysis headings that happen to
# match the label grammar (e.g. "### Sources for the Analysis"
# with prose body).
non_empty_lines = []
line_offset = 0
for ln in block_text.splitlines(keepends=True):
line_start = after_heading + line_offset
if ln.strip() and not _is_in_spans(line_start, fence_spans):
non_empty_lines.append(ln)
line_offset += len(ln)
if not non_empty_lines:
# Empty block — treat as bibliography (heading with no body)
is_bib_block = True
else:
# Intervening prose safeguard: a bibliography heading must introduce
# citations directly (or via an immediate note / 1-line lead-in).
# If substantive prose precedes citations, the heading is introducing
# substantive analysis, not an embedded bibliography.
first_line = non_empty_lines[0]
first_is_bib = (
bib_line_re.match(first_line)
or _BIB_NOTE_RE.match(first_line)
or _HR_LINE_RE.match(first_line)
)
if not first_is_bib:
# Check if it's a short 1-line lead-in immediately followed by citations
second_is_citation = len(
non_empty_lines
) > 1 and bib_line_re.match(non_empty_lines[1])
is_lead_in = (
len(first_line.strip()) < 120
and second_is_citation
and bool(
re.search(
r"\b(?:sources?|references?|consulted|cited|following|basis)\b",
first_line,
re.IGNORECASE,
)
)
)
has_early_note = any(
_BIB_NOTE_RE.match(ln) for ln in non_empty_lines[:3]
)
if not (is_lead_in or has_early_note):
continue
citation_lines = sum(
1 for ln in non_empty_lines if bib_line_re.match(ln)
)
# Require explicit italic note boilerplate (e.g. "*(Note: ... bibliography ...)*")
# rather than matching arbitrary prose containing "bibliography" (S1).
has_bib_note = any(
_BIB_NOTE_RE.match(ln) for ln in non_empty_lines[:3]
)
if citation_lines == 0 and not has_bib_note:
# No citation shape and no italic bib note — preserve heading + body
continue
# If citation lines present, require they are not a tiny minority
# unless block is short (e.g. "- item" single line)
if citation_lines == 0:
is_bib_block = has_bib_note
elif len(non_empty_lines) <= 3:
is_bib_block = citation_lines >= 1
else:
is_bib_block = (
citation_lines / len(non_empty_lines)
) >= 0.3 or citation_lines >= 2
if not is_bib_block:
continue
# Passed safeguard — this is a real bibliography block to remove
bib_removed = True
block_start = match.start()
prefix = content[cursor:block_start]
prefix_lines = prefix.splitlines(keepends=True)
cut_idx = len(prefix)
curr_offset = len(prefix)
found_delimiter = False
for pline in reversed(prefix_lines):
line_start = curr_offset - len(pline)
abs_pos = cursor + line_start
if _is_in_spans(abs_pos, fence_spans):
break
pstripped = pline.strip()
if not pstripped:
curr_offset = line_start
continue
if _EMPTY_HEADING_OR_HR_RE.match(pstripped):
found_delimiter = True
cut_idx = line_start
curr_offset = line_start
else:
break
if found_delimiter and cut_idx < len(prefix):
block_start = cursor + cut_idx
pieces.append(content[cursor:block_start])
# R8: consecutive bib headings — end at citation list, not at next bib heading
if next_heading_is_bib:
# Find end of citation list within block_text
lines = block_text.splitlines(keepends=True)
offset = 0
last_citation_end = 0
for ln in lines:
line_start = after_heading + offset
if not _is_in_spans(line_start, fence_spans) and (
bib_line_re.match(ln)
or (_BIB_NOTE_RE.match(ln) and offset < 300)
):
last_citation_end = offset + len(ln)
# Also keep consecutive citation lines; stop at first
# non-citation prose that is not a blank/bib note?
offset += len(ln)
# Also include trailing blank lines after citations, but not
# intervening prose before next bib heading
if last_citation_end > 0:
# Advance cursor to after the citation list (preserve
# intervening prose between two bib headings)
cursor = after_heading + last_citation_end
# Skip following blank lines / HRs but not prose
while cursor < block_end and content[cursor] in " \t\r\n-":
# Only skip whitespace/HRs, stop at prose
# Peek next non-whitespace chunk
nxt = content[cursor:].lstrip(" \t\r\n")
if nxt.startswith("-") and nxt[1:2] in " \t":
# Another HR-like line, skip it
cursor += len(content[cursor:]) - len(nxt)
# consume the HR line
eol = content.find("\n", cursor)
cursor = eol + 1 if eol != -1 else len(content)
else:
break
if cursor >= block_end:
break
# Ensure we don't overshoot into next bib heading's prefix;
# if cursor is before next_heading_start, keep it, else
# fall back to next_heading_start
if cursor > block_end:
cursor = block_end
else:
cursor = block_end
else:
# End the removal at the citation list rather than at the end
# of the block, so a table, list or paragraph the model wrote
# after its bibliography survives the strip.
offset = 0
citation_end = 0
non_empty_seen = 0
trailing_content = False
for ln in block_text.splitlines(keepends=True):
line_start = after_heading + offset
substantive = bool(ln.strip())
outside_fence = not _is_in_spans(line_start, fence_spans)
non_empty_seen += substantive and outside_fence
# The note window is the classifier's first-three-non-empty-
# lines rule, so a block accepted as a bibliography always
# has a qualifying line here and cannot fall through.
is_bib_line = bool(
outside_fence
and (
bib_line_re.match(ln)
or (_BIB_NOTE_RE.match(ln) and non_empty_seen <= 3)
or _HR_LINE_RE.match(ln)
)
)
if citation_end and substantive and not is_bib_line:
trailing_content = True
break
offset += len(ln)
if substantive and is_bib_line:
citation_end = offset
if not trailing_content:
# Nothing substantive follows, so the whole block is the
# bibliography and is removed as before.
citation_end = len(block_text)
cursor = after_heading + citation_end
if bib_removed:
pieces.append(content[cursor:])
content = "".join(pieces)
return self._strip_trailing_citation_blocks(content)
def _strip_trailing_citation_blocks(self, content: str) -> str:
"""Remove unheaded trailing citation blocks or concatenated citations.
When the LLM is instructed not to include a sources section heading,
some models omit the heading entirely but append a list or block of
citations at the end of the response (e.g. consecutive lines of
`[[111]](/library/...)` or a concatenated line of citations).
This defensive helper scans backwards from the end of the content
outside code fences and strips trailing citation blocks while
preserving all substantive prose, tables, code fences, reference-style
link definitions, further-reading URLs, quiz answer keys, and CJK notes.
"""
if not content:
return content
fence_spans = _get_code_fence_spans(content)
lines = content.splitlines(keepends=True)
if not lines:
return content
line_offsets: List[int] = []
curr = 0
for ln in lines:
line_offsets.append(curr)
curr += len(ln)
# Skip trailing empty / whitespace lines
cut_idx = len(lines)
while cut_idx > 0 and not lines[cut_idx - 1].strip():
cut_idx -= 1
if cut_idx == 0:
return content
# Never strip inside code fences
if _is_in_spans(line_offsets[cut_idx - 1], fence_spans):
return content
citation_lines_count = 0
concatenated_lines_count = 0
bib_heading_preceded = False
idx = cut_idx
while idx > 0:
offset = line_offsets[idx - 1]
if _is_in_spans(offset, fence_spans):
break
line_str = lines[idx - 1].strip()
if not line_str:
idx -= 1
continue
# Reference-style link definitions ([1]: https://...) must NEVER be stripped
if _MD_LINK_DEF_RE.match(line_str):
break
# Plain numbered or bulleted list items must NEVER be stripped as unheaded citations
if _ORDERED_OR_BULLET_LIST_RE.match(line_str):
break
# Concatenated citation clusters (e.g. [[217]][[218]] or [[1]](/a), [2], [[3]])
if _CONCATENATED_CITATIONS_LINE_RE.match(line_str):
# Must contain double brackets [[...]], cite tags, or adjacent bracket tokens;
# exclude plain numbered markdown links like [1](url) [2](url)
if (
"[[" in line_str
or "[cite" in line_str.lower()
or bool(re.search(r"\]\s*\[|】\s*[【\[]|\]\s*【", line_str))
):
concatenated_lines_count += 1
idx -= 1
continue
break
if _UNHEADED_CITATION_ITEM_RE.match(line_str):
citation_lines_count += 1
idx -= 1
continue
# Support indented continuation lines within a citation entry.
# When scanning backwards, find the preceding non-indented line.
if lines[idx - 1].startswith((" ", "\t")):
p = idx - 2
while p >= 0 and lines[p].startswith((" ", "\t")):
p -= 1
if p >= 0 and _UNHEADED_CITATION_ITEM_RE.match(
lines[p].strip()
):
idx -= 1
continue
if _BIB_NOTE_RE.match(line_str):
idx -= 1
continue
# Check if preceded by an explicit bibliography heading or label
if citation_lines_count > 0 or concatenated_lines_count > 0:
if _BIBLIOGRAPHY_HEADING_RE.match(line_str):
bib_heading_preceded = True
idx -= 1
continue
if _EMPTY_HEADING_OR_HR_RE.match(line_str):
idx -= 1
continue
break
# Only strip if:
# 1. Found a concatenated citation cluster
# 2. Preceded by an explicit bibliography heading/label and found >= 1 citation items
# 3. Found >= 2 citation items in an unheaded block (bare --- does NOT arm single items)
should_strip = False
if concatenated_lines_count > 0:
should_strip = True
elif bib_heading_preceded and citation_lines_count >= 1:
should_strip = True
elif citation_lines_count >= 2:
should_strip = True
if should_strip:
# Also consume any preceding empty headings, labels, or HRs
while idx > 0:
offset = line_offsets[idx - 1]
if _is_in_spans(offset, fence_spans):
break
pstripped = lines[idx - 1].strip()
if not pstripped:
idx -= 1
continue
if _EMPTY_HEADING_OR_HR_RE.match(
pstripped
) or _BIBLIOGRAPHY_HEADING_RE.match(pstripped):
idx -= 1
else:
break
return "".join(lines[:idx])
return content
def _strip_subsection_boilerplate(
self,
content: str,
subsection_name: str,
section_name: str,
sibling_subsection_names: Optional[List[str]] = None,
purpose: Optional[str] = None,
) -> str:
"""Strip boilerplate the LLM tends to emit around subsection content.
Small and/or quantized local models routinely produce three artefacts
that pollute the rendered report even when the OUTPUT FORMAT RULES
spelled out in the subsection prompt explicitly forbid them:
1. A redundant leading heading (stripped by :meth:`_strip_leading_heading`)
2. An italic purpose statement (stripped by :meth:`_strip_leading_italic_purpose`)
3. An embedded bibliography block (stripped by :meth:`_strip_embedded_bibliographies`)
This helper normalises the per-subsection content so the rendered
report stays clean even when the model ignored the prompt. It is a
defensive cleanup, not a substitute for the prompt rules.
Args:
content: Raw ``current_knowledge`` returned for one subsection.
subsection_name: The subsection's name (used for matching and
logging).
section_name: The parent section's name (used for logging).
sibling_subsection_names: Optional names of other subsections
in the same section. When provided, a leading heading that
restates a *sibling's* name (context-bleed) is also stripped.
purpose: Optional purpose string for the subsection. Used to
verify if a leading italic line mirrors the subsection purpose.
Returns:
``content`` with a redundant leading heading removed when it
restates this or a sibling subsection name, a leading
italic-purpose statement removed, and any embedded bibliography
blocks removed. Returns content unchanged if no artefacts are
found, aside from collapsing 3+ blank lines to 2 and
normalising trailing whitespace (collapsed outside code fences
only, so fenced blocks are never rewritten).
"""
if not content:
return content
original_len = len(content)
new_content = self._strip_leading_heading(
content, subsection_name, sibling_subsection_names
)
new_content = self._strip_leading_italic_purpose(new_content, purpose)
pre_bib_len = len(new_content)
new_content = self._strip_embedded_bibliographies(new_content)
pre_trailing_len = len(new_content)
new_content = self._strip_trailing_citation_blocks(new_content)
bib_chars_removed = (pre_bib_len - pre_trailing_len) + (
pre_trailing_len - len(new_content)
)
# R2: fence-aware whitespace collapse — never rewrite inside code fences
fence_spans = _get_code_fence_spans(new_content)
if not fence_spans:
collapsed = re.sub(r"\n{3,}", "\n\n", new_content).strip()
else:
parts: List[str] = []
last = 0
for s, e in sorted(fence_spans):
# collapse outside fence
parts.append(re.sub(r"\n{3,}", "\n\n", new_content[last:s]))
# keep fence content verbatim
parts.append(new_content[s:e])
last = e
parts.append(re.sub(r"\n{3,}", "\n\n", new_content[last:]))
collapsed = "".join(parts).strip()
new_content = collapsed
if new_content:
new_content += "\n"
if len(new_content) != original_len:
# R5: make silent deletions observable
if not new_content.strip() and content.strip():
logger.warning(
"Stripped subsection boilerplate emptied non-trivial content for "
f"'{section_name} > {subsection_name}': "
f"{original_len} -> {len(new_content)} chars — "
"check for over-strip (R1/R3 regression)"
)
elif len(new_content) < 0.5 * original_len:
# If the truncation below 50% was driven by stripping an
# embedded bibliography (where pre-bibliography content was
# >= 50% and bib removal did the rest), this is expected
# cleanup rather than an unexpected over-strip.
if (
bib_chars_removed > 0
and pre_bib_len >= 0.5 * original_len
and (original_len - pre_bib_len) < 0.25 * original_len
):
logger.info(
"Stripped embedded bibliography from subsection "
f"'{section_name} > {subsection_name}': "
f"{original_len} -> {len(new_content)} chars "
"(expected cleanup; warning demoted to INFO)"
)
else:
logger.warning(
"Stripped subsection boilerplate heavily truncated content for "
f"'{section_name} > {subsection_name}': "
f"{original_len} -> {len(new_content)} chars"
)
else:
logger.debug(
"Stripped subsection boilerplate for "
f"'{section_name} > {subsection_name}': "
f"{original_len} -> {len(new_content)} chars"
)
return new_content
def _build_previous_context(self, accumulated_findings: List[str]) -> str:
"""Build context block from previously generated sections.
Creates a formatted context block containing content from the last
N sections (defined by self.max_context_sections) with explicit instructions
not to repeat this content. Context is truncated if it exceeds
self.max_context_chars to stay safe for smaller local models.
Args:
accumulated_findings: List of previously generated section content,
each formatted as "[Section > Subsection]\\n{content}"
Returns:
Formatted context block with delimiters, or empty string if no
previous findings exist
"""
if not accumulated_findings:
return ""
recent_findings = accumulated_findings[-self.max_context_sections :]
previous_context = "\n\n---\n\n".join(recent_findings)
# Truncate at sentence boundary if too long
if len(previous_context) > self.max_context_chars:
previous_context = self._truncate_at_sentence_boundary(
previous_context, self.max_context_chars
)
return (
f"\n\n=== CONTENT ALREADY WRITTEN (DO NOT REPEAT) ===\n"
f"{previous_context}\n"
f"=== END OF PREVIOUS CONTENT ===\n\n"
f"CRITICAL: The above content has already been written. Do NOT repeat "
f"these points, examples, or explanations. Focus on NEW information "
f"not covered above.\n"
)
def _research_and_generate_sections(
self,
initial_findings: Dict,
structure: List[Dict],
query: str,
progress_callback=None,
) -> Dict[str, str]:
"""Research and generate content for each section in one step.
This method processes sections sequentially, accumulating generated
content as it goes. For each new section/subsection, it passes context
from the last few previously generated sections to help the LLM avoid
repetition.
The context accumulation mechanism:
- Tracks all generated content in accumulated_findings list
- Before generating each section, builds context from recent findings
- Uses self.max_context_sections (configurable, default: 3) to limit context size
- Truncates context to self.max_context_chars (configurable, default: 4000) for safety
- Includes explicit "DO NOT REPEAT" instructions with actual content
Args:
initial_findings: Results from initial research phase, may contain
questions_by_iteration to preserve search continuity
structure: List of section definitions, each with name and subsections
query: Original user query for context
Returns:
Dict mapping section names to their generated markdown content
"""
sections = {}
# Accumulate content from previous sections to avoid repetition
accumulated_findings: List[str] = []
# Count total subsections for progress tracking
total_subsections = sum(
max(len(section.get("subsections", [])), 1) for section in structure
)
completed_subsections = 0
# Preserve questions from initial research to avoid repetition
# This follows the same pattern as citation tracking (all_links_of_system)
existing_questions = initial_findings.get("questions_by_iteration", {})
if existing_questions:
# Set questions on both search system and its strategy
if hasattr(self.search_system, "questions_by_iteration"):
self.search_system.questions_by_iteration = (
existing_questions.copy()
)
# More importantly, set it on the strategy which actually uses it
if hasattr(self.search_system, "strategy") and hasattr(
self.search_system.strategy, "questions_by_iteration"
):
self.search_system.strategy.questions_by_iteration = (
existing_questions.copy()
)
logger.info(
f"Initialized strategy with {len(existing_questions)} iterations of previous questions"
)
for i, section in enumerate(structure, 1):
logger.info(f"Processing section: {section['name']}")
section_content = []
section_content.append(f"# {i}. {section['name']}\n")
# If section has no subsections, create one from the section itself
if not section["subsections"]:
# Parse section name for purpose
if "|" in section["name"]:
parts = section["name"].split("|", 1)
section["subsections"] = [
{"name": parts[0].strip(), "purpose": parts[1].strip()}
]
else:
# No purpose provided - use section name as subsection
section["subsections"] = [
{
"name": section["name"],
"purpose": f"Provide comprehensive content for {section['name']}",
}
]
# Process each subsection by directly researching it
for j, subsection in enumerate(section["subsections"], 1):
# Only add subsection header if there are multiple subsections
if len(section["subsections"]) > 1:
section_content.append(f"## {i}.{j} {subsection['name']}\n")
section_content.append(f"_{subsection['purpose']}_\n\n")
# Get other subsections in this section for context
other_subsections = [
f"- {s['name']}: {s['purpose']}"
for s in section["subsections"]
if s["name"] != subsection["name"]
]
other_subsections_text = (
"\n".join(other_subsections)
if other_subsections
else "None"
)
# Get all other sections for broader context
other_sections = [
f"- {s['name']}"
for s in structure
if s["name"] != section["name"]
]
other_sections_text = (
"\n".join(other_sections) if other_sections else "None"
)
# Check if this is actually a section-level content (only one subsection, likely auto-created)
is_section_level = len(section["subsections"]) == 1
# Build context from previously generated sections to avoid repetition
previous_context_section = self._build_previous_context(
accumulated_findings
)
# Generate appropriate search query
if is_section_level:
# Section-level prompt - more comprehensive
subsection_query = (
f"Research task: Create comprehensive content for the '{subsection['name']}' section in a report about '{query}'. "
f"Section purpose: {subsection['purpose']} "
f"\n"
f"Other sections in the report:\n{other_sections_text}\n"
f"{previous_context_section}"
f"This is a standalone section requiring comprehensive coverage of its topic. "
f"Provide a thorough exploration that may include synthesis of information from previous sections where relevant. "
f"Include unique insights, specific examples, and concrete data. "
f"Use tables to organize information where applicable. "
f"For conclusion sections: synthesize key findings and provide forward-looking insights. "
f"Build upon the research findings from earlier sections to create a cohesive narrative."
f"{_SUBSECTION_OUTPUT_GUIDANCE}"
)
else:
# Subsection-level prompt - more focused
subsection_query = (
f"Research task: Create content for subsection '{subsection['name']}' in a report about '{query}'. "
f"This subsection's purpose: {subsection['purpose']} "
f"Part of section: '{section['name']}' "
f"\n"
f"Other sections in the report:\n{other_sections_text}\n"
f"\n"
f"Other subsections in this section will cover:\n{other_subsections_text}\n"
f"{previous_context_section}"
f"Focus ONLY on information specific to your subsection's purpose. "
f"Include unique details, specific examples, and concrete data. "
f"Use tables to organize information where applicable. "
f"IMPORTANT: Avoid repeating information that would logically be covered in other sections - focus on what makes this subsection unique. "
f"Previous research exists - find specific angles for this subsection."
f"{_SUBSECTION_OUTPUT_GUIDANCE}"
)
logger.info(
f"Researching subsection: {subsection['name']} with query: {subsection_query}"
)
# Report progress and check for cancellation
if progress_callback:
pct = int(
10
+ (completed_subsections / max(total_subsections, 1))
* 80
)
progress_callback(
f"Researching: {section['name']} > {subsection['name']}",
pct,
{
"phase": "report_section_research",
"subsection": subsection["name"],
},
)
# Fix iteration override: modify strategy's settings_snapshot
# which is read dynamically via get_setting()
strategy = self.search_system.strategy
original_iterations = strategy.settings_snapshot.get(
"search.iterations"
)
had_iterations_key = (
"search.iterations" in strategy.settings_snapshot
)
strategy.settings_snapshot["search.iterations"] = 1
# Belt-and-suspenders: also override max_iterations for
# strategies that cache it at __init__ time
original_max_iter = getattr(strategy, "max_iterations", None)
strategy.max_iterations = 1
# If the strategy wraps a delegate (e.g.
# EnhancedContextualFollowUpStrategy), the actual research
# happens inside ``delegate_strategy.analyze_topic`` and the
# delegate has its own cached ``max_iterations`` from
# construction — without this extra override the per-subsection
# cap never reaches the worker and each subsection runs the
# full configured loop instead of one iteration.
delegate = getattr(strategy, "delegate_strategy", None)
original_delegate_max_iter = (
getattr(delegate, "max_iterations", None)
if delegate is not None
else None
)
if delegate is not None and hasattr(delegate, "max_iterations"):
delegate.max_iterations = 1
try:
# Perform search for this subsection
subsection_results = self.search_system.analyze_topic(
subsection_query
)
finally:
# Restore original iteration settings
if had_iterations_key:
strategy.settings_snapshot["search.iterations"] = (
original_iterations
)
else:
strategy.settings_snapshot.pop(
"search.iterations", None
)
if original_max_iter is not None:
strategy.max_iterations = original_max_iter
if delegate is not None and hasattr(
delegate, "max_iterations"
):
if original_delegate_max_iter is not None:
delegate.max_iterations = original_delegate_max_iter
else:
delegate.max_iterations = None
completed_subsections += 1
# Add the researched content for this subsection
if subsection_results.get("current_knowledge"):
generated_content = subsection_results["current_knowledge"]
# Strip the boilerplate the LLM tends to emit around
# subsections: a redundant leading heading that mirrors
# the framework's own `## i.j Name` heading, an italic
# purpose statement that mirrors the framework's
# `_<purpose>_` subtitle, and an embedded
# '## Sources' / '## References' bibliography block
# (the framework appends one master '## Sources' to
# the whole report). Small/quantized local models
# routinely ignore the OUTPUT FORMAT RULES in the
# prompt above; this normalisation step guarantees a
# clean rendered output regardless.
sibling_names = [
s["name"]
for s in section["subsections"]
if s["name"] != subsection["name"]
]
try:
generated_content = self._strip_subsection_boilerplate(
generated_content,
subsection_name=subsection["name"],
section_name=section["name"],
sibling_subsection_names=sibling_names,
purpose=subsection.get("purpose"),
)
except Exception:
logger.exception(
"Boilerplate strip failed for "
f"'{section['name']} > {subsection['name']}' — "
"using raw LLM content"
)
# Fall back to raw content so a cosmetic step never
# aborts the entire multi-section report.
generated_content = (
subsection_results.get("current_knowledge", "")
or ""
)
if generated_content.strip():
section_content.append(generated_content)
# Accumulate for context in subsequent sections
accumulated_findings.append(
f"[{section['name']} > {subsection['name']}]\n{generated_content}"
)
else:
section_content.append(
"*Limited information was found for this subsection.*\n"
)
else:
section_content.append(
"*Limited information was found for this subsection.*\n"
)
section_content.append("\n\n")
# Combine all content for this section
sections[section["name"]] = "\n".join(section_content)
return sections
def _generate_sections(
self,
initial_findings: Dict,
_section_research: Dict[str, List[Dict]],
structure: List[Dict],
query: str,
) -> Dict[str, str]:
"""
This method is kept for compatibility but no longer used.
The functionality has been moved to _research_and_generate_sections.
"""
return {}
def _format_final_report(
self,
sections: Dict[str, str],
structure: List[Dict],
query: str,
) -> Dict:
"""Format the final report with table of contents and sections."""
# Generate TOC
toc = ["# Table of Contents\n"]
for i, section in enumerate(structure, 1):
toc.append(f"{i}. **{section['name']}**")
if len(section["subsections"]) > 1:
for j, subsection in enumerate(section["subsections"], 1):
toc.append(
f" {i}.{j} {subsection['name']} | _{subsection['purpose']}_"
)
# Combine TOC and sections
report_parts = ["\n".join(toc), ""]
# Add a summary of the research
report_parts.append("# Research Summary")
report_parts.append(
"This report was researched using an advanced search system."
)
report_parts.append(
"Research included targeted searches for each section and subsection."
)
report_parts.append("\n---\n")
# Add each section's content
for section in structure:
if section["name"] in sections:
report_parts.append(sections[section["name"]])
report_parts.append("")
# Format links from search system
# Get utilities module dynamically to avoid circular imports
utilities = importlib.import_module("local_deep_research.utilities")
# Imported here for the same reason, and directly rather than off
# the module object above so the count is computed by the real
# grouping helper.
from .utilities.search_utilities import (
count_distinct_sources,
filter_cited_links,
resolve_uncited_sources_mode,
)
# Body prose BEFORE the Sources tail: citation indices/URLs are
# parsed from this, so uncited accumulated hits stay out of the
# bibliography. Honors report.uncited_sources_mode.
body_prose = "\n\n".join(report_parts)
mode = resolve_uncited_sources_mode()
visible_links = filter_cited_links(
self.search_system.all_links_of_system,
body_prose,
mode=mode,
)
formatted_all_links = (
utilities.search_utilities.format_links_to_markdown(
all_links=visible_links,
)
)
# Create final report with all parts. The Sources tail is
# kept here so in-memory consumers (MCP `generate_report`,
# programmatic API) get the full assembled blob unchanged.
# The DB save site (research_service.py) strips this Sources
# section via format_document_split before persisting, so the
# answer-only invariant on report_content still holds.
final_report_content = body_prose
# Explicit "\n\n" separator: downstream regex consumers
# (_SOURCES_SECTION_PATTERNS in text_optimization/citation_formatter.py
# and _LEGACY_SOURCES_RE in web/services/report_assembly_service.py)
# use line-anchored `re.MULTILINE` matching. Today the trailing
# newlines produced by `"\n\n".join` happen to keep `## Sources`
# at the start of a line, but that is incidental; an explicit
# separator preserves the invariant against future section
# template changes.
#
# The HTML-comment sentinel around the appended `## Sources`
# block lets ``format_document_split`` locate this section
# unambiguously, even when the LLM has emitted its own
# `## Sources` header earlier in the prose. The legacy regex
# patterns would otherwise match the first `## Sources` they
# find and could over-strip a multi-section report where the
# LLM happened to include an inline sources block. Without the
# sentinel, a 1380-source run legitimately pushes the
# answer/sources ratio below the 50% safety threshold and the
# splitter logs a misleading "over-stripped" warning.
# The sentinel is imported at module top; no circular-import
# concern at use time.
final_report_content += (
f"\n\n{LDR_APPENDED_SOURCES_SENTINEL}\n\n"
f"## Sources\n\n{formatted_all_links}"
)
# Create metadata dictionary
metadata = {
"generated_at": datetime.now(UTC).isoformat(),
# Count the same visible, canonically grouped source set that
# the Sources block above renders. Document-level: one document
# cited via several chunk anchors still counts once even though
# it renders several bibliography lines (distinct sources <=
# bibliography lines <= citation indices) — so a count below
# the rendered line total is expected, not a bug.
"initial_sources": count_distinct_sources(visible_links),
"sections_researched": len(structure),
"searches_per_section": self.searches_per_section,
"query": query,
}
# Return both content and metadata
return {"content": final_report_content, "metadata": metadata}
def _generate_error_report(self, query: str, error_msg: str) -> str:
return f"=== ERROR REPORT ===\nQuery: {query}\nError: {error_msg}"