~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(citation): make format_document_split resilient to LLM-emitted Sources headers (#5283)
S367df4cebfad
diff --git a/changelog.d/5283.bugfix.md b/changelog.d/5283.bugfix.mdnew file mode 100644index 0000000000..527146f254--- /dev/null+++ b/changelog.d/5283.bugfix.md@@ -0,0 +1 @@+Detailed-mode runs no longer log a misleading `format_document_split appears to have over-stripped` warning when the answer body is legitimately smaller than the appended sources list (e.g. runs with hundreds of sources); `IntegratedReportGenerator` now wraps the appended `## Sources` block in a sentinel that the splitter keys off, and `report_content` is correctly persisted as the answer body only. The sentinel is matched at its last occurrence so a spoofed earlier occurrence inside LLM-generated prose cannot silently truncate the answer or disable the over-strip safety check, and the quick-mode save path no longer trusts the sentinel (only the detailed-mode report generator emits it).diff --git a/src/local_deep_research/report_generator.py b/src/local_deep_research/report_generator.pyindex cc42359cb6..299b14a5bf 100644--- a/src/local_deep_research/report_generator.py+++ b/src/local_deep_research/report_generator.py@@ -9,6 +9,9 @@ from loguru import loggerfrom .config.llm_config import get_llmfrom .config.thread_settings import get_setting_from_snapshotfrom .search_system import AdvancedSearchSystem+from .text_optimization.citation_formatter import (+ LDR_APPENDED_SOURCES_SENTINEL,+)from .utilities.json_utils import get_llm_response_text# Default constants for context accumulation to avoid repetition@@ -599,7 +602,23 @@ class IntegratedReportGenerator:# at the start of a line, but that is incidental; an explicit# separator preserves the invariant against future section# template changes.- final_report_content += "\n\n## Sources\n\n" + formatted_all_links+ #+ # The HTML-comment sentinel around the appended `## Sources`+ # block lets ``format_document_split`` locate this section+ # unambiguously, even when the LLM has emitted its own+ # `## Sources` header earlier in the prose. The legacy regex+ # patterns would otherwise match the first `## Sources` they+ # find and could over-strip a multi-section report where the+ # LLM happened to include an inline sources block. Without the+ # sentinel, a 1380-source run legitimately pushes the+ # answer/sources ratio below the 50% safety threshold and the+ # splitter logs a misleading "over-stripped" warning.+ # The sentinel is imported at module top; no circular-import+ # concern at use time.+ final_report_content += (+ f"\n\n{LDR_APPENDED_SOURCES_SENTINEL}\n\n"+ f"## Sources\n\n{formatted_all_links}"+ )# Create metadata dictionarymetadata = {diff --git a/src/local_deep_research/text_optimization/citation_formatter.py b/src/local_deep_research/text_optimization/citation_formatter.pyindex e2af5237b9..0ce57f12c3 100644--- a/src/local_deep_research/text_optimization/citation_formatter.py+++ b/src/local_deep_research/text_optimization/citation_formatter.py@@ -10,6 +10,15 @@ from slugify import slugifyfrom ..content_fetcher.url_classifier import URLClassifier, URLType+# Marker emitted by ``IntegratedReportGenerator._format_final_report``+# around the ``## Sources`` block it appends to the end of a detailed-mode+# report. Looked up by ``find_sources_section`` so the splitter can locate+# the appended sources section unambiguously, even when the LLM has+# emitted its own ``## Sources`` (or ``References:`` etc.) header earlier+# in the prose. The sentinel is a markdown HTML comment so it round-trips+# through markdown renderers without affecting the displayed output.+LDR_APPENDED_SOURCES_SENTINEL = "<!-- LDR_APPENDED_SOURCES_START -->"+_SOURCES_SECTION_PATTERNS = [re.compile(r"^#{1,3}\s*(?:Sources|References|Bibliography|Citations)",@@ -22,16 +31,52 @@ _SOURCES_SECTION_PATTERNS = []-def find_sources_section(content: str) -> int:- """Find the start position of the sources/references section in *content*.-- Returns -1 if no section is found.+def find_sources_section(+ content: str, *, trust_sentinel: bool = True+) -> Tuple[int, bool]:+ """Locate the sources/references section in *content*.++ Returns ``(position, on_sentinel)``. ``position`` is the character+ offset of the section start, or ``-1`` if no section is found.+ ``on_sentinel`` is ``True`` iff the unique sentinel emitted by the+ report generator was matched (in which case the boundary is correct+ by construction); ``False`` if the position came from the legacy+ regex patterns (which can match an early ``## Sources`` line inside+ the answer body and need an over-strip safety check downstream).++ The sentinel is matched at its **last** occurrence (``rfind``) so a+ spoofed earlier occurrence inside LLM-generated prose — e.g. the+ LLM quoting the marker string verbatim while summarizing the+ codebase — cannot silently truncate the answer body or disable the+ over-strip safety check. The report generator only ever emits the+ sentinel once at the end of the assembled report, so the last+ occurrence is the trusted one.++ When ``trust_sentinel`` is ``False`` (e.g. raw LLM output from the+ quick-mode save path, which never adds the sentinel itself), the+ sentinel branch is skipped — only the legacy regex patterns run, so+ a spurious ``on_sentinel=True`` cannot arise from the LLM quoting+ the marker."""+ if trust_sentinel:+ sentinel_pos = content.rfind(LDR_APPENDED_SOURCES_SENTINEL)+ while sentinel_pos != -1:+ preceded = sentinel_pos == 0 or content[sentinel_pos - 1] in (+ "\n",+ "\r",+ )+ after = content[sentinel_pos + len(LDR_APPENDED_SOURCES_SENTINEL) :]+ followed = not after or after[0] in ("\n", "\r")+ if preceded and followed:+ return sentinel_pos, True+ sentinel_pos = content.rfind(+ LDR_APPENDED_SOURCES_SENTINEL, 0, sentinel_pos+ )for pattern in _SOURCES_SECTION_PATTERNS:match = pattern.search(content)if match:- return match.start()- return -1+ return match.start(), False+ return -1, Falseclass CitationMode(Enum):@@ -133,33 +178,50 @@ class CitationFormatter:return self.comma_citation_pattern.sub(_replacer, content)def format_document(self, content: str) -> str:- """Format citations and return the concatenated answer + sources blob.+ """Format citations in *content* and return the full document.- Kept for backward compatibility — most call sites only need the- concatenated string. New code that needs to persist answer-only+ This is the legacy single-string entry point: it concatenates+ the answer half and the sources half produced by+ :meth:`format_document_split`. New code that needs to know+ where the boundary between answer and sources actually fellshould use :meth:`format_document_split` instead so the boundaryis returned explicitly (no re-parsing of the concatenated output)."""- formatted_answer, sources_md = self.format_document_split(content)+ formatted_answer, sources_md, _on_sentinel = self.format_document_split(+ content+ )return formatted_answer + sources_md- def format_document_split(self, content: str) -> Tuple[str, str]:- """Format citations and return (answer, sources_md) separately.+ def format_document_split(+ self, content: str, *, trust_sentinel: bool = True+ ) -> Tuple[str, str, bool]:+ """Format citations and return ``(answer, sources_md, on_sentinel)``.The boundary between the LLM's answer and the trailing Sourcessection is computed inside this method. Callers that only wantthe answer (e.g. the chat-mode save site) get a clean splitwithout re-applying a regex on concatenated output downstream.-- Returns ``(content, "")`` when the formatter is in NO_HYPERLINKS- mode or when no Sources section can be found in ``content``.+ ``on_sentinel`` mirrors the second element of+ :func:`find_sources_section`'s return — ``True`` iff the split+ boundary came from the appended-sources sentinel emitted by+ ``IntegratedReportGenerator``, in which case the boundary is+ correct by construction and downstream over-strip safety checks+ should be skipped. Pass ``trust_sentinel=False`` for raw LLM+ output (quick-mode save path) where the sentinel would only+ appear if the LLM quoted the marker itself.++ Returns ``(content, "", False)`` when the formatter is in+ NO_HYPERLINKS mode or when no Sources section can be found in+ ``content``."""if self.mode == CitationMode.NO_HYPERLINKS:- return content, ""+ return content, "", False- sources_start = self._find_sources_section(content)+ sources_start, on_sentinel = find_sources_section(+ content, trust_sentinel=trust_sentinel+ )if sources_start == -1:- return content, ""+ return content, "", Falsedocument_content = content[:sources_start]sources_content = content[sources_start:]@@ -191,7 +253,7 @@ class CitationFormatter:else:formatted_content = document_content- return formatted_content, sources_content+ return formatted_content, sources_content, on_sentineldef apply_inline_hyperlinks(self, content: str, sources: List[Dict[str, Any]]@@ -270,9 +332,17 @@ class CitationFormatter:# mode added later that doesn't have an explicit branch above.return self._format_number_hyperlinks(content, adapted)- def _find_sources_section(self, content: str) -> int:- """Find the start of the sources/references section."""- return find_sources_section(content)+ def _find_sources_section(+ self, content: str, trust_sentinel: bool = True+ ) -> Tuple[int, bool]:+ """Find the start of the sources/references section.++ Returns ``(position, on_sentinel)`` — see+ :func:`find_sources_section` for the semantics. ``on_sentinel``+ is True iff the boundary came from the unique sentinel emitted+ by the report generator (i.e. it is correct by construction).+ """+ return find_sources_section(content, trust_sentinel=trust_sentinel)def _parse_sources(self, sources_content: str@@ -886,7 +956,7 @@ class RISExporter:RIS formatted references"""# Find sources section- sources_start = find_sources_section(content)+ sources_start, _on_sentinel = find_sources_section(content)if sources_start == -1:return ""@@ -1276,7 +1346,7 @@ class LaTeXExporter:def _create_bibliography(self, content: str) -> str:"""Extract sources and create LaTeX bibliography."""- sources_start = find_sources_section(content)+ sources_start, _on_sentinel = find_sources_section(content)if sources_start == -1:return ""diff --git a/src/local_deep_research/utilities/search_utilities.py b/src/local_deep_research/utilities/search_utilities.pyindex ac94218a04..fac22d182c 100644--- a/src/local_deep_research/utilities/search_utilities.py+++ b/src/local_deep_research/utilities/search_utilities.py@@ -3,6 +3,9 @@ from typing import Dict, Listfrom loguru import logger+from local_deep_research.text_optimization.citation_formatter import (+ LDR_APPENDED_SOURCES_SENTINEL,+)from .url_utils import canonical_url_key@@ -258,6 +261,8 @@ def format_links_to_markdown(all_links: List[Dict]) -> str:if not canon or canon in seen:continuetitle = canon_to_title[canon]+ if title:+ title = title.replace(LDR_APPENDED_SOURCES_SENTINEL, "")# Indices arrive as int (from strategy enumeration) or str (from# _build_sources_markdown's fallback). Coerce so dedup collapses# 1 and "1", and sorted() doesn't TypeError on mixed types.diff --git a/src/local_deep_research/web/services/research_service.py b/src/local_deep_research/web/services/research_service.pyindex 51e40363d6..1f2330664a 100644--- a/src/local_deep_research/web/services/research_service.py+++ b/src/local_deep_research/web/services/research_service.py@@ -1883,8 +1883,17 @@ def run_research_process(research_id, query, mode, **kwargs):# concatenated formatter output downstream.formatter = get_citation_formatter()try:- answer_with_links, llm_sources = (- formatter.format_document_split(clean_markdown)+ # Quick-mode (chat summary) input is raw LLM+ # output; the report generator never wraps it+ # in the appended-sources sentinel, so disable+ # sentinel trust. Otherwise an LLM that quotes+ # the marker verbatim would silently trip the+ # over-strip safety bypass on a path where the+ # bypass can only ever fire as a false-positive.+ answer_with_links, llm_sources, on_sentinel = (+ formatter.format_document_split(+ clean_markdown, trust_sentinel=False+ ))if not llm_sources:answer_with_links = (@@ -1898,9 +1907,19 @@ def run_research_process(research_id, query, mode, **kwargs):# Fall back to structured-source hyperlinking# on the full text. Min-length floor prevents# false-fires on legitimately short answers.+ # Skip the check when the boundary came from+ # the unique appended-sources sentinel emitted+ # by IntegratedReportGenerator — that boundary+ # is correct by construction (e.g. a 1380-source+ # detailed-mode run legitimately pushes the+ # answer/sources ratio below 50% without any+ # over-strip having occurred). Quick-mode+ # always passes trust_sentinel=False above, so+ # ``on_sentinel`` can never be True here.SAFETY_MIN_LEN = 800if (- llm_sources+ not on_sentinel+ and llm_sourcesand len(clean_markdown) > SAFETY_MIN_LENand len(answer_with_links)< len(clean_markdown) * 0.5@@ -1925,7 +1944,8 @@ def run_research_process(research_id, query, mode, **kwargs):# requirement. If anything blows up, save the# raw LLM text rather than fail the research.logger.exception(- "Citation formatter failed; saving raw answer"+ "Citation formatter failed for research {}; saving raw answer",+ research_id,)answer_with_links = clean_markdown@@ -2204,16 +2224,29 @@ def run_research_process(research_id, query, mode, **kwargs):)formatter = get_citation_formatter()try:- answer_with_links, llm_sources = (+ # Detailed-mode input comes from+ # ``IntegratedReportGenerator._format_final_report``,+ # which wraps the appended ``## Sources`` block in the+ # unique sentinel. Trust the sentinel so a legitimate+ # large-tail run (e.g. 1380 sources) doesn't trip the+ # over-strip safety check on the right answer half.+ answer_with_links, llm_sources, on_sentinel = (formatter.format_document_split(final_report["content"]))if not llm_sources:answer_with_links = formatter.apply_inline_hyperlinks(final_report["content"], all_links)+ # Skip the over-strip safety check when the boundary+ # came from the unique appended-sources sentinel emitted+ # by IntegratedReportGenerator — that boundary is+ # correct by construction (e.g. a 1380-source detailed+ # run legitimately pushes the answer/sources ratio+ # below 50% without any over-strip having occurred).SAFETY_MIN_LEN = 800if (- llm_sources+ not on_sentinel+ and llm_sourcesand len(final_report["content"]) > SAFETY_MIN_LENand len(answer_with_links)< len(final_report["content"]) * 0.5@@ -2229,7 +2262,10 @@ def run_research_process(research_id, query, mode, **kwargs):final_report["content"], all_links)except Exception:- logger.exception("Citation formatter failed; saving raw answer")+ logger.exception(+ "Citation formatter failed for research {}; saving raw answer",+ research_id,+ )answer_with_links = final_report["content"]formatted_content = answer_with_linksdiff --git a/tests/text_optimization/test_citation_formatter.py b/tests/text_optimization/test_citation_formatter.pyindex 4bd3e70fa0..e5c2696159 100644--- a/tests/text_optimization/test_citation_formatter.py+++ b/tests/text_optimization/test_citation_formatter.py@@ -1557,7 +1557,7 @@ Some content here.[1] First source"""- result = find_sources_section(content)+ result, _on_sentinel = find_sources_section(content)assert result == content.index("## Sources")def test_find_h2_references_header(self):@@ -1574,7 +1574,7 @@ Some content here.[1] First reference"""- result = find_sources_section(content)+ result, _on_sentinel = find_sources_section(content)assert result == content.index("## References")def test_find_h2_bibliography_header(self):@@ -1591,7 +1591,7 @@ Some content here.[1] First entry"""- result = find_sources_section(content)+ result, _on_sentinel = find_sources_section(content)assert result == content.index("## Bibliography")def test_find_h2_citations_header(self):@@ -1608,7 +1608,7 @@ Some content here.[1] First citation"""- result = find_sources_section(content)+ result, _on_sentinel = find_sources_section(content)assert result == content.index("## Citations")def test_find_h3_sources_header(self):@@ -1625,7 +1625,7 @@ Some content here.[1] First source"""- result = find_sources_section(content)+ result, _on_sentinel = find_sources_section(content)assert result == content.index("### Sources")def test_find_h1_sources_header(self):@@ -1642,7 +1642,7 @@ Some content.[1] First source"""- result = find_sources_section(content)+ result, _on_sentinel = find_sources_section(content)assert result == content.index("# Sources")def test_case_insensitivity(self):@@ -1663,9 +1663,9 @@ Some content.[1] First"""- assert find_sources_section(content_lower) == 0- assert find_sources_section(content_upper) == 0- assert find_sources_section(content_mixed) == 0+ assert find_sources_section(content_lower)[0] == 0+ assert find_sources_section(content_upper)[0] == 0+ assert find_sources_section(content_mixed)[0] == 0def test_no_sources_section_returns_negative_one(self):"""Test that -1 is returned when no sources section exists."""@@ -1681,7 +1681,7 @@ Some content with [1] citation but no sources section.The end."""- result = find_sources_section(content)+ result, _on_sentinel = find_sources_section(content)assert result == -1def test_empty_content_returns_negative_one(self):@@ -1690,7 +1690,7 @@ The end.find_sources_section,)- assert find_sources_section("") == -1+ assert find_sources_section("") == (-1, False)def test_plain_sources_colon_format(self):"""Test finding 'Sources:' format without markdown headers."""@@ -1703,7 +1703,7 @@ The end.Sources:[1] First source"""- result = find_sources_section(content)+ result, _on_sentinel = find_sources_section(content)assert result == content.index("Sources:")def test_references_without_colon(self):@@ -1717,7 +1717,7 @@ Sources:References[1] First reference"""- result = find_sources_section(content)+ result, _on_sentinel = find_sources_section(content)assert result == content.index("References")diff --git a/tests/text_optimization/test_citation_formatter_behavior.py b/tests/text_optimization/test_citation_formatter_behavior.pyindex 78cd579ee2..5b0a918ad6 100644--- a/tests/text_optimization/test_citation_formatter_behavior.py+++ b/tests/text_optimization/test_citation_formatter_behavior.py@@ -185,7 +185,15 @@ class TestCitationFormatterExtractDomain:class TestCitationFormatterFindSourcesSection:- """Tests for _find_sources_section method."""+ """Tests for _find_sources_section method.++ The method returns ``(position, on_sentinel)``:+ - ``position`` is the character offset of the section start, or ``-1``+ if no section is found.+ - ``on_sentinel`` is True iff the unique appended-sources sentinel+ emitted by ``IntegratedReportGenerator`` was matched (in which+ case the boundary is correct by construction).+ """def test_finds_sources_header(self):"""Finds ## Sources header."""@@ -195,9 +203,10 @@ class TestCitationFormatterFindSourcesSection:formatter = CitationFormatter()content = "Some content\n\n## Sources\n\n[1] Source one"- result = formatter._find_sources_section(content)- assert result != -1- assert content[result:].startswith("## Sources")+ position, on_sentinel = formatter._find_sources_section(content)+ assert position != -1+ assert on_sentinel is False+ assert content[position:].startswith("## Sources")def test_finds_references_header(self):"""Finds ## References header."""@@ -207,8 +216,8 @@ class TestCitationFormatterFindSourcesSection:formatter = CitationFormatter()content = "Some content\n\n## References\n\n[1] Ref one"- result = formatter._find_sources_section(content)- assert result != -1+ position, _on_sentinel = formatter._find_sources_section(content)+ assert position != -1def test_finds_bibliography_header(self):"""Finds ## Bibliography header."""@@ -218,8 +227,8 @@ class TestCitationFormatterFindSourcesSection:formatter = CitationFormatter()content = "Some content\n\n## Bibliography\n\n[1] Bib one"- result = formatter._find_sources_section(content)- assert result != -1+ position, _on_sentinel = formatter._find_sources_section(content)+ assert position != -1def test_finds_citations_header(self):"""Finds ## Citations header."""@@ -229,8 +238,8 @@ class TestCitationFormatterFindSourcesSection:formatter = CitationFormatter()content = "Some content\n\n## Citations\n\n[1] Citation one"- result = formatter._find_sources_section(content)- assert result != -1+ position, _on_sentinel = formatter._find_sources_section(content)+ assert position != -1def test_case_insensitive(self):"""Finds sources header case-insensitively."""@@ -240,19 +249,20 @@ class TestCitationFormatterFindSourcesSection:formatter = CitationFormatter()content = "Some content\n\n## SOURCES\n\n[1] Source one"- result = formatter._find_sources_section(content)- assert result != -1+ position, _on_sentinel = formatter._find_sources_section(content)+ assert position != -1def test_returns_minus_one_when_not_found(self):- """Returns -1 when no sources section found."""+ """Returns (-1, False) when no sources section found."""from local_deep_research.text_optimization.citation_formatter import (CitationFormatter,)formatter = CitationFormatter()content = "Some content without sources section."- result = formatter._find_sources_section(content)- assert result == -1+ position, on_sentinel = formatter._find_sources_section(content)+ assert position == -1+ assert on_sentinel is Falsedef test_finds_single_hash_header(self):"""Finds # Sources header."""@@ -262,8 +272,8 @@ class TestCitationFormatterFindSourcesSection:formatter = CitationFormatter()content = "Some content\n\n# Sources\n\n[1] Source one"- result = formatter._find_sources_section(content)- assert result != -1+ position, _on_sentinel = formatter._find_sources_section(content)+ assert position != -1def test_finds_triple_hash_header(self):"""Finds ### Sources header."""@@ -273,8 +283,89 @@ class TestCitationFormatterFindSourcesSection:formatter = CitationFormatter()content = "Some content\n\n### Sources\n\n[1] Source one"- result = formatter._find_sources_section(content)- assert result != -1+ position, _on_sentinel = formatter._find_sources_section(content)+ assert position != -1++ def test_sentinel_takes_precedence_over_legacy_patterns(self):+ """Sentinel matches before the legacy regex patterns do.++ Regression for the over-strip false-positive that fired on+ 1380-source detailed-mode runs: when the report generator+ wraps its appended ``## Sources`` block in the sentinel,+ the splitter must use the sentinel position (correct by+ construction) and not the first legacy ``## Sources`` that+ the LLM may have emitted earlier in the section content.+ """+ from local_deep_research.text_optimization.citation_formatter import (+ CitationFormatter,+ LDR_APPENDED_SOURCES_SENTINEL,+ )++ formatter = CitationFormatter()+ # Earlier LLM-emitted ## Sources inside the section, then the+ # sentinel, then the appended sources block.+ content = (+ "Section body\n\n"+ "## Sources\n\n"+ "Inline LLM citations [1] http://inline.example\n\n"+ f"{LDR_APPENDED_SOURCES_SENTINEL}\n\n"+ "## Sources\n\n"+ "[1] Appended source\n URL: http://appended.example"+ )+ position, on_sentinel = formatter._find_sources_section(content)+ assert on_sentinel is True+ assert content[position:].startswith(LDR_APPENDED_SOURCES_SENTINEL)+ # Position must point at the sentinel, not the earlier+ # LLM-emitted header.+ assert position > content.index("## Sources\n\nInline LLM")++ def test_sentinel_recognized_even_without_trailing_section_header(self):+ """Sentinel alone is enough; trailing ## Sources isn't required.++ Defensive: the report generator always emits both, but the+ splitter must key off the sentinel alone so a future change to+ the appended header style (e.g. switching to ``## Bibliography``)+ cannot silently re-introduce the false-positive.+ """+ from local_deep_research.text_optimization.citation_formatter import (+ CitationFormatter,+ LDR_APPENDED_SOURCES_SENTINEL,+ )++ formatter = CitationFormatter()+ content = (+ "Section body.\n\n"+ f"{LDR_APPENDED_SOURCES_SENTINEL}\n\n"+ "Some appended content."+ )+ position, on_sentinel = formatter._find_sources_section(content)+ assert position != -1+ assert on_sentinel is True++ def test_spoofed_later_sentinel_in_sources_tail_ignored(self):+ """A spoofed sentinel inside a source title in the tail is ignored.++ If an indexed source title contains the sentinel string embedded inside+ a line, find_sources_section must skip it and match the real+ standalone sentinel line before the ## Sources header.+ """+ from local_deep_research.text_optimization.citation_formatter import (+ CitationFormatter,+ LDR_APPENDED_SOURCES_SENTINEL,+ )++ formatter = CitationFormatter()+ real_sentinel_pos = len("Section body.\n\n")+ content = (+ "Section body.\n\n"+ f"{LDR_APPENDED_SOURCES_SENTINEL}\n\n"+ "## Sources\n\n"+ f"[1] Title containing {LDR_APPENDED_SOURCES_SENTINEL} inline (source nr: 1)\n"+ " URL: http://example.com"+ )+ position, on_sentinel = formatter._find_sources_section(content)+ assert position == real_sentinel_pos+ assert on_sentinel is Trueclass TestCitationFormatterParseSources:@@ -1197,13 +1288,16 @@ class TestCitationFormatterFormatDocumentSplit:"See [1] for details.\n\n""## Sources\n\n[1] Test Source\n URL: https://test.com")- answer, sources = formatter.format_document_split(content)+ answer, sources, on_sentinel = formatter.format_document_split(content)assert "[[1]](https://test.com)" in answerassert sources.startswith("## Sources")assert "Test Source" in sources+ # No sentinel present in this input — boundary came from the+ # legacy `## Sources` regex.+ assert on_sentinel is Falsedef test_returns_empty_sources_when_no_sources_section(self):- """No Sources section in input → returns (content, '')."""+ """No Sources section in input → returns (content, '', False)."""from local_deep_research.text_optimization.citation_formatter import (CitationFormatter,CitationMode,@@ -1211,12 +1305,13 @@ class TestCitationFormatterFormatDocumentSplit:formatter = CitationFormatter(mode=CitationMode.NUMBER_HYPERLINKS)content = "Just an answer with [1] but no Sources section."- answer, sources = formatter.format_document_split(content)+ answer, sources, on_sentinel = formatter.format_document_split(content)assert answer == contentassert sources == ""+ assert on_sentinel is Falsedef test_returns_unchanged_in_no_hyperlinks_mode(self):- """NO_HYPERLINKS mode short-circuits to (content, '')."""+ """NO_HYPERLINKS mode short-circuits to (content, '', False)."""from local_deep_research.text_optimization.citation_formatter import (CitationFormatter,CitationMode,@@ -1226,9 +1321,10 @@ class TestCitationFormatterFormatDocumentSplit:content = ("Text with [1].\n\n## Sources\n\n[1] Source\n URL: https://x.com")- answer, sources = formatter.format_document_split(content)+ answer, sources, on_sentinel = formatter.format_document_split(content)assert answer == contentassert sources == ""+ assert on_sentinel is Falsedef test_format_document_compat_equals_concatenation(self):"""format_document() == answer + sources from format_document_split."""@@ -1243,9 +1339,126 @@ class TestCitationFormatterFormatDocumentSplit:"## Sources\n\n[1] One\n URL: https://one.com")full = formatter.format_document(content)- answer, sources = formatter.format_document_split(content)+ answer, sources, _on_sentinel = formatter.format_document_split(content)assert full == answer + sources+ def test_sentinel_boundary_wins_over_inline_sources_header(self):+ """Sentinel positions split at the sentinel, not earlier ## Sources.++ Regression for triage item #4: a 1380-source detailed-mode+ run logged ``format_document_split appears to have over-stripped+ (answer=84716 chars, original=196122 chars)`` because the+ legacy regex picked up an LLM-emitted ``## Sources`` header+ inside the section body, cutting the answer ~57% shorter than+ the input. With the sentinel wrapped around the appended+ sources block by ``IntegratedReportGenerator``, the splitter+ keys off the sentinel and returns the full answer body+ instead of the truncated prefix.+ """+ from local_deep_research.text_optimization.citation_formatter import (+ CitationFormatter,+ CitationMode,+ LDR_APPENDED_SOURCES_SENTINEL,+ )++ formatter = CitationFormatter(mode=CitationMode.NUMBER_HYPERLINKS)+ # Section body contains an inline LLM-emitted ## Sources block,+ # then the sentinel, then the appended structured sources list.+ inline_sources_block = (+ "## Sources\n\n"+ "[1] Inline LLM cite\n URL: https://inline.example\n"+ )+ appended_sources_block = (+ "## Sources\n\n[1] Appended\n URL: https://appended.example\n"+ )+ content = (+ "Long answer body that the LLM wrote. " * 100+ + f"\n\n{inline_sources_block}"+ + f"\n\n{LDR_APPENDED_SOURCES_SENTINEL}\n\n"+ + appended_sources_block+ )+ answer, sources, on_sentinel = formatter.format_document_split(content)++ # The answer must include the full LLM prose (the entire+ # "Long answer body..." preamble) — not stop at the inline+ # ## Sources that the LLM emitted.+ assert answer.count("Long answer body") == 100+ # The sources tail must begin at the sentinel.+ assert sources.startswith(LDR_APPENDED_SOURCES_SENTINEL)+ assert "appended.example" in sources+ # Boundary came from the sentinel — caller can skip the+ # over-strip safety check on this result.+ assert on_sentinel is True++ def test_spoofed_earlier_sentinel_ignored(self):+ """An LLM-quoted sentinel earlier in the prose is ignored.++ The sentinel is matched at its **last** occurrence (``rfind``),+ so an LLM that quotes the marker verbatim earlier in the answer+ body — e.g. while summarizing the LDR codebase — cannot push+ the split boundary into the middle of the answer. The trusted+ sentinel is the one ``_format_final_report`` actually appended+ at the end.+ """+ from local_deep_research.text_optimization.citation_formatter import (+ CitationFormatter,+ CitationMode,+ LDR_APPENDED_SOURCES_SENTINEL,+ )++ formatter = CitationFormatter(mode=CitationMode.NUMBER_HYPERLINKS)+ # LLM quoted the marker verbatim in its prose, then the+ # genuine appended sentinel + Sources block at the end.+ content = (+ "Answer body that quotes "+ f"{LDR_APPENDED_SOURCES_SENTINEL} "+ "as a string literal.\n\n"+ f"{LDR_APPENDED_SOURCES_SENTINEL}\n\n"+ "## Sources\n\n[1] Real\n URL: https://real.example"+ )+ answer, sources, on_sentinel = formatter.format_document_split(content)++ # The split must land on the LAST sentinel — the one the+ # report generator actually appended.+ assert sources.startswith(LDR_APPENDED_SOURCES_SENTINEL)+ # And the position must be after the LLM's quoted sentinel,+ # not at it. We verify by checking that the answer contains+ # the LLM's quoted sentinel verbatim.+ assert LDR_APPENDED_SOURCES_SENTINEL in answer+ assert on_sentinel is True++ def test_trust_sentinel_false_returns_on_sentinel_false(self):+ """trust_sentinel=False disables the sentinel branch entirely.++ The quick-mode save path passes trust_sentinel=False because+ the report generator never emits the sentinel on that path.+ If the LLM happens to quote the marker verbatim, the result+ must be on_sentinel=False so the over-strip safety check still+ runs and a spoofing split cannot silently truncate the answer.+ """+ from local_deep_research.text_optimization.citation_formatter import (+ CitationFormatter,+ CitationMode,+ LDR_APPENDED_SOURCES_SENTINEL,+ )++ formatter = CitationFormatter(mode=CitationMode.NUMBER_HYPERLINKS)+ content = (+ "Answer body.\n\n"+ f"{LDR_APPENDED_SOURCES_SENTINEL}\n\n"+ "## Sources\n\n[1] Real\n URL: https://real.example"+ )+ answer, sources, on_sentinel = formatter.format_document_split(+ content, trust_sentinel=False+ )+ # The sentinel was present but caller doesn't trust it, so the+ # splitter falls back to the legacy regex on `## Sources`.+ assert on_sentinel is False+ # The split still finds the boundary (via the legacy regex),+ # so the answer half doesn't include the appended sources.+ assert "Real" not in answer+ assert "real.example" not in answer+class TestCitationFormatterApplyInlineHyperlinks:"""Tests for apply_inline_hyperlinks — structured-source delegation."""diff --git a/tests/text_optimization/test_citation_formatter_high_value.py b/tests/text_optimization/test_citation_formatter_high_value.pyindex 1ce52724df..e27a258d00 100644--- a/tests/text_optimization/test_citation_formatter_high_value.py+++ b/tests/text_optimization/test_citation_formatter_high_value.py@@ -46,29 +46,29 @@ class TestCitationMode:class TestFindSourcesSection:def test_markdown_heading_sources(self):text = "Some text\n## Sources\n[1] Foo"- assert find_sources_section(text) == text.index("## Sources")+ assert find_sources_section(text)[0] == text.index("## Sources")def test_markdown_heading_references(self):text = "Body\n# References\n[1] Bar"- assert find_sources_section(text) == text.index("# References")+ assert find_sources_section(text)[0] == text.index("# References")def test_plain_label_sources(self):text = "Body paragraph\nSources:\n[1] Item"- assert find_sources_section(text) == text.index("Sources:")+ assert find_sources_section(text)[0] == text.index("Sources:")def test_case_insensitive(self):text = "Body\n## BIBLIOGRAPHY\n[1] X"- assert find_sources_section(text) != -1+ assert find_sources_section(text)[0] != -1def test_no_section_returns_negative_one(self):- assert find_sources_section("No references here.") == -1+ assert find_sources_section("No references here.") == (-1, False)def test_empty_string(self):- assert find_sources_section("") == -1+ assert find_sources_section("") == (-1, False)def test_citations_heading_detected(self):text = "Content\n### Citations\nSome refs"- assert find_sources_section(text) == text.index("### Citations")+ assert find_sources_section(text)[0] == text.index("### Citations")# ---------------------------------------------------------------------------diff --git a/tests/utilities/test_search_utilities_formatting.py b/tests/utilities/test_search_utilities_formatting.pyindex bf9bd91c4e..4a68de693c 100644--- a/tests/utilities/test_search_utilities_formatting.py+++ b/tests/utilities/test_search_utilities_formatting.py@@ -811,3 +811,23 @@ class TestFormatLinksToMarkdownCollections:# Single entry (URLs deduped), carries the first collection.assert "Collection: first" in resultassert "Collection: second" not in result++ def test_sentinel_stripped_from_title(self):+ """LDR_APPENDED_SOURCES_SENTINEL in link title is stripped."""+ from local_deep_research.text_optimization.citation_formatter import (+ LDR_APPENDED_SOURCES_SENTINEL,+ )+ from local_deep_research.utilities.search_utilities import (+ format_links_to_markdown,+ )++ links = [+ {+ "title": f"Title with {LDR_APPENDED_SOURCES_SENTINEL} sentinel",+ "url": "https://example.com",+ "index": "1",+ }+ ]+ result = format_links_to_markdown(links)+ assert LDR_APPENDED_SOURCES_SENTINEL not in result+ assert "Title with sentinel" in resultdiff --git a/tests/web/services/test_research_service_coverage.py b/tests/web/services/test_research_service_coverage.pyindex c794ffd219..3c81461b23 100644--- a/tests/web/services/test_research_service_coverage.py+++ b/tests/web/services/test_research_service_coverage.py@@ -1103,9 +1103,11 @@ class TestRunResearchProcessDetailedMode:# skipped and the over-strip safety check is not triggered for this# short content. (Without this stub the test passes only via the# formatter's exception fallback, which is fragile.)+ # format_document_split now returns (answer, sources, on_sentinel).mock_formatter.format_document_split.return_value = ("# Full Report",[{"url": "u"}],+ False,)mock_report_gen = MagicMock()@@ -1230,9 +1232,11 @@ class TestRunResearchProcessDetailedMode:# Exercise the normal citation path (see test_detailed_mode_success)# rather than the formatter's exception fallback.mock_formatter = MagicMock()+ # format_document_split now returns (answer, sources, on_sentinel).mock_formatter.format_document_split.return_value = ("# Full Report",[{"url": "u"}],+ False,)mock_file_storage = MagicMock()@@ -1403,6 +1407,312 @@ class TestRunResearchProcessDetailedMode:# Error handler should queue error updatemock_qp.queue_error_update.assert_called()+ def test_detailed_mode_sentinel_skips_overstrip_warning(+ self, loguru_caplog+ ):+ """The over-strip safety check is bypassed when final_report content+ contains the appended-sources sentinel emitted by+ IntegratedReportGenerator.++ Regression for triage item #4: a 1380-source detailed-mode run+ logged a misleading ``format_document_split appears to have+ over-stripped (answer=84716 chars, original=196122 chars)``+ warning even though the splitter had correctly identified the+ trailing sources section. The warning fired because the+ answer/sources ratio (43%) was below the 50% safety threshold.+ With the sentinel, the splitter knows the boundary is correct+ by construction and the check no longer fires.+ """+ from local_deep_research.text_optimization.citation_formatter import (+ LDR_APPENDED_SOURCES_SENTINEL,+ )++ mock_session = MagicMock()+ research = _make_research_mock()+ mock_session.query.return_value.filter_by.return_value.first.return_value = research++ mock_system = MagicMock()+ mock_search_system = MagicMock()+ mock_search_system.all_links_of_system = [+ {"url": "http://a.com", "title": "A"}+ ]+ results = {+ "findings": [{"content": "data"}],+ "formatted_findings": "# Report",+ "iterations": 5,+ "search_system": mock_search_system,+ }+ mock_system.analyze_topic.return_value = results++ # Build a content blob that triggers the over-strip safety+ # check by ratio: a long input with the answer portion well+ # under 50% of the total. Without the sentinel, the safety+ # check would fire and log the misleading warning. With the+ # sentinel, the check is bypassed.+ answer = "A" * 900 # > SAFETY_MIN_LEN (800) so the check would fire+ # Pad the sources tail to push the answer below 50% of the total.+ # 2900 chars of sources = answer ratio ~900 / (900 + 2900) = 24%.+ sources_tail = "[1] S\n URL: https://s.example\n" * 100+ content = (+ answer+ + f"\n\n{LDR_APPENDED_SOURCES_SENTINEL}\n\n"+ + "## Sources\n\n"+ + sources_tail+ )+ assert len(answer) < len(content) * 0.5++ mock_report_gen = MagicMock()+ mock_report_gen.generate_report.return_value = {+ "content": content,+ "metadata": {"sections": 3},+ }++ # The formatter's format_document_split must return what the+ # real split produces: everything before the sentinel is the+ # answer (84716 chars), everything from the sentinel onward is+ # the sources tail.+ mock_formatter = MagicMock()+ # format_document_split now returns (answer, sources, on_sentinel).+ # The mock returns on_sentinel=True to mimic the detailed-mode+ # path where the report generator emitted the sentinel.+ mock_formatter.format_document_split.return_value = (+ answer,+ content[len(answer) :],+ True,+ )++ mock_storage = MagicMock()+ mock_storage.save_report.return_value = True++ mock_sources_service = MagicMock()+ mock_sources_service.save_research_sources.return_value = 1+ mock_qp = MagicMock()++ with loguru_caplog.at_level("WARNING"):+ with (+ patch(+ "local_deep_research.web.routes.globals.is_termination_requested",+ return_value=False,+ ),+ patch(+ "local_deep_research.web.routes.globals.is_research_active",+ return_value=True,+ ),+ patch(+ "local_deep_research.web.routes.globals.update_progress_and_check_active",+ return_value=(50, True),+ ),+ patch(f"{RS}.get_llm", return_value=MagicMock()),+ patch(f"{RS}.get_search", return_value=MagicMock()),+ patch(f"{RS}.AdvancedSearchSystem", return_value=mock_system),+ patch(+ f"{RS}.get_citation_formatter", return_value=mock_formatter+ ),+ patch(+ f"{RS}.get_user_db_session",+ side_effect=_fake_session_ctx(mock_session),+ ),+ patch(f"{RS}.cleanup_research_resources"),+ patch(f"{RS}.SocketIOService"),+ patch(f"{RS}.set_search_context"),+ patch(f"{RS}.calculate_duration", return_value=20.0),+ patch(+ f"{RS}.IntegratedReportGenerator",+ return_value=mock_report_gen,+ ),+ patch(+ "local_deep_research.storage.get_report_storage",+ return_value=mock_storage,+ ),+ patch(+ f"{RS}.extract_links_from_search_results",+ return_value=[],+ ),+ patch(+ "local_deep_research.web.services.research_sources_service.ResearchSourcesService",+ return_value=mock_sources_service,+ ),+ patch(+ "local_deep_research.web.queue.processor_v2.queue_processor",+ mock_qp,+ ),+ patch("local_deep_research.settings.logger.log_settings"),+ patch(+ "local_deep_research.config.thread_settings.set_settings_context"+ ),+ ):+ raw_fn = _get_raw_run_research_process()+ raw_fn(+ "r1",+ "query",+ "detailed",+ username="alice",+ settings_snapshot={"search.tool": "searxng"},+ model="m",+ search_engine="s",+ )++ # The over-strip warning must NOT appear in any log line.+ assert (+ "format_document_split appears to have over-stripped"+ not in loguru_caplog.text+ )++ # The save path should have persisted the truncated answer (the+ # body before the sentinel), not the hyperlinked full blob. The+ # latter would be the fallback the safety check would otherwise+ # take on a non-sentinel match.+ mock_storage.save_report.assert_called_once()+ persisted_content = mock_storage.save_report.call_args.kwargs["content"]+ assert persisted_content == answer++ def test_detailed_mode_legacy_oversplit_still_fires_warning(+ self, loguru_caplog+ ):+ """The over-strip safety check still fires when no sentinel is+ present and the splitter returned a suspiciously short answer.++ Counterpart to ``test_detailed_mode_sentinel_skips_overstrip_warning``.+ The sentinel is an opt-in safety net for the detailed-mode report+ generator; callers that bypass it (e.g. chat-mode quick path,+ programmatic API, or a future section template that forgets to+ to emit the sentinel) must still get the warning + fallback, so a+ genuine over-strip bug doesn't go silent.+ """+ mock_session = MagicMock()+ research = _make_research_mock()+ mock_session.query.return_value.filter_by.return_value.first.return_value = research++ mock_system = MagicMock()+ mock_search_system = MagicMock()+ mock_search_system.all_links_of_system = [+ {"url": "http://a.com", "title": "A"}+ ]+ results = {+ "findings": [{"content": "data"}],+ "formatted_findings": "# Report",+ "iterations": 5,+ "search_system": mock_search_system,+ }+ mock_system.analyze_topic.return_value = results++ # Long content WITHOUT the sentinel — the splitter returns a+ # short answer (legacy regex matched an early "## Sources"+ # line), triggering the safety check.+ answer = "A" * 900 # > SAFETY_MIN_LEN (800)+ # Pad the sources tail to push the answer below 50% of the total.+ # 2900 chars of sources = answer ratio ~900 / (900 + 2900) = 24%.+ sources_tail = "[1] S\n URL: https://s.example\n" * 100+ content = answer + "\n\n## Sources\n\n" + sources_tail+ assert len(answer) < len(content) * 0.5++ mock_report_gen = MagicMock()+ mock_report_gen.generate_report.return_value = {+ "content": content,+ "metadata": {"sections": 3},+ }++ mock_formatter = MagicMock()+ # Simulate what the splitter does on an early "## Sources"+ # match: return the truncated answer, not the full content.+ # format_document_split now returns (answer, sources, on_sentinel);+ # the mock returns on_sentinel=False because no sentinel was in+ # the input (the legacy regex matched an early ## Sources).+ mock_formatter.format_document_split.return_value = (+ answer,+ content[len(answer) :],+ False,+ )+ # The fallback applies inline hyperlinks to the full content.+ mock_formatter.apply_inline_hyperlinks.return_value = (+ "hyperlinked full content"+ )++ mock_storage = MagicMock()+ mock_storage.save_report.return_value = True++ mock_sources_service = MagicMock()+ mock_sources_service.save_research_sources.return_value = 1+ mock_qp = MagicMock()++ with loguru_caplog.at_level("WARNING"):+ with (+ patch(+ "local_deep_research.web.routes.globals.is_termination_requested",+ return_value=False,+ ),+ patch(+ "local_deep_research.web.routes.globals.is_research_active",+ return_value=True,+ ),+ patch(+ "local_deep_research.web.routes.globals.update_progress_and_check_active",+ return_value=(50, True),+ ),+ patch(f"{RS}.get_llm", return_value=MagicMock()),+ patch(f"{RS}.get_search", return_value=MagicMock()),+ patch(f"{RS}.AdvancedSearchSystem", return_value=mock_system),+ patch(+ f"{RS}.get_citation_formatter", return_value=mock_formatter+ ),+ patch(+ f"{RS}.get_user_db_session",+ side_effect=_fake_session_ctx(mock_session),+ ),+ patch(f"{RS}.cleanup_research_resources"),+ patch(f"{RS}.SocketIOService"),+ patch(f"{RS}.set_search_context"),+ patch(f"{RS}.calculate_duration", return_value=20.0),+ patch(+ f"{RS}.IntegratedReportGenerator",+ return_value=mock_report_gen,+ ),+ patch(+ "local_deep_research.storage.get_report_storage",+ return_value=mock_storage,+ ),+ patch(+ f"{RS}.extract_links_from_search_results",+ return_value=[],+ ),+ patch(+ "local_deep_research.web.services.research_sources_service.ResearchSourcesService",+ return_value=mock_sources_service,+ ),+ patch(+ "local_deep_research.web.queue.processor_v2.queue_processor",+ mock_qp,+ ),+ patch("local_deep_research.settings.logger.log_settings"),+ patch(+ "local_deep_research.config.thread_settings.set_settings_context"+ ),+ ):+ raw_fn = _get_raw_run_research_process()+ raw_fn(+ "r1",+ "query",+ "detailed",+ username="alice",+ settings_snapshot={"search.tool": "searxng"},+ model="m",+ search_engine="s",+ )++ # The over-strip warning must have been emitted.+ assert (+ "format_document_split appears to have over-stripped"+ in loguru_caplog.text+ )++ # The fallback should have been invoked — apply_inline_hyperlinks+ # was called with the full content, and the persisted content is+ # the hyperlinked fallback, not the truncated answer.+ mock_formatter.apply_inline_hyperlinks.assert_called()+ mock_storage.save_report.assert_called_once()+ persisted_content = mock_storage.save_report.call_args.kwargs["content"]+ assert persisted_content == "hyperlinked full content"+class TestRunResearchProcessSearchErrors:"""Test search error handling in run_research_process."""@@ -2465,3 +2775,83 @@ class TestRunResearchProcessSaveReportFailure:)# Should queue errormock_qp.queue_error_update.assert_called_once()+++class TestRunResearchProcessQuickModeSentinel:+ """Test that quick mode passes trust_sentinel=False to format_document_split."""++ def test_quick_mode_passes_trust_sentinel_false(self):+ mock_session = MagicMock()+ research = _make_research_mock()+ mock_session.query.return_value.filter_by.return_value.first.return_value = research++ mock_system = MagicMock()+ results = {+ "findings": [{"phase": "s", "content": "data"}],+ "formatted_findings": "# Results",+ "iterations": 1,+ }+ mock_system.analyze_topic.return_value = results+ mock_formatter = MagicMock()+ mock_formatter.format_document_split.return_value = (+ "answer",+ "sources",+ False,+ )+ mock_storage = MagicMock()+ mock_storage.save_report.return_value = True+ mock_qp = MagicMock()++ with (+ patch(+ "local_deep_research.web.routes.globals.is_termination_requested",+ return_value=False,+ ),+ patch(+ "local_deep_research.web.routes.globals.is_research_active",+ return_value=True,+ ),+ patch(+ "local_deep_research.web.routes.globals.update_progress_and_check_active",+ return_value=(50, True),+ ),+ patch(f"{RS}.get_llm", return_value=MagicMock()),+ patch(f"{RS}.get_search", return_value=MagicMock()),+ patch(f"{RS}.AdvancedSearchSystem", return_value=mock_system),+ patch(f"{RS}.get_citation_formatter", return_value=mock_formatter),+ patch(+ f"{RS}.get_user_db_session",+ side_effect=_fake_session_ctx(mock_session),+ ),+ patch(f"{RS}.cleanup_research_resources"),+ patch(f"{RS}.SocketIOService"),+ patch(f"{RS}.set_search_context"),+ patch(f"{RS}.calculate_duration", return_value=5.0),+ patch(+ "local_deep_research.storage.get_report_storage",+ return_value=mock_storage,+ ),+ patch(f"{RS}.extract_links_from_search_results", return_value=[]),+ patch(+ "local_deep_research.web.queue.processor_v2.queue_processor",+ mock_qp,+ ),+ patch("local_deep_research.settings.logger.log_settings"),+ patch(+ "local_deep_research.config.thread_settings.set_settings_context"+ ),+ ):+ raw_fn = _get_raw_run_research_process()+ raw_fn(+ "r1",+ "query",+ "quick",+ username="alice",+ settings_snapshot={"search.tool": "searxng"},+ model="m",+ search_engine="s",+ )++ mock_formatter.format_document_split.assert_called_once_with(+ "# Results", trust_sentinel=False+ )