~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Behavioral tests for citation_formatter module.
Tests CitationFormatter, QuartoExporter, RISExporter, and LaTeXExporter classes.
"""
import pytest
class TestCitationModeEnum:
"""Tests for CitationMode enum."""
def test_number_hyperlinks_mode_exists(self):
"""NUMBER_HYPERLINKS mode exists."""
from local_deep_research.text_optimization.citation_formatter import (
CitationMode,
)
assert hasattr(CitationMode, "NUMBER_HYPERLINKS")
def test_domain_hyperlinks_mode_exists(self):
"""DOMAIN_HYPERLINKS mode exists."""
from local_deep_research.text_optimization.citation_formatter import (
CitationMode,
)
assert hasattr(CitationMode, "DOMAIN_HYPERLINKS")
def test_domain_id_hyperlinks_mode_exists(self):
"""DOMAIN_ID_HYPERLINKS mode exists."""
from local_deep_research.text_optimization.citation_formatter import (
CitationMode,
)
assert hasattr(CitationMode, "DOMAIN_ID_HYPERLINKS")
def test_domain_id_always_hyperlinks_mode_exists(self):
"""DOMAIN_ID_ALWAYS_HYPERLINKS mode exists."""
from local_deep_research.text_optimization.citation_formatter import (
CitationMode,
)
assert hasattr(CitationMode, "DOMAIN_ID_ALWAYS_HYPERLINKS")
def test_no_hyperlinks_mode_exists(self):
"""NO_HYPERLINKS mode exists."""
from local_deep_research.text_optimization.citation_formatter import (
CitationMode,
)
assert hasattr(CitationMode, "NO_HYPERLINKS")
def test_mode_values_are_strings(self):
"""All mode values are strings."""
from local_deep_research.text_optimization.citation_formatter import (
CitationMode,
)
for mode in CitationMode:
assert isinstance(mode.value, str)
class TestCitationFormatterInit:
"""Tests for CitationFormatter initialization."""
def test_default_mode_is_number_hyperlinks(self):
"""Default mode is NUMBER_HYPERLINKS."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
CitationMode,
)
formatter = CitationFormatter()
assert formatter.mode == CitationMode.NUMBER_HYPERLINKS
def test_accepts_custom_mode(self):
"""Accepts custom citation mode."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
CitationMode,
)
formatter = CitationFormatter(mode=CitationMode.DOMAIN_HYPERLINKS)
assert formatter.mode == CitationMode.DOMAIN_HYPERLINKS
def test_has_citation_pattern(self):
"""Has compiled citation pattern regex."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
assert formatter.citation_pattern is not None
def test_has_comma_citation_pattern(self):
"""Has compiled comma citation pattern regex."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
assert formatter.comma_citation_pattern is not None
def test_has_source_word_pattern(self):
"""Has compiled source word pattern regex."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
assert formatter.source_word_pattern is not None
class TestCitationFormatterExtractDomain:
"""Tests for _extract_domain method."""
def test_extracts_simple_domain(self):
"""Extracts simple domain from URL."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
result = formatter._extract_domain("https://example.com/page")
assert result == "example.com"
def test_removes_www_prefix(self):
"""Removes www. prefix from domain."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
result = formatter._extract_domain("https://www.example.com/page")
assert result == "example.com"
def test_recognizes_known_domains(self):
"""Recognizes and preserves known domains."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
known_urls = [
("https://arxiv.org/abs/1234", "arxiv.org"),
("https://github.com/user/repo", "github.com"),
("https://reddit.com/r/test", "reddit.com"),
("https://youtube.com/watch", "youtube.com"),
("https://pypi.org/project/test", "pypi.org"),
("https://milvus.io/docs", "milvus.io"),
("https://medium.com/article", "medium.com"),
]
for url, expected in known_urls:
result = formatter._extract_domain(url)
assert result == expected, f"Failed for {url}"
def test_handles_subdomain(self):
"""Handles subdomain in URL."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
result = formatter._extract_domain("https://docs.python.org/page")
assert result == "python.org"
def test_handles_invalid_url(self):
"""Returns 'source' for invalid URL."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
result = formatter._extract_domain("not-a-valid-url")
# Should return something sensible, not crash
assert isinstance(result, str)
def test_handles_empty_url(self):
"""Handles empty URL string."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
result = formatter._extract_domain("")
assert isinstance(result, str)
class TestCitationFormatterFindSourcesSection:
"""Tests for _find_sources_section method.
The method returns ``(position, on_sentinel)``:
- ``position`` is the character offset of the section start, or ``-1``
if no section is found.
- ``on_sentinel`` is True iff the unique appended-sources sentinel
emitted by ``IntegratedReportGenerator`` was matched (in which
case the boundary is correct by construction).
"""
def test_finds_sources_header(self):
"""Finds ## Sources header."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
content = "Some content\n\n## Sources\n\n[1] Source one"
position, on_sentinel = formatter._find_sources_section(content)
assert position != -1
assert on_sentinel is False
assert content[position:].startswith("## Sources")
def test_finds_references_header(self):
"""Finds ## References header."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
content = "Some content\n\n## References\n\n[1] Ref one"
position, _on_sentinel = formatter._find_sources_section(content)
assert position != -1
def test_finds_bibliography_header(self):
"""Finds ## Bibliography header."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
content = "Some content\n\n## Bibliography\n\n[1] Bib one"
position, _on_sentinel = formatter._find_sources_section(content)
assert position != -1
def test_finds_citations_header(self):
"""Finds ## Citations header."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
content = "Some content\n\n## Citations\n\n[1] Citation one"
position, _on_sentinel = formatter._find_sources_section(content)
assert position != -1
def test_case_insensitive(self):
"""Finds sources header case-insensitively."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
content = "Some content\n\n## SOURCES\n\n[1] Source one"
position, _on_sentinel = formatter._find_sources_section(content)
assert position != -1
def test_returns_minus_one_when_not_found(self):
"""Returns (-1, False) when no sources section found."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
content = "Some content without sources section."
position, on_sentinel = formatter._find_sources_section(content)
assert position == -1
assert on_sentinel is False
def test_finds_single_hash_header(self):
"""Finds # Sources header."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
content = "Some content\n\n# Sources\n\n[1] Source one"
position, _on_sentinel = formatter._find_sources_section(content)
assert position != -1
def test_finds_triple_hash_header(self):
"""Finds ### Sources header."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
content = "Some content\n\n### Sources\n\n[1] Source one"
position, _on_sentinel = formatter._find_sources_section(content)
assert position != -1
def test_sentinel_takes_precedence_over_legacy_patterns(self):
"""Sentinel matches before the legacy regex patterns do.
Regression for the over-strip false-positive that fired on
1380-source detailed-mode runs: when the report generator
wraps its appended ``## Sources`` block in the sentinel,
the splitter must use the sentinel position (correct by
construction) and not the first legacy ``## Sources`` that
the LLM may have emitted earlier in the section content.
"""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
LDR_APPENDED_SOURCES_SENTINEL,
)
formatter = CitationFormatter()
# Earlier LLM-emitted ## Sources inside the section, then the
# sentinel, then the appended sources block.
content = (
"Section body\n\n"
"## Sources\n\n"
"Inline LLM citations [1] http://inline.example\n\n"
f"{LDR_APPENDED_SOURCES_SENTINEL}\n\n"
"## Sources\n\n"
"[1] Appended source\n URL: http://appended.example"
)
position, on_sentinel = formatter._find_sources_section(content)
assert on_sentinel is True
assert content[position:].startswith(LDR_APPENDED_SOURCES_SENTINEL)
# Position must point at the sentinel, not the earlier
# LLM-emitted header.
assert position > content.index("## Sources\n\nInline LLM")
def test_sentinel_recognized_even_without_trailing_section_header(self):
"""Sentinel alone is enough; trailing ## Sources isn't required.
Defensive: the report generator always emits both, but the
splitter must key off the sentinel alone so a future change to
the appended header style (e.g. switching to ``## Bibliography``)
cannot silently re-introduce the false-positive.
"""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
LDR_APPENDED_SOURCES_SENTINEL,
)
formatter = CitationFormatter()
content = (
"Section body.\n\n"
f"{LDR_APPENDED_SOURCES_SENTINEL}\n\n"
"Some appended content."
)
position, on_sentinel = formatter._find_sources_section(content)
assert position != -1
assert on_sentinel is True
def test_spoofed_later_sentinel_in_sources_tail_ignored(self):
"""A spoofed sentinel inside a source title in the tail is ignored.
If an indexed source title contains the sentinel string embedded inside
a line, find_sources_section must skip it and match the real
standalone sentinel line before the ## Sources header.
"""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
LDR_APPENDED_SOURCES_SENTINEL,
)
formatter = CitationFormatter()
real_sentinel_pos = len("Section body.\n\n")
content = (
"Section body.\n\n"
f"{LDR_APPENDED_SOURCES_SENTINEL}\n\n"
"## Sources\n\n"
f"[1] Title containing {LDR_APPENDED_SOURCES_SENTINEL} inline (source nr: 1)\n"
" URL: http://example.com"
)
position, on_sentinel = formatter._find_sources_section(content)
assert position == real_sentinel_pos
assert on_sentinel is True
class TestStripAppendedSourcesSentinel:
"""strip_appended_sources_sentinel removes only the generator's marker.
The sentinel is an HTML comment and renderMarkdown() (services/ui.js)
shows raw HTML as literal text, so the copy IntegratedReportGenerator
appended must not reach displayed report text. Copies an LLM wrote are
report text and stay.
"""
def test_removes_the_appended_sentinel_and_keeps_the_report(self):
from local_deep_research.text_optimization.citation_formatter import (
LDR_APPENDED_SOURCES_SENTINEL,
strip_appended_sources_sentinel,
)
answer = "The answer [1]."
sources = "## Sources\n\n[1] Source\n URL: https://s.example\n"
content = f"{answer}\n\n{LDR_APPENDED_SOURCES_SENTINEL}\n\n{sources}"
assert (
strip_appended_sources_sentinel(content)
== f"{answer}\n\n\n\n{sources}"
)
def test_keeps_copies_that_are_not_the_trusted_sentinel(self):
"""An inline copy, and a standalone copy before the trusted last
one, are text find_sources_section does not trust either."""
from local_deep_research.text_optimization.citation_formatter import (
LDR_APPENDED_SOURCES_SENTINEL,
strip_appended_sources_sentinel,
)
quoted = f"The marker is {LDR_APPENDED_SOURCES_SENTINEL} here."
content = (
f"{quoted}\n\n{LDR_APPENDED_SOURCES_SENTINEL}\n\nMore."
f"\n\n{LDR_APPENDED_SOURCES_SENTINEL}\n\n## Sources\n"
)
stripped = strip_appended_sources_sentinel(content)
assert stripped == (
f"{quoted}\n\n{LDR_APPENDED_SOURCES_SENTINEL}\n\nMore."
"\n\n\n\n## Sources\n"
)
assert strip_appended_sources_sentinel(quoted) == quoted
@pytest.mark.parametrize(
"content",
[
"",
None,
"No sources here.",
"Answer.\n\n## Sources\n\n[1] S\n URL: https://s.example",
],
)
def test_leaves_text_without_the_sentinel_unchanged(self, content):
from local_deep_research.text_optimization.citation_formatter import (
strip_appended_sources_sentinel,
)
assert strip_appended_sources_sentinel(content) == content
class TestCitationFormatterParseSources:
"""Tests for _parse_sources method."""
def test_parses_single_source(self):
"""Parses a single source entry."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
sources_content = "[1] Test Source Title\n URL: https://example.com"
result = formatter._parse_sources(sources_content)
assert "1" in result
assert result["1"][0] == "Test Source Title"
assert result["1"][1] == "https://example.com"
def test_parses_multiple_sources(self):
"""Parses multiple source entries."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
sources_content = """[1] First Source
URL: https://first.com
[2] Second Source
URL: https://second.com"""
result = formatter._parse_sources(sources_content)
assert "1" in result
assert "2" in result
def test_parses_source_without_url(self):
"""Parses source without URL."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
sources_content = "[1] Source Without URL"
result = formatter._parse_sources(sources_content)
assert "1" in result
assert result["1"][1] == "" # Empty URL
def test_parses_comma_separated_citation_numbers(self):
"""Parses comma-separated citation numbers like [36, 3]."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
sources_content = (
"[36, 3] Shared Source Title\n URL: https://shared.com"
)
result = formatter._parse_sources(sources_content)
assert "36" in result
assert "3" in result
assert result["36"][0] == "Shared Source Title"
assert result["3"][0] == "Shared Source Title"
def test_returns_empty_dict_for_empty_content(self):
"""Returns empty dict for empty content."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
result = formatter._parse_sources("")
assert result == {}
class TestCitationFormatterFormatDocument:
"""Tests for format_document method."""
def test_returns_unchanged_for_no_hyperlinks_mode(self):
"""Returns unchanged content in NO_HYPERLINKS mode."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
CitationMode,
)
formatter = CitationFormatter(mode=CitationMode.NO_HYPERLINKS)
content = "Text with [1] citation.\n\n## Sources\n\n[1] Source\n URL: https://test.com"
result = formatter.format_document(content)
assert result == content
def test_returns_unchanged_when_no_sources_section(self):
"""Returns unchanged when no sources section found."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
content = "Text with [1] citation but no sources section."
result = formatter.format_document(content)
assert result == content
def test_formats_number_hyperlinks(self):
"""Formats citations with number hyperlinks."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
CitationMode,
)
formatter = CitationFormatter(mode=CitationMode.NUMBER_HYPERLINKS)
content = "See [1] for details.\n\n## Sources\n\n[1] Test Source\n URL: https://test.com"
result = formatter.format_document(content)
assert "[[1]](https://test.com)" in result
def test_formats_domain_hyperlinks(self):
"""Formats citations with domain hyperlinks."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
CitationMode,
)
formatter = CitationFormatter(mode=CitationMode.DOMAIN_HYPERLINKS)
content = "See [1] for details.\n\n## Sources\n\n[1] Test Source\n URL: https://example.com/page"
result = formatter.format_document(content)
assert "example.com" in result
def test_preserves_sources_section(self):
"""Preserves the sources section unchanged."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
sources_section = (
"## Sources\n\n[1] Test Source\n URL: https://test.com"
)
content = f"See [1] here.\n\n{sources_section}"
result = formatter.format_document(content)
assert sources_section in result
class TestCitationFormatterFormatNumberHyperlinks:
"""Tests for _format_number_hyperlinks method."""
def test_formats_single_citation(self):
"""Formats a single citation with number hyperlink."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
content = "Text with [1] here."
sources = {"1": ("Title", "https://example.com")}
result = formatter._format_number_hyperlinks(content, sources)
assert "[[1]](https://example.com)" in result
def test_formats_comma_separated_citations(self):
"""Formats comma-separated citations."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
content = "Text with [1, 2] here."
sources = {
"1": ("Title 1", "https://one.com"),
"2": ("Title 2", "https://two.com"),
}
result = formatter._format_number_hyperlinks(content, sources)
assert "[[1]](https://one.com)" in result
assert "[[2]](https://two.com)" in result
def test_formats_source_word_pattern(self):
"""Formats 'Source X' patterns."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
content = "According to Source 1, this is true."
sources = {"1": ("Title", "https://example.com")}
result = formatter._format_number_hyperlinks(content, sources)
assert "[[1]](https://example.com)" in result
def test_leaves_citation_without_url_unchanged(self):
"""Leaves citation without URL unchanged."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
content = "Text with [1] here."
sources = {"1": ("Title", "")} # No URL
result = formatter._format_number_hyperlinks(content, sources)
assert "[1]" in result
# Should not have hyperlink
assert "[[1]](" not in result
class TestCitationFormatterFormatDomainHyperlinks:
"""Tests for _format_domain_hyperlinks method."""
def test_replaces_number_with_domain(self):
"""Replaces citation number with domain."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
content = "Text with [1] here."
sources = {"1": ("Title", "https://github.com/repo")}
result = formatter._format_domain_hyperlinks(content, sources)
assert "github.com" in result
def test_handles_comma_separated_citations(self):
"""Handles comma-separated citations with domains."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
content = "Text with [1, 2] here."
sources = {
"1": ("Title 1", "https://github.com"),
"2": ("Title 2", "https://arxiv.org"),
}
result = formatter._format_domain_hyperlinks(content, sources)
assert "github.com" in result
assert "arxiv.org" in result
class TestCitationFormatterFormatDomainIdHyperlinks:
"""Tests for _format_domain_id_hyperlinks method."""
def test_no_id_for_single_domain_citation(self):
"""No ID suffix when only one citation from domain."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
content = "Text with [1] here."
sources = {"1": ("Title", "https://github.com/repo")}
result = formatter._format_domain_id_hyperlinks(content, sources)
assert "[[github.com]](" in result
assert "-1" not in result.split("github.com")[1].split("]")[0]
def test_adds_id_for_multiple_domain_citations(self):
"""Adds ID suffix when multiple citations from same domain."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
content = "Text with [1] and [2] here."
sources = {
"1": ("Title 1", "https://github.com/repo1"),
"2": ("Title 2", "https://github.com/repo2"),
}
result = formatter._format_domain_id_hyperlinks(content, sources)
assert "github.com-1" in result
assert "github.com-2" in result
class TestCitationFormatterFormatDomainIdAlwaysHyperlinks:
"""Tests for _format_domain_id_always_hyperlinks method."""
def test_always_adds_id(self):
"""Always adds ID suffix even for single citation."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
content = "Text with [1] here."
sources = {"1": ("Title", "https://github.com/repo")}
result = formatter._format_domain_id_always_hyperlinks(content, sources)
assert "github.com-1" in result
class TestCitationFormatterUnicodeBrackets:
"""Tests for handling Unicode lenticular brackets."""
def test_matches_unicode_lenticular_brackets(self):
"""Matches Unicode lenticular brackets 【】."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
content = "Text with 【1】 here."
sources = {"1": ("Title", "https://example.com")}
result = formatter._format_number_hyperlinks(content, sources)
assert "[[1]](https://example.com)" in result
class TestQuartoExporterInit:
"""Tests for QuartoExporter initialization."""
def test_can_instantiate(self):
"""Can instantiate QuartoExporter."""
from local_deep_research.text_optimization.citation_formatter import (
QuartoExporter,
)
exporter = QuartoExporter()
assert exporter is not None
def test_has_citation_pattern(self):
"""Has citation pattern regex."""
from local_deep_research.text_optimization.citation_formatter import (
QuartoExporter,
)
exporter = QuartoExporter()
assert exporter.citation_pattern is not None
def test_has_comma_citation_pattern(self):
"""Has comma citation pattern regex."""
from local_deep_research.text_optimization.citation_formatter import (
QuartoExporter,
)
exporter = QuartoExporter()
assert exporter.comma_citation_pattern is not None
class TestQuartoExporterExportToQuarto:
"""Tests for export_to_quarto method."""
def test_adds_yaml_header(self):
"""Adds YAML front matter header."""
from local_deep_research.text_optimization.citation_formatter import (
QuartoExporter,
)
exporter = QuartoExporter()
content = "# My Report\n\nSome content."
result = exporter.export_to_quarto(content)
assert result.startswith("---")
assert "title:" in result
def test_uses_provided_title(self):
"""Uses provided title in YAML header."""
from local_deep_research.text_optimization.citation_formatter import (
QuartoExporter,
)
exporter = QuartoExporter()
content = "Some content."
result = exporter.export_to_quarto(content, title="Custom Title")
assert "Custom Title" in result
def test_extracts_title_from_content(self):
"""Extracts title from content if not provided."""
from local_deep_research.text_optimization.citation_formatter import (
QuartoExporter,
)
exporter = QuartoExporter()
content = "# My Research Report\n\nSome content."
result = exporter.export_to_quarto(content)
assert "My Research Report" in result
def test_converts_citations_to_quarto_format(self):
"""Converts [1] citations to [@ref1] format."""
from local_deep_research.text_optimization.citation_formatter import (
QuartoExporter,
)
exporter = QuartoExporter()
content = "See [1] for details.\n\n## Sources\n\n[1] A Source\n"
result = exporter.export_to_quarto(content)
assert "[@ref1]" in result
def test_converts_comma_separated_citations(self):
"""Converts comma-separated citations to Quarto format."""
from local_deep_research.text_optimization.citation_formatter import (
QuartoExporter,
)
exporter = QuartoExporter()
content = (
"See [1, 2, 3] for details.\n\n## Sources\n\n"
"[1] A Source\n\n[2] B Source\n\n[3] C Source\n"
)
result = exporter.export_to_quarto(content)
assert "@ref1" in result
assert "@ref2" in result
assert "@ref3" in result
def test_adds_bibliography_note(self):
"""Adds bibliography file note."""
from local_deep_research.text_optimization.citation_formatter import (
QuartoExporter,
)
exporter = QuartoExporter()
content = (
"Content.\n\n## Sources\n\n[1] Source One\n URL: https://test.com"
)
result = exporter.export_to_quarto(content)
assert "references.bib" in result
class TestQuartoExporterGenerateBibliography:
"""Tests for _generate_bibliography method."""
def test_generates_bibtex_entries(self):
"""Generates BibTeX entries from sources."""
from local_deep_research.text_optimization.citation_formatter import (
QuartoExporter,
)
exporter = QuartoExporter()
content = "[1] Test Source Title\n URL: https://example.com"
result = exporter._generate_bibliography(content)
assert "@misc{ref1" in result
# BibTeX uses braces and quotes: title = "{Title}"
assert "Test Source Title" in result
assert "title" in result
def test_includes_url_in_bibtex(self):
"""Includes URL in BibTeX entry."""
from local_deep_research.text_optimization.citation_formatter import (
QuartoExporter,
)
exporter = QuartoExporter()
content = "[1] Test Source Title\n URL: https://example.com"
result = exporter._generate_bibliography(content)
assert "url = {https://example.com}" in result
def test_generates_multiple_entries(self):
"""Generates multiple BibTeX entries."""
from local_deep_research.text_optimization.citation_formatter import (
QuartoExporter,
)
exporter = QuartoExporter()
content = "[1] First Source\n URL: https://first.com\n[2] Second Source\n URL: https://second.com"
result = exporter._generate_bibliography(content)
assert "@misc{ref1" in result
assert "@misc{ref2" in result
class TestRISExporterInit:
"""Tests for RISExporter initialization."""
def test_can_instantiate(self):
"""Can instantiate RISExporter."""
from local_deep_research.text_optimization.citation_formatter import (
RISExporter,
)
exporter = RISExporter()
assert exporter is not None
def test_has_sources_pattern(self):
"""Has sources pattern regex."""
from local_deep_research.text_optimization.citation_formatter import (
RISExporter,
)
exporter = RISExporter()
assert exporter.sources_pattern is not None
class TestRISExporterExportToRIS:
"""Tests for export_to_ris method."""
def test_returns_empty_for_no_sources(self):
"""Returns empty string when no sources section."""
from local_deep_research.text_optimization.citation_formatter import (
RISExporter,
)
exporter = RISExporter()
content = "Content without sources section."
result = exporter.export_to_ris(content)
assert result == ""
def test_finds_sources_section(self):
"""Finds ## Sources section."""
from local_deep_research.text_optimization.citation_formatter import (
RISExporter,
)
exporter = RISExporter()
content = "Content.\n\n## Sources\n\n[1] Test Source\n URL: https://test.com"
result = exporter.export_to_ris(content)
assert "TY - ELEC" in result
def test_finds_references_section(self):
"""Finds ## References section."""
from local_deep_research.text_optimization.citation_formatter import (
RISExporter,
)
exporter = RISExporter()
content = "Content.\n\n## References\n\n[1] Test Ref\n URL: https://test.com"
result = exporter.export_to_ris(content)
assert "TY - ELEC" in result
def test_includes_title(self):
"""Includes title in RIS entry."""
from local_deep_research.text_optimization.citation_formatter import (
RISExporter,
)
exporter = RISExporter()
content = "## Sources\n\n[1] My Test Title"
result = exporter.export_to_ris(content)
assert "TI -" in result
def test_includes_url(self):
"""Includes URL in RIS entry."""
from local_deep_research.text_optimization.citation_formatter import (
RISExporter,
)
exporter = RISExporter()
content = "## Sources\n\n[1] Test Source\nURL: https://example.com"
result = exporter.export_to_ris(content)
assert "UR - https://example.com" in result
def test_includes_end_of_reference(self):
"""Includes ER - end of reference marker."""
from local_deep_research.text_optimization.citation_formatter import (
RISExporter,
)
exporter = RISExporter()
content = "## Sources\n\n[1] Test Source"
result = exporter.export_to_ris(content)
assert "ER -" in result
def test_includes_language(self):
"""Includes language field in RIS."""
from local_deep_research.text_optimization.citation_formatter import (
RISExporter,
)
exporter = RISExporter()
content = "## Sources\n\n[1] Test Source"
result = exporter.export_to_ris(content)
assert "LA - en" in result
class TestRISExporterCreateRISEntry:
"""Tests for _create_ris_entry method."""
def test_creates_entry_with_id(self):
"""Creates RIS entry with reference ID."""
from local_deep_research.text_optimization.citation_formatter import (
RISExporter,
)
exporter = RISExporter()
result = exporter._create_ris_entry("1", "Test Title")
assert "ID - ref1" in result
def test_extracts_year_from_text(self):
"""Extracts year from text if present."""
from local_deep_research.text_optimization.citation_formatter import (
RISExporter,
)
exporter = RISExporter()
result = exporter._create_ris_entry("1", "Test Title published in 2023")
assert "PY - 2023" in result
def test_extracts_doi_from_metadata(self):
"""Extracts DOI from metadata."""
from local_deep_research.text_optimization.citation_formatter import (
RISExporter,
)
exporter = RISExporter()
result = exporter._create_ris_entry(
"1", "Test Title", metadata={"doi": "10.1234/test"}
)
assert "DO - 10.1234/test" in result
def test_extracts_publisher_from_github_url(self):
"""Extracts GitHub as publisher from URL."""
from local_deep_research.text_optimization.citation_formatter import (
RISExporter,
)
exporter = RISExporter()
result = exporter._create_ris_entry(
"1", "Test Title", url="https://github.com/user/repo"
)
assert "PB - GitHub" in result
def test_extracts_publisher_from_arxiv_url(self):
"""Extracts arXiv as publisher from URL."""
from local_deep_research.text_optimization.citation_formatter import (
RISExporter,
)
exporter = RISExporter()
result = exporter._create_ris_entry(
"1", "Test Title", url="https://arxiv.org/abs/1234"
)
assert "PB - arXiv" in result
class TestLaTeXExporterInit:
"""Tests for LaTeXExporter initialization."""
def test_can_instantiate(self):
"""Can instantiate LaTeXExporter."""
from local_deep_research.text_optimization.citation_formatter import (
LaTeXExporter,
)
exporter = LaTeXExporter()
assert exporter is not None
def test_has_citation_pattern(self):
"""Has citation pattern regex."""
from local_deep_research.text_optimization.citation_formatter import (
LaTeXExporter,
)
exporter = LaTeXExporter()
assert exporter.citation_pattern is not None
def test_has_heading_patterns(self):
"""Has heading patterns list."""
from local_deep_research.text_optimization.citation_formatter import (
LaTeXExporter,
)
exporter = LaTeXExporter()
assert len(exporter.heading_patterns) > 0
def test_has_emphasis_patterns(self):
"""Has emphasis patterns list."""
from local_deep_research.text_optimization.citation_formatter import (
LaTeXExporter,
)
exporter = LaTeXExporter()
assert len(exporter.emphasis_patterns) > 0
class TestLaTeXExporterExportToLatex:
"""Tests for export_to_latex method."""
def test_adds_document_class(self):
"""Adds documentclass declaration."""
from local_deep_research.text_optimization.citation_formatter import (
LaTeXExporter,
)
exporter = LaTeXExporter()
result = exporter.export_to_latex("Content")
assert "\\documentclass" in result
def test_adds_begin_document(self):
"""Adds begin{document}."""
from local_deep_research.text_optimization.citation_formatter import (
LaTeXExporter,
)
exporter = LaTeXExporter()
result = exporter.export_to_latex("Content")
assert "\\begin{document}" in result
def test_adds_end_document(self):
"""Adds end{document}."""
from local_deep_research.text_optimization.citation_formatter import (
LaTeXExporter,
)
exporter = LaTeXExporter()
result = exporter.export_to_latex("Content")
assert "\\end{document}" in result
def test_converts_h1_to_section(self):
"""Converts # heading to \\section."""
from local_deep_research.text_optimization.citation_formatter import (
LaTeXExporter,
)
exporter = LaTeXExporter()
result = exporter.export_to_latex("# My Heading\n\nContent")
assert "\\section{My Heading}" in result
def test_converts_h2_to_subsection(self):
"""Converts ## heading to \\subsection."""
from local_deep_research.text_optimization.citation_formatter import (
LaTeXExporter,
)
exporter = LaTeXExporter()
result = exporter.export_to_latex("## My Subheading\n\nContent")
assert "\\subsection{My Subheading}" in result
def test_converts_h3_to_subsubsection(self):
"""Converts ### heading to \\subsubsection."""
from local_deep_research.text_optimization.citation_formatter import (
LaTeXExporter,
)
exporter = LaTeXExporter()
result = exporter.export_to_latex("### My Sub-Subheading\n\nContent")
assert "\\subsubsection{My Sub-Subheading}" in result
def test_converts_bold_to_textbf(self):
"""Converts **bold** to \\textbf."""
from local_deep_research.text_optimization.citation_formatter import (
LaTeXExporter,
)
exporter = LaTeXExporter()
result = exporter.export_to_latex("This is **bold** text.")
assert "\\textbf{bold}" in result
def test_converts_italic_to_textit(self):
"""Converts *italic* to \\textit."""
from local_deep_research.text_optimization.citation_formatter import (
LaTeXExporter,
)
exporter = LaTeXExporter()
result = exporter.export_to_latex("This is *italic* text.")
assert "\\textit{italic}" in result
def test_converts_code_to_texttt(self):
"""Converts `code` to \\texttt."""
from local_deep_research.text_optimization.citation_formatter import (
LaTeXExporter,
)
exporter = LaTeXExporter()
result = exporter.export_to_latex("This is `code` text.")
assert "\\texttt{code}" in result
def test_converts_citations_to_cite(self):
"""Converts [1] to \\cite{1}."""
from local_deep_research.text_optimization.citation_formatter import (
LaTeXExporter,
)
exporter = LaTeXExporter()
result = exporter.export_to_latex(
"See [1] for details.\n\n## Sources\n\n[1] A Source\n"
)
assert "\\cite{1}" in result
def test_adds_bibliography(self):
"""Adds bibliography section when sources exist."""
from local_deep_research.text_optimization.citation_formatter import (
LaTeXExporter,
)
exporter = LaTeXExporter()
content = "Content.\n\n## Sources\n\n[1] Test Source\n URL: https://test.com"
result = exporter.export_to_latex(content)
assert "\\begin{thebibliography}" in result
assert "\\end{thebibliography}" in result
class TestLaTeXExporterConvertLists:
"""Tests for _convert_lists method."""
def test_converts_bullet_to_item(self):
"""Converts - bullet to \\item."""
from local_deep_research.text_optimization.citation_formatter import (
LaTeXExporter,
)
exporter = LaTeXExporter()
result = exporter._convert_lists("- First item\n- Second item")
assert "\\item First item" in result
assert "\\item Second item" in result
def test_adds_itemize_environment(self):
"""Adds itemize environment around list."""
from local_deep_research.text_optimization.citation_formatter import (
LaTeXExporter,
)
exporter = LaTeXExporter()
result = exporter._convert_lists("- First item\n- Second item")
assert "\\begin{itemize}" in result
assert "\\end{itemize}" in result
class TestLaTeXExporterEscapeLatex:
"""Tests for _escape_latex method."""
def test_escapes_ampersand(self):
"""Escapes & character."""
from local_deep_research.text_optimization.citation_formatter import (
LaTeXExporter,
)
exporter = LaTeXExporter()
result = exporter._escape_latex("A & B")
assert "\\&" in result
def test_escapes_percent(self):
"""Escapes % character."""
from local_deep_research.text_optimization.citation_formatter import (
LaTeXExporter,
)
exporter = LaTeXExporter()
result = exporter._escape_latex("50%")
assert "\\%" in result
def test_escapes_dollar(self):
"""Escapes $ character."""
from local_deep_research.text_optimization.citation_formatter import (
LaTeXExporter,
)
exporter = LaTeXExporter()
result = exporter._escape_latex("$100")
assert "\\$" in result
def test_escapes_hash(self):
"""Escapes # character."""
from local_deep_research.text_optimization.citation_formatter import (
LaTeXExporter,
)
exporter = LaTeXExporter()
result = exporter._escape_latex("#1")
assert "\\#" in result
def test_escapes_underscore(self):
"""Escapes _ character."""
from local_deep_research.text_optimization.citation_formatter import (
LaTeXExporter,
)
exporter = LaTeXExporter()
result = exporter._escape_latex("file_name")
assert "\\_" in result
def test_escapes_braces(self):
"""Escapes { and } characters."""
from local_deep_research.text_optimization.citation_formatter import (
LaTeXExporter,
)
exporter = LaTeXExporter()
result = exporter._escape_latex("{test}")
assert "\\{" in result
assert "\\}" in result
def test_escapes_tilde(self):
"""Escapes ~ character."""
from local_deep_research.text_optimization.citation_formatter import (
LaTeXExporter,
)
exporter = LaTeXExporter()
result = exporter._escape_latex("~test")
assert "\\textasciitilde{}" in result
def test_escapes_caret(self):
"""Escapes ^ character."""
from local_deep_research.text_optimization.citation_formatter import (
LaTeXExporter,
)
exporter = LaTeXExporter()
result = exporter._escape_latex("x^2")
assert "\\textasciicircum{}" in result
class TestLaTeXExporterCreateBibliography:
"""Tests for _create_bibliography method."""
def test_returns_empty_for_no_sources(self):
"""Returns empty string when no sources section."""
from local_deep_research.text_optimization.citation_formatter import (
LaTeXExporter,
)
exporter = LaTeXExporter()
result = exporter._create_bibliography("Content without sources.")
assert result == ""
def test_creates_bibitem_entries(self):
"""Creates bibitem entries for sources."""
from local_deep_research.text_optimization.citation_formatter import (
LaTeXExporter,
)
exporter = LaTeXExporter()
content = "## Sources\n\n[1] Test Source Title"
result = exporter._create_bibliography(content)
assert "\\bibitem[1]{1}" in result
def test_includes_url_in_bibitem(self):
"""Includes URL in bibitem entry."""
from local_deep_research.text_optimization.citation_formatter import (
LaTeXExporter,
)
exporter = LaTeXExporter()
content = (
"## Sources\n\n[1] Test Source Title\n URL: https://example.com"
)
result = exporter._create_bibliography(content)
assert "\\url{https://example.com}" in result
class TestCitationFormatterFormatDocumentSplit:
"""Tests for format_document_split — answer / sources tuple return."""
def test_returns_tuple_with_answer_and_sources(self):
"""Happy path: returns hyperlinked answer and raw sources block."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
CitationMode,
)
formatter = CitationFormatter(mode=CitationMode.NUMBER_HYPERLINKS)
content = (
"See [1] for details.\n\n"
"## Sources\n\n[1] Test Source\n URL: https://test.com"
)
answer, sources, on_sentinel = formatter.format_document_split(content)
assert "[[1]](https://test.com)" in answer
assert sources.startswith("## Sources")
assert "Test Source" in sources
# No sentinel present in this input — boundary came from the
# legacy `## Sources` regex.
assert on_sentinel is False
def test_returns_empty_sources_when_no_sources_section(self):
"""No Sources section in input → returns (content, '', False)."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
CitationMode,
)
formatter = CitationFormatter(mode=CitationMode.NUMBER_HYPERLINKS)
content = "Just an answer with [1] but no Sources section."
answer, sources, on_sentinel = formatter.format_document_split(content)
assert answer == content
assert sources == ""
assert on_sentinel is False
def test_returns_unchanged_in_no_hyperlinks_mode(self):
"""NO_HYPERLINKS mode short-circuits to (content, '', False)."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
CitationMode,
)
formatter = CitationFormatter(mode=CitationMode.NO_HYPERLINKS)
content = (
"Text with [1].\n\n## Sources\n\n[1] Source\n URL: https://x.com"
)
answer, sources, on_sentinel = formatter.format_document_split(content)
assert answer == content
assert sources == ""
assert on_sentinel is False
def test_format_document_compat_equals_concatenation(self):
"""format_document() == answer + sources from format_document_split."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
CitationMode,
)
formatter = CitationFormatter(mode=CitationMode.NUMBER_HYPERLINKS)
content = (
"Body with [1] cite.\n\n"
"## Sources\n\n[1] One\n URL: https://one.com"
)
full = formatter.format_document(content)
answer, sources, _on_sentinel = formatter.format_document_split(content)
assert full == answer + sources
def test_sentinel_boundary_wins_over_inline_sources_header(self):
"""Sentinel positions split at the sentinel, not earlier ## Sources.
Regression for triage item #4: a 1380-source detailed-mode
run logged ``format_document_split appears to have over-stripped
(answer=84716 chars, original=196122 chars)`` because the
legacy regex picked up an LLM-emitted ``## Sources`` header
inside the section body, cutting the answer ~57% shorter than
the input. With the sentinel wrapped around the appended
sources block by ``IntegratedReportGenerator``, the splitter
keys off the sentinel and returns the full answer body
instead of the truncated prefix.
"""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
CitationMode,
LDR_APPENDED_SOURCES_SENTINEL,
)
formatter = CitationFormatter(mode=CitationMode.NUMBER_HYPERLINKS)
# Section body contains an inline LLM-emitted ## Sources block,
# then the sentinel, then the appended structured sources list.
inline_sources_block = (
"## Sources\n\n"
"[1] Inline LLM cite\n URL: https://inline.example\n"
)
appended_sources_block = (
"## Sources\n\n[1] Appended\n URL: https://appended.example\n"
)
content = (
"Long answer body that the LLM wrote. " * 100
+ f"\n\n{inline_sources_block}"
+ f"\n\n{LDR_APPENDED_SOURCES_SENTINEL}\n\n"
+ appended_sources_block
)
answer, sources, on_sentinel = formatter.format_document_split(content)
# The answer must include the full LLM prose (the entire
# "Long answer body..." preamble) — not stop at the inline
# ## Sources that the LLM emitted.
assert answer.count("Long answer body") == 100
# The sources tail must begin at the sentinel.
assert sources.startswith(LDR_APPENDED_SOURCES_SENTINEL)
assert "appended.example" in sources
# Boundary came from the sentinel — caller can skip the
# over-strip safety check on this result.
assert on_sentinel is True
def test_spoofed_earlier_sentinel_ignored(self):
"""An LLM-quoted sentinel earlier in the prose is ignored.
The sentinel is matched at its **last** occurrence (``rfind``),
so an LLM that quotes the marker verbatim earlier in the answer
body — e.g. while summarizing the LDR codebase — cannot push
the split boundary into the middle of the answer. The trusted
sentinel is the one ``_format_final_report`` actually appended
at the end.
"""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
CitationMode,
LDR_APPENDED_SOURCES_SENTINEL,
)
formatter = CitationFormatter(mode=CitationMode.NUMBER_HYPERLINKS)
# LLM quoted the marker verbatim in its prose, then the
# genuine appended sentinel + Sources block at the end.
content = (
"Answer body that quotes "
f"{LDR_APPENDED_SOURCES_SENTINEL} "
"as a string literal.\n\n"
f"{LDR_APPENDED_SOURCES_SENTINEL}\n\n"
"## Sources\n\n[1] Real\n URL: https://real.example"
)
answer, sources, on_sentinel = formatter.format_document_split(content)
# The split must land on the LAST sentinel — the one the
# report generator actually appended.
assert sources.startswith(LDR_APPENDED_SOURCES_SENTINEL)
# And the position must be after the LLM's quoted sentinel,
# not at it. We verify by checking that the answer contains
# the LLM's quoted sentinel verbatim.
assert LDR_APPENDED_SOURCES_SENTINEL in answer
assert on_sentinel is True
def test_trust_sentinel_false_returns_on_sentinel_false(self):
"""trust_sentinel=False disables the sentinel branch entirely.
The quick-mode save path passes trust_sentinel=False because
the report generator never emits the sentinel on that path.
If the LLM happens to quote the marker verbatim, the result
must be on_sentinel=False so the over-strip safety check still
runs and a spoofing split cannot silently truncate the answer.
"""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
CitationMode,
LDR_APPENDED_SOURCES_SENTINEL,
)
formatter = CitationFormatter(mode=CitationMode.NUMBER_HYPERLINKS)
content = (
"Answer body.\n\n"
f"{LDR_APPENDED_SOURCES_SENTINEL}\n\n"
"## Sources\n\n[1] Real\n URL: https://real.example"
)
answer, sources, on_sentinel = formatter.format_document_split(
content, trust_sentinel=False
)
# The sentinel was present but caller doesn't trust it, so the
# splitter falls back to the legacy regex on `## Sources`.
assert on_sentinel is False
# The split still finds the boundary (via the legacy regex),
# so the answer half doesn't include the appended sources.
assert "Real" not in answer
assert "real.example" not in answer
class TestCitationFormatterApplyInlineHyperlinks:
"""Tests for apply_inline_hyperlinks — structured-source delegation."""
@pytest.mark.parametrize(
"mode",
[
"number_hyperlinks",
"domain_hyperlinks",
"domain_id_hyperlinks",
"domain_id_always_hyperlinks",
"source_tagged_hyperlinks",
],
)
@pytest.mark.parametrize("source_key", ["url", "link"])
@pytest.mark.parametrize("at_entity", ["@", "@", "@"])
def test_inline_destination_refuses_userinfo_remaining_after_stripping(
self, mode, source_key, at_entity
):
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
CitationMode,
)
source = {
"index": 1,
"title": "Doc",
source_key: f"https://first:password@secret{at_entity}example.org/doc",
}
assert (
CitationFormatter(CitationMode(mode)).apply_inline_hyperlinks(
"Claim [1].", [source]
)
== "Claim [1]."
)
@pytest.mark.parametrize(
"mode",
[
"number_hyperlinks",
"domain_hyperlinks",
"domain_id_hyperlinks",
"domain_id_always_hyperlinks",
"source_tagged_hyperlinks",
],
)
@pytest.mark.parametrize("source_key", ["url", "link"])
def test_inline_destination_removes_userinfo_without_changing_query(
self, mode, source_key
):
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
CitationMode,
)
source = {
"index": 1,
"title": "Doc",
source_key: "https://admin:[email protected]/doc?utm_source=x",
}
result = CitationFormatter(CitationMode(mode)).apply_inline_hyperlinks(
"Claim [1].", [source]
)
assert "https://example.test/doc?utm_source=x" in result
assert "admin" not in result
assert "hunter2" not in result
def test_inline_destination_refuses_entity_encoded_userinfo(self):
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
source = {
"index": 1,
"title": "Doc",
"url": "https://admin@example.test/doc",
}
assert (
CitationFormatter().apply_inline_hyperlinks("Claim [1].", [source])
== "Claim [1]."
)
@pytest.mark.parametrize(
"destination",
[
"https:user:[email protected]/article",
"https:/user:[email protected]/x",
"https:///user:[email protected]/x",
"https:\\user:[email protected]/x",
"https:\\\\user:[email protected]/x",
],
)
def test_inline_destination_refuses_authority_without_double_slash(
self, destination
):
"""Renderers parse `https:user:pass@host` (and `/`, `///`, `\\`
spellings) as a real authority even though ``urlsplit`` does not
populate ``netloc`` for it, so the userinfo must not survive."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
source = {"index": 1, "title": "Doc", "url": destination}
result = CitationFormatter().apply_inline_hyperlinks(
"Claim [1].", [source]
)
assert result == "Claim [1]."
assert "user:pass" not in result
assert "user:secret" not in result
@pytest.mark.parametrize(
"destination",
[
"https://user:[email protected]/a",
"https://user:[email protected]/a",
"https://user:[email protected]/a",
],
)
def test_inline_destination_refuses_entity_encoded_scheme(
self, destination
):
"""An entity-encoded `:` breaks the scheme token for ``urlsplit``,
but a rendered link decodes it back into a live credential URL."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
source = {"index": 1, "title": "Doc", "url": destination}
result = CitationFormatter().apply_inline_hyperlinks(
"Claim [1].", [source]
)
assert result == "Claim [1]."
assert "user:pass" not in result
@pytest.mark.parametrize(
"destination",
[
pytest.param(" https:/user:[email protected]/x", id="decimal"),
pytest.param(" https:/user:[email protected]/x", id="hex"),
pytest.param(" https:/user:[email protected]/x", id="no-semi"),
pytest.param(" https:/user:[email protected]/x", id="padded"),
pytest.param(
"  https:/user:[email protected]/x", id="two-spaces"
),
pytest.param(" https:///user:[email protected]/x", id="slash3"),
pytest.param(" HTTPS:/user:[email protected]/x", id="upper"),
pytest.param(
" https:///user:[email protected]/x", id="entity-slash"
),
pytest.param(
" https:/user:[email protected]/x", id="c0-then-space"
),
],
)
def test_inline_destination_refuses_userinfo_after_encoded_leading_space(
self, destination
):
"""`` `` decodes to a leading space, which ``urlsplit`` and a
browser both drop before they read the scheme. A browser then
follows ``https:/user:[email protected]/x`` as a credential URL from
an ``http:`` page such as LDR's default origin or from an exported
file, and the ``https:///`` spelling from any page. The authority
must be read from the parse, not from an offset counted from the
start of the decoded string."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
source = {"index": 1, "title": "Doc", "url": destination}
result = CitationFormatter().apply_inline_hyperlinks(
"Claim [1].", [source]
)
assert result == "Claim [1]."
assert "user:pass" not in result
@pytest.mark.parametrize(
"destination",
[
"	https:/user:[email protected]/x",
" https:/user:[email protected]/x",
"https:/user:[email protected]/x",
"https:/user:[email protected]/x",
],
)
def test_inline_destination_refuses_userinfo_after_encoded_leading_control(
self, destination
):
"""A browser drops a leading C0 control just as it drops a space.
``	`` and `` `` decode to a control character, which is
refused outright; ``html.unescape`` decodes ```` and ````
to nothing, which leaves the scheme at the start."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
source = {"index": 1, "title": "Doc", "url": destination}
result = CitationFormatter().apply_inline_hyperlinks(
"Claim [1].", [source]
)
assert result == "Claim [1]."
assert "user:pass" not in result
@pytest.mark.parametrize(
"destination",
[
pytest.param(
"https://[@v1.user:hunter2]@v1.evil.example/doc",
id="raw-unparseable-decimal",
),
pytest.param(
"https://[@v1.user:hunter2]@v1.evil.example/doc",
id="raw-unparseable-hex",
),
pytest.param(
"https://[@v1.user:hunter2]@v1.evil.example/doc",
id="raw-unparseable-named",
),
pytest.param(
"https://[@v1.user:hunter2]@v1.evil.example"
"/doc",
id="decoded-unparseable",
),
],
)
def test_inline_destination_refuses_what_stripping_leaves_unparseable(
self, destination
):
"""Each URL parses, but its userinfo opens a bracket, so removing it
leaves an unbalanced ``]`` (in the raw URL, or once it is decoded)
that ``urlsplit`` rejects. A browser still decodes that remainder
into ``https://v1.user:hunter2]@v1.evil.example/doc``. A reading the
guard cannot parse has to count as hiding userinfo, not as clean."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
source = {"index": 1, "title": "Doc", "url": destination}
result = CitationFormatter().apply_inline_hyperlinks(
"Claim [1].", [source]
)
assert result == "Claim [1]."
assert "hunter2" not in result
@pytest.mark.parametrize(
"destination,expected",
[
(" https://example.org/doc", " https://example.org/doc"),
(
" https:/example.org/@author",
" https:/example.org/@author",
),
(
"https://[2001:db8::1]:8443/doc",
"https://[2001:db8::1]:8443/doc",
),
(
"https://admin:hunter2@[2001:db8::1]:8443/doc?q=1",
"https://[2001:db8::1]:8443/doc?q=1",
),
],
)
def test_inline_destination_keeps_entity_led_and_bracketed_urls(
self, destination, expected
):
"""Controls for the two refusals above. A leading `` `` with no
userinfo after it keeps its link, and so does a bracketed host that
still parses once its userinfo is removed."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
source = {"index": 1, "title": "Doc", "url": destination}
result = CitationFormatter().apply_inline_hyperlinks(
"Claim [1].", [source]
)
assert result == f"Claim [[1]]({expected})."
@pytest.mark.parametrize(
"mode",
[
"number_hyperlinks",
"domain_hyperlinks",
"domain_id_hyperlinks",
"domain_id_always_hyperlinks",
"source_tagged_hyperlinks",
],
)
@pytest.mark.parametrize(
"destination",
[
"https://medium.com/@author/article",
"https://example.org/[email protected]",
"https://example.org/wiki/Ünïcødé_(disambiguation)",
"https://api.example.org/v1?key=SECRET",
],
)
def test_inline_destination_keeps_at_outside_the_authority(
self, mode, destination
):
"""The authority-head check must not eat @s that live in the path,
query, or a plain host — only userinfo before the host."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
CitationMode,
)
source = {"index": 1, "title": "Doc", "url": destination}
result = CitationFormatter(CitationMode(mode)).apply_inline_hyperlinks(
"Claim [1].", [source]
)
assert destination in result
def test_replaces_bracket_refs_with_markdown_links(self):
"""[1] [2] become [[1]](url) [[2]](url) given a structured list."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
content = "See [1] and [2]."
sources = [
{"index": "1", "title": "First", "url": "https://a.com"},
{"index": "2", "title": "Second", "url": "https://b.com"},
]
result = formatter.apply_inline_hyperlinks(content, sources)
assert "[[1]](https://a.com)" in result
assert "[[2]](https://b.com)" in result
def test_handles_missing_index_gracefully(self):
"""[7] left unchanged when no source with index 7."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
content = "See [1] and [7]."
sources = [
{"index": "1", "title": "Only", "url": "https://a.com"},
]
result = formatter.apply_inline_hyperlinks(content, sources)
assert "[[1]](https://a.com)" in result
# [7] should remain as-is — formatter passes through unknown indices
assert "[7]" in result
def test_handles_comma_lists(self):
"""[1, 2] becomes individual links via comma_citation_pattern."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
content = "Per [1, 2] this is true."
sources = [
{"index": "1", "title": "First", "url": "https://a.com"},
{"index": "2", "title": "Second", "url": "https://b.com"},
]
result = formatter.apply_inline_hyperlinks(content, sources)
# _format_number_hyperlinks expands comma lists into individual links
assert "https://a.com" in result
assert "https://b.com" in result
def test_skips_when_no_bracket_refs(self):
"""Content without [N] refs returns unchanged."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
content = "Plain text with no citations."
sources = [
{"index": "1", "title": "Unused", "url": "https://x.com"},
]
result = formatter.apply_inline_hyperlinks(content, sources)
assert result == content
def test_returns_content_unchanged_when_sources_empty(self):
"""Empty sources list short-circuits to content."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
content = "See [1]."
assert formatter.apply_inline_hyperlinks(content, []) == content
def test_returns_empty_string_when_content_empty(self):
"""Empty content returns empty string, no crash."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
sources = [{"index": "1", "title": "x", "url": "https://x.com"}]
assert formatter.apply_inline_hyperlinks("", sources) == ""
assert formatter.apply_inline_hyperlinks(None, sources) == ""
def test_skips_sources_with_missing_url(self):
"""Sources missing url are filtered out before delegation."""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
content = "See [1] and [2]."
sources = [
{"index": "1", "title": "Has URL", "url": "https://a.com"},
{"index": "2", "title": "No URL"}, # missing url key
]
result = formatter.apply_inline_hyperlinks(content, sources)
assert "[[1]](https://a.com)" in result
# [2] left unchanged because that source was filtered
assert "[2]" in result
def test_accepts_link_key_for_searxng_compatibility(self):
"""Regression: Searxng-sourced results use 'link' not 'url'.
search_engine_searxng.py emits ``{"link": url, "title": ...,
"snippet": ...}``. The langgraph-agent strategy's collector stores
these dicts verbatim in ``all_links_of_system`` and the chat-mode
fallback path (``apply_inline_hyperlinks``) consumed them. Looking
up only ``s["url"]`` silently dropped every Searxng-sourced
citation — the answer body shipped with plain ``[N]`` brackets
even though the Sources section beneath was fully populated.
Both keys must be accepted.
"""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
content = "Per [1] and [2], yes."
sources = [
# Searxng shape
{"index": "1", "title": "Searxng result", "link": "https://sx.com"},
# Other-engine shape (still works)
{"index": "2", "title": "Other engine", "url": "https://o.com"},
]
result = formatter.apply_inline_hyperlinks(content, sources)
assert "[[1]](https://sx.com)" in result, (
"Searxng-style 'link' key must produce a hyperlink"
)
assert "[[2]](https://o.com)" in result
def test_prefers_url_over_link_when_both_present(self):
"""When a source carries both keys, ``url`` wins.
The langgraph collector copies the raw engine dict and additionally
sets ``r['link'] = r['url']`` when only ``url`` is present; other
callers may set both for legacy reasons. The canonical destination
is ``url`` — fall back to ``link`` only when ``url`` is missing.
"""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
)
formatter = CitationFormatter()
content = "See [1]."
sources = [
{
"index": "1",
"title": "Both",
"url": "https://canonical.com",
"link": "https://legacy.com",
}
]
result = formatter.apply_inline_hyperlinks(content, sources)
assert "[[1]](https://canonical.com)" in result
def test_dispatches_on_self_mode_for_domain_hyperlinks(self):
"""Regression: ``apply_inline_hyperlinks`` was hard-coded to
the NUMBER_HYPERLINKS formatter, so the chat-mode fallback
path ignored ``report.citation_format``. A user who picked
DOMAIN_HYPERLINKS in settings still got ``[[1]](url)`` on
every chat answer instead of ``[[arxiv.org]](url)``.
"""
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
CitationMode,
)
formatter = CitationFormatter(mode=CitationMode.DOMAIN_HYPERLINKS)
content = "Per [1] and [2]."
sources = [
{"index": "1", "title": "T", "url": "https://arxiv.org/abs/123"},
{"index": "2", "title": "T", "url": "https://nytimes.com/a"},
]
result = formatter.apply_inline_hyperlinks(content, sources)
assert "[[arxiv.org]](https://arxiv.org/abs/123)" in result
assert "[[nytimes.com]](https://nytimes.com/a)" in result
assert "[[1]](" not in result # must NOT fall back to NUMBER mode
def test_dispatches_on_self_mode_for_domain_id_hyperlinks(self):
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
CitationMode,
)
formatter = CitationFormatter(mode=CitationMode.DOMAIN_ID_HYPERLINKS)
content = "See [1] and [2]."
sources = [
{"index": "1", "title": "T", "url": "https://arxiv.org/abs/1"},
{"index": "2", "title": "T", "url": "https://arxiv.org/abs/2"},
]
result = formatter.apply_inline_hyperlinks(content, sources)
# Smart domain numbering: same domain twice → [arxiv.org-1] /
# [arxiv.org-2] (or similar — the exact suffix is the formatter's
# business; we assert it does NOT collapse to plain [N] mode).
assert "arxiv.org" in result
assert "[[1]](" not in result
def test_dispatches_on_self_mode_for_source_tagged_hyperlinks(self):
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
CitationMode,
)
formatter = CitationFormatter(
mode=CitationMode.SOURCE_TAGGED_HYPERLINKS
)
content = "Per [1] and [2]."
sources = [
# A real arXiv identifier: the URLClassifier tag follows the
# paper identifier, so a placeholder id gets the domain label.
{
"index": "1",
"title": "T",
"url": "https://arxiv.org/abs/2301.00001",
},
# Library hit: collection_name in metadata should win over
# the URL-derived domain tag.
{
"index": "2",
"title": "T",
"url": "https://example.com/doc",
"metadata": {"collection_name": "mypapers"},
},
]
result = formatter.apply_inline_hyperlinks(content, sources)
# arxiv URL → [arxiv-1] (URLClassifier tag), collection → [mypapers-2].
assert "arxiv-1" in result
assert "mypapers-2" in result
def test_no_hyperlinks_mode_returns_content_unchanged(self):
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
CitationMode,
)
formatter = CitationFormatter(mode=CitationMode.NO_HYPERLINKS)
content = "Per [1] and [2]."
sources = [
{"index": "1", "title": "T", "url": "https://a.com"},
{"index": "2", "title": "T", "url": "https://b.com"},
]
assert formatter.apply_inline_hyperlinks(content, sources) == content