Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

6.6 KB
Raw Blame
"""Escaping contracts for the bibliography exporters.

These pin behaviour that regressed once already: an earlier revision escaped
BibTeX braces as ``\\{``/``\\}``, which is a LaTeX escape. BibTeX's field
scanner counts brace CHARACTERS and has no escape mechanism, so the value
stayed unbalanced and the scanner ran to EOF, swallowing the rest of the
file. Nothing asserted the contract, so it shipped.
"""

from __future__ import annotations

import itertools

import pytest

from local_deep_research.text_optimization.citation_formatter import (
    LaTeXExporter,
    QuartoExporter,
    RISExporter,
    _escape_bibtex,
    _safe_bibtex_url,
    _safe_ris_url,
    is_line_breaking_char,
)
from local_deep_research.utilities.search_utilities import (
    _sanitize_sources_field,
)


def _scan_quoted_value(text: str) -> str:
    """Model BibTeX's scanner over ``title = "<text>"``.

    The value ends at the first ``"`` seen at brace depth 0. Returns the
    consumed value, or a marker for the two failure modes.
    """
    depth = 0
    for position, char in enumerate(text):
        if char == "{":
            depth += 1
        elif char == "}":
            depth -= 1
            if depth < 0:
                return "UNBALANCED"
        elif char == '"' and depth == 0:
            return text[:position]
    return "RUNAWAY"


@pytest.mark.parametrize(
    "hostile",
    [
        'Benign}", url = "http://evil.example", note = "{x',
        "Benign{X",
        "Benign}X",
        "{{{{",
        "}}}}",
        "\\textbraceleft",
        "a\\b",
        "100% of $5 & #1 ~ ^2 _x",
    ],
)
def test_escaped_title_terminates_exactly_at_the_closing_quote(hostile):
    """No hostile title may end the field early or run to EOF."""
    field = f'{_escape_bibtex(hostile)}", year = {{2024}}'
    assert _scan_quoted_value(field) == _escape_bibtex(hostile)


def test_no_short_string_of_brace_and_quote_chars_breaks_the_field():
    """Exhaustive over the characters that could break out."""
    for length in range(1, 4):
        for combo in itertools.product('{}"\\', repeat=length):
            hostile = "".join(combo)
            escaped = _escape_bibtex(hostile)
            field = f'{escaped}", year = {{2024}}'
            assert _scan_quoted_value(field) == escaped, hostile


def test_escaped_title_is_brace_balanced():
    for hostile in ("{", "}", "{}", "}{", "a{b}c", '"{'):
        escaped = _escape_bibtex(hostile)
        assert escaped.count("{") == escaped.count("}"), hostile


@pytest.mark.parametrize(
    "code",
    [
        0x00,
        0x0D,  # CR — closes the ```bibtex fence in the Quarto export
        0x1F,
        0x7F,
        0x85,  # NEL
        0x9B,  # C1
        0x2028,  # LINE SEPARATOR
        0x2029,  # PARAGRAPH SEPARATOR
    ],
)
def test_escape_bibtex_flattens_line_breaking_characters(code):
    """The set must match ``_sanitize_sources_field``.

    Fixing CR alone was not enough: the .bib path is reachable without
    going through the Sources renderer, so every character either function
    treats as a break has to be handled by both.
    """
    char = chr(code)
    assert char not in _escape_bibtex(f"Pwn{char}```bibtex{char}junk")
    assert _safe_bibtex_url(f"http://a.test/{char}") == ""


def test_unsafe_urls_are_omitted_not_repaired():
    """Dropping characters can rewrite a URL's host.

    Removing the backslash from the first case turns the trusted host into
    userinfo, so the export would link somewhere the reader never saw.
    """
    assert _safe_bibtex_url("//trusted.example.com\\@evil.example/pwn") == ""
    assert _safe_bibtex_url("/docs/a b/c") == ""
    assert _safe_bibtex_url("https://real.example/p?a=1#frag") == (
        "https://real.example/p?a=1#frag"
    )


@pytest.mark.parametrize(
    "url",
    [
        "https://user:[email protected]/path",
        "https://[email protected]/p?a=1#f",
        "https://user%40x:[email protected]/",
        "ftp://u:[email protected]/x",
        "https:user:[email protected]/",
        "https://h.example/?next=https://u:[email protected]/",
        "https://medium.com/@user/post",
        "https://real.example/p?a=1#frag",
        "/docs/a.pdf",
    ],
)
def test_bibliography_url_matches_ris_on_userinfo(url):
    """LaTeX/BibTeX links drop userinfo exactly as the RIS export does.

    The bibliography used to re-emit ``user:secret@`` while RIS stripped
    it, so a credential in a source URL leaked into one export format
    only (#7914). A URL RIS drops, the bibliography drops too. The converse
    does not hold: the LaTeX-breakout check runs first and is deliberately
    stricter (see ``test_unsafe_urls_are_omitted_not_repaired``). These cases
    all pass that check, which is why the two results are equal here.
    """
    assert _safe_bibtex_url(url) == _safe_ris_url(url)


def test_exported_bibliographies_carry_no_url_credentials():
    """End to end: the LaTeX ``thebibliography`` block, the generated .bib
    and the RIS export keep the link but not ``user:secret@``.

    Only the bibliography is checked for LaTeX: the report's own Sources
    section is copied into the document body as written, which is a
    separate question from how the bibliography renders its links.
    """
    report = (
        "# Report\n\nA claim [1].\n\n## Sources\n\n"
        "[1] Paper\n   URL: https://user:[email protected]/paper\n"
    )
    latex = LaTeXExporter().export_to_latex(report)
    thebibliography = latex[latex.index("\\begin{thebibliography}") :]
    bib = QuartoExporter()._generate_bibliography(report)
    ris = RISExporter().export_to_ris(report)
    for exported in (thebibliography, bib, ris):
        assert "user:secret" not in exported
        assert "host.example/paper" in exported
    assert "\\url{https://host.example/paper}" in thebibliography


def test_both_producers_agree_on_every_bmp_codepoint():
    """The Sources renderer and the .bib escaper must not drift.

    They were previously two hand-rolled expressions differing on 31
    codepoints while a comment claimed they matched "exactly". Comparing
    behaviour rather than reading the source is the only way that claim
    stays true.
    """
    disagreements = [
        code
        for code in range(0x10000)
        if (_sanitize_sources_field(f"a{chr(code)}b") != f"a{chr(code)}b")
        != is_line_breaking_char(chr(code))
    ]
    assert disagreements == []


def test_legitimate_non_ascii_titles_are_untouched():
    """Widening the control set must not reject real text."""
    for title in (
        "Étude sur les systèmes",
        "汉字标题 — 测试",
        "Ελληνικά",
        "naïve café",
    ):
        assert _escape_bibtex(title) == title
        assert _sanitize_sources_field(title) == title