Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

24.1 KB
Raw Blame
r"""Math-mode parity contracts for the LaTeX exporter's body escaper.

The promoted contract in
``tests/web/services/test_export_generation_contracts.py`` and the math
preservation tests in ``test_citation_exporter_edge_cases.py`` already
pin the core scenarios (escaping stays active after a lone ``$``;
inline and display pairs preserved). This file pins only the edges
those suites do not: the lone ``$`` itself must be escaped as a
literal; a well-formed pair AFTER a lone dollar is still recognised as
math; inline spans stay inside one paragraph and close at the first
live dollar, failing the opener when that dollar is whitespace-preceded
or digit-followed; a markdown ``\$`` in prose exports as ``\$`` while
every other backslash becomes ``\textbackslash{}``; the block-scoping
and opener guards are pinned against unspaced math openers (a
space-preceded opener lets the whitespace closer rule mask a scoping
revert); backslash runs of odd length -- only odd -- neutralise the
dollar that ends them in the scanner; and a ``#`` in a text segment
that resumes mid-line after a math span is escaped, not kept as a
heading marker.
"""

from __future__ import annotations

import time

from local_deep_research.text_optimization import citation_formatter
from local_deep_research.text_optimization.citation_formatter import (
    LaTeXExporter,
    _escape_latex_body_text,
    _split_math_segments,
)
from hypothesis import given, settings, strategies as st

_META_ALPHABET = "$\\{}[]#%&~^【】,0123456789 \n\t-URL:" + "ab"
_meta_text = st.text(alphabet=_META_ALPHABET, min_size=0, max_size=400)


class TestLoneDollarDoesNotDisableEscaping:
    """An unpaired ``$`` is prose, not a math-mode opener."""

    def test_lone_dollar_itself_is_escaped_as_a_literal(self):
        exporter = LaTeXExporter()
        content = "# Title\n\nThis costs $100 per unit.\n"
        result = exporter.export_to_latex(content)

        # A bare $ in text mode is a TeX error; it must be escaped.
        assert "\\$100" in result

    def test_dollar_followed_by_whitespace_never_opens_math(self):
        r"""A ``$`` followed by whitespace is prose, not an opener.

        Pins the OUTCOME, not the opener guard: the lone ``$`` lands
        in a text segment (escaped) and the genuine pair behind it
        still exports as math. These assertions hold with the guard
        deleted too, because the pair's space-preceded opening ``$``
        is a whitespace-rejected closer that fails the lone dollar as
        an opener (pandoc's backtrack rule) before the guard ever
        runs. The guard itself is pinned by
        ``test_space_followed_dollar_cannot_steal_a_pairs_opener``,
        whose pair opener is not space-preceded.
        """
        exporter = LaTeXExporter()
        content = "# Title\n\nTotal: $ and $x$ here.\n"
        result = exporter.export_to_latex(content)

        assert "\\$ and" in result  # the lone dollar: escaped as text
        assert "$x$" in result  # the genuine pair: survives as math

    def test_space_followed_dollar_cannot_steal_a_pairs_opener(self):
        r"""Pin the opener whitespace guard itself.

        ``Total: $ ($x$) here.`` -- the lone ``$`` is followed by a
        space, and the pair's opening ``$`` sits right after ``(``,
        which no closer rule rejects. Without the guard the lone
        dollar opens a span ending at the pair's opening ``$``
        (``$ ($`` rides through as math) and destroys the pair; with
        it, the lone dollar is escaped prose and the pair survives
        intact. (In the ``...never_opens_math`` test above, the space
        before the pair's ``$`` masks the guard -- which is why this
        pin needs an unspaced opener.)
        """
        exporter = LaTeXExporter()
        content = "# Title\n\nTotal: $ ($x$) here.\n"
        result = exporter.export_to_latex(content)

        assert "Total: \\$ ($x$) here." in result


class TestPreEscapedDollarSurvivesVerbatim:
    r"""A markdown ``\$`` is a literal dollar and exports as ``\$``.

    The app's frontend (marked-katex-extension) renders ``\$`` as a
    literal dollar and never opens math on it. The body escaper
    consumes the ``\$`` pair as one token and emits ``\$``; escaping
    the backslash and the dollar separately would print a stray
    backslash (``\textbackslash{}\$``), and escaping only the dollar
    would emit ``\\$`` -- a line-break command followed by a live
    math-mode dollar.
    """

    def test_pre_escaped_dollar_in_prose_is_not_double_escaped(self):
        exporter = LaTeXExporter()
        content = "# Title\n\nIt costs \\$5 at 100% of R&D.\n"
        result = exporter.export_to_latex(content)

        assert "It costs \\$5 at 100\\% of R\\&D." in result
        assert "\\textbackslash{}\\$5" not in result
        assert "\\\\$5" not in result

    def test_body_escaper_pairs_only_the_backslash_next_to_the_dollar(self):
        r"""One regex pass; only ``\$`` is special, other backslashes are text.

        In a run of backslashes before a dollar only the last one pairs
        with it; the rest are literal backslashes, exactly as main's
        ``_escape_latex_text`` renders them. No output holds ``\\``
        (a line break) or a dollar not preceded by a single backslash.
        """
        assert _escape_latex_body_text("\\$5") == "\\$5"
        assert _escape_latex_body_text("\\\\$5") == "\\textbackslash{}\\$5"
        assert (
            _escape_latex_body_text("\\\\\\$5")
            == "\\textbackslash{}\\textbackslash{}\\$5"
        )
        assert _escape_latex_body_text("a\\b$c") == "a\\textbackslash{}b\\$c"
        assert _escape_latex_body_text("\\") == "\\textbackslash{}"

    def test_pre_escaped_dollar_never_opens_math_in_the_scanner(self):
        segments = _split_math_segments(r"It costs \$5 and $x$ today.")

        assert segments == [
            ("It costs \\$5 and ", False),
            ("$x$", True),
            (" today.", False),
        ]


class TestMathSpansSurviveIntact:
    """Matched, allowed math spans reach the output untouched."""

    def test_math_pair_after_a_lone_dollar_is_still_recognized(self):
        r"""A genuine pair after a lone ``$`` is math in its own right.

        Pins the prose AFTER the span being escaped and ``_``/``^``
        staying raw inside the math span (main's edge-case tests pin
        ``^`` only). The recognition assertions are belt-and-braces:
        they hold with the opener guard deleted, because the pair's
        space-preceded opening ``$`` fails the lone dollar as a
        closer before the guard ever runs -- so this test does not
        pin the guard, and the ``\$ and`` assertion in the
        ``...never_opens_math`` test above does not either; the
        guard's regression signal lives in
        ``test_space_followed_dollar_cannot_steal_a_pairs_opener``.
        """
        exporter = LaTeXExporter()
        content = "# Title\n\nCosts $ 5, and $x_2^2$ holds 100% of cases.\n"
        result = exporter.export_to_latex(content)

        assert "$x_2^2$" in result  # the pair itself, delimiters intact
        assert "100\\% of cases" in result  # prose after it: text, escaped

    def test_display_math_delimiters_survive(self):
        """The ``$$`` delimiters themselves must reach the output.

        The edge-case suite pins the math CONTENT (``E = mc^2``
        survives, no ``\\textasciicircum``); nothing else pins that
        the delimiters are emitted, which the scanner must do by
        returning matched spans verbatim.
        """
        exporter = LaTeXExporter()
        content = "# Title\n\n$$E = mc^2$$\n\nSome text.\n"
        result = exporter.export_to_latex(content)

        assert "$$E = mc^2$$" in result


class TestInlineMathIsBlockScoped:
    """Inline ``$...$`` spans never cross a blank line.

    Pandoc scopes inline math to a single block, so a price in one
    paragraph must not pair with the closer of genuine math in a later
    paragraph and swallow both paragraphs as one span.
    """

    def test_price_paragraph_does_not_swallow_a_later_math_paragraph(self):
        exporter = LaTeXExporter()
        content = (
            "# Title\n\n"
            "Licences cost $5 per seat at 100% of R&D budget.\n\n"
            "The identity $x^2$ holds.\n"
        )
        result = exporter.export_to_latex(content)

        # Paragraph A: prose escaped, the price escaped as a literal.
        assert "\\$5" in result
        assert "100\\%" in result
        assert "R\\&D" in result
        # Paragraph B: the math pair survives as math.
        assert "$x^2$" in result

    def test_crlf_blank_line_still_scopes_the_price_paragraph(self):
        r"""A CRLF blank line is as much a boundary as ``\n\n``.

        Legacy ``report_content`` rows carry CRLF newlines (nothing
        normalizes them before the exporter), so a price in paragraph
        A must not pair with paragraph B's math when the blank line
        between them is ``\r\n\r\n``.
        """
        exporter = LaTeXExporter()
        content = (
            "# Title\r\n\r\n"
            "Licences cost $5 per seat at 100% of R&D budget.\r\n\r\n"
            "The identity $x^2$ holds.\r\n"
        )
        result = exporter.export_to_latex(content)

        # Paragraph A: prose escaped, the price escaped as a literal.
        assert "\\$5" in result
        assert "100\\%" in result
        assert "R\\&D" in result
        # Paragraph B: the math pair survives as math.
        assert "$x^2$" in result

    def test_blank_line_of_spaces_still_scopes_the_price_paragraph(self):
        r"""A blank line holding spaces is as much a boundary as ``\n\n``.

        LLM prose often leaves trailing spaces on the otherwise blank
        line (``\n \n``); that is still one paragraph boundary, so
        the price paragraph must stay separate from the math
        paragraph.
        """
        exporter = LaTeXExporter()
        content = (
            "# Title\n \n"
            "Licences cost $5 per seat at 100% of R&D budget.\n \n"
            "The identity $x^2$ holds.\n"
        )
        result = exporter.export_to_latex(content)

        # Paragraph A: prose escaped, the price escaped as a literal.
        assert "\\$5" in result
        assert "100\\%" in result
        assert "R\\&D" in result
        # Paragraph B: the math pair survives as math.
        assert "$x^2$" in result

    def test_lf_blank_line_scopes_price_against_unspaced_math_opener(self):
        r"""The block machinery ALONE must block the cross-block pair.

        The three tests above use ``The identity $x^2$ holds.``, whose
        math opener is space-preceded: the whitespace closer rule
        independently rejects the cross-block candidate, so those
        assertions stay green even with the block machinery deleted
        and cannot see a scoping revert. Here paragraph B's math
        opener sits directly after a non-space character
        (``notes($x$)``), so its ``$`` is an acceptable closer
        candidate and ONLY the blank line stops paragraph A's price
        from pairing with it across blocks.
        """
        exporter = LaTeXExporter()
        content = "# Title\n\nCosts $5 at 100% of R&D.\n\nnotes($x$) ok.\n"
        result = exporter.export_to_latex(content)

        # Paragraph A: prose escaped, the price escaped as a literal.
        assert "\\$5 at 100\\% of R\\&D." in result
        # Paragraph B: the unspaced-opener math pair survives as math.
        assert "notes($x$) ok." in result

    def test_crlf_blank_line_scopes_price_against_unspaced_math_opener(self):
        r"""CRLF form of the unspaced-opener scoping pin.

        With ``_BLOCK_END`` reverted to a literal ``\n\n`` probe, the
        ``\r\n\r\n`` boundary is invisible and the price pairs with
        the unspaced ``notes($`` opener across blocks; the structural
        probe must catch it.
        """
        exporter = LaTeXExporter()
        content = (
            "# Title\r\n\r\nCosts $5 at 100% of R&D.\r\n\r\nnotes($x$) ok.\r\n"
        )
        result = exporter.export_to_latex(content)

        assert "\\$5 at 100\\% of R\\&D." in result
        assert "notes($x$) ok." in result

    def test_space_padded_blank_line_scopes_price_against_unspaced_opener(
        self,
    ):
        r"""Space-padded blank line form of the unspaced-opener pin.

        A literal ``\n\n`` probe cannot see the ``\n \n`` boundary
        either; the structural probe must.
        """
        exporter = LaTeXExporter()
        content = "# Title\n \nCosts $5 at 100% of R&D.\n \nnotes($x$) ok.\n"
        result = exporter.export_to_latex(content)

        assert "\\$5 at 100\\% of R\\&D." in result
        assert "notes($x$) ok." in result


class _CountingPattern:
    """Stand-in for ``_BLOCK_END`` that counts ``search`` calls."""

    def __init__(self, inner):
        self.inner = inner
        self.calls = 0

    def search(self, *args):
        self.calls += 1
        return self.inner.search(*args)


class TestDollarDenseProseScansInLinearTime:
    """The scanner must not rescan the block per opener."""

    def test_blank_line_search_runs_once_per_block_not_per_opener(
        self, monkeypatch
    ):
        r"""Every ``$a`` below is an opener that fails on its closer.

        The first blank line after an opener is cached until the scan
        passes it. Searching again per opener walks to the end of the
        block every time -- quadratic on a paragraph of prices, on the
        synchronous export route. Counting the searches pins the cache
        deterministically; wall-clock bounds alone are too loose to
        see it at any size a test can afford.
        """
        spy = _CountingPattern(citation_formatter._BLOCK_END)
        monkeypatch.setattr(citation_formatter, "_BLOCK_END", spy)
        blocks = 20
        text = ("$a " * 200 + "\n\n") * blocks

        segments = _split_math_segments(text)

        assert not any(is_math for _, is_math in segments)
        assert spy.calls <= blocks + 1

    def test_price_wall_export_is_not_quadratic(self):
        r"""A ``$a `` wall of 60 000 failed openers exports in linear time.

        Each opener's closer is the next dollar, rejected because a
        space precedes it. The scan advances one candidate pointer;
        restarting it (or the blank-line search) per opener makes this
        wall take minutes. The bound is deliberately generous: the
        linear scan takes well under a second here.
        """
        exporter = LaTeXExporter()
        wall = "$a " * 60000

        started = time.perf_counter()
        result = exporter.export_to_latex(wall)
        elapsed = time.perf_counter() - started

        assert result.count("\\$a") == 60000  # every dollar: prose
        assert elapsed < 15.0


class TestCloserFollowedByDigitIsRejected:
    """A closing ``$`` followed by a digit reads as a price.

    Pandoc (and the app's frontend) reject such closers, so ``$5-$10``
    is two prices in prose -- not ``$5-$`` of math around ``10``. The
    rejection fails the OPENER: stepping past the ``$10`` to a farther
    closer would emit one span holding that raw interior dollar. The
    rejected ``$10`` is a price too and never opens a span of its own.
    """

    def test_price_range_stays_prose(self):
        exporter = LaTeXExporter()
        content = "# Title\n\nIt ranged from $5-$10 per unit.\n"
        result = exporter.export_to_latex(content)

        assert "from \\$5-\\$10 per unit" in result

    def test_price_range_before_a_genuine_pair_does_not_swallow_it(self):
        text = "Prices of $5-$10 per unit ($n$) apply."

        assert _split_math_segments(text) == [
            ("Prices of $5-$10 per unit (", False),
            ("$n$", True),
            (") apply.", False),
        ]
        result = LaTeXExporter().export_to_latex("# Title\n\n" + text + "\n")
        assert "Prices of \\$5-\\$10 per unit ($n$) apply." in result

    def test_price_range_cannot_open_a_span_over_prose(self):
        r"""The rejected ``$10`` must not pair with ``US$``.

        Were it an opener, ``$10 (50% off), paid in US$`` would ride
        raw as math -- and ``%`` in math is a TeX comment that eats
        the rest of the line.
        """
        text = "Plans cost $5-$10 (50% off), paid in US$ or A$."

        assert _split_math_segments(text) == [(text, False)]
        result = LaTeXExporter().export_to_latex("# Title\n\n" + text + "\n")
        assert (
            "Plans cost \\$5-\\$10 (50\\% off), paid in US\\$ or A\\$."
            in result
        )


class TestBackslashedDollarNeverClosesMath:
    r"""A ``\$`` sitting between two genuine dollars is not a closer.

    The scanner promises a backslash-preceded ``$`` is a literal
    dollar in BOTH directions, but the closer precompute accepted it:
    ``$x\$y$`` closed at the ``\$``, the genuine closer was then
    escaped as prose, and the math left the document unbalanced.
    """

    def test_backslashed_dollar_is_not_a_closer_in_the_scanner(self):
        segments = _split_math_segments(r"Cost $x\$y$ total")

        assert segments == [
            ("Cost ", False),
            ("$x\\$y$", True),
            (" total", False),
        ]

    def test_span_closes_at_the_genuine_dollar_not_the_escaped_one(self):
        exporter = LaTeXExporter()
        content = "# Title\n\nCost $x\\$y$ and 100% of R&D.\n"
        result = exporter.export_to_latex(content)

        # One math span, closed by the real "$"; the escaped interior
        # dollar rides verbatim inside it.
        assert "$x\\$y$" in result
        # The prose AFTER the span is still escaped as text.
        assert "100\\% of R\\&D" in result


class TestEvenBackslashRunKeepsTheDollarLive:
    r"""Only an odd backslash run escapes the ``$`` that ends it.

    ``\\$`` is an escaped backslash followed by a live dollar. In the
    scanner it is a legitimate closer (excluding it left ``$x\\$``
    unpaired -- a strict regression: the pre-scanner exporter kept
    this shape one balanced math span). In prose, the escaper renders
    the first backslash as ``\textbackslash{}`` and the ``\$`` pair as
    an escaped dollar: never a live dollar, never ``\\``.
    """

    def test_even_run_dollar_in_prose_is_escaped(self):
        r"""``\\$5`` in prose exports as ``\textbackslash{}\$5``.

        No line break (``\\``) and no live dollar reach the ``.tex``;
        the ``100\%`` and ``R\&D`` around it stay escaped text.
        """
        exporter = LaTeXExporter()
        content = "# Title\n\nIt costs \\\\$5 today at 100% of R&D.\n"
        result = exporter.export_to_latex(content)

        assert (
            "It costs \\textbackslash{}\\$5 today at 100\\% of R\\&D." in result
        )

    def test_even_run_dollar_is_a_closer_in_the_scanner(self):
        r"""``$x\\$`` pairs as one span, closed by the even-run ``$``."""
        segments = _split_math_segments(r"a $x\\$ b")

        assert segments == [
            ("a ", False),
            ("$x\\\\$", True),
            (" b", False),
        ]

    def test_even_run_closer_keeps_the_span_balanced_in_export(self):
        r"""The balanced-span behavior reaches the exported output.

        The math span ``$x\\$`` rides through verbatim (balanced
        delimiters, no unclosed math) and the prose after it is
        escaped as text.
        """
        exporter = LaTeXExporter()
        content = "# Title\n\na $x\\\\$ b and 100% after.\n"
        result = exporter.export_to_latex(content)

        assert "$x\\\\$ b and 100\\% after." in result


class TestWhitespaceRejectedCloserFailsTheOpener:
    r"""A closer ``$`` preceded by whitespace fails the whole opener.

    Pandoc's rule: when the closing candidate is preceded by
    whitespace, the opening ``$`` was prose. Skipping past such a
    candidate to a farther ``$`` emits one math span holding a raw
    interior dollar -- byte-identical to the pre-PR vulnerable
    exporter. Unlike a digit-rejected candidate, a whitespace-rejected
    one may still open its own span.
    """

    def test_price_then_pair_in_one_block_is_not_one_span(self):
        r"""A price sentence and a genuine pair share one paragraph.

        The closer candidate after ``so `` is whitespace-preceded, so
        the price's ``$`` must fail as an opener (escaped prose) and
        the pair behind it must pair on its own -- not one span
        ``$5 today, so $x$`` with a raw interior dollar.
        """
        exporter = LaTeXExporter()
        content = "# Title\n\nCosts $5 today, so $x$ holds 100%.\n"
        result = exporter.export_to_latex(content)

        assert "\\$5 today, so $x$ holds" in result
        assert "100\\%" in result

    def test_digit_followed_candidate_after_a_space_still_fails_opener(self):
        r"""``$fee $5$``: the interior candidate has BOTH flaws.

        The ``$5`` candidate is whitespace-preceded AND digit-followed;
        whitespace wins, so ``$fee`` is prose and ``$5$`` still pairs
        on its own (a digit-rejected candidate could not open).
        Stepping over the candidate would emit the unbalanced span
        ``$fee $5$`` with a raw interior dollar.
        """
        segments = _split_math_segments("The $fee $5$ now")

        assert segments == [
            ("The $fee ", False),
            ("$5$", True),
            (" now", False),
        ]


class TestDisplayMathIsBlockScoped:
    r"""A ``$$`` display span pairs only within its own block.

    The display closer search used to run document-globally, so ONE
    stray ``$$`` in an earlier paragraph captured the next genuine
    display opener: the corrupted span swallowed the blank line
    between the paragraphs (never valid inside math), the prose
    between them rode raw, and the genuine pair was destroyed.
    """

    def test_stray_display_dollars_do_not_capture_a_later_span(self):
        exporter = LaTeXExporter()
        content = (
            "# Title\n\n"
            "Budget rising $$ at 100% & pace.\n\n"
            "$$E = mc^2$$\n\n"
            "More R&D prose.\n"
        )
        result = exporter.export_to_latex(content)

        # The stray $$ is prose: escaped, and the % and & around it
        # stay escaped text instead of riding raw inside a span that
        # runs across the blank line.
        assert "rising \\$\\$ at 100\\% \\& pace." in result
        # The genuine display pair in the next block survives intact.
        assert "$$E = mc^2$$" in result
        assert "R\\&D prose" in result


class TestHashAfterMathIsNotAHeading:
    r"""Only a true line start can carry a heading marker.

    A text segment that resumes mid-line after a math span begins with
    whatever follows the closer; ``$x$ # y`` must escape that ``#``
    rather than keep it raw as if it opened a heading.
    """

    def test_hash_after_inline_math_is_escaped(self):
        result = LaTeXExporter().export_to_latex(
            "# Title\n\nWe have $x$ # 5 and 100% more.\n"
        )

        assert "We have $x$ \\# 5 and 100\\% more." in result

    def test_heading_with_math_is_escaped_after_the_span(self):
        result = LaTeXExporter().export_to_latex(
            "# Energy $E=mc^2$ # 50% & more\n\nBody.\n"
        )

        assert "\\section{Energy $E=mc^2$ \\# 50\\% \\& more}" in result


class TestMathScannerPartitionsExactly:
    """``_split_math_segments`` must be a lossless partition.

    The scanner this PR introduces is the new input→output boundary for
    every exported document; these Hypothesis properties pin that it
    neither drops nor duplicates text, and that anything it calls math
    is a well-formed span.
    """

    @settings(max_examples=100, deadline=None)
    @given(_meta_text)
    def test_segments_reconstruct_the_input(self, text):
        segments = _split_math_segments(text)

        assert "".join(segment for segment, _ in segments) == text

    @settings(max_examples=100, deadline=None)
    @given(_meta_text)
    def test_math_segments_are_well_formed(self, text):
        # Lone dollars legitimately live in TEXT segments (the escaper
        # turns them into \\$ downstream — that is the parity fix); what
        # must hold is that anything the scanner CALLS math is a
        # well-formed, dollar-delimited span with a non-empty interior.
        for segment, is_math in _split_math_segments(text):
            if is_math:
                assert len(segment) >= 3
                assert segment.startswith("$") and segment.endswith("$")