~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
r"""Math-mode parity contracts for the LaTeX exporter's body escaper.
The promoted contract in
``tests/web/services/test_export_generation_contracts.py`` and the math
preservation tests in ``test_citation_exporter_edge_cases.py`` already
pin the core scenarios (escaping stays active after a lone ``$``;
inline and display pairs preserved). This file pins only the edges
those suites do not: the lone ``$`` itself must be escaped as a
literal; a well-formed pair AFTER a lone dollar is still recognised as
math; inline spans stay inside one paragraph and close at the first
live dollar, failing the opener when that dollar is whitespace-preceded
or digit-followed; a markdown ``\$`` in prose exports as ``\$`` while
every other backslash becomes ``\textbackslash{}``; the block-scoping
and opener guards are pinned against unspaced math openers (a
space-preceded opener lets the whitespace closer rule mask a scoping
revert); backslash runs of odd length -- only odd -- neutralise the
dollar that ends them in the scanner; and a ``#`` in a text segment
that resumes mid-line after a math span is escaped, not kept as a
heading marker.
"""
from __future__ import annotations
import time
from local_deep_research.text_optimization import citation_formatter
from local_deep_research.text_optimization.citation_formatter import (
LaTeXExporter,
_escape_latex_body_text,
_split_math_segments,
)
from hypothesis import given, settings, strategies as st
_META_ALPHABET = "$\\{}[]#%&~^【】,0123456789 \n\t-URL:" + "ab"
_meta_text = st.text(alphabet=_META_ALPHABET, min_size=0, max_size=400)
class TestLoneDollarDoesNotDisableEscaping:
"""An unpaired ``$`` is prose, not a math-mode opener."""
def test_lone_dollar_itself_is_escaped_as_a_literal(self):
exporter = LaTeXExporter()
content = "# Title\n\nThis costs $100 per unit.\n"
result = exporter.export_to_latex(content)
# A bare $ in text mode is a TeX error; it must be escaped.
assert "\\$100" in result
def test_dollar_followed_by_whitespace_never_opens_math(self):
r"""A ``$`` followed by whitespace is prose, not an opener.
Pins the OUTCOME, not the opener guard: the lone ``$`` lands
in a text segment (escaped) and the genuine pair behind it
still exports as math. These assertions hold with the guard
deleted too, because the pair's space-preceded opening ``$``
is a whitespace-rejected closer that fails the lone dollar as
an opener (pandoc's backtrack rule) before the guard ever
runs. The guard itself is pinned by
``test_space_followed_dollar_cannot_steal_a_pairs_opener``,
whose pair opener is not space-preceded.
"""
exporter = LaTeXExporter()
content = "# Title\n\nTotal: $ and $x$ here.\n"
result = exporter.export_to_latex(content)
assert "\\$ and" in result # the lone dollar: escaped as text
assert "$x$" in result # the genuine pair: survives as math
def test_space_followed_dollar_cannot_steal_a_pairs_opener(self):
r"""Pin the opener whitespace guard itself.
``Total: $ ($x$) here.`` -- the lone ``$`` is followed by a
space, and the pair's opening ``$`` sits right after ``(``,
which no closer rule rejects. Without the guard the lone
dollar opens a span ending at the pair's opening ``$``
(``$ ($`` rides through as math) and destroys the pair; with
it, the lone dollar is escaped prose and the pair survives
intact. (In the ``...never_opens_math`` test above, the space
before the pair's ``$`` masks the guard -- which is why this
pin needs an unspaced opener.)
"""
exporter = LaTeXExporter()
content = "# Title\n\nTotal: $ ($x$) here.\n"
result = exporter.export_to_latex(content)
assert "Total: \\$ ($x$) here." in result
class TestPreEscapedDollarSurvivesVerbatim:
r"""A markdown ``\$`` is a literal dollar and exports as ``\$``.
The app's frontend (marked-katex-extension) renders ``\$`` as a
literal dollar and never opens math on it. The body escaper
consumes the ``\$`` pair as one token and emits ``\$``; escaping
the backslash and the dollar separately would print a stray
backslash (``\textbackslash{}\$``), and escaping only the dollar
would emit ``\\$`` -- a line-break command followed by a live
math-mode dollar.
"""
def test_pre_escaped_dollar_in_prose_is_not_double_escaped(self):
exporter = LaTeXExporter()
content = "# Title\n\nIt costs \\$5 at 100% of R&D.\n"
result = exporter.export_to_latex(content)
assert "It costs \\$5 at 100\\% of R\\&D." in result
assert "\\textbackslash{}\\$5" not in result
assert "\\\\$5" not in result
def test_body_escaper_pairs_only_the_backslash_next_to_the_dollar(self):
r"""One regex pass; only ``\$`` is special, other backslashes are text.
In a run of backslashes before a dollar only the last one pairs
with it; the rest are literal backslashes, exactly as main's
``_escape_latex_text`` renders them. No output holds ``\\``
(a line break) or a dollar not preceded by a single backslash.
"""
assert _escape_latex_body_text("\\$5") == "\\$5"
assert _escape_latex_body_text("\\\\$5") == "\\textbackslash{}\\$5"
assert (
_escape_latex_body_text("\\\\\\$5")
== "\\textbackslash{}\\textbackslash{}\\$5"
)
assert _escape_latex_body_text("a\\b$c") == "a\\textbackslash{}b\\$c"
assert _escape_latex_body_text("\\") == "\\textbackslash{}"
def test_pre_escaped_dollar_never_opens_math_in_the_scanner(self):
segments = _split_math_segments(r"It costs \$5 and $x$ today.")
assert segments == [
("It costs \\$5 and ", False),
("$x$", True),
(" today.", False),
]
class TestMathSpansSurviveIntact:
"""Matched, allowed math spans reach the output untouched."""
def test_math_pair_after_a_lone_dollar_is_still_recognized(self):
r"""A genuine pair after a lone ``$`` is math in its own right.
Pins the prose AFTER the span being escaped and ``_``/``^``
staying raw inside the math span (main's edge-case tests pin
``^`` only). The recognition assertions are belt-and-braces:
they hold with the opener guard deleted, because the pair's
space-preceded opening ``$`` fails the lone dollar as a
closer before the guard ever runs -- so this test does not
pin the guard, and the ``\$ and`` assertion in the
``...never_opens_math`` test above does not either; the
guard's regression signal lives in
``test_space_followed_dollar_cannot_steal_a_pairs_opener``.
"""
exporter = LaTeXExporter()
content = "# Title\n\nCosts $ 5, and $x_2^2$ holds 100% of cases.\n"
result = exporter.export_to_latex(content)
assert "$x_2^2$" in result # the pair itself, delimiters intact
assert "100\\% of cases" in result # prose after it: text, escaped
def test_display_math_delimiters_survive(self):
"""The ``$$`` delimiters themselves must reach the output.
The edge-case suite pins the math CONTENT (``E = mc^2``
survives, no ``\\textasciicircum``); nothing else pins that
the delimiters are emitted, which the scanner must do by
returning matched spans verbatim.
"""
exporter = LaTeXExporter()
content = "# Title\n\n$$E = mc^2$$\n\nSome text.\n"
result = exporter.export_to_latex(content)
assert "$$E = mc^2$$" in result
class TestInlineMathIsBlockScoped:
"""Inline ``$...$`` spans never cross a blank line.
Pandoc scopes inline math to a single block, so a price in one
paragraph must not pair with the closer of genuine math in a later
paragraph and swallow both paragraphs as one span.
"""
def test_price_paragraph_does_not_swallow_a_later_math_paragraph(self):
exporter = LaTeXExporter()
content = (
"# Title\n\n"
"Licences cost $5 per seat at 100% of R&D budget.\n\n"
"The identity $x^2$ holds.\n"
)
result = exporter.export_to_latex(content)
# Paragraph A: prose escaped, the price escaped as a literal.
assert "\\$5" in result
assert "100\\%" in result
assert "R\\&D" in result
# Paragraph B: the math pair survives as math.
assert "$x^2$" in result
def test_crlf_blank_line_still_scopes_the_price_paragraph(self):
r"""A CRLF blank line is as much a boundary as ``\n\n``.
Legacy ``report_content`` rows carry CRLF newlines (nothing
normalizes them before the exporter), so a price in paragraph
A must not pair with paragraph B's math when the blank line
between them is ``\r\n\r\n``.
"""
exporter = LaTeXExporter()
content = (
"# Title\r\n\r\n"
"Licences cost $5 per seat at 100% of R&D budget.\r\n\r\n"
"The identity $x^2$ holds.\r\n"
)
result = exporter.export_to_latex(content)
# Paragraph A: prose escaped, the price escaped as a literal.
assert "\\$5" in result
assert "100\\%" in result
assert "R\\&D" in result
# Paragraph B: the math pair survives as math.
assert "$x^2$" in result
def test_blank_line_of_spaces_still_scopes_the_price_paragraph(self):
r"""A blank line holding spaces is as much a boundary as ``\n\n``.
LLM prose often leaves trailing spaces on the otherwise blank
line (``\n \n``); that is still one paragraph boundary, so
the price paragraph must stay separate from the math
paragraph.
"""
exporter = LaTeXExporter()
content = (
"# Title\n \n"
"Licences cost $5 per seat at 100% of R&D budget.\n \n"
"The identity $x^2$ holds.\n"
)
result = exporter.export_to_latex(content)
# Paragraph A: prose escaped, the price escaped as a literal.
assert "\\$5" in result
assert "100\\%" in result
assert "R\\&D" in result
# Paragraph B: the math pair survives as math.
assert "$x^2$" in result
def test_lf_blank_line_scopes_price_against_unspaced_math_opener(self):
r"""The block machinery ALONE must block the cross-block pair.
The three tests above use ``The identity $x^2$ holds.``, whose
math opener is space-preceded: the whitespace closer rule
independently rejects the cross-block candidate, so those
assertions stay green even with the block machinery deleted
and cannot see a scoping revert. Here paragraph B's math
opener sits directly after a non-space character
(``notes($x$)``), so its ``$`` is an acceptable closer
candidate and ONLY the blank line stops paragraph A's price
from pairing with it across blocks.
"""
exporter = LaTeXExporter()
content = "# Title\n\nCosts $5 at 100% of R&D.\n\nnotes($x$) ok.\n"
result = exporter.export_to_latex(content)
# Paragraph A: prose escaped, the price escaped as a literal.
assert "\\$5 at 100\\% of R\\&D." in result
# Paragraph B: the unspaced-opener math pair survives as math.
assert "notes($x$) ok." in result
def test_crlf_blank_line_scopes_price_against_unspaced_math_opener(self):
r"""CRLF form of the unspaced-opener scoping pin.
With ``_BLOCK_END`` reverted to a literal ``\n\n`` probe, the
``\r\n\r\n`` boundary is invisible and the price pairs with
the unspaced ``notes($`` opener across blocks; the structural
probe must catch it.
"""
exporter = LaTeXExporter()
content = (
"# Title\r\n\r\nCosts $5 at 100% of R&D.\r\n\r\nnotes($x$) ok.\r\n"
)
result = exporter.export_to_latex(content)
assert "\\$5 at 100\\% of R\\&D." in result
assert "notes($x$) ok." in result
def test_space_padded_blank_line_scopes_price_against_unspaced_opener(
self,
):
r"""Space-padded blank line form of the unspaced-opener pin.
A literal ``\n\n`` probe cannot see the ``\n \n`` boundary
either; the structural probe must.
"""
exporter = LaTeXExporter()
content = "# Title\n \nCosts $5 at 100% of R&D.\n \nnotes($x$) ok.\n"
result = exporter.export_to_latex(content)
assert "\\$5 at 100\\% of R\\&D." in result
assert "notes($x$) ok." in result
class _CountingPattern:
"""Stand-in for ``_BLOCK_END`` that counts ``search`` calls."""
def __init__(self, inner):
self.inner = inner
self.calls = 0
def search(self, *args):
self.calls += 1
return self.inner.search(*args)
class TestDollarDenseProseScansInLinearTime:
"""The scanner must not rescan the block per opener."""
def test_blank_line_search_runs_once_per_block_not_per_opener(
self, monkeypatch
):
r"""Every ``$a`` below is an opener that fails on its closer.
The first blank line after an opener is cached until the scan
passes it. Searching again per opener walks to the end of the
block every time -- quadratic on a paragraph of prices, on the
synchronous export route. Counting the searches pins the cache
deterministically; wall-clock bounds alone are too loose to
see it at any size a test can afford.
"""
spy = _CountingPattern(citation_formatter._BLOCK_END)
monkeypatch.setattr(citation_formatter, "_BLOCK_END", spy)
blocks = 20
text = ("$a " * 200 + "\n\n") * blocks
segments = _split_math_segments(text)
assert not any(is_math for _, is_math in segments)
assert spy.calls <= blocks + 1
def test_price_wall_export_is_not_quadratic(self):
r"""A ``$a `` wall of 60 000 failed openers exports in linear time.
Each opener's closer is the next dollar, rejected because a
space precedes it. The scan advances one candidate pointer;
restarting it (or the blank-line search) per opener makes this
wall take minutes. The bound is deliberately generous: the
linear scan takes well under a second here.
"""
exporter = LaTeXExporter()
wall = "$a " * 60000
started = time.perf_counter()
result = exporter.export_to_latex(wall)
elapsed = time.perf_counter() - started
assert result.count("\\$a") == 60000 # every dollar: prose
assert elapsed < 15.0
class TestCloserFollowedByDigitIsRejected:
"""A closing ``$`` followed by a digit reads as a price.
Pandoc (and the app's frontend) reject such closers, so ``$5-$10``
is two prices in prose -- not ``$5-$`` of math around ``10``. The
rejection fails the OPENER: stepping past the ``$10`` to a farther
closer would emit one span holding that raw interior dollar. The
rejected ``$10`` is a price too and never opens a span of its own.
"""
def test_price_range_stays_prose(self):
exporter = LaTeXExporter()
content = "# Title\n\nIt ranged from $5-$10 per unit.\n"
result = exporter.export_to_latex(content)
assert "from \\$5-\\$10 per unit" in result
def test_price_range_before_a_genuine_pair_does_not_swallow_it(self):
text = "Prices of $5-$10 per unit ($n$) apply."
assert _split_math_segments(text) == [
("Prices of $5-$10 per unit (", False),
("$n$", True),
(") apply.", False),
]
result = LaTeXExporter().export_to_latex("# Title\n\n" + text + "\n")
assert "Prices of \\$5-\\$10 per unit ($n$) apply." in result
def test_price_range_cannot_open_a_span_over_prose(self):
r"""The rejected ``$10`` must not pair with ``US$``.
Were it an opener, ``$10 (50% off), paid in US$`` would ride
raw as math -- and ``%`` in math is a TeX comment that eats
the rest of the line.
"""
text = "Plans cost $5-$10 (50% off), paid in US$ or A$."
assert _split_math_segments(text) == [(text, False)]
result = LaTeXExporter().export_to_latex("# Title\n\n" + text + "\n")
assert (
"Plans cost \\$5-\\$10 (50\\% off), paid in US\\$ or A\\$."
in result
)
class TestBackslashedDollarNeverClosesMath:
r"""A ``\$`` sitting between two genuine dollars is not a closer.
The scanner promises a backslash-preceded ``$`` is a literal
dollar in BOTH directions, but the closer precompute accepted it:
``$x\$y$`` closed at the ``\$``, the genuine closer was then
escaped as prose, and the math left the document unbalanced.
"""
def test_backslashed_dollar_is_not_a_closer_in_the_scanner(self):
segments = _split_math_segments(r"Cost $x\$y$ total")
assert segments == [
("Cost ", False),
("$x\\$y$", True),
(" total", False),
]
def test_span_closes_at_the_genuine_dollar_not_the_escaped_one(self):
exporter = LaTeXExporter()
content = "# Title\n\nCost $x\\$y$ and 100% of R&D.\n"
result = exporter.export_to_latex(content)
# One math span, closed by the real "$"; the escaped interior
# dollar rides verbatim inside it.
assert "$x\\$y$" in result
# The prose AFTER the span is still escaped as text.
assert "100\\% of R\\&D" in result
class TestEvenBackslashRunKeepsTheDollarLive:
r"""Only an odd backslash run escapes the ``$`` that ends it.
``\\$`` is an escaped backslash followed by a live dollar. In the
scanner it is a legitimate closer (excluding it left ``$x\\$``
unpaired -- a strict regression: the pre-scanner exporter kept
this shape one balanced math span). In prose, the escaper renders
the first backslash as ``\textbackslash{}`` and the ``\$`` pair as
an escaped dollar: never a live dollar, never ``\\``.
"""
def test_even_run_dollar_in_prose_is_escaped(self):
r"""``\\$5`` in prose exports as ``\textbackslash{}\$5``.
No line break (``\\``) and no live dollar reach the ``.tex``;
the ``100\%`` and ``R\&D`` around it stay escaped text.
"""
exporter = LaTeXExporter()
content = "# Title\n\nIt costs \\\\$5 today at 100% of R&D.\n"
result = exporter.export_to_latex(content)
assert (
"It costs \\textbackslash{}\\$5 today at 100\\% of R\\&D." in result
)
def test_even_run_dollar_is_a_closer_in_the_scanner(self):
r"""``$x\\$`` pairs as one span, closed by the even-run ``$``."""
segments = _split_math_segments(r"a $x\\$ b")
assert segments == [
("a ", False),
("$x\\\\$", True),
(" b", False),
]
def test_even_run_closer_keeps_the_span_balanced_in_export(self):
r"""The balanced-span behavior reaches the exported output.
The math span ``$x\\$`` rides through verbatim (balanced
delimiters, no unclosed math) and the prose after it is
escaped as text.
"""
exporter = LaTeXExporter()
content = "# Title\n\na $x\\\\$ b and 100% after.\n"
result = exporter.export_to_latex(content)
assert "$x\\\\$ b and 100\\% after." in result
class TestWhitespaceRejectedCloserFailsTheOpener:
r"""A closer ``$`` preceded by whitespace fails the whole opener.
Pandoc's rule: when the closing candidate is preceded by
whitespace, the opening ``$`` was prose. Skipping past such a
candidate to a farther ``$`` emits one math span holding a raw
interior dollar -- byte-identical to the pre-PR vulnerable
exporter. Unlike a digit-rejected candidate, a whitespace-rejected
one may still open its own span.
"""
def test_price_then_pair_in_one_block_is_not_one_span(self):
r"""A price sentence and a genuine pair share one paragraph.
The closer candidate after ``so `` is whitespace-preceded, so
the price's ``$`` must fail as an opener (escaped prose) and
the pair behind it must pair on its own -- not one span
``$5 today, so $x$`` with a raw interior dollar.
"""
exporter = LaTeXExporter()
content = "# Title\n\nCosts $5 today, so $x$ holds 100%.\n"
result = exporter.export_to_latex(content)
assert "\\$5 today, so $x$ holds" in result
assert "100\\%" in result
def test_digit_followed_candidate_after_a_space_still_fails_opener(self):
r"""``$fee $5$``: the interior candidate has BOTH flaws.
The ``$5`` candidate is whitespace-preceded AND digit-followed;
whitespace wins, so ``$fee`` is prose and ``$5$`` still pairs
on its own (a digit-rejected candidate could not open).
Stepping over the candidate would emit the unbalanced span
``$fee $5$`` with a raw interior dollar.
"""
segments = _split_math_segments("The $fee $5$ now")
assert segments == [
("The $fee ", False),
("$5$", True),
(" now", False),
]
class TestDisplayMathIsBlockScoped:
r"""A ``$$`` display span pairs only within its own block.
The display closer search used to run document-globally, so ONE
stray ``$$`` in an earlier paragraph captured the next genuine
display opener: the corrupted span swallowed the blank line
between the paragraphs (never valid inside math), the prose
between them rode raw, and the genuine pair was destroyed.
"""
def test_stray_display_dollars_do_not_capture_a_later_span(self):
exporter = LaTeXExporter()
content = (
"# Title\n\n"
"Budget rising $$ at 100% & pace.\n\n"
"$$E = mc^2$$\n\n"
"More R&D prose.\n"
)
result = exporter.export_to_latex(content)
# The stray $$ is prose: escaped, and the % and & around it
# stay escaped text instead of riding raw inside a span that
# runs across the blank line.
assert "rising \\$\\$ at 100\\% \\& pace." in result
# The genuine display pair in the next block survives intact.
assert "$$E = mc^2$$" in result
assert "R\\&D prose" in result
class TestHashAfterMathIsNotAHeading:
r"""Only a true line start can carry a heading marker.
A text segment that resumes mid-line after a math span begins with
whatever follows the closer; ``$x$ # y`` must escape that ``#``
rather than keep it raw as if it opened a heading.
"""
def test_hash_after_inline_math_is_escaped(self):
result = LaTeXExporter().export_to_latex(
"# Title\n\nWe have $x$ # 5 and 100% more.\n"
)
assert "We have $x$ \\# 5 and 100\\% more." in result
def test_heading_with_math_is_escaped_after_the_span(self):
result = LaTeXExporter().export_to_latex(
"# Energy $E=mc^2$ # 50% & more\n\nBody.\n"
)
assert "\\section{Energy $E=mc^2$ \\# 50\\% \\& more}" in result
class TestMathScannerPartitionsExactly:
"""``_split_math_segments`` must be a lossless partition.
The scanner this PR introduces is the new input→output boundary for
every exported document; these Hypothesis properties pin that it
neither drops nor duplicates text, and that anything it calls math
is a well-formed span.
"""
@settings(max_examples=100, deadline=None)
@given(_meta_text)
def test_segments_reconstruct_the_input(self, text):
segments = _split_math_segments(text)
assert "".join(segment for segment, _ in segments) == text
@settings(max_examples=100, deadline=None)
@given(_meta_text)
def test_math_segments_are_well_formed(self, text):
# Lone dollars legitimately live in TEXT segments (the escaper
# turns them into \\$ downstream — that is the parity fix); what
# must hold is that anything the scanner CALLS math is a
# well-formed, dollar-delimited span with a non-empty interior.
for segment, is_math in _split_math_segments(text):
if is_math:
assert len(segment) >= 3
assert segment.startswith("$") and segment.endswith("$")