~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Contract tests for citation identity, bounds and URL handling.
Scope note: LaTeX/BibTeX *escaping* is covered by
``test_bibliography_escaping.py`` and is deliberately not retested here.
What this file pins is everything else the module does to
LLM-generated text:
* citation extraction / index-to-source binding (a ``[N]`` in the body
must resolve to the source the Sources block gives index ``N``, and to
no other),
* idempotency of ``format_document`` in every ``CitationMode``,
* round-tripping of citation identity through the Quarto / LaTeX / RIS
exporters,
* regex bounds (super-linear blowup on hostile input),
* URL scheme handling (``javascript:`` / ``data:`` and markdown link
injection).
Tests marked ``xfail(strict=True)`` assert the behaviour that *should*
hold and currently does not. They are live defect records: when the
underlying bug is fixed the strict marker turns the XPASS into a
failure, which is the signal to delete the marker. Each one names the
defect in its ``reason``.
"""
import html
import html.entities
import json
import random
import re
import subprocess
import time
import pytest
from local_deep_research.text_optimization.citation_formatter import (
CitationFormatter,
CitationMode,
LaTeXExporter,
QuartoExporter,
RISExporter,
_defuse_quarto_body,
_defuse_quarto_cells,
)
ALL_MODES = list(CitationMode)
# Modes that render a citation as a markdown hyperlink built directly
# from the Sources ``URL:`` value. SOURCE_TAGGED is excluded on purpose:
# it is the only mode that gates the destination through
# ``_is_linkable_url``, and TestUrlSchemeHandling pins that difference.
UNGATED_LINK_MODES = [
CitationMode.NUMBER_HYPERLINKS,
CitationMode.DOMAIN_HYPERLINKS,
CitationMode.DOMAIN_ID_HYPERLINKS,
CitationMode.DOMAIN_ID_ALWAYS_HYPERLINKS,
]
def make_doc(body: str, sources: str) -> str:
"""Assemble the report shape the formatter expects."""
return f"# Research Report\n\n{body}\n\n## Sources\n\n{sources}\n"
def answer_of(document: str) -> str:
"""Return the answer half (everything before the Sources header)."""
return document.split("## Sources")[0]
def cite_through_both_entry_points(mode: CitationMode, url: str) -> list:
"""Render ``See [1].`` with ``url`` as source 1, once per map builder.
The Sources-block regex ends a URL at the first newline and ``\\s*`` eats
leading whitespace, so only the structured source list hands the gate a
destination that still carries them.
"""
formatter = CitationFormatter(mode)
document = make_doc("See [1].", f"[1] Evil\nURL: {url}")
return [
answer_of(formatter.format_document(document)),
formatter.apply_inline_hyperlinks(
"See [1].", [{"index": 1, "title": "Evil", "url": url}]
),
]
def bibliography_of_latex(latex: str) -> str:
start = latex.index("\\begin{thebibliography}")
end = latex.index("\\end{thebibliography}")
return latex[start:end]
def bibtex_of_quarto(quarto: str) -> str:
start = quarto.index("```bibtex")
return quarto[start : quarto.rindex("\n```\n")]
class TestCitationIdentity:
"""A ``[N]`` must bind to source ``N`` and to nothing else."""
def test_every_index_binds_to_its_own_source_at_scale(self):
"""2000 citations: each ``[N]`` links to source ``N``'s URL.
The failure this guards against is a citation silently pointing
at a neighbouring source. Expected fragments are written out by
hand rather than recomputed with the production formatting rule.
"""
count = 2000
body = " ".join(f"claim [{i}]." for i in range(1, count + 1))
sources = "\n\n".join(
f"[{i}] Title {i}\nURL: https://s{i}.example/p{i}"
for i in range(1, count + 1)
)
document = make_doc(body, sources)
started = time.perf_counter()
result = CitationFormatter(
CitationMode.NUMBER_HYPERLINKS
).format_document(document)
elapsed = time.perf_counter() - started
# Bound, not a benchmark: the pass is linear in document size and
# a 2000-citation report is a realistic worst case.
assert elapsed < 10.0, f"formatting 2000 citations took {elapsed:.1f}s"
answer = answer_of(result)
# Hand-written expectations, including the digit-prefix pairs
# (1/10/100/1000, 199/1999) where an off-by-one or a prefix match
# would cross-link two different sources.
assert "claim [[1]](https://s1.example/p1)." in answer
assert "claim [[10]](https://s10.example/p10)." in answer
assert "claim [[100]](https://s100.example/p100)." in answer
assert "claim [[199]](https://s199.example/p199)." in answer
assert "claim [[1000]](https://s1000.example/p1000)." in answer
assert "claim [[1999]](https://s1999.example/p1999)." in answer
assert "claim [[2000]](https://s2000.example/p2000)." in answer
# Nothing was left unformatted and nothing was cross-linked.
assert "claim [1]." not in answer
pairs = re.findall(
r"\[\[(\d+)\]\]\(https://s(\d+)\.example/p\d+\)", answer
)
assert len(pairs) == count
mismatched = [p for p in pairs if p[0] != p[1]]
assert mismatched == [], (
f"citations bound to wrong source: {mismatched}"
)
def test_comma_group_expands_in_order_to_per_index_targets(self):
document = make_doc(
"Combined [1, 2, 3].",
"[1] Alpha\nURL: https://alpha.example/a\n\n"
"[2] Beta\nURL: https://beta.example/b\n\n"
"[3] Gamma\nURL: https://gamma.example/c",
)
result = CitationFormatter(
CitationMode.NUMBER_HYPERLINKS
).format_document(document)
assert (
"Combined [[1]](https://alpha.example/a)"
"[[2]](https://beta.example/b)"
"[[3]](https://gamma.example/c)." in answer_of(result)
)
def test_source_word_form_binds_to_the_same_target_as_the_bracket(self):
document = make_doc(
"Bracket [2] and prose Source 2 agree.",
"[1] Alpha\nURL: https://alpha.example/a\n\n"
"[2] Beta\nURL: https://beta.example/b",
)
result = answer_of(
CitationFormatter(CitationMode.NUMBER_HYPERLINKS).format_document(
document
)
)
assert (
"Bracket [[2]](https://beta.example/b) and prose "
"[[2]](https://beta.example/b) agree." in result
)
def test_leading_zero_index_is_a_distinct_citation(self):
"""``[007]`` and ``[7]`` are separate keys, each keeping its URL."""
document = make_doc(
"See [7] and [007].",
"[7] Seven\nURL: https://seven.example\n\n"
"[007] Bond\nURL: https://bond.example",
)
result = answer_of(CitationFormatter().format_document(document))
assert (
"See [[7]](https://seven.example) and "
"[[007]](https://bond.example)." in result
)
def test_unknown_indices_in_a_group_become_individual_markers(self):
"""A group whose indices have no source is split, not dropped.
Pinned because it is the one input where formatting rewrites text
without resolving anything; the rewrite must stay lossless.
"""
document = make_doc(
"See [7, 8].", "[1] Alpha\nURL: https://alpha.example/a"
)
result = answer_of(CitationFormatter().format_document(document))
assert "See [7][8]." in result
def test_url_less_source_leaves_its_citation_untouched(self):
document = make_doc("Local claim [3].", "[3] Local Doc Without Url")
result = answer_of(CitationFormatter().format_document(document))
assert "Local claim [3]." in result
class TestMalformedMarkers:
"""Hostile / malformed markers must not produce a wrong binding."""
def test_already_hyperlinked_citation_is_not_rewritten(self):
document = make_doc(
"A [1] B [[1]] C [[[1]]] D",
"[1] Alpha\nURL: https://alpha.example/a",
)
result = answer_of(CitationFormatter().format_document(document))
assert "A [[1]](https://alpha.example/a) B [[1]] C [[[1]]] D" in result
def test_deeply_nested_bracket_run_is_left_alone(self):
depth = 20
marker = "[" * depth + "1" + "]" * depth
document = make_doc(marker, "[1] Alpha\nURL: https://alpha.example/a")
result = answer_of(CitationFormatter().format_document(document))
assert marker in result
def test_mixed_lenticular_brackets_bind_to_the_same_source(self):
document = make_doc(
"See \u30101] and [2\u3011.",
"[1] Alpha\nURL: https://alpha.example/a\n\n"
"[2] Beta\nURL: https://beta.example/b",
)
result = answer_of(CitationFormatter().format_document(document))
assert (
"See [[1]](https://alpha.example/a) and "
"[[2]](https://beta.example/b)." in result
)
class TestIdempotency:
"""Formatting an already-formatted report must be a no-op."""
RICH_DOCUMENT = make_doc(
"Alpha [1], beta [2], gamma [3]. Group [1, 2, 3]. Source 2 "
"confirms.\nLenticular \u30101\u3011 and mixed \u30102] here. "
"Unknown [99] and group [98, 99].\n"
"Already linked [[1]](https://arxiv.org/abs/2401.00001) stays.",
"[1] Alpha Paper\nURL: https://arxiv.org/abs/2401.00001\n\n"
"[2] Beta Blog\nURL: https://beta.example/post\n\n"
"[3] Local Doc Without Url",
)
@pytest.mark.parametrize("mode", ALL_MODES, ids=lambda m: m.value)
def test_second_pass_is_a_fixpoint(self, mode):
formatter = CitationFormatter(mode)
once = formatter.format_document(self.RICH_DOCUMENT)
twice = formatter.format_document(once)
assert once == twice
@pytest.mark.parametrize("mode", ALL_MODES, ids=lambda m: m.value)
def test_third_pass_is_still_the_same_fixpoint(self, mode):
"""Guards a two-step oscillation that a single re-run would miss."""
formatter = CitationFormatter(mode)
twice = formatter.format_document(
formatter.format_document(self.RICH_DOCUMENT)
)
thrice = formatter.format_document(twice)
assert twice == thrice
def test_formatting_never_drops_the_sources_block(self):
formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS)
once = formatter.format_document(self.RICH_DOCUMENT)
assert "[1] Alpha Paper" in once
assert "URL: https://arxiv.org/abs/2401.00001" in once
assert "[3] Local Doc Without Url" in once
class TestRoundTripCitationIdentity:
"""markdown -> export format must preserve index/URL pairing."""
DOCUMENT = make_doc(
"Claims [1] and [2].",
"[1] Alpha Paper\nURL: https://arxiv.org/abs/2401.00001\n\n"
"[2] Beta Blog\nURL: https://beta.example/post",
)
def test_quarto_keeps_index_to_url_pairing(self):
quarto = QuartoExporter().export_to_quarto(self.DOCUMENT)
assert "Claims [@ref1] and [@ref2]." in quarto
bib = bibtex_of_quarto(quarto)
assert '@misc{ref1,\n title = "{Alpha Paper}",' in bib
assert "url = {https://arxiv.org/abs/2401.00001}," in bib
assert '@misc{ref2,\n title = "{Beta Blog}",' in bib
assert "url = {https://beta.example/post}," in bib
def test_latex_keeps_index_to_url_pairing(self):
latex = LaTeXExporter().export_to_latex(self.DOCUMENT)
assert "Claims \\cite{1} and \\cite{2}." in latex
bib = bibliography_of_latex(latex)
assert (
"\\bibitem[1]{1} Alpha Paper. \\url{https://arxiv.org/abs/2401.00001}"
in bib
)
assert (
"\\bibitem[2]{2} Beta Blog. \\url{https://beta.example/post}" in bib
)
def test_ris_keeps_index_to_url_pairing(self):
ris = RISExporter().export_to_ris(self.DOCUMENT)
assert "ID - ref1\nTI - Alpha Paper\n" in ris
assert "UR - https://arxiv.org/abs/2401.00001\n" in ris
assert "ID - ref2\nTI - Beta Blog\n" in ris
assert "UR - https://beta.example/post\n" in ris
def test_latex_export_of_a_formatted_report_keeps_the_link_intact(self):
formatted = CitationFormatter(
CitationMode.NUMBER_HYPERLINKS
).format_document(self.DOCUMENT)
assert "[[1]](https://arxiv.org/abs/2401.00001)" in formatted
latex = LaTeXExporter().export_to_latex(formatted)
body = latex[: latex.index("\\begin{thebibliography}")]
assert "[\\cite{1}](" not in body
assert "\\cite{1}" in body
def test_latex_bibitem_survives_a_gap_in_the_index_sequence(self):
document = make_doc(
"See [1] and [5].",
"[1] First\nURL: https://one.example\n\n"
"[5] Fifth\nURL: https://five.example",
)
bib = bibliography_of_latex(LaTeXExporter().export_to_latex(document))
# A labelled bibitem is what makes \cite{5} render as [5].
assert "\\bibitem[5]{5}" in bib
def test_latex_bibitem_label_keeps_the_index_as_written(self):
document = make_doc(
"See [7] and [007].",
"[7] Seven\nURL: https://seven.example\n\n"
"[007] Bond\nURL: https://bond.example",
)
bib = bibliography_of_latex(LaTeXExporter().export_to_latex(document))
assert "\\bibitem[7]{7} Seven." in bib
assert "\\bibitem[007]{007} Bond." in bib
def test_latex_widest_label_grows_with_the_longest_index(self):
few = LaTeXExporter().export_to_latex(
make_doc("See [1].", "[1] One\nURL: https://one.example")
)
many = LaTeXExporter().export_to_latex(
make_doc(
"See [1] and [120].",
"[1] One\nURL: https://one.example\n\n"
"[120] Many\nURL: https://many.example",
)
)
assert "\\begin{thebibliography}{99}" in few
assert "\\begin{thebibliography}{999}" in many
class TestUrlSchemeHandling:
"""``javascript:`` / ``data:`` destinations and link injection."""
@pytest.mark.parametrize(
"url",
[
"javascript:alert(document.domain)",
"data:text/html;base64,PHNjcmlwdD5hbGVydCgxKTwvc2NyaXB0Pg==",
],
)
def test_source_tagged_mode_neutralises_dangerous_schemes(self, url):
"""SOURCE_TAGGED gates the destination and emits no hyperlink."""
document = make_doc("See [1].", f"[1] Evil\nURL: {url}")
result = answer_of(
CitationFormatter(
CitationMode.SOURCE_TAGGED_HYPERLINKS
).format_document(document)
)
assert "See [local-1]." in result
assert "javascript:" not in result
assert "data:text/html" not in result
@pytest.mark.parametrize("mode", UNGATED_LINK_MODES, ids=lambda m: m.value)
@pytest.mark.parametrize(
"url",
[
"javascript:alert(document.domain)",
"data:text/html;base64,PHNjcmlwdD5hbGVydCgxKTwvc2NyaXB0Pg==",
],
)
def test_dangerous_scheme_is_never_emitted_as_a_destination(
self, mode, url
):
document = make_doc("See [1].", f"[1] Evil\nURL: {url}")
result = answer_of(CitationFormatter(mode).format_document(document))
assert "javascript:" not in result
assert "data:text/html" not in result
def test_source_url_cannot_inject_a_second_markdown_link(self):
hostile = (
"https://good.example/x) [click here](https://evil.example/pwn"
)
document = make_doc("See [1].", f"[1] Benign\nURL: {hostile}")
result = answer_of(CitationFormatter().format_document(document))
assert "[click here](https://evil.example/pwn)" not in result
@pytest.mark.parametrize("mode", UNGATED_LINK_MODES, ids=lambda m: m.value)
@pytest.mark.parametrize(
"url",
[
pytest.param("JaVaScRiPt:alert(1)", id="mixed case"),
pytest.param(" javascript:alert(1)", id="leading spaces"),
pytest.param(
"\tjava\nscript:alert(1)", id="tab and newline inside"
),
pytest.param("\x01javascript:alert(1)", id="leading control char"),
pytest.param("vbscript:msgbox(1)", id="vbscript"),
pytest.param("blob:https://good.example/x", id="blob"),
pytest.param("javascript:alert(1)", id="named colon"),
pytest.param("javascript:alert(1)", id="decimal colon"),
pytest.param("javascript:alert(1)", id="decimal letter"),
pytest.param("java	script:alert(1)", id="named tab"),
],
)
def test_a_dangerous_scheme_survives_no_spelling_of_itself(self, mode, url):
"""A browser lowercases the scheme, strips leading C0 controls and
spaces, and removes tabs and newlines before it reads one. A check on
the raw string therefore passes strings that still execute, which is
why the allowlist is applied to the trimmed form.
Before any of that, the HTML parser decodes character references in an
``href``, and marked and Python-Markdown copy the destination there as
written, so ``javascript:`` is a spelling as well. The result is
decoded and lowercased the same way before it is searched.
``blob:`` is in here for the other half: an allowlist is what makes a
scheme nobody enumerated fail by default.
"""
for result in cite_through_both_entry_points(mode, url):
as_the_browser_reads_it = html.unescape(result).lower()
assert "script:" not in as_the_browser_reads_it
assert "blob:" not in as_the_browser_reads_it
@pytest.mark.parametrize("mode", UNGATED_LINK_MODES, ids=lambda m: m.value)
def test_an_unparseable_url_is_refused_rather_than_passed_through(
self, mode
):
"""``urlsplit`` raises on a broken authority, and a dangerous scheme
can carry one: ``javascript://[`` raises before the scheme is read.
Treating "cannot parse" as "nothing to object to" would hand that
straight to the renderer.
"""
document = make_doc(
"See [1].", "[1] Evil\nURL: javascript://[/alert(1)"
)
result = answer_of(CitationFormatter(mode).format_document(document))
assert "javascript:" not in result
def test_a_structured_source_list_is_gated_the_same_way(self):
"""The other entry point. ``format_document_with_sources`` is the
fallback used when the LLM emits no Sources block, and it builds the
same map from search-result dicts, where the URL came from a search
engine rather than from the model.
"""
sources = [
{"index": 1, "title": "Evil", "url": "javascript:alert(1)"},
{"index": 2, "title": "Real", "link": "https://good.example/x"},
]
result = CitationFormatter().apply_inline_hyperlinks(
"See [1] and [2].", sources
)
assert "javascript:" not in result
assert "](https://good.example/x)" in result
@pytest.mark.parametrize("mode", UNGATED_LINK_MODES, ids=lambda m: m.value)
@pytest.mark.parametrize(
"url",
[
pytest.param(
"https://ok.example/x\n\n<img src=x onerror=alert(1)>",
id="newline ends the destination",
),
pytest.param("java\x00script:alert(1)", id="NUL inside the scheme"),
pytest.param(
"https://ok.example/\x07bell", id="bell inside the path"
),
],
)
def test_a_control_character_refuses_the_destination(self, mode, url):
"""``urlsplit`` removes tabs and newlines and strips leading C0
controls to PARSE, and leaves them in the string. A destination that
carries one ends there, and what follows is emitted as document body:
an exported .md or .qmd then contains attacker-written markdown that
never meets DOMPurify. NUL is not stripped at all, so it turns a
dangerous scheme into a scheme-less string that would otherwise pass.
"""
for result in cite_through_both_entry_points(mode, url):
assert "onerror" not in result
assert "script:" not in result
assert "\x00" not in result
assert "\x07" not in result
@pytest.mark.parametrize("mode", UNGATED_LINK_MODES, ids=lambda m: m.value)
def test_a_closing_paren_cannot_reopen_markdown(self, mode):
"""The scheme gate reads the whole string; markdown reads only up to
the first unbalanced ``)``. Without escaping, everything after it is
document body, so a destination can write its own link and put back
exactly the scheme the gate refuses.
"""
hostile = "https://good.example/x) [click here](javascript:alert(1)"
document = make_doc("See [1].", f"[1] Benign\nURL: {hostile}")
result = answer_of(CitationFormatter(mode).format_document(document))
assert "](javascript:" not in result
assert "javascript:alert(1))" not in result
@pytest.mark.parametrize("mode", UNGATED_LINK_MODES, ids=lambda m: m.value)
@pytest.mark.parametrize(
("character", "encoded"),
[
pytest.param(" ", "%20", id="space"),
pytest.param("<", "%3C", id="angle open"),
pytest.param(">", "%3E", id="angle close"),
pytest.param('"', "%22", id="double quote"),
pytest.param("'", "%27", id="single quote"),
pytest.param("\\", "%5C", id="backslash"),
pytest.param("`", "%60", id="backtick"),
],
)
def test_the_other_destination_escapes_each_close_the_same_breakout(
self, mode, character, encoded
):
"""A paren is not the only way out of ``](...)``.
Whitespace ends a bare destination outright and can open a title;
``<`` at the head switches markdown to the angle-bracket form, whose
``>`` then ends it; a quote or a backtick hands the rest of the line to
another inline construct. Each one puts the attacker's text back into
document body with markdown active, which is the same breakout the
paren cell measures, so each is pinned rather than trusted to the table
being complete.
"""
hostile = f"https://good.example/x{character}TAIL"
document = make_doc("See [1].", f"[1] Benign\nURL: {hostile}")
result = answer_of(CitationFormatter(mode).format_document(document))
# The whole destination, closing paren included: asserting only that
# the character was encoded would still pass if everything after it had
# been emitted as document body.
assert f"](https://good.example/x{encoded}TAIL)" in result
def test_a_balanced_pair_of_parens_is_left_verbatim(self):
"""Accept control for the escaping, and the reason it is not applied to
every paren.
Markdown ends a bare destination at the first *unbalanced* ``)`` and
carries a balanced pair through unharmed, so there is nothing to defend
against here. Encoding anyway rewrites an ordinary citation into a URL
the source never served, which the reader sees and copies, and
Wikipedia alone titles thousands of articles ``..._(disambiguation)``.
"""
document = make_doc(
"See [1].",
"[1] Foo\nURL: https://en.wikipedia.org/wiki/Foo_(bar)",
)
result = answer_of(CitationFormatter().format_document(document))
assert "](https://en.wikipedia.org/wiki/Foo_(bar))" in result
assert "%28" not in result
def test_an_unbalanced_paren_is_encoded(self):
"""The case the encoding exists for, and the one the balanced rule must
not let through. A lone ``)`` ends the destination where it stands, so
everything the source put after it would be emitted as document body
with markdown active.
"""
document = make_doc(
"See [1].",
"[1] Evil\nURL: https://example.com/a)[x](javascript:alert(1))",
)
result = answer_of(CitationFormatter().format_document(document))
assert "https://example.com/a%29" in result
assert "](javascript:" not in result
def test_parens_that_balance_by_count_but_not_in_order_are_encoded(self):
"""Counting parens is not the same as balancing them. ``a)b(c`` has one
of each and still ends the destination at the ``)``, so a rule that only
compared totals would hand the attacker the same breakout with one
character added.
"""
document = make_doc(
"See [1].",
"[1] Evil\nURL: https://example.com/a)b(c",
)
result = answer_of(CitationFormatter().format_document(document))
assert "https://example.com/a%29b%28c" in result
def test_an_opener_that_is_never_closed_is_encoded(self):
"""The other direction. An unmatched ``(`` does not end the destination
early, it makes the link fail to parse at all, so the citation renders
as literal text with its URL exposed instead of as a link.
"""
document = make_doc(
"See [1].",
"[1] Truncated\nURL: https://example.com/a(b",
)
result = answer_of(CitationFormatter().format_document(document))
assert "https://example.com/a%28b" in result
def test_an_ipv6_literal_is_left_alone(self):
"""Second accept control: brackets delimit an IPv6 host and do not end
a markdown destination, so they must not be encoded.
"""
document = make_doc("See [1].", "[1] Local\nURL: http://[::1]/x")
result = answer_of(CitationFormatter().format_document(document))
assert "](http://[::1]/x)" in result
@pytest.mark.parametrize("mode", UNGATED_LINK_MODES, ids=lambda m: m.value)
@pytest.mark.parametrize(
"url",
[
pytest.param("https://good.example/q?b=1&c=2", id="amp"),
pytest.param(
"https://good.example/q?b=1&c=2©=3", id="bare ampersands"
),
pytest.param("https://good.example/q?q=x<y", id="lt"),
],
)
def test_a_query_string_with_character_references_stays_linkable(
self, mode, url
):
"""Accept control for the decoded check. Each of these decodes to
another ``https`` URL, so it is linked, and linked as written: encoding
``&`` to ``%26`` would merge every multi-parameter query into one.
"""
for result in cite_through_both_entry_points(mode, url):
assert f"]({url})" in result
@pytest.mark.parametrize("mode", UNGATED_LINK_MODES, ids=lambda m: m.value)
@pytest.mark.parametrize(
"url",
[
pytest.param("//evil.example/x", id="two slashes"),
pytest.param("///evil.example/x", id="three slashes"),
pytest.param("/\evil.example/x", id="encoded backslash"),
pytest.param("//evil.example/x", id="encoded slashes"),
pytest.param(" //evil.example/x", id="encoded leading space"),
],
)
def test_a_scheme_less_destination_cannot_name_a_host(self, mode, url):
"""A relative URL stays linkable because a library document is cited
as ``/library/...``. On an http(s) page a browser reads all of these
as a link to ``evil.example`` instead, and a report opened from disk
resolves the two-slash and backslash forms to
``file://evil.example/x``. ``urlsplit`` gives the three-slash form an
empty netloc, which is why the rule is not a netloc check.
"""
for result in cite_through_both_entry_points(mode, url):
assert "evil.example" not in result
def test_relative_url_still_produces_a_readable_label(self):
"""Control: a benign non-http URL must not be dropped silently."""
document = make_doc(
"See [1].", "[1] Internal Handbook\nURL: /library/document/7"
)
result = answer_of(
CitationFormatter(CitationMode.DOMAIN_HYPERLINKS).format_document(
document
)
)
assert "[[internal-handbook]](/library/document/7)" in result
class TestSourceLabelFidelity:
"""The visible citation label must not misname the host."""
@pytest.mark.parametrize(
"url,impostor",
[
("https://arxiv.org.evil.example/paper", "arxiv.org"),
("https://not-arxiv.org.attacker.test/p", "arxiv.org"),
("https://evil-github.com/x", "github.com"),
("https://youtube.com.phish.test/v", "youtube.com"),
# A registrable name that merely ends in the known one, with no
# dot in between, one per known domain.
("https://notarxiv.org/abs/1", "arxiv.org"),
("https://evilreddit.com/r/x", "reddit.com"),
("https://not-youtube.com/v", "youtube.com"),
("https://mypypi.org/project/x", "pypi.org"),
("https://fakemilvus.io/docs", "milvus.io"),
("https://fakemedium.com/a", "medium.com"),
],
)
def test_lookalike_host_is_not_labelled_with_the_known_domain(
self, url, impostor
):
assert CitationFormatter()._extract_domain(url) != impostor
@pytest.mark.parametrize(
"url,host",
[
("https://[email protected]/abs/1", "evil.example"),
("https://user:[email protected]/report", "example.com"),
],
)
def test_userinfo_does_not_name_the_host(self, url, host):
"""The label names the host the link reaches, and no credentials."""
assert CitationFormatter()._extract_domain(url) == host
@pytest.mark.parametrize(
"url",
[
"https://evil.example\\@arxiv.org/abs/1",
"https://evil.example\\.arxiv.org/abs/1",
"https://arxiv.org:443.evil.example/abs/1",
],
)
def test_an_authority_the_http_stack_reads_differently_names_where_it_goes(
self, url
):
"""Browsers and requests end the authority at a backslash, and a port
that is not a number hides the rest of it from urllib's hostname."""
assert CitationFormatter()._extract_domain(url) == "evil.example"
def test_a_lookalike_citation_renders_its_real_host(self):
document = make_doc(
"See [1].", "[1] Paper\nURL: https://arxiv.org.evil.example/p"
)
answer = answer_of(
CitationFormatter(CitationMode.DOMAIN_HYPERLINKS).format_document(
document
)
)
assert "See [[evil.example]](https://arxiv.org.evil.example/p)." in (
answer
)
@pytest.mark.parametrize(
"url",
[
"https://export.arxiv.org/abs/2401.12345",
"https://arxiv.org:443/abs/2401.12345",
"https://ArXiv.org/abs/2401.12345",
"https://arxiv.org./abs/2401.12345",
],
)
def test_a_subdomain_port_capital_or_root_dot_keeps_the_known_label(
self, url
):
assert CitationFormatter()._extract_domain(url) == "arxiv.org"
@pytest.mark.parametrize(
"url,label",
[
# A port requests refuses earns no known label, but the label
# still leaves it out, as main did for known domains.
("https://arxiv.org:99999/abs/1", "arxiv.org"),
("https://arxiv.org:+443/abs/1", "arxiv.org"),
("https://example.com:8080/report", "example.com:8080"),
("http://[::1]:8080/x", "[::1]:8080"),
("http://[::1]/x", "[::1]"),
],
)
def test_port_labels_preserve_compatibility(self, url, label):
assert CitationFormatter()._extract_domain(url) == label
@pytest.mark.parametrize(
"known",
[
"arxiv.org",
"github.com",
"reddit.com",
"youtube.com",
"pypi.org",
"milvus.io",
"medium.com",
],
)
def test_a_root_dot_keeps_every_known_label(self, known):
# Every entry, not just arxiv.org: stripping more than the dot
# (rstrip(".o")) leaves "arxiv.org" whole but turns milvus.io. into io.
assert (
CitationFormatter()._extract_domain(f"https://{known}./x") == known
)
def test_arxiv_in_the_path_does_not_earn_its_tag(self):
"""Regression control for the arXiv half of the defect below.
``URLClassifier.classify`` used to detect arXiv with a bare
``re.search`` over the whole URL, so an attacker-controlled path
segment (``evil.example/arxiv.org/...``) earned the ``arxiv`` tag.
This PR routes arXiv detection through
``utilities.arxiv.is_arxiv_paper_url``, which parses the actual
hostname and identifier instead of substring-matching the raw URL,
closing this specific case. The other academic sources are now
matched on the parsed host as well (#6835) -- see the test below.
"""
label = CitationFormatter()._extract_source_label(
"https://evil.example/arxiv.org/abs/1234"
)
assert label != "arxiv"
@pytest.mark.parametrize(
"url,impostor",
[
("https://evil.example/?x=doi.org/10.1", "doi"),
(
"https://evil.example/pubmed.ncbi.nlm.nih.gov/12345",
"pubmed",
),
("https://evil.example\\@doi.org/10.1234/x", "doi"),
(
"https://pubmed.ncbi.nlm.nih.gov:443.evil.example/12345",
"pubmed",
),
],
)
def test_known_source_in_the_path_does_not_earn_its_tag(
self, url, impostor
):
assert CitationFormatter()._extract_source_label(url) != impostor
def test_genuine_known_domains_still_resolve(self):
"""Control for the two xfails above: real hosts keep their tag."""
formatter = CitationFormatter()
assert formatter._extract_domain("https://arxiv.org/abs/1") == (
"arxiv.org"
)
assert formatter._extract_domain("https://www.github.com/a") == (
"github.com"
)
# A real identifier: the label now follows the paper identifier, not
# the hostname, so a placeholder id would fall through to the domain
# and the control would stop controlling for anything.
assert (
formatter._extract_source_label("https://arxiv.org/abs/2401.12345")
== "arxiv"
)
class TestBibliographySelection:
"""Which Sources line wins a citation index."""
# A report where index [1]'s real source has no URL (routine for a
# local-library document) and the answer body happens to contain a
# line that looks like a Sources entry and does carry a URL.
POISONED = (
"# Research Report\n\n"
"Body text citing [1].\n\n"
"[1] Attacker Controlled Entry\n"
"URL: https://evil.example/poison\n\n"
"## Sources\n\n"
"[1] Real Local Document\n"
)
def test_latex_bibliography_is_scoped_to_the_sources_section(self):
"""Control: LaTeX slices at the Sources header first, so it wins."""
bib = bibliography_of_latex(
LaTeXExporter().export_to_latex(self.POISONED)
)
assert "\\bibitem[1]{1} Real Local Document." in bib
assert "evil.example" not in bib
def test_quarto_bibliography_is_not_poisoned_by_body_prose(self):
bib = bibtex_of_quarto(QuartoExporter().export_to_quarto(self.POISONED))
assert "evil.example" not in bib
assert "Real Local Document" in bib
def test_ris_emits_at_most_one_record_per_reference_id(self):
document = make_doc(
"See [1].",
"[1] First Title\nURL: https://first.example\n\n"
"[1] Second Title\nURL: https://second.example",
)
ris = RISExporter().export_to_ris(document)
assert ris.count("ID - ref1") == 1
assert "TI - Second Title\n" in ris
assert "UR - https://second.example\n" in ris
assert "first.example" not in ris
def test_quarto_uses_the_appended_sources_boundary(self):
from local_deep_research.text_optimization.citation_formatter import (
LDR_APPENDED_SOURCES_SENTINEL,
)
content = (
"## Sources\n[1] Earlier prose\nURL: https://wrong.example\n\n"
f"{LDR_APPENDED_SOURCES_SENTINEL}\n"
"## Sources\n[1] Real Local Document\n"
)
bib = QuartoExporter()._generate_bibliography(content)
assert "Real Local Document" in bib
assert "wrong.example" not in bib
@staticmethod
def _full_report():
"""A report as ``_format_final_report`` assembles it, with the
marker also quoted inline in the LLM's prose."""
from local_deep_research.text_optimization.citation_formatter import (
LDR_APPENDED_SOURCES_SENTINEL,
)
prose = (
"# Research Report\n\n"
"Body text citing [1]. The tool marks sources with "
f"{LDR_APPENDED_SOURCES_SENTINEL} internally.\n\n"
"[1] Attacker Controlled Entry\n"
"URL: https://evil.example/poison"
)
return (
prose
+ f"\n\n{LDR_APPENDED_SOURCES_SENTINEL}\n\n"
+ "## Sources\n\n[1] Real Local Document\n"
+ "URL: https://real.example/doc\n"
)
@staticmethod
def _quarto_body(qmd):
front_end = qmd.index("\n---\n\n", 3) + len("\n---\n\n")
note = qmd.rindex("\n\n**Bibliography File Required**")
return qmd[:front_end], qmd[front_end:note]
def test_quarto_drops_the_appended_sources_sentinel(self):
"""The trusted sentinel line is the generator's markup, not report
text: it is removed rather than defused into visible text, while
the bibliography is still scoped by it and a copy quoted in the
prose is still broken like any other comment."""
qmd = QuartoExporter().export_to_quarto(self._full_report())
_front, body = self._quarto_body(qmd)
assert body.count("LDR_APPENDED_SOURCES_START") == 1, body
assert "internally" in body.split("LDR_APPENDED_SOURCES_START")[1]
assert "<!--" not in body
assert "\n\n## Sources\n\n[@ref1] Real Local Document" in body
bib = bibtex_of_quarto(qmd)
assert "Real Local Document" in bib
assert "evil.example" not in bib
def test_pandoc_shows_no_sentinel_text_in_the_quarto_body(self):
"""Read the exported body with the file's own ``from`` (raw HTML
off), as Quarto does: no paragraph is left holding the sentinel."""
pypandoc = pytest.importorskip("pypandoc")
front, body = self._quarto_body(
QuartoExporter().export_to_quarto(self._full_report())
)
line = next(x for x in front.splitlines() if x.startswith("from:"))
reader = line[len("from:") :].strip()
document = json.loads(
pypandoc.convert_text(body, "json", format=reader)
)
holding = [
block
for block in document["blocks"]
if "LDR_APPENDED_SOURCES_START" in json.dumps(block)
]
# Only the copy quoted in the prose, inside its own sentence.
assert len(holding) == 1, holding
assert "internally" in json.dumps(holding[0])
class TestRegexBounds:
"""Hostile input must not drive a pattern super-linear.
Inputs are deliberately small (10-40 KB). The formerly quadratic
parsers take seconds at these sizes, while a linear implementation
stays comfortably within the bounds below.
These synchronous scans use calling-thread CPU time so scheduling
delays and unrelated background threads do not consume their budgets.
The global pytest timeout still bounds wall time.
"""
def test_pathological_bracket_input_stays_fast(self):
pathological = [
"[" * 200 + "1" + "]" * 200,
"[1][2][3][4][5]" * 200,
"[" + ",".join(["1"] * 2000) + "X",
"\u3010" * 500 + "1" + "\u3011" * 500,
"[1]" + " " * 20000,
]
sources = (
"[1] Alpha\nURL: https://alpha.example/a\n\n"
"[2] Beta\nURL: https://beta.example/b"
)
formatter = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS)
for chunk in pathological:
started = time.thread_time()
formatter.format_document(make_doc(chunk, sources))
elapsed = time.thread_time() - started
assert elapsed < 2.0, f"input {chunk[:24]!r}... took {elapsed:.2f}s"
def test_ris_author_split_is_not_quadratic_in_whitespace(self):
run = " " * 10000
sources_block = (
f"## Sources\n\n[1] Study by A{run}B.\nURL: https://x.example/p\n"
)
started = time.thread_time()
RISExporter().export_to_ris(sources_block)
elapsed = time.thread_time() - started
# A linear split handles 10 KB in well under a millisecond; the
# 0.25s budget leaves ~6x headroom over the measured 1.6s so a
# fast machine cannot XPASS this by accident.
assert elapsed < 0.25, (
f"author split took {elapsed:.2f}s on a 10KB whitespace run"
)
@pytest.mark.parametrize(
"title",
[
# The run sits outside any "by ..." span, so the author cut
# does not remove it before the title-clause cleanup runs.
"A{run}B",
"A{run}B by C.",
"A.{run}B",
"A{run}\t{run}B",
"A{run}DOI",
"A{run}Published i",
"A{run}Volum",
"A{run}Page",
"A{run}Volume 3",
"A{run}Pages 4",
],
ids=[
"plain",
"before-by",
"after-period",
"tabs",
"near-doi",
"near-published",
"near-volume",
"near-pages",
"volume",
"pages",
],
)
def test_ris_title_cleanup_is_not_quadratic_in_whitespace(self, title):
run = " " * 20000
sources_block = (
"## Sources\n\n[1] "
+ title.format(run=run)
+ "\nURL: https://x.example/p\n"
)
started = time.thread_time()
RISExporter().export_to_ris(sources_block)
elapsed = time.thread_time() - started
# Linear cleanup takes a few milliseconds here. Each of the four
# former ``\s*``-led substitutions took about 2s at 20k.
assert elapsed < 0.25, (
f"title cleanup took {elapsed:.2f}s on a 20KB whitespace run"
)
@pytest.mark.parametrize(
("title", "expected"),
[
("Deep Learning DOI: 10.1/x Tail", "Deep Learning Tail"),
("Deep Learning Published in Nature", "Deep Learning"),
("Audio \t Volume 3 Pages 1-9", "Audio"),
("Title doi:10.1/a DOI:10.2/b end", "Title end"),
("Mixed DOI:10.1/a", "Mixed"),
("Plain Title", "Plain Title"),
],
)
def test_ris_title_cleanup_output(self, title, expected):
"""Control: the linear cleanup cuts what the old regexes cut."""
entry = RISExporter()._create_ris_entry("1", title)
assert f"TI - {expected}" in entry.splitlines()
@pytest.mark.parametrize(
"line",
["```{rRUN", "```{rRUNx", " ````{pythonRUN, echo"],
ids=["run-to-end", "run-then-text", "indented-options"],
)
def test_quarto_cell_fence_scan_is_not_quadratic_in_spaces(self, line):
body = "# Report\n\n" + line.replace("RUN", " " * 10000) + "\n\nText.\n"
started = time.thread_time()
QuartoExporter().export_to_quarto(body, "Report")
elapsed = time.thread_time() - started
# The former `` *[ ,].*`` options group took 2.5-3.5s here.
assert elapsed < 0.25, (
f"cell-fence scan took {elapsed:.2f}s on a 10KB run of spaces"
)
def test_quarto_cell_fence_with_spaced_options_is_still_defused(self):
"""Control: an options group led by several spaces still matches."""
body = "# Report\n\n```{python , echo=false}\nprint(1)\n```\n"
output = QuartoExporter().export_to_quarto(body, "Report")
assert "```{python" not in output
assert "```python" in output
@pytest.mark.parametrize(
"unit",
[" ```{a}\n", "\t````{r, echo=false}\n", " ```{python}\ncode\n"],
ids=["indented-cells", "tab-indented-options", "indented-with-code"],
)
def test_quarto_cell_defusion_is_not_quadratic_in_indented_cells(
self, unit
):
# Each demoted indented cell stops shielding the line after it, so
# the former re-read-until-stable loop needed one full pass per
# cell: 8000 cells took 20-50s.
body = "# Report\n\n" + unit * 8000 + "\nText.\n"
started = time.thread_time()
output = QuartoExporter().export_to_quarto(body, "Report")
elapsed = time.thread_time() - started
assert "{a}" not in output
assert "{r" not in output
assert "{python" not in output
assert elapsed < 1.0, (
f"cell defusion took {elapsed:.2f}s on 8000 indented cells"
)
@pytest.mark.parametrize(
("body", "expected"),
[
# A demoted indented cell is prose, so the next tag is live.
(" ```{r}\n```{python}\nx\n```", " ```r\n```python\nx\n```"),
# Fence state is not modelled: a cell inside a cell is demoted.
("```{r}\n```{python}\n```", "```r\n```python\n```"),
(
" ```{r}\n````\n```\n```{python}\n````\n```",
" ```r\n````\n```\n```python\n````\n```",
),
# Quoted as an example inside a longer fence: demoted as well.
# Quarto's engine detection and knitr do not track display
# fences, and the Jupyter engine's reader lets a cell open
# inside one, so a quoted header is live to them.
("````\n```{r}\n````\n", "````\n```r\n````\n"),
(
"````markdown\n```{r}\nsummary(cars)\n```\n````",
"````markdown\n```r\nsummary(cars)\n```\n````",
),
(
"```\n```{python}\nimport os; os.system('id')\n```\n```",
"```\n```python\nimport os; os.system('id')\n```\n```",
),
(
"```\n ```{python}\nprint(1)\n ```\n```",
"```\n ```python\nprint(1)\n ```\n```",
),
# knitr and engine detection accept a ``>`` blockquote prefix.
(
"> ```{r}\n> system('id')\n> ```",
"> ```r\n> system('id')\n> ```",
),
(">> \t```{r, echo=FALSE}\n", ">> \t```r, echo=FALSE\n"),
# A bare \r ending a prose line is a break to Quarto but not
# one every reader shares; demoting regardless covers both.
(
"Some prose.\r```\n```{python}\nimport os\n```\n",
"Some prose.\r```\n```python\nimport os\n```\n",
),
# Engine detection's ``\s*`` and ``[^}]*`` span line breaks, so
# a header split across lines is broken at its brace.
("```\r{python}\nimport os\n```", "```\rpython}\nimport os\n```"),
(
"```\n{r}\n```\n`r system('id')`",
"```\nr}\n```\n`r system('id')`",
),
("x
```{r}
y", "x
```r}
y"),
# Inside prose, too.
("use ```{r}``` here", "use ```r}``` here"),
],
ids=[
"unshielded",
"cell-in-cell",
"demoted-once-stays",
"quoted-in-four-ticks",
"verbatim-rmd",
"python-in-display-fence",
"indented-in-display-fence",
"blockquote",
"nested-blockquote",
"lone-cr-prose",
"lone-cr-split-header",
"newline-split-header",
"u2028-split-header",
"mid-line",
],
)
def test_quarto_cell_defusion_ignores_fence_state(self, body, expected):
assert _defuse_quarto_cells(body) == expected
@pytest.mark.parametrize(
("body", "expected"),
[
("```\ufeff{python}\nx\n```", "```python\nx\n```"),
("\ufeff```{python}\nx\n```", "\ufeff```python\nx\n```"),
("```{python}\ufeff\nx\n```", "```python\nx\n```"),
(
"```\n\x1c```\n```\n```{r}\nx\n```",
"```\n\x1c```\n```\n```r\nx\n```",
),
(
"```\n```\x85\n```\n```{r}\nx\n```",
"```\n```\x85\n```\n```r\nx\n```",
),
("x\r```{python}\nx\n```", "x\r```python\nx\n```"),
("```\r```\r```{r}\r\nx\r\n```", "```\r```\r```r\r\nx\r\n```"),
],
ids=[
"bom-before-brace",
"bom-indent",
"bom-trailing",
"x1c-line",
"x85-line",
"lone-cr-line",
"lone-cr-closer",
],
)
def test_quarto_cell_defusion_reads_any_whitespace_and_line_break(
self, body, expected
):
assert _defuse_quarto_cells(body) == expected
@pytest.mark.parametrize(
"body",
["```{.python}\nx\n```", "```{#id}\nx\n```", "```{{python}}\nx\n```"],
ids=["pandoc-class", "pandoc-id", "escaped-cell"],
)
def test_quarto_defusion_leaves_non_cells_alone(self, body):
"""Control: no reader runs these, so they stay as written."""
assert _defuse_quarto_cells(body) == body
def test_quarto_cell_whitespace_is_every_readers(self):
"""The whitespace class is Python's ``\\s`` plus JavaScript's.
JavaScript's ``\\s`` (ECMAScript WhiteSpace and LineTerminator) adds
U+FEFF to Python's, Python's adds \\x1c-\\x1f and \\x85, and U+180E
was a space before Unicode 6.3. A character either reader skips must
not shield a header in any of the three positions.
"""
spaces = {
*(chr(c) for c in range(0x10000) if chr(c).isspace()),
"\ufeff",
"\u180e",
}
for char in sorted(spaces):
for body in (
f"{char}```{char}{{r}}{char}\nx\n```",
f"```{char}{{r}}\nx\n```",
f"{char}```{{r, echo=FALSE}}{char}",
):
assert "{r" not in _defuse_quarto_cells(body), hex(ord(char))
def test_quarto_defusion_output_matches_no_reader(self):
"""Fuzz: no reader that decides execution sees a cell header.
The regexes are the upstream ones, transcribed with ``\\s`` widened
to the union above and every line splitting a reader could use:
* quarto-cli ``languagesWithClasses`` (pandoc-partition.ts), /gm over
the whole text: ``^[\\t >]*```+\\s*\\{([a-zA-Z][a-zA-Z0-9_.]*)``
``([^}]*)?\\}\\s*$``
* quarto-cli ``breakQuartoMd`` (break-quarto-md.ts):
``^\\s*(```+)\\s*\\{([=A-Za-z][=A-Za-z0-9._]*)( *[ ,].*)?\\}\\s*$``
* quarto-cli ``quartoMdToJupyter`` (jupyter.ts):
``^(\\s*)(```+)\\s*\\{<language>( *[ ,].*)?\\}\\s*$``
* knitr md ``chunk.begin`` (R/pattern.R):
``^[\\t >]*```+\\s*\\{([a-zA-Z0-9_]+( *[ ,].*)?)\\}\\s*$``
"""
space = r"[\s\ufeff\u180e]"
ends = "\n\r
"
engine_detection = re.compile(
rf"(?:^|(?<=[{ends}]))[\t >]*`{{3,}}{space}*"
rf"\{{[a-zA-Z][a-zA-Z0-9_.]*[^}}]*\}}{space}*(?=$|[{ends}])"
)
line_readers = [
re.compile(
rf"^{space}*`{{3,}}{space}*\{{[=A-Za-z][=A-Za-z0-9._]*"
rf"( *[ ,].*)?\}}{space}*$"
),
re.compile(
rf"^{space}*`{{3,}}{space}*\{{[a-z0-9_]+( *[ ,].*)?\}}{space}*$",
re.IGNORECASE,
),
re.compile(
rf"^[\t >]*`{{3,}}{space}*\{{[a-zA-Z0-9_]+( *[ ,].*)?\}}"
rf"{space}*$"
),
]
splits = [r"\r\n?|\n", r"\r?\n", r"\n", "\r\n|[\n\r
\x85]"]
def live(text):
if engine_detection.search(text):
return True
return any(
reader.match(line)
for split in splits
for line in re.split(split, text)
for reader in line_readers
)
rng = random.Random(6982)
spacing = ["", " ", "\t", "\ufeff", "\x1c", "\x85", "
"]
spacing += ["\xa0", "\u180e", "> ", "\r", "\n"]
langs = ["r", "python", "=html", ".r", "R", "r-x", "1r", "é", "ojs"]
rests = ["", " echo=FALSE", ', x="`}`"', "}", " ", "x"]
lines = ["```", "````", " ```", "```python", "~~~", "---", "text"]
lines += ["", "`r x`", "```{{python}}", "```{.python}", "{r}"]
breaks = ["\n", "\n", "\r\n", "\r", "
"]
for _ in range(3000):
body = ""
for _ in range(rng.randint(0, 20)):
if rng.random() < 0.55:
line = (
f"{rng.choice(spacing)}{'`' * rng.choice([3, 4])}"
f"{rng.choice(spacing)}{{{rng.choice(langs)}"
f"{rng.choice(rests)}}}{rng.choice(spacing)}"
)
else:
line = rng.choice(lines)
body += line + rng.choice(breaks)
output = _defuse_quarto_cells(body)
assert not live(output), repr(body)
assert _defuse_quarto_cells(output) == output, repr(body)
@pytest.mark.parametrize(
("body", "expected"),
[
# Prose that starts with a fence and ends in ``}`` is no header
# to any reader. Only the opening brace goes; the text stays.
(
"```{r}``` chunks run R, see {Figure 1}",
"```r}``` chunks run R, see {Figure 1}",
),
(
"```{python}: Quarto cell syntax {docs}",
"```python}: Quarto cell syntax {docs}",
),
# A real header keeps its options; only the braces go.
(
"```{python, echo=false}\nx\n```",
"```python, echo=false\nx\n```",
),
("```{r, x='}'}\nx\n```", "```r, x='}'}\nx\n```"),
],
ids=["prose-code-span", "prose-colon", "options", "brace-in-options"],
)
def test_quarto_cell_demotion_drops_no_text(self, body, expected):
assert _defuse_quarto_cells(body) == expected
@staticmethod
def _qmd_body(body, title="Report"):
qmd = QuartoExporter().export_to_quarto(body, title)
front_end = qmd.index("\n---\n\n", 3) + len("\n---\n\n")
note = qmd.rindex("\n\n**Bibliography File Required**")
return qmd[:front_end], qmd[front_end:note]
@pytest.mark.parametrize(
"body",
[
"{{< include ../../../usr/lib/R/x/skeleton.Rmd >}}\n`r system('id')`",
"Key: {{< env OPENAI_API_KEY >}}",
"{{< meta title >}} and {{<meta x>}}",
"{{% env OPENAI_API_KEY %}}",
"{{ < env OPENAI_API_KEY >}}",
"{{{< env OPENAI_API_KEY >}}}",
"{{</* include x.qmd */>}}",
"```\n{{< include x.qmd >}}\n```",
],
ids=[
"include",
"env",
"meta",
"percent",
"spaced",
"escaped",
"comment-escaped",
"in-code-block",
],
)
def test_quarto_shortcodes_in_the_body_are_broken(self, body):
"""quarto-cli opens a shortcode at ``{{<`` (lpegshortcode.lua) and
expands includes before it picks the engine, so an include can bring
in a file with cells and ``env`` prints the renderer's secrets."""
_front, rendered = self._qmd_body(body)
assert re.search(r"\{\s*\{\s*[<%]", rendered) is None, rendered
# The text stays readable: only a zero-width space was inserted.
assert rendered.replace("\u200b", "") == body
def test_quarto_shortcode_in_the_title_is_broken(self):
"""Quarto expands shortcodes in metadata, the title included."""
front, _body = self._qmd_body(
"Body.", title="{{< env OPENAI_API_KEY >}}"
)
assert "{{<" not in front
line = next(x for x in front.splitlines() if x.startswith("title"))
value = line[len('title: "') : -1].replace("\\\\", "\\")
assert self._pandoc_decode(value) == "{\u200b{< env OPENAI_API_KEY >}}"
# What still decodes body text once the reader format has switched off
# raw content and attributes (quarto-cli v1.10.18): lpegshortcode.lua's
# ``wrap_lpeg_match`` restores ``cf5733e5-...&#x<hex>;`` after its
# whole-file pass, Pandoc then decodes entities and backslash escapes
# once (text, link targets, image sources; not code or math), and
# readqmd.lua hex-decodes ``b58fc729-...;<hex>;`` into a shortcode.
_SENTINEL = "b58fc729-690b-4000-b19f-365a4093b2ff"
_UNICODE_SENTINEL = "cf5733e5-0370-4aae-8689-61bad1dd9ec0"
_ENV_HEX = "7B7B3C20656E76204F50454E41495F4150495F4B4559203E7D7D"
@staticmethod
def _pandoc_decode(text, entities=True):
"""Pandoc's markdown decoding: ``escapedChar'`` with
all_symbols_escapable (a backslash before any character that is not
alphanumeric or a line break) and ``characterReference`` (``&``,
then non-space characters up to the first ``;`` after the first
one, looked up as an entity)."""
out = []
index = 0
while index < len(text):
char = text[index]
following = text[index + 1 : index + 2]
if (
char == "\\"
and following
and following not in "\n\r"
and not following.isalnum()
):
out.append(following)
index += 2
continue
if char == "&" and entities:
end = index + 2
while end < len(text) and text[end] != ";":
if text[end] in " \t\n\r":
break
end += 1
if (
end < len(text)
and text[end] == ";"
and text[index + 1] not in " \t\n\r"
):
value = html.unescape(text[index : end + 1])
if value != text[index : end + 1]:
out.append(value)
index = end + 1
continue
out.append(char)
index += 1
return "".join(out)
@classmethod
def _unescape_unicode(cls, text):
"""lpegshortcode.lua ``unescape_unicode``."""
return re.sub(
re.escape(cls._UNICODE_SENTINEL) + "&#x([0-9a-fA-F]+);",
lambda m: chr(int(m.group(1), 16)),
text,
)
@classmethod
def _decodings(cls, text):
"""*text* as each shortcode finder sees it: the raw file, the file
after ``unescape_unicode`` (code, code blocks and math), and that
decoded once by Pandoc or by its escapes alone (text, link targets
and image sources)."""
restored = cls._unescape_unicode(text)
return {
text,
restored,
cls._pandoc_decode(restored),
cls._pandoc_decode(restored, entities=False),
}
@classmethod
def _expanded_by_quarto(cls, text):
"""A decoding of *text* holding a shortcode opener or encoding."""
opener = re.compile(r"\{\{[\s\ufeff\u180e]*[<%]")
for decoded in cls._decodings(text):
if (
opener.search(decoded)
or cls._SENTINEL in decoded.lower()
or "quarto-shortcode__" in decoded.lower()
):
return decoded
return None
@pytest.mark.parametrize(
"body",
[
# readqmd.lua hex sentinel; the hex is {{< env OPENAI_API_KEY >}}.
f"Key: {_SENTINEL};{_ENV_HEX}; end",
f"[x](https://e.example/{_SENTINEL};{_ENV_HEX};)",
f"`{_SENTINEL};{_ENV_HEX};`",
f"b58fc729-690b\\-4000-b19f-365a4093b2ff;{_ENV_HEX};",
f"b58fc729-690b-4000-b19f-365a4093b2ff;{_ENV_HEX};",
f"B58FC729-690b-4000-b19f-365a4093b2ff;{_ENV_HEX};",
# lpegshortcode.lua restores the unicode escape after its pass:
# code, a fence, math, prose and a link target.
f"`{{{{{_UNICODE_SENTINEL}< env OPENAI_API_KEY >}}}}`",
f"```\n{{{{{_UNICODE_SENTINEL}< env HOME >}}}}\n```",
f"${{{{{_UNICODE_SENTINEL}< meta title >}}}}$",
f"{{{{{_UNICODE_SENTINEL}< env X >}}}}",
f"[x](https://e.example/{{{{{_UNICODE_SENTINEL}<)",
f"{_UNICODE_SENTINEL}{{_UNICODE_SENTINEL}{< env X >}}}}",
# Pandoc decodes escapes and entities in text and link targets.
"[x](https://e.example/\\{\\{<)",
"[x](https://e.example/{\\{\\<)",
"{{< env X >}}",
"{{< env X >}}",
"{{< env X >}}",
"{{ < env X >}}",
"{{< env X >}}",
"{{< env X >}}",
"&{{< env X >}}",
# The class ast/parse.lua turns into a shortcode.
"quarto-shortcode__",
"quarto\\-shortcode\\__",
],
)
def test_quarto_shortcodes_are_broken_after_any_decoding(self, body):
"""Breaking only the literal ``{{<`` left the env shortcode
reachable through every other spelling a reader decodes."""
assert self._expanded_by_quarto(body) is not None, "control"
_front, rendered = self._qmd_body(body)
assert self._expanded_by_quarto(rendered) is None, rendered
# Only the shortcode pass promises to insert zero-width spaces
# without other edits. The full exporter may also escape unsafe
# math; keep checking its shortcode safety above.
assert _defuse_quarto_body(body).replace("\u200b", "") == body
# Each of these needs a reader the exported file switches off, named
# with the payload: attribute values Pandoc decodes and shortcodes.lua
# expands, a callout title read twice, ``qmd``/``qmd-base64`` and
# ``pandoc-*`` raw formats (normalize/extractquartodom.lua), raw LaTeX
# ``\QuartoMarkdownBase64{}`` and raw HTML attributes.
@pytest.mark.parametrize(
("body", "extensions"),
[
(
'[x]{title="{\\{< env OPENAI_API_KEY >}}"}',
["bracketed_spans"],
),
("[x]{data-k=\\{\\{<}", ["bracketed_spans"]),
(
'::: {data-k="{\\{< meta title >}}"}\nx\n:::',
["fenced_divs"],
),
(
"## H {data-k='{{< env HOME >}}'}",
["header_attributes"],
),
('[x](y){k="{{ < env X >}}"}', ["link_attributes"]),
('{k="{{< env X >}}"}', ["link_attributes"]),
('`x`{k="{{< env X >}}"}', ["inline_code_attributes"]),
(
'```{.x k="{{< env X >}}"}\nx\n```',
["fenced_code_attributes"],
),
(
'::: {title="&#123;&#123;< env X >}}" .callout-note}'
"\nx\n:::",
["fenced_divs"],
),
('[x]{qmd-base64="e3s8IGVudiBYID59fQ=="}', ["bracketed_spans"]),
(
'::: {qmd="{{< env X >}}"}\nx\n:::',
["fenced_divs"],
),
(
'`[Str "x"]`{=pandoc-native}',
["raw_attribute"],
),
(
'~~~{=pandoc-reader-html}\n<span class="quarto-'
'shortcode__">x</span>\n~~~',
["raw_attribute"],
),
(
"\\QuartoMarkdownBase64{e3s8IGVudiBYID59fQ==}",
["raw_tex"],
),
(
'<span title="{{< env X >}}">x</span>',
["raw_html", "native_spans"],
),
(
'<div qmd-base64="e3s8IGVudiBYID59fQ==">x</div>',
["raw_html", "native_divs"],
),
(
"+--------------+\n| --- |\n| engine: x |\n"
"| --- |\n+--------------+",
["yaml_metadata_block"],
),
],
)
def test_quarto_reader_format_switches_off_decoding_readers(
self, body, extensions
):
"""Modelling these readers one spelling at a time kept missing
one, so the exported file reads with Pandoc's markdown minus every
extension that makes an attribute, a raw block or inline, a Div or
a Span. Quarto passes ``from`` to readqmd.lua as
``user-defined-from``, for the document and every re-read."""
front, rendered = self._qmd_body(body)
line = next(x for x in front.splitlines() if x.startswith("from:"))
reader = line[len("from:") :].strip()
# Quarto's ``parse_format`` (_format.lua) reads the whole string.
assert re.fullmatch(r"markdown(?:-[a-z_]+)+", reader), reader
disabled = set(reader.split("-")[1:])
assert set(extensions) <= disabled, (body, reader)
assert rendered.replace("\u200b", "") == body
def test_quarto_title_shortcodes_are_broken_after_any_decoding(self):
"""Pandoc reads the title as markdown, so it decodes it too."""
for title in (
"{{< env OPENAI_API_KEY >}}",
"\\{\\{< env OPENAI_API_KEY >}}",
f"{self._SENTINEL};{self._ENV_HEX};",
f"{{{{{self._UNICODE_SENTINEL}< env OPENAI_API_KEY >}}}}",
):
front, _body = self._qmd_body("Body.", title=title)
line = next(x for x in front.splitlines() if x.startswith("title"))
value = line[len('title: "') : -1].replace("\\\\", "\\")
assert self._expanded_by_quarto(value) is None, title
@pytest.mark.parametrize(
"payload",
[
f"{_SENTINEL};{_ENV_HEX};",
f"{_UNICODE_SENTINEL}{{_UNICODE_SENTINEL}{< env X",
],
)
def test_quarto_bibliography_note_breaks_the_sentinels(self, payload):
"""readqmd.lua hex-decodes the sentinel in CodeBlock text, the
whole-file pass restores the unicode escape, and shortcodes.lua
then expands the result, so the bibtex fence is a sink."""
qmd = QuartoExporter().export_to_quarto(
f"See [1].\n\n## Sources\n\n[1] {payload}\n"
f"URL: https://e.example/{payload}\n",
"Report",
)
bibtex = bibtex_of_quarto(qmd)
assert "\u200b" in bibtex
assert self._SENTINEL not in qmd
assert self._UNICODE_SENTINEL not in qmd
@pytest.mark.parametrize(
"body",
[
"Johnson & Johnson, R&D and Q&A",
"a b <tag> "q" — ' © &",
"{{ jinja }} and {{x}} and x^{{2}}",
"AT & T, 5 % off, R & D, Q&A, R&D, AT&T, S&P 500",
"$$\\frac{{\\partial f}}{{\\partial x}}$$",
'```c\nscanf("%d", &x); int *p = &arr[i];\n```',
"```cpp\nvoid f(std::string &name, int &&n);\n```",
"[the paper](https://scholar.google.com/scholar?q=llm&hl=en)",
"<https://example.com/a?x=1&y=2> and a&b;c",
],
)
def test_quarto_defusion_keeps_harmless_text(self, body):
_front, rendered = self._qmd_body(body)
assert rendered == body
_SOURCE_URLS = (
"https://scholar.google.com/scholar?q=llm&hl=en&as_sdt=0,5",
"https://www.youtube.com/watch?v=dQw4w9WgXcQ&t=10s",
"https://example.com/a?x=1&y=2&z=&w",
"https://example.com/search?q=a%26b&lang=en#results",
"https://example.com/page;jsessionid=ABC123?x=1&y=2",
"https://example.com/a?b=1&c=2;d=3&e",
"https://example.com/a?b=1&c=2&d=3",
"https://example.com/a?b=1<c&d=2&e=3",
"https://en.wikipedia.org/wiki/AT%26T#History",
"https://doi.org/10.1000/xyz123-456_789",
"https://arxiv.org/abs/2401.01234v2",
"https://example.com/a---",
"https://example.com/{x}?y={z}",
"https://example.com/path/to/%7B%7B%3C/?a=b&c",
"https://example.com/550e8400-e29b-41d4-a716-446655440000?a=1&b=2",
)
def test_quarto_sources_urls_round_trip(self):
"""The ``&`` rule used to insert a zero-width space after every
``&`` before a letter, so ``?q=a&hl=en`` reached the server as
``&%E2%80%8Bhl=en`` and every query link in Sources broke. A URL
must remain byte for byte through the shortcode pass. The full
exporter can escape punctuation, so compare that output after
Markdown escape and entity decoding."""
rng = random.Random(69822)
urls = list(self._SOURCE_URLS)
alphabet = (
"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
"-._~:/?#@!$&'()*+,=%"
)
# Without a ``;`` no ``&`` can start an entity. Brackets are left
# out: the exporter turns ``[5]`` into a citation wherever it is.
for _ in range(2000):
urls.append(
"https://e.example/"
+ "".join(rng.choice(alphabet) for _ in range(40))
)
sources = "".join(
f"[{index}] Source {index}\n URL: {url}\n"
for index, url in enumerate(urls, 1)
)
cited = " ".join(f"[{index}]" for index in range(1, len(urls) + 1))
_front, rendered = self._qmd_body(
f"Body {cited}.\n\n## Sources\n\n{sources}"
)
# A separate math guard can escape punctuation in a Sources line.
# Compare its decoded text, while retaining the byte-for-byte
# contract for the shortcode pass itself below.
decoded_lines = [
self._pandoc_decode(line) for line in rendered.splitlines()
]
lines = [
line[len(" URL: ") :]
for line in decoded_lines
if line.startswith(" URL: ")
]
assert lines == [self._pandoc_decode(url) for url in urls]
for url in urls:
link = f"[x]({url})"
assert _defuse_quarto_body(link) == link
assert self._pandoc_decode(
_defuse_quarto_body(url)
) == self._pandoc_decode(url)
def test_quarto_shortcode_defusion_differential(self):
"""Fuzz: bodies of brace, escape, entity and whitespace spellings
and of the internal encodings spelled out a character at a time in
every form, against the decoding chain above. The output is never
expanded, only zero-width spaces are inserted, and defusing again
changes nothing."""
rng = random.Random(69821)
def spell(char):
forms = [char, f"&#{ord(char)};", f"&#x{ord(char):x};"]
if not char.isalnum():
forms += ["\\" + char, "\\\\\\" + char]
forms.append(f"{self._UNICODE_SENTINEL}&#x{ord(char):x};")
forms += [
f"&{name}"
for name, value in html.entities.html5.items()
if value == char and name.endswith(";")
]
return rng.choice(forms)
pieces = ["{", "\\{", "\\\\{", "{", "{", "{"]
pieces += ["&#123;", "&lcub;", "<", "<", "%", "\\<"]
pieces += [" ", " ", "\\ ", "\n", "&", "\\", "#", ";", "x"]
pieces += ["&", "}", "\u200b", "b", "q", "c", "-", "_"]
pieces += [f"{self._UNICODE_SENTINEL}<", " ", "`", "$"]
expanded_before = 0
for _ in range(400):
body = ""
for _ in range(rng.randint(1, 8)):
roll = rng.random()
if roll < 0.1:
word = rng.choice([self._SENTINEL, "quarto-shortcode__"])
body += "".join(spell(char) for char in word)
elif roll < 0.25:
body += "".join(spell(char) for char in "{{<")
else:
body += rng.choice(pieces)
if self._expanded_by_quarto(body) is not None:
expanded_before += 1
output = _defuse_quarto_body(body)
assert self._expanded_by_quarto(output) is None, repr(body)
assert (
output.replace("\u200b", "") == body.replace("\u200b", "")
or "---" in body
)
assert _defuse_quarto_body(output) == output, repr(body)
assert expanded_before > 100
@staticmethod
def _quarto_yaml_blocks(qmd):
"""Every YAML block quarto-cli's ``readYamlFromMarkdown`` merges.
src/core/yaml.ts: newlines normalised by ``lines()``, HTML comments
and fenced code removed, then ``kRegExYAML`` with /gm (JavaScript's
``^``/``$`` also break at U+2028 and U+2029). Fenced code is kept
here, which only finds more blocks.
"""
text = re.sub(r"\r\n?", "\n", qmd)
text = re.sub(r"<!--[\W\w]*?-->", "", text)
text = text.replace("\u2028", "\n").replace("\u2029", "\n")
yaml_block = re.compile(
r"(^)(---[ \t]*[\r\n]+(?![ \t]*[\r\n]+)[\W\w]*?[\r\n]+"
r"(?:---|\.\.\.))([ \t]*)$",
re.MULTILINE,
)
return [m.group(2) for m in yaml_block.finditer(text)]
@pytest.mark.parametrize(
"body",
[
"Intro.\n\n---\nengine: knitr\n---\n\n`r system('id')`",
'---\npdf-engine-opt: "-shell-escape"\n...\n',
"```yaml\n---\nfilters: [x.lua]\n---\n```",
"> ---\n> engine: knitr\n> ---",
"x\r---\r\nengine: knitr\r---",
"x\u2028--- \u2028engine: knitr\u2028---",
"<!-- c -->---\nengine: knitr\n---",
"-<!-- -->--\nengine: knitr\n<!-- -->---",
],
ids=[
"plain",
"dot-closer",
"in-code-block",
"blockquote",
"cr-lines",
"u2028-lines",
"after-comment",
"assembled-by-comments",
],
)
def test_quarto_body_cannot_open_a_yaml_metadata_block(self, body):
"""quarto-cli merges every ``---`` block in the file into the render
metadata (src/core/yaml.ts, ``kRegExYAML`` /gm), so a block in the
body could set ``engine``, ``filters`` or ``pdf-engine-opt``."""
qmd = QuartoExporter().export_to_quarto(body, "Report")
blocks = self._quarto_yaml_blocks(qmd)
assert len(blocks) == 1, blocks
assert blocks[0].startswith("---\ntitle:")
@pytest.mark.parametrize(
"body",
[
"Intro.\n\n+--------------+\n| --- |\n"
"| injected: x |\n| --- |\n+--------------+\n",
"+-----+---------------+\n| a | --- |\n"
"| | injected: x |\n| | ... |\n"
"+-----+---------------+\n",
"- item\n\n ---\n injected: x\n ---\n",
"> ---\n> injected: x\n> ---\n",
"Text[^1].\n\n[^1]: Note.\n\n ---\n injected: x\n ---\n",
"Term\n\n: ---\n injected: x\n ---\n",
"Intro.\n\n---\ninjected: x\n---\n",
"% Injected title\n% Injected author\n\nBody.\n",
],
ids=[
"grid-table-cell",
"grid-table-second-cell",
"list-item",
"blockquote",
"footnote",
"definition",
"top-level",
"title-block",
],
)
def test_pandoc_reads_no_metadata_block_from_the_quarto_body(self, body):
"""Pandoc's yaml_metadata_block opens in any block context, a
grid-table cell included, where the ``---`` does not end the line,
and its pandoc_title_block at the start of the input. Quarto
removes the front matter and passes the metadata with
``--metadata-file`` (src/command/render/pandoc.ts), then reads the
rest with the file's ``from``; this runs the bundled Pandoc the
same way."""
pypandoc = pytest.importorskip("pypandoc")
front, rendered = self._qmd_body(body)
line = next(x for x in front.splitlines() if x.startswith("from:"))
reader = line[len("from:") :].strip()
control = json.loads(
pypandoc.convert_text(
body,
"json",
format=reader + "+yaml_metadata_block+pandoc_title_block",
)
)
assert control["meta"], "control: Pandoc reads a block from the body"
document = json.loads(
pypandoc.convert_text(rendered, "json", format=reader)
)
assert document["meta"] == {}, document["meta"]
@classmethod
def _inline_types(cls, node, found):
"""Every Pandoc JSON node type (``"t"``) under *node*."""
if isinstance(node, dict):
if "t" in node:
found.add(node["t"])
for value in node.values():
cls._inline_types(value, found)
elif isinstance(node, list):
for value in node:
cls._inline_types(value, found)
return found
@pytest.mark.parametrize(
("title", "unescaped"),
[
(
'`[RawInline (Format "html") "<b>x</b>"]`{=pandoc-native}',
"RawInline",
),
('[x]{qmd-base64="e3s8IGVudiBIT01FID59fQ=="}', "Span"),
("<b>raw</b>", "RawInline"),
("Intro\n\n code", "Code"),
("*em* $x$ [@ref1] <https://e.example>", "Emph"),
],
ids=["pandoc-native", "qmd-base64-span", "raw-html", "code", "markup"],
)
def test_pandoc_reads_the_quarto_title_as_plain_text(
self, tmp_path, title, unescaped
):
"""Quarto passes the front matter with ``--metadata-file`` and reads
the body with a Lua custom reader (filters/qmd-reader.lua). Pandoc
then parses metadata with all of its default markdown extensions,
not with the file's ``from``, so raw inlines and spans in the title
would reach Quarto's ``pandoc-native`` and ``qmd-base64`` decoders.
This runs the bundled Pandoc the same way: the exported title must
come out as plain text, and the same title unescaped must not."""
pypandoc = pytest.importorskip("pypandoc")
front, _body = self._qmd_body("Body.", title=title)
line = next(x for x in front.splitlines() if x.startswith("from:"))
reader = line[len("from:") :].strip()
lua_reader = tmp_path / "reader.lua"
lua_reader.write_text(
"function Reader(input, opts)\n"
f' return pandoc.read(tostring(input), "{reader}", opts)\n'
"end\n",
encoding="utf-8",
)
source = tmp_path / "body.md"
source.write_text("Body.\n", encoding="utf-8")
def meta_title(front_matter):
metadata = tmp_path / "metadata.yaml"
metadata.write_text(front_matter, encoding="utf-8")
completed = subprocess.run(
[
pypandoc.get_pandoc_path(),
"--from",
str(lua_reader),
"--metadata-file",
str(metadata),
"--to",
"json",
str(source),
],
capture_output=True,
text=True,
check=True,
timeout=60,
)
return json.loads(completed.stdout)["meta"]["title"]
# Control: the title as plain YAML parses into markup.
quoted = json.dumps(title)
control = self._inline_types(meta_title(f"title: {quoted}\n"), set())
assert unescaped in control, control
# The exported front matter, as Quarto hands it to Pandoc.
exported = meta_title(front[len("---\n") : -len("---\n\n")])
assert exported["t"] == "MetaInlines", exported
assert self._inline_types(exported, set()) <= {
"MetaInlines",
"Str",
"Space",
}, exported
text = "".join(
node["c"] if node["t"] == "Str" else " " for node in exported["c"]
)
assert text.replace("\u200b", "") == " ".join(title.split()), text
@pytest.mark.parametrize(
("body", "expected"),
[
# Thematic break and setext underline: ``----`` renders the same.
("a\n\n---\n\nb", "a\n\n----\n\nb"),
("Heading\n---\ntext", "Heading\n----\ntext"),
(" --- \t", " ---- \t"),
# Not three dashes ending a line: left alone.
("----\n- - -\n---x\n-----", "----\n- - -\n---x\n-----"),
# Three dashes ending a URL or a word open no block either.
(
"URL: https://e.example/a---\nx---",
"URL: https://e.example/a---\nx---",
),
# After a block prefix they could: ``>``, a footnote's ``:``.
("> ---\n[^1]:---", "> ----\n[^1]:----"),
# Any run of whitespace, prefix characters and footnote labels.
(
"> > ---\n: ---\n~ ---\n| ---\n\t> [^a b]: ---\n>|:~---",
"> > ----\n: ----\n~ ----\n| ----\n\t> [^a b]: ----\n"
">|:~----",
),
# Any other text before them on the line: left alone, so prose
# keeps its em dash.
(
"He paused ---\nthen went on.\nx>---\n1 ---\nx:---\n- ---\n"
"[^1] ---\n[x]: ---\n[^]: ---",
"He paused ---\nthen went on.\nx>---\n1 ---\nx:---\n- ---\n"
"[^1] ---\n[x]: ---\n[^]: ---",
),
],
ids=[
"thematic-break",
"setext",
"indented",
"controls",
"url",
"block-prefix",
"nested-prefixes",
"prose",
],
)
def test_quarto_dash_lines_keep_their_markdown_meaning(
self, body, expected
):
_front, rendered = self._qmd_body(body)
assert rendered == expected
@pytest.mark.parametrize(
"body",
[
"He paused ---\nthen went on.",
"The answer is clear ---",
"a\n\n---\n\nb",
"Heading\n---\ntext",
"> quoted\n> ---\n> more",
"Term\n\n: ---\n",
"Text[^1].\n\n[^1]: ---\n",
# A rule directly followed by text, closed after a blank line.
# The four dashes of the defused line must be the same document
# as the three that were written.
"---\nParagraph after the first rule.\n\n---",
],
ids=[
"prose",
"prose-at-end",
"thematic-break",
"setext",
"blockquote",
"definition",
"footnote",
"rule-followed-by-text",
],
)
def test_pandoc_renders_dash_lines_as_written(self, body):
"""The bundled Pandoc reads the exported body, with the file's
``from``, to the same document as the body as written: prose
ending in `` ---`` keeps its em dash, and the four dashes of a
defused line are the same thematic break or setext underline."""
pypandoc = pytest.importorskip("pypandoc")
front, rendered = self._qmd_body(body)
line = next(x for x in front.splitlines() if x.startswith("from:"))
reader = line[len("from:") :].strip()
def blocks(text):
return json.loads(
pypandoc.convert_text(text, "json", format=reader)
)["blocks"]
assert blocks(rendered) == blocks(body)
if body.startswith(("He paused", "The answer")):
assert rendered == body
assert "\u2014" in json.dumps(blocks(rendered), ensure_ascii=False)
@pytest.mark.parametrize(
"body",
[
"---\nParagraph after the first rule.\n\n---",
"Intro text.\n\n---\n**Total Sources Organized:** 14 across 3"
" topics\n\n## Sources\n\n[1] Example source\n\n---\nFooter.",
],
ids=["rule-followed-by-text", "topic-organization"],
)
def test_pandoc_reads_a_rule_followed_by_text_as_a_rule(self, body):
"""A ``---`` line directly followed by text, with the next rule
after a blank line, stays two horizontal rules under the exported
reader.
``topic_organization_strategy.py`` emits this shape
(``\\n---\\n**Total Sources Organized:**``). With
``multiline_tables`` on, Pandoc reads the defused ``----`` lines
as a one-column table and the rest of the section, headings
included, lands in that cell. This pins the blocks the file's
``from`` produces against a control that turns the extension
back on.
"""
pypandoc = pytest.importorskip("pypandoc")
front, rendered = self._qmd_body(body)
line = next(x for x in front.splitlines() if x.startswith("from:"))
reader = line[len("from:") :].strip()
def blocks_of(fmt):
return json.loads(
pypandoc.convert_text(rendered, "json", format=fmt)
)["blocks"]
control_blocks = blocks_of(reader + "+multiline_tables")
blocks = blocks_of(reader)
assert any(b["t"] == "Table" for b in control_blocks), control_blocks
assert all(b["t"] != "Table" for b in blocks), blocks
assert [b["t"] for b in blocks].count("HorizontalRule") == 2, [
b["t"] for b in blocks
]
if "## Sources" in body:
assert any(b["t"] == "Header" for b in blocks), blocks
def test_pandoc_keeps_lines_after_an_indented_rule_whole(self):
"""An indented ``---`` followed by text, closed after a blank
line, does not cut the lines after it.
Read as a multiline table, the indent of the rule is the column
start, so ``ABCDEF`` / ``GHIJKL`` become ``CDEF`` / ``IJKL``
inside one cell. The exported reader keeps both rules and both
lines whole.
"""
pypandoc = pytest.importorskip("pypandoc")
body = "Intro.\n\n ---\nABCDEF line\nGHIJKL line\n\n---\nAfter.\n"
front, rendered = self._qmd_body(body)
line = next(x for x in front.splitlines() if x.startswith("from:"))
reader = line[len("from:") :].strip()
def blocks_of(fmt):
return json.loads(
pypandoc.convert_text(rendered, "json", format=fmt)
)["blocks"]
def strings(node):
found = []
def walk(value):
if isinstance(value, dict):
if value.get("t") == "Str":
found.append(value["c"])
for child in value.values():
walk(child)
elif isinstance(value, list):
for child in value:
walk(child)
walk(node)
return found
control_blocks = blocks_of(reader + "+multiline_tables")
blocks = blocks_of(reader)
control_cell = strings(
[block for block in control_blocks if block["t"] == "Table"]
)
assert "CDEF" in control_cell, control_cell
assert "IJKL" in control_cell, control_cell
assert "ABCDEF" not in control_cell, control_cell
assert "GHIJKL" not in control_cell, control_cell
assert all(b["t"] != "Table" for b in blocks), blocks
assert [b["t"] for b in blocks].count("HorizontalRule") == 2, [
b["t"] for b in blocks
]
assert "ABCDEF" in strings(blocks), strings(blocks)
assert "GHIJKL" in strings(blocks), strings(blocks)
@pytest.mark.parametrize(
"body",
[
"| Column A | Column B |\n|----------|----------|\n| 1 | two |\n",
"---\n 1 two\n 3 four\n---\n",
],
ids=["pipe-table", "simple-table"],
)
def test_pandoc_still_reads_ordinary_tables(self, body):
"""Pipe tables, and a headerless simple table whose rules have no
blank line between them, stay tables with ``multiline_tables`` off."""
pypandoc = pytest.importorskip("pypandoc")
front, rendered = self._qmd_body(body)
line = next(x for x in front.splitlines() if x.startswith("from:"))
reader = line[len("from:") :].strip()
blocks = json.loads(
pypandoc.convert_text(rendered, "json", format=reader)
)["blocks"]
assert any(b["t"] == "Table" for b in blocks), blocks
@pytest.mark.parametrize(
"unit",
[
"`",
"{{< ",
"<!-",
"--- ",
"---\n",
"{{{{",
"{{{<",
"```{r} ",
"{",
"&x",
"{\\{ ",
"\\",
"{ ",
"b58fc729\\-690b-",
"cf5733e5-0370-",
"quarto-shortcode_",
"&",
"&a",
"&#",
"&a;",
"&;",
"{{\\",
],
)
def test_quarto_body_defusion_is_a_linear_fixed_point(self, unit):
body = unit * 50000
started = time.thread_time()
once = self._qmd_body(body)[1]
elapsed = time.thread_time() - started
assert elapsed < 1.0, f"body defusion took {elapsed:.2f}s"
assert self._qmd_body(once)[1] == once
@pytest.mark.parametrize(
"body",
[
"&" * 200000 + ";",
"&a" * 100000 + ";",
"&#" * 100000 + ";",
"&lbrace" * 30000 + ";",
"&a " * 70000 + ";",
],
ids=["ampersands", "names", "numeric", "long-name", "spaced"],
)
def test_quarto_entity_scan_is_linear(self, body):
"""Each ``&`` reads up to the next ``;`` (Pandoc's
``characterReference``); re-reading that span per ``&`` would be
quadratic in a run of them."""
started = time.thread_time()
_defuse_quarto_body(body)
elapsed = time.thread_time() - started
assert elapsed < 1.0, f"entity scan took {elapsed:.2f}s"
def test_collection_line_scan_is_not_quadratic_in_blank_lines(self):
sources_block = (
"[1] Title\nURL: https://x.example\n" + "\n" * 24000 + "end\n"
)
formatter = CitationFormatter(CitationMode.SOURCE_TAGGED_HYPERLINKS)
started = time.thread_time()
collections = formatter._parse_collections(sources_block)
elapsed = time.thread_time() - started
assert collections == {}
assert elapsed < 0.1, (
f"collection scan took {elapsed:.2f}s on 24k blank lines"
)
def test_collection_line_scan_still_finds_a_real_collection(self):
"""Control: the pattern the previous test measures still works."""
formatter = CitationFormatter(CitationMode.SOURCE_TAGGED_HYPERLINKS)
block = (
"[1] Handbook\nURL: /library/document/7\nCollection: My Papers\n"
)
assert formatter._parse_collections(block) == {"1": "My Papers"}
def test_collection_fields_stay_with_their_header_and_group(self):
formatter = CitationFormatter(CitationMode.SOURCE_TAGGED_HYPERLINKS)
block = (
"Collection: Unattached\n"
"[1, 003] Grouped document\nURL: /library/document/7?q=[value]\n"
"\n Collection: My Papers\nCollection: Ignored second field\n"
"[2] No collection\nURL: https://example.org\n"
"[broken] Invalid header\nCollection: Must not attach to 2\n"
"[4] Last\nCollection: \n Collection: Final collection"
)
assert formatter._parse_collections(block) == {
"1": "My Papers",
"003": "My Papers",
"4": "Final collection",
}
def test_author_delimiters_preserve_names_and_whitespace(self):
ris = RISExporter()._create_ris_entry(
"1", "Title by Sandy Anderson, Alice\tand\tBob & Carol."
)
authors = [
line for line in ris.splitlines() if line.startswith("AU -")
]
assert authors == [
"AU - Sandy Anderson",
"AU - Alice",
"AU - Bob",
"AU - Carol",
]