Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

92.0 KB
Raw Blame
"""Contract tests for citation identity, bounds and URL handling.

Scope note: LaTeX/BibTeX *escaping* is covered by
``test_bibliography_escaping.py`` and is deliberately not retested here.
What this file pins is everything else the module does to
LLM-generated text:

* citation extraction / index-to-source binding (a ``[N]`` in the body
  must resolve to the source the Sources block gives index ``N``, and to
  no other),
* idempotency of ``format_document`` in every ``CitationMode``,
* round-tripping of citation identity through the Quarto / LaTeX / RIS
  exporters,
* regex bounds (super-linear blowup on hostile input),
* URL scheme handling (``javascript:`` / ``data:`` and markdown link
  injection).

Tests marked ``xfail(strict=True)`` assert the behaviour that *should*
hold and currently does not. They are live defect records: when the
underlying bug is fixed the strict marker turns the XPASS into a
failure, which is the signal to delete the marker. Each one names the
defect in its ``reason``.
"""

import html
import html.entities
import json
import random
import re
import subprocess
import time

import pytest

from local_deep_research.text_optimization.citation_formatter import (
    CitationFormatter,
    CitationMode,
    LaTeXExporter,
    QuartoExporter,
    RISExporter,
    _defuse_quarto_body,
    _defuse_quarto_cells,
)

ALL_MODES = list(CitationMode)

# Modes that render a citation as a markdown hyperlink built directly
# from the Sources ``URL:`` value. SOURCE_TAGGED is excluded on purpose:
# it is the only mode that gates the destination through
# ``_is_linkable_url``, and TestUrlSchemeHandling pins that difference.
UNGATED_LINK_MODES = [
    CitationMode.NUMBER_HYPERLINKS,
    CitationMode.DOMAIN_HYPERLINKS,
    CitationMode.DOMAIN_ID_HYPERLINKS,
    CitationMode.DOMAIN_ID_ALWAYS_HYPERLINKS,
]


def make_doc(body: str, sources: str) -> str:
    """Assemble the report shape the formatter expects."""
    return f"# Research Report\n\n{body}\n\n## Sources\n\n{sources}\n"


def answer_of(document: str) -> str:
    """Return the answer half (everything before the Sources header)."""
    return document.split("## Sources")[0]


def cite_through_both_entry_points(mode: CitationMode, url: str) -> list:
    """Render ``See [1].`` with ``url`` as source 1, once per map builder.

    The Sources-block regex ends a URL at the first newline and ``\\s*`` eats
    leading whitespace, so only the structured source list hands the gate a
    destination that still carries them.
    """
    formatter = CitationFormatter(mode)
    document = make_doc("See [1].", f"[1] Evil\nURL: {url}")
    return [
        answer_of(formatter.format_document(document)),
        formatter.apply_inline_hyperlinks(
            "See [1].", [{"index": 1, "title": "Evil", "url": url}]
        ),
    ]


def bibliography_of_latex(latex: str) -> str:
    start = latex.index("\\begin{thebibliography}")
    end = latex.index("\\end{thebibliography}")
    return latex[start:end]


def bibtex_of_quarto(quarto: str) -> str:
    start = quarto.index("```bibtex")
    return quarto[start : quarto.rindex("\n```\n")]


class TestCitationIdentity:
    """A ``[N]`` must bind to source ``N`` and to nothing else."""

    def test_every_index_binds_to_its_own_source_at_scale(self):
        """2000 citations: each ``[N]`` links to source ``N``'s URL.

        The failure this guards against is a citation silently pointing
        at a neighbouring source. Expected fragments are written out by
        hand rather than recomputed with the production formatting rule.
        """
        count = 2000
        body = " ".join(f"claim [{i}]." for i in range(1, count + 1))
        sources = "\n\n".join(
            f"[{i}] Title {i}\nURL: https://s{i}.example/p{i}"
            for i in range(1, count + 1)
        )
        document = make_doc(body, sources)

        started = time.perf_counter()
        result = CitationFormatter(
            CitationMode.NUMBER_HYPERLINKS
        ).format_document(document)
        elapsed = time.perf_counter() - started

        # Bound, not a benchmark: the pass is linear in document size and
        # a 2000-citation report is a realistic worst case.
        assert elapsed < 10.0, f"formatting 2000 citations took {elapsed:.1f}s"

        answer = answer_of(result)
        # Hand-written expectations, including the digit-prefix pairs
        # (1/10/100/1000, 199/1999) where an off-by-one or a prefix match
        # would cross-link two different sources.
        assert "claim [[1]](https://s1.example/p1)." in answer
        assert "claim [[10]](https://s10.example/p10)." in answer
        assert "claim [[100]](https://s100.example/p100)." in answer
        assert "claim [[199]](https://s199.example/p199)." in answer
        assert "claim [[1000]](https://s1000.example/p1000)." in answer
        assert "claim [[1999]](https://s1999.example/p1999)." in answer
        assert "claim [[2000]](https://s2000.example/p2000)." in answer

        # Nothing was left unformatted and nothing was cross-linked.
        assert "claim [1]." not in answer
        pairs = re.findall(
            r"\[\[(\d+)\]\]\(https://s(\d+)\.example/p\d+\)", answer
        )
        assert len(pairs) == count
        mismatched = [p for p in pairs if p[0] != p[1]]
        assert mismatched == [], (
            f"citations bound to wrong source: {mismatched}"
        )

    def test_comma_group_expands_in_order_to_per_index_targets(self):
        document = make_doc(
            "Combined [1, 2, 3].",
            "[1] Alpha\nURL: https://alpha.example/a\n\n"
            "[2] Beta\nURL: https://beta.example/b\n\n"
            "[3] Gamma\nURL: https://gamma.example/c",
        )
        result = CitationFormatter(
            CitationMode.NUMBER_HYPERLINKS
        ).format_document(document)
        assert (
            "Combined [[1]](https://alpha.example/a)"
            "[[2]](https://beta.example/b)"
            "[[3]](https://gamma.example/c)." in answer_of(result)
        )

    def test_source_word_form_binds_to_the_same_target_as_the_bracket(self):
        document = make_doc(
            "Bracket [2] and prose Source 2 agree.",
            "[1] Alpha\nURL: https://alpha.example/a\n\n"
            "[2] Beta\nURL: https://beta.example/b",
        )
        result = answer_of(
            CitationFormatter(CitationMode.NUMBER_HYPERLINKS).format_document(
                document
            )
        )
        assert (
            "Bracket [[2]](https://beta.example/b) and prose "
            "[[2]](https://beta.example/b) agree." in result
        )

    def test_leading_zero_index_is_a_distinct_citation(self):
        """``[007]`` and ``[7]`` are separate keys, each keeping its URL."""
        document = make_doc(
            "See [7] and [007].",
            "[7] Seven\nURL: https://seven.example\n\n"
            "[007] Bond\nURL: https://bond.example",
        )
        result = answer_of(CitationFormatter().format_document(document))
        assert (
            "See [[7]](https://seven.example) and "
            "[[007]](https://bond.example)." in result
        )

    def test_unknown_indices_in_a_group_become_individual_markers(self):
        """A group whose indices have no source is split, not dropped.

        Pinned because it is the one input where formatting rewrites text
        without resolving anything; the rewrite must stay lossless.
        """
        document = make_doc(
            "See [7, 8].", "[1] Alpha\nURL: https://alpha.example/a"
        )
        result = answer_of(CitationFormatter().format_document(document))
        assert "See [7][8]." in result

    def test_url_less_source_leaves_its_citation_untouched(self):
        document = make_doc("Local claim [3].", "[3] Local Doc Without Url")
        result = answer_of(CitationFormatter().format_document(document))
        assert "Local claim [3]." in result


class TestMalformedMarkers:
    """Hostile / malformed markers must not produce a wrong binding."""

    def test_already_hyperlinked_citation_is_not_rewritten(self):
        document = make_doc(
            "A [1] B [[1]] C [[[1]]] D",
            "[1] Alpha\nURL: https://alpha.example/a",
        )
        result = answer_of(CitationFormatter().format_document(document))
        assert "A [[1]](https://alpha.example/a) B [[1]] C [[[1]]] D" in result

    def test_deeply_nested_bracket_run_is_left_alone(self):
        depth = 20
        marker = "[" * depth + "1" + "]" * depth
        document = make_doc(marker, "[1] Alpha\nURL: https://alpha.example/a")
        result = answer_of(CitationFormatter().format_document(document))
        assert marker in result

    def test_mixed_lenticular_brackets_bind_to_the_same_source(self):
        document = make_doc(
            "See \u30101] and [2\u3011.",
            "[1] Alpha\nURL: https://alpha.example/a\n\n"
            "[2] Beta\nURL: https://beta.example/b",
        )
        result = answer_of(CitationFormatter().format_document(document))
        assert (
            "See [[1]](https://alpha.example/a) and "
            "[[2]](https://beta.example/b)." in result
        )


class TestIdempotency:
    """Formatting an already-formatted report must be a no-op."""

    RICH_DOCUMENT = make_doc(
        "Alpha [1], beta [2], gamma [3]. Group [1, 2, 3]. Source 2 "
        "confirms.\nLenticular \u30101\u3011 and mixed \u30102] here. "
        "Unknown [99] and group [98, 99].\n"
        "Already linked [[1]](https://arxiv.org/abs/2401.00001) stays.",
        "[1] Alpha Paper\nURL: https://arxiv.org/abs/2401.00001\n\n"
        "[2] Beta Blog\nURL: https://beta.example/post\n\n"
        "[3] Local Doc Without Url",
    )

    @pytest.mark.parametrize("mode", ALL_MODES, ids=lambda m: m.value)
    def test_second_pass_is_a_fixpoint(self, mode):
        formatter = CitationFormatter(mode)
        once = formatter.format_document(self.RICH_DOCUMENT)
        twice = formatter.format_document(once)
        assert once == twice

    @pytest.mark.parametrize("mode", ALL_MODES, ids=lambda m: m.value)
    def test_third_pass_is_still_the_same_fixpoint(self, mode):
        """Guards a two-step oscillation that a single re-run would miss."""
        formatter = CitationFormatter(mode)
        twice = formatter.format_document(
            formatter.format_document(self.RICH_DOCUMENT)
        )
        thrice = formatter.format_document(twice)
        assert twice == thrice

    def test_formatting_never_drops_the_sources_block(self):
        formatter = CitationFormatter(CitationMode.NUMBER_HYPERLINKS)
        once = formatter.format_document(self.RICH_DOCUMENT)
        assert "[1] Alpha Paper" in once
        assert "URL: https://arxiv.org/abs/2401.00001" in once
        assert "[3] Local Doc Without Url" in once


class TestRoundTripCitationIdentity:
    """markdown -> export format must preserve index/URL pairing."""

    DOCUMENT = make_doc(
        "Claims [1] and [2].",
        "[1] Alpha Paper\nURL: https://arxiv.org/abs/2401.00001\n\n"
        "[2] Beta Blog\nURL: https://beta.example/post",
    )

    def test_quarto_keeps_index_to_url_pairing(self):
        quarto = QuartoExporter().export_to_quarto(self.DOCUMENT)
        assert "Claims [@ref1] and [@ref2]." in quarto
        bib = bibtex_of_quarto(quarto)
        assert '@misc{ref1,\n  title = "{Alpha Paper}",' in bib
        assert "url = {https://arxiv.org/abs/2401.00001}," in bib
        assert '@misc{ref2,\n  title = "{Beta Blog}",' in bib
        assert "url = {https://beta.example/post}," in bib

    def test_latex_keeps_index_to_url_pairing(self):
        latex = LaTeXExporter().export_to_latex(self.DOCUMENT)
        assert "Claims \\cite{1} and \\cite{2}." in latex
        bib = bibliography_of_latex(latex)
        assert (
            "\\bibitem[1]{1} Alpha Paper. \\url{https://arxiv.org/abs/2401.00001}"
            in bib
        )
        assert (
            "\\bibitem[2]{2} Beta Blog. \\url{https://beta.example/post}" in bib
        )

    def test_ris_keeps_index_to_url_pairing(self):
        ris = RISExporter().export_to_ris(self.DOCUMENT)
        assert "ID  - ref1\nTI  - Alpha Paper\n" in ris
        assert "UR  - https://arxiv.org/abs/2401.00001\n" in ris
        assert "ID  - ref2\nTI  - Beta Blog\n" in ris
        assert "UR  - https://beta.example/post\n" in ris

    def test_latex_export_of_a_formatted_report_keeps_the_link_intact(self):
        formatted = CitationFormatter(
            CitationMode.NUMBER_HYPERLINKS
        ).format_document(self.DOCUMENT)
        assert "[[1]](https://arxiv.org/abs/2401.00001)" in formatted

        latex = LaTeXExporter().export_to_latex(formatted)
        body = latex[: latex.index("\\begin{thebibliography}")]
        assert "[\\cite{1}](" not in body
        assert "\\cite{1}" in body

    def test_latex_bibitem_survives_a_gap_in_the_index_sequence(self):
        document = make_doc(
            "See [1] and [5].",
            "[1] First\nURL: https://one.example\n\n"
            "[5] Fifth\nURL: https://five.example",
        )
        bib = bibliography_of_latex(LaTeXExporter().export_to_latex(document))
        # A labelled bibitem is what makes \cite{5} render as [5].
        assert "\\bibitem[5]{5}" in bib

    def test_latex_bibitem_label_keeps_the_index_as_written(self):
        document = make_doc(
            "See [7] and [007].",
            "[7] Seven\nURL: https://seven.example\n\n"
            "[007] Bond\nURL: https://bond.example",
        )
        bib = bibliography_of_latex(LaTeXExporter().export_to_latex(document))
        assert "\\bibitem[7]{7} Seven." in bib
        assert "\\bibitem[007]{007} Bond." in bib

    def test_latex_widest_label_grows_with_the_longest_index(self):
        few = LaTeXExporter().export_to_latex(
            make_doc("See [1].", "[1] One\nURL: https://one.example")
        )
        many = LaTeXExporter().export_to_latex(
            make_doc(
                "See [1] and [120].",
                "[1] One\nURL: https://one.example\n\n"
                "[120] Many\nURL: https://many.example",
            )
        )
        assert "\\begin{thebibliography}{99}" in few
        assert "\\begin{thebibliography}{999}" in many


class TestUrlSchemeHandling:
    """``javascript:`` / ``data:`` destinations and link injection."""

    @pytest.mark.parametrize(
        "url",
        [
            "javascript:alert(document.domain)",
            "data:text/html;base64,PHNjcmlwdD5hbGVydCgxKTwvc2NyaXB0Pg==",
        ],
    )
    def test_source_tagged_mode_neutralises_dangerous_schemes(self, url):
        """SOURCE_TAGGED gates the destination and emits no hyperlink."""
        document = make_doc("See [1].", f"[1] Evil\nURL: {url}")
        result = answer_of(
            CitationFormatter(
                CitationMode.SOURCE_TAGGED_HYPERLINKS
            ).format_document(document)
        )
        assert "See [local-1]." in result
        assert "javascript:" not in result
        assert "data:text/html" not in result

    @pytest.mark.parametrize("mode", UNGATED_LINK_MODES, ids=lambda m: m.value)
    @pytest.mark.parametrize(
        "url",
        [
            "javascript:alert(document.domain)",
            "data:text/html;base64,PHNjcmlwdD5hbGVydCgxKTwvc2NyaXB0Pg==",
        ],
    )
    def test_dangerous_scheme_is_never_emitted_as_a_destination(
        self, mode, url
    ):
        document = make_doc("See [1].", f"[1] Evil\nURL: {url}")
        result = answer_of(CitationFormatter(mode).format_document(document))
        assert "javascript:" not in result
        assert "data:text/html" not in result

    def test_source_url_cannot_inject_a_second_markdown_link(self):
        hostile = (
            "https://good.example/x) [click here](https://evil.example/pwn"
        )
        document = make_doc("See [1].", f"[1] Benign\nURL: {hostile}")
        result = answer_of(CitationFormatter().format_document(document))
        assert "[click here](https://evil.example/pwn)" not in result

    @pytest.mark.parametrize("mode", UNGATED_LINK_MODES, ids=lambda m: m.value)
    @pytest.mark.parametrize(
        "url",
        [
            pytest.param("JaVaScRiPt:alert(1)", id="mixed case"),
            pytest.param("  javascript:alert(1)", id="leading spaces"),
            pytest.param(
                "\tjava\nscript:alert(1)", id="tab and newline inside"
            ),
            pytest.param("\x01javascript:alert(1)", id="leading control char"),
            pytest.param("vbscript:msgbox(1)", id="vbscript"),
            pytest.param("blob:https://good.example/x", id="blob"),
            pytest.param("javascript&colon;alert(1)", id="named colon"),
            pytest.param("javascript&#58;alert(1)", id="decimal colon"),
            pytest.param("&#106;avascript:alert(1)", id="decimal letter"),
            pytest.param("java&Tab;script:alert(1)", id="named tab"),
        ],
    )
    def test_a_dangerous_scheme_survives_no_spelling_of_itself(self, mode, url):
        """A browser lowercases the scheme, strips leading C0 controls and
        spaces, and removes tabs and newlines before it reads one. A check on
        the raw string therefore passes strings that still execute, which is
        why the allowlist is applied to the trimmed form.

        Before any of that, the HTML parser decodes character references in an
        ``href``, and marked and Python-Markdown copy the destination there as
        written, so ``javascript&colon;`` is a spelling as well. The result is
        decoded and lowercased the same way before it is searched.

        ``blob:`` is in here for the other half: an allowlist is what makes a
        scheme nobody enumerated fail by default.
        """
        for result in cite_through_both_entry_points(mode, url):
            as_the_browser_reads_it = html.unescape(result).lower()

            assert "script:" not in as_the_browser_reads_it
            assert "blob:" not in as_the_browser_reads_it

    @pytest.mark.parametrize("mode", UNGATED_LINK_MODES, ids=lambda m: m.value)
    def test_an_unparseable_url_is_refused_rather_than_passed_through(
        self, mode
    ):
        """``urlsplit`` raises on a broken authority, and a dangerous scheme
        can carry one: ``javascript://[`` raises before the scheme is read.
        Treating "cannot parse" as "nothing to object to" would hand that
        straight to the renderer.
        """
        document = make_doc(
            "See [1].", "[1] Evil\nURL: javascript://[/alert(1)"
        )

        result = answer_of(CitationFormatter(mode).format_document(document))

        assert "javascript:" not in result

    def test_a_structured_source_list_is_gated_the_same_way(self):
        """The other entry point. ``format_document_with_sources`` is the
        fallback used when the LLM emits no Sources block, and it builds the
        same map from search-result dicts, where the URL came from a search
        engine rather than from the model.
        """
        sources = [
            {"index": 1, "title": "Evil", "url": "javascript:alert(1)"},
            {"index": 2, "title": "Real", "link": "https://good.example/x"},
        ]

        result = CitationFormatter().apply_inline_hyperlinks(
            "See [1] and [2].", sources
        )

        assert "javascript:" not in result
        assert "](https://good.example/x)" in result

    @pytest.mark.parametrize("mode", UNGATED_LINK_MODES, ids=lambda m: m.value)
    @pytest.mark.parametrize(
        "url",
        [
            pytest.param(
                "https://ok.example/x\n\n<img src=x onerror=alert(1)>",
                id="newline ends the destination",
            ),
            pytest.param("java\x00script:alert(1)", id="NUL inside the scheme"),
            pytest.param(
                "https://ok.example/\x07bell", id="bell inside the path"
            ),
        ],
    )
    def test_a_control_character_refuses_the_destination(self, mode, url):
        """``urlsplit`` removes tabs and newlines and strips leading C0
        controls to PARSE, and leaves them in the string. A destination that
        carries one ends there, and what follows is emitted as document body:
        an exported .md or .qmd then contains attacker-written markdown that
        never meets DOMPurify. NUL is not stripped at all, so it turns a
        dangerous scheme into a scheme-less string that would otherwise pass.
        """
        for result in cite_through_both_entry_points(mode, url):
            assert "onerror" not in result
            assert "script:" not in result
            assert "\x00" not in result
            assert "\x07" not in result

    @pytest.mark.parametrize("mode", UNGATED_LINK_MODES, ids=lambda m: m.value)
    def test_a_closing_paren_cannot_reopen_markdown(self, mode):
        """The scheme gate reads the whole string; markdown reads only up to
        the first unbalanced ``)``. Without escaping, everything after it is
        document body, so a destination can write its own link and put back
        exactly the scheme the gate refuses.
        """
        hostile = "https://good.example/x) [click here](javascript:alert(1)"
        document = make_doc("See [1].", f"[1] Benign\nURL: {hostile}")

        result = answer_of(CitationFormatter(mode).format_document(document))

        assert "](javascript:" not in result
        assert "javascript:alert(1))" not in result

    @pytest.mark.parametrize("mode", UNGATED_LINK_MODES, ids=lambda m: m.value)
    @pytest.mark.parametrize(
        ("character", "encoded"),
        [
            pytest.param(" ", "%20", id="space"),
            pytest.param("<", "%3C", id="angle open"),
            pytest.param(">", "%3E", id="angle close"),
            pytest.param('"', "%22", id="double quote"),
            pytest.param("'", "%27", id="single quote"),
            pytest.param("\\", "%5C", id="backslash"),
            pytest.param("`", "%60", id="backtick"),
        ],
    )
    def test_the_other_destination_escapes_each_close_the_same_breakout(
        self, mode, character, encoded
    ):
        """A paren is not the only way out of ``](...)``.

        Whitespace ends a bare destination outright and can open a title;
        ``<`` at the head switches markdown to the angle-bracket form, whose
        ``>`` then ends it; a quote or a backtick hands the rest of the line to
        another inline construct. Each one puts the attacker's text back into
        document body with markdown active, which is the same breakout the
        paren cell measures, so each is pinned rather than trusted to the table
        being complete.
        """
        hostile = f"https://good.example/x{character}TAIL"
        document = make_doc("See [1].", f"[1] Benign\nURL: {hostile}")

        result = answer_of(CitationFormatter(mode).format_document(document))

        # The whole destination, closing paren included: asserting only that
        # the character was encoded would still pass if everything after it had
        # been emitted as document body.
        assert f"](https://good.example/x{encoded}TAIL)" in result

    def test_a_balanced_pair_of_parens_is_left_verbatim(self):
        """Accept control for the escaping, and the reason it is not applied to
        every paren.

        Markdown ends a bare destination at the first *unbalanced* ``)`` and
        carries a balanced pair through unharmed, so there is nothing to defend
        against here. Encoding anyway rewrites an ordinary citation into a URL
        the source never served, which the reader sees and copies, and
        Wikipedia alone titles thousands of articles ``..._(disambiguation)``.
        """
        document = make_doc(
            "See [1].",
            "[1] Foo\nURL: https://en.wikipedia.org/wiki/Foo_(bar)",
        )

        result = answer_of(CitationFormatter().format_document(document))

        assert "](https://en.wikipedia.org/wiki/Foo_(bar))" in result
        assert "%28" not in result

    def test_an_unbalanced_paren_is_encoded(self):
        """The case the encoding exists for, and the one the balanced rule must
        not let through. A lone ``)`` ends the destination where it stands, so
        everything the source put after it would be emitted as document body
        with markdown active.
        """
        document = make_doc(
            "See [1].",
            "[1] Evil\nURL: https://example.com/a)[x](javascript:alert(1))",
        )

        result = answer_of(CitationFormatter().format_document(document))

        assert "https://example.com/a%29" in result
        assert "](javascript:" not in result

    def test_parens_that_balance_by_count_but_not_in_order_are_encoded(self):
        """Counting parens is not the same as balancing them. ``a)b(c`` has one
        of each and still ends the destination at the ``)``, so a rule that only
        compared totals would hand the attacker the same breakout with one
        character added.
        """
        document = make_doc(
            "See [1].",
            "[1] Evil\nURL: https://example.com/a)b(c",
        )

        result = answer_of(CitationFormatter().format_document(document))

        assert "https://example.com/a%29b%28c" in result

    def test_an_opener_that_is_never_closed_is_encoded(self):
        """The other direction. An unmatched ``(`` does not end the destination
        early, it makes the link fail to parse at all, so the citation renders
        as literal text with its URL exposed instead of as a link.
        """
        document = make_doc(
            "See [1].",
            "[1] Truncated\nURL: https://example.com/a(b",
        )

        result = answer_of(CitationFormatter().format_document(document))

        assert "https://example.com/a%28b" in result

    def test_an_ipv6_literal_is_left_alone(self):
        """Second accept control: brackets delimit an IPv6 host and do not end
        a markdown destination, so they must not be encoded.
        """
        document = make_doc("See [1].", "[1] Local\nURL: http://[::1]/x")

        result = answer_of(CitationFormatter().format_document(document))

        assert "](http://[::1]/x)" in result

    @pytest.mark.parametrize("mode", UNGATED_LINK_MODES, ids=lambda m: m.value)
    @pytest.mark.parametrize(
        "url",
        [
            pytest.param("https://good.example/q?b=1&amp;c=2", id="amp"),
            pytest.param(
                "https://good.example/q?b=1&c=2&copy=3", id="bare ampersands"
            ),
            pytest.param("https://good.example/q?q=x&lt;y", id="lt"),
        ],
    )
    def test_a_query_string_with_character_references_stays_linkable(
        self, mode, url
    ):
        """Accept control for the decoded check. Each of these decodes to
        another ``https`` URL, so it is linked, and linked as written: encoding
        ``&`` to ``%26`` would merge every multi-parameter query into one.
        """
        for result in cite_through_both_entry_points(mode, url):
            assert f"]({url})" in result

    @pytest.mark.parametrize("mode", UNGATED_LINK_MODES, ids=lambda m: m.value)
    @pytest.mark.parametrize(
        "url",
        [
            pytest.param("//evil.example/x", id="two slashes"),
            pytest.param("///evil.example/x", id="three slashes"),
            pytest.param("/&#92;evil.example/x", id="encoded backslash"),
            pytest.param("&#47;&#47;evil.example/x", id="encoded slashes"),
            pytest.param("&#32;//evil.example/x", id="encoded leading space"),
        ],
    )
    def test_a_scheme_less_destination_cannot_name_a_host(self, mode, url):
        """A relative URL stays linkable because a library document is cited
        as ``/library/...``. On an http(s) page a browser reads all of these
        as a link to ``evil.example`` instead, and a report opened from disk
        resolves the two-slash and backslash forms to
        ``file://evil.example/x``. ``urlsplit`` gives the three-slash form an
        empty netloc, which is why the rule is not a netloc check.
        """
        for result in cite_through_both_entry_points(mode, url):
            assert "evil.example" not in result

    def test_relative_url_still_produces_a_readable_label(self):
        """Control: a benign non-http URL must not be dropped silently."""
        document = make_doc(
            "See [1].", "[1] Internal Handbook\nURL: /library/document/7"
        )
        result = answer_of(
            CitationFormatter(CitationMode.DOMAIN_HYPERLINKS).format_document(
                document
            )
        )
        assert "[[internal-handbook]](/library/document/7)" in result


class TestSourceLabelFidelity:
    """The visible citation label must not misname the host."""

    @pytest.mark.parametrize(
        "url,impostor",
        [
            ("https://arxiv.org.evil.example/paper", "arxiv.org"),
            ("https://not-arxiv.org.attacker.test/p", "arxiv.org"),
            ("https://evil-github.com/x", "github.com"),
            ("https://youtube.com.phish.test/v", "youtube.com"),
            # A registrable name that merely ends in the known one, with no
            # dot in between, one per known domain.
            ("https://notarxiv.org/abs/1", "arxiv.org"),
            ("https://evilreddit.com/r/x", "reddit.com"),
            ("https://not-youtube.com/v", "youtube.com"),
            ("https://mypypi.org/project/x", "pypi.org"),
            ("https://fakemilvus.io/docs", "milvus.io"),
            ("https://fakemedium.com/a", "medium.com"),
        ],
    )
    def test_lookalike_host_is_not_labelled_with_the_known_domain(
        self, url, impostor
    ):
        assert CitationFormatter()._extract_domain(url) != impostor

    @pytest.mark.parametrize(
        "url,host",
        [
            ("https://[email protected]/abs/1", "evil.example"),
            ("https://user:[email protected]/report", "example.com"),
        ],
    )
    def test_userinfo_does_not_name_the_host(self, url, host):
        """The label names the host the link reaches, and no credentials."""
        assert CitationFormatter()._extract_domain(url) == host

    @pytest.mark.parametrize(
        "url",
        [
            "https://evil.example\\@arxiv.org/abs/1",
            "https://evil.example\\.arxiv.org/abs/1",
            "https://arxiv.org:443.evil.example/abs/1",
        ],
    )
    def test_an_authority_the_http_stack_reads_differently_names_where_it_goes(
        self, url
    ):
        """Browsers and requests end the authority at a backslash, and a port
        that is not a number hides the rest of it from urllib's hostname."""
        assert CitationFormatter()._extract_domain(url) == "evil.example"

    def test_a_lookalike_citation_renders_its_real_host(self):
        document = make_doc(
            "See [1].", "[1] Paper\nURL: https://arxiv.org.evil.example/p"
        )
        answer = answer_of(
            CitationFormatter(CitationMode.DOMAIN_HYPERLINKS).format_document(
                document
            )
        )
        assert "See [[evil.example]](https://arxiv.org.evil.example/p)." in (
            answer
        )

    @pytest.mark.parametrize(
        "url",
        [
            "https://export.arxiv.org/abs/2401.12345",
            "https://arxiv.org:443/abs/2401.12345",
            "https://ArXiv.org/abs/2401.12345",
            "https://arxiv.org./abs/2401.12345",
        ],
    )
    def test_a_subdomain_port_capital_or_root_dot_keeps_the_known_label(
        self, url
    ):
        assert CitationFormatter()._extract_domain(url) == "arxiv.org"

    @pytest.mark.parametrize(
        "url,label",
        [
            # A port requests refuses earns no known label, but the label
            # still leaves it out, as main did for known domains.
            ("https://arxiv.org:99999/abs/1", "arxiv.org"),
            ("https://arxiv.org:+443/abs/1", "arxiv.org"),
            ("https://example.com:8080/report", "example.com:8080"),
            ("http://[::1]:8080/x", "[::1]:8080"),
            ("http://[::1]/x", "[::1]"),
        ],
    )
    def test_port_labels_preserve_compatibility(self, url, label):
        assert CitationFormatter()._extract_domain(url) == label

    @pytest.mark.parametrize(
        "known",
        [
            "arxiv.org",
            "github.com",
            "reddit.com",
            "youtube.com",
            "pypi.org",
            "milvus.io",
            "medium.com",
        ],
    )
    def test_a_root_dot_keeps_every_known_label(self, known):
        # Every entry, not just arxiv.org: stripping more than the dot
        # (rstrip(".o")) leaves "arxiv.org" whole but turns milvus.io. into io.
        assert (
            CitationFormatter()._extract_domain(f"https://{known}./x") == known
        )

    def test_arxiv_in_the_path_does_not_earn_its_tag(self):
        """Regression control for the arXiv half of the defect below.

        ``URLClassifier.classify`` used to detect arXiv with a bare
        ``re.search`` over the whole URL, so an attacker-controlled path
        segment (``evil.example/arxiv.org/...``) earned the ``arxiv`` tag.
        This PR routes arXiv detection through
        ``utilities.arxiv.is_arxiv_paper_url``, which parses the actual
        hostname and identifier instead of substring-matching the raw URL,
        closing this specific case. The other academic sources are now
        matched on the parsed host as well (#6835) -- see the test below.
        """
        label = CitationFormatter()._extract_source_label(
            "https://evil.example/arxiv.org/abs/1234"
        )
        assert label != "arxiv"

    @pytest.mark.parametrize(
        "url,impostor",
        [
            ("https://evil.example/?x=doi.org/10.1", "doi"),
            (
                "https://evil.example/pubmed.ncbi.nlm.nih.gov/12345",
                "pubmed",
            ),
            ("https://evil.example\\@doi.org/10.1234/x", "doi"),
            (
                "https://pubmed.ncbi.nlm.nih.gov:443.evil.example/12345",
                "pubmed",
            ),
        ],
    )
    def test_known_source_in_the_path_does_not_earn_its_tag(
        self, url, impostor
    ):
        assert CitationFormatter()._extract_source_label(url) != impostor

    def test_genuine_known_domains_still_resolve(self):
        """Control for the two xfails above: real hosts keep their tag."""
        formatter = CitationFormatter()
        assert formatter._extract_domain("https://arxiv.org/abs/1") == (
            "arxiv.org"
        )
        assert formatter._extract_domain("https://www.github.com/a") == (
            "github.com"
        )
        # A real identifier: the label now follows the paper identifier, not
        # the hostname, so a placeholder id would fall through to the domain
        # and the control would stop controlling for anything.
        assert (
            formatter._extract_source_label("https://arxiv.org/abs/2401.12345")
            == "arxiv"
        )


class TestBibliographySelection:
    """Which Sources line wins a citation index."""

    # A report where index [1]'s real source has no URL (routine for a
    # local-library document) and the answer body happens to contain a
    # line that looks like a Sources entry and does carry a URL.
    POISONED = (
        "# Research Report\n\n"
        "Body text citing [1].\n\n"
        "[1] Attacker Controlled Entry\n"
        "URL: https://evil.example/poison\n\n"
        "## Sources\n\n"
        "[1] Real Local Document\n"
    )

    def test_latex_bibliography_is_scoped_to_the_sources_section(self):
        """Control: LaTeX slices at the Sources header first, so it wins."""
        bib = bibliography_of_latex(
            LaTeXExporter().export_to_latex(self.POISONED)
        )
        assert "\\bibitem[1]{1} Real Local Document." in bib
        assert "evil.example" not in bib

    def test_quarto_bibliography_is_not_poisoned_by_body_prose(self):
        bib = bibtex_of_quarto(QuartoExporter().export_to_quarto(self.POISONED))
        assert "evil.example" not in bib
        assert "Real Local Document" in bib

    def test_ris_emits_at_most_one_record_per_reference_id(self):
        document = make_doc(
            "See [1].",
            "[1] First Title\nURL: https://first.example\n\n"
            "[1] Second Title\nURL: https://second.example",
        )
        ris = RISExporter().export_to_ris(document)
        assert ris.count("ID  - ref1") == 1
        assert "TI  - Second Title\n" in ris
        assert "UR  - https://second.example\n" in ris
        assert "first.example" not in ris

    def test_quarto_uses_the_appended_sources_boundary(self):
        from local_deep_research.text_optimization.citation_formatter import (
            LDR_APPENDED_SOURCES_SENTINEL,
        )

        content = (
            "## Sources\n[1] Earlier prose\nURL: https://wrong.example\n\n"
            f"{LDR_APPENDED_SOURCES_SENTINEL}\n"
            "## Sources\n[1] Real Local Document\n"
        )
        bib = QuartoExporter()._generate_bibliography(content)
        assert "Real Local Document" in bib
        assert "wrong.example" not in bib

    @staticmethod
    def _full_report():
        """A report as ``_format_final_report`` assembles it, with the
        marker also quoted inline in the LLM's prose."""
        from local_deep_research.text_optimization.citation_formatter import (
            LDR_APPENDED_SOURCES_SENTINEL,
        )

        prose = (
            "# Research Report\n\n"
            "Body text citing [1]. The tool marks sources with "
            f"{LDR_APPENDED_SOURCES_SENTINEL} internally.\n\n"
            "[1] Attacker Controlled Entry\n"
            "URL: https://evil.example/poison"
        )
        return (
            prose
            + f"\n\n{LDR_APPENDED_SOURCES_SENTINEL}\n\n"
            + "## Sources\n\n[1] Real Local Document\n"
            + "URL: https://real.example/doc\n"
        )

    @staticmethod
    def _quarto_body(qmd):
        front_end = qmd.index("\n---\n\n", 3) + len("\n---\n\n")
        note = qmd.rindex("\n\n**Bibliography File Required**")
        return qmd[:front_end], qmd[front_end:note]

    def test_quarto_drops_the_appended_sources_sentinel(self):
        """The trusted sentinel line is the generator's markup, not report
        text: it is removed rather than defused into visible text, while
        the bibliography is still scoped by it and a copy quoted in the
        prose is still broken like any other comment."""
        qmd = QuartoExporter().export_to_quarto(self._full_report())
        _front, body = self._quarto_body(qmd)
        assert body.count("LDR_APPENDED_SOURCES_START") == 1, body
        assert "internally" in body.split("LDR_APPENDED_SOURCES_START")[1]
        assert "<!--" not in body
        assert "\n\n## Sources\n\n[@ref1] Real Local Document" in body
        bib = bibtex_of_quarto(qmd)
        assert "Real Local Document" in bib
        assert "evil.example" not in bib

    def test_pandoc_shows_no_sentinel_text_in_the_quarto_body(self):
        """Read the exported body with the file's own ``from`` (raw HTML
        off), as Quarto does: no paragraph is left holding the sentinel."""
        pypandoc = pytest.importorskip("pypandoc")
        front, body = self._quarto_body(
            QuartoExporter().export_to_quarto(self._full_report())
        )
        line = next(x for x in front.splitlines() if x.startswith("from:"))
        reader = line[len("from:") :].strip()
        document = json.loads(
            pypandoc.convert_text(body, "json", format=reader)
        )
        holding = [
            block
            for block in document["blocks"]
            if "LDR_APPENDED_SOURCES_START" in json.dumps(block)
        ]
        # Only the copy quoted in the prose, inside its own sentence.
        assert len(holding) == 1, holding
        assert "internally" in json.dumps(holding[0])


class TestRegexBounds:
    """Hostile input must not drive a pattern super-linear.

    Inputs are deliberately small (10-40 KB). The formerly quadratic
    parsers take seconds at these sizes, while a linear implementation
    stays comfortably within the bounds below.

    These synchronous scans use calling-thread CPU time so scheduling
    delays and unrelated background threads do not consume their budgets.
    The global pytest timeout still bounds wall time.
    """

    def test_pathological_bracket_input_stays_fast(self):
        pathological = [
            "[" * 200 + "1" + "]" * 200,
            "[1][2][3][4][5]" * 200,
            "[" + ",".join(["1"] * 2000) + "X",
            "\u3010" * 500 + "1" + "\u3011" * 500,
            "[1]" + " " * 20000,
        ]
        sources = (
            "[1] Alpha\nURL: https://alpha.example/a\n\n"
            "[2] Beta\nURL: https://beta.example/b"
        )
        formatter = CitationFormatter(CitationMode.DOMAIN_HYPERLINKS)
        for chunk in pathological:
            started = time.thread_time()
            formatter.format_document(make_doc(chunk, sources))
            elapsed = time.thread_time() - started
            assert elapsed < 2.0, f"input {chunk[:24]!r}... took {elapsed:.2f}s"

    def test_ris_author_split_is_not_quadratic_in_whitespace(self):
        run = " " * 10000
        sources_block = (
            f"## Sources\n\n[1] Study by A{run}B.\nURL: https://x.example/p\n"
        )
        started = time.thread_time()
        RISExporter().export_to_ris(sources_block)
        elapsed = time.thread_time() - started
        # A linear split handles 10 KB in well under a millisecond; the
        # 0.25s budget leaves ~6x headroom over the measured 1.6s so a
        # fast machine cannot XPASS this by accident.
        assert elapsed < 0.25, (
            f"author split took {elapsed:.2f}s on a 10KB whitespace run"
        )

    @pytest.mark.parametrize(
        "title",
        [
            # The run sits outside any "by ..." span, so the author cut
            # does not remove it before the title-clause cleanup runs.
            "A{run}B",
            "A{run}B by C.",
            "A.{run}B",
            "A{run}\t{run}B",
            "A{run}DOI",
            "A{run}Published i",
            "A{run}Volum",
            "A{run}Page",
            "A{run}Volume 3",
            "A{run}Pages 4",
        ],
        ids=[
            "plain",
            "before-by",
            "after-period",
            "tabs",
            "near-doi",
            "near-published",
            "near-volume",
            "near-pages",
            "volume",
            "pages",
        ],
    )
    def test_ris_title_cleanup_is_not_quadratic_in_whitespace(self, title):
        run = " " * 20000
        sources_block = (
            "## Sources\n\n[1] "
            + title.format(run=run)
            + "\nURL: https://x.example/p\n"
        )
        started = time.thread_time()
        RISExporter().export_to_ris(sources_block)
        elapsed = time.thread_time() - started
        # Linear cleanup takes a few milliseconds here. Each of the four
        # former ``\s*``-led substitutions took about 2s at 20k.
        assert elapsed < 0.25, (
            f"title cleanup took {elapsed:.2f}s on a 20KB whitespace run"
        )

    @pytest.mark.parametrize(
        ("title", "expected"),
        [
            ("Deep Learning  DOI: 10.1/x  Tail", "Deep Learning  Tail"),
            ("Deep Learning Published in Nature", "Deep Learning"),
            ("Audio \t Volume 3 Pages 1-9", "Audio"),
            ("Title doi:10.1/a DOI:10.2/b end", "Title end"),
            ("Mixed DOI:10.1/a", "Mixed"),
            ("Plain Title", "Plain Title"),
        ],
    )
    def test_ris_title_cleanup_output(self, title, expected):
        """Control: the linear cleanup cuts what the old regexes cut."""
        entry = RISExporter()._create_ris_entry("1", title)
        assert f"TI  - {expected}" in entry.splitlines()

    @pytest.mark.parametrize(
        "line",
        ["```{rRUN", "```{rRUNx", "  ````{pythonRUN, echo"],
        ids=["run-to-end", "run-then-text", "indented-options"],
    )
    def test_quarto_cell_fence_scan_is_not_quadratic_in_spaces(self, line):
        body = "# Report\n\n" + line.replace("RUN", " " * 10000) + "\n\nText.\n"
        started = time.thread_time()
        QuartoExporter().export_to_quarto(body, "Report")
        elapsed = time.thread_time() - started
        # The former `` *[ ,].*`` options group took 2.5-3.5s here.
        assert elapsed < 0.25, (
            f"cell-fence scan took {elapsed:.2f}s on a 10KB run of spaces"
        )

    def test_quarto_cell_fence_with_spaced_options_is_still_defused(self):
        """Control: an options group led by several spaces still matches."""
        body = "# Report\n\n```{python   , echo=false}\nprint(1)\n```\n"
        output = QuartoExporter().export_to_quarto(body, "Report")
        assert "```{python" not in output
        assert "```python" in output

    @pytest.mark.parametrize(
        "unit",
        [" ```{a}\n", "\t````{r, echo=false}\n", "  ```{python}\ncode\n"],
        ids=["indented-cells", "tab-indented-options", "indented-with-code"],
    )
    def test_quarto_cell_defusion_is_not_quadratic_in_indented_cells(
        self, unit
    ):
        # Each demoted indented cell stops shielding the line after it, so
        # the former re-read-until-stable loop needed one full pass per
        # cell: 8000 cells took 20-50s.
        body = "# Report\n\n" + unit * 8000 + "\nText.\n"
        started = time.thread_time()
        output = QuartoExporter().export_to_quarto(body, "Report")
        elapsed = time.thread_time() - started
        assert "{a}" not in output
        assert "{r" not in output
        assert "{python" not in output
        assert elapsed < 1.0, (
            f"cell defusion took {elapsed:.2f}s on 8000 indented cells"
        )

    @pytest.mark.parametrize(
        ("body", "expected"),
        [
            # A demoted indented cell is prose, so the next tag is live.
            (" ```{r}\n```{python}\nx\n```", " ```r\n```python\nx\n```"),
            # Fence state is not modelled: a cell inside a cell is demoted.
            ("```{r}\n```{python}\n```", "```r\n```python\n```"),
            (
                " ```{r}\n````\n```\n```{python}\n````\n```",
                " ```r\n````\n```\n```python\n````\n```",
            ),
            # Quoted as an example inside a longer fence: demoted as well.
            # Quarto's engine detection and knitr do not track display
            # fences, and the Jupyter engine's reader lets a cell open
            # inside one, so a quoted header is live to them.
            ("````\n```{r}\n````\n", "````\n```r\n````\n"),
            (
                "````markdown\n```{r}\nsummary(cars)\n```\n````",
                "````markdown\n```r\nsummary(cars)\n```\n````",
            ),
            (
                "```\n```{python}\nimport os; os.system('id')\n```\n```",
                "```\n```python\nimport os; os.system('id')\n```\n```",
            ),
            (
                "```\n  ```{python}\nprint(1)\n  ```\n```",
                "```\n  ```python\nprint(1)\n  ```\n```",
            ),
            # knitr and engine detection accept a ``>`` blockquote prefix.
            (
                "> ```{r}\n> system('id')\n> ```",
                "> ```r\n> system('id')\n> ```",
            ),
            (">> \t```{r, echo=FALSE}\n", ">> \t```r, echo=FALSE\n"),
            # A bare \r ending a prose line is a break to Quarto but not
            # one every reader shares; demoting regardless covers both.
            (
                "Some prose.\r```\n```{python}\nimport os\n```\n",
                "Some prose.\r```\n```python\nimport os\n```\n",
            ),
            # Engine detection's ``\s*`` and ``[^}]*`` span line breaks, so
            # a header split across lines is broken at its brace.
            ("```\r{python}\nimport os\n```", "```\rpython}\nimport os\n```"),
            (
                "```\n{r}\n```\n`r system('id')`",
                "```\nr}\n```\n`r system('id')`",
            ),
            ("x
```{r}
y", "x
```r}
y"),
            # Inside prose, too.
            ("use ```{r}``` here", "use ```r}``` here"),
        ],
        ids=[
            "unshielded",
            "cell-in-cell",
            "demoted-once-stays",
            "quoted-in-four-ticks",
            "verbatim-rmd",
            "python-in-display-fence",
            "indented-in-display-fence",
            "blockquote",
            "nested-blockquote",
            "lone-cr-prose",
            "lone-cr-split-header",
            "newline-split-header",
            "u2028-split-header",
            "mid-line",
        ],
    )
    def test_quarto_cell_defusion_ignores_fence_state(self, body, expected):
        assert _defuse_quarto_cells(body) == expected

    @pytest.mark.parametrize(
        ("body", "expected"),
        [
            ("```\ufeff{python}\nx\n```", "```python\nx\n```"),
            ("\ufeff```{python}\nx\n```", "\ufeff```python\nx\n```"),
            ("```{python}\ufeff\nx\n```", "```python\nx\n```"),
            (
                "```\n\x1c```\n```\n```{r}\nx\n```",
                "```\n\x1c```\n```\n```r\nx\n```",
            ),
            (
                "```\n```\x85\n```\n```{r}\nx\n```",
                "```\n```\x85\n```\n```r\nx\n```",
            ),
            ("x\r```{python}\nx\n```", "x\r```python\nx\n```"),
            ("```\r```\r```{r}\r\nx\r\n```", "```\r```\r```r\r\nx\r\n```"),
        ],
        ids=[
            "bom-before-brace",
            "bom-indent",
            "bom-trailing",
            "x1c-line",
            "x85-line",
            "lone-cr-line",
            "lone-cr-closer",
        ],
    )
    def test_quarto_cell_defusion_reads_any_whitespace_and_line_break(
        self, body, expected
    ):
        assert _defuse_quarto_cells(body) == expected

    @pytest.mark.parametrize(
        "body",
        ["```{.python}\nx\n```", "```{#id}\nx\n```", "```{{python}}\nx\n```"],
        ids=["pandoc-class", "pandoc-id", "escaped-cell"],
    )
    def test_quarto_defusion_leaves_non_cells_alone(self, body):
        """Control: no reader runs these, so they stay as written."""
        assert _defuse_quarto_cells(body) == body

    def test_quarto_cell_whitespace_is_every_readers(self):
        """The whitespace class is Python's ``\\s`` plus JavaScript's.

        JavaScript's ``\\s`` (ECMAScript WhiteSpace and LineTerminator) adds
        U+FEFF to Python's, Python's adds \\x1c-\\x1f and \\x85, and U+180E
        was a space before Unicode 6.3. A character either reader skips must
        not shield a header in any of the three positions.
        """
        spaces = {
            *(chr(c) for c in range(0x10000) if chr(c).isspace()),
            "\ufeff",
            "\u180e",
        }
        for char in sorted(spaces):
            for body in (
                f"{char}```{char}{{r}}{char}\nx\n```",
                f"```{char}{{r}}\nx\n```",
                f"{char}```{{r, echo=FALSE}}{char}",
            ):
                assert "{r" not in _defuse_quarto_cells(body), hex(ord(char))

    def test_quarto_defusion_output_matches_no_reader(self):
        """Fuzz: no reader that decides execution sees a cell header.

        The regexes are the upstream ones, transcribed with ``\\s`` widened
        to the union above and every line splitting a reader could use:

        * quarto-cli ``languagesWithClasses`` (pandoc-partition.ts), /gm over
          the whole text: ``^[\\t >]*```+\\s*\\{([a-zA-Z][a-zA-Z0-9_.]*)``
          ``([^}]*)?\\}\\s*$``
        * quarto-cli ``breakQuartoMd`` (break-quarto-md.ts):
          ``^\\s*(```+)\\s*\\{([=A-Za-z][=A-Za-z0-9._]*)( *[ ,].*)?\\}\\s*$``
        * quarto-cli ``quartoMdToJupyter`` (jupyter.ts):
          ``^(\\s*)(```+)\\s*\\{<language>( *[ ,].*)?\\}\\s*$``
        * knitr md ``chunk.begin`` (R/pattern.R):
          ``^[\\t >]*```+\\s*\\{([a-zA-Z0-9_]+( *[ ,].*)?)\\}\\s*$``
        """
        space = r"[\s\ufeff\u180e]"
        ends = "\n\r

"
        engine_detection = re.compile(
            rf"(?:^|(?<=[{ends}]))[\t >]*`{{3,}}{space}*"
            rf"\{{[a-zA-Z][a-zA-Z0-9_.]*[^}}]*\}}{space}*(?=$|[{ends}])"
        )
        line_readers = [
            re.compile(
                rf"^{space}*`{{3,}}{space}*\{{[=A-Za-z][=A-Za-z0-9._]*"
                rf"( *[ ,].*)?\}}{space}*$"
            ),
            re.compile(
                rf"^{space}*`{{3,}}{space}*\{{[a-z0-9_]+( *[ ,].*)?\}}{space}*$",
                re.IGNORECASE,
            ),
            re.compile(
                rf"^[\t >]*`{{3,}}{space}*\{{[a-zA-Z0-9_]+( *[ ,].*)?\}}"
                rf"{space}*$"
            ),
        ]
        splits = [r"\r\n?|\n", r"\r?\n", r"\n", "\r\n|[\n\r

\x85]"]

        def live(text):
            if engine_detection.search(text):
                return True
            return any(
                reader.match(line)
                for split in splits
                for line in re.split(split, text)
                for reader in line_readers
            )

        rng = random.Random(6982)
        spacing = ["", " ", "\t", "\ufeff", "\x1c", "\x85", "
"]
        spacing += ["\xa0", "\u180e", "> ", "\r", "\n"]
        langs = ["r", "python", "=html", ".r", "R", "r-x", "1r", "é", "ojs"]
        rests = ["", " echo=FALSE", ', x="`}`"', "}", " ", "x"]
        lines = ["```", "````", " ```", "```python", "~~~", "---", "text"]
        lines += ["", "`r x`", "```{{python}}", "```{.python}", "{r}"]
        breaks = ["\n", "\n", "\r\n", "\r", "
"]
        for _ in range(3000):
            body = ""
            for _ in range(rng.randint(0, 20)):
                if rng.random() < 0.55:
                    line = (
                        f"{rng.choice(spacing)}{'`' * rng.choice([3, 4])}"
                        f"{rng.choice(spacing)}{{{rng.choice(langs)}"
                        f"{rng.choice(rests)}}}{rng.choice(spacing)}"
                    )
                else:
                    line = rng.choice(lines)
                body += line + rng.choice(breaks)
            output = _defuse_quarto_cells(body)
            assert not live(output), repr(body)
            assert _defuse_quarto_cells(output) == output, repr(body)

    @pytest.mark.parametrize(
        ("body", "expected"),
        [
            # Prose that starts with a fence and ends in ``}`` is no header
            # to any reader. Only the opening brace goes; the text stays.
            (
                "```{r}``` chunks run R, see {Figure 1}",
                "```r}``` chunks run R, see {Figure 1}",
            ),
            (
                "```{python}: Quarto cell syntax {docs}",
                "```python}: Quarto cell syntax {docs}",
            ),
            # A real header keeps its options; only the braces go.
            (
                "```{python, echo=false}\nx\n```",
                "```python, echo=false\nx\n```",
            ),
            ("```{r, x='}'}\nx\n```", "```r, x='}'}\nx\n```"),
        ],
        ids=["prose-code-span", "prose-colon", "options", "brace-in-options"],
    )
    def test_quarto_cell_demotion_drops_no_text(self, body, expected):
        assert _defuse_quarto_cells(body) == expected

    @staticmethod
    def _qmd_body(body, title="Report"):
        qmd = QuartoExporter().export_to_quarto(body, title)
        front_end = qmd.index("\n---\n\n", 3) + len("\n---\n\n")
        note = qmd.rindex("\n\n**Bibliography File Required**")
        return qmd[:front_end], qmd[front_end:note]

    @pytest.mark.parametrize(
        "body",
        [
            "{{< include ../../../usr/lib/R/x/skeleton.Rmd >}}\n`r system('id')`",
            "Key: {{< env OPENAI_API_KEY >}}",
            "{{< meta title >}} and {{<meta x>}}",
            "{{% env OPENAI_API_KEY %}}",
            "{{ < env OPENAI_API_KEY >}}",
            "{{{< env OPENAI_API_KEY >}}}",
            "{{</* include x.qmd */>}}",
            "```\n{{< include x.qmd >}}\n```",
        ],
        ids=[
            "include",
            "env",
            "meta",
            "percent",
            "spaced",
            "escaped",
            "comment-escaped",
            "in-code-block",
        ],
    )
    def test_quarto_shortcodes_in_the_body_are_broken(self, body):
        """quarto-cli opens a shortcode at ``{{<`` (lpegshortcode.lua) and
        expands includes before it picks the engine, so an include can bring
        in a file with cells and ``env`` prints the renderer's secrets."""
        _front, rendered = self._qmd_body(body)
        assert re.search(r"\{\s*\{\s*[<%]", rendered) is None, rendered
        # The text stays readable: only a zero-width space was inserted.
        assert rendered.replace("\u200b", "") == body

    def test_quarto_shortcode_in_the_title_is_broken(self):
        """Quarto expands shortcodes in metadata, the title included."""
        front, _body = self._qmd_body(
            "Body.", title="{{< env OPENAI_API_KEY >}}"
        )
        assert "{{<" not in front
        line = next(x for x in front.splitlines() if x.startswith("title"))
        value = line[len('title: "') : -1].replace("\\\\", "\\")
        assert self._pandoc_decode(value) == "{\u200b{< env OPENAI_API_KEY >}}"

    # What still decodes body text once the reader format has switched off
    # raw content and attributes (quarto-cli v1.10.18): lpegshortcode.lua's
    # ``wrap_lpeg_match`` restores ``cf5733e5-...&#x<hex>;`` after its
    # whole-file pass, Pandoc then decodes entities and backslash escapes
    # once (text, link targets, image sources; not code or math), and
    # readqmd.lua hex-decodes ``b58fc729-...;<hex>;`` into a shortcode.
    _SENTINEL = "b58fc729-690b-4000-b19f-365a4093b2ff"
    _UNICODE_SENTINEL = "cf5733e5-0370-4aae-8689-61bad1dd9ec0"
    _ENV_HEX = "7B7B3C20656E76204F50454E41495F4150495F4B4559203E7D7D"

    @staticmethod
    def _pandoc_decode(text, entities=True):
        """Pandoc's markdown decoding: ``escapedChar'`` with
        all_symbols_escapable (a backslash before any character that is not
        alphanumeric or a line break) and ``characterReference`` (``&``,
        then non-space characters up to the first ``;`` after the first
        one, looked up as an entity)."""
        out = []
        index = 0
        while index < len(text):
            char = text[index]
            following = text[index + 1 : index + 2]
            if (
                char == "\\"
                and following
                and following not in "\n\r"
                and not following.isalnum()
            ):
                out.append(following)
                index += 2
                continue
            if char == "&" and entities:
                end = index + 2
                while end < len(text) and text[end] != ";":
                    if text[end] in " \t\n\r":
                        break
                    end += 1
                if (
                    end < len(text)
                    and text[end] == ";"
                    and text[index + 1] not in " \t\n\r"
                ):
                    value = html.unescape(text[index : end + 1])
                    if value != text[index : end + 1]:
                        out.append(value)
                        index = end + 1
                        continue
            out.append(char)
            index += 1
        return "".join(out)

    @classmethod
    def _unescape_unicode(cls, text):
        """lpegshortcode.lua ``unescape_unicode``."""
        return re.sub(
            re.escape(cls._UNICODE_SENTINEL) + "&#x([0-9a-fA-F]+);",
            lambda m: chr(int(m.group(1), 16)),
            text,
        )

    @classmethod
    def _decodings(cls, text):
        """*text* as each shortcode finder sees it: the raw file, the file
        after ``unescape_unicode`` (code, code blocks and math), and that
        decoded once by Pandoc or by its escapes alone (text, link targets
        and image sources)."""
        restored = cls._unescape_unicode(text)
        return {
            text,
            restored,
            cls._pandoc_decode(restored),
            cls._pandoc_decode(restored, entities=False),
        }

    @classmethod
    def _expanded_by_quarto(cls, text):
        """A decoding of *text* holding a shortcode opener or encoding."""
        opener = re.compile(r"\{\{[\s\ufeff\u180e]*[<%]")
        for decoded in cls._decodings(text):
            if (
                opener.search(decoded)
                or cls._SENTINEL in decoded.lower()
                or "quarto-shortcode__" in decoded.lower()
            ):
                return decoded
        return None

    @pytest.mark.parametrize(
        "body",
        [
            # readqmd.lua hex sentinel; the hex is {{< env OPENAI_API_KEY >}}.
            f"Key: {_SENTINEL};{_ENV_HEX}; end",
            f"[x](https://e.example/{_SENTINEL};{_ENV_HEX};)",
            f"`{_SENTINEL};{_ENV_HEX};`",
            f"b58fc729&#45;690b\\-4000-b19f-365a4093b2ff;{_ENV_HEX};",
            f"&#98;58fc729-690b-4000-b19f-365a4093b2ff;{_ENV_HEX};",
            f"B58FC729&#x2D;690b-4000-b19f-365a4093b2ff;{_ENV_HEX};",
            # lpegshortcode.lua restores the unicode escape after its pass:
            # code, a fence, math, prose and a link target.
            f"`{{{{{_UNICODE_SENTINEL}&#x3c; env OPENAI_API_KEY >}}}}`",
            f"```\n{{{{{_UNICODE_SENTINEL}&#x3c; env HOME >}}}}\n```",
            f"${{{{{_UNICODE_SENTINEL}&#x3c; meta title >}}}}$",
            f"{{{{{_UNICODE_SENTINEL}&#x3c; env X >}}}}",
            f"[x](https://e.example/{{{{{_UNICODE_SENTINEL}&#x3c;)",
            f"{_UNICODE_SENTINEL}&#x7b;{_UNICODE_SENTINEL}&#x7b;< env X >}}}}",
            # Pandoc decodes escapes and entities in text and link targets.
            "[x](https://e.example/\\{\\{<)",
            "[x](https://e.example/{\\{\\<)",
            "&#123;&#123;< env X >&#125;&#125;",
            "&#0123;&#X7B;&lt; env X >}}",
            "&lbrace;&lcub;< env X >}}",
            "{{&#32;< env X >}}",
            "{{&lt; env X >}}",
            "{&#x7b;< env X >}}",
            "&&lbrace;{< env X >}}",
            # The class ast/parse.lua turns into a shortcode.
            "quarto-shortcode__",
            "quarto\\-shortcode\\_&lowbar;",
        ],
    )
    def test_quarto_shortcodes_are_broken_after_any_decoding(self, body):
        """Breaking only the literal ``{{<`` left the env shortcode
        reachable through every other spelling a reader decodes."""
        assert self._expanded_by_quarto(body) is not None, "control"
        _front, rendered = self._qmd_body(body)
        assert self._expanded_by_quarto(rendered) is None, rendered
        # Only the shortcode pass promises to insert zero-width spaces
        # without other edits. The full exporter may also escape unsafe
        # math; keep checking its shortcode safety above.
        assert _defuse_quarto_body(body).replace("\u200b", "") == body

    # Each of these needs a reader the exported file switches off, named
    # with the payload: attribute values Pandoc decodes and shortcodes.lua
    # expands, a callout title read twice, ``qmd``/``qmd-base64`` and
    # ``pandoc-*`` raw formats (normalize/extractquartodom.lua), raw LaTeX
    # ``\QuartoMarkdownBase64{}`` and raw HTML attributes.
    @pytest.mark.parametrize(
        ("body", "extensions"),
        [
            (
                '[x]{title="{\\{< env OPENAI_API_KEY >}}"}',
                ["bracketed_spans"],
            ),
            ("[x]{data-k=\\{\\{<}", ["bracketed_spans"]),
            (
                '::: {data-k="{\\{< meta title >}}"}\nx\n:::',
                ["fenced_divs"],
            ),
            (
                "## H {data-k='&#x7b;&#x7B;< env HOME >}}'}",
                ["header_attributes"],
            ),
            ('[x](y){k="{{&#32;< env X >}}"}', ["link_attributes"]),
            ('![x](y){k="&#123;&#123;< env X >}}"}', ["link_attributes"]),
            ('`x`{k="&#123;&#123;< env X >}}"}', ["inline_code_attributes"]),
            (
                '```{.x k="&#123;&#123;< env X >}}"}\nx\n```',
                ["fenced_code_attributes"],
            ),
            (
                '::: {title="&amp;#123;&amp;#123;< env X >}}" .callout-note}'
                "\nx\n:::",
                ["fenced_divs"],
            ),
            ('[x]{qmd-base64="e3s8IGVudiBYID59fQ=="}', ["bracketed_spans"]),
            (
                '::: {qmd="&#123;&#123;< env X >}}"}\nx\n:::',
                ["fenced_divs"],
            ),
            (
                '`[Str "x"]`{=pandoc-native}',
                ["raw_attribute"],
            ),
            (
                '~~~{=pandoc-reader-html}\n<span class="quarto&#45;'
                'shortcode__">x</span>\n~~~',
                ["raw_attribute"],
            ),
            (
                "\\QuartoMarkdownBase64{e3s8IGVudiBYID59fQ==}",
                ["raw_tex"],
            ),
            (
                '<span title="&#123&#123< env X >}}">x</span>',
                ["raw_html", "native_spans"],
            ),
            (
                '<div qmd-base64="e3s8IGVudiBYID59fQ==">x</div>',
                ["raw_html", "native_divs"],
            ),
            (
                "+--------------+\n| ---          |\n| engine: x    |\n"
                "| ---          |\n+--------------+",
                ["yaml_metadata_block"],
            ),
        ],
    )
    def test_quarto_reader_format_switches_off_decoding_readers(
        self, body, extensions
    ):
        """Modelling these readers one spelling at a time kept missing
        one, so the exported file reads with Pandoc's markdown minus every
        extension that makes an attribute, a raw block or inline, a Div or
        a Span. Quarto passes ``from`` to readqmd.lua as
        ``user-defined-from``, for the document and every re-read."""
        front, rendered = self._qmd_body(body)
        line = next(x for x in front.splitlines() if x.startswith("from:"))
        reader = line[len("from:") :].strip()
        # Quarto's ``parse_format`` (_format.lua) reads the whole string.
        assert re.fullmatch(r"markdown(?:-[a-z_]+)+", reader), reader
        disabled = set(reader.split("-")[1:])
        assert set(extensions) <= disabled, (body, reader)
        assert rendered.replace("\u200b", "") == body

    def test_quarto_title_shortcodes_are_broken_after_any_decoding(self):
        """Pandoc reads the title as markdown, so it decodes it too."""
        for title in (
            "&#123;&#123;< env OPENAI_API_KEY >}}",
            "\\{\\{< env OPENAI_API_KEY >}}",
            f"{self._SENTINEL};{self._ENV_HEX};",
            f"{{{{{self._UNICODE_SENTINEL}&#x3c; env OPENAI_API_KEY >}}}}",
        ):
            front, _body = self._qmd_body("Body.", title=title)
            line = next(x for x in front.splitlines() if x.startswith("title"))
            value = line[len('title: "') : -1].replace("\\\\", "\\")
            assert self._expanded_by_quarto(value) is None, title

    @pytest.mark.parametrize(
        "payload",
        [
            f"{_SENTINEL};{_ENV_HEX};",
            f"{_UNICODE_SENTINEL}&#x7b;{_UNICODE_SENTINEL}&#x7b;&lt; env X",
        ],
    )
    def test_quarto_bibliography_note_breaks_the_sentinels(self, payload):
        """readqmd.lua hex-decodes the sentinel in CodeBlock text, the
        whole-file pass restores the unicode escape, and shortcodes.lua
        then expands the result, so the bibtex fence is a sink."""
        qmd = QuartoExporter().export_to_quarto(
            f"See [1].\n\n## Sources\n\n[1] {payload}\n"
            f"URL: https://e.example/{payload}\n",
            "Report",
        )
        bibtex = bibtex_of_quarto(qmd)
        assert "\u200b" in bibtex
        assert self._SENTINEL not in qmd
        assert self._UNICODE_SENTINEL not in qmd

    @pytest.mark.parametrize(
        "body",
        [
            "Johnson &amp; Johnson, R&amp;D and Q&amp;A",
            "a&nbsp;b &lt;tag&gt; &quot;q&quot; &mdash; &#39; &copy; &#38;",
            "{{ jinja }} and {{x}} and x^{{2}}",
            "AT & T, 5 % off, R & D, Q&A, R&D, AT&T, S&P 500",
            "$$\\frac{{\\partial f}}{{\\partial x}}$$",
            '```c\nscanf("%d", &x); int *p = &arr[i];\n```',
            "```cpp\nvoid f(std::string &name, int &&n);\n```",
            "[the paper](https://scholar.google.com/scholar?q=llm&hl=en)",
            "<https://example.com/a?x=1&y=2> and a&b;c",
        ],
    )
    def test_quarto_defusion_keeps_harmless_text(self, body):
        _front, rendered = self._qmd_body(body)
        assert rendered == body

    _SOURCE_URLS = (
        "https://scholar.google.com/scholar?q=llm&hl=en&as_sdt=0,5",
        "https://www.youtube.com/watch?v=dQw4w9WgXcQ&t=10s",
        "https://example.com/a?x=1&y=2&z=&w",
        "https://example.com/search?q=a%26b&lang=en#results",
        "https://example.com/page;jsessionid=ABC123?x=1&y=2",
        "https://example.com/a?b=1&c=2;d=3&e",
        "https://example.com/a?b=1&amp;c=2&amp;d=3",
        "https://example.com/a?b=1&lt;c&#38;d=2&#x26;e=3",
        "https://en.wikipedia.org/wiki/AT%26T#History",
        "https://doi.org/10.1000/xyz123-456_789",
        "https://arxiv.org/abs/2401.01234v2",
        "https://example.com/a---",
        "https://example.com/{x}?y={z}",
        "https://example.com/path/to/%7B%7B%3C/?a=b&c",
        "https://example.com/550e8400-e29b-41d4-a716-446655440000?a=1&b=2",
    )

    def test_quarto_sources_urls_round_trip(self):
        """The ``&`` rule used to insert a zero-width space after every
        ``&`` before a letter, so ``?q=a&hl=en`` reached the server as
        ``&%E2%80%8Bhl=en`` and every query link in Sources broke. A URL
        must remain byte for byte through the shortcode pass. The full
        exporter can escape punctuation, so compare that output after
        Markdown escape and entity decoding."""
        rng = random.Random(69822)
        urls = list(self._SOURCE_URLS)
        alphabet = (
            "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
            "-._~:/?#@!$&'()*+,=%"
        )
        # Without a ``;`` no ``&`` can start an entity. Brackets are left
        # out: the exporter turns ``[5]`` into a citation wherever it is.
        for _ in range(2000):
            urls.append(
                "https://e.example/"
                + "".join(rng.choice(alphabet) for _ in range(40))
            )
        sources = "".join(
            f"[{index}] Source {index}\n   URL: {url}\n"
            for index, url in enumerate(urls, 1)
        )
        cited = " ".join(f"[{index}]" for index in range(1, len(urls) + 1))
        _front, rendered = self._qmd_body(
            f"Body {cited}.\n\n## Sources\n\n{sources}"
        )
        # A separate math guard can escape punctuation in a Sources line.
        # Compare its decoded text, while retaining the byte-for-byte
        # contract for the shortcode pass itself below.
        decoded_lines = [
            self._pandoc_decode(line) for line in rendered.splitlines()
        ]
        lines = [
            line[len("   URL: ") :]
            for line in decoded_lines
            if line.startswith("   URL: ")
        ]
        assert lines == [self._pandoc_decode(url) for url in urls]
        for url in urls:
            link = f"[x]({url})"
            assert _defuse_quarto_body(link) == link
            assert self._pandoc_decode(
                _defuse_quarto_body(url)
            ) == self._pandoc_decode(url)

    def test_quarto_shortcode_defusion_differential(self):
        """Fuzz: bodies of brace, escape, entity and whitespace spellings
        and of the internal encodings spelled out a character at a time in
        every form, against the decoding chain above. The output is never
        expanded, only zero-width spaces are inserted, and defusing again
        changes nothing."""
        rng = random.Random(69821)

        def spell(char):
            forms = [char, f"&#{ord(char)};", f"&#x{ord(char):x};"]
            if not char.isalnum():
                forms += ["\\" + char, "\\\\\\" + char]
            forms.append(f"{self._UNICODE_SENTINEL}&#x{ord(char):x};")
            forms += [
                f"&{name}"
                for name, value in html.entities.html5.items()
                if value == char and name.endswith(";")
            ]
            return rng.choice(forms)

        pieces = ["{", "\\{", "\\\\{", "&#123;", "&#x7B;", "&lbrace;"]
        pieces += ["&amp;#123;", "&#38;lcub;", "<", "&lt;", "%", "\\<"]
        pieces += [" ", "&nbsp;", "\\ ", "\n", "&", "\\", "#", ";", "x"]
        pieces += ["&amp;", "}", "\u200b", "b", "q", "c", "-", "_"]
        pieces += [f"{self._UNICODE_SENTINEL}&#x3c;", "&#32;", "`", "$"]
        expanded_before = 0
        for _ in range(400):
            body = ""
            for _ in range(rng.randint(1, 8)):
                roll = rng.random()
                if roll < 0.1:
                    word = rng.choice([self._SENTINEL, "quarto-shortcode__"])
                    body += "".join(spell(char) for char in word)
                elif roll < 0.25:
                    body += "".join(spell(char) for char in "{{<")
                else:
                    body += rng.choice(pieces)
            if self._expanded_by_quarto(body) is not None:
                expanded_before += 1
            output = _defuse_quarto_body(body)
            assert self._expanded_by_quarto(output) is None, repr(body)
            assert (
                output.replace("\u200b", "") == body.replace("\u200b", "")
                or "---" in body
            )
            assert _defuse_quarto_body(output) == output, repr(body)
        assert expanded_before > 100

    @staticmethod
    def _quarto_yaml_blocks(qmd):
        """Every YAML block quarto-cli's ``readYamlFromMarkdown`` merges.

        src/core/yaml.ts: newlines normalised by ``lines()``, HTML comments
        and fenced code removed, then ``kRegExYAML`` with /gm (JavaScript's
        ``^``/``$`` also break at U+2028 and U+2029). Fenced code is kept
        here, which only finds more blocks.
        """
        text = re.sub(r"\r\n?", "\n", qmd)
        text = re.sub(r"<!--[\W\w]*?-->", "", text)
        text = text.replace("\u2028", "\n").replace("\u2029", "\n")
        yaml_block = re.compile(
            r"(^)(---[ \t]*[\r\n]+(?![ \t]*[\r\n]+)[\W\w]*?[\r\n]+"
            r"(?:---|\.\.\.))([ \t]*)$",
            re.MULTILINE,
        )
        return [m.group(2) for m in yaml_block.finditer(text)]

    @pytest.mark.parametrize(
        "body",
        [
            "Intro.\n\n---\nengine: knitr\n---\n\n`r system('id')`",
            '---\npdf-engine-opt: "-shell-escape"\n...\n',
            "```yaml\n---\nfilters: [x.lua]\n---\n```",
            "> ---\n> engine: knitr\n> ---",
            "x\r---\r\nengine: knitr\r---",
            "x\u2028--- \u2028engine: knitr\u2028---",
            "<!-- c -->---\nengine: knitr\n---",
            "-<!-- -->--\nengine: knitr\n<!-- -->---",
        ],
        ids=[
            "plain",
            "dot-closer",
            "in-code-block",
            "blockquote",
            "cr-lines",
            "u2028-lines",
            "after-comment",
            "assembled-by-comments",
        ],
    )
    def test_quarto_body_cannot_open_a_yaml_metadata_block(self, body):
        """quarto-cli merges every ``---`` block in the file into the render
        metadata (src/core/yaml.ts, ``kRegExYAML`` /gm), so a block in the
        body could set ``engine``, ``filters`` or ``pdf-engine-opt``."""
        qmd = QuartoExporter().export_to_quarto(body, "Report")
        blocks = self._quarto_yaml_blocks(qmd)
        assert len(blocks) == 1, blocks
        assert blocks[0].startswith("---\ntitle:")

    @pytest.mark.parametrize(
        "body",
        [
            "Intro.\n\n+--------------+\n| ---          |\n"
            "| injected: x  |\n| ---          |\n+--------------+\n",
            "+-----+---------------+\n| a   | ---           |\n"
            "|     | injected: x   |\n|     | ...           |\n"
            "+-----+---------------+\n",
            "- item\n\n  ---\n  injected: x\n  ---\n",
            "> ---\n> injected: x\n> ---\n",
            "Text[^1].\n\n[^1]: Note.\n\n    ---\n    injected: x\n    ---\n",
            "Term\n\n:   ---\n    injected: x\n    ---\n",
            "Intro.\n\n---\ninjected: x\n---\n",
            "% Injected title\n% Injected author\n\nBody.\n",
        ],
        ids=[
            "grid-table-cell",
            "grid-table-second-cell",
            "list-item",
            "blockquote",
            "footnote",
            "definition",
            "top-level",
            "title-block",
        ],
    )
    def test_pandoc_reads_no_metadata_block_from_the_quarto_body(self, body):
        """Pandoc's yaml_metadata_block opens in any block context, a
        grid-table cell included, where the ``---`` does not end the line,
        and its pandoc_title_block at the start of the input. Quarto
        removes the front matter and passes the metadata with
        ``--metadata-file`` (src/command/render/pandoc.ts), then reads the
        rest with the file's ``from``; this runs the bundled Pandoc the
        same way."""
        pypandoc = pytest.importorskip("pypandoc")
        front, rendered = self._qmd_body(body)
        line = next(x for x in front.splitlines() if x.startswith("from:"))
        reader = line[len("from:") :].strip()
        control = json.loads(
            pypandoc.convert_text(
                body,
                "json",
                format=reader + "+yaml_metadata_block+pandoc_title_block",
            )
        )
        assert control["meta"], "control: Pandoc reads a block from the body"
        document = json.loads(
            pypandoc.convert_text(rendered, "json", format=reader)
        )
        assert document["meta"] == {}, document["meta"]

    @classmethod
    def _inline_types(cls, node, found):
        """Every Pandoc JSON node type (``"t"``) under *node*."""
        if isinstance(node, dict):
            if "t" in node:
                found.add(node["t"])
            for value in node.values():
                cls._inline_types(value, found)
        elif isinstance(node, list):
            for value in node:
                cls._inline_types(value, found)
        return found

    @pytest.mark.parametrize(
        ("title", "unescaped"),
        [
            (
                '`[RawInline (Format "html") "<b>x</b>"]`{=pandoc-native}',
                "RawInline",
            ),
            ('[x]{qmd-base64="e3s8IGVudiBIT01FID59fQ=="}', "Span"),
            ("<b>raw</b>", "RawInline"),
            ("Intro\n\n    code", "Code"),
            ("*em* $x$ [@ref1] <https://e.example>", "Emph"),
        ],
        ids=["pandoc-native", "qmd-base64-span", "raw-html", "code", "markup"],
    )
    def test_pandoc_reads_the_quarto_title_as_plain_text(
        self, tmp_path, title, unescaped
    ):
        """Quarto passes the front matter with ``--metadata-file`` and reads
        the body with a Lua custom reader (filters/qmd-reader.lua). Pandoc
        then parses metadata with all of its default markdown extensions,
        not with the file's ``from``, so raw inlines and spans in the title
        would reach Quarto's ``pandoc-native`` and ``qmd-base64`` decoders.
        This runs the bundled Pandoc the same way: the exported title must
        come out as plain text, and the same title unescaped must not."""
        pypandoc = pytest.importorskip("pypandoc")
        front, _body = self._qmd_body("Body.", title=title)
        line = next(x for x in front.splitlines() if x.startswith("from:"))
        reader = line[len("from:") :].strip()
        lua_reader = tmp_path / "reader.lua"
        lua_reader.write_text(
            "function Reader(input, opts)\n"
            f'  return pandoc.read(tostring(input), "{reader}", opts)\n'
            "end\n",
            encoding="utf-8",
        )
        source = tmp_path / "body.md"
        source.write_text("Body.\n", encoding="utf-8")

        def meta_title(front_matter):
            metadata = tmp_path / "metadata.yaml"
            metadata.write_text(front_matter, encoding="utf-8")
            completed = subprocess.run(
                [
                    pypandoc.get_pandoc_path(),
                    "--from",
                    str(lua_reader),
                    "--metadata-file",
                    str(metadata),
                    "--to",
                    "json",
                    str(source),
                ],
                capture_output=True,
                text=True,
                check=True,
                timeout=60,
            )
            return json.loads(completed.stdout)["meta"]["title"]

        # Control: the title as plain YAML parses into markup.
        quoted = json.dumps(title)
        control = self._inline_types(meta_title(f"title: {quoted}\n"), set())
        assert unescaped in control, control

        # The exported front matter, as Quarto hands it to Pandoc.
        exported = meta_title(front[len("---\n") : -len("---\n\n")])
        assert exported["t"] == "MetaInlines", exported
        assert self._inline_types(exported, set()) <= {
            "MetaInlines",
            "Str",
            "Space",
        }, exported
        text = "".join(
            node["c"] if node["t"] == "Str" else " " for node in exported["c"]
        )
        assert text.replace("\u200b", "") == " ".join(title.split()), text

    @pytest.mark.parametrize(
        ("body", "expected"),
        [
            # Thematic break and setext underline: ``----`` renders the same.
            ("a\n\n---\n\nb", "a\n\n----\n\nb"),
            ("Heading\n---\ntext", "Heading\n----\ntext"),
            ("  --- \t", "  ---- \t"),
            # Not three dashes ending a line: left alone.
            ("----\n- - -\n---x\n-----", "----\n- - -\n---x\n-----"),
            # Three dashes ending a URL or a word open no block either.
            (
                "URL: https://e.example/a---\nx---",
                "URL: https://e.example/a---\nx---",
            ),
            # After a block prefix they could: ``>``, a footnote's ``:``.
            ("> ---\n[^1]:---", "> ----\n[^1]:----"),
            # Any run of whitespace, prefix characters and footnote labels.
            (
                "> > ---\n:   ---\n~ ---\n| ---\n\t> [^a b]: ---\n>|:~---",
                "> > ----\n:   ----\n~ ----\n| ----\n\t> [^a b]: ----\n"
                ">|:~----",
            ),
            # Any other text before them on the line: left alone, so prose
            # keeps its em dash.
            (
                "He paused ---\nthen went on.\nx>---\n1 ---\nx:---\n- ---\n"
                "[^1] ---\n[x]: ---\n[^]: ---",
                "He paused ---\nthen went on.\nx>---\n1 ---\nx:---\n- ---\n"
                "[^1] ---\n[x]: ---\n[^]: ---",
            ),
        ],
        ids=[
            "thematic-break",
            "setext",
            "indented",
            "controls",
            "url",
            "block-prefix",
            "nested-prefixes",
            "prose",
        ],
    )
    def test_quarto_dash_lines_keep_their_markdown_meaning(
        self, body, expected
    ):
        _front, rendered = self._qmd_body(body)
        assert rendered == expected

    @pytest.mark.parametrize(
        "body",
        [
            "He paused ---\nthen went on.",
            "The answer is clear ---",
            "a\n\n---\n\nb",
            "Heading\n---\ntext",
            "> quoted\n> ---\n> more",
            "Term\n\n:   ---\n",
            "Text[^1].\n\n[^1]: ---\n",
            # A rule directly followed by text, closed after a blank line.
            # The four dashes of the defused line must be the same document
            # as the three that were written.
            "---\nParagraph after the first rule.\n\n---",
        ],
        ids=[
            "prose",
            "prose-at-end",
            "thematic-break",
            "setext",
            "blockquote",
            "definition",
            "footnote",
            "rule-followed-by-text",
        ],
    )
    def test_pandoc_renders_dash_lines_as_written(self, body):
        """The bundled Pandoc reads the exported body, with the file's
        ``from``, to the same document as the body as written: prose
        ending in `` ---`` keeps its em dash, and the four dashes of a
        defused line are the same thematic break or setext underline."""
        pypandoc = pytest.importorskip("pypandoc")
        front, rendered = self._qmd_body(body)
        line = next(x for x in front.splitlines() if x.startswith("from:"))
        reader = line[len("from:") :].strip()

        def blocks(text):
            return json.loads(
                pypandoc.convert_text(text, "json", format=reader)
            )["blocks"]

        assert blocks(rendered) == blocks(body)
        if body.startswith(("He paused", "The answer")):
            assert rendered == body
            assert "\u2014" in json.dumps(blocks(rendered), ensure_ascii=False)

    @pytest.mark.parametrize(
        "body",
        [
            "---\nParagraph after the first rule.\n\n---",
            "Intro text.\n\n---\n**Total Sources Organized:** 14 across 3"
            " topics\n\n## Sources\n\n[1] Example source\n\n---\nFooter.",
        ],
        ids=["rule-followed-by-text", "topic-organization"],
    )
    def test_pandoc_reads_a_rule_followed_by_text_as_a_rule(self, body):
        """A ``---`` line directly followed by text, with the next rule
        after a blank line, stays two horizontal rules under the exported
        reader.

        ``topic_organization_strategy.py`` emits this shape
        (``\\n---\\n**Total Sources Organized:**``). With
        ``multiline_tables`` on, Pandoc reads the defused ``----`` lines
        as a one-column table and the rest of the section, headings
        included, lands in that cell. This pins the blocks the file's
        ``from`` produces against a control that turns the extension
        back on.
        """
        pypandoc = pytest.importorskip("pypandoc")
        front, rendered = self._qmd_body(body)
        line = next(x for x in front.splitlines() if x.startswith("from:"))
        reader = line[len("from:") :].strip()

        def blocks_of(fmt):
            return json.loads(
                pypandoc.convert_text(rendered, "json", format=fmt)
            )["blocks"]

        control_blocks = blocks_of(reader + "+multiline_tables")
        blocks = blocks_of(reader)
        assert any(b["t"] == "Table" for b in control_blocks), control_blocks
        assert all(b["t"] != "Table" for b in blocks), blocks
        assert [b["t"] for b in blocks].count("HorizontalRule") == 2, [
            b["t"] for b in blocks
        ]
        if "## Sources" in body:
            assert any(b["t"] == "Header" for b in blocks), blocks

    def test_pandoc_keeps_lines_after_an_indented_rule_whole(self):
        """An indented ``---`` followed by text, closed after a blank
        line, does not cut the lines after it.

        Read as a multiline table, the indent of the rule is the column
        start, so ``ABCDEF`` / ``GHIJKL`` become ``CDEF`` / ``IJKL``
        inside one cell. The exported reader keeps both rules and both
        lines whole.
        """
        pypandoc = pytest.importorskip("pypandoc")
        body = "Intro.\n\n  ---\nABCDEF line\nGHIJKL line\n\n---\nAfter.\n"
        front, rendered = self._qmd_body(body)
        line = next(x for x in front.splitlines() if x.startswith("from:"))
        reader = line[len("from:") :].strip()

        def blocks_of(fmt):
            return json.loads(
                pypandoc.convert_text(rendered, "json", format=fmt)
            )["blocks"]

        def strings(node):
            found = []

            def walk(value):
                if isinstance(value, dict):
                    if value.get("t") == "Str":
                        found.append(value["c"])
                    for child in value.values():
                        walk(child)
                elif isinstance(value, list):
                    for child in value:
                        walk(child)

            walk(node)
            return found

        control_blocks = blocks_of(reader + "+multiline_tables")
        blocks = blocks_of(reader)
        control_cell = strings(
            [block for block in control_blocks if block["t"] == "Table"]
        )
        assert "CDEF" in control_cell, control_cell
        assert "IJKL" in control_cell, control_cell
        assert "ABCDEF" not in control_cell, control_cell
        assert "GHIJKL" not in control_cell, control_cell
        assert all(b["t"] != "Table" for b in blocks), blocks
        assert [b["t"] for b in blocks].count("HorizontalRule") == 2, [
            b["t"] for b in blocks
        ]
        assert "ABCDEF" in strings(blocks), strings(blocks)
        assert "GHIJKL" in strings(blocks), strings(blocks)

    @pytest.mark.parametrize(
        "body",
        [
            "| Column A | Column B |\n|----------|----------|\n| 1 | two |\n",
            "---\n  1     two\n  3     four\n---\n",
        ],
        ids=["pipe-table", "simple-table"],
    )
    def test_pandoc_still_reads_ordinary_tables(self, body):
        """Pipe tables, and a headerless simple table whose rules have no
        blank line between them, stay tables with ``multiline_tables`` off."""
        pypandoc = pytest.importorskip("pypandoc")
        front, rendered = self._qmd_body(body)
        line = next(x for x in front.splitlines() if x.startswith("from:"))
        reader = line[len("from:") :].strip()
        blocks = json.loads(
            pypandoc.convert_text(rendered, "json", format=reader)
        )["blocks"]
        assert any(b["t"] == "Table" for b in blocks), blocks

    @pytest.mark.parametrize(
        "unit",
        [
            "`",
            "{{< ",
            "<!-",
            "--- ",
            "---\n",
            "{{{{",
            "{{{<",
            "```{r} ",
            "&#123;",
            "&amp;x",
            "{\\{ ",
            "\\",
            "{ ",
            "b58fc729\\-690b-",
            "cf5733e5-0370-",
            "quarto-shortcode_",
            "&",
            "&a",
            "&#",
            "&a;",
            "&;",
            "{{\\",
        ],
    )
    def test_quarto_body_defusion_is_a_linear_fixed_point(self, unit):
        body = unit * 50000
        started = time.thread_time()
        once = self._qmd_body(body)[1]
        elapsed = time.thread_time() - started
        assert elapsed < 1.0, f"body defusion took {elapsed:.2f}s"
        assert self._qmd_body(once)[1] == once

    @pytest.mark.parametrize(
        "body",
        [
            "&" * 200000 + ";",
            "&a" * 100000 + ";",
            "&#" * 100000 + ";",
            "&lbrace" * 30000 + ";",
            "&a " * 70000 + ";",
        ],
        ids=["ampersands", "names", "numeric", "long-name", "spaced"],
    )
    def test_quarto_entity_scan_is_linear(self, body):
        """Each ``&`` reads up to the next ``;`` (Pandoc's
        ``characterReference``); re-reading that span per ``&`` would be
        quadratic in a run of them."""
        started = time.thread_time()
        _defuse_quarto_body(body)
        elapsed = time.thread_time() - started
        assert elapsed < 1.0, f"entity scan took {elapsed:.2f}s"

    def test_collection_line_scan_is_not_quadratic_in_blank_lines(self):
        sources_block = (
            "[1] Title\nURL: https://x.example\n" + "\n" * 24000 + "end\n"
        )
        formatter = CitationFormatter(CitationMode.SOURCE_TAGGED_HYPERLINKS)
        started = time.thread_time()
        collections = formatter._parse_collections(sources_block)
        elapsed = time.thread_time() - started
        assert collections == {}
        assert elapsed < 0.1, (
            f"collection scan took {elapsed:.2f}s on 24k blank lines"
        )

    def test_collection_line_scan_still_finds_a_real_collection(self):
        """Control: the pattern the previous test measures still works."""
        formatter = CitationFormatter(CitationMode.SOURCE_TAGGED_HYPERLINKS)
        block = (
            "[1] Handbook\nURL: /library/document/7\nCollection: My Papers\n"
        )
        assert formatter._parse_collections(block) == {"1": "My Papers"}

    def test_collection_fields_stay_with_their_header_and_group(self):
        formatter = CitationFormatter(CitationMode.SOURCE_TAGGED_HYPERLINKS)
        block = (
            "Collection: Unattached\n"
            "[1, 003] Grouped document\nURL: /library/document/7?q=[value]\n"
            "\n   Collection: My Papers\nCollection: Ignored second field\n"
            "[2] No collection\nURL: https://example.org\n"
            "[broken] Invalid header\nCollection: Must not attach to 2\n"
            "[4] Last\nCollection: \n  Collection: Final collection"
        )
        assert formatter._parse_collections(block) == {
            "1": "My Papers",
            "003": "My Papers",
            "4": "Final collection",
        }

    def test_author_delimiters_preserve_names_and_whitespace(self):
        ris = RISExporter()._create_ris_entry(
            "1", "Title by Sandy Anderson, Alice\tand\tBob & Carol."
        )
        authors = [
            line for line in ris.splitlines() if line.startswith("AU  -")
        ]
        assert authors == [
            "AU  - Sandy Anderson",
            "AU  - Alice",
            "AU  - Bob",
            "AU  - Carol",
        ]