Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

76.8 KB
Raw Blame
"""
Tests for RISExporter._create_ris_entry() edge cases.

Existing tests cover: basic ID, year extraction, DOI from metadata,
GitHub publisher, and arXiv publisher.

This file adds coverage for the many remaining branches:
- Author parsing (split by 'and', '&', commas)
- DOI extraction from text (not just metadata)
- Publisher detection for Reddit, YouTube, Medium, PyPI, generic domains
- www. prefix stripping
- Title cleaning (DOI, "Published in", "Volume", "Pages" patterns)
- Empty/minimal inputs
- Author match overlapping with title cleaning
"""

import re
import time

import pytest
from freezegun import freeze_time

from local_deep_research.text_optimization import citation_formatter
from local_deep_research.text_optimization.citation_formatter import RISExporter

# Passwords for the credential fixtures, each with the parts of it that no
# field may hold. The fixtures are written with ``secret`` in a URL's
# userinfo. It matches no pattern that cuts a field out of text, so with it
# alone such a cut could not show. The others match those patterns: a year
# (PY), a ``DOI:`` label (DO), and the commas, ampersands and full stops at
# which the author list is split and ended (AU).
_PASSWORDS = [
    pytest.param("secret", ("secret",), id="word"),
    pytest.param("1987", ("1987",), id="year"),
    pytest.param("tok-1999-x", ("1999",), id="year-in-a-token"),
    pytest.param("DOI:10.5555", ("5555",), id="doi"),
    pytest.param(
        "Xq7,Zr9&Wk3.Vm5", ("Xq7", "Zr9", "Wk3", "Vm5"), id="punctuation"
    ),
]


def _with_password(value, password):
    """``value`` with ``password`` in place of ``secret``."""
    assert "secret" in value, value
    return value.replace("secret", password)


def _assert_holds_no_part(result, parts):
    for part in parts:
        assert part not in result, (part, result)


class TestRISMultilineCredentialSources:
    """Check the unsplit source before line-based metadata can hide its @."""

    @pytest.mark.parametrize("separator", ["\n", "\r\n"])
    @pytest.mark.parametrize(
        "title",
        [
            "A harmless title by https: user:[email protected]/x. "
            "https://user:CutPassword957",
            "https:/by Smith./user:CutPassword957",
            "https:/by Smith./user:CutPassword957 Volume",
        ],
    )
    def test_title_cleanup_cannot_hide_a_remaining_fragment(
        self, title, separator
    ):
        exported = RISExporter().export_to_ris(
            f"## Sources\n[1] {title}{separator}@example.net/paper\n"
        )

        assert "CutPassword957" not in exported
        assert "TI  - [source URL omitted]" in exported

    @pytest.mark.parametrize("prefix", ["by", "URL:", "DOI:"])
    def test_source_guard_does_not_backtrack_over_whitespace(self, prefix):
        source = prefix + " " * 16_000 + "[email protected]\nPlain text"
        started = time.thread_time()

        assert not citation_formatter._ris_multiline_line_carries_userinfo(
            source, 0
        )

        # CPU time excludes competing test processes and VM suspension.
        # Ambiguous adjacent whitespace repeats take seconds here.
        assert time.thread_time() - started < 1.0

    @pytest.mark.parametrize("password,parts", _PASSWORDS)
    @pytest.mark.parametrize("separator", ["\n", "\r\n"])
    @pytest.mark.parametrize("field", ["title", "URL", "DOI", "author"])
    def test_line_split_cannot_export_a_password_fragment(
        self, password, parts, separator, field
    ):
        url = f"https://user:{password}{separator}@example.org/paper"
        source = {
            "title": url,
            "URL": f"A harmless title\nURL: {url}",
            "DOI": f"A harmless title\nDOI: {url}",
            "author": f"A harmless title\nby {url}",
        }[field]

        exported = RISExporter().export_to_ris(f"## Sources\n[1] {source}\n")

        assert exported.startswith("TY  - ELEC")
        _assert_holds_no_part(exported, parts)
        if field == "title":
            assert "TI  - [source URL omitted]" in exported
        else:
            assert "TI  - A harmless title" in exported

    @pytest.mark.parametrize(
        "url",
        [
            "https://user:secret%4\n0example.org/paper",
            "https://user:secret&com\nmat;example.org/paper",
            "ht\ntps://user:secret\n@example.org/paper",
            "https://user:\nsecret\n@example.org/paper",
            "https://user:secret&#6\n4;example.org/paper",
            "https://user:secret@\nexample.org/paper",
            "https: user:secret\n@example.org\nPublished in 2023",
            "https: user:sec\n[email protected]\nPublished in 2023",
        ],
    )
    def test_decoding_and_line_splitting_compose(self, url):
        exported = RISExporter().export_to_ris(
            f"## Sources\n[1] {url}\nDOI: {url}\nURL: {url}\n"
        )

        assert exported.startswith("TY  - ELEC")
        assert "secret" not in exported
        assert "TI  - [source URL omitted]" in exported

    @pytest.mark.parametrize(
        "title",
        [
            "HTTP: [email protected] explains",
            "HTTP: email me at [email protected]",
            "HTTP: @mention",
        ],
    )
    def test_multiline_record_keeps_prose_title_exception(self, title):
        exported = RISExporter().export_to_ris(
            f"## Sources\n[1] {title}\nPublished in 2024\n"
            "URL: https://example.org/paper\n"
        )

        assert f"TI  - {title}" in exported
        assert "PY  - 2024" in exported
        assert "UR  - https://example.org/paper" in exported

    @pytest.mark.parametrize("line_ending", ["\n", "\r\n"])
    def test_safe_fields_outside_a_split_credential_keep_their_values(
        self, line_ending
    ):
        source = line_ending.join(
            [
                "## Sources",
                "[1] A harmless title",
                "by Alice and Bob.",
                "Published in 2024",
                "URL: https://user:secret",
                "@example.org/paper",
                "DOI: 10.1234/safe",
            ]
        )

        exported = RISExporter().export_to_ris(source)

        assert "secret" not in exported
        for field in (
            "TI  - A harmless title",
            "AU  - Alice",
            "AU  - Bob",
            "PY  - 2024",
            "DO  - 10.1234/safe",
        ):
            assert field in exported

    def test_source_lines_without_credentials_keep_their_metadata(self):
        source = (
            "## Sources\n[1] A harmless title\nby Alice and Bob.\n"
            "Published in 2024\nDOI: 10.1234/safe\n"
            "URL: https://example.org/@author\n"
        )

        exported = RISExporter().export_to_ris(source)

        for field in (
            "TI  - A harmless title",
            "AU  - Alice",
            "AU  - Bob",
            "PY  - 2024",
            "DO  - 10.1234/safe",
            "UR  - https://example.org/@author",
        ):
            assert field in exported

    def test_a_complete_inline_credential_does_not_discard_the_cleaned_title(
        self,
    ):
        exported = RISExporter().export_to_ris(
            "## Sources\n[1] A harmless title by https: user:secret@host\n"
            "URL: https://example.org/paper\n"
        )

        assert "secret" not in exported
        assert "TI  - A harmless title" in exported
        assert "UR  - https://example.org/paper" in exported


class TestRISEntryAuthorParsing:
    """Tests for author extraction from 'by Author1, Author2' patterns."""

    def _make(self):
        return RISExporter()

    def test_single_author(self):
        """Extracts a single author."""
        exporter = self._make()
        result = exporter._create_ris_entry(
            "1", "Some Title\nWritten by Jane Doe."
        )
        assert "AU  - Jane Doe" in result

    def test_authors_separated_by_and(self):
        """Authors separated by 'and' are split correctly."""
        exporter = self._make()
        result = exporter._create_ris_entry(
            "1", "Title\nby Alice Smith and Bob Jones."
        )
        assert "AU  - Alice Smith" in result
        assert "AU  - Bob Jones" in result

    def test_authors_separated_by_ampersand(self):
        """Authors separated by '&' are split correctly."""
        exporter = self._make()
        result = exporter._create_ris_entry(
            "1", "Title\nby Alice Smith & Bob Jones."
        )
        assert "AU  - Alice Smith" in result
        assert "AU  - Bob Jones" in result

    def test_authors_separated_by_comma(self):
        """Authors separated by commas are split correctly."""
        exporter = self._make()
        result = exporter._create_ris_entry(
            "1", "Title\nby Alice Smith, Bob Jones, Carol White."
        )
        assert "AU  - Alice Smith" in result
        assert "AU  - Bob Jones" in result
        assert "AU  - Carol White" in result

    def test_authors_mixed_separators(self):
        """Authors with mixed 'and' and comma separators."""
        exporter = self._make()
        result = exporter._create_ris_entry(
            "1", "Title\nby Alice, Bob and Carol."
        )
        assert "AU  - Alice" in result
        assert "AU  - Bob" in result
        assert "AU  - Carol" in result

    def test_no_authors(self):
        """No author line produces no AU entries."""
        exporter = self._make()
        result = exporter._create_ris_entry("1", "Title with no authors")
        assert "AU  -" not in result

    def test_by_keyword_case_insensitive(self):
        """The 'by' keyword match is case-insensitive."""
        exporter = self._make()
        result = exporter._create_ris_entry("1", "Title\nBY John Smith.")
        assert "AU  - John Smith" in result

    def test_empty_author_parts_filtered(self):
        """Empty parts from splitting are filtered out."""
        exporter = self._make()
        # Trailing comma leaves an empty part
        result = exporter._create_ris_entry("1", "Title\nby Alice, , Bob.")
        lines = [
            line for line in result.split("\n") if line.startswith("AU  -")
        ]
        # Should only have real authors, not empty strings
        for line in lines:
            assert line.strip() != "AU  -"


class TestRISEntryDOIExtraction:
    """Tests for DOI extraction from metadata and text."""

    def _make(self):
        return RISExporter()

    def test_doi_from_metadata_takes_precedence(self):
        """DOI from metadata dict is used over DOI in text."""
        exporter = self._make()
        result = exporter._create_ris_entry(
            "1",
            "Title DOI: 10.9999/text-doi",
            metadata={"doi": "10.1234/meta-doi"},
        )
        assert "DO  - 10.1234/meta-doi" in result
        assert (
            "10.9999/text-doi" not in result.split("DO  - ")[1].split("\n")[0]
        )

    def test_doi_from_text(self):
        """DOI extracted from text when not in metadata."""
        exporter = self._make()
        result = exporter._create_ris_entry(
            "1", "Title\nDOI: 10.1234/test.5678"
        )
        assert "DO  - 10.1234/test.5678" in result

    def test_doi_from_text_case_insensitive(self):
        """DOI extraction from text is case-insensitive."""
        exporter = self._make()
        result = exporter._create_ris_entry(
            "1", "Title\ndoi: 10.1234/lowercase"
        )
        assert "DO  - 10.1234/lowercase" in result

    def test_no_doi(self):
        """No DOI produces no DO entry."""
        exporter = self._make()
        result = exporter._create_ris_entry("1", "Title with no DOI")
        assert "DO  -" not in result

    def test_metadata_none_falls_back_to_text(self):
        """When metadata is None, DOI is extracted from text."""
        exporter = self._make()
        result = exporter._create_ris_entry("1", "Title\nDOI: 10.5555/fallback")
        assert "DO  - 10.5555/fallback" in result

    def test_metadata_without_doi_key(self):
        """Metadata dict without 'doi' key falls back to text."""
        exporter = self._make()
        result = exporter._create_ris_entry(
            "1",
            "Title\nDOI: 10.7777/textdoi",
            metadata={"other": "value"},
        )
        assert "DO  - 10.7777/textdoi" in result


class TestRISEntryPublisherDetection:
    """Tests for publisher extraction from URL domains."""

    def _make(self):
        return RISExporter()

    def test_reddit_publisher(self):
        """Reddit URL produces Reddit publisher."""
        exporter = self._make()
        result = exporter._create_ris_entry(
            "1", "Title", url="https://www.reddit.com/r/python/post"
        )
        assert "PB  - Reddit" in result

    def test_reddit_subdomain(self):
        """Reddit subdomain also detected."""
        exporter = self._make()
        result = exporter._create_ris_entry(
            "1", "Title", url="https://old.reddit.com/r/science"
        )
        assert "PB  - Reddit" in result

    def test_youtube_publisher(self):
        """YouTube URL produces YouTube publisher."""
        exporter = self._make()
        result = exporter._create_ris_entry(
            "1", "Title", url="https://www.youtube.com/watch?v=abc123"
        )
        assert "PB  - YouTube" in result

    def test_youtube_mobile(self):
        """Mobile YouTube URL produces YouTube publisher."""
        exporter = self._make()
        result = exporter._create_ris_entry(
            "1", "Title", url="https://m.youtube.com/watch?v=abc123"
        )
        assert "PB  - YouTube" in result

    def test_medium_publisher(self):
        """Medium URL produces Medium publisher."""
        exporter = self._make()
        result = exporter._create_ris_entry(
            "1", "Title", url="https://medium.com/@author/article"
        )
        assert "PB  - Medium" in result

    def test_medium_subdomain(self):
        """Medium subdomain (custom publication) detected."""
        exporter = self._make()
        result = exporter._create_ris_entry(
            "1", "Title", url="https://blog.medium.com/post"
        )
        assert "PB  - Medium" in result

    def test_pypi_publisher(self):
        """PyPI URL produces PyPI publisher."""
        exporter = self._make()
        result = exporter._create_ris_entry(
            "1", "Title", url="https://pypi.org/project/requests/"
        )
        assert "PB  - Python Package Index (PyPI)" in result

    def test_generic_domain_as_publisher(self):
        """Unknown domain used as publisher name."""
        exporter = self._make()
        result = exporter._create_ris_entry(
            "1", "Title", url="https://example.com/article"
        )
        assert "PB  - example.com" in result

    def test_www_stripped_from_domain(self):
        """www. prefix is stripped from domain."""
        exporter = self._make()
        result = exporter._create_ris_entry(
            "1", "Title", url="https://www.example.org/page"
        )
        assert "PB  - example.org" in result

    def test_no_url_no_publisher(self):
        """No URL produces no PB or UR entries."""
        exporter = self._make()
        result = exporter._create_ris_entry("1", "Title")
        assert "PB  -" not in result
        assert "UR  -" not in result

    def test_empty_url_no_publisher(self):
        """Empty string URL produces no PB or UR entries."""
        exporter = self._make()
        result = exporter._create_ris_entry("1", "Title", url="")
        assert "PB  -" not in result
        assert "UR  -" not in result

    def test_github_subdomain(self):
        """GitHub subdomain detected as GitHub."""
        exporter = self._make()
        result = exporter._create_ris_entry(
            "1", "Title", url="https://pages.github.com/project"
        )
        assert "PB  - GitHub" in result

    def test_arxiv_subdomain(self):
        """arXiv subdomain detected as arXiv."""
        exporter = self._make()
        result = exporter._create_ris_entry(
            "1", "Title", url="https://export.arxiv.org/abs/1234"
        )
        assert "PB  - arXiv" in result

    @pytest.mark.parametrize(
        "url,publisher",
        [
            ("https://user:[email protected]/abs/1", "arXiv"),
            ("https://user:[email protected]/report", "example.com"),
            ("https://[email protected]/abs/1", "evil.example"),
            ("https://example.com:8080/article", "example.com"),
            ("https://ArXiv.org/abs/1", "arXiv"),
            ("https://arxiv.org./abs/1", "arXiv"),
        ],
    )
    def test_the_publisher_is_the_host_alone(self, url, publisher):
        """Userinfo and port are not the publisher; credentials in the
        netloc were written into the .ris file (#6837)."""
        result = self._make()._create_ris_entry("1", "Title", url=url)
        assert f"PB  - {publisher}\n" in result


class TestRISEntryTitleCleaning:
    """Tests for title cleaning patterns."""

    def _make(self):
        return RISExporter()

    def test_doi_removed_from_title(self):
        """DOI text is removed from the title."""
        exporter = self._make()
        result = exporter._create_ris_entry(
            "1", "My Paper DOI: 10.1234/test\nMore content"
        )
        # The title line should not contain DOI
        ti_line = [
            line for line in result.split("\n") if line.startswith("TI  -")
        ][0]
        assert "DOI:" not in ti_line

    def test_published_in_removed_from_title(self):
        """'Published in ...' is removed from the title."""
        exporter = self._make()
        result = exporter._create_ris_entry(
            "1", "My Paper Published in Nature 2023\nContent"
        )
        ti_line = [
            line for line in result.split("\n") if line.startswith("TI  -")
        ][0]
        assert "Published in" not in ti_line

    def test_volume_removed_from_title(self):
        """'Volume ...' is removed from the title."""
        exporter = self._make()
        result = exporter._create_ris_entry(
            "1", "My Paper Volume 42 Issue 3\nContent"
        )
        ti_line = [
            line for line in result.split("\n") if line.startswith("TI  -")
        ][0]
        assert "Volume" not in ti_line

    def test_pages_removed_from_title(self):
        """'Pages ...' is removed from the title."""
        exporter = self._make()
        result = exporter._create_ris_entry(
            "1", "My Paper Pages 100-120\nContent"
        )
        ti_line = [
            line for line in result.split("\n") if line.startswith("TI  -")
        ][0]
        assert "Pages" not in ti_line

    def test_fallback_to_original_title_when_clean_empty(self):
        """If cleaning empties the title, original title is used."""
        exporter = self._make()
        # Title that's entirely "by Author" where author match starts at 0
        result = exporter._create_ris_entry("1", "by John Smith\nContent")
        # Should still have a TI entry (either cleaned or fallback)
        ti_line = [
            line for line in result.split("\n") if line.startswith("TI  -")
        ][0]
        assert "TI  - " in ti_line


class TestRISEntryStructure:
    """Tests for overall RIS entry structure."""

    def _make(self):
        return RISExporter()

    def test_contains_ty_elec(self):
        """Entry contains TY - ELEC type marker."""
        exporter = self._make()
        result = exporter._create_ris_entry("1", "Title")
        assert "TY  - ELEC" in result

    def test_ends_with_er(self):
        """Entry ends with ER marker."""
        exporter = self._make()
        result = exporter._create_ris_entry("1", "Title")
        lines = result.split("\n")
        assert lines[-1] == "ER  - "

    def test_contains_language(self):
        """Entry contains language tag."""
        exporter = self._make()
        result = exporter._create_ris_entry("1", "Title")
        assert "LA  - en" in result

    @freeze_time("2025-06-15")
    def test_contains_access_year(self):
        """Entry contains current access year."""
        exporter = self._make()
        result = exporter._create_ris_entry("1", "Title")
        assert "Y1  - 2025" in result

    @freeze_time("2025-06-15")
    def test_contains_access_date(self):
        """Entry contains formatted access date."""
        exporter = self._make()
        result = exporter._create_ris_entry("1", "Title")
        assert "DA  - 2025/06/15" in result

    def test_ref_id_format(self):
        """Reference ID formatted as 'refN'."""
        exporter = self._make()
        result = exporter._create_ris_entry("42", "Title")
        assert "ID  - ref42" in result


class TestRISEntryYearExtraction:
    """Tests for year extraction from text."""

    def _make(self):
        return RISExporter()

    def test_year_1990s(self):
        """Extracts 1990s year."""
        exporter = self._make()
        result = exporter._create_ris_entry("1", "Published in 1995")
        assert "PY  - 1995" in result

    def test_year_2000s(self):
        """Extracts 2000s year."""
        exporter = self._make()
        result = exporter._create_ris_entry("1", "Released 2003")
        assert "PY  - 2003" in result

    def test_no_year(self):
        """No year in text produces no PY entry."""
        exporter = self._make()
        result = exporter._create_ris_entry("1", "Title with no date")
        assert "PY  -" not in result

    def test_first_year_extracted(self):
        """First valid year in text is extracted."""
        exporter = self._make()
        result = exporter._create_ris_entry(
            "1", "Written in 2020, revised in 2023"
        )
        assert "PY  - 2020" in result

    def test_year_boundary_1900(self):
        """Year 1900 is valid (19xx pattern)."""
        exporter = self._make()
        result = exporter._create_ris_entry("1", "Historical text from 1900")
        assert "PY  - 1900" in result

    def test_non_year_four_digit_ignored(self):
        """Four-digit numbers outside 19xx-20xx range are not matched."""
        exporter = self._make()
        result = exporter._create_ris_entry(
            "1", "Article number 3456 in series"
        )
        assert "PY  -" not in result


class TestRISEntryURLHandling:
    """Tests for URL handling in RIS entries."""

    def _make(self):
        return RISExporter()

    def test_url_included(self):
        """URL is included as UR entry."""
        exporter = self._make()
        result = exporter._create_ris_entry(
            "1", "Title", url="https://example.com"
        )
        assert "UR  - https://example.com" in result

    def test_url_with_path(self):
        """Full URL with path is preserved."""
        exporter = self._make()
        url = "https://example.com/path/to/article?id=123"
        result = exporter._create_ris_entry("1", "Title", url=url)
        assert f"UR  - {url}" in result

    def test_export_strips_userinfo_without_losing_path_or_query(self):
        document = (
            "Report\n\n## Sources\n\n"
            "[1] Credential leak probe\n"
            "URL: https://user:[email protected]/abs/2401.00001?q=a&b=2"
        )

        result = self._make().export_to_ris(document)

        assert "UR  - https://arxiv.org/abs/2401.00001?q=a&b=2" in result
        assert "PB  - arXiv" in result
        assert "secret-token" not in result

    @pytest.mark.parametrize("at_entity", ["&commat;", "&#64;", "&#x40;"])
    def test_export_refuses_userinfo_remaining_after_stripping(self, at_entity):
        url = f"https://first:password@secret{at_entity}example.org/doc"

        result = self._make()._create_ris_entry("1", url, url=url)

        assert "TI  - [source URL omitted]" in result
        assert "UR  -" not in result
        assert "PB  -" not in result
        assert "secret" not in result
        assert "password" not in result

    def test_url_title_does_not_export_credentials(self):
        result = self._make()._create_ris_entry(
            "1",
            "https://user:[email protected]/article",
            url="https://user:[email protected]/article",
        )

        assert "TI  - https://example.org/article" in result
        assert "UR  - https://example.org/article" in result
        assert "secret-token" not in result

    def test_ris_url_with_control_is_omitted(self):
        result = self._make()._create_ris_entry(
            "1", "Title", url="https://example.org/a\rER  - injected"
        )

        assert "UR  -" not in result
        assert result.count("ER  -") == 1

    def test_malformed_credential_url_is_omitted(self):
        result = self._make()._create_ris_entry(
            "1", "Title", url="https:user:[email protected]/article"
        )

        assert "UR  -" not in result
        assert "secret-token" not in result

    def test_ris_url_with_at_in_path_is_preserved(self):
        result = self._make()._create_ris_entry(
            "1", "Title", url="https://medium.com/@author/article"
        )

        assert "UR  - https://medium.com/@author/article" in result

    @pytest.mark.parametrize(
        "url",
        [
            "https&#58;//user:[email protected]/a",
            "https&colon;//user:[email protected]/a",
            "https&#x3a;//user:[email protected]/a",
        ],
    )
    def test_ris_url_with_entity_encoded_scheme_is_omitted(self, url):
        """The entity breaks the scheme token for ``urlsplit``, but a
        consumer that decodes HTML entities follows a live credential
        URL, so the field must not survive verbatim."""
        result = self._make()._create_ris_entry("1", "Title", url=url)

        assert "UR  -" not in result
        assert "user:pass" not in result

    @pytest.mark.parametrize(
        "title",
        [
            "https:user:[email protected]/article",
            "https:/user:[email protected]/x",
            "https://user:[email protected]/x",
            "ftp://user:[email protected]/x",
            "https&#58;//user:[email protected]/a",
            "https&colon;//user:[email protected]/a",
        ],
    )
    def test_url_title_gate_covers_non_prefix_credential_spellings(self, title):
        """The title must not keep a credential spelling the shared check
        flags, not only one after a literal ``http(s)://`` prefix."""
        result = self._make()._create_ris_entry("1", title, url="")

        ti_line = [
            line for line in result.split("\n") if line.startswith("TI  -")
        ][0]
        assert "user:pass" not in ti_line
        assert "user:secret" not in ti_line
        assert "@example.org" not in ti_line

    @pytest.mark.parametrize(
        "title",
        [
            "https://example.org/article",
            "https://medium.com/@author/article",
            "Notes from [email protected] on https://example.org/x",
        ],
    )
    def test_url_title_gate_keeps_non_credential_titles(self, title):
        """A plain URL title, an @ in the path, and an email in prose are
        not credentials and must survive the gate verbatim."""
        result = self._make()._create_ris_entry("1", title)

        assert f"TI  - {title}" in result


class TestRISSharedCredentialCheck:
    """One credential check gates UR, TI, AU and DO, and no author, year or
    DOI is cut out of a line that carries a credential.

    Every refused value below carries ``secret`` in a URL's userinfo, so
    no spelling of it may reach any field of the RIS record. The tests that
    take ``password`` also run each value with the other passwords of
    ``_PASSWORDS`` in place of ``secret``, and no part of one may reach a
    field either.
    """

    def _make(self):
        return RISExporter()

    @pytest.mark.parametrize("password,parts", _PASSWORDS)
    @pytest.mark.parametrize(
        "value",
        [
            # urlsplit raises ValueError on each of these. The UR sanitizer
            # already failed closed; the title check read the error as "no
            # credential" and exported the title verbatim.
            pytest.param(
                "https://user:[email protected]\uff1a8443/x",
                id="nfkc-fullwidth-colon",
            ),
            pytest.param(
                "https://[email protected]\uff0fowner/repo",
                id="nfkc-fullwidth-solidus",
            ),
            pytest.param(
                "https://user:secret@exa\u2100mple.org/x",
                id="nfkc-account-of",
            ),
            pytest.param(
                "https://user:[email protected]\ufe56/x",
                id="nfkc-small-question-mark",
            ),
            pytest.param(
                "https://user&colon;secret&commat;example.org&incare;x",
                id="entities-then-care-of",
            ),
            pytest.param(
                "https://a:b@user&colon;secret&commat;example.org&incare;x",
                id="literal-then-entities-then-care-of",
            ),
            pytest.param(
                "https://secret-token&commat;github.com&incare;x",
                id="entity-username-then-care-of",
            ),
            pytest.param(
                "https://user:[email protected]]/x",
                id="unbalanced-close-bracket",
            ),
            pytest.param(
                "https://user:secret@[broken/x",
                id="unbalanced-open-bracket",
            ),
            pytest.param(
                "https://[@usersecret]&#64;example.org/x",
                id="bracketed-userinfo-then-entity-at",
            ),
        ],
    )
    def test_unparseable_credential_url_fails_closed(
        self, value, password, parts
    ):
        value = _with_password(value, password)
        result = self._make()._create_ris_entry("1", value, url=value)

        assert "TI  - [source URL omitted]\n" in result
        assert "UR  -" not in result
        _assert_holds_no_part(result, parts)

    @pytest.mark.parametrize("password,parts", _PASSWORDS)
    @pytest.mark.parametrize(
        "value,expected",
        [
            (
                "git+https://user:[email protected]/o/r",
                "git+https://github.com/o/r",
            ),
            ("sftp://user:[email protected]/x", "sftp://example.org/x"),
            ("ssh://git:[email protected]/r", "ssh://gitlab.com/r"),
            ("wss://user:[email protected]/x", "wss://example.org/x"),
            ("//user:[email protected]/x", "//example.org/x"),
        ],
    )
    def test_userinfo_is_stripped_under_any_scheme(
        self, value, expected, password, parts
    ):
        """The title check once accepted only http, https and ftp, so these
        titles kept the credentials the UR field stripped."""
        value = _with_password(value, password)
        result = self._make()._create_ris_entry("1", value, url=value)

        assert f"TI  - {expected}\n" in result
        assert f"UR  - {expected}\n" in result
        _assert_holds_no_part(result, parts)

    @pytest.mark.parametrize("password,parts", _PASSWORDS)
    @pytest.mark.parametrize(
        "value",
        [
            pytest.param(
                "&#32;https:/user:[email protected]/x", id="decimal-entity"
            ),
            pytest.param(
                "&#x20;https:/user:[email protected]/x", id="hex-entity"
            ),
            pytest.param(
                "&#32https:/user:[email protected]/x", id="no-semicolon"
            ),
            pytest.param(
                "&#0032;https:/user:[email protected]/x", id="zero-padded"
            ),
            pytest.param(
                "&#32;&#32;https:/user:[email protected]/x", id="two-spaces"
            ),
            pytest.param(
                "&#32;https:///user:[email protected]/x", id="triple-slash"
            ),
            pytest.param("&#9;https:/user:[email protected]/x", id="tab"),
        ],
    )
    def test_entity_encoded_leading_space_does_not_hide_userinfo(
        self, value, password, parts
    ):
        """urlsplit drops the decoded leading whitespace before it reads the
        scheme, so an authority cut at ``len(scheme) + 1`` began one
        character early, stopped at the first ``/`` and missed the @."""
        value = _with_password(value, password)
        result = self._make()._create_ris_entry("1", value, url=value)

        assert "TI  - [source URL omitted]\n" in result
        assert "UR  -" not in result
        _assert_holds_no_part(result, parts)

    @pytest.mark.parametrize("password,parts", _PASSWORDS)
    @pytest.mark.parametrize(
        "value",
        [
            pytest.param("wss://secret&#64;example.org/x", id="entity-at"),
            pytest.param(
                "https://user:secret%40example.org/x", id="percent-encoded-at"
            ),
            pytest.param(
                "https://user:secret\uff20example.org/x", id="fullwidth-at"
            ),
        ],
    )
    def test_encoded_at_sign_is_refused(self, value, password, parts):
        value = _with_password(value, password)
        result = self._make()._create_ris_entry("1", value, url=value)

        assert "TI  - [source URL omitted]\n" in result
        assert "UR  -" not in result
        assert "PB  -" not in result
        _assert_holds_no_part(result, parts)

    @pytest.mark.parametrize("password,parts", _PASSWORDS)
    @pytest.mark.parametrize(
        "value",
        [
            pytest.param(
                "<https://user:[email protected]/abs/1>", id="angle-brackets"
            ),
            pytest.param(
                "[arXiv](https://user:[email protected]/abs/1)",
                id="markdown-link",
            ),
            pytest.param(
                '"https://user:[email protected]/abs/1"', id="double-quotes"
            ),
            pytest.param(
                "'https://user:[email protected]/abs/1'", id="single-quotes"
            ),
            pytest.param(
                "(https://user:[email protected]/abs/1)", id="parentheses"
            ),
            pytest.param(
                "`https://user:[email protected]/abs/1`", id="backticks"
            ),
            pytest.param(
                "*https://user:[email protected]/abs/1*", id="emphasis"
            ),
            pytest.param(
                "**https://user:[email protected]/abs/1**", id="strong-emphasis"
            ),
            pytest.param(
                "_https://user:[email protected]/abs/1_",
                id="underscore-emphasis",
            ),
            pytest.param("see https://user:[email protected]/abs/1", id="prose"),
            pytest.param(
                "&lt;https://user:[email protected]/abs/1&gt;",
                id="entity-angle-brackets",
            ),
            pytest.param(
                "\u200bhttps://user:[email protected]/abs/1",
                id="zero-width-space",
            ),
            pytest.param(
                "https%3A//user:[email protected]/abs/1",
                id="percent-encoded-colon",
            ),
        ],
    )
    def test_wrapped_credential_url_is_refused(self, value, password, parts):
        """urlsplit finds no scheme and no authority in these, so they were
        exported verbatim as the source link and as the title."""
        value = _with_password(value, password)
        result = self._make()._create_ris_entry("1", value, url=value)

        assert "TI  - [source URL omitted]\n" in result
        assert "UR  -" not in result
        assert "PB  -" not in result
        _assert_holds_no_part(result, parts)

    def test_export_refuses_wrapped_source_urls(self):
        document = (
            "Report\n\n## Sources\n\n"
            "[1] Autolinked source\n"
            "URL: <https://user:[email protected]/abs/1>\n\n"
            "[2] Linked source\n"
            "URL: [arXiv](https://user:[email protected]/abs/2)\n"
        )

        result = self._make().export_to_ris(document)

        assert result.count("ER  - ") == 2
        assert "UR  -" not in result
        assert "secret" not in result

    def test_doi_url_loses_its_userinfo(self):
        document = (
            "Report\n\n## Sources\n\n"
            "[1] Paper title\n"
            "URL: https://example.org/p\n"
            "DOI: https://user:[email protected]/10.1234/x\n"
        )

        result = self._make().export_to_ris(document)

        assert "DO  - https://doi.org/10.1234/x\n" in result
        assert "secret" not in result

    @pytest.mark.parametrize("password,parts", _PASSWORDS)
    @pytest.mark.parametrize(
        "line",
        [
            "Mirrored by https://user:[email protected]/p",
            # Splitting authors on "&" cut this into parts that each passed.
            "Mirrored by https://user:secret&commat;mirror.example/p",
        ],
    )
    def test_author_text_is_credential_checked(self, line, password, parts):
        line = _with_password(line, password)
        result = self._make()._create_ris_entry("1", f"Paper title\n{line}")

        assert "TI  - Paper title\n" in result
        _assert_holds_no_part(result, parts)

    @pytest.mark.parametrize("password,parts", _PASSWORDS)
    @pytest.mark.parametrize(
        "title",
        [
            "https://DOI:[email protected]/x",
            "https://x:doi:[email protected]/x",
            "https://user:[email protected]/x",
            "https://user:[email protected]/x",
            "https://user:secret Published [email protected]/x",
            "https://u by [email protected]/x",
        ],
    )
    def test_nothing_is_cut_out_of_a_credential_url_title(
        self, title, password, parts
    ):
        """Title cleaning and the author, DOI and year matches cut text out
        of the title. A cut that drops the @ of a credential URL left its
        userinfo in TI, AU or DO, and a year in the password was written as
        PY; the title is now sanitized whole, and gives no author, DOI or
        year."""
        title = _with_password(title, password)
        result = self._make()._create_ris_entry("1", title)

        assert "TI  - https://example.org/x\n" in result
        assert "AU  -" not in result
        assert "DO  -" not in result
        assert "PY  -" not in result
        _assert_holds_no_part(result, parts)

    @pytest.mark.parametrize("password,parts", _PASSWORDS)
    @pytest.mark.parametrize(
        "title,expected",
        [
            (
                "https:/by Smith./user:[email protected]/x",
                "https://example.org/x",
            ),
            ("/by Smith./user:[email protected]/x", "//example.org/x"),
            (
                "https:/by Smith./user:secret [email protected]/x",
                "https://example.org/x",
            ),
            (
                "https:/by Smith./user:DOI:[email protected]/x",
                "https://example.org/x",
            ),
        ],
    )
    def test_title_that_cleaning_joins_into_a_credential_url_is_sanitized(
        self, title, expected, password, parts
    ):
        """No title is a credential URL as written. Cleaning cuts the
        author phrase out, and what is left joins into one, so the cleaned
        title is checked as well as the title as written. In the last two,
        a later cut, at ``Volume`` or at ``DOI:``, cut that URL's @ away
        again: the title was written as ``https://user:secret`` or
        ``https://user:``, with the DOI ``[email protected]/x``. So every
        form a cut makes is checked, the first one flagged is sanitized,
        and the title gives no DOI, nor a year, which a password can hold.
        """
        title = _with_password(title, password)
        assert not citation_formatter._ris_spells_userinfo(title)

        result = self._make()._create_ris_entry("1", title)

        assert f"TI  - {expected}\n" in result
        assert "DO  -" not in result
        assert "PY  -" not in result
        _assert_holds_no_part(result, parts)

    @pytest.mark.parametrize("password,parts", _PASSWORDS)
    def test_doi_is_not_cut_out_of_a_credential_url_line(self, password, parts):
        result = self._make()._create_ris_entry(
            "1",
            _with_password(
                "Paper title\nMirror: https://DOI:[email protected]/x",
                password,
            ),
        )

        assert "TI  - Paper title\n" in result
        assert "DO  -" not in result
        assert "PY  -" not in result
        _assert_holds_no_part(result, parts)

    @freeze_time("2026-01-01")
    def test_a_year_in_a_password_is_not_written_as_the_year(self):
        """The year pattern ran over the whole source text, so it took the
        password 2024 out of the link's line, next to the link written
        without it, and 1999 out of a token in a title written without it.
        """
        url = "https://admin:[email protected]/"
        result = self._make()._create_ris_entry(
            "1", f"Internal dashboard\nURL: {url}", url=url
        )

        assert "UR  - https://intranet.example/\n" in result
        assert "PY  -" not in result

        result = self._make()._create_ris_entry(
            "1", "https://deploy:[email protected]/"
        )

        assert "TI  - https://host.example/\n" in result
        assert "PY  -" not in result

    @pytest.mark.parametrize(
        "password,parts",
        [
            pytest.param("2024", ("2024",), id="year"),
            pytest.param("tok-1999-x", ("1999",), id="year-in-a-token"),
            pytest.param("Kv3 1987", ("Kv3", "1987"), id="year-after-a-space"),
            pytest.param("DOI:10.1234%2Fx", ("1234",), id="doi"),
            pytest.param("by Smith", ("Smith",), id="name-after-by"),
            pytest.param(
                "by Smith", ("Smith",), id="name-after-by-no-break-space"
            ),
            pytest.param(
                "Xq7,Zr9&Wk3.Vm5",
                ("Xq7", "Zr9", "Wk3", "Vm5"),
                id="punctuation",
            ),
        ],
    )
    @pytest.mark.parametrize(
        "where",
        ["title", "URL line", "DOI line", "lowercase doi line", "export"],
    )
    @freeze_time("2026-01-01")
    def test_a_password_shaped_like_a_field_is_not_cut_into_it(
        self, where, password, parts
    ):
        """Each password matches a pattern that cuts a field out of text: a
        year (PY), a ``DOI:`` label (DO), ``by`` and a name (AU), and the
        commas, ampersands and full stops at which the author list is split
        and ended. The URL that holds it is a value as a whole: a title, or
        the value of a ``URL:`` or ``DOI:`` line, which the exporter reads
        as the link or the DOI and strips the userinfo from. The year
        pattern took ``2024`` out of any of them. From a ``URL:`` or
        ``DOI:`` line, the author pattern also took ``Smith@intranet`` out
        of the password ``by Smith``, and the DOI pattern took
        ``https://admin:Kv3`` out of a lowercase ``doi:`` line: in a line, a
        URL after a label ends at whitespace, so the check passed the line.
        """
        url = f"https://admin:{password}@intranet.example/"
        exporter = self._make()
        if where == "title":
            result = exporter._create_ris_entry("1", url, url=url)
            kept = ["TI  - https://intranet.example/\n"]
        elif where == "URL line":
            result = exporter._create_ris_entry(
                "1", f"Internal dashboard\nURL: {url}", url=url
            )
            kept = ["UR  - https://intranet.example/\n"]
        elif where == "DOI line":
            result = exporter._create_ris_entry(
                "1", f"Internal dashboard\nDOI: {url}", metadata={"doi": url}
            )
            kept = ["DO  - https://intranet.example/\n"]
        elif where == "lowercase doi line":
            result = exporter._create_ris_entry(
                "1", f"Internal dashboard\ndoi: {url}"
            )
            kept = ["TI  - Internal dashboard\n"]
        else:
            result = exporter.export_to_ris(
                f"## Sources\n\n[1] Internal dashboard\nURL: {url}\n"
                f"DOI: {url}\n"
            )
            kept = [
                "UR  - https://intranet.example/\n",
                "DO  - https://intranet.example/\n",
            ]

        for line in kept:
            assert line in result
        assert "AU  -" not in result
        assert "PY  -" not in result
        _assert_holds_no_part(result, parts)

    @pytest.mark.parametrize("password,parts", _PASSWORDS)
    @pytest.mark.parametrize(
        "line",
        [
            pytest.param(
                "Mirrored by Jane Doe, https: user:secret@localhost/x",
                id="after-a-comma",
            ),
            pytest.param(
                "Mirrored by Jane Doe and https: user:secret@localhost/x",
                id="after-and",
            ),
            pytest.param(
                "Mirrored by Jane Doe & https: user:secret@localhost/x",
                id="after-an-ampersand",
            ),
            pytest.param(
                "Mirrored by Jane Doe, https://user name:secret@localhost/x",
                id="space-in-the-userinfo",
            ),
        ],
    )
    def test_a_later_author_is_read_as_the_url_it_starts(
        self, line, password, parts
    ):
        """The author pattern ends the list at a full stop and splits it at
        commas, ``and`` and ampersands, any of which a password can hold,
        as ``Xq7,Zr9&Wk3.Vm5`` does. Each author was checked as cut, so the
        cut hid the @ of a URL that only a reading as a whole finds: the
        record held the author ``https: user:Xq7`` and the authors ``Zr9``
        and ``Wk3``. Each author is now read as the URL it starts, to the
        end of its line, and the line gives no author or year."""
        line = _with_password(line, password)
        result = self._make()._create_ris_entry("1", f"Paper title\n{line}")

        assert "TI  - Paper title\n" in result
        assert "AU  -" not in result
        _assert_holds_no_part(result, parts)

    @pytest.mark.parametrize(
        "value,flagged",
        [
            pytest.param(
                "https: user:a.b@localhost/x", True, id="at-the-start"
            ),
            pytest.param(
                "Jane Doe, https: user:a.b@localhost/x",
                True,
                id="after-a-comma",
            ),
            pytest.param(
                "Jane Doe and \ufeffhttps://user name:a@localhost/x",
                True,
                id="after-and-and-a-byte-order-mark",
            ),
            # The first URL's authority ends at the second one's slashes.
            pytest.param(
                "https:a, https://u.v w@localhost/x", True, id="overlapping"
            ),
            pytest.param("https://@@localhost/x", False, id="at-signs-only"),
            pytest.param(
                "Jane Doe, https://example.org/@jane", False, id="at-in-a-path"
            ),
            pytest.param(
                "Jane Doe, mail [email protected]", False, id="no-url-start"
            ),
            pytest.param(
                "Grand and https: email me at [email protected]",
                False,
                id="prose-after-and",
            ),
        ],
    )
    def test_values_are_read_as_the_urls_their_authors_start(
        self, value, flagged
    ):
        """A URL as a whole is read where a value, or an author in it,
        starts: past whitespace in its authority, to the first ``/``, ``?``
        or ``#``, with an @ there after any leading @ signs counting as
        userinfo, as a whole value is read. Prose after a scheme's colon
        is kept as it is in a title."""
        assert citation_formatter._ris_value_starts_userinfo(value) is flagged

    def test_value_starts_are_checked_in_linear_time(self):
        """Each of the 20,000 authors here starts a URL whose authority
        starts inside the first one's and ends where it ends, so the
        authority is scanned once, not once per URL: about forty
        milliseconds, against about four seconds when each URL scans it."""
        value = "https:a," * 20000
        started = time.perf_counter()
        flagged = citation_formatter._ris_value_starts_userinfo(value)
        elapsed = time.perf_counter() - started

        assert not flagged
        assert elapsed < 1.0, f"the check took {elapsed:.2f}s"

    @pytest.mark.parametrize(
        "full_text,url,expected",
        [
            pytest.param(
                "Quarterly report 2023\n"
                "URL: https://admin:[email protected]/",
                "https://admin:[email protected]/",
                "PY  - 2023\n",
                id="year-in-the-title",
            ),
            pytest.param(
                "Archived post\nURL: https://example.org/2019/post",
                "https://example.org/2019/post",
                "PY  - 2019\n",
                id="year-in-a-plain-link",
            ),
            pytest.param(
                "Archived post\nurl: https://example.org/2019/post",
                "",
                "PY  - 2019\n",
                id="year-in-a-plain-lowercase-link",
            ),
            pytest.param(
                "Paper by Jane Doe\n"
                "URL: https://admin:[email protected]/",
                "https://admin:[email protected]/",
                "AU  - Jane Doe\n",
                id="author-in-the-title",
            ),
        ],
    )
    def test_a_field_outside_a_credential_line_is_kept(
        self, full_text, url, expected
    ):
        """Only a line that carries a credential gives no year, author or
        DOI. A plain ``URL:`` line, in any letter case, still gives a year,
        and so does a title beside a credential URL's line."""
        result = self._make()._create_ris_entry("1", full_text, url=url)

        assert expected in result
        assert "2024" not in result
        assert "secret" not in result

    @pytest.mark.parametrize(
        "url",
        [
            "https://evil.example\\@arxiv.org/abs/1",
            "https://evil.example\\.arxiv.org/abs/1",
        ],
    )
    def test_backslash_in_authority_is_omitted(self, url):
        """WHATWG parsers end the authority at the backslash, so the host is
        evil.example, while urlsplit reads arxiv.org; neither reading is
        exported as the link, and arXiv is not credited as publisher."""
        result = self._make()._create_ris_entry("1", "Title", url=url)

        assert "UR  -" not in result
        assert "PB  -" not in result

    @pytest.mark.parametrize("password,parts", _PASSWORDS)
    @pytest.mark.parametrize(
        "value",
        [
            pytest.param("\\\\user:[email protected]/x", id="backslashes"),
            pytest.param("/\\user:[email protected]/x", id="slash-backslash"),
            pytest.param("\\/user:[email protected]/x", id="backslash-slash"),
            pytest.param(
                "\\\\\\user:[email protected]/x", id="three-backslashes"
            ),
            pytest.param("///user:[email protected]/x", id="three-slashes"),
            pytest.param(
                "\\/\\/user:[email protected]/x", id="escaped-slashes"
            ),
            pytest.param(
                "<\\\\user:[email protected]/x>", id="angle-brackets"
            ),
            pytest.param("see \\\\user:[email protected]/x", id="prose"),
            pytest.param(
                "\\\\ user:[email protected]/x", id="space-after-the-run"
            ),
            pytest.param(
                "https:\\\\ user:[email protected]/x",
                id="https-backslashes-then-space",
            ),
            pytest.param(
                "https:/// user:[email protected]/x",
                id="https-three-slashes-then-space",
            ),
        ],
    )
    def test_run_of_slashes_and_backslashes_opens_an_authority(
        self, value, password, parts
    ):
        """A browser reads any run of two or more slashes or backslashes
        after https: as //, and does the same with no scheme against an
        https page, so each value is a credential URL to it. The check knew
        only //, so these were written verbatim as titles, and the
        scheme-less ones as links too: urlsplit finds no authority in them
        to strip."""
        value = _with_password(value, password)
        result = self._make()._create_ris_entry("1", value, url=value)

        assert "TI  - [source URL omitted]\n" in result
        assert "UR  -" not in result
        assert "PB  -" not in result
        _assert_holds_no_part(result, parts)

    @pytest.mark.parametrize("password,parts", _PASSWORDS)
    @pytest.mark.parametrize(
        "value",
        [
            # Inside the scheme.
            pytest.param(
                "ht&Tab;tps:user:[email protected]/x", id="scheme-tab-entity"
            ),
            pytest.param(
                "ht&#10;tps:/user:[email protected]/x", id="scheme-lf-decimal"
            ),
            pytest.param(
                "ht%0Dtps:user:[email protected]/x", id="scheme-cr-percent"
            ),
            # Between the slashes.
            pytest.param(
                "https:/&#9;/user:[email protected]/x",
                id="slashes-tab-decimal",
            ),
            pytest.param(
                "https:/&NewLine;/user:[email protected]/x",
                id="slashes-lf-entity",
            ),
            pytest.param(
                "https:/%0D/user:[email protected]/x", id="slashes-cr-percent"
            ),
            pytest.param(
                "ftp:/%09/user:[email protected]/x", id="ftp-slashes-tab"
            ),
            pytest.param(
                "/&#10;/user:[email protected]/x", id="bare-slashes-lf"
            ),
            # After the scheme's colon.
            pytest.param(
                "https:&#x9;user:[email protected]/x", id="colon-tab-hex"
            ),
            pytest.param(
                "https:%0A/user:[email protected]/x", id="colon-lf-percent"
            ),
            pytest.param(
                "https:&#13;user:[email protected]/x", id="colon-cr-decimal"
            ),
            # Written as it is, not encoded.
            pytest.param(
                "https:/\t/user:[email protected]/x", id="slashes-raw-tab"
            ),
        ],
    )
    def test_tab_or_line_break_does_not_hide_userinfo(
        self, value, password, parts
    ):
        """A browser deletes every tab, LF and CR from a URL before it
        parses it, so each value, with its entity or escape decoded, is a
        credential URL to it: ``ht&Tab;tps:user:secret@...`` reads as
        ``https:user:secret@...``. The check took the decoded character for
        whitespace that ends or blocks an authority, so the title and the
        DOI were written verbatim, and the link too wherever urlsplit found
        no scheme in it. A raw tab, which serialization turns into a space,
        is deleted for the check as well."""
        value = _with_password(value, password)
        result = self._make()._create_ris_entry(
            "1", f"{value}\nDOI: {value}", url=value
        )

        assert "TI  - [source URL omitted]\n" in result
        assert "UR  -" not in result
        assert "PB  -" not in result
        assert "DO  -" not in result
        _assert_holds_no_part(result, parts)

        from_metadata = self._make()._create_ris_entry(
            "1", "Paper title", metadata={"doi": value}
        )

        assert "TI  - Paper title\n" in from_metadata
        assert "DO  -" not in from_metadata
        _assert_holds_no_part(from_metadata, parts)

    @pytest.mark.parametrize(
        "title,expected",
        [
            ("Notes\tfrom [email protected]", "Notes from [email protected]"),
            ("Notes\rfrom [email protected]", "Notes from [email protected]"),
        ],
    )
    def test_tab_or_carriage_return_in_a_plain_title_keeps_it(
        self, title, expected
    ):
        """Deleting the character only feeds the check. A title it does not
        flag is written as before, with the character turned into a space."""
        result = self._make()._create_ris_entry("1", title)

        assert f"TI  - {expected}\n" in result

    @pytest.mark.parametrize("password,parts", _PASSWORDS)
    @pytest.mark.parametrize(
        "value",
        [
            pytest.param(
                "\ufeffhttps://user name:[email protected]/x",
                id="space-in-userinfo",
            ),
            pytest.param(
                "\ufeff\\\\ user:[email protected]/x", id="space-after-the-run"
            ),
        ],
    )
    def test_leading_byte_order_mark_does_not_hide_userinfo(
        self, value, password, parts
    ):
        """JavaScript's trim() removes a leading U+FEFF, which Python's
        strip() keeps, so a reader that trims the field reads each value as
        a URL as a whole, whitespace in its authority included. The check
        read it as text, where a URL ends at whitespace, so the title and
        the link were written verbatim."""
        value = _with_password(value, password)
        result = self._make()._create_ris_entry("1", value, url=value)

        assert "TI  - [source URL omitted]\n" in result
        assert "UR  -" not in result
        assert "PB  -" not in result
        _assert_holds_no_part(result, parts)

    @pytest.mark.parametrize("password,parts", _PASSWORDS)
    @pytest.mark.parametrize("scheme", ["https", "HTTP", "ftp"])
    @pytest.mark.parametrize(
        "gap",
        [
            pytest.param(" ", id="space"),
            pytest.param("\u00a0", id="no-break-space"),
            pytest.param("\x0b", id="vertical-tab"),
            pytest.param("\u3000", id="ideographic-space"),
            pytest.param("&#32;", id="entity-space"),
            pytest.param("/ ", id="slash-then-space"),
            pytest.param("\\ ", id="backslash-then-space"),
        ],
    )
    def test_whitespace_after_a_scheme_colon_does_not_hide_userinfo(
        self, scheme, gap, password, parts
    ):
        """Parsed with no base, each value, its entity decoded, is a
        credential URL to a browser, whose userinfo starts with the
        whitespace, encoded: ``https: user:[email protected]/x`` reads as
        ``https://%20user:[email protected]/x``. The check took whitespace
        right after a special scheme's colon, or after its colon and one
        slash or backslash, for the end of the URL, so the title, the
        author and a metadata DOI were written verbatim. The source link
        was already left out."""
        value = f"{scheme}:{gap}user:{password}@example.org/x"
        result = self._make()._create_ris_entry(
            "1", f"{value}\nDOI: {value}\nMirrored by {value}", url=value
        )

        assert "TI  - [source URL omitted]\n" in result
        assert "AU  -" not in result
        assert "UR  -" not in result
        assert "PB  -" not in result
        _assert_holds_no_part(result, parts)

        from_metadata = self._make()._create_ris_entry(
            "1", "Paper title", metadata={"doi": value}
        )

        assert "TI  - Paper title\n" in from_metadata
        assert "DO  -" not in from_metadata
        _assert_holds_no_part(from_metadata, parts)

    @pytest.mark.parametrize("password,parts", _PASSWORDS)
    @pytest.mark.parametrize(
        "value",
        [
            pytest.param(
                "https: user:secret @example.org/x", id="space-before-the-at"
            ),
            pytest.param("HTTP: [email protected]", id="token-only"),
            pytest.param("ftp: user:[email protected] ", id="trailing-space"),
            pytest.param(
                "https:\u00a0 user:[email protected]/x", id="whitespace-run"
            ),
        ],
    )
    def test_one_token_after_a_scheme_colon_and_whitespace_is_flagged(
        self, value, password, parts
    ):
        """Past whitespace right after a special scheme's colon, one token,
        an @ and a host without whitespace open an authority. Whitespace
        before the @ does not prevent it, nor does whitespace after the host
        at the end of the value, which a browser trims. A browser reads the
        values, as written, with the password ``secret%20``, the username
        ``%20secret-token``, the URL ``ftp://%20user:[email protected]/``
        and the username ``%C2%A0%20user``."""
        value = _with_password(value, password)
        assert citation_formatter._ris_spells_userinfo(value)

        result = self._make()._create_ris_entry("1", value, url=value)

        assert "TI  - [source URL omitted]\n" in result
        assert "UR  -" not in result
        _assert_holds_no_part(result, parts)

    @pytest.mark.parametrize("password,parts", _PASSWORDS)
    @pytest.mark.parametrize(
        "control",
        [
            pytest.param("\x00", id="null"),
            pytest.param("\x1b", id="escape"),
            pytest.param("\x7f", id="delete"),
            pytest.param("\x9f", id="c1-control"),
        ],
    )
    @pytest.mark.parametrize(
        "shape",
        [
            pytest.param(
                "https: user:secret@localhost {}", id="after-the-host"
            ),
            pytest.param(
                "https: user:secret {}@localhost/x", id="before-the-at"
            ),
            pytest.param(
                "https: {} user:secret@localhost/x", id="after-the-colon"
            ),
        ],
    )
    def test_control_among_whitespace_after_a_scheme_colon_is_flagged(
        self, shape, control, password, parts
    ):
        """Serialization writes a control character as a space, so it
        writes each value in a form the check flags, such as
        ``https: user:secret@localhost`` and two spaces, and a browser reads
        each written form as userinfo at localhost. Python's ``re`` does not
        count these controls as whitespace, so the check read each as text,
        found no credential, and the title, the author and a metadata DOI
        were written verbatim. The source link was already left out."""
        value = _with_password(shape.format(control), password)
        result = self._make()._create_ris_entry(
            "1", f"{value}\nDOI: {value}\nMirrored by {value}", url=value
        )

        assert "TI  - [source URL omitted]\n" in result
        assert "AU  -" not in result
        assert "UR  -" not in result
        _assert_holds_no_part(result, parts)

        from_metadata = self._make()._create_ris_entry(
            "1", "Paper title", metadata={"doi": value}
        )

        assert "TI  - Paper title\n" in from_metadata
        assert "DO  -" not in from_metadata
        _assert_holds_no_part(from_metadata, parts)

    @pytest.mark.parametrize("password,parts", _PASSWORDS)
    @pytest.mark.parametrize(
        "value",
        [
            pytest.param(
                "https: user:secret@localhost %1B", id="after-the-host"
            ),
            pytest.param(
                "https: user:secret %1B@localhost/x", id="before-the-at"
            ),
            pytest.param(
                "https: %1B user:secret@localhost/x", id="after-the-colon"
            ),
        ],
    )
    def test_encoded_control_among_whitespace_is_flagged(
        self, value, password, parts
    ):
        """Percent-decoded, each value holds an escape character after
        whitespace, which a browser trims after the host and keeps in the
        userinfo anywhere else, so it reads each as userinfo at localhost.
        The check reads every decoded form with its controls as spaces, as
        it reads the value itself. It read the decoded control as text, and
        the title was written verbatim."""
        value = _with_password(value, password)
        result = self._make()._create_ris_entry("1", value, url=value)

        assert citation_formatter._ris_spells_userinfo(value)
        assert "TI  - [source URL omitted]\n" in result
        _assert_holds_no_part(result, parts)

    @pytest.mark.parametrize("password,parts", _PASSWORDS)
    @pytest.mark.parametrize(
        "value",
        [
            pytest.param(
                "https: user:secret@localhost \ufeff", id="after-a-space"
            ),
            pytest.param(
                "HTTP: secret-token@localhost \ufeff \ufeff", id="in-a-run"
            ),
            pytest.param(
                "ftp: user:secret @localhost\u3000\ufeff",
                id="after-an-ideographic-space",
            ),
        ],
    )
    def test_trailing_byte_order_mark_does_not_hide_userinfo(
        self, value, password, parts
    ):
        """JavaScript's trim() removes a trailing U+FEFF as it removes a
        leading one, so a reader that trims the field reads each value, as
        written, as a credential URL: ``https: user:secret@localhost`` with
        the username ``%20user``, ``HTTP: secret-token@localhost`` with the
        username ``%20secret-token``, and ``ftp: user:secret @localhost``
        with the password ``secret%20``. After the host the check accepted
        only whitespace before the end of the value, so the title, the
        author and a metadata DOI were written verbatim."""
        value = _with_password(value, password)
        result = self._make()._create_ris_entry(
            "1", f"{value}\nDOI: {value}\nMirrored by {value}", url=value
        )

        assert "TI  - [source URL omitted]\n" in result
        assert "AU  -" not in result
        assert "UR  -" not in result
        _assert_holds_no_part(result, parts)

        from_metadata = self._make()._create_ris_entry(
            "1", "Paper title", metadata={"doi": value}
        )

        assert "DO  -" not in from_metadata
        _assert_holds_no_part(from_metadata, parts)

    def test_controls_after_a_host_are_checked_in_linear_time(self):
        """After the host, the value may end in whitespace, controls and
        U+FEFF, and the host may hold controls and U+FEFF itself. The host
        is matched possessively, so a long run of them followed by text is
        scanned once. If the match could give characters back, each shorter
        host would rescan the rest of the value: about two seconds for this
        run, against about ten milliseconds."""
        value = "https: a@" + "\x01" * 20000 + " x"
        started = time.perf_counter()
        flagged = citation_formatter._ris_spells_userinfo(value)
        elapsed = time.perf_counter() - started

        assert not flagged
        assert elapsed < 0.25, f"the check took {elapsed:.2f}s"

    @pytest.mark.parametrize(
        "title",
        [
            "HTTPS: why you should email [email protected]",
            "http: a guide to the @ sign",
            "HTTP: email me at [email protected]",
            "HTTP:\u00a0email me at [email protected]",
            "HTTP: [email protected] explains caching",
            "HTTP: @mention",
        ],
    )
    def test_prose_after_a_scheme_word_keeps_its_title(self, title):
        """A browser reads four of these as URLs whose userinfo is what
        precedes the @: ``HTTP: email me at [email protected]`` as
        ``http://%20email%20me%20at%[email protected]/``, and
        ``HTTP: @mention`` with a space for its username. The other two,
        ``http: a guide to the @ sign`` and
        ``HTTP: [email protected] explains caching``, do not parse, since their
        would-be hosts hold whitespace. They are titles all the same. Past
        whitespace right after a special scheme's colon, the check reads an
        authority only when one token, an @ and a host without whitespace
        follow. Each of these holds whitespace in its would-be userinfo or
        host, or has no token before its @."""
        result = self._make()._create_ris_entry("1", title)

        assert f"TI  - {title}\n" in result

    @pytest.mark.parametrize(
        "value",
        [
            "https://example.org/article",
            "https://medium.com/@author/article",
            "https://web.archive.org/web/2020/https://medium.com/@author/post",
            "https://example.org/?email=bob%40example.org",
            "https://www.google.com/url?q=https%3A%2F%2Fexample.org%2F%40user",
            "https://de.wikipedia.org/wiki/Stra%C3%9Fe",
            "https://example.org//double//slash",
            "https://example.org/a%09b%0Ac",
            "https://example.org/?q=a&Tab;b",
        ],
    )
    def test_url_without_credentials_is_unchanged(self, value):
        result = self._make()._create_ris_entry("1", value, url=value)

        assert f"TI  - {value}\n" in result
        assert f"UR  - {value}\n" in result

    @pytest.mark.parametrize(
        "title",
        [
            "Re: question from [email protected]",
            "news:[email protected]",
            "Using //@ts-ignore in TypeScript",
            "Using \\\\@ts-ignore in TypeScript",
            "\\\\fileserver\\share\\notes.txt",
            "HTTP: The Definitive Guide",
            "Tom & Jerry @ Home",
            "Tabs&Tab;and&#10;breaks @ home",
            "Comments open with /&Tab;/ and @ marks a decorator",
        ],
    )
    def test_title_without_url_credentials_is_unchanged(self, title):
        result = self._make()._create_ris_entry("1", title)

        assert f"TI  - {title}\n" in result

    @pytest.mark.parametrize("password,parts", _PASSWORDS)
    def test_no_field_keeps_a_value_the_shared_check_flags(
        self, password, parts
    ):
        """UR, TI, AU and DO cannot drift apart: whatever the shared check
        flags is absent from every field of the record, and no field, PY
        included, holds any part of its password."""
        from local_deep_research.text_optimization.citation_formatter import (
            _ris_spells_userinfo,
        )

        values = [
            "https://user:[email protected]\uff1a8443/x",
            "git+https://user:[email protected]/o/r",
            "//user:[email protected]/x",
            "\\\\user:[email protected]/x",
            "///user:[email protected]/x",
            "&#32;https:/user:[email protected]/x",
            "https://user:secret%40example.org/x",
            "[arXiv](https://user:[email protected]/abs/1)",
            "https://first:secret@x&commat;example.org/doc",
            "https://DOI:[email protected]/x",
            "ht&Tab;tps:user:[email protected]/x",
            "/&#10;/user:[email protected]/x",
            "\ufeffhttps://user name:[email protected]/x",
            "https: user:[email protected]/x",
            "HTTP:/ user:[email protected]/x",
        ]
        for value in values:
            value = _with_password(value, password)
            assert _ris_spells_userinfo(value), value
            result = self._make()._create_ris_entry(
                "1",
                f"{value}\nWritten by {value}\nDOI: {value}",
                url=value,
            )
            for line in result.split("\n"):
                assert not _ris_spells_userinfo(line[6:]), (value, line)
            _assert_holds_no_part(result, parts)

    def test_every_field_is_written_in_a_form_the_check_passes(self):
        """Serialization turns every control character, U+2028 and U+2029
        into a space, and the check reads each value that way too, so no
        field of a record is written with a value it flags. The values put
        a C0, DEL or C1 control, U+2028, U+FEFF, or an entity- or
        percent-encoded control into credential URLs and titles, around and
        in place of their whitespace. Before, a credential URL with a
        control after its whitespace passed the check and was written with
        a space in place of the control, which the check flags."""
        from local_deep_research.text_optimization.citation_formatter import (
            _ris_spells_userinfo,
        )

        shapes = [
            "https: user:secret@localhost{}",
            "https: user:secret@localhost {}",
            "https: user:secret {}@localhost/x",
            "https: {} user:secret@localhost/x",
            "https:{}user:secret@localhost/x",
            "https://user:secret@localhost{}/x",
            "<https://user:secret@localhost/x>{}",
            "HTTP: email me at{} [email protected]",
            "HTTP: {}@mention",
            "Notes{}from [email protected]",
        ]
        characters = [
            "\x00",
            "\x1b",
            "\x7f",
            "\x85",
            "\x9f",
            "\u2028",
            "\ufeff",
            "&#1;",
            "%01",
        ]
        exporter = self._make()
        for shape in shapes:
            for character in characters:
                value = shape.format(character)
                records = [
                    exporter._create_ris_entry(
                        "1",
                        f"{value}\nDOI: {value}\nMirrored by {value}",
                        url=value,
                    ),
                    exporter._create_ris_entry(
                        "1", "Paper title", metadata={"doi": value}
                    ),
                    exporter.export_to_ris(
                        f"## Sources\n\n[1] {value}\nURL: {value}\n"
                        f"DOI: {value}\n"
                    ),
                ]
                for record in records:
                    for line in record.split("\n"):
                        assert not _ris_spells_userinfo(line[6:]), (
                            value,
                            line,
                        )

    @pytest.mark.parametrize(
        "bound,limit,value",
        [
            # "%41" has 3 characters; its readings are "%41" and "A".
            pytest.param(
                "_RIS_MAX_DECODED_LENGTH", 3, "%41", id="length-of-the-value"
            ),
            # "a\ufb01" has 2 characters; NFKC reads it as "afi", with 3.
            pytest.param(
                "_RIS_MAX_DECODED_LENGTH",
                3,
                "a\ufb01",
                id="length-of-a-decoded-reading",
            ),
            # The readings "a%41" and "aA" have 6 characters together.
            pytest.param(
                "_RIS_MAX_DECODED_TOTAL_LENGTH",
                6,
                "a%41",
                id="length-of-all-readings",
            ),
            pytest.param("_RIS_MAX_READINGS", 2, "%41", id="reading-count"),
        ],
    )
    def test_value_past_a_reading_bound_is_left_out(
        self, monkeypatch, bound, limit, value
    ):
        """Each bound is lowered to what a few characters reach. At the
        bound the value is checked as usual; past it the value counts as
        flagged and is left out of the title and the link. Only the length
        of the value itself was bounded before, though NFKC can make a
        reading longer than the value, and the readings together were not
        bounded at all."""
        monkeypatch.setattr(citation_formatter, bound, limit)
        kept = self._make()._create_ris_entry("1", value, url=value)

        assert f"TI  - {value}\n" in kept
        assert f"UR  - {value}\n" in kept

        monkeypatch.setattr(citation_formatter, bound, limit - 1)
        result = self._make()._create_ris_entry("1", value, url=value)

        assert citation_formatter._ris_spells_userinfo(value)
        assert "TI  - [source URL omitted]\n" in result
        assert "UR  -" not in result
        assert value not in result

    def test_value_with_a_control_character_is_measured(self, monkeypatch):
        """One reading of a value is the value with its control characters
        written as spaces, so an ASCII value holding a control is measured
        against the bounds like a value that needs decoding, and counts as
        flagged past them. It was its only reading, and was not measured."""
        value = "Notes\x01from bob"
        monkeypatch.setattr(citation_formatter, "_RIS_MAX_READINGS", 2)
        kept = self._make()._create_ris_entry("1", value)

        assert "TI  - Notes from bob\n" in kept

        monkeypatch.setattr(citation_formatter, "_RIS_MAX_READINGS", 1)
        result = self._make()._create_ris_entry("1", value)

        assert citation_formatter._ris_spells_userinfo(value)
        assert "TI  - [source URL omitted]\n" in result


class TestRISEntryStructuralValidity:
    """Regression guard for the accumulator-clobber bug.

    ``_create_ris_entry`` initialized ``lines = []`` as the RIS-output
    accumulator, then immediately reassigned ``lines = full_text.split("\\n")``
    to read the title — overwriting the accumulator with the raw source body.
    Every RIS field was then appended *after* those source lines, so each entry
    emitted the source text before the mandatory leading ``TY  - `` tag and
    reference managers (Zotero/Mendeley/EndNote) rejected the record. The
    pre-existing tests only used ``"<tag>" in result`` substring checks, so they
    never caught the leaked prefix.
    """

    # RIS records are newline-separated ``XX  - value`` lines where the tag is
    # two uppercase-or-digit chars (TY, ID, TI, AU, DO, PY, UR, PB, Y1, DA, LA,
    # ER), two spaces, a hyphen, then a space.
    _RIS_LINE = re.compile(r"^[A-Z][A-Z0-9]  - ")

    def test_entry_starts_with_ty_tag(self):
        """RIS requires ``TY`` to be the first tag of the record."""
        exporter = RISExporter()
        source = (
            "Understanding Widgets\n"
            "URL: https://example.com/widgets\n"
            "Collection: Gadgets"
        )
        result = exporter._create_ris_entry(
            "7", source, url="https://example.com/widgets"
        )
        assert result.startswith("TY  - "), result

    def test_no_raw_source_lines_leak(self):
        """Every non-blank output line is a RIS tag, never raw source body."""
        exporter = RISExporter()
        source = (
            "My Title\n"
            "BODYLEAKMARKER this descriptive line must never be emitted\n"
            "Collection: Internal"
        )
        result = exporter._create_ris_entry("1", source)
        assert "BODYLEAKMARKER" not in result
        for line in result.split("\n"):
            if not line.strip():
                continue
            assert self._RIS_LINE.match(line), f"non-RIS line leaked: {line!r}"