~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Tests for RISExporter._create_ris_entry() edge cases.
Existing tests cover: basic ID, year extraction, DOI from metadata,
GitHub publisher, and arXiv publisher.
This file adds coverage for the many remaining branches:
- Author parsing (split by 'and', '&', commas)
- DOI extraction from text (not just metadata)
- Publisher detection for Reddit, YouTube, Medium, PyPI, generic domains
- www. prefix stripping
- Title cleaning (DOI, "Published in", "Volume", "Pages" patterns)
- Empty/minimal inputs
- Author match overlapping with title cleaning
"""
import re
import time
import pytest
from freezegun import freeze_time
from local_deep_research.text_optimization import citation_formatter
from local_deep_research.text_optimization.citation_formatter import RISExporter
# Passwords for the credential fixtures, each with the parts of it that no
# field may hold. The fixtures are written with ``secret`` in a URL's
# userinfo. It matches no pattern that cuts a field out of text, so with it
# alone such a cut could not show. The others match those patterns: a year
# (PY), a ``DOI:`` label (DO), and the commas, ampersands and full stops at
# which the author list is split and ended (AU).
_PASSWORDS = [
pytest.param("secret", ("secret",), id="word"),
pytest.param("1987", ("1987",), id="year"),
pytest.param("tok-1999-x", ("1999",), id="year-in-a-token"),
pytest.param("DOI:10.5555", ("5555",), id="doi"),
pytest.param(
"Xq7,Zr9&Wk3.Vm5", ("Xq7", "Zr9", "Wk3", "Vm5"), id="punctuation"
),
]
def _with_password(value, password):
"""``value`` with ``password`` in place of ``secret``."""
assert "secret" in value, value
return value.replace("secret", password)
def _assert_holds_no_part(result, parts):
for part in parts:
assert part not in result, (part, result)
class TestRISMultilineCredentialSources:
"""Check the unsplit source before line-based metadata can hide its @."""
@pytest.mark.parametrize("separator", ["\n", "\r\n"])
@pytest.mark.parametrize(
"title",
[
"A harmless title by https: user:[email protected]/x. "
"https://user:CutPassword957",
"https:/by Smith./user:CutPassword957",
"https:/by Smith./user:CutPassword957 Volume",
],
)
def test_title_cleanup_cannot_hide_a_remaining_fragment(
self, title, separator
):
exported = RISExporter().export_to_ris(
f"## Sources\n[1] {title}{separator}@example.net/paper\n"
)
assert "CutPassword957" not in exported
assert "TI - [source URL omitted]" in exported
@pytest.mark.parametrize("prefix", ["by", "URL:", "DOI:"])
def test_source_guard_does_not_backtrack_over_whitespace(self, prefix):
source = prefix + " " * 16_000 + "[email protected]\nPlain text"
started = time.thread_time()
assert not citation_formatter._ris_multiline_line_carries_userinfo(
source, 0
)
# CPU time excludes competing test processes and VM suspension.
# Ambiguous adjacent whitespace repeats take seconds here.
assert time.thread_time() - started < 1.0
@pytest.mark.parametrize("password,parts", _PASSWORDS)
@pytest.mark.parametrize("separator", ["\n", "\r\n"])
@pytest.mark.parametrize("field", ["title", "URL", "DOI", "author"])
def test_line_split_cannot_export_a_password_fragment(
self, password, parts, separator, field
):
url = f"https://user:{password}{separator}@example.org/paper"
source = {
"title": url,
"URL": f"A harmless title\nURL: {url}",
"DOI": f"A harmless title\nDOI: {url}",
"author": f"A harmless title\nby {url}",
}[field]
exported = RISExporter().export_to_ris(f"## Sources\n[1] {source}\n")
assert exported.startswith("TY - ELEC")
_assert_holds_no_part(exported, parts)
if field == "title":
assert "TI - [source URL omitted]" in exported
else:
assert "TI - A harmless title" in exported
@pytest.mark.parametrize(
"url",
[
"https://user:secret%4\n0example.org/paper",
"https://user:secret&com\nmat;example.org/paper",
"ht\ntps://user:secret\n@example.org/paper",
"https://user:\nsecret\n@example.org/paper",
"https://user:secret\n4;example.org/paper",
"https://user:secret@\nexample.org/paper",
"https: user:secret\n@example.org\nPublished in 2023",
"https: user:sec\n[email protected]\nPublished in 2023",
],
)
def test_decoding_and_line_splitting_compose(self, url):
exported = RISExporter().export_to_ris(
f"## Sources\n[1] {url}\nDOI: {url}\nURL: {url}\n"
)
assert exported.startswith("TY - ELEC")
assert "secret" not in exported
assert "TI - [source URL omitted]" in exported
@pytest.mark.parametrize(
"title",
[
"HTTP: [email protected] explains",
"HTTP: email me at [email protected]",
"HTTP: @mention",
],
)
def test_multiline_record_keeps_prose_title_exception(self, title):
exported = RISExporter().export_to_ris(
f"## Sources\n[1] {title}\nPublished in 2024\n"
"URL: https://example.org/paper\n"
)
assert f"TI - {title}" in exported
assert "PY - 2024" in exported
assert "UR - https://example.org/paper" in exported
@pytest.mark.parametrize("line_ending", ["\n", "\r\n"])
def test_safe_fields_outside_a_split_credential_keep_their_values(
self, line_ending
):
source = line_ending.join(
[
"## Sources",
"[1] A harmless title",
"by Alice and Bob.",
"Published in 2024",
"URL: https://user:secret",
"@example.org/paper",
"DOI: 10.1234/safe",
]
)
exported = RISExporter().export_to_ris(source)
assert "secret" not in exported
for field in (
"TI - A harmless title",
"AU - Alice",
"AU - Bob",
"PY - 2024",
"DO - 10.1234/safe",
):
assert field in exported
def test_source_lines_without_credentials_keep_their_metadata(self):
source = (
"## Sources\n[1] A harmless title\nby Alice and Bob.\n"
"Published in 2024\nDOI: 10.1234/safe\n"
"URL: https://example.org/@author\n"
)
exported = RISExporter().export_to_ris(source)
for field in (
"TI - A harmless title",
"AU - Alice",
"AU - Bob",
"PY - 2024",
"DO - 10.1234/safe",
"UR - https://example.org/@author",
):
assert field in exported
def test_a_complete_inline_credential_does_not_discard_the_cleaned_title(
self,
):
exported = RISExporter().export_to_ris(
"## Sources\n[1] A harmless title by https: user:secret@host\n"
"URL: https://example.org/paper\n"
)
assert "secret" not in exported
assert "TI - A harmless title" in exported
assert "UR - https://example.org/paper" in exported
class TestRISEntryAuthorParsing:
"""Tests for author extraction from 'by Author1, Author2' patterns."""
def _make(self):
return RISExporter()
def test_single_author(self):
"""Extracts a single author."""
exporter = self._make()
result = exporter._create_ris_entry(
"1", "Some Title\nWritten by Jane Doe."
)
assert "AU - Jane Doe" in result
def test_authors_separated_by_and(self):
"""Authors separated by 'and' are split correctly."""
exporter = self._make()
result = exporter._create_ris_entry(
"1", "Title\nby Alice Smith and Bob Jones."
)
assert "AU - Alice Smith" in result
assert "AU - Bob Jones" in result
def test_authors_separated_by_ampersand(self):
"""Authors separated by '&' are split correctly."""
exporter = self._make()
result = exporter._create_ris_entry(
"1", "Title\nby Alice Smith & Bob Jones."
)
assert "AU - Alice Smith" in result
assert "AU - Bob Jones" in result
def test_authors_separated_by_comma(self):
"""Authors separated by commas are split correctly."""
exporter = self._make()
result = exporter._create_ris_entry(
"1", "Title\nby Alice Smith, Bob Jones, Carol White."
)
assert "AU - Alice Smith" in result
assert "AU - Bob Jones" in result
assert "AU - Carol White" in result
def test_authors_mixed_separators(self):
"""Authors with mixed 'and' and comma separators."""
exporter = self._make()
result = exporter._create_ris_entry(
"1", "Title\nby Alice, Bob and Carol."
)
assert "AU - Alice" in result
assert "AU - Bob" in result
assert "AU - Carol" in result
def test_no_authors(self):
"""No author line produces no AU entries."""
exporter = self._make()
result = exporter._create_ris_entry("1", "Title with no authors")
assert "AU -" not in result
def test_by_keyword_case_insensitive(self):
"""The 'by' keyword match is case-insensitive."""
exporter = self._make()
result = exporter._create_ris_entry("1", "Title\nBY John Smith.")
assert "AU - John Smith" in result
def test_empty_author_parts_filtered(self):
"""Empty parts from splitting are filtered out."""
exporter = self._make()
# Trailing comma leaves an empty part
result = exporter._create_ris_entry("1", "Title\nby Alice, , Bob.")
lines = [
line for line in result.split("\n") if line.startswith("AU -")
]
# Should only have real authors, not empty strings
for line in lines:
assert line.strip() != "AU -"
class TestRISEntryDOIExtraction:
"""Tests for DOI extraction from metadata and text."""
def _make(self):
return RISExporter()
def test_doi_from_metadata_takes_precedence(self):
"""DOI from metadata dict is used over DOI in text."""
exporter = self._make()
result = exporter._create_ris_entry(
"1",
"Title DOI: 10.9999/text-doi",
metadata={"doi": "10.1234/meta-doi"},
)
assert "DO - 10.1234/meta-doi" in result
assert (
"10.9999/text-doi" not in result.split("DO - ")[1].split("\n")[0]
)
def test_doi_from_text(self):
"""DOI extracted from text when not in metadata."""
exporter = self._make()
result = exporter._create_ris_entry(
"1", "Title\nDOI: 10.1234/test.5678"
)
assert "DO - 10.1234/test.5678" in result
def test_doi_from_text_case_insensitive(self):
"""DOI extraction from text is case-insensitive."""
exporter = self._make()
result = exporter._create_ris_entry(
"1", "Title\ndoi: 10.1234/lowercase"
)
assert "DO - 10.1234/lowercase" in result
def test_no_doi(self):
"""No DOI produces no DO entry."""
exporter = self._make()
result = exporter._create_ris_entry("1", "Title with no DOI")
assert "DO -" not in result
def test_metadata_none_falls_back_to_text(self):
"""When metadata is None, DOI is extracted from text."""
exporter = self._make()
result = exporter._create_ris_entry("1", "Title\nDOI: 10.5555/fallback")
assert "DO - 10.5555/fallback" in result
def test_metadata_without_doi_key(self):
"""Metadata dict without 'doi' key falls back to text."""
exporter = self._make()
result = exporter._create_ris_entry(
"1",
"Title\nDOI: 10.7777/textdoi",
metadata={"other": "value"},
)
assert "DO - 10.7777/textdoi" in result
class TestRISEntryPublisherDetection:
"""Tests for publisher extraction from URL domains."""
def _make(self):
return RISExporter()
def test_reddit_publisher(self):
"""Reddit URL produces Reddit publisher."""
exporter = self._make()
result = exporter._create_ris_entry(
"1", "Title", url="https://www.reddit.com/r/python/post"
)
assert "PB - Reddit" in result
def test_reddit_subdomain(self):
"""Reddit subdomain also detected."""
exporter = self._make()
result = exporter._create_ris_entry(
"1", "Title", url="https://old.reddit.com/r/science"
)
assert "PB - Reddit" in result
def test_youtube_publisher(self):
"""YouTube URL produces YouTube publisher."""
exporter = self._make()
result = exporter._create_ris_entry(
"1", "Title", url="https://www.youtube.com/watch?v=abc123"
)
assert "PB - YouTube" in result
def test_youtube_mobile(self):
"""Mobile YouTube URL produces YouTube publisher."""
exporter = self._make()
result = exporter._create_ris_entry(
"1", "Title", url="https://m.youtube.com/watch?v=abc123"
)
assert "PB - YouTube" in result
def test_medium_publisher(self):
"""Medium URL produces Medium publisher."""
exporter = self._make()
result = exporter._create_ris_entry(
"1", "Title", url="https://medium.com/@author/article"
)
assert "PB - Medium" in result
def test_medium_subdomain(self):
"""Medium subdomain (custom publication) detected."""
exporter = self._make()
result = exporter._create_ris_entry(
"1", "Title", url="https://blog.medium.com/post"
)
assert "PB - Medium" in result
def test_pypi_publisher(self):
"""PyPI URL produces PyPI publisher."""
exporter = self._make()
result = exporter._create_ris_entry(
"1", "Title", url="https://pypi.org/project/requests/"
)
assert "PB - Python Package Index (PyPI)" in result
def test_generic_domain_as_publisher(self):
"""Unknown domain used as publisher name."""
exporter = self._make()
result = exporter._create_ris_entry(
"1", "Title", url="https://example.com/article"
)
assert "PB - example.com" in result
def test_www_stripped_from_domain(self):
"""www. prefix is stripped from domain."""
exporter = self._make()
result = exporter._create_ris_entry(
"1", "Title", url="https://www.example.org/page"
)
assert "PB - example.org" in result
def test_no_url_no_publisher(self):
"""No URL produces no PB or UR entries."""
exporter = self._make()
result = exporter._create_ris_entry("1", "Title")
assert "PB -" not in result
assert "UR -" not in result
def test_empty_url_no_publisher(self):
"""Empty string URL produces no PB or UR entries."""
exporter = self._make()
result = exporter._create_ris_entry("1", "Title", url="")
assert "PB -" not in result
assert "UR -" not in result
def test_github_subdomain(self):
"""GitHub subdomain detected as GitHub."""
exporter = self._make()
result = exporter._create_ris_entry(
"1", "Title", url="https://pages.github.com/project"
)
assert "PB - GitHub" in result
def test_arxiv_subdomain(self):
"""arXiv subdomain detected as arXiv."""
exporter = self._make()
result = exporter._create_ris_entry(
"1", "Title", url="https://export.arxiv.org/abs/1234"
)
assert "PB - arXiv" in result
@pytest.mark.parametrize(
"url,publisher",
[
("https://user:[email protected]/abs/1", "arXiv"),
("https://user:[email protected]/report", "example.com"),
("https://[email protected]/abs/1", "evil.example"),
("https://example.com:8080/article", "example.com"),
("https://ArXiv.org/abs/1", "arXiv"),
("https://arxiv.org./abs/1", "arXiv"),
],
)
def test_the_publisher_is_the_host_alone(self, url, publisher):
"""Userinfo and port are not the publisher; credentials in the
netloc were written into the .ris file (#6837)."""
result = self._make()._create_ris_entry("1", "Title", url=url)
assert f"PB - {publisher}\n" in result
class TestRISEntryTitleCleaning:
"""Tests for title cleaning patterns."""
def _make(self):
return RISExporter()
def test_doi_removed_from_title(self):
"""DOI text is removed from the title."""
exporter = self._make()
result = exporter._create_ris_entry(
"1", "My Paper DOI: 10.1234/test\nMore content"
)
# The title line should not contain DOI
ti_line = [
line for line in result.split("\n") if line.startswith("TI -")
][0]
assert "DOI:" not in ti_line
def test_published_in_removed_from_title(self):
"""'Published in ...' is removed from the title."""
exporter = self._make()
result = exporter._create_ris_entry(
"1", "My Paper Published in Nature 2023\nContent"
)
ti_line = [
line for line in result.split("\n") if line.startswith("TI -")
][0]
assert "Published in" not in ti_line
def test_volume_removed_from_title(self):
"""'Volume ...' is removed from the title."""
exporter = self._make()
result = exporter._create_ris_entry(
"1", "My Paper Volume 42 Issue 3\nContent"
)
ti_line = [
line for line in result.split("\n") if line.startswith("TI -")
][0]
assert "Volume" not in ti_line
def test_pages_removed_from_title(self):
"""'Pages ...' is removed from the title."""
exporter = self._make()
result = exporter._create_ris_entry(
"1", "My Paper Pages 100-120\nContent"
)
ti_line = [
line for line in result.split("\n") if line.startswith("TI -")
][0]
assert "Pages" not in ti_line
def test_fallback_to_original_title_when_clean_empty(self):
"""If cleaning empties the title, original title is used."""
exporter = self._make()
# Title that's entirely "by Author" where author match starts at 0
result = exporter._create_ris_entry("1", "by John Smith\nContent")
# Should still have a TI entry (either cleaned or fallback)
ti_line = [
line for line in result.split("\n") if line.startswith("TI -")
][0]
assert "TI - " in ti_line
class TestRISEntryStructure:
"""Tests for overall RIS entry structure."""
def _make(self):
return RISExporter()
def test_contains_ty_elec(self):
"""Entry contains TY - ELEC type marker."""
exporter = self._make()
result = exporter._create_ris_entry("1", "Title")
assert "TY - ELEC" in result
def test_ends_with_er(self):
"""Entry ends with ER marker."""
exporter = self._make()
result = exporter._create_ris_entry("1", "Title")
lines = result.split("\n")
assert lines[-1] == "ER - "
def test_contains_language(self):
"""Entry contains language tag."""
exporter = self._make()
result = exporter._create_ris_entry("1", "Title")
assert "LA - en" in result
@freeze_time("2025-06-15")
def test_contains_access_year(self):
"""Entry contains current access year."""
exporter = self._make()
result = exporter._create_ris_entry("1", "Title")
assert "Y1 - 2025" in result
@freeze_time("2025-06-15")
def test_contains_access_date(self):
"""Entry contains formatted access date."""
exporter = self._make()
result = exporter._create_ris_entry("1", "Title")
assert "DA - 2025/06/15" in result
def test_ref_id_format(self):
"""Reference ID formatted as 'refN'."""
exporter = self._make()
result = exporter._create_ris_entry("42", "Title")
assert "ID - ref42" in result
class TestRISEntryYearExtraction:
"""Tests for year extraction from text."""
def _make(self):
return RISExporter()
def test_year_1990s(self):
"""Extracts 1990s year."""
exporter = self._make()
result = exporter._create_ris_entry("1", "Published in 1995")
assert "PY - 1995" in result
def test_year_2000s(self):
"""Extracts 2000s year."""
exporter = self._make()
result = exporter._create_ris_entry("1", "Released 2003")
assert "PY - 2003" in result
def test_no_year(self):
"""No year in text produces no PY entry."""
exporter = self._make()
result = exporter._create_ris_entry("1", "Title with no date")
assert "PY -" not in result
def test_first_year_extracted(self):
"""First valid year in text is extracted."""
exporter = self._make()
result = exporter._create_ris_entry(
"1", "Written in 2020, revised in 2023"
)
assert "PY - 2020" in result
def test_year_boundary_1900(self):
"""Year 1900 is valid (19xx pattern)."""
exporter = self._make()
result = exporter._create_ris_entry("1", "Historical text from 1900")
assert "PY - 1900" in result
def test_non_year_four_digit_ignored(self):
"""Four-digit numbers outside 19xx-20xx range are not matched."""
exporter = self._make()
result = exporter._create_ris_entry(
"1", "Article number 3456 in series"
)
assert "PY -" not in result
class TestRISEntryURLHandling:
"""Tests for URL handling in RIS entries."""
def _make(self):
return RISExporter()
def test_url_included(self):
"""URL is included as UR entry."""
exporter = self._make()
result = exporter._create_ris_entry(
"1", "Title", url="https://example.com"
)
assert "UR - https://example.com" in result
def test_url_with_path(self):
"""Full URL with path is preserved."""
exporter = self._make()
url = "https://example.com/path/to/article?id=123"
result = exporter._create_ris_entry("1", "Title", url=url)
assert f"UR - {url}" in result
def test_export_strips_userinfo_without_losing_path_or_query(self):
document = (
"Report\n\n## Sources\n\n"
"[1] Credential leak probe\n"
"URL: https://user:[email protected]/abs/2401.00001?q=a&b=2"
)
result = self._make().export_to_ris(document)
assert "UR - https://arxiv.org/abs/2401.00001?q=a&b=2" in result
assert "PB - arXiv" in result
assert "secret-token" not in result
@pytest.mark.parametrize("at_entity", ["@", "@", "@"])
def test_export_refuses_userinfo_remaining_after_stripping(self, at_entity):
url = f"https://first:password@secret{at_entity}example.org/doc"
result = self._make()._create_ris_entry("1", url, url=url)
assert "TI - [source URL omitted]" in result
assert "UR -" not in result
assert "PB -" not in result
assert "secret" not in result
assert "password" not in result
def test_url_title_does_not_export_credentials(self):
result = self._make()._create_ris_entry(
"1",
"https://user:[email protected]/article",
url="https://user:[email protected]/article",
)
assert "TI - https://example.org/article" in result
assert "UR - https://example.org/article" in result
assert "secret-token" not in result
def test_ris_url_with_control_is_omitted(self):
result = self._make()._create_ris_entry(
"1", "Title", url="https://example.org/a\rER - injected"
)
assert "UR -" not in result
assert result.count("ER -") == 1
def test_malformed_credential_url_is_omitted(self):
result = self._make()._create_ris_entry(
"1", "Title", url="https:user:[email protected]/article"
)
assert "UR -" not in result
assert "secret-token" not in result
def test_ris_url_with_at_in_path_is_preserved(self):
result = self._make()._create_ris_entry(
"1", "Title", url="https://medium.com/@author/article"
)
assert "UR - https://medium.com/@author/article" in result
@pytest.mark.parametrize(
"url",
[
"https://user:[email protected]/a",
"https://user:[email protected]/a",
"https://user:[email protected]/a",
],
)
def test_ris_url_with_entity_encoded_scheme_is_omitted(self, url):
"""The entity breaks the scheme token for ``urlsplit``, but a
consumer that decodes HTML entities follows a live credential
URL, so the field must not survive verbatim."""
result = self._make()._create_ris_entry("1", "Title", url=url)
assert "UR -" not in result
assert "user:pass" not in result
@pytest.mark.parametrize(
"title",
[
"https:user:[email protected]/article",
"https:/user:[email protected]/x",
"https://user:[email protected]/x",
"ftp://user:[email protected]/x",
"https://user:[email protected]/a",
"https://user:[email protected]/a",
],
)
def test_url_title_gate_covers_non_prefix_credential_spellings(self, title):
"""The title must not keep a credential spelling the shared check
flags, not only one after a literal ``http(s)://`` prefix."""
result = self._make()._create_ris_entry("1", title, url="")
ti_line = [
line for line in result.split("\n") if line.startswith("TI -")
][0]
assert "user:pass" not in ti_line
assert "user:secret" not in ti_line
assert "@example.org" not in ti_line
@pytest.mark.parametrize(
"title",
[
"https://example.org/article",
"https://medium.com/@author/article",
"Notes from [email protected] on https://example.org/x",
],
)
def test_url_title_gate_keeps_non_credential_titles(self, title):
"""A plain URL title, an @ in the path, and an email in prose are
not credentials and must survive the gate verbatim."""
result = self._make()._create_ris_entry("1", title)
assert f"TI - {title}" in result
class TestRISSharedCredentialCheck:
"""One credential check gates UR, TI, AU and DO, and no author, year or
DOI is cut out of a line that carries a credential.
Every refused value below carries ``secret`` in a URL's userinfo, so
no spelling of it may reach any field of the RIS record. The tests that
take ``password`` also run each value with the other passwords of
``_PASSWORDS`` in place of ``secret``, and no part of one may reach a
field either.
"""
def _make(self):
return RISExporter()
@pytest.mark.parametrize("password,parts", _PASSWORDS)
@pytest.mark.parametrize(
"value",
[
# urlsplit raises ValueError on each of these. The UR sanitizer
# already failed closed; the title check read the error as "no
# credential" and exported the title verbatim.
pytest.param(
"https://user:[email protected]\uff1a8443/x",
id="nfkc-fullwidth-colon",
),
pytest.param(
"https://[email protected]\uff0fowner/repo",
id="nfkc-fullwidth-solidus",
),
pytest.param(
"https://user:secret@exa\u2100mple.org/x",
id="nfkc-account-of",
),
pytest.param(
"https://user:[email protected]\ufe56/x",
id="nfkc-small-question-mark",
),
pytest.param(
"https://user:secret@example.org℅x",
id="entities-then-care-of",
),
pytest.param(
"https://a:b@user:secret@example.org℅x",
id="literal-then-entities-then-care-of",
),
pytest.param(
"https://secret-token@github.com℅x",
id="entity-username-then-care-of",
),
pytest.param(
"https://user:[email protected]]/x",
id="unbalanced-close-bracket",
),
pytest.param(
"https://user:secret@[broken/x",
id="unbalanced-open-bracket",
),
pytest.param(
"https://[@usersecret]@example.org/x",
id="bracketed-userinfo-then-entity-at",
),
],
)
def test_unparseable_credential_url_fails_closed(
self, value, password, parts
):
value = _with_password(value, password)
result = self._make()._create_ris_entry("1", value, url=value)
assert "TI - [source URL omitted]\n" in result
assert "UR -" not in result
_assert_holds_no_part(result, parts)
@pytest.mark.parametrize("password,parts", _PASSWORDS)
@pytest.mark.parametrize(
"value,expected",
[
(
"git+https://user:[email protected]/o/r",
"git+https://github.com/o/r",
),
("sftp://user:[email protected]/x", "sftp://example.org/x"),
("ssh://git:[email protected]/r", "ssh://gitlab.com/r"),
("wss://user:[email protected]/x", "wss://example.org/x"),
("//user:[email protected]/x", "//example.org/x"),
],
)
def test_userinfo_is_stripped_under_any_scheme(
self, value, expected, password, parts
):
"""The title check once accepted only http, https and ftp, so these
titles kept the credentials the UR field stripped."""
value = _with_password(value, password)
result = self._make()._create_ris_entry("1", value, url=value)
assert f"TI - {expected}\n" in result
assert f"UR - {expected}\n" in result
_assert_holds_no_part(result, parts)
@pytest.mark.parametrize("password,parts", _PASSWORDS)
@pytest.mark.parametrize(
"value",
[
pytest.param(
" https:/user:[email protected]/x", id="decimal-entity"
),
pytest.param(
" https:/user:[email protected]/x", id="hex-entity"
),
pytest.param(
" https:/user:[email protected]/x", id="no-semicolon"
),
pytest.param(
" https:/user:[email protected]/x", id="zero-padded"
),
pytest.param(
"  https:/user:[email protected]/x", id="two-spaces"
),
pytest.param(
" https:///user:[email protected]/x", id="triple-slash"
),
pytest.param("	https:/user:[email protected]/x", id="tab"),
],
)
def test_entity_encoded_leading_space_does_not_hide_userinfo(
self, value, password, parts
):
"""urlsplit drops the decoded leading whitespace before it reads the
scheme, so an authority cut at ``len(scheme) + 1`` began one
character early, stopped at the first ``/`` and missed the @."""
value = _with_password(value, password)
result = self._make()._create_ris_entry("1", value, url=value)
assert "TI - [source URL omitted]\n" in result
assert "UR -" not in result
_assert_holds_no_part(result, parts)
@pytest.mark.parametrize("password,parts", _PASSWORDS)
@pytest.mark.parametrize(
"value",
[
pytest.param("wss://secret@example.org/x", id="entity-at"),
pytest.param(
"https://user:secret%40example.org/x", id="percent-encoded-at"
),
pytest.param(
"https://user:secret\uff20example.org/x", id="fullwidth-at"
),
],
)
def test_encoded_at_sign_is_refused(self, value, password, parts):
value = _with_password(value, password)
result = self._make()._create_ris_entry("1", value, url=value)
assert "TI - [source URL omitted]\n" in result
assert "UR -" not in result
assert "PB -" not in result
_assert_holds_no_part(result, parts)
@pytest.mark.parametrize("password,parts", _PASSWORDS)
@pytest.mark.parametrize(
"value",
[
pytest.param(
"<https://user:[email protected]/abs/1>", id="angle-brackets"
),
pytest.param(
"[arXiv](https://user:[email protected]/abs/1)",
id="markdown-link",
),
pytest.param(
'"https://user:[email protected]/abs/1"', id="double-quotes"
),
pytest.param(
"'https://user:[email protected]/abs/1'", id="single-quotes"
),
pytest.param(
"(https://user:[email protected]/abs/1)", id="parentheses"
),
pytest.param(
"`https://user:[email protected]/abs/1`", id="backticks"
),
pytest.param(
"*https://user:[email protected]/abs/1*", id="emphasis"
),
pytest.param(
"**https://user:[email protected]/abs/1**", id="strong-emphasis"
),
pytest.param(
"_https://user:[email protected]/abs/1_",
id="underscore-emphasis",
),
pytest.param("see https://user:[email protected]/abs/1", id="prose"),
pytest.param(
"<https://user:[email protected]/abs/1>",
id="entity-angle-brackets",
),
pytest.param(
"\u200bhttps://user:[email protected]/abs/1",
id="zero-width-space",
),
pytest.param(
"https%3A//user:[email protected]/abs/1",
id="percent-encoded-colon",
),
],
)
def test_wrapped_credential_url_is_refused(self, value, password, parts):
"""urlsplit finds no scheme and no authority in these, so they were
exported verbatim as the source link and as the title."""
value = _with_password(value, password)
result = self._make()._create_ris_entry("1", value, url=value)
assert "TI - [source URL omitted]\n" in result
assert "UR -" not in result
assert "PB -" not in result
_assert_holds_no_part(result, parts)
def test_export_refuses_wrapped_source_urls(self):
document = (
"Report\n\n## Sources\n\n"
"[1] Autolinked source\n"
"URL: <https://user:[email protected]/abs/1>\n\n"
"[2] Linked source\n"
"URL: [arXiv](https://user:[email protected]/abs/2)\n"
)
result = self._make().export_to_ris(document)
assert result.count("ER - ") == 2
assert "UR -" not in result
assert "secret" not in result
def test_doi_url_loses_its_userinfo(self):
document = (
"Report\n\n## Sources\n\n"
"[1] Paper title\n"
"URL: https://example.org/p\n"
"DOI: https://user:[email protected]/10.1234/x\n"
)
result = self._make().export_to_ris(document)
assert "DO - https://doi.org/10.1234/x\n" in result
assert "secret" not in result
@pytest.mark.parametrize("password,parts", _PASSWORDS)
@pytest.mark.parametrize(
"line",
[
"Mirrored by https://user:[email protected]/p",
# Splitting authors on "&" cut this into parts that each passed.
"Mirrored by https://user:secret@mirror.example/p",
],
)
def test_author_text_is_credential_checked(self, line, password, parts):
line = _with_password(line, password)
result = self._make()._create_ris_entry("1", f"Paper title\n{line}")
assert "TI - Paper title\n" in result
_assert_holds_no_part(result, parts)
@pytest.mark.parametrize("password,parts", _PASSWORDS)
@pytest.mark.parametrize(
"title",
[
"https://DOI:[email protected]/x",
"https://x:doi:[email protected]/x",
"https://user:[email protected]/x",
"https://user:[email protected]/x",
"https://user:secret Published [email protected]/x",
"https://u by [email protected]/x",
],
)
def test_nothing_is_cut_out_of_a_credential_url_title(
self, title, password, parts
):
"""Title cleaning and the author, DOI and year matches cut text out
of the title. A cut that drops the @ of a credential URL left its
userinfo in TI, AU or DO, and a year in the password was written as
PY; the title is now sanitized whole, and gives no author, DOI or
year."""
title = _with_password(title, password)
result = self._make()._create_ris_entry("1", title)
assert "TI - https://example.org/x\n" in result
assert "AU -" not in result
assert "DO -" not in result
assert "PY -" not in result
_assert_holds_no_part(result, parts)
@pytest.mark.parametrize("password,parts", _PASSWORDS)
@pytest.mark.parametrize(
"title,expected",
[
(
"https:/by Smith./user:[email protected]/x",
"https://example.org/x",
),
("/by Smith./user:[email protected]/x", "//example.org/x"),
(
"https:/by Smith./user:secret [email protected]/x",
"https://example.org/x",
),
(
"https:/by Smith./user:DOI:[email protected]/x",
"https://example.org/x",
),
],
)
def test_title_that_cleaning_joins_into_a_credential_url_is_sanitized(
self, title, expected, password, parts
):
"""No title is a credential URL as written. Cleaning cuts the
author phrase out, and what is left joins into one, so the cleaned
title is checked as well as the title as written. In the last two,
a later cut, at ``Volume`` or at ``DOI:``, cut that URL's @ away
again: the title was written as ``https://user:secret`` or
``https://user:``, with the DOI ``[email protected]/x``. So every
form a cut makes is checked, the first one flagged is sanitized,
and the title gives no DOI, nor a year, which a password can hold.
"""
title = _with_password(title, password)
assert not citation_formatter._ris_spells_userinfo(title)
result = self._make()._create_ris_entry("1", title)
assert f"TI - {expected}\n" in result
assert "DO -" not in result
assert "PY -" not in result
_assert_holds_no_part(result, parts)
@pytest.mark.parametrize("password,parts", _PASSWORDS)
def test_doi_is_not_cut_out_of_a_credential_url_line(self, password, parts):
result = self._make()._create_ris_entry(
"1",
_with_password(
"Paper title\nMirror: https://DOI:[email protected]/x",
password,
),
)
assert "TI - Paper title\n" in result
assert "DO -" not in result
assert "PY -" not in result
_assert_holds_no_part(result, parts)
@freeze_time("2026-01-01")
def test_a_year_in_a_password_is_not_written_as_the_year(self):
"""The year pattern ran over the whole source text, so it took the
password 2024 out of the link's line, next to the link written
without it, and 1999 out of a token in a title written without it.
"""
url = "https://admin:[email protected]/"
result = self._make()._create_ris_entry(
"1", f"Internal dashboard\nURL: {url}", url=url
)
assert "UR - https://intranet.example/\n" in result
assert "PY -" not in result
result = self._make()._create_ris_entry(
"1", "https://deploy:[email protected]/"
)
assert "TI - https://host.example/\n" in result
assert "PY -" not in result
@pytest.mark.parametrize(
"password,parts",
[
pytest.param("2024", ("2024",), id="year"),
pytest.param("tok-1999-x", ("1999",), id="year-in-a-token"),
pytest.param("Kv3 1987", ("Kv3", "1987"), id="year-after-a-space"),
pytest.param("DOI:10.1234%2Fx", ("1234",), id="doi"),
pytest.param("by Smith", ("Smith",), id="name-after-by"),
pytest.param(
"by Smith", ("Smith",), id="name-after-by-no-break-space"
),
pytest.param(
"Xq7,Zr9&Wk3.Vm5",
("Xq7", "Zr9", "Wk3", "Vm5"),
id="punctuation",
),
],
)
@pytest.mark.parametrize(
"where",
["title", "URL line", "DOI line", "lowercase doi line", "export"],
)
@freeze_time("2026-01-01")
def test_a_password_shaped_like_a_field_is_not_cut_into_it(
self, where, password, parts
):
"""Each password matches a pattern that cuts a field out of text: a
year (PY), a ``DOI:`` label (DO), ``by`` and a name (AU), and the
commas, ampersands and full stops at which the author list is split
and ended. The URL that holds it is a value as a whole: a title, or
the value of a ``URL:`` or ``DOI:`` line, which the exporter reads
as the link or the DOI and strips the userinfo from. The year
pattern took ``2024`` out of any of them. From a ``URL:`` or
``DOI:`` line, the author pattern also took ``Smith@intranet`` out
of the password ``by Smith``, and the DOI pattern took
``https://admin:Kv3`` out of a lowercase ``doi:`` line: in a line, a
URL after a label ends at whitespace, so the check passed the line.
"""
url = f"https://admin:{password}@intranet.example/"
exporter = self._make()
if where == "title":
result = exporter._create_ris_entry("1", url, url=url)
kept = ["TI - https://intranet.example/\n"]
elif where == "URL line":
result = exporter._create_ris_entry(
"1", f"Internal dashboard\nURL: {url}", url=url
)
kept = ["UR - https://intranet.example/\n"]
elif where == "DOI line":
result = exporter._create_ris_entry(
"1", f"Internal dashboard\nDOI: {url}", metadata={"doi": url}
)
kept = ["DO - https://intranet.example/\n"]
elif where == "lowercase doi line":
result = exporter._create_ris_entry(
"1", f"Internal dashboard\ndoi: {url}"
)
kept = ["TI - Internal dashboard\n"]
else:
result = exporter.export_to_ris(
f"## Sources\n\n[1] Internal dashboard\nURL: {url}\n"
f"DOI: {url}\n"
)
kept = [
"UR - https://intranet.example/\n",
"DO - https://intranet.example/\n",
]
for line in kept:
assert line in result
assert "AU -" not in result
assert "PY -" not in result
_assert_holds_no_part(result, parts)
@pytest.mark.parametrize("password,parts", _PASSWORDS)
@pytest.mark.parametrize(
"line",
[
pytest.param(
"Mirrored by Jane Doe, https: user:secret@localhost/x",
id="after-a-comma",
),
pytest.param(
"Mirrored by Jane Doe and https: user:secret@localhost/x",
id="after-and",
),
pytest.param(
"Mirrored by Jane Doe & https: user:secret@localhost/x",
id="after-an-ampersand",
),
pytest.param(
"Mirrored by Jane Doe, https://user name:secret@localhost/x",
id="space-in-the-userinfo",
),
],
)
def test_a_later_author_is_read_as_the_url_it_starts(
self, line, password, parts
):
"""The author pattern ends the list at a full stop and splits it at
commas, ``and`` and ampersands, any of which a password can hold,
as ``Xq7,Zr9&Wk3.Vm5`` does. Each author was checked as cut, so the
cut hid the @ of a URL that only a reading as a whole finds: the
record held the author ``https: user:Xq7`` and the authors ``Zr9``
and ``Wk3``. Each author is now read as the URL it starts, to the
end of its line, and the line gives no author or year."""
line = _with_password(line, password)
result = self._make()._create_ris_entry("1", f"Paper title\n{line}")
assert "TI - Paper title\n" in result
assert "AU -" not in result
_assert_holds_no_part(result, parts)
@pytest.mark.parametrize(
"value,flagged",
[
pytest.param(
"https: user:a.b@localhost/x", True, id="at-the-start"
),
pytest.param(
"Jane Doe, https: user:a.b@localhost/x",
True,
id="after-a-comma",
),
pytest.param(
"Jane Doe and \ufeffhttps://user name:a@localhost/x",
True,
id="after-and-and-a-byte-order-mark",
),
# The first URL's authority ends at the second one's slashes.
pytest.param(
"https:a, https://u.v w@localhost/x", True, id="overlapping"
),
pytest.param("https://@@localhost/x", False, id="at-signs-only"),
pytest.param(
"Jane Doe, https://example.org/@jane", False, id="at-in-a-path"
),
pytest.param(
"Jane Doe, mail [email protected]", False, id="no-url-start"
),
pytest.param(
"Grand and https: email me at [email protected]",
False,
id="prose-after-and",
),
],
)
def test_values_are_read_as_the_urls_their_authors_start(
self, value, flagged
):
"""A URL as a whole is read where a value, or an author in it,
starts: past whitespace in its authority, to the first ``/``, ``?``
or ``#``, with an @ there after any leading @ signs counting as
userinfo, as a whole value is read. Prose after a scheme's colon
is kept as it is in a title."""
assert citation_formatter._ris_value_starts_userinfo(value) is flagged
def test_value_starts_are_checked_in_linear_time(self):
"""Each of the 20,000 authors here starts a URL whose authority
starts inside the first one's and ends where it ends, so the
authority is scanned once, not once per URL: about forty
milliseconds, against about four seconds when each URL scans it."""
value = "https:a," * 20000
started = time.perf_counter()
flagged = citation_formatter._ris_value_starts_userinfo(value)
elapsed = time.perf_counter() - started
assert not flagged
assert elapsed < 1.0, f"the check took {elapsed:.2f}s"
@pytest.mark.parametrize(
"full_text,url,expected",
[
pytest.param(
"Quarterly report 2023\n"
"URL: https://admin:[email protected]/",
"https://admin:[email protected]/",
"PY - 2023\n",
id="year-in-the-title",
),
pytest.param(
"Archived post\nURL: https://example.org/2019/post",
"https://example.org/2019/post",
"PY - 2019\n",
id="year-in-a-plain-link",
),
pytest.param(
"Archived post\nurl: https://example.org/2019/post",
"",
"PY - 2019\n",
id="year-in-a-plain-lowercase-link",
),
pytest.param(
"Paper by Jane Doe\n"
"URL: https://admin:[email protected]/",
"https://admin:[email protected]/",
"AU - Jane Doe\n",
id="author-in-the-title",
),
],
)
def test_a_field_outside_a_credential_line_is_kept(
self, full_text, url, expected
):
"""Only a line that carries a credential gives no year, author or
DOI. A plain ``URL:`` line, in any letter case, still gives a year,
and so does a title beside a credential URL's line."""
result = self._make()._create_ris_entry("1", full_text, url=url)
assert expected in result
assert "2024" not in result
assert "secret" not in result
@pytest.mark.parametrize(
"url",
[
"https://evil.example\\@arxiv.org/abs/1",
"https://evil.example\\.arxiv.org/abs/1",
],
)
def test_backslash_in_authority_is_omitted(self, url):
"""WHATWG parsers end the authority at the backslash, so the host is
evil.example, while urlsplit reads arxiv.org; neither reading is
exported as the link, and arXiv is not credited as publisher."""
result = self._make()._create_ris_entry("1", "Title", url=url)
assert "UR -" not in result
assert "PB -" not in result
@pytest.mark.parametrize("password,parts", _PASSWORDS)
@pytest.mark.parametrize(
"value",
[
pytest.param("\\\\user:[email protected]/x", id="backslashes"),
pytest.param("/\\user:[email protected]/x", id="slash-backslash"),
pytest.param("\\/user:[email protected]/x", id="backslash-slash"),
pytest.param(
"\\\\\\user:[email protected]/x", id="three-backslashes"
),
pytest.param("///user:[email protected]/x", id="three-slashes"),
pytest.param(
"\\/\\/user:[email protected]/x", id="escaped-slashes"
),
pytest.param(
"<\\\\user:[email protected]/x>", id="angle-brackets"
),
pytest.param("see \\\\user:[email protected]/x", id="prose"),
pytest.param(
"\\\\ user:[email protected]/x", id="space-after-the-run"
),
pytest.param(
"https:\\\\ user:[email protected]/x",
id="https-backslashes-then-space",
),
pytest.param(
"https:/// user:[email protected]/x",
id="https-three-slashes-then-space",
),
],
)
def test_run_of_slashes_and_backslashes_opens_an_authority(
self, value, password, parts
):
"""A browser reads any run of two or more slashes or backslashes
after https: as //, and does the same with no scheme against an
https page, so each value is a credential URL to it. The check knew
only //, so these were written verbatim as titles, and the
scheme-less ones as links too: urlsplit finds no authority in them
to strip."""
value = _with_password(value, password)
result = self._make()._create_ris_entry("1", value, url=value)
assert "TI - [source URL omitted]\n" in result
assert "UR -" not in result
assert "PB -" not in result
_assert_holds_no_part(result, parts)
@pytest.mark.parametrize("password,parts", _PASSWORDS)
@pytest.mark.parametrize(
"value",
[
# Inside the scheme.
pytest.param(
"ht	tps:user:[email protected]/x", id="scheme-tab-entity"
),
pytest.param(
"ht tps:/user:[email protected]/x", id="scheme-lf-decimal"
),
pytest.param(
"ht%0Dtps:user:[email protected]/x", id="scheme-cr-percent"
),
# Between the slashes.
pytest.param(
"https:/	/user:[email protected]/x",
id="slashes-tab-decimal",
),
pytest.param(
"https:/
/user:[email protected]/x",
id="slashes-lf-entity",
),
pytest.param(
"https:/%0D/user:[email protected]/x", id="slashes-cr-percent"
),
pytest.param(
"ftp:/%09/user:[email protected]/x", id="ftp-slashes-tab"
),
pytest.param(
"/ /user:[email protected]/x", id="bare-slashes-lf"
),
# After the scheme's colon.
pytest.param(
"https:	user:[email protected]/x", id="colon-tab-hex"
),
pytest.param(
"https:%0A/user:[email protected]/x", id="colon-lf-percent"
),
pytest.param(
"https: user:[email protected]/x", id="colon-cr-decimal"
),
# Written as it is, not encoded.
pytest.param(
"https:/\t/user:[email protected]/x", id="slashes-raw-tab"
),
],
)
def test_tab_or_line_break_does_not_hide_userinfo(
self, value, password, parts
):
"""A browser deletes every tab, LF and CR from a URL before it
parses it, so each value, with its entity or escape decoded, is a
credential URL to it: ``ht	tps:user:secret@...`` reads as
``https:user:secret@...``. The check took the decoded character for
whitespace that ends or blocks an authority, so the title and the
DOI were written verbatim, and the link too wherever urlsplit found
no scheme in it. A raw tab, which serialization turns into a space,
is deleted for the check as well."""
value = _with_password(value, password)
result = self._make()._create_ris_entry(
"1", f"{value}\nDOI: {value}", url=value
)
assert "TI - [source URL omitted]\n" in result
assert "UR -" not in result
assert "PB -" not in result
assert "DO -" not in result
_assert_holds_no_part(result, parts)
from_metadata = self._make()._create_ris_entry(
"1", "Paper title", metadata={"doi": value}
)
assert "TI - Paper title\n" in from_metadata
assert "DO -" not in from_metadata
_assert_holds_no_part(from_metadata, parts)
@pytest.mark.parametrize(
"title,expected",
[
("Notes\tfrom [email protected]", "Notes from [email protected]"),
("Notes\rfrom [email protected]", "Notes from [email protected]"),
],
)
def test_tab_or_carriage_return_in_a_plain_title_keeps_it(
self, title, expected
):
"""Deleting the character only feeds the check. A title it does not
flag is written as before, with the character turned into a space."""
result = self._make()._create_ris_entry("1", title)
assert f"TI - {expected}\n" in result
@pytest.mark.parametrize("password,parts", _PASSWORDS)
@pytest.mark.parametrize(
"value",
[
pytest.param(
"\ufeffhttps://user name:[email protected]/x",
id="space-in-userinfo",
),
pytest.param(
"\ufeff\\\\ user:[email protected]/x", id="space-after-the-run"
),
],
)
def test_leading_byte_order_mark_does_not_hide_userinfo(
self, value, password, parts
):
"""JavaScript's trim() removes a leading U+FEFF, which Python's
strip() keeps, so a reader that trims the field reads each value as
a URL as a whole, whitespace in its authority included. The check
read it as text, where a URL ends at whitespace, so the title and
the link were written verbatim."""
value = _with_password(value, password)
result = self._make()._create_ris_entry("1", value, url=value)
assert "TI - [source URL omitted]\n" in result
assert "UR -" not in result
assert "PB -" not in result
_assert_holds_no_part(result, parts)
@pytest.mark.parametrize("password,parts", _PASSWORDS)
@pytest.mark.parametrize("scheme", ["https", "HTTP", "ftp"])
@pytest.mark.parametrize(
"gap",
[
pytest.param(" ", id="space"),
pytest.param("\u00a0", id="no-break-space"),
pytest.param("\x0b", id="vertical-tab"),
pytest.param("\u3000", id="ideographic-space"),
pytest.param(" ", id="entity-space"),
pytest.param("/ ", id="slash-then-space"),
pytest.param("\\ ", id="backslash-then-space"),
],
)
def test_whitespace_after_a_scheme_colon_does_not_hide_userinfo(
self, scheme, gap, password, parts
):
"""Parsed with no base, each value, its entity decoded, is a
credential URL to a browser, whose userinfo starts with the
whitespace, encoded: ``https: user:[email protected]/x`` reads as
``https://%20user:[email protected]/x``. The check took whitespace
right after a special scheme's colon, or after its colon and one
slash or backslash, for the end of the URL, so the title, the
author and a metadata DOI were written verbatim. The source link
was already left out."""
value = f"{scheme}:{gap}user:{password}@example.org/x"
result = self._make()._create_ris_entry(
"1", f"{value}\nDOI: {value}\nMirrored by {value}", url=value
)
assert "TI - [source URL omitted]\n" in result
assert "AU -" not in result
assert "UR -" not in result
assert "PB -" not in result
_assert_holds_no_part(result, parts)
from_metadata = self._make()._create_ris_entry(
"1", "Paper title", metadata={"doi": value}
)
assert "TI - Paper title\n" in from_metadata
assert "DO -" not in from_metadata
_assert_holds_no_part(from_metadata, parts)
@pytest.mark.parametrize("password,parts", _PASSWORDS)
@pytest.mark.parametrize(
"value",
[
pytest.param(
"https: user:secret @example.org/x", id="space-before-the-at"
),
pytest.param("HTTP: [email protected]", id="token-only"),
pytest.param("ftp: user:[email protected] ", id="trailing-space"),
pytest.param(
"https:\u00a0 user:[email protected]/x", id="whitespace-run"
),
],
)
def test_one_token_after_a_scheme_colon_and_whitespace_is_flagged(
self, value, password, parts
):
"""Past whitespace right after a special scheme's colon, one token,
an @ and a host without whitespace open an authority. Whitespace
before the @ does not prevent it, nor does whitespace after the host
at the end of the value, which a browser trims. A browser reads the
values, as written, with the password ``secret%20``, the username
``%20secret-token``, the URL ``ftp://%20user:[email protected]/``
and the username ``%C2%A0%20user``."""
value = _with_password(value, password)
assert citation_formatter._ris_spells_userinfo(value)
result = self._make()._create_ris_entry("1", value, url=value)
assert "TI - [source URL omitted]\n" in result
assert "UR -" not in result
_assert_holds_no_part(result, parts)
@pytest.mark.parametrize("password,parts", _PASSWORDS)
@pytest.mark.parametrize(
"control",
[
pytest.param("\x00", id="null"),
pytest.param("\x1b", id="escape"),
pytest.param("\x7f", id="delete"),
pytest.param("\x9f", id="c1-control"),
],
)
@pytest.mark.parametrize(
"shape",
[
pytest.param(
"https: user:secret@localhost {}", id="after-the-host"
),
pytest.param(
"https: user:secret {}@localhost/x", id="before-the-at"
),
pytest.param(
"https: {} user:secret@localhost/x", id="after-the-colon"
),
],
)
def test_control_among_whitespace_after_a_scheme_colon_is_flagged(
self, shape, control, password, parts
):
"""Serialization writes a control character as a space, so it
writes each value in a form the check flags, such as
``https: user:secret@localhost`` and two spaces, and a browser reads
each written form as userinfo at localhost. Python's ``re`` does not
count these controls as whitespace, so the check read each as text,
found no credential, and the title, the author and a metadata DOI
were written verbatim. The source link was already left out."""
value = _with_password(shape.format(control), password)
result = self._make()._create_ris_entry(
"1", f"{value}\nDOI: {value}\nMirrored by {value}", url=value
)
assert "TI - [source URL omitted]\n" in result
assert "AU -" not in result
assert "UR -" not in result
_assert_holds_no_part(result, parts)
from_metadata = self._make()._create_ris_entry(
"1", "Paper title", metadata={"doi": value}
)
assert "TI - Paper title\n" in from_metadata
assert "DO -" not in from_metadata
_assert_holds_no_part(from_metadata, parts)
@pytest.mark.parametrize("password,parts", _PASSWORDS)
@pytest.mark.parametrize(
"value",
[
pytest.param(
"https: user:secret@localhost %1B", id="after-the-host"
),
pytest.param(
"https: user:secret %1B@localhost/x", id="before-the-at"
),
pytest.param(
"https: %1B user:secret@localhost/x", id="after-the-colon"
),
],
)
def test_encoded_control_among_whitespace_is_flagged(
self, value, password, parts
):
"""Percent-decoded, each value holds an escape character after
whitespace, which a browser trims after the host and keeps in the
userinfo anywhere else, so it reads each as userinfo at localhost.
The check reads every decoded form with its controls as spaces, as
it reads the value itself. It read the decoded control as text, and
the title was written verbatim."""
value = _with_password(value, password)
result = self._make()._create_ris_entry("1", value, url=value)
assert citation_formatter._ris_spells_userinfo(value)
assert "TI - [source URL omitted]\n" in result
_assert_holds_no_part(result, parts)
@pytest.mark.parametrize("password,parts", _PASSWORDS)
@pytest.mark.parametrize(
"value",
[
pytest.param(
"https: user:secret@localhost \ufeff", id="after-a-space"
),
pytest.param(
"HTTP: secret-token@localhost \ufeff \ufeff", id="in-a-run"
),
pytest.param(
"ftp: user:secret @localhost\u3000\ufeff",
id="after-an-ideographic-space",
),
],
)
def test_trailing_byte_order_mark_does_not_hide_userinfo(
self, value, password, parts
):
"""JavaScript's trim() removes a trailing U+FEFF as it removes a
leading one, so a reader that trims the field reads each value, as
written, as a credential URL: ``https: user:secret@localhost`` with
the username ``%20user``, ``HTTP: secret-token@localhost`` with the
username ``%20secret-token``, and ``ftp: user:secret @localhost``
with the password ``secret%20``. After the host the check accepted
only whitespace before the end of the value, so the title, the
author and a metadata DOI were written verbatim."""
value = _with_password(value, password)
result = self._make()._create_ris_entry(
"1", f"{value}\nDOI: {value}\nMirrored by {value}", url=value
)
assert "TI - [source URL omitted]\n" in result
assert "AU -" not in result
assert "UR -" not in result
_assert_holds_no_part(result, parts)
from_metadata = self._make()._create_ris_entry(
"1", "Paper title", metadata={"doi": value}
)
assert "DO -" not in from_metadata
_assert_holds_no_part(from_metadata, parts)
def test_controls_after_a_host_are_checked_in_linear_time(self):
"""After the host, the value may end in whitespace, controls and
U+FEFF, and the host may hold controls and U+FEFF itself. The host
is matched possessively, so a long run of them followed by text is
scanned once. If the match could give characters back, each shorter
host would rescan the rest of the value: about two seconds for this
run, against about ten milliseconds."""
value = "https: a@" + "\x01" * 20000 + " x"
started = time.perf_counter()
flagged = citation_formatter._ris_spells_userinfo(value)
elapsed = time.perf_counter() - started
assert not flagged
assert elapsed < 0.25, f"the check took {elapsed:.2f}s"
@pytest.mark.parametrize(
"title",
[
"HTTPS: why you should email [email protected]",
"http: a guide to the @ sign",
"HTTP: email me at [email protected]",
"HTTP:\u00a0email me at [email protected]",
"HTTP: [email protected] explains caching",
"HTTP: @mention",
],
)
def test_prose_after_a_scheme_word_keeps_its_title(self, title):
"""A browser reads four of these as URLs whose userinfo is what
precedes the @: ``HTTP: email me at [email protected]`` as
``http://%20email%20me%20at%[email protected]/``, and
``HTTP: @mention`` with a space for its username. The other two,
``http: a guide to the @ sign`` and
``HTTP: [email protected] explains caching``, do not parse, since their
would-be hosts hold whitespace. They are titles all the same. Past
whitespace right after a special scheme's colon, the check reads an
authority only when one token, an @ and a host without whitespace
follow. Each of these holds whitespace in its would-be userinfo or
host, or has no token before its @."""
result = self._make()._create_ris_entry("1", title)
assert f"TI - {title}\n" in result
@pytest.mark.parametrize(
"value",
[
"https://example.org/article",
"https://medium.com/@author/article",
"https://web.archive.org/web/2020/https://medium.com/@author/post",
"https://example.org/?email=bob%40example.org",
"https://www.google.com/url?q=https%3A%2F%2Fexample.org%2F%40user",
"https://de.wikipedia.org/wiki/Stra%C3%9Fe",
"https://example.org//double//slash",
"https://example.org/a%09b%0Ac",
"https://example.org/?q=a	b",
],
)
def test_url_without_credentials_is_unchanged(self, value):
result = self._make()._create_ris_entry("1", value, url=value)
assert f"TI - {value}\n" in result
assert f"UR - {value}\n" in result
@pytest.mark.parametrize(
"title",
[
"Re: question from [email protected]",
"news:[email protected]",
"Using //@ts-ignore in TypeScript",
"Using \\\\@ts-ignore in TypeScript",
"\\\\fileserver\\share\\notes.txt",
"HTTP: The Definitive Guide",
"Tom & Jerry @ Home",
"Tabs	and breaks @ home",
"Comments open with /	/ and @ marks a decorator",
],
)
def test_title_without_url_credentials_is_unchanged(self, title):
result = self._make()._create_ris_entry("1", title)
assert f"TI - {title}\n" in result
@pytest.mark.parametrize("password,parts", _PASSWORDS)
def test_no_field_keeps_a_value_the_shared_check_flags(
self, password, parts
):
"""UR, TI, AU and DO cannot drift apart: whatever the shared check
flags is absent from every field of the record, and no field, PY
included, holds any part of its password."""
from local_deep_research.text_optimization.citation_formatter import (
_ris_spells_userinfo,
)
values = [
"https://user:[email protected]\uff1a8443/x",
"git+https://user:[email protected]/o/r",
"//user:[email protected]/x",
"\\\\user:[email protected]/x",
"///user:[email protected]/x",
" https:/user:[email protected]/x",
"https://user:secret%40example.org/x",
"[arXiv](https://user:[email protected]/abs/1)",
"https://first:secret@x@example.org/doc",
"https://DOI:[email protected]/x",
"ht	tps:user:[email protected]/x",
"/ /user:[email protected]/x",
"\ufeffhttps://user name:[email protected]/x",
"https: user:[email protected]/x",
"HTTP:/ user:[email protected]/x",
]
for value in values:
value = _with_password(value, password)
assert _ris_spells_userinfo(value), value
result = self._make()._create_ris_entry(
"1",
f"{value}\nWritten by {value}\nDOI: {value}",
url=value,
)
for line in result.split("\n"):
assert not _ris_spells_userinfo(line[6:]), (value, line)
_assert_holds_no_part(result, parts)
def test_every_field_is_written_in_a_form_the_check_passes(self):
"""Serialization turns every control character, U+2028 and U+2029
into a space, and the check reads each value that way too, so no
field of a record is written with a value it flags. The values put
a C0, DEL or C1 control, U+2028, U+FEFF, or an entity- or
percent-encoded control into credential URLs and titles, around and
in place of their whitespace. Before, a credential URL with a
control after its whitespace passed the check and was written with
a space in place of the control, which the check flags."""
from local_deep_research.text_optimization.citation_formatter import (
_ris_spells_userinfo,
)
shapes = [
"https: user:secret@localhost{}",
"https: user:secret@localhost {}",
"https: user:secret {}@localhost/x",
"https: {} user:secret@localhost/x",
"https:{}user:secret@localhost/x",
"https://user:secret@localhost{}/x",
"<https://user:secret@localhost/x>{}",
"HTTP: email me at{} [email protected]",
"HTTP: {}@mention",
"Notes{}from [email protected]",
]
characters = [
"\x00",
"\x1b",
"\x7f",
"\x85",
"\x9f",
"\u2028",
"\ufeff",
"",
"%01",
]
exporter = self._make()
for shape in shapes:
for character in characters:
value = shape.format(character)
records = [
exporter._create_ris_entry(
"1",
f"{value}\nDOI: {value}\nMirrored by {value}",
url=value,
),
exporter._create_ris_entry(
"1", "Paper title", metadata={"doi": value}
),
exporter.export_to_ris(
f"## Sources\n\n[1] {value}\nURL: {value}\n"
f"DOI: {value}\n"
),
]
for record in records:
for line in record.split("\n"):
assert not _ris_spells_userinfo(line[6:]), (
value,
line,
)
@pytest.mark.parametrize(
"bound,limit,value",
[
# "%41" has 3 characters; its readings are "%41" and "A".
pytest.param(
"_RIS_MAX_DECODED_LENGTH", 3, "%41", id="length-of-the-value"
),
# "a\ufb01" has 2 characters; NFKC reads it as "afi", with 3.
pytest.param(
"_RIS_MAX_DECODED_LENGTH",
3,
"a\ufb01",
id="length-of-a-decoded-reading",
),
# The readings "a%41" and "aA" have 6 characters together.
pytest.param(
"_RIS_MAX_DECODED_TOTAL_LENGTH",
6,
"a%41",
id="length-of-all-readings",
),
pytest.param("_RIS_MAX_READINGS", 2, "%41", id="reading-count"),
],
)
def test_value_past_a_reading_bound_is_left_out(
self, monkeypatch, bound, limit, value
):
"""Each bound is lowered to what a few characters reach. At the
bound the value is checked as usual; past it the value counts as
flagged and is left out of the title and the link. Only the length
of the value itself was bounded before, though NFKC can make a
reading longer than the value, and the readings together were not
bounded at all."""
monkeypatch.setattr(citation_formatter, bound, limit)
kept = self._make()._create_ris_entry("1", value, url=value)
assert f"TI - {value}\n" in kept
assert f"UR - {value}\n" in kept
monkeypatch.setattr(citation_formatter, bound, limit - 1)
result = self._make()._create_ris_entry("1", value, url=value)
assert citation_formatter._ris_spells_userinfo(value)
assert "TI - [source URL omitted]\n" in result
assert "UR -" not in result
assert value not in result
def test_value_with_a_control_character_is_measured(self, monkeypatch):
"""One reading of a value is the value with its control characters
written as spaces, so an ASCII value holding a control is measured
against the bounds like a value that needs decoding, and counts as
flagged past them. It was its only reading, and was not measured."""
value = "Notes\x01from bob"
monkeypatch.setattr(citation_formatter, "_RIS_MAX_READINGS", 2)
kept = self._make()._create_ris_entry("1", value)
assert "TI - Notes from bob\n" in kept
monkeypatch.setattr(citation_formatter, "_RIS_MAX_READINGS", 1)
result = self._make()._create_ris_entry("1", value)
assert citation_formatter._ris_spells_userinfo(value)
assert "TI - [source URL omitted]\n" in result
class TestRISEntryStructuralValidity:
"""Regression guard for the accumulator-clobber bug.
``_create_ris_entry`` initialized ``lines = []`` as the RIS-output
accumulator, then immediately reassigned ``lines = full_text.split("\\n")``
to read the title — overwriting the accumulator with the raw source body.
Every RIS field was then appended *after* those source lines, so each entry
emitted the source text before the mandatory leading ``TY - `` tag and
reference managers (Zotero/Mendeley/EndNote) rejected the record. The
pre-existing tests only used ``"<tag>" in result`` substring checks, so they
never caught the leaked prefix.
"""
# RIS records are newline-separated ``XX - value`` lines where the tag is
# two uppercase-or-digit chars (TY, ID, TI, AU, DO, PY, UR, PB, Y1, DA, LA,
# ER), two spaces, a hyphen, then a space.
_RIS_LINE = re.compile(r"^[A-Z][A-Z0-9] - ")
def test_entry_starts_with_ty_tag(self):
"""RIS requires ``TY`` to be the first tag of the record."""
exporter = RISExporter()
source = (
"Understanding Widgets\n"
"URL: https://example.com/widgets\n"
"Collection: Gadgets"
)
result = exporter._create_ris_entry(
"7", source, url="https://example.com/widgets"
)
assert result.startswith("TY - "), result
def test_no_raw_source_lines_leak(self):
"""Every non-blank output line is a RIS tag, never raw source body."""
exporter = RISExporter()
source = (
"My Title\n"
"BODYLEAKMARKER this descriptive line must never be emitted\n"
"Collection: Internal"
)
result = exporter._create_ris_entry("1", source)
assert "BODYLEAKMARKER" not in result
for line in result.split("\n"):
if not line.strip():
continue
assert self._RIS_LINE.match(line), f"non-RIS line leaked: {line!r}"