Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(ris): remove credentials from exported URLs (#6991)

LearningCircuitf7b3e4d16d0f
diff --git a/changelog.d/6863.security.md b/changelog.d/6863.security.md
new file mode 100644
index 0000000000..1ff4c6ec24
--- /dev/null
+++ b/changelog.d/6863.security.md
@@ -0,0 +1,3 @@
+RIS exports no longer write URL credentials into any field: the `user:password@` userinfo before a URL's host, as a browser's URL parser reads it. Text that the rules below for a URL in other text keep is still written as it is. A source link keeps its host, path and query without them, and so does a title or a `DOI:` line that is a credential URL. A value is checked as it is and in its decoded forms: with HTML entities and percent-escapes decoded, NFKC-normalized, with every tab, line feed and carriage return deleted, as a browser deletes them from a URL, and with every control character, U+2028 and U+2029 turned into a space, as the RIS file writes them. A value whose credentials cannot be removed cleanly is left out instead, and a title is then written as `[source URL omitted]`: a credential URL wrapped in `<…>`, a markdown link, quotes or other text, one whose delimiters are spelled with HTML entities, percent-encoding or full-width characters, one that a tab, line feed or carriage return breaks up, one after a bare run of slashes and backslashes other than `//`, and one that does not parse. In every form, a URL inside a wrapper or other text ends at the first whitespace, as a linkifier ends it. A value that is a URL as a whole, past any leading whitespace, controls or U+FEFF, is read past whitespace in its host part, as in `https:/ user:pass@host` and `https: user:pass@host`. Where whitespace directly follows the colon of `http:`, `https:`, `ftp:`, `ws:` or `wss:`, it is read past only when a word follows it, then an `@` and a host, so that a title such as `HTTP: email me at [email protected]` is kept: the word and the host hold no whitespace, `/`, `\`, `?` or `#`, the host holds no `@`, whitespace may come between the word and the `@`, and the host must be followed by `/`, `\`, `?`, `#` or nothing but whitespace, controls or U+FEFF. A value that needs decoding, one containing `&`, `%`, a control character or a non-ASCII character, counts as carrying credentials when its decoded forms, itself included, are too long or too many to check. An author, a publication year, or a DOI found in other text, is no longer taken from a line that carries a credential URL, so a password such as the `2024` of `URL: https://admin:[email protected]/` is not written as the year. A line carries one when it does as written, when the value of a `URL:` or `DOI:` line does, or when the author list, an author in it, or a DOI after `DOI:` starts a value that does: each such value is read as a URL as a whole, to the end of its line, past the full stop, comma, `and`, ampersand or whitespace at which the author or the DOI is cut, as in `by https: user:a.b@host`. A title carrying one is sanitized as written rather than after its metadata is trimmed, and is checked again after each cut the trimming makes, since a cut can join what is left into a credential URL and a later cut can split it again; such a title gives no year or DOI either. Credentials are recognised under any scheme; after a bare `//`, `\\`, `/\`, `///` or other run of two or more slashes and backslashes, all of which a browser on an `https` page reads as `//`; and in the `https:user:pass@host` form browsers accept. Source links containing control characters, or a backslash in the host part, are left out too. An ordinary `@` in a path, as in `medium.com/@author`, is kept, but two rare shapes are left out to be safe: an `@` in the segment after a `//` inside a path, as in `/img//logo@2x`, and one in the segment after `http:`, `https:`, `ftp:`, `ws:` or `wss:` inside a path or query, as in `?q=ws:foo@bar`. A password that holds an unencoded `/`, `?` or `#` ends the authority there, so text such as `https://user:2024/Secret@host/x` holds no userinfo and is kept.
+
+Source lines are also checked in their unsplit record before title and metadata extraction can separate a password from its `@`. Credential fragments split by LF or CRLF are omitted, including splits inside encoded delimiters; unrelated titles, authors, years and DOI lines remain available. This source-context check uses the same decoding limits and omits fields when it cannot complete within those limits.
diff --git a/src/local_deep_research/text_optimization/citation_formatter.py b/src/local_deep_research/text_optimization/citation_formatter.py
index 6b600601ec..76a8852881 100644
--- a/src/local_deep_research/text_optimization/citation_formatter.py
+++ b/src/local_deep_research/text_optimization/citation_formatter.py
@@ -2,6 +2,8 @@
import html
import re
+import unicodedata
+import urllib.parse
from array import array
from enum import Enum
from html.entities import html5
@@ -1849,6 +1851,474 @@ csl: apa.csl
return bibliography.strip()
+# One credential check, ``_ris_spells_userinfo``, decides for every RIS
+# field built from source text: UR, TI, AU and DO are never written with a
+# value it flags. It also reads each value as serialization writes it
+# (``_ris_flatten``), so no field is written in a form it flags either. A
+# source URL goes through ``_safe_ris_url``, which runs the check last; a
+# title, author or DOI the check flags is replaced by ``_safe_ris_url``'s
+# userinfo-free URL for it, or dropped (a title, by a placeholder). A
+# password can hold a name, a year or a DOI, so no author, no year (PY)
+# and no DOI matched in other text is cut out of a line that carries a
+# credential (``_ris_line_carries_userinfo``): a line the check flags, or
+# one in which a value read from it holds a credential when read as a URL
+# as a whole from where the value starts (``_ris_value_starts_userinfo``).
+# Nor is a year or DOI taken from a title that its cleaning joins into a
+# credential URL. PB is derived from the host of the sanitized UR. TY, Y1,
+# DA, LA and ER are fixed or dated, and ID holds the citation number.
+# ``_safe_ris_url`` also omits URLs it cannot parse unambiguously; a title,
+# author or DOI need not be a URL, so such text stays in those fields
+# unless the check flags it.
+#
+# Userinfo is what precedes the last @ of a URL's authority, as the WHATWG
+# URL Standard parses it. The authority ends at the first ``/``, ``?`` or
+# ``#``, so a password that holds one of them unencoded is not userinfo,
+# and the check does not flag it: ``https://user:2024/Secret@host/x`` has
+# the host ``user``, the port 2024 and no credential. Under any scheme,
+# and under none (``//host``), an authority follows ``//``. Under
+# the schemes the WHATWG URL Standard calls special, it also follows the
+# colon and any run of ``/`` or ``\`` (none included): a browser reads
+# ``https:user:pass@host`` and ``https:\\user:pass@host`` as credential
+# URLs. ``file`` is special too, but a file URL carries no userinfo. A
+# value with no scheme is resolved against a base URL, and against a
+# special one, such as an ``https`` page, any run of two or more ``/`` or
+# ``\`` opens an authority as ``//`` does: a browser reads
+# ``\\user:pass@host``, ``/\user:pass@host`` and ``///user:pass@host``
+# there as ``https://user:pass@host``.
+_RIS_SPECIAL_SCHEME = r"(?:https?|ftp|wss?)"
+
+# An authority opened anywhere in a value, so a credential URL wrapped in
+# ``<...>``, a markdown link, quotes, parentheses, backticks or emphasis, or
+# written inside prose, is still seen; a run of two or more ``/`` or ``\``
+# opens one wherever it appears, as ``//`` does. In each reading of a
+# value (``_ris_readings``), a URL inside text ends at whitespace, as a
+# linkifier ends it, so an e-mail address later in the text is not read
+# as its userinfo. Neither is what follows whitespace inside a wrapped
+# URL's authority, as in ``<https://user :pass@host>``; only a value that
+# is a URL as a whole is read past it. The lookbehind keeps the ``ws:`` of
+# ``news:`` and the ``ftp:`` of ``sftp:`` from opening an authority.
+_RIS_EMBEDDED_AUTHORITY_RE = re.compile(
+ rf"(?:(?<![a-z0-9+.\-]){_RIS_SPECIAL_SCHEME}:[/\\]*|[/\\]{{2,}})"
+ r"([^\s/?#]*)",
+ re.IGNORECASE,
+)
+
+# What a value that is a URL as a whole may begin and end with: whitespace,
+# controls and U+FEFF. RIS serialization turns controls into spaces, and a
+# reader may trim a field: JavaScript's ``trim()`` removes U+FEFF as well,
+# which Python's ``strip()`` keeps. Inside a value, U+FEFF is not
+# whitespace.
+_RIS_TRIMMED = r"[\s\x00-\x20\x7f-\x9f\ufeff]"
+
+# The authority of a value that is a URL as a whole, past any leading
+# ``_RIS_TRIMMED`` characters. It runs up to the first ``/``, ``?`` or
+# ``#``, whitespace included, as ``urlsplit`` delimits a netloc after
+# ``//``. So a title spelling a URL that ``_safe_ris_url`` would strip a
+# credential from is flagged too, as is ``\\ user:pass@host``, which a
+# browser reads with the space encoded.
+#
+# Unless a base URL of the same scheme makes the value relative, a browser
+# also reads whitespace after a special scheme's colon, or after the colon
+# and one ``/`` or ``\``, as part of the authority: to it,
+# ``https:/ user:pass@host`` and ``https: user:pass@host`` carry
+# credentials, and so does the title ``HTTP: email me at [email protected]``,
+# whose userinfo it reads as `` email me at bob``. After the colon and one
+# ``/`` or ``\``, the authority runs past whitespace as above; no title
+# begins that way. So that prose stays a title, an authority opened by the
+# colon alone, with no ``/`` or ``\`` after it, may begin with whitespace
+# only in the shape ``_RIS_SPACED_AUTHORITY`` matches. Past that
+# whitespace must come a token of one or more characters other than
+# whitespace, ``/``, ``\``, ``?`` and ``#``; optional whitespace; an @;
+# and a host of characters other than those and @, followed by ``/``,
+# ``\``, ``?``, ``#`` or nothing but ``_RIS_TRIMMED`` characters, as the
+# value may begin with them. ``https: user:pass@host/x`` and
+# ``https: user:pass @host`` are flagged. Three titles are not:
+# ``HTTP: email me at [email protected]``, whose would-be userinfo holds
+# whitespace; ``HTTP: [email protected] explains``, whose would-be host
+# does; and ``HTTP: @mention``, which has no token before its @. Every
+# reading of a value is matched, so a control where this shape takes
+# whitespace is read as the space serialization writes for it
+# (``_ris_flatten``). The host is matched possessively: it shares controls
+# and U+FEFF with ``_RIS_TRIMMED``, and giving them back could only make
+# the end of the value be scanned again, in time quadratic in its length.
+_RIS_SPACED_AUTHORITY = "".join(
+ (
+ r"\s+[^\s/?#\\]+\s*@[^\s/?#@\\]++(?:[/?#\\]|",
+ _RIS_TRIMMED,
+ r"*\Z)",
+ )
+)
+# What opens the authority of a URL as a whole.
+_RIS_WHOLE_URL_START = (
+ rf"(?:(?:{_RIS_SPECIAL_SCHEME}:)?[/\\]{{2,}}|[a-z][a-z0-9+.\-]*://"
+ rf"|{_RIS_SPECIAL_SCHEME}:"
+ rf"(?:[/\\]|(?!\s)|(?={_RIS_SPACED_AUTHORITY})))"
+)
+_RIS_WHOLE_URL_AUTHORITY_RE = re.compile(
+ rf"{_RIS_TRIMMED}*{_RIS_WHOLE_URL_START}([^/?#]*)",
+ re.IGNORECASE,
+)
+
+# Where a URL as a whole can start in a value read from a line
+# (``_ris_line_carries_userinfo``): at the value's start, and after each
+# comma, ``and`` or ampersand, where the author pattern splits a list into
+# authors. As at the start of a value, the URL may follow whitespace,
+# controls and U+FEFF, matched possessively, since none of them can open
+# an authority. ``_ris_value_starts_userinfo`` finds where each such
+# authority ends.
+_RIS_VALUE_URL_START_RE = re.compile(
+ rf"(?:\A|,|(?<=\s)(?-i:and)(?=\s)|&){_RIS_TRIMMED}*+"
+ rf"{_RIS_WHOLE_URL_START}",
+ re.IGNORECASE,
+)
+_RIS_AUTHORITY_END_RE = re.compile(r"[/?#]")
+
+# Bounds on the readings of one value, which keep the work of checking it
+# small; a value past any of them counts as flagged. Each layer of encoding
+# a value carries adds a reading, so an ordinary value has a few, and 64
+# leaves room for deliberate nesting. A reading can be longer than the
+# value, since NFKC turns U+FDFA alone into 18 characters, so every
+# reading is measured, not only the value. RFC 9110 recommends supporting
+# URIs of 8,000 octets; no title, author, DOI or link needs eight times
+# that. The total leaves room for the value plus three more readings of
+# that length.
+_RIS_MAX_READINGS = 64
+_RIS_MAX_DECODED_LENGTH = 1 << 16
+_RIS_MAX_DECODED_TOTAL_LENGTH = 4 * _RIS_MAX_DECODED_LENGTH
+
+# The WHATWG URL parser deletes every tab, LF and CR from its input before
+# it parses it: a browser reads ``ht<TAB>tps:/<LF>/user:pass@host`` as
+# ``https://user:pass@host``. So the readings of a value include it with
+# them deleted, or one that an entity or percent-escape spells would hide
+# a credential URL once decoded: ``&Tab;`` splits the scheme of
+# ``ht&Tab;tps:user:pass@host``, and ``%0A`` the slashes of
+# ``https:/%0A/user:pass@host``.
+_RIS_URL_DELETED_CHARACTERS = str.maketrans("", "", "\t\n\r")
+
+
+def _ris_flatten(value: str) -> str:
+ """``value`` as RIS serialization writes it: every character
+ ``is_line_breaking_char`` matches is turned into a space, so only the
+ exporter's own line breaks separate RIS fields."""
+ return "".join(
+ " " if is_line_breaking_char(char) else char for char in value
+ )
+
+
+def _ris_readings(value: str) -> set[str] | None:
+ """``value`` and every form decoding turns it into; None past a bound.
+
+ The decodings are HTML character references, NFKC normalization,
+ percent-decoding, the deletion of every tab, LF and CR, as the WHATWG
+ URL parser deletes them, and ``_ris_flatten``, which turns every
+ control character, U+2028 and U+2029 into a space, as serialization
+ writes a field. Each is applied to every reading until none yields a
+ new one, so they compose in any order and repeat. An ASCII value
+ without ``&``, ``%`` or a control character, which none of them can
+ change, is its only reading. Any other value is measured, and so is
+ each new reading as a decoding yields it: the result is None once one
+ of them is longer than ``_RIS_MAX_DECODED_LENGTH``, once they are
+ longer together than ``_RIS_MAX_DECODED_TOTAL_LENGTH``, or once there
+ would be more than ``_RIS_MAX_READINGS`` of them.
+ """
+ if value.isascii() and not any(
+ char in "&%" or is_line_breaking_char(char) for char in value
+ ):
+ return {value} # None of the decodings can change it.
+ if len(value) > _RIS_MAX_DECODED_LENGTH:
+ return None
+ readings = {value}
+ total_length = len(value)
+ pending = [value]
+ while pending:
+ form = pending.pop()
+ for decoded in (
+ html.unescape(form),
+ unicodedata.normalize("NFKC", form),
+ urllib.parse.unquote(form),
+ form.translate(_RIS_URL_DELETED_CHARACTERS),
+ _ris_flatten(form),
+ ):
+ if decoded in readings:
+ continue
+ total_length += len(decoded)
+ if (
+ len(readings) == _RIS_MAX_READINGS
+ or len(decoded) > _RIS_MAX_DECODED_LENGTH
+ or total_length > _RIS_MAX_DECODED_TOTAL_LENGTH
+ ):
+ return None
+ readings.add(decoded)
+ pending.append(decoded)
+ return readings
+
+
+def _ris_spells_userinfo(value: str) -> bool:
+ """Whether some reading of ``value`` puts a credential in a URL authority.
+
+ A credential is userinfo holding anything other than ``@`` signs; an @
+ outside every authority, as in ``medium.com/@author``, is not one. The
+ readings are ``value`` and its decoded forms (``_ris_readings``): a
+ consumer that renders ``&commat;``, decodes ``%40`` or folds a
+ full-width at sign (U+FF20) reads the @ they spell, and a browser
+ deletes the tab, LF or CR that ``&Tab;``, ``&#10;`` or ``%0D`` spells,
+ as one reading does. Another is the value as serialization writes it
+ (``_ris_flatten``), so whatever the check passes is written in a form
+ it passes too. The check is lexical, so no parse error can let a value
+ through, and a value past the reading bounds counts as flagged.
+ """
+ readings = _ris_readings(value)
+ if readings is None:
+ return True
+ for form in readings:
+ whole = _RIS_WHOLE_URL_AUTHORITY_RE.match(form)
+ if whole and "@" in whole.group(1).lstrip("@"):
+ return True
+ if any(
+ "@" in match.group(1).lstrip("@")
+ for match in _RIS_EMBEDDED_AUTHORITY_RE.finditer(form)
+ ):
+ return True
+ return False
+
+
+def _safe_ris_url(raw_url: str | None) -> str:
+ """Return a source URL ready for a RIS field without its userinfo, or "".
+
+ The literal userinfo of a parseable authority is removed and the rest
+ of the URL kept. The URL is omitted instead when it contains a
+ line-breaking character; when ``urlsplit`` cannot parse it, or finds no
+ host or an empty or invalid port in its authority; when its authority
+ holds a backslash; when it has a scheme but no authority, and the scheme
+ is http(s) or the URL holds an @; and when ``_ris_spells_userinfo``
+ flags what would be exported: an entity-, percent- or NFKC-spelled
+ delimiter left in the authority; a credential URL with no authority
+ ``urlsplit`` could strip, inside a wrapper such as ``<...>`` or
+ ``[text](...)``, inside prose, after a scheme-less run of slashes and
+ backslashes other than ``//``, after U+FEFF, or with its scheme or
+ slashes split by an encoded tab, LF or CR; or a value past the bounds
+ on its readings.
+ """
+ if not isinstance(raw_url, str):
+ return ""
+ candidate = raw_url.strip()
+ if not candidate or any(is_line_breaking_char(char) for char in candidate):
+ # urlsplit silently discards some controls while parsing, but RIS
+ # consumers may read the original bytes as additional fields.
+ return ""
+ try:
+ parsed = urlsplit(candidate)
+ if parsed.netloc:
+ if "\\" in parsed.netloc:
+ # WHATWG parsers end a special authority at a backslash and
+ # urlsplit does not, so the two disagree about the host.
+ return ""
+ authority = parsed.netloc.rpartition("@")[2]
+ clean = parsed._replace(netloc=authority)
+ if not clean.hostname or authority.endswith(":"):
+ return ""
+ _ = clean.port # ValueError for malformed or out-of-range ports.
+ candidate = urlunsplit(clean)
+ elif parsed.scheme and (
+ parsed.scheme.lower() in _LINKABLE_URL_SCHEMES or "@" in candidate
+ ):
+ # A scheme followed by user:password@host without // is not a
+ # parseable authority; it must not survive as a raw RIS field.
+ return ""
+ except ValueError:
+ return ""
+ if _ris_spells_userinfo(candidate):
+ return ""
+ return candidate
+
+
+def _ris_field_text(value: str) -> str:
+ """``value`` for a free-text RIS field, or ``_safe_ris_url(value)`` when
+ the credential check flags it ("" when no URL is kept)."""
+ if _ris_spells_userinfo(value):
+ return _safe_ris_url(value)
+ return value
+
+
+def _ris_value_starts_userinfo(value: str) -> bool:
+ """Whether, in some reading of ``value``, a URL as a whole that starts
+ at its start, or after a comma, ``and`` or ampersand in it
+ (``_RIS_VALUE_URL_START_RE``), holds a credential: an @ in its
+ authority after any leading @ signs, as ``_ris_spells_userinfo`` reads
+ the authority of a value that is a URL as a whole. The authority runs
+ to the first ``/``, ``?`` or ``#``. An authority that starts inside
+ another ends where that one ends, so each is scanned once, and the work
+ is linear in the length of the readings. A value past the reading
+ bounds counts as holding one.
+ """
+ readings = _ris_readings(value)
+ if readings is None:
+ return True
+ for form in readings:
+ authority_end = last_at = -1
+ for start in _RIS_VALUE_URL_START_RE.finditer(form):
+ position = start.end()
+ if position >= authority_end:
+ end = _RIS_AUTHORITY_END_RE.search(form, position)
+ authority_end = end.start() if end else len(form)
+ last_at = form.rfind("@", position, authority_end)
+ while position < authority_end and form[position] == "@":
+ position += 1
+ if last_at >= position:
+ return True
+ return False
+
+
+# The labels of the lines whose value ``export_to_ris`` reads whole: the
+# source URL from a ``URL:`` line and the DOI from a ``DOI:`` line, each
+# matched as ``export_to_ris`` matches it.
+_RIS_VALUE_LABEL_RE = re.compile(r"URL:|DOI:")
+
+
+_RIS_SOURCE_VALUE_START = r"(?:\A|URL:|DOI:|by\s++|,|(?<=\s)and(?=\s)|&)"
+_RIS_SOURCE_VALUE_AUTHORITY_RE = re.compile(
+ rf"{_RIS_SOURCE_VALUE_START}"
+ rf"{_RIS_TRIMMED}*+{_RIS_WHOLE_URL_START}([^/?#]*)",
+ re.IGNORECASE,
+)
+# A source has metadata after its URL, so a host need not end the record.
+# This relaxed tail applies only when userinfo crosses the checked physical
+# line. A one-line prose title such as "HTTP: [email protected] explains"
+# therefore keeps the field guard's documented prose exception.
+_RIS_SOURCE_SPACED_AUTHORITY_RE = re.compile(
+ rf"{_RIS_SOURCE_VALUE_START}{_RIS_TRIMMED}*+{_RIS_SPECIAL_SCHEME}:"
+ r"\s++([^\s/?#\\]+\s*@[^\s/?#@\\]++)(?=[/?#\\]|\s|\Z)",
+ re.IGNORECASE,
+)
+
+
+def _ris_multiline_line_carries_userinfo(text: str, position: int) -> bool:
+ """Check a source line in its unsplit record, before cutting fields.
+
+ A raw LF can cut the @ off a credential URL before a title, URL, DOI,
+ author or year is checked. Track the line's span through the same
+ decodings as the field guard, so unrelated lines keep their metadata.
+ Prefix/suffix comparisons conservatively include a decoded token that
+ crosses a span boundary (for example, ``%4<LF>0`` becoming ``@``).
+ No raw source text is rewritten or used as a replacement field value.
+ """
+ if "\n" not in text:
+ return False
+ if text.isascii() and not any(char in text for char in "@%&"):
+ return False
+ if len(text) > _RIS_MAX_DECODED_LENGTH:
+ return True
+
+ start = text.rfind("\n", 0, position) + 1
+ end = text.find("\n", position)
+ if end == -1:
+ end = len(text)
+ pending = [(text, start, end)]
+ seen = set(pending)
+ total_length = len(text)
+ decoders = (
+ html.unescape,
+ lambda value: unicodedata.normalize("NFKC", value),
+ urllib.parse.unquote,
+ lambda value: value.translate(_RIS_URL_DELETED_CHARACTERS),
+ _ris_flatten,
+ )
+
+ def common_edge(left: str, right: str, *, reverse: bool = False) -> int:
+ if reverse:
+ left, right = left[::-1], right[::-1]
+ if left.startswith(right):
+ return len(right)
+ return next(
+ (i for i, (a, b) in enumerate(zip(left, right)) if a != b),
+ min(len(left), len(right)),
+ )
+
+ while pending:
+ form, start, end = pending.pop()
+ for pattern in (
+ _RIS_EMBEDDED_AUTHORITY_RE,
+ _RIS_SOURCE_VALUE_AUTHORITY_RE,
+ _RIS_SOURCE_SPACED_AUTHORITY_RE,
+ ):
+ for match in pattern.finditer(form):
+ authority = match.group(1)
+ if "@" not in authority.lstrip("@"):
+ continue
+ credential_end = match.start(1) + authority.rfind("@")
+ crosses_line = (
+ match.start(1) < start or credential_end + 1 >= end
+ )
+ if (
+ start < credential_end
+ and end > match.start()
+ and (
+ pattern is not _RIS_SOURCE_SPACED_AUTHORITY_RE
+ or crosses_line
+ )
+ ):
+ return True
+
+ for decode in decoders:
+ decoded = decode(form)
+ if decoded == form:
+ continue
+ if len(decoded) > _RIS_MAX_DECODED_LENGTH:
+ return True
+ before = common_edge(decoded, decode(form[:start]))
+ after = len(decoded) - common_edge(
+ decoded, decode(form[end:]), reverse=True
+ )
+ state = (decoded, min(before, after), max(before, after))
+ if state in seen:
+ continue
+ total_length += len(decoded)
+ if (
+ len(seen) == _RIS_MAX_READINGS
+ or len(decoded) > _RIS_MAX_DECODED_LENGTH
+ or total_length > _RIS_MAX_DECODED_TOTAL_LENGTH
+ ):
+ return True
+ seen.add(state)
+ pending.append(state)
+ return False
+
+
+def _ris_line_carries_userinfo(
+ text: str, position: int, value_starts: tuple[int, ...] = ()
+) -> bool:
+ """Whether the line of ``text`` that holds ``position`` carries a
+ credential, so that no field may be cut out of it.
+
+ The credential check reads the line itself, and each value read from
+ the line from where it starts to the end of the line
+ (``_ris_value_starts_userinfo``): the value after a ``URL:`` or ``DOI:``
+ label that begins the line, which ``export_to_ris`` reads as the link
+ or the DOI, and the values that start in the line at ``value_starts``,
+ where the author and DOI patterns begin to read. In the line, a URL
+ after a label or a word ends at whitespace, as in other text, and the
+ author and DOI patterns end a value at a full stop, a comma, ``and``,
+ an ampersand or whitespace, past any of which a URL as a whole can go
+ on. So ``URL: https://user:by Smith@host/`` is not flagged as a line,
+ but its value is, with the password ``by Smith``, and the author
+ pattern would cut ``Smith@host/`` out of the line.
+ """
+ start = text.rfind("\n", 0, position) + 1
+ end = text.find("\n", position)
+ if end == -1:
+ end = len(text)
+ if _ris_spells_userinfo(text[start:end]):
+ return True
+ starts = [value for value in value_starts if start <= value <= end]
+ stripped = text[start:end].lstrip()
+ label = _RIS_VALUE_LABEL_RE.match(stripped)
+ if label:
+ starts.append(end - len(stripped) + label.end())
+ return any(
+ _ris_value_starts_userinfo(text[value:end]) for value in starts
+ ) or _ris_multiline_line_carries_userinfo(text, position)
+
+
# Metadata clauses cut from a RIS title, applied in this order. Each one
# starts at its keyword: a leading ``\s*`` would retry from every position
# of a whitespace run before failing, which is quadratic in the run length.
@@ -1982,12 +2452,18 @@ class RISExporter:
# Extract URL, DOI, and other metadata from subsequent lines
url = ""
metadata = {}
+ value_positions = {}
+ line_start = len(lines[0]) + 1
for line in lines[1:]:
+ position = line_start
+ line_start += len(line) + 1
line = line.strip()
if line.startswith("URL:"):
url = line[4:].strip()
+ value_positions["url"] = position
elif line.startswith("DOI:"):
metadata["doi"] = line[4:].strip()
+ value_positions["doi"] = position
elif line.startswith("Published in"):
metadata["journal"] = line[12:].strip()
# Add more metadata parsing as needed
@@ -1999,6 +2475,26 @@ class RISExporter:
if isinstance(additional, list):
additional.append(line)
+ # A line split may already have removed the credential's @.
+ # Check the selected values against the unsplit source before
+ # the ordinary whole-field sanitizer handles parseable URLs.
+ if (
+ url
+ and not _ris_spells_userinfo(url)
+ and _ris_multiline_line_carries_userinfo(
+ entry_text, value_positions["url"]
+ )
+ ):
+ url = ""
+ if (
+ metadata.get("doi")
+ and not _ris_spells_userinfo(metadata["doi"])
+ and _ris_multiline_line_carries_userinfo(
+ entry_text, value_positions["doi"]
+ )
+ ):
+ metadata["doi"] = ""
+
# Combine title with additional metadata lines for full context
full_text = entry_text
@@ -2040,42 +2536,114 @@ class RISExporter:
text_lines = full_text.split("\n")
title = text_lines[0].strip()
- # Extract year from full text (looks for 4-digit year)
+ # The year, the author list, and the DOI when no ``DOI:`` line gave
+ # one, are cut out of whatever line their match captures from, and
+ # the title is cleaned. A cut can drop a credential URL's @ and keep
+ # its userinfo (the title ``https://DOI:pass@host`` yields the DOI
+ # ``pass@host``), and a password can hold a year or a name
+ # (``https://admin:2024@host`` yields the year 2024), so none of the
+ # three is cut from a line that carries a credential
+ # (``_ris_line_carries_userinfo``), and a flagged title is sanitized
+ # whole, not cleaned. A title that its cleaning joins into a
+ # credential URL gives no year or DOI either (``flagged_title``).
year_match = re.search(r"\b(19\d{2}|20\d{2})\b", full_text)
- year = year_match.group(1) if year_match else None
-
- # Extract authors if present (looks for "by Author1, Author2")
authors_match = re.search(
r"\bby\s+([^.\n]+?)(?:\.|\n|$)", full_text, re.IGNORECASE
)
+ doi = metadata.get("doi")
+ doi_match = (
+ None
+ if doi
+ else re.search(r"DOI:\s*([^\s\n]+)", full_text, re.IGNORECASE)
+ )
+ # Where the author and DOI patterns begin to read a value; a line
+ # is checked from each of them too. Each line is checked once.
+ value_starts = tuple(
+ match.start(1)
+ for match in (authors_match, doi_match)
+ if match is not None
+ )
+ line_carries_userinfo: dict[int, bool] = {}
+
+ def cut_is_safe(match: re.Match) -> bool:
+ line_start = full_text.rfind("\n", 0, match.start(1)) + 1
+ if line_start not in line_carries_userinfo:
+ line_carries_userinfo[line_start] = _ris_line_carries_userinfo(
+ full_text, match.start(1), value_starts
+ )
+ return not line_carries_userinfo[line_start]
+
+ # Extract year from full text (looks for 4-digit year)
+ year = (
+ year_match.group(1)
+ if year_match and cut_is_safe(year_match)
+ else None
+ )
+
+ # Extract authors if present (looks for "by Author1, Author2")
authors = []
- if authors_match:
- authors_text = authors_match.group(1)
+ if authors_match and cut_is_safe(authors_match):
+ # The captured text and each author are checked as written too.
+ authors_text = _ris_field_text(authors_match.group(1))
# Split at delimiters only, then trim each author. Consuming
# optional whitespace before a delimiter makes a long run with
# no delimiter quadratic; lookarounds keep "and" word-scoped.
author_parts = re.split(r",|(?<=\s)and(?=\s)|&", authors_text)
- authors = [a.strip() for a in author_parts if a.strip()]
+ authors = [
+ author
+ for author in (_ris_field_text(a.strip()) for a in author_parts)
+ if author
+ ]
# Extract DOI from metadata or text
- doi = metadata.get("doi")
- if not doi:
- doi_match = re.search(
- r"DOI:\s*([^\s\n]+)", full_text, re.IGNORECASE
- )
- doi = doi_match.group(1) if doi_match else None
+ if doi_match is not None:
+ doi = doi_match.group(1) if cut_is_safe(doi_match) else None
- # Clean title - remove author and metadata info for cleaner title
+ # Clean title - remove author and metadata info for cleaner title.
+ # Every form a cut makes is kept for the credential check below.
clean_title = title
+ title_forms = [title]
if authors_match and authors_match.start() < len(title):
clean_title = (
title[: authors_match.start()] + title[authors_match.end() :]
if authors_match.end() < len(title)
else title[: authors_match.start()]
)
+ title_forms.append(clean_title)
for clause in _RIS_TITLE_CLAUSES:
clean_title = _cut_title_clause(clause, clean_title)
+ title_forms.append(clean_title)
clean_title = clean_title.strip()
+ title_forms.append(clean_title)
+
+ # A title carries a credential when the check flags it as written or
+ # in any form a cut makes of it: what a cut leaves can join into a
+ # credential URL, and a later cut can cut that URL's @ away again,
+ # leaving its userinfo. The first form the check flags is sanitized
+ # whole, and the title gives no year and no DOI.
+ flagged_title = next(
+ (
+ form
+ for form in dict.fromkeys(title_forms)
+ if _ris_spells_userinfo(form)
+ ),
+ None,
+ )
+ # Cleaning can remove a complete credential clause while leaving a
+ # different fragment, or join a fragment to the continuation lines.
+ # Check the title we would write together with the uncut remainder.
+ fragmented_title = (
+ flagged_title is None
+ and _ris_multiline_line_carries_userinfo(
+ (clean_title or title) + full_text[len(text_lines[0]) :], 0
+ )
+ )
+ if flagged_title is not None or fragmented_title:
+ title_end = len(text_lines[0])
+ if year_match and year_match.start(1) < title_end:
+ year = None
+ if doi_match and doi_match.start(1) < title_end:
+ doi = None
# TY - Type of reference (ELEC for electronic source/website)
lines.append("TY - ELEC")
@@ -2083,30 +2651,50 @@ class RISExporter:
# ID - Reference ID
lines.append(f"ID - ref{ref_id}")
- # TI - Title
- lines.append(f"TI - {clean_title if clean_title else title}")
+ # TI - Title. A source's entire title can itself be a URL, or hold
+ # one: a title the shared credential check flags is written as its
+ # userinfo-free URL, or as a placeholder when ``_safe_ris_url``
+ # keeps no URL for it. The title as written is checked first, since
+ # cleaning can cut the @ out of a credential URL; each form a cut
+ # makes is checked too, since joining what a cut leaves can spell
+ # one (``flagged_title`` above).
+ if fragmented_title:
+ display_title = "[source URL omitted]"
+ elif flagged_title is None:
+ display_title = clean_title if clean_title else title
+ else:
+ display_title = (
+ _safe_ris_url(flagged_title) or "[source URL omitted]"
+ )
+ lines.append(f"TI - {display_title}")
# AU - Authors
for author in authors:
lines.append(f"AU - {author}")
- # DO - DOI
+ # DO - DOI. Often written as a doi.org URL, so it gets the same
+ # credential check as the title; one it flags without a URL that
+ # ``_safe_ris_url`` keeps is left out.
+ if doi:
+ doi = _ris_field_text(str(doi))
if doi:
lines.append(f"DO - {doi}")
- # PY - Publication year (if found in title)
+ # PY - Publication year (if one was found above)
if year:
lines.append(f"PY - {year}")
- # UR - URL
- if url:
- lines.append(f"UR - {url}")
+ # UR - URL. Legacy reports and LLM-authored Sources may still carry
+ # user:password@host even though new reports canonicalize their URLs.
+ safe_url = _safe_ris_url(url)
+ if safe_url:
+ lines.append(f"UR - {safe_url}")
# Try to extract domain as publisher
try:
from urllib.parse import urlparse
- parsed = urlparse(url)
+ parsed = urlparse(safe_url)
# The host alone. The netloc also carries userinfo and port,
# so https://user:[email protected]/ wrote the token into the
# .ris file as the publisher (#6837). The root dot of a fully
@@ -2156,12 +2744,8 @@ class RISExporter:
# A field can contain bare CR or Unicode line separators even when
# extracted from one LF-delimited source line. Flatten them at the
# serialization boundary so only our separators create RIS fields.
- return "\n".join(
- "".join(
- " " if is_line_breaking_char(char) else char for char in line
- )
- for line in lines
- )
+ # The credential check reads every value flattened the same way.
+ return "\n".join(_ris_flatten(line) for line in lines)
# Body prose is markdown, where "\$" is the escape for a literal
diff --git a/tests/exporters/test_ris_injection_contracts.py b/tests/exporters/test_ris_injection_contracts.py
index 0dd3df82a7..adfa0e9064 100644
--- a/tests/exporters/test_ris_injection_contracts.py
+++ b/tests/exporters/test_ris_injection_contracts.py
@@ -148,6 +148,11 @@ def test_embedded_line_separators_cannot_inject_records(separator, field):
]
assert lines.count("ER - ") == 1
assert "TI - Injected" not in lines
+ if field == "url":
+ # A URL containing a line separator is discarded before RIS output;
+ # urlsplit can otherwise silently strip some of these characters.
+ assert not any(line.startswith("UR - ") for line in lines)
+ return
expected_tag = {"title": "TI", "url": "UR", "doi": "DO", "author": "AU"}[
field
]
diff --git a/tests/text_optimization/test_ris_entry_edge_cases.py b/tests/text_optimization/test_ris_entry_edge_cases.py
index f08d55e8b2..bba5a6fcdb 100644
--- a/tests/text_optimization/test_ris_entry_edge_cases.py
+++ b/tests/text_optimization/test_ris_entry_edge_cases.py
@@ -15,12 +15,200 @@ This file adds coverage for the many remaining branches:
"""
import re
+import time
import pytest
from freezegun import freeze_time
+from local_deep_research.text_optimization import citation_formatter
from local_deep_research.text_optimization.citation_formatter import RISExporter
+# Passwords for the credential fixtures, each with the parts of it that no
+# field may hold. The fixtures are written with ``secret`` in a URL's
+# userinfo. It matches no pattern that cuts a field out of text, so with it
+# alone such a cut could not show. The others match those patterns: a year
+# (PY), a ``DOI:`` label (DO), and the commas, ampersands and full stops at
+# which the author list is split and ended (AU).
+_PASSWORDS = [
+ pytest.param("secret", ("secret",), id="word"),
+ pytest.param("1987", ("1987",), id="year"),
+ pytest.param("tok-1999-x", ("1999",), id="year-in-a-token"),
+ pytest.param("DOI:10.5555", ("5555",), id="doi"),
+ pytest.param(
+ "Xq7,Zr9&Wk3.Vm5", ("Xq7", "Zr9", "Wk3", "Vm5"), id="punctuation"
+ ),
+]
+
+
+def _with_password(value, password):
+ """``value`` with ``password`` in place of ``secret``."""
+ assert "secret" in value, value
+ return value.replace("secret", password)
+
+
+def _assert_holds_no_part(result, parts):
+ for part in parts:
+ assert part not in result, (part, result)
+
+
+class TestRISMultilineCredentialSources:
+ """Check the unsplit source before line-based metadata can hide its @."""
+
+ @pytest.mark.parametrize("separator", ["\n", "\r\n"])
+ @pytest.mark.parametrize(
+ "title",
+ [
+ "A harmless title by https: user:[email protected]/x. "
+ "https://user:CutPassword957",
+ "https:/by Smith./user:CutPassword957",
+ "https:/by Smith./user:CutPassword957 Volume",
+ ],
+ )
+ def test_title_cleanup_cannot_hide_a_remaining_fragment(
+ self, title, separator
+ ):
+ exported = RISExporter().export_to_ris(
+ f"## Sources\n[1] {title}{separator}@example.net/paper\n"
+ )
+
+ assert "CutPassword957" not in exported
+ assert "TI - [source URL omitted]" in exported
+
+ @pytest.mark.parametrize("prefix", ["by", "URL:", "DOI:"])
+ def test_source_guard_does_not_backtrack_over_whitespace(self, prefix):
+ source = prefix + " " * 16_000 + "[email protected]\nPlain text"
+ started = time.thread_time()
+
+ assert not citation_formatter._ris_multiline_line_carries_userinfo(
+ source, 0
+ )
+
+ # CPU time excludes competing test processes and VM suspension.
+ # Ambiguous adjacent whitespace repeats take seconds here.
+ assert time.thread_time() - started < 1.0
+
+ @pytest.mark.parametrize("password,parts", _PASSWORDS)
+ @pytest.mark.parametrize("separator", ["\n", "\r\n"])
+ @pytest.mark.parametrize("field", ["title", "URL", "DOI", "author"])
+ def test_line_split_cannot_export_a_password_fragment(
+ self, password, parts, separator, field
+ ):
+ url = f"https://user:{password}{separator}@example.org/paper"
+ source = {
+ "title": url,
+ "URL": f"A harmless title\nURL: {url}",
+ "DOI": f"A harmless title\nDOI: {url}",
+ "author": f"A harmless title\nby {url}",
+ }[field]
+
+ exported = RISExporter().export_to_ris(f"## Sources\n[1] {source}\n")
+
+ assert exported.startswith("TY - ELEC")
+ _assert_holds_no_part(exported, parts)
+ if field == "title":
+ assert "TI - [source URL omitted]" in exported
+ else:
+ assert "TI - A harmless title" in exported
+
+ @pytest.mark.parametrize(
+ "url",
+ [
+ "https://user:secret%4\n0example.org/paper",
+ "https://user:secret&com\nmat;example.org/paper",
+ "ht\ntps://user:secret\[email protected]/paper",
+ "https://user:\nsecret\[email protected]/paper",
+ "https://user:secret&#6\n4;example.org/paper",
+ "https://user:secret@\nexample.org/paper",
+ "https: user:secret\[email protected]\nPublished in 2023",
+ "https: user:sec\[email protected]\nPublished in 2023",
+ ],
+ )
+ def test_decoding_and_line_splitting_compose(self, url):
+ exported = RISExporter().export_to_ris(
+ f"## Sources\n[1] {url}\nDOI: {url}\nURL: {url}\n"
+ )
+
+ assert exported.startswith("TY - ELEC")
+ assert "secret" not in exported
+ assert "TI - [source URL omitted]" in exported
+
+ @pytest.mark.parametrize(
+ "title",
+ [
+ "HTTP: [email protected] explains",
+ "HTTP: email me at [email protected]",
+ "HTTP: @mention",
+ ],
+ )
+ def test_multiline_record_keeps_prose_title_exception(self, title):
+ exported = RISExporter().export_to_ris(
+ f"## Sources\n[1] {title}\nPublished in 2024\n"
+ "URL: https://example.org/paper\n"
+ )
+
+ assert f"TI - {title}" in exported
+ assert "PY - 2024" in exported
+ assert "UR - https://example.org/paper" in exported
+
+ @pytest.mark.parametrize("line_ending", ["\n", "\r\n"])
+ def test_safe_fields_outside_a_split_credential_keep_their_values(
+ self, line_ending
+ ):
+ source = line_ending.join(
+ [
+ "## Sources",
+ "[1] A harmless title",
+ "by Alice and Bob.",
+ "Published in 2024",
+ "URL: https://user:secret",
+ "@example.org/paper",
+ "DOI: 10.1234/safe",
+ ]
+ )
+
+ exported = RISExporter().export_to_ris(source)
+
+ assert "secret" not in exported
+ for field in (
+ "TI - A harmless title",
+ "AU - Alice",
+ "AU - Bob",
+ "PY - 2024",
+ "DO - 10.1234/safe",
+ ):
+ assert field in exported
+
+ def test_source_lines_without_credentials_keep_their_metadata(self):
+ source = (
+ "## Sources\n[1] A harmless title\nby Alice and Bob.\n"
+ "Published in 2024\nDOI: 10.1234/safe\n"
+ "URL: https://example.org/@author\n"
+ )
+
+ exported = RISExporter().export_to_ris(source)
+
+ for field in (
+ "TI - A harmless title",
+ "AU - Alice",
+ "AU - Bob",
+ "PY - 2024",
+ "DO - 10.1234/safe",
+ "UR - https://example.org/@author",
+ ):
+ assert field in exported
+
+ def test_a_complete_inline_credential_does_not_discard_the_cleaned_title(
+ self,
+ ):
+ exported = RISExporter().export_to_ris(
+ "## Sources\n[1] A harmless title by https: user:secret@host\n"
+ "URL: https://example.org/paper\n"
+ )
+
+ assert "secret" not in exported
+ assert "TI - A harmless title" in exported
+ assert "UR - https://example.org/paper" in exported
+
class TestRISEntryAuthorParsing:
"""Tests for author extraction from 'by Author1, Author2' patterns."""
@@ -460,6 +648,1339 @@ class TestRISEntryURLHandling:
result = exporter._create_ris_entry("1", "Title", url=url)
assert f"UR - {url}" in result
+ def test_export_strips_userinfo_without_losing_path_or_query(self):
+ document = (
+ "Report\n\n## Sources\n\n"
+ "[1] Credential leak probe\n"
+ "URL: https://user:[email protected]/abs/2401.00001?q=a&b=2"
+ )
+
+ result = self._make().export_to_ris(document)
+
+ assert "UR - https://arxiv.org/abs/2401.00001?q=a&b=2" in result
+ assert "PB - arXiv" in result
+ assert "secret-token" not in result
+
+ @pytest.mark.parametrize("at_entity", ["&commat;", "&#64;", "&#x40;"])
+ def test_export_refuses_userinfo_remaining_after_stripping(self, at_entity):
+ url = f"https://first:password@secret{at_entity}example.org/doc"
+
+ result = self._make()._create_ris_entry("1", url, url=url)
+
+ assert "TI - [source URL omitted]" in result
+ assert "UR -" not in result
+ assert "PB -" not in result
+ assert "secret" not in result
+ assert "password" not in result
+
+ def test_url_title_does_not_export_credentials(self):
+ result = self._make()._create_ris_entry(
+ "1",
+ "https://user:[email protected]/article",
+ url="https://user:[email protected]/article",
+ )
+
+ assert "TI - https://example.org/article" in result
+ assert "UR - https://example.org/article" in result
+ assert "secret-token" not in result
+
+ def test_ris_url_with_control_is_omitted(self):
+ result = self._make()._create_ris_entry(
+ "1", "Title", url="https://example.org/a\rER - injected"
+ )
+
+ assert "UR -" not in result
+ assert result.count("ER -") == 1
+
+ def test_malformed_credential_url_is_omitted(self):
+ result = self._make()._create_ris_entry(
+ "1", "Title", url="https:user:[email protected]/article"
+ )
+
+ assert "UR -" not in result
+ assert "secret-token" not in result
+
+ def test_ris_url_with_at_in_path_is_preserved(self):
+ result = self._make()._create_ris_entry(
+ "1", "Title", url="https://medium.com/@author/article"
+ )
+
+ assert "UR - https://medium.com/@author/article" in result
+
+ @pytest.mark.parametrize(
+ "url",
+ [
+ "https&#58;//user:[email protected]/a",
+ "https&colon;//user:[email protected]/a",
+ "https&#x3a;//user:[email protected]/a",
+ ],
+ )
+ def test_ris_url_with_entity_encoded_scheme_is_omitted(self, url):
+ """The entity breaks the scheme token for ``urlsplit``, but a
+ consumer that decodes HTML entities follows a live credential
+ URL, so the field must not survive verbatim."""
+ result = self._make()._create_ris_entry("1", "Title", url=url)
+
+ assert "UR -" not in result
+ assert "user:pass" not in result
+
+ @pytest.mark.parametrize(
+ "title",
+ [
+ "https:user:[email protected]/article",
+ "https:/user:[email protected]/x",
+ "https://user:[email protected]/x",
+ "ftp://user:[email protected]/x",
+ "https&#58;//user:[email protected]/a",
+ "https&colon;//user:[email protected]/a",
+ ],
+ )
+ def test_url_title_gate_covers_non_prefix_credential_spellings(self, title):
+ """The title must not keep a credential spelling the shared check
+ flags, not only one after a literal ``http(s)://`` prefix."""
+ result = self._make()._create_ris_entry("1", title, url="")
+
+ ti_line = [
+ line for line in result.split("\n") if line.startswith("TI -")
+ ][0]
+ assert "user:pass" not in ti_line
+ assert "user:secret" not in ti_line
+ assert "@example.org" not in ti_line
+
+ @pytest.mark.parametrize(
+ "title",
+ [
+ "https://example.org/article",
+ "https://medium.com/@author/article",
+ "Notes from [email protected] on https://example.org/x",
+ ],
+ )
+ def test_url_title_gate_keeps_non_credential_titles(self, title):
+ """A plain URL title, an @ in the path, and an email in prose are
+ not credentials and must survive the gate verbatim."""
+ result = self._make()._create_ris_entry("1", title)
+
+ assert f"TI - {title}" in result
+
+
+class TestRISSharedCredentialCheck:
+ """One credential check gates UR, TI, AU and DO, and no author, year or
+ DOI is cut out of a line that carries a credential.
+
+ Every refused value below carries ``secret`` in a URL's userinfo, so
+ no spelling of it may reach any field of the RIS record. The tests that
+ take ``password`` also run each value with the other passwords of
+ ``_PASSWORDS`` in place of ``secret``, and no part of one may reach a
+ field either.
+ """
+
+ def _make(self):
+ return RISExporter()
+
+ @pytest.mark.parametrize("password,parts", _PASSWORDS)
+ @pytest.mark.parametrize(
+ "value",
+ [
+ # urlsplit raises ValueError on each of these. The UR sanitizer
+ # already failed closed; the title check read the error as "no
+ # credential" and exported the title verbatim.
+ pytest.param(
+ "https://user:[email protected]\uff1a8443/x",
+ id="nfkc-fullwidth-colon",
+ ),
+ pytest.param(
+ "https://[email protected]\uff0fowner/repo",
+ id="nfkc-fullwidth-solidus",
+ ),
+ pytest.param(
+ "https://user:secret@exa\u2100mple.org/x",
+ id="nfkc-account-of",
+ ),
+ pytest.param(
+ "https://user:[email protected]\ufe56/x",
+ id="nfkc-small-question-mark",
+ ),
+ pytest.param(
+ "https://user&colon;secret&commat;example.org&incare;x",
+ id="entities-then-care-of",
+ ),
+ pytest.param(
+ "https://a:b@user&colon;secret&commat;example.org&incare;x",
+ id="literal-then-entities-then-care-of",
+ ),
+ pytest.param(
+ "https://secret-token&commat;github.com&incare;x",
+ id="entity-username-then-care-of",
+ ),
+ pytest.param(
+ "https://user:[email protected]]/x",
+ id="unbalanced-close-bracket",
+ ),
+ pytest.param(
+ "https://user:secret@[broken/x",
+ id="unbalanced-open-bracket",
+ ),
+ pytest.param(
+ "https://[@usersecret]&#64;example.org/x",
+ id="bracketed-userinfo-then-entity-at",
+ ),
+ ],
+ )
+ def test_unparseable_credential_url_fails_closed(
+ self, value, password, parts
+ ):
+ value = _with_password(value, password)
+ result = self._make()._create_ris_entry("1", value, url=value)
+
+ assert "TI - [source URL omitted]\n" in result
+ assert "UR -" not in result
+ _assert_holds_no_part(result, parts)
+
+ @pytest.mark.parametrize("password,parts", _PASSWORDS)
+ @pytest.mark.parametrize(
+ "value,expected",
+ [
+ (
+ "git+https://user:[email protected]/o/r",
+ "git+https://github.com/o/r",
+ ),
+ ("sftp://user:[email protected]/x", "sftp://example.org/x"),
+ ("ssh://git:[email protected]/r", "ssh://gitlab.com/r"),
+ ("wss://user:[email protected]/x", "wss://example.org/x"),
+ ("//user:[email protected]/x", "//example.org/x"),
+ ],
+ )
+ def test_userinfo_is_stripped_under_any_scheme(
+ self, value, expected, password, parts
+ ):
+ """The title check once accepted only http, https and ftp, so these
+ titles kept the credentials the UR field stripped."""
+ value = _with_password(value, password)
+ result = self._make()._create_ris_entry("1", value, url=value)
+
+ assert f"TI - {expected}\n" in result
+ assert f"UR - {expected}\n" in result
+ _assert_holds_no_part(result, parts)
+
+ @pytest.mark.parametrize("password,parts", _PASSWORDS)
+ @pytest.mark.parametrize(
+ "value",
+ [
+ pytest.param(
+ "&#32;https:/user:[email protected]/x", id="decimal-entity"
+ ),
+ pytest.param(
+ "&#x20;https:/user:[email protected]/x", id="hex-entity"
+ ),
+ pytest.param(
+ "&#32https:/user:[email protected]/x", id="no-semicolon"
+ ),
+ pytest.param(
+ "&#0032;https:/user:[email protected]/x", id="zero-padded"
+ ),
+ pytest.param(
+ "&#32;&#32;https:/user:[email protected]/x", id="two-spaces"
+ ),
+ pytest.param(
+ "&#32;https:///user:[email protected]/x", id="triple-slash"
+ ),
+ pytest.param("&#9;https:/user:[email protected]/x", id="tab"),
+ ],
+ )
+ def test_entity_encoded_leading_space_does_not_hide_userinfo(
+ self, value, password, parts
+ ):
+ """urlsplit drops the decoded leading whitespace before it reads the
+ scheme, so an authority cut at ``len(scheme) + 1`` began one
+ character early, stopped at the first ``/`` and missed the @."""
+ value = _with_password(value, password)
+ result = self._make()._create_ris_entry("1", value, url=value)
+
+ assert "TI - [source URL omitted]\n" in result
+ assert "UR -" not in result
+ _assert_holds_no_part(result, parts)
+
+ @pytest.mark.parametrize("password,parts", _PASSWORDS)
+ @pytest.mark.parametrize(
+ "value",
+ [
+ pytest.param("wss://secret&#64;example.org/x", id="entity-at"),
+ pytest.param(
+ "https://user:secret%40example.org/x", id="percent-encoded-at"
+ ),
+ pytest.param(
+ "https://user:secret\uff20example.org/x", id="fullwidth-at"
+ ),
+ ],
+ )
+ def test_encoded_at_sign_is_refused(self, value, password, parts):
+ value = _with_password(value, password)
+ result = self._make()._create_ris_entry("1", value, url=value)
+
+ assert "TI - [source URL omitted]\n" in result
+ assert "UR -" not in result
+ assert "PB -" not in result
+ _assert_holds_no_part(result, parts)
+
+ @pytest.mark.parametrize("password,parts", _PASSWORDS)
+ @pytest.mark.parametrize(
+ "value",
+ [
+ pytest.param(
+ "<https://user:[email protected]/abs/1>", id="angle-brackets"
+ ),
+ pytest.param(
+ "[arXiv](https://user:[email protected]/abs/1)",
+ id="markdown-link",
+ ),
+ pytest.param(
+ '"https://user:[email protected]/abs/1"', id="double-quotes"
+ ),
+ pytest.param(
+ "'https://user:[email protected]/abs/1'", id="single-quotes"
+ ),
+ pytest.param(
+ "(https://user:[email protected]/abs/1)", id="parentheses"
+ ),
+ pytest.param(
+ "`https://user:[email protected]/abs/1`", id="backticks"
+ ),
+ pytest.param(
+ "*https://user:[email protected]/abs/1*", id="emphasis"
+ ),
+ pytest.param(
+ "**https://user:[email protected]/abs/1**", id="strong-emphasis"
+ ),
+ pytest.param(
+ "_https://user:[email protected]/abs/1_",
+ id="underscore-emphasis",
+ ),
+ pytest.param("see https://user:[email protected]/abs/1", id="prose"),
+ pytest.param(
+ "&lt;https://user:[email protected]/abs/1&gt;",
+ id="entity-angle-brackets",
+ ),
+ pytest.param(
+ "\u200bhttps://user:[email protected]/abs/1",
+ id="zero-width-space",
+ ),
+ pytest.param(
+ "https%3A//user:[email protected]/abs/1",
+ id="percent-encoded-colon",
+ ),
+ ],
+ )
+ def test_wrapped_credential_url_is_refused(self, value, password, parts):
+ """urlsplit finds no scheme and no authority in these, so they were
+ exported verbatim as the source link and as the title."""
+ value = _with_password(value, password)
+ result = self._make()._create_ris_entry("1", value, url=value)
+
+ assert "TI - [source URL omitted]\n" in result
+ assert "UR -" not in result
+ assert "PB -" not in result
+ _assert_holds_no_part(result, parts)
+
+ def test_export_refuses_wrapped_source_urls(self):
+ document = (
+ "Report\n\n## Sources\n\n"
+ "[1] Autolinked source\n"
+ "URL: <https://user:[email protected]/abs/1>\n\n"
+ "[2] Linked source\n"
+ "URL: [arXiv](https://user:[email protected]/abs/2)\n"
+ )
+
+ result = self._make().export_to_ris(document)
+
+ assert result.count("ER - ") == 2
+ assert "UR -" not in result
+ assert "secret" not in result
+
+ def test_doi_url_loses_its_userinfo(self):
+ document = (
+ "Report\n\n## Sources\n\n"
+ "[1] Paper title\n"
+ "URL: https://example.org/p\n"
+ "DOI: https://user:[email protected]/10.1234/x\n"
+ )
+
+ result = self._make().export_to_ris(document)
+
+ assert "DO - https://doi.org/10.1234/x\n" in result
+ assert "secret" not in result
+
+ @pytest.mark.parametrize("password,parts", _PASSWORDS)
+ @pytest.mark.parametrize(
+ "line",
+ [
+ "Mirrored by https://user:[email protected]/p",
+ # Splitting authors on "&" cut this into parts that each passed.
+ "Mirrored by https://user:secret&commat;mirror.example/p",
+ ],
+ )
+ def test_author_text_is_credential_checked(self, line, password, parts):
+ line = _with_password(line, password)
+ result = self._make()._create_ris_entry("1", f"Paper title\n{line}")
+
+ assert "TI - Paper title\n" in result
+ _assert_holds_no_part(result, parts)
+
+ @pytest.mark.parametrize("password,parts", _PASSWORDS)
+ @pytest.mark.parametrize(
+ "title",
+ [
+ "https://DOI:[email protected]/x",
+ "https://x:doi:[email protected]/x",
+ "https://user:[email protected]/x",
+ "https://user:[email protected]/x",
+ "https://user:secret Published [email protected]/x",
+ "https://u by [email protected]/x",
+ ],
+ )
+ def test_nothing_is_cut_out_of_a_credential_url_title(
+ self, title, password, parts
+ ):
+ """Title cleaning and the author, DOI and year matches cut text out
+ of the title. A cut that drops the @ of a credential URL left its
+ userinfo in TI, AU or DO, and a year in the password was written as
+ PY; the title is now sanitized whole, and gives no author, DOI or
+ year."""
+ title = _with_password(title, password)
+ result = self._make()._create_ris_entry("1", title)
+
+ assert "TI - https://example.org/x\n" in result
+ assert "AU -" not in result
+ assert "DO -" not in result
+ assert "PY -" not in result
+ _assert_holds_no_part(result, parts)
+
+ @pytest.mark.parametrize("password,parts", _PASSWORDS)
+ @pytest.mark.parametrize(
+ "title,expected",
+ [
+ (
+ "https:/by Smith./user:[email protected]/x",
+ "https://example.org/x",
+ ),
+ ("/by Smith./user:[email protected]/x", "//example.org/x"),
+ (
+ "https:/by Smith./user:secret [email protected]/x",
+ "https://example.org/x",
+ ),
+ (
+ "https:/by Smith./user:DOI:[email protected]/x",
+ "https://example.org/x",
+ ),
+ ],
+ )
+ def test_title_that_cleaning_joins_into_a_credential_url_is_sanitized(
+ self, title, expected, password, parts
+ ):
+ """No title is a credential URL as written. Cleaning cuts the
+ author phrase out, and what is left joins into one, so the cleaned
+ title is checked as well as the title as written. In the last two,
+ a later cut, at ``Volume`` or at ``DOI:``, cut that URL's @ away
+ again: the title was written as ``https://user:secret`` or
+ ``https://user:``, with the DOI ``[email protected]/x``. So every
+ form a cut makes is checked, the first one flagged is sanitized,
+ and the title gives no DOI, nor a year, which a password can hold.
+ """
+ title = _with_password(title, password)
+ assert not citation_formatter._ris_spells_userinfo(title)
+
+ result = self._make()._create_ris_entry("1", title)
+
+ assert f"TI - {expected}\n" in result
+ assert "DO -" not in result
+ assert "PY -" not in result
+ _assert_holds_no_part(result, parts)
+
+ @pytest.mark.parametrize("password,parts", _PASSWORDS)
+ def test_doi_is_not_cut_out_of_a_credential_url_line(self, password, parts):
+ result = self._make()._create_ris_entry(
+ "1",
+ _with_password(
+ "Paper title\nMirror: https://DOI:[email protected]/x",
+ password,
+ ),
+ )
+
+ assert "TI - Paper title\n" in result
+ assert "DO -" not in result
+ assert "PY -" not in result
+ _assert_holds_no_part(result, parts)
+
+ @freeze_time("2026-01-01")
+ def test_a_year_in_a_password_is_not_written_as_the_year(self):
+ """The year pattern ran over the whole source text, so it took the
+ password 2024 out of the link's line, next to the link written
+ without it, and 1999 out of a token in a title written without it.
+ """
+ url = "https://admin:[email protected]/"
+ result = self._make()._create_ris_entry(
+ "1", f"Internal dashboard\nURL: {url}", url=url
+ )
+
+ assert "UR - https://intranet.example/\n" in result
+ assert "PY -" not in result
+
+ result = self._make()._create_ris_entry(
+ "1", "https://deploy:[email protected]/"
+ )
+
+ assert "TI - https://host.example/\n" in result
+ assert "PY -" not in result
+
+ @pytest.mark.parametrize(
+ "password,parts",
+ [
+ pytest.param("2024", ("2024",), id="year"),
+ pytest.param("tok-1999-x", ("1999",), id="year-in-a-token"),
+ pytest.param("Kv3 1987", ("Kv3", "1987"), id="year-after-a-space"),
+ pytest.param("DOI:10.1234%2Fx", ("1234",), id="doi"),
+ pytest.param("by Smith", ("Smith",), id="name-after-by"),
+ pytest.param(
+ "by Smith", ("Smith",), id="name-after-by-no-break-space"
+ ),
+ pytest.param(
+ "Xq7,Zr9&Wk3.Vm5",
+ ("Xq7", "Zr9", "Wk3", "Vm5"),
+ id="punctuation",
+ ),
+ ],
+ )
+ @pytest.mark.parametrize(
+ "where",
+ ["title", "URL line", "DOI line", "lowercase doi line", "export"],
+ )
+ @freeze_time("2026-01-01")
+ def test_a_password_shaped_like_a_field_is_not_cut_into_it(
+ self, where, password, parts
+ ):
+ """Each password matches a pattern that cuts a field out of text: a
+ year (PY), a ``DOI:`` label (DO), ``by`` and a name (AU), and the
+ commas, ampersands and full stops at which the author list is split
+ and ended. The URL that holds it is a value as a whole: a title, or
+ the value of a ``URL:`` or ``DOI:`` line, which the exporter reads
+ as the link or the DOI and strips the userinfo from. The year
+ pattern took ``2024`` out of any of them. From a ``URL:`` or
+ ``DOI:`` line, the author pattern also took ``Smith@intranet`` out
+ of the password ``by Smith``, and the DOI pattern took
+ ``https://admin:Kv3`` out of a lowercase ``doi:`` line: in a line, a
+ URL after a label ends at whitespace, so the check passed the line.
+ """
+ url = f"https://admin:{password}@intranet.example/"
+ exporter = self._make()
+ if where == "title":
+ result = exporter._create_ris_entry("1", url, url=url)
+ kept = ["TI - https://intranet.example/\n"]
+ elif where == "URL line":
+ result = exporter._create_ris_entry(
+ "1", f"Internal dashboard\nURL: {url}", url=url
+ )
+ kept = ["UR - https://intranet.example/\n"]
+ elif where == "DOI line":
+ result = exporter._create_ris_entry(
+ "1", f"Internal dashboard\nDOI: {url}", metadata={"doi": url}
+ )
+ kept = ["DO - https://intranet.example/\n"]
+ elif where == "lowercase doi line":
+ result = exporter._create_ris_entry(
+ "1", f"Internal dashboard\ndoi: {url}"
+ )
+ kept = ["TI - Internal dashboard\n"]
+ else:
+ result = exporter.export_to_ris(
+ f"## Sources\n\n[1] Internal dashboard\nURL: {url}\n"
+ f"DOI: {url}\n"
+ )
+ kept = [
+ "UR - https://intranet.example/\n",
+ "DO - https://intranet.example/\n",
+ ]
+
+ for line in kept:
+ assert line in result
+ assert "AU -" not in result
+ assert "PY -" not in result
+ _assert_holds_no_part(result, parts)
+
+ @pytest.mark.parametrize("password,parts", _PASSWORDS)
+ @pytest.mark.parametrize(
+ "line",
+ [
+ pytest.param(
+ "Mirrored by Jane Doe, https: user:secret@localhost/x",
+ id="after-a-comma",
+ ),
+ pytest.param(
+ "Mirrored by Jane Doe and https: user:secret@localhost/x",
+ id="after-and",
+ ),
+ pytest.param(
+ "Mirrored by Jane Doe & https: user:secret@localhost/x",
+ id="after-an-ampersand",
+ ),
+ pytest.param(
+ "Mirrored by Jane Doe, https://user name:secret@localhost/x",
+ id="space-in-the-userinfo",
+ ),
+ ],
+ )
+ def test_a_later_author_is_read_as_the_url_it_starts(
+ self, line, password, parts
+ ):
+ """The author pattern ends the list at a full stop and splits it at
+ commas, ``and`` and ampersands, any of which a password can hold,
+ as ``Xq7,Zr9&Wk3.Vm5`` does. Each author was checked as cut, so the
+ cut hid the @ of a URL that only a reading as a whole finds: the
+ record held the author ``https: user:Xq7`` and the authors ``Zr9``
+ and ``Wk3``. Each author is now read as the URL it starts, to the
+ end of its line, and the line gives no author or year."""
+ line = _with_password(line, password)
+ result = self._make()._create_ris_entry("1", f"Paper title\n{line}")
+
+ assert "TI - Paper title\n" in result
+ assert "AU -" not in result
+ _assert_holds_no_part(result, parts)
+
+ @pytest.mark.parametrize(
+ "value,flagged",
+ [
+ pytest.param(
+ "https: user:a.b@localhost/x", True, id="at-the-start"
+ ),
+ pytest.param(
+ "Jane Doe, https: user:a.b@localhost/x",
+ True,
+ id="after-a-comma",
+ ),
+ pytest.param(
+ "Jane Doe and https://user name:a@localhost/x",
+ True,
+ id="after-and-and-a-byte-order-mark",
+ ),
+ # The first URL's authority ends at the second one's slashes.
+ pytest.param(
+ "https:a, https://u.v w@localhost/x", True, id="overlapping"
+ ),
+ pytest.param("https://@@localhost/x", False, id="at-signs-only"),
+ pytest.param(
+ "Jane Doe, https://example.org/@jane", False, id="at-in-a-path"
+ ),
+ pytest.param(
+ "Jane Doe, mail [email protected]", False, id="no-url-start"
+ ),
+ pytest.param(
+ "Grand and https: email me at [email protected]",
+ False,
+ id="prose-after-and",
+ ),
+ ],
+ )
+ def test_values_are_read_as_the_urls_their_authors_start(
+ self, value, flagged
+ ):
+ """A URL as a whole is read where a value, or an author in it,
+ starts: past whitespace in its authority, to the first ``/``, ``?``
+ or ``#``, with an @ there after any leading @ signs counting as
+ userinfo, as a whole value is read. Prose after a scheme's colon
+ is kept as it is in a title."""
+ assert citation_formatter._ris_value_starts_userinfo(value) is flagged
+
+ def test_value_starts_are_checked_in_linear_time(self):
+ """Each of the 20,000 authors here starts a URL whose authority
+ starts inside the first one's and ends where it ends, so the
+ authority is scanned once, not once per URL: about forty
+ milliseconds, against about four seconds when each URL scans it."""
+ value = "https:a," * 20000
+ started = time.perf_counter()
+ flagged = citation_formatter._ris_value_starts_userinfo(value)
+ elapsed = time.perf_counter() - started
+
+ assert not flagged
+ assert elapsed < 1.0, f"the check took {elapsed:.2f}s"
+
+ @pytest.mark.parametrize(
+ "full_text,url,expected",
+ [
+ pytest.param(
+ "Quarterly report 2023\n"
+ "URL: https://admin:[email protected]/",
+ "https://admin:[email protected]/",
+ "PY - 2023\n",
+ id="year-in-the-title",
+ ),
+ pytest.param(
+ "Archived post\nURL: https://example.org/2019/post",
+ "https://example.org/2019/post",
+ "PY - 2019\n",
+ id="year-in-a-plain-link",
+ ),
+ pytest.param(
+ "Archived post\nurl: https://example.org/2019/post",
+ "",
+ "PY - 2019\n",
+ id="year-in-a-plain-lowercase-link",
+ ),
+ pytest.param(
+ "Paper by Jane Doe\n"
+ "URL: https://admin:[email protected]/",
+ "https://admin:[email protected]/",
+ "AU - Jane Doe\n",
+ id="author-in-the-title",
+ ),
+ ],
+ )
+ def test_a_field_outside_a_credential_line_is_kept(
+ self, full_text, url, expected
+ ):
+ """Only a line that carries a credential gives no year, author or
+ DOI. A plain ``URL:`` line, in any letter case, still gives a year,
+ and so does a title beside a credential URL's line."""
+ result = self._make()._create_ris_entry("1", full_text, url=url)
+
+ assert expected in result
+ assert "2024" not in result
+ assert "secret" not in result
+
+ @pytest.mark.parametrize(
+ "url",
+ [
+ "https://evil.example\\@arxiv.org/abs/1",
+ "https://evil.example\\.arxiv.org/abs/1",
+ ],
+ )
+ def test_backslash_in_authority_is_omitted(self, url):
+ """WHATWG parsers end the authority at the backslash, so the host is
+ evil.example, while urlsplit reads arxiv.org; neither reading is
+ exported as the link, and arXiv is not credited as publisher."""
+ result = self._make()._create_ris_entry("1", "Title", url=url)
+
+ assert "UR -" not in result
+ assert "PB -" not in result
+
+ @pytest.mark.parametrize("password,parts", _PASSWORDS)
+ @pytest.mark.parametrize(
+ "value",
+ [
+ pytest.param("\\\\user:[email protected]/x", id="backslashes"),
+ pytest.param("/\\user:[email protected]/x", id="slash-backslash"),
+ pytest.param("\\/user:[email protected]/x", id="backslash-slash"),
+ pytest.param(
+ "\\\\\\user:[email protected]/x", id="three-backslashes"
+ ),
+ pytest.param("///user:[email protected]/x", id="three-slashes"),
+ pytest.param(
+ "\\/\\/user:[email protected]/x", id="escaped-slashes"
+ ),
+ pytest.param(
+ "<\\\\user:[email protected]/x>", id="angle-brackets"
+ ),
+ pytest.param("see \\\\user:[email protected]/x", id="prose"),
+ pytest.param(
+ "\\\\ user:[email protected]/x", id="space-after-the-run"
+ ),
+ pytest.param(
+ "https:\\\\ user:[email protected]/x",
+ id="https-backslashes-then-space",
+ ),
+ pytest.param(
+ "https:/// user:[email protected]/x",
+ id="https-three-slashes-then-space",
+ ),
+ ],
+ )
+ def test_run_of_slashes_and_backslashes_opens_an_authority(
+ self, value, password, parts
+ ):
+ """A browser reads any run of two or more slashes or backslashes
+ after https: as //, and does the same with no scheme against an
+ https page, so each value is a credential URL to it. The check knew
+ only //, so these were written verbatim as titles, and the
+ scheme-less ones as links too: urlsplit finds no authority in them
+ to strip."""
+ value = _with_password(value, password)
+ result = self._make()._create_ris_entry("1", value, url=value)
+
+ assert "TI - [source URL omitted]\n" in result
+ assert "UR -" not in result
+ assert "PB -" not in result
+ _assert_holds_no_part(result, parts)
+
+ @pytest.mark.parametrize("password,parts", _PASSWORDS)
+ @pytest.mark.parametrize(
+ "value",
+ [
+ # Inside the scheme.
+ pytest.param(
+ "ht&Tab;tps:user:[email protected]/x", id="scheme-tab-entity"
+ ),
+ pytest.param(
+ "ht&#10;tps:/user:[email protected]/x", id="scheme-lf-decimal"
+ ),
+ pytest.param(
+ "ht%0Dtps:user:[email protected]/x", id="scheme-cr-percent"
+ ),
+ # Between the slashes.
+ pytest.param(
+ "https:/&#9;/user:[email protected]/x",
+ id="slashes-tab-decimal",
+ ),
+ pytest.param(
+ "https:/&NewLine;/user:[email protected]/x",
+ id="slashes-lf-entity",
+ ),
+ pytest.param(
+ "https:/%0D/user:[email protected]/x", id="slashes-cr-percent"
+ ),
+ pytest.param(
+ "ftp:/%09/user:[email protected]/x", id="ftp-slashes-tab"
+ ),
+ pytest.param(
+ "/&#10;/user:[email protected]/x", id="bare-slashes-lf"
+ ),
+ # After the scheme's colon.
+ pytest.param(
+ "https:&#x9;user:[email protected]/x", id="colon-tab-hex"
+ ),
+ pytest.param(
+ "https:%0A/user:[email protected]/x", id="colon-lf-percent"
+ ),
+ pytest.param(
+ "https:&#13;user:[email protected]/x", id="colon-cr-decimal"
+ ),
+ # Written as it is, not encoded.
+ pytest.param(
+ "https:/\t/user:[email protected]/x", id="slashes-raw-tab"
+ ),
+ ],
+ )
+ def test_tab_or_line_break_does_not_hide_userinfo(
+ self, value, password, parts
+ ):
+ """A browser deletes every tab, LF and CR from a URL before it
+ parses it, so each value, with its entity or escape decoded, is a
+ credential URL to it: ``ht&Tab;tps:user:secret@...`` reads as
+ ``https:user:secret@...``. The check took the decoded character for
+ whitespace that ends or blocks an authority, so the title and the
+ DOI were written verbatim, and the link too wherever urlsplit found
+ no scheme in it. A raw tab, which serialization turns into a space,
+ is deleted for the check as well."""
+ value = _with_password(value, password)
+ result = self._make()._create_ris_entry(
+ "1", f"{value}\nDOI: {value}", url=value
+ )
+
+ assert "TI - [source URL omitted]\n" in result
+ assert "UR -" not in result
+ assert "PB -" not in result
+ assert "DO -" not in result
+ _assert_holds_no_part(result, parts)
+
+ from_metadata = self._make()._create_ris_entry(
+ "1", "Paper title", metadata={"doi": value}
+ )
+
+ assert "TI - Paper title\n" in from_metadata
+ assert "DO -" not in from_metadata
+ _assert_holds_no_part(from_metadata, parts)
+
+ @pytest.mark.parametrize(
+ "title,expected",
+ [
+ ("Notes\tfrom [email protected]", "Notes from [email protected]"),
+ ("Notes\rfrom [email protected]", "Notes from [email protected]"),
+ ],
+ )
+ def test_tab_or_carriage_return_in_a_plain_title_keeps_it(
+ self, title, expected
+ ):
+ """Deleting the character only feeds the check. A title it does not
+ flag is written as before, with the character turned into a space."""
+ result = self._make()._create_ris_entry("1", title)
+
+ assert f"TI - {expected}\n" in result
+
+ @pytest.mark.parametrize("password,parts", _PASSWORDS)
+ @pytest.mark.parametrize(
+ "value",
+ [
+ pytest.param(
+ "\ufeffhttps://user name:[email protected]/x",
+ id="space-in-userinfo",
+ ),
+ pytest.param(
+ "\ufeff\\\\ user:[email protected]/x", id="space-after-the-run"
+ ),
+ ],
+ )
+ def test_leading_byte_order_mark_does_not_hide_userinfo(
+ self, value, password, parts
+ ):
+ """JavaScript's trim() removes a leading U+FEFF, which Python's
+ strip() keeps, so a reader that trims the field reads each value as
+ a URL as a whole, whitespace in its authority included. The check
+ read it as text, where a URL ends at whitespace, so the title and
+ the link were written verbatim."""
+ value = _with_password(value, password)
+ result = self._make()._create_ris_entry("1", value, url=value)
+
+ assert "TI - [source URL omitted]\n" in result
+ assert "UR -" not in result
+ assert "PB -" not in result
+ _assert_holds_no_part(result, parts)
+
+ @pytest.mark.parametrize("password,parts", _PASSWORDS)
+ @pytest.mark.parametrize("scheme", ["https", "HTTP", "ftp"])
+ @pytest.mark.parametrize(
+ "gap",
+ [
+ pytest.param(" ", id="space"),
+ pytest.param("\u00a0", id="no-break-space"),
+ pytest.param("\x0b", id="vertical-tab"),
+ pytest.param("\u3000", id="ideographic-space"),
+ pytest.param("&#32;", id="entity-space"),
+ pytest.param("/ ", id="slash-then-space"),
+ pytest.param("\\ ", id="backslash-then-space"),
+ ],
+ )
+ def test_whitespace_after_a_scheme_colon_does_not_hide_userinfo(
+ self, scheme, gap, password, parts
+ ):
+ """Parsed with no base, each value, its entity decoded, is a
+ credential URL to a browser, whose userinfo starts with the
+ whitespace, encoded: ``https: user:[email protected]/x`` reads as
+ ``https://%20user:[email protected]/x``. The check took whitespace
+ right after a special scheme's colon, or after its colon and one
+ slash or backslash, for the end of the URL, so the title, the
+ author and a metadata DOI were written verbatim. The source link
+ was already left out."""
+ value = f"{scheme}:{gap}user:{password}@example.org/x"
+ result = self._make()._create_ris_entry(
+ "1", f"{value}\nDOI: {value}\nMirrored by {value}", url=value
+ )
+
+ assert "TI - [source URL omitted]\n" in result
+ assert "AU -" not in result
+ assert "UR -" not in result
+ assert "PB -" not in result
+ _assert_holds_no_part(result, parts)
+
+ from_metadata = self._make()._create_ris_entry(
+ "1", "Paper title", metadata={"doi": value}
+ )
+
+ assert "TI - Paper title\n" in from_metadata
+ assert "DO -" not in from_metadata
+ _assert_holds_no_part(from_metadata, parts)
+
+ @pytest.mark.parametrize("password,parts", _PASSWORDS)
+ @pytest.mark.parametrize(
+ "value",
+ [
+ pytest.param(
+ "https: user:secret @example.org/x", id="space-before-the-at"
+ ),
+ pytest.param("HTTP: [email protected]", id="token-only"),
+ pytest.param("ftp: user:[email protected] ", id="trailing-space"),
+ pytest.param(
+ "https:\u00a0 user:[email protected]/x", id="whitespace-run"
+ ),
+ ],
+ )
+ def test_one_token_after_a_scheme_colon_and_whitespace_is_flagged(
+ self, value, password, parts
+ ):
+ """Past whitespace right after a special scheme's colon, one token,
+ an @ and a host without whitespace open an authority. Whitespace
+ before the @ does not prevent it, nor does whitespace after the host
+ at the end of the value, which a browser trims. A browser reads the
+ values, as written, with the password ``secret%20``, the username
+ ``%20secret-token``, the URL ``ftp://%20user:[email protected]/``
+ and the username ``%C2%A0%20user``."""
+ value = _with_password(value, password)
+ assert citation_formatter._ris_spells_userinfo(value)
+
+ result = self._make()._create_ris_entry("1", value, url=value)
+
+ assert "TI - [source URL omitted]\n" in result
+ assert "UR -" not in result
+ _assert_holds_no_part(result, parts)
+
+ @pytest.mark.parametrize("password,parts", _PASSWORDS)
+ @pytest.mark.parametrize(
+ "control",
+ [
+ pytest.param("\x00", id="null"),
+ pytest.param("\x1b", id="escape"),
+ pytest.param("\x7f", id="delete"),
+ pytest.param("\x9f", id="c1-control"),
+ ],
+ )
+ @pytest.mark.parametrize(
+ "shape",
+ [
+ pytest.param(
+ "https: user:secret@localhost {}", id="after-the-host"
+ ),
+ pytest.param(
+ "https: user:secret {}@localhost/x", id="before-the-at"
+ ),
+ pytest.param(
+ "https: {} user:secret@localhost/x", id="after-the-colon"
+ ),
+ ],
+ )
+ def test_control_among_whitespace_after_a_scheme_colon_is_flagged(
+ self, shape, control, password, parts
+ ):
+ """Serialization writes a control character as a space, so it
+ writes each value in a form the check flags, such as
+ ``https: user:secret@localhost`` and two spaces, and a browser reads
+ each written form as userinfo at localhost. Python's ``re`` does not
+ count these controls as whitespace, so the check read each as text,
+ found no credential, and the title, the author and a metadata DOI
+ were written verbatim. The source link was already left out."""
+ value = _with_password(shape.format(control), password)
+ result = self._make()._create_ris_entry(
+ "1", f"{value}\nDOI: {value}\nMirrored by {value}", url=value
+ )
+
+ assert "TI - [source URL omitted]\n" in result
+ assert "AU -" not in result
+ assert "UR -" not in result
+ _assert_holds_no_part(result, parts)
+
+ from_metadata = self._make()._create_ris_entry(
+ "1", "Paper title", metadata={"doi": value}
+ )
+
+ assert "TI - Paper title\n" in from_metadata
+ assert "DO -" not in from_metadata
+ _assert_holds_no_part(from_metadata, parts)
+
+ @pytest.mark.parametrize("password,parts", _PASSWORDS)
+ @pytest.mark.parametrize(
+ "value",
+ [
+ pytest.param(
+ "https: user:secret@localhost %1B", id="after-the-host"
+ ),
+ pytest.param(
+ "https: user:secret %1B@localhost/x", id="before-the-at"
+ ),
+ pytest.param(
+ "https: %1B user:secret@localhost/x", id="after-the-colon"
+ ),
+ ],
+ )
+ def test_encoded_control_among_whitespace_is_flagged(
+ self, value, password, parts
+ ):
+ """Percent-decoded, each value holds an escape character after
+ whitespace, which a browser trims after the host and keeps in the
+ userinfo anywhere else, so it reads each as userinfo at localhost.
+ The check reads every decoded form with its controls as spaces, as
+ it reads the value itself. It read the decoded control as text, and
+ the title was written verbatim."""
+ value = _with_password(value, password)
+ result = self._make()._create_ris_entry("1", value, url=value)
+
+ assert citation_formatter._ris_spells_userinfo(value)
+ assert "TI - [source URL omitted]\n" in result
+ _assert_holds_no_part(result, parts)
+
+ @pytest.mark.parametrize("password,parts", _PASSWORDS)
+ @pytest.mark.parametrize(
+ "value",
+ [
+ pytest.param(
+ "https: user:secret@localhost \ufeff", id="after-a-space"
+ ),
+ pytest.param(
+ "HTTP: secret-token@localhost \ufeff \ufeff", id="in-a-run"
+ ),
+ pytest.param(
+ "ftp: user:secret @localhost\u3000\ufeff",
+ id="after-an-ideographic-space",
+ ),
+ ],
+ )
+ def test_trailing_byte_order_mark_does_not_hide_userinfo(
+ self, value, password, parts
+ ):
+ """JavaScript's trim() removes a trailing U+FEFF as it removes a
+ leading one, so a reader that trims the field reads each value, as
+ written, as a credential URL: ``https: user:secret@localhost`` with
+ the username ``%20user``, ``HTTP: secret-token@localhost`` with the
+ username ``%20secret-token``, and ``ftp: user:secret @localhost``
+ with the password ``secret%20``. After the host the check accepted
+ only whitespace before the end of the value, so the title, the
+ author and a metadata DOI were written verbatim."""
+ value = _with_password(value, password)
+ result = self._make()._create_ris_entry(
+ "1", f"{value}\nDOI: {value}\nMirrored by {value}", url=value
+ )
+
+ assert "TI - [source URL omitted]\n" in result
+ assert "AU -" not in result
+ assert "UR -" not in result
+ _assert_holds_no_part(result, parts)
+
+ from_metadata = self._make()._create_ris_entry(
+ "1", "Paper title", metadata={"doi": value}
+ )
+
+ assert "DO -" not in from_metadata
+ _assert_holds_no_part(from_metadata, parts)
+
+ def test_controls_after_a_host_are_checked_in_linear_time(self):
+ """After the host, the value may end in whitespace, controls and
+ U+FEFF, and the host may hold controls and U+FEFF itself. The host
+ is matched possessively, so a long run of them followed by text is
+ scanned once. If the match could give characters back, each shorter
+ host would rescan the rest of the value: about two seconds for this
+ run, against about ten milliseconds."""
+ value = "https: a@" + "\x01" * 20000 + " x"
+ started = time.perf_counter()
+ flagged = citation_formatter._ris_spells_userinfo(value)
+ elapsed = time.perf_counter() - started
+
+ assert not flagged
+ assert elapsed < 0.25, f"the check took {elapsed:.2f}s"
+
+ @pytest.mark.parametrize(
+ "title",
+ [
+ "HTTPS: why you should email [email protected]",
+ "http: a guide to the @ sign",
+ "HTTP: email me at [email protected]",
+ "HTTP:\u00a0email me at [email protected]",
+ "HTTP: [email protected] explains caching",
+ "HTTP: @mention",
+ ],
+ )
+ def test_prose_after_a_scheme_word_keeps_its_title(self, title):
+ """A browser reads four of these as URLs whose userinfo is what
+ precedes the @: ``HTTP: email me at [email protected]`` as
+ ``http://%20email%20me%20at%[email protected]/``, and
+ ``HTTP: @mention`` with a space for its username. The other two,
+ ``http: a guide to the @ sign`` and
+ ``HTTP: [email protected] explains caching``, do not parse, since their
+ would-be hosts hold whitespace. They are titles all the same. Past
+ whitespace right after a special scheme's colon, the check reads an
+ authority only when one token, an @ and a host without whitespace
+ follow. Each of these holds whitespace in its would-be userinfo or
+ host, or has no token before its @."""
+ result = self._make()._create_ris_entry("1", title)
+
+ assert f"TI - {title}\n" in result
+
+ @pytest.mark.parametrize(
+ "value",
+ [
+ "https://example.org/article",
+ "https://medium.com/@author/article",
+ "https://web.archive.org/web/2020/https://medium.com/@author/post",
+ "https://example.org/?email=bob%40example.org",
+ "https://www.google.com/url?q=https%3A%2F%2Fexample.org%2F%40user",
+ "https://de.wikipedia.org/wiki/Stra%C3%9Fe",
+ "https://example.org//double//slash",
+ "https://example.org/a%09b%0Ac",
+ "https://example.org/?q=a&Tab;b",
+ ],
+ )
+ def test_url_without_credentials_is_unchanged(self, value):
+ result = self._make()._create_ris_entry("1", value, url=value)
+
+ assert f"TI - {value}\n" in result
+ assert f"UR - {value}\n" in result
+
+ @pytest.mark.parametrize(
+ "title",
+ [
+ "Re: question from [email protected]",
+ "Using //@ts-ignore in TypeScript",
+ "Using \\\\@ts-ignore in TypeScript",
+ "\\\\fileserver\\share\\notes.txt",
+ "HTTP: The Definitive Guide",
+ "Tom & Jerry @ Home",
+ "Tabs&Tab;and&#10;breaks @ home",
+ "Comments open with /&Tab;/ and @ marks a decorator",
+ ],
+ )
+ def test_title_without_url_credentials_is_unchanged(self, title):
+ result = self._make()._create_ris_entry("1", title)
+
+ assert f"TI - {title}\n" in result
+
+ @pytest.mark.parametrize("password,parts", _PASSWORDS)
+ def test_no_field_keeps_a_value_the_shared_check_flags(
+ self, password, parts
+ ):
+ """UR, TI, AU and DO cannot drift apart: whatever the shared check
+ flags is absent from every field of the record, and no field, PY
+ included, holds any part of its password."""
+ from local_deep_research.text_optimization.citation_formatter import (
+ _ris_spells_userinfo,
+ )
+
+ values = [
+ "https://user:[email protected]\uff1a8443/x",
+ "git+https://user:[email protected]/o/r",
+ "//user:[email protected]/x",
+ "\\\\user:[email protected]/x",
+ "///user:[email protected]/x",
+ "&#32;https:/user:[email protected]/x",
+ "https://user:secret%40example.org/x",
+ "[arXiv](https://user:[email protected]/abs/1)",
+ "https://first:secret@x&commat;example.org/doc",
+ "https://DOI:[email protected]/x",
+ "ht&Tab;tps:user:[email protected]/x",
+ "/&#10;/user:[email protected]/x",
+ "\ufeffhttps://user name:[email protected]/x",
+ "https: user:[email protected]/x",
+ "HTTP:/ user:[email protected]/x",
+ ]
+ for value in values:
+ value = _with_password(value, password)
+ assert _ris_spells_userinfo(value), value
+ result = self._make()._create_ris_entry(
+ "1",
+ f"{value}\nWritten by {value}\nDOI: {value}",
+ url=value,
+ )
+ for line in result.split("\n"):
+ assert not _ris_spells_userinfo(line[6:]), (value, line)
+ _assert_holds_no_part(result, parts)
+
+ def test_every_field_is_written_in_a_form_the_check_passes(self):
+ """Serialization turns every control character, U+2028 and U+2029
+ into a space, and the check reads each value that way too, so no
+ field of a record is written with a value it flags. The values put
+ a C0, DEL or C1 control, U+2028, U+FEFF, or an entity- or
+ percent-encoded control into credential URLs and titles, around and
+ in place of their whitespace. Before, a credential URL with a
+ control after its whitespace passed the check and was written with
+ a space in place of the control, which the check flags."""
+ from local_deep_research.text_optimization.citation_formatter import (
+ _ris_spells_userinfo,
+ )
+
+ shapes = [
+ "https: user:secret@localhost{}",
+ "https: user:secret@localhost {}",
+ "https: user:secret {}@localhost/x",
+ "https: {} user:secret@localhost/x",
+ "https:{}user:secret@localhost/x",
+ "https://user:secret@localhost{}/x",
+ "<https://user:secret@localhost/x>{}",
+ "HTTP: email me at{} [email protected]",
+ "HTTP: {}@mention",
+ "Notes{}from [email protected]",
+ ]
+ characters = [
+ "\x00",
+ "\x1b",
+ "\x7f",
+ "\x85",
+ "\x9f",
+ "\u2028",
+ "\ufeff",
+ "&#1;",
+ "%01",
+ ]
+ exporter = self._make()
+ for shape in shapes:
+ for character in characters:
+ value = shape.format(character)
+ records = [
+ exporter._create_ris_entry(
+ "1",
+ f"{value}\nDOI: {value}\nMirrored by {value}",
+ url=value,
+ ),
+ exporter._create_ris_entry(
+ "1", "Paper title", metadata={"doi": value}
+ ),
+ exporter.export_to_ris(
+ f"## Sources\n\n[1] {value}\nURL: {value}\n"
+ f"DOI: {value}\n"
+ ),
+ ]
+ for record in records:
+ for line in record.split("\n"):
+ assert not _ris_spells_userinfo(line[6:]), (
+ value,
+ line,
+ )
+
+ @pytest.mark.parametrize(
+ "bound,limit,value",
+ [
+ # "%41" has 3 characters; its readings are "%41" and "A".
+ pytest.param(
+ "_RIS_MAX_DECODED_LENGTH", 3, "%41", id="length-of-the-value"
+ ),
+ # "a\ufb01" has 2 characters; NFKC reads it as "afi", with 3.
+ pytest.param(
+ "_RIS_MAX_DECODED_LENGTH",
+ 3,
+ "a\ufb01",
+ id="length-of-a-decoded-reading",
+ ),
+ # The readings "a%41" and "aA" have 6 characters together.
+ pytest.param(
+ "_RIS_MAX_DECODED_TOTAL_LENGTH",
+ 6,
+ "a%41",
+ id="length-of-all-readings",
+ ),
+ pytest.param("_RIS_MAX_READINGS", 2, "%41", id="reading-count"),
+ ],
+ )
+ def test_value_past_a_reading_bound_is_left_out(
+ self, monkeypatch, bound, limit, value
+ ):
+ """Each bound is lowered to what a few characters reach. At the
+ bound the value is checked as usual; past it the value counts as
+ flagged and is left out of the title and the link. Only the length
+ of the value itself was bounded before, though NFKC can make a
+ reading longer than the value, and the readings together were not
+ bounded at all."""
+ monkeypatch.setattr(citation_formatter, bound, limit)
+ kept = self._make()._create_ris_entry("1", value, url=value)
+
+ assert f"TI - {value}\n" in kept
+ assert f"UR - {value}\n" in kept
+
+ monkeypatch.setattr(citation_formatter, bound, limit - 1)
+ result = self._make()._create_ris_entry("1", value, url=value)
+
+ assert citation_formatter._ris_spells_userinfo(value)
+ assert "TI - [source URL omitted]\n" in result
+ assert "UR -" not in result
+ assert value not in result
+
+ def test_value_with_a_control_character_is_measured(self, monkeypatch):
+ """One reading of a value is the value with its control characters
+ written as spaces, so an ASCII value holding a control is measured
+ against the bounds like a value that needs decoding, and counts as
+ flagged past them. It was its only reading, and was not measured."""
+ value = "Notes\x01from bob"
+ monkeypatch.setattr(citation_formatter, "_RIS_MAX_READINGS", 2)
+ kept = self._make()._create_ris_entry("1", value)
+
+ assert "TI - Notes from bob\n" in kept
+
+ monkeypatch.setattr(citation_formatter, "_RIS_MAX_READINGS", 1)
+ result = self._make()._create_ris_entry("1", value)
+
+ assert citation_formatter._ris_spells_userinfo(value)
+ assert "TI - [source URL omitted]\n" in result
+
class TestRISEntryStructuralValidity:
"""Regression guard for the accumulator-clobber bug.