Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(security): strip credentials from inline citation links (#6990)

LearningCircuite3ff33c55e07
diff --git a/changelog.d/5916.security.md b/changelog.d/5916.security.md
new file mode 100644
index 0000000000..c9bff8da8b
--- /dev/null
+++ b/changelog.d/5916.security.md
@@ -0,0 +1 @@
+Inline citation links no longer write URL userinfo (`user:pass@`) into saved reports, streams, or exports. Literal userinfo is removed, and the host, path and query are kept. A citation is left unlinked when its userinfo is spelled where the plain parser does not look: without `//` (`https:user:pass@host`), or through HTML character references that a rendered link decodes, such as an encoded `@` or `:` or a leading ` `. It is also left unlinked when its URL no longer parses once the userinfo is removed.
diff --git a/src/local_deep_research/text_optimization/citation_formatter.py b/src/local_deep_research/text_optimization/citation_formatter.py
index bb023eb101..6b600601ec 100644
--- a/src/local_deep_research/text_optimization/citation_formatter.py
+++ b/src/local_deep_research/text_optimization/citation_formatter.py
@@ -6,7 +6,7 @@ from array import array
from enum import Enum
from html.entities import html5
from typing import Any, Dict, List, Optional, Tuple
-from urllib.parse import urlparse, urlsplit
+from urllib.parse import urlparse, urlsplit, urlunsplit
from loguru import logger
from slugify import slugify
@@ -82,10 +82,79 @@ def _destination_is_refused(form: str) -> bool:
return form.lstrip().replace("\\", "/").startswith("//")
+def _authority_head_has_userinfo(netloc: str, path: str) -> bool:
+ """Whether a linkable-scheme URL spells its authority without ``//``.
+
+ WHATWG-consuming renderers read ``https:user:[email protected]/x`` — and
+ its ``https:/``, ``https:///`` and ``https:\\`` spellings — as the
+ authority ``user:[email protected]``: any run of slashes or backslashes
+ stands in for the ``//``. ``urlsplit`` populates ``netloc`` only for the
+ ``//``-delimited form, so an @-in-netloc test alone lets those spellings
+ keep their userinfo. The authority-ish head — what follows the scheme,
+ past leading slashes or backslashes, up to the first ``/``, ``?`` or
+ ``#`` — is where those parsers read it; an @ there is treated as a
+ credential. An @ further along (``.../@author/...``, ``?q=a@b``) never
+ lands in it.
+
+ The head is rebuilt from the ``netloc`` and ``path`` that ``urlsplit``
+ parsed, never cut out of the input at the scheme's length. ``urlsplit``
+ drops leading spaces and C0 controls, and every tab and newline, before
+ it reads the scheme, as a browser does. `` https:/user:pass@host``
+ decodes to a leading space and then the URL, so a cut at
+ ``len("https") + 1`` started on the ``:``, found no ``@`` before the
+ next ``/``, and let the credential through. ``?`` and ``#`` need no
+ search here: ``urlsplit`` has already split them off.
+ """
+ rest = f"//{netloc}{path}" if netloc else path
+ head = rest.strip().lstrip("/\\").partition("/")[0]
+ return "@" in head
+
+
+def _spells_hidden_userinfo(candidate: str) -> bool:
+ """Whether a raw or entity-decoded reading of ``candidate`` hides userinfo.
+
+ Consulted after stripping literal userinfo, so every reading that finds
+ an @ is a spelling the plain parse missed:
+ an authority not delimited by ``//``, or one assembled by rendered HTML
+ entities — ``https://user:pass@host`` breaks the scheme token, parses
+ as scheme-less, and decodes into a live credential URL in a browser.
+
+ A reading that ``urlsplit`` rejects counts as hiding userinfo, just as
+ ``_destination_is_refused`` refuses one: the guard cannot vouch for what
+ it cannot parse. Stripping can produce such a reading.
+ ``https://[@v1.u:p]@v1.example/`` parses, but its userinfo is the
+ ``[`` before the first ``@``, and once that is removed what is left has
+ an unbalanced ``]``, which ``urlsplit`` rejects. A browser still decodes
+ that remainder into ``https://v1.u:p]@v1.example/``, whose userinfo is
+ ``v1.u:p]``.
+ """
+ try:
+ parsed = urlsplit(candidate)
+ except ValueError:
+ return True
+ if (
+ parsed.scheme in _LINKABLE_URL_SCHEMES
+ and "@" not in parsed.netloc
+ and _authority_head_has_userinfo(parsed.netloc, parsed.path)
+ ):
+ return True
+ decoded = html.unescape(candidate)
+ if decoded == candidate:
+ return False
+ try:
+ cooked = urlsplit(decoded)
+ except ValueError:
+ return True
+ return cooked.scheme in _LINKABLE_URL_SCHEMES and (
+ "@" in cooked.netloc
+ or _authority_head_has_userinfo(cooked.netloc, cooked.path)
+ )
+
+
def _safe_link_destination(url: str) -> str:
"""Return ``url`` ready to be interpolated into ``](...)``, or "" to refuse.
- Three things have to hold, and what comes back is the string that was
+ Four things have to hold, and what comes back is the string that was
checked rather than the one that came in:
1. No control character survives. ``urlsplit`` removes tabs and newlines
@@ -98,7 +167,13 @@ def _safe_link_destination(url: str) -> str:
what SOURCE_TAGGED uses to decide whether to render a link at all; the
other modes keep linking the relative URLs a library document uses. A
scheme-less destination that names a host (``//host/x``) is refused.
- 3. What markdown would read as syntax is percent-encoded, so the gate
+ 3. HTTP userinfo is removed before the URL reaches a report or export.
+ It is a credential even when the destination itself is otherwise
+ safe, and it is refused rather than stripped when it is spelled
+ where the plain parser does not look — an authority not delimited
+ by ``//``, or one assembled from HTML entities — or when what is
+ left once it is removed no longer parses.
+ 4. What markdown would read as syntax is percent-encoded, so the gate
cannot be walked around by ending the destination early.
Rules 1 and 2 are checked on the character-reference-decoded form too.
@@ -113,8 +188,10 @@ def _safe_link_destination(url: str) -> str:
the only one today: an exported .md / .qmd / .tex and every other markdown
consumer gets whatever this returns.
- Callers already skip sources whose URL is empty, so a refused destination
- leaves the citation as a plain ``[N]`` bracket instead of dropping it.
+ A refused destination leaves the citation unlinked rather than dropping
+ it: callers treat an empty URL as no URL, so the citation stays a plain
+ ``[N]`` bracket, or an unlinked tag such as ``[local-N]`` in
+ SOURCE_TAGGED mode.
"""
candidate = url.strip()
if not candidate:
@@ -123,6 +200,19 @@ def _safe_link_destination(url: str) -> str:
html.unescape(candidate)
):
return ""
+ parsed = urlsplit(candidate)
+ if parsed.scheme in _LINKABLE_URL_SCHEMES and "@" in parsed.netloc:
+ # A parseable authority keeps its host and loses its userinfo.
+ host = parsed.netloc.rpartition("@")[2]
+ if not host:
+ return ""
+ candidate = urlunsplit(parsed._replace(netloc=host))
+ if _spells_hidden_userinfo(candidate):
+ # The userinfo is spelled where the plain parse does not look —
+ # without the ``//``, or through HTML entities that a rendered
+ # link decodes back into a live credential URL — or what is left
+ # no longer parses. Refuse rather than emit it.
+ return ""
escapes = dict(_DESTINATION_ESCAPES)
if not _parentheses_are_balanced(candidate):
escapes.update(_PARENTHESIS_ESCAPES)
diff --git a/tests/text_optimization/test_citation_formatter_behavior.py b/tests/text_optimization/test_citation_formatter_behavior.py
index 870c495a4d..e19d3929bb 100644
--- a/tests/text_optimization/test_citation_formatter_behavior.py
+++ b/tests/text_optimization/test_citation_formatter_behavior.py
@@ -4,6 +4,8 @@ Behavioral tests for citation_formatter module.
Tests CitationFormatter, QuartoExporter, RISExporter, and LaTeXExporter classes.
"""
+import pytest
+
class TestCitationModeEnum:
"""Tests for CitationMode enum."""
@@ -1463,6 +1465,323 @@ class TestCitationFormatterFormatDocumentSplit:
class TestCitationFormatterApplyInlineHyperlinks:
"""Tests for apply_inline_hyperlinks — structured-source delegation."""
+ @pytest.mark.parametrize(
+ "mode",
+ [
+ "number_hyperlinks",
+ "domain_hyperlinks",
+ "domain_id_hyperlinks",
+ "domain_id_always_hyperlinks",
+ "source_tagged_hyperlinks",
+ ],
+ )
+ @pytest.mark.parametrize("source_key", ["url", "link"])
+ @pytest.mark.parametrize("at_entity", ["@", "@", "@"])
+ def test_inline_destination_refuses_userinfo_remaining_after_stripping(
+ self, mode, source_key, at_entity
+ ):
+ from local_deep_research.text_optimization.citation_formatter import (
+ CitationFormatter,
+ CitationMode,
+ )
+
+ source = {
+ "index": 1,
+ "title": "Doc",
+ source_key: f"https://first:password@secret{at_entity}example.org/doc",
+ }
+
+ assert (
+ CitationFormatter(CitationMode(mode)).apply_inline_hyperlinks(
+ "Claim [1].", [source]
+ )
+ == "Claim [1]."
+ )
+
+ @pytest.mark.parametrize(
+ "mode",
+ [
+ "number_hyperlinks",
+ "domain_hyperlinks",
+ "domain_id_hyperlinks",
+ "domain_id_always_hyperlinks",
+ "source_tagged_hyperlinks",
+ ],
+ )
+ @pytest.mark.parametrize("source_key", ["url", "link"])
+ def test_inline_destination_removes_userinfo_without_changing_query(
+ self, mode, source_key
+ ):
+ from local_deep_research.text_optimization.citation_formatter import (
+ CitationFormatter,
+ CitationMode,
+ )
+
+ source = {
+ "index": 1,
+ "title": "Doc",
+ source_key: "https://admin:[email protected]/doc?utm_source=x",
+ }
+ result = CitationFormatter(CitationMode(mode)).apply_inline_hyperlinks(
+ "Claim [1].", [source]
+ )
+
+ assert "https://example.test/doc?utm_source=x" in result
+ assert "admin" not in result
+ assert "hunter2" not in result
+
+ def test_inline_destination_refuses_entity_encoded_userinfo(self):
+ from local_deep_research.text_optimization.citation_formatter import (
+ CitationFormatter,
+ )
+
+ source = {
+ "index": 1,
+ "title": "Doc",
+ "url": "https://admin@example.test/doc",
+ }
+ assert (
+ CitationFormatter().apply_inline_hyperlinks("Claim [1].", [source])
+ == "Claim [1]."
+ )
+
+ @pytest.mark.parametrize(
+ "destination",
+ [
+ "https:user:[email protected]/article",
+ "https:/user:[email protected]/x",
+ "https:///user:[email protected]/x",
+ "https:\\user:[email protected]/x",
+ "https:\\\\user:[email protected]/x",
+ ],
+ )
+ def test_inline_destination_refuses_authority_without_double_slash(
+ self, destination
+ ):
+ """Renderers parse `https:user:pass@host` (and `/`, `///`, `\\`
+ spellings) as a real authority even though ``urlsplit`` does not
+ populate ``netloc`` for it, so the userinfo must not survive."""
+ from local_deep_research.text_optimization.citation_formatter import (
+ CitationFormatter,
+ )
+
+ source = {"index": 1, "title": "Doc", "url": destination}
+ result = CitationFormatter().apply_inline_hyperlinks(
+ "Claim [1].", [source]
+ )
+
+ assert result == "Claim [1]."
+ assert "user:pass" not in result
+ assert "user:secret" not in result
+
+ @pytest.mark.parametrize(
+ "destination",
+ [
+ "https://user:[email protected]/a",
+ "https://user:[email protected]/a",
+ "https://user:[email protected]/a",
+ ],
+ )
+ def test_inline_destination_refuses_entity_encoded_scheme(
+ self, destination
+ ):
+ """An entity-encoded `:` breaks the scheme token for ``urlsplit``,
+ but a rendered link decodes it back into a live credential URL."""
+ from local_deep_research.text_optimization.citation_formatter import (
+ CitationFormatter,
+ )
+
+ source = {"index": 1, "title": "Doc", "url": destination}
+ result = CitationFormatter().apply_inline_hyperlinks(
+ "Claim [1].", [source]
+ )
+
+ assert result == "Claim [1]."
+ assert "user:pass" not in result
+
+ @pytest.mark.parametrize(
+ "destination",
+ [
+ pytest.param(" https:/user:[email protected]/x", id="decimal"),
+ pytest.param(" https:/user:[email protected]/x", id="hex"),
+ pytest.param("&#32https:/user:[email protected]/x", id="no-semi"),
+ pytest.param(" https:/user:[email protected]/x", id="padded"),
+ pytest.param(
+ "  https:/user:[email protected]/x", id="two-spaces"
+ ),
+ pytest.param(" https:///user:[email protected]/x", id="slash3"),
+ pytest.param(" HTTPS:/user:[email protected]/x", id="upper"),
+ pytest.param(
+ " https:///user:[email protected]/x", id="entity-slash"
+ ),
+ pytest.param(
+ " https:/user:[email protected]/x", id="c0-then-space"
+ ),
+ ],
+ )
+ def test_inline_destination_refuses_userinfo_after_encoded_leading_space(
+ self, destination
+ ):
+ """`` `` decodes to a leading space, which ``urlsplit`` and a
+ browser both drop before they read the scheme. A browser then
+ follows ``https:/user:[email protected]/x`` as a credential URL from
+ an ``http:`` page such as LDR's default origin or from an exported
+ file, and the ``https:///`` spelling from any page. The authority
+ must be read from the parse, not from an offset counted from the
+ start of the decoded string."""
+ from local_deep_research.text_optimization.citation_formatter import (
+ CitationFormatter,
+ )
+
+ source = {"index": 1, "title": "Doc", "url": destination}
+ result = CitationFormatter().apply_inline_hyperlinks(
+ "Claim [1].", [source]
+ )
+
+ assert result == "Claim [1]."
+ assert "user:pass" not in result
+
+ @pytest.mark.parametrize(
+ "destination",
+ [
+ "	https:/user:[email protected]/x",
+ "
https:/user:[email protected]/x",
+ "https:/user:[email protected]/x",
+ "https:/user:[email protected]/x",
+ ],
+ )
+ def test_inline_destination_refuses_userinfo_after_encoded_leading_control(
+ self, destination
+ ):
+ """A browser drops a leading C0 control just as it drops a space.
+ ``	`` and ``
`` decode to a control character, which is
+ refused outright; ``html.unescape`` decodes ```` and ````
+ to nothing, which leaves the scheme at the start."""
+ from local_deep_research.text_optimization.citation_formatter import (
+ CitationFormatter,
+ )
+
+ source = {"index": 1, "title": "Doc", "url": destination}
+ result = CitationFormatter().apply_inline_hyperlinks(
+ "Claim [1].", [source]
+ )
+
+ assert result == "Claim [1]."
+ assert "user:pass" not in result
+
+ @pytest.mark.parametrize(
+ "destination",
+ [
+ pytest.param(
+ "https://[@v1.user:hunter2]@v1.evil.example/doc",
+ id="raw-unparseable-decimal",
+ ),
+ pytest.param(
+ "https://[@v1.user:hunter2]@v1.evil.example/doc",
+ id="raw-unparseable-hex",
+ ),
+ pytest.param(
+ "https://[@v1.user:hunter2]@v1.evil.example/doc",
+ id="raw-unparseable-named",
+ ),
+ pytest.param(
+ "https://[@v1.user:hunter2]@v1.evil.example"
+ "/doc",
+ id="decoded-unparseable",
+ ),
+ ],
+ )
+ def test_inline_destination_refuses_what_stripping_leaves_unparseable(
+ self, destination
+ ):
+ """Each URL parses, but its userinfo opens a bracket, so removing it
+ leaves an unbalanced ``]`` (in the raw URL, or once it is decoded)
+ that ``urlsplit`` rejects. A browser still decodes that remainder
+ into ``https://v1.user:hunter2]@v1.evil.example/doc``. A reading the
+ guard cannot parse has to count as hiding userinfo, not as clean."""
+ from local_deep_research.text_optimization.citation_formatter import (
+ CitationFormatter,
+ )
+
+ source = {"index": 1, "title": "Doc", "url": destination}
+ result = CitationFormatter().apply_inline_hyperlinks(
+ "Claim [1].", [source]
+ )
+
+ assert result == "Claim [1]."
+ assert "hunter2" not in result
+
+ @pytest.mark.parametrize(
+ "destination,expected",
+ [
+ (" https://example.org/doc", " https://example.org/doc"),
+ (
+ " https:/example.org/@author",
+ " https:/example.org/@author",
+ ),
+ (
+ "https://[2001:db8::1]:8443/doc",
+ "https://[2001:db8::1]:8443/doc",
+ ),
+ (
+ "https://admin:hunter2@[2001:db8::1]:8443/doc?q=1",
+ "https://[2001:db8::1]:8443/doc?q=1",
+ ),
+ ],
+ )
+ def test_inline_destination_keeps_entity_led_and_bracketed_urls(
+ self, destination, expected
+ ):
+ """Controls for the two refusals above. A leading `` `` with no
+ userinfo after it keeps its link, and so does a bracketed host that
+ still parses once its userinfo is removed."""
+ from local_deep_research.text_optimization.citation_formatter import (
+ CitationFormatter,
+ )
+
+ source = {"index": 1, "title": "Doc", "url": destination}
+ result = CitationFormatter().apply_inline_hyperlinks(
+ "Claim [1].", [source]
+ )
+
+ assert result == f"Claim [[1]]({expected})."
+
+ @pytest.mark.parametrize(
+ "mode",
+ [
+ "number_hyperlinks",
+ "domain_hyperlinks",
+ "domain_id_hyperlinks",
+ "domain_id_always_hyperlinks",
+ "source_tagged_hyperlinks",
+ ],
+ )
+ @pytest.mark.parametrize(
+ "destination",
+ [
+ "https://medium.com/@author/article",
+ "https://example.org/[email protected]",
+ "https://example.org/wiki/Ünïcødé_(disambiguation)",
+ "https://api.example.org/v1?key=SECRET",
+ ],
+ )
+ def test_inline_destination_keeps_at_outside_the_authority(
+ self, mode, destination
+ ):
+ """The authority-head check must not eat @s that live in the path,
+ query, or a plain host — only userinfo before the host."""
+ from local_deep_research.text_optimization.citation_formatter import (
+ CitationFormatter,
+ CitationMode,
+ )
+
+ source = {"index": 1, "title": "Doc", "url": destination}
+ result = CitationFormatter(CitationMode(mode)).apply_inline_hyperlinks(
+ "Claim [1].", [source]
+ )
+
+ assert destination in result
+
def test_replaces_bracket_refs_with_markdown_links(self):
"""[1] [2] become [[1]](url) [[2]](url) given a structured list."""
from local_deep_research.text_optimization.citation_formatter import (