~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(security): strip credentials from inline citation links (#6990)
LearningCircuite3ff33c55e07
diff --git a/changelog.d/5916.security.md b/changelog.d/5916.security.mdnew file mode 100644index 0000000000..c9bff8da8b--- /dev/null+++ b/changelog.d/5916.security.md@@ -0,0 +1 @@+Inline citation links no longer write URL userinfo (`user:pass@`) into saved reports, streams, or exports. Literal userinfo is removed, and the host, path and query are kept. A citation is left unlinked when its userinfo is spelled where the plain parser does not look: without `//` (`https:user:pass@host`), or through HTML character references that a rendered link decodes, such as an encoded `@` or `:` or a leading ` `. It is also left unlinked when its URL no longer parses once the userinfo is removed.diff --git a/src/local_deep_research/text_optimization/citation_formatter.py b/src/local_deep_research/text_optimization/citation_formatter.pyindex bb023eb101..6b600601ec 100644--- a/src/local_deep_research/text_optimization/citation_formatter.py+++ b/src/local_deep_research/text_optimization/citation_formatter.py@@ -6,7 +6,7 @@ from array import arrayfrom enum import Enumfrom html.entities import html5from typing import Any, Dict, List, Optional, Tuple-from urllib.parse import urlparse, urlsplit+from urllib.parse import urlparse, urlsplit, urlunsplitfrom loguru import loggerfrom slugify import slugify@@ -82,10 +82,79 @@ def _destination_is_refused(form: str) -> bool:return form.lstrip().replace("\\", "/").startswith("//")+def _authority_head_has_userinfo(netloc: str, path: str) -> bool:+ """Whether a linkable-scheme URL spells its authority without ``//``.++ WHATWG-consuming renderers read ``https:user:[email protected]/x`` — and+ its ``https:/``, ``https:///`` and ``https:\\`` spellings — as the+ authority ``user:[email protected]``: any run of slashes or backslashes+ stands in for the ``//``. ``urlsplit`` populates ``netloc`` only for the+ ``//``-delimited form, so an @-in-netloc test alone lets those spellings+ keep their userinfo. The authority-ish head — what follows the scheme,+ past leading slashes or backslashes, up to the first ``/``, ``?`` or+ ``#`` — is where those parsers read it; an @ there is treated as a+ credential. An @ further along (``.../@author/...``, ``?q=a@b``) never+ lands in it.++ The head is rebuilt from the ``netloc`` and ``path`` that ``urlsplit``+ parsed, never cut out of the input at the scheme's length. ``urlsplit``+ drops leading spaces and C0 controls, and every tab and newline, before+ it reads the scheme, as a browser does. `` https:/user:pass@host``+ decodes to a leading space and then the URL, so a cut at+ ``len("https") + 1`` started on the ``:``, found no ``@`` before the+ next ``/``, and let the credential through. ``?`` and ``#`` need no+ search here: ``urlsplit`` has already split them off.+ """+ rest = f"//{netloc}{path}" if netloc else path+ head = rest.strip().lstrip("/\\").partition("/")[0]+ return "@" in head+++def _spells_hidden_userinfo(candidate: str) -> bool:+ """Whether a raw or entity-decoded reading of ``candidate`` hides userinfo.++ Consulted after stripping literal userinfo, so every reading that finds+ an @ is a spelling the plain parse missed:+ an authority not delimited by ``//``, or one assembled by rendered HTML+ entities — ``https://user:pass@host`` breaks the scheme token, parses+ as scheme-less, and decodes into a live credential URL in a browser.++ A reading that ``urlsplit`` rejects counts as hiding userinfo, just as+ ``_destination_is_refused`` refuses one: the guard cannot vouch for what+ it cannot parse. Stripping can produce such a reading.+ ``https://[@v1.u:p]@v1.example/`` parses, but its userinfo is the+ ``[`` before the first ``@``, and once that is removed what is left has+ an unbalanced ``]``, which ``urlsplit`` rejects. A browser still decodes+ that remainder into ``https://v1.u:p]@v1.example/``, whose userinfo is+ ``v1.u:p]``.+ """+ try:+ parsed = urlsplit(candidate)+ except ValueError:+ return True+ if (+ parsed.scheme in _LINKABLE_URL_SCHEMES+ and "@" not in parsed.netloc+ and _authority_head_has_userinfo(parsed.netloc, parsed.path)+ ):+ return True+ decoded = html.unescape(candidate)+ if decoded == candidate:+ return False+ try:+ cooked = urlsplit(decoded)+ except ValueError:+ return True+ return cooked.scheme in _LINKABLE_URL_SCHEMES and (+ "@" in cooked.netloc+ or _authority_head_has_userinfo(cooked.netloc, cooked.path)+ )++def _safe_link_destination(url: str) -> str:"""Return ``url`` ready to be interpolated into ``](...)``, or "" to refuse.- Three things have to hold, and what comes back is the string that was+ Four things have to hold, and what comes back is the string that waschecked rather than the one that came in:1. No control character survives. ``urlsplit`` removes tabs and newlines@@ -98,7 +167,13 @@ def _safe_link_destination(url: str) -> str:what SOURCE_TAGGED uses to decide whether to render a link at all; theother modes keep linking the relative URLs a library document uses. Ascheme-less destination that names a host (``//host/x``) is refused.- 3. What markdown would read as syntax is percent-encoded, so the gate+ 3. HTTP userinfo is removed before the URL reaches a report or export.+ It is a credential even when the destination itself is otherwise+ safe, and it is refused rather than stripped when it is spelled+ where the plain parser does not look — an authority not delimited+ by ``//``, or one assembled from HTML entities — or when what is+ left once it is removed no longer parses.+ 4. What markdown would read as syntax is percent-encoded, so the gatecannot be walked around by ending the destination early.Rules 1 and 2 are checked on the character-reference-decoded form too.@@ -113,8 +188,10 @@ def _safe_link_destination(url: str) -> str:the only one today: an exported .md / .qmd / .tex and every other markdownconsumer gets whatever this returns.- Callers already skip sources whose URL is empty, so a refused destination- leaves the citation as a plain ``[N]`` bracket instead of dropping it.+ A refused destination leaves the citation unlinked rather than dropping+ it: callers treat an empty URL as no URL, so the citation stays a plain+ ``[N]`` bracket, or an unlinked tag such as ``[local-N]`` in+ SOURCE_TAGGED mode."""candidate = url.strip()if not candidate:@@ -123,6 +200,19 @@ def _safe_link_destination(url: str) -> str:html.unescape(candidate)):return ""+ parsed = urlsplit(candidate)+ if parsed.scheme in _LINKABLE_URL_SCHEMES and "@" in parsed.netloc:+ # A parseable authority keeps its host and loses its userinfo.+ host = parsed.netloc.rpartition("@")[2]+ if not host:+ return ""+ candidate = urlunsplit(parsed._replace(netloc=host))+ if _spells_hidden_userinfo(candidate):+ # The userinfo is spelled where the plain parse does not look —+ # without the ``//``, or through HTML entities that a rendered+ # link decodes back into a live credential URL — or what is left+ # no longer parses. Refuse rather than emit it.+ return ""escapes = dict(_DESTINATION_ESCAPES)if not _parentheses_are_balanced(candidate):escapes.update(_PARENTHESIS_ESCAPES)diff --git a/tests/text_optimization/test_citation_formatter_behavior.py b/tests/text_optimization/test_citation_formatter_behavior.pyindex 870c495a4d..e19d3929bb 100644--- a/tests/text_optimization/test_citation_formatter_behavior.py+++ b/tests/text_optimization/test_citation_formatter_behavior.py@@ -4,6 +4,8 @@ Behavioral tests for citation_formatter module.Tests CitationFormatter, QuartoExporter, RISExporter, and LaTeXExporter classes."""+import pytest+class TestCitationModeEnum:"""Tests for CitationMode enum."""@@ -1463,6 +1465,323 @@ class TestCitationFormatterFormatDocumentSplit:class TestCitationFormatterApplyInlineHyperlinks:"""Tests for apply_inline_hyperlinks — structured-source delegation."""+ @pytest.mark.parametrize(+ "mode",+ [+ "number_hyperlinks",+ "domain_hyperlinks",+ "domain_id_hyperlinks",+ "domain_id_always_hyperlinks",+ "source_tagged_hyperlinks",+ ],+ )+ @pytest.mark.parametrize("source_key", ["url", "link"])+ @pytest.mark.parametrize("at_entity", ["@", "@", "@"])+ def test_inline_destination_refuses_userinfo_remaining_after_stripping(+ self, mode, source_key, at_entity+ ):+ from local_deep_research.text_optimization.citation_formatter import (+ CitationFormatter,+ CitationMode,+ )++ source = {+ "index": 1,+ "title": "Doc",+ source_key: f"https://first:password@secret{at_entity}example.org/doc",+ }++ assert (+ CitationFormatter(CitationMode(mode)).apply_inline_hyperlinks(+ "Claim [1].", [source]+ )+ == "Claim [1]."+ )++ @pytest.mark.parametrize(+ "mode",+ [+ "number_hyperlinks",+ "domain_hyperlinks",+ "domain_id_hyperlinks",+ "domain_id_always_hyperlinks",+ "source_tagged_hyperlinks",+ ],+ )+ @pytest.mark.parametrize("source_key", ["url", "link"])+ def test_inline_destination_removes_userinfo_without_changing_query(+ self, mode, source_key+ ):+ from local_deep_research.text_optimization.citation_formatter import (+ CitationFormatter,+ CitationMode,+ )++ source = {+ "index": 1,+ "title": "Doc",+ source_key: "https://admin:[email protected]/doc?utm_source=x",+ }+ result = CitationFormatter(CitationMode(mode)).apply_inline_hyperlinks(+ "Claim [1].", [source]+ )++ assert "https://example.test/doc?utm_source=x" in result+ assert "admin" not in result+ assert "hunter2" not in result++ def test_inline_destination_refuses_entity_encoded_userinfo(self):+ from local_deep_research.text_optimization.citation_formatter import (+ CitationFormatter,+ )++ source = {+ "index": 1,+ "title": "Doc",+ "url": "https://admin@example.test/doc",+ }+ assert (+ CitationFormatter().apply_inline_hyperlinks("Claim [1].", [source])+ == "Claim [1]."+ )++ @pytest.mark.parametrize(+ "destination",+ [+ "https:user:[email protected]/article",+ "https:/user:[email protected]/x",+ "https:///user:[email protected]/x",+ "https:\\user:[email protected]/x",+ "https:\\\\user:[email protected]/x",+ ],+ )+ def test_inline_destination_refuses_authority_without_double_slash(+ self, destination+ ):+ """Renderers parse `https:user:pass@host` (and `/`, `///`, `\\`+ spellings) as a real authority even though ``urlsplit`` does not+ populate ``netloc`` for it, so the userinfo must not survive."""+ from local_deep_research.text_optimization.citation_formatter import (+ CitationFormatter,+ )++ source = {"index": 1, "title": "Doc", "url": destination}+ result = CitationFormatter().apply_inline_hyperlinks(+ "Claim [1].", [source]+ )++ assert result == "Claim [1]."+ assert "user:pass" not in result+ assert "user:secret" not in result++ @pytest.mark.parametrize(+ "destination",+ [+ "https://user:[email protected]/a",+ "https://user:[email protected]/a",+ "https://user:[email protected]/a",+ ],+ )+ def test_inline_destination_refuses_entity_encoded_scheme(+ self, destination+ ):+ """An entity-encoded `:` breaks the scheme token for ``urlsplit``,+ but a rendered link decodes it back into a live credential URL."""+ from local_deep_research.text_optimization.citation_formatter import (+ CitationFormatter,+ )++ source = {"index": 1, "title": "Doc", "url": destination}+ result = CitationFormatter().apply_inline_hyperlinks(+ "Claim [1].", [source]+ )++ assert result == "Claim [1]."+ assert "user:pass" not in result++ @pytest.mark.parametrize(+ "destination",+ [+ pytest.param(" https:/user:[email protected]/x", id="decimal"),+ pytest.param(" https:/user:[email protected]/x", id="hex"),+ pytest.param(" https:/user:[email protected]/x", id="no-semi"),+ pytest.param(" https:/user:[email protected]/x", id="padded"),+ pytest.param(+ "  https:/user:[email protected]/x", id="two-spaces"+ ),+ pytest.param(" https:///user:[email protected]/x", id="slash3"),+ pytest.param(" HTTPS:/user:[email protected]/x", id="upper"),+ pytest.param(+ " https:///user:[email protected]/x", id="entity-slash"+ ),+ pytest.param(+ " https:/user:[email protected]/x", id="c0-then-space"+ ),+ ],+ )+ def test_inline_destination_refuses_userinfo_after_encoded_leading_space(+ self, destination+ ):+ """`` `` decodes to a leading space, which ``urlsplit`` and a+ browser both drop before they read the scheme. A browser then+ follows ``https:/user:[email protected]/x`` as a credential URL from+ an ``http:`` page such as LDR's default origin or from an exported+ file, and the ``https:///`` spelling from any page. The authority+ must be read from the parse, not from an offset counted from the+ start of the decoded string."""+ from local_deep_research.text_optimization.citation_formatter import (+ CitationFormatter,+ )++ source = {"index": 1, "title": "Doc", "url": destination}+ result = CitationFormatter().apply_inline_hyperlinks(+ "Claim [1].", [source]+ )++ assert result == "Claim [1]."+ assert "user:pass" not in result++ @pytest.mark.parametrize(+ "destination",+ [+ "	https:/user:[email protected]/x",+ " https:/user:[email protected]/x",+ "https:/user:[email protected]/x",+ "https:/user:[email protected]/x",+ ],+ )+ def test_inline_destination_refuses_userinfo_after_encoded_leading_control(+ self, destination+ ):+ """A browser drops a leading C0 control just as it drops a space.+ ``	`` and `` `` decode to a control character, which is+ refused outright; ``html.unescape`` decodes ```` and ````+ to nothing, which leaves the scheme at the start."""+ from local_deep_research.text_optimization.citation_formatter import (+ CitationFormatter,+ )++ source = {"index": 1, "title": "Doc", "url": destination}+ result = CitationFormatter().apply_inline_hyperlinks(+ "Claim [1].", [source]+ )++ assert result == "Claim [1]."+ assert "user:pass" not in result++ @pytest.mark.parametrize(+ "destination",+ [+ pytest.param(+ "https://[@v1.user:hunter2]@v1.evil.example/doc",+ id="raw-unparseable-decimal",+ ),+ pytest.param(+ "https://[@v1.user:hunter2]@v1.evil.example/doc",+ id="raw-unparseable-hex",+ ),+ pytest.param(+ "https://[@v1.user:hunter2]@v1.evil.example/doc",+ id="raw-unparseable-named",+ ),+ pytest.param(+ "https://[@v1.user:hunter2]@v1.evil.example"+ "/doc",+ id="decoded-unparseable",+ ),+ ],+ )+ def test_inline_destination_refuses_what_stripping_leaves_unparseable(+ self, destination+ ):+ """Each URL parses, but its userinfo opens a bracket, so removing it+ leaves an unbalanced ``]`` (in the raw URL, or once it is decoded)+ that ``urlsplit`` rejects. A browser still decodes that remainder+ into ``https://v1.user:hunter2]@v1.evil.example/doc``. A reading the+ guard cannot parse has to count as hiding userinfo, not as clean."""+ from local_deep_research.text_optimization.citation_formatter import (+ CitationFormatter,+ )++ source = {"index": 1, "title": "Doc", "url": destination}+ result = CitationFormatter().apply_inline_hyperlinks(+ "Claim [1].", [source]+ )++ assert result == "Claim [1]."+ assert "hunter2" not in result++ @pytest.mark.parametrize(+ "destination,expected",+ [+ (" https://example.org/doc", " https://example.org/doc"),+ (+ " https:/example.org/@author",+ " https:/example.org/@author",+ ),+ (+ "https://[2001:db8::1]:8443/doc",+ "https://[2001:db8::1]:8443/doc",+ ),+ (+ "https://admin:hunter2@[2001:db8::1]:8443/doc?q=1",+ "https://[2001:db8::1]:8443/doc?q=1",+ ),+ ],+ )+ def test_inline_destination_keeps_entity_led_and_bracketed_urls(+ self, destination, expected+ ):+ """Controls for the two refusals above. A leading `` `` with no+ userinfo after it keeps its link, and so does a bracketed host that+ still parses once its userinfo is removed."""+ from local_deep_research.text_optimization.citation_formatter import (+ CitationFormatter,+ )++ source = {"index": 1, "title": "Doc", "url": destination}+ result = CitationFormatter().apply_inline_hyperlinks(+ "Claim [1].", [source]+ )++ assert result == f"Claim [[1]]({expected})."++ @pytest.mark.parametrize(+ "mode",+ [+ "number_hyperlinks",+ "domain_hyperlinks",+ "domain_id_hyperlinks",+ "domain_id_always_hyperlinks",+ "source_tagged_hyperlinks",+ ],+ )+ @pytest.mark.parametrize(+ "destination",+ [+ "https://medium.com/@author/article",+ "https://example.org/[email protected]",+ "https://example.org/wiki/Ünïcødé_(disambiguation)",+ "https://api.example.org/v1?key=SECRET",+ ],+ )+ def test_inline_destination_keeps_at_outside_the_authority(+ self, mode, destination+ ):+ """The authority-head check must not eat @s that live in the path,+ query, or a plain host — only userinfo before the host."""+ from local_deep_research.text_optimization.citation_formatter import (+ CitationFormatter,+ CitationMode,+ )++ source = {"index": 1, "title": "Doc", "url": destination}+ result = CitationFormatter(CitationMode(mode)).apply_inline_hyperlinks(+ "Claim [1].", [source]+ )++ assert destination in result+def test_replaces_bracket_refs_with_markdown_links(self):"""[1] [2] become [[1]](url) [[2]](url) given a structured list."""from local_deep_research.text_optimization.citation_formatter import (