~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""``redact_url_credentials``: credentials embedded in a configured URL.
Used by the log lines listed in
``changelog.d/+redact-base-url-query-credentials.security.md`` (the
OpenAI-compatible, Ollama and OpenAI-embeddings provider logs, the research
route fallbacks, the friendly error rewrite and the debug settings dump).
The worker's parameters line keeps main's stricter origin-only redaction.
Other URL-printing log lines do not use this helper.
These tests pin the
redaction scope; ``tests/error_handling/test_openai_compat_errors.py``
covers the historical ``_strip_credentials`` alias.
"""
import time
import pytest
from local_deep_research.security.url_redaction import (
MAX_REDACTABLE_URL_LEN,
is_credential_param_name,
redact_url_credentials,
)
SECRET = "sk-leak1234567890abcdef"
class TestRepeatedQuestionMark:
def test_credential_after_a_second_question_mark(self):
out = redact_url_credentials(f"https://h/v1?x=1?api_key={SECRET}")
assert SECRET not in out
assert out == "https://h/v1?x=1?api_key=REDACTED"
def test_credential_inside_an_embedded_url_value(self):
out = redact_url_credentials(
f"https://h/v1?next=https://a/b?token={SECRET}&y=1"
)
assert SECRET not in out
assert out == "https://h/v1?next=https://a/b?token=REDACTED&y=1"
class TestBareItemBoundary:
"""A value-less query or fragment item is redacted from 17 characters
on; a 16-character one is kept."""
@pytest.mark.parametrize("separator", ["?", "#"])
def test_sixteen_characters_are_kept(self, separator):
item = "b" * 16
url = f"https://h/v1{separator}{item}"
assert redact_url_credentials(url) == url
@pytest.mark.parametrize("separator", ["?", "#"])
def test_seventeen_characters_are_redacted(self, separator):
out = redact_url_credentials(f"https://h/v1{separator}{'b' * 17}")
assert out == f"https://h/v1{separator}REDACTED"
class TestCredentialNames:
@pytest.mark.parametrize(
"name",
[
"accessToken",
"authToken",
"clientSecret",
"apiKey",
"dbPassword",
"sessionToken",
"pass",
"credential",
"credentials",
"jwt",
"session",
"user.password",
"X-Api-Key",
# Array-style names, a ``+``-encoded space, an acronym prefix.
"api_key[0]",
"key[]",
"auth[token]",
"data[api_key]",
"api+key",
"XAPIKey",
# Run-together names: a token/secret/password/passwd ending, or
# a listed ``...key`` compound.
"authtoken",
"accesstoken",
"apitoken",
"APITOKEN",
"refreshtoken",
"clientsecret",
"dbpassword",
"rootpasswd",
"secretkey",
"privatekey",
"xapikey",
"APIKEY",
"ocpapimsubscriptionkey",
# Every bracket part is checked, and a stray bracket or a
# non-alphanumeric byte ends a word.
"api_key[a][b]",
"api_key[prod]",
"token[value]",
"api_key[a][0]",
"api_key]",
"api_key[]]",
"key%00",
"auth%00",
# OAuth grant secrets: RFC 7521/7523 assertions, the RFC 7636
# PKCE verifier, the RFC 5849 verifier, the RFC 8628 device
# code, an OpenID Connect ID token hint, and a bare refresh
# token.
"assertion",
"client_assertion",
"clientAssertion",
"clientassertion",
"code_verifier",
"codeVerifier",
"oauth_verifier",
"device_code",
"deviceCode",
"id_token_hint",
"refresh",
# A passphrase, and the RFC 6749 authorization code.
"passphrase",
"sslPassphrase",
"keypassphrase",
"auth_code",
"authCode",
"authorization_code",
"AuthorizationCode",
# Run-together spellings of the whole-name credentials.
"devicecode",
"DEVICECODE",
"idtokenhint",
"IDTOKENHINT",
"authcode",
"authorizationcode",
# A trailing number, version or plural does not change what a
# name holds.
"api_key_2",
"api_key_1",
"apiKey2",
"APIKEY2",
"XAPIKey2",
"key1",
"key2",
"keyv2",
"token2",
"authtoken2",
"access_token_v2",
"accessTokenV2",
"apiKeyV2",
"key_v2",
"x-api-key-v2",
"api_key_v2_1",
"api_keys",
"apikeys",
"api_keys[0]",
"keys",
"tokens",
"tokensv2",
"client_secrets",
"passwords",
"credentials2",
"pass2",
"sas2",
"device_code_2",
"auth_codes",
"code2",
"refresh_v2",
"id_token_hint_2",
],
)
def test_is_credential(self, name):
assert is_credential_param_name(name)
out = redact_url_credentials(f"https://h/v1?{name}={SECRET}")
assert out == f"https://h/v1?{name}=REDACTED"
@pytest.mark.parametrize(
"name",
[
"tokenizer",
"passthrough",
"monkey",
"keyboard",
"oauth",
"error_code",
"sessionId",
"passage",
"keyword",
"items[0]",
"data[name]",
"OAuth",
# Words that merely end in ``key`` / ``token``-like letters.
"turkey",
"hockey",
"whiskey",
"fookey",
"secretary",
"items[0][1]",
# Non-secret neighbours of the OAuth grant names: only the
# whole names ``refresh``, ``device_code`` and ``id_token_hint``
# match, not every name ending in their last word.
"client_assertion_type",
"code_challenge",
"code_challenge_method",
"login_hint",
"force_refresh",
"refresh_interval",
"country_code",
# Non-secret neighbours of the number / version / plural rule:
# taking the suffix off leaves an ordinary word, and a name
# that is only a version keeps it.
"page2",
"page_2",
"items",
"ids",
"v2",
"api_v2",
"limit_v2",
"version2",
"refresh_interval_2",
"country_codes",
"error_codes",
"login_hints",
"codecs",
"monkeys",
"hotkeys",
"fookeys",
"keyboards",
"keywords",
"tokenizers",
"passthrough2",
# A plural ``s`` is not taken off after another ``s``, and a
# name ending in ``es`` keeps its ``e``.
"access",
"bypass",
"class",
"sass",
"passes",
"status",
# Digits that are part of an ordinary word.
"oauth2",
"s3",
"md5",
"sha256",
"ipv4",
],
)
def test_lookalike_is_not_credential(self, name):
assert not is_credential_param_name(name)
url = f"https://h/v1?{name}=visible"
assert redact_url_credentials(url) == url
@pytest.mark.parametrize(
"name", ["sort_key", "sort_keys", "max_tokens", "codes", "sessions"]
)
def test_documented_over_redaction(self, name):
"""Names that hold no secret but end in a credential word (also
after the plural is taken off) are redacted -- the safe direction
the rule errs towards, as its comment says."""
assert is_credential_param_name(name)
out = redact_url_credentials(f"https://h/v1?{name}=visible")
assert out == f"https://h/v1?{name}=REDACTED"
@pytest.mark.parametrize(
"url,expected",
[
(
f"https://h/v1?api_key_2={SECRET}&page2=7",
"https://h/v1?api_key_2=REDACTED&page2=7",
),
(
f"https://h/v1#access_token_v2={SECRET}&v2=1",
"https://h/v1#access_token_v2=REDACTED&v2=1",
),
(
f"https://h/v1;passphrase={SECRET}/x",
"https://h/v1;passphrase=REDACTED/x",
),
(
f"https://h/v1?items=3&api_keys%5B%5D={SECRET}",
"https://h/v1?items=3&api_keys%5B%5D=REDACTED",
),
],
ids=["query", "fragment", "path-parameter", "encoded-array"],
)
def test_suffixed_names_are_redacted_in_every_position(self, url, expected):
out = redact_url_credentials(url)
assert SECRET not in out
assert out == expected
def test_overlong_name_is_treated_as_a_credential(self):
"""A name longer than the inspected limit is not parsed word by
word; its value is redacted (fail closed)."""
name = "q" * 300
assert is_credential_param_name(name)
out = redact_url_credentials(f"https://h/v1?{name}={SECRET}")
assert out == f"https://h/v1?{name}=REDACTED"
# 200 KB inputs shaped to hit the per-name loop that used to be quadratic
# (one rescan per trailing ``[..]`` group). Explicit ids keep the test node
# ids short.
_BIG = 200_000
_SLOW_NAMES = [
pytest.param("k" + "[]" * (_BIG // 2), id="empty-index-groups"),
pytest.param("k" + "[0]" * (_BIG // 3), id="digit-index-groups"),
pytest.param("k" + "[a]" * (_BIG // 3), id="named-groups"),
pytest.param("k" + "%5B%5D" * (_BIG // 6), id="percent-encoded-groups"),
pytest.param("aB" * (_BIG // 2), id="camel-case"),
pytest.param("a" * _BIG, id="plain"),
]
class TestLargeInputsAreCheap:
@pytest.mark.parametrize("name", _SLOW_NAMES)
def test_long_name_is_classified_quickly(self, name):
start = time.perf_counter()
assert is_credential_param_name(name)
assert time.perf_counter() - start < 0.5
@pytest.mark.parametrize("name", _SLOW_NAMES)
def test_long_url_is_redacted_quickly(self, name):
start = time.perf_counter()
out = redact_url_credentials(f"http://localhost:1234/v1?{name}=1")
assert time.perf_counter() - start < 0.5
assert out == "<unknown>"
def test_url_over_the_cap_is_withheld(self):
url = "https://h/v1?x=" + "a" * MAX_REDACTABLE_URL_LEN
assert redact_url_credentials(url) == "<unknown>"
def test_url_at_the_cap_is_still_redacted(self):
prefix = f"https://h/v1?api_key={SECRET}&x="
url = prefix + "a" * (MAX_REDACTABLE_URL_LEN - len(prefix))
assert len(url) == MAX_REDACTABLE_URL_LEN
out = redact_url_credentials(url)
assert SECRET not in out
assert out.startswith("https://h/v1?api_key=REDACTED&x=aaa")
class TestPathSegments:
def test_sk_key_segment_is_redacted(self):
out = redact_url_credentials(f"https://h/{SECRET}/v1")
assert out == "https://h/REDACTED/v1"
def test_long_dashless_alnum_segment_is_redacted(self):
segment = "hf_AbCdEfGhIjKlMnOpQrStUvWxYz0123456789"
out = redact_url_credentials(f"https://h/{segment}/v1")
assert out == "https://h/REDACTED/v1"
def test_key_prefixed_segment_is_redacted(self):
out = redact_url_credentials("https://h/key-abcdefghijklmnopqrst/v1")
assert out == "https://h/REDACTED/v1"
@pytest.mark.parametrize(
"segment",
[
"SK-leak1234567890abcdef",
"Sk-leak1234567890abcdef",
"KEY-abcdefghijklmnopqrst",
"Key-abcdefghijklmnopqrst",
],
)
def test_key_prefix_matches_in_any_letter_case(self, segment):
out = redact_url_credentials(f"https://h/{segment}/v1")
assert out == "https://h/REDACTED/v1"
def test_32_hex_id_is_redacted(self):
"""Documented over-redaction: a 32-hex account id (e.g. a
Cloudflare account id) is indistinguishable from a hex key."""
out = redact_url_credentials(
"https://h/accounts/0123456789abcdef0123456789abcdef/ai/v1"
)
assert out == "https://h/accounts/REDACTED/ai/v1"
@pytest.mark.parametrize(
"segment",
[
"text-embedding-3-large-2024-preview-model",
"Qwen2-VL-72B-Instruct-Preview-Model-Extra",
"abcdefghijklmnopqrstuvwxyzabcdefghij",
],
)
def test_hyphenated_or_letter_only_names_are_kept(self, segment):
url = f"https://h/models/{segment}/v1"
assert redact_url_credentials(url) == url
def test_uuid_and_short_segments_are_kept(self):
url = (
"https://h/openai/deployments/"
"123e4567-e89b-12d3-a456-426614174000/v1/sk-short"
)
assert redact_url_credentials(url) == url
class TestSchemelessUserinfo:
def test_userinfo_without_scheme_is_dropped(self):
out = redact_url_credentials("user:[email protected]:8443/v1?x=1")
assert "SECRETPW" not in out
assert out == "gw.example:8443/v1?x=1"
def test_host_port_without_userinfo_passes_through(self):
assert redact_url_credentials("localhost:1234/v1") == (
"localhost:1234/v1"
)
class TestAuthorityIsKeptExactly:
def test_host_case_and_port_spelling_are_preserved(self):
out = redact_url_credentials("https://u:[email protected]:0443/v1")
assert out == "https://GW.Example.COM:0443/v1"
def test_ipv6_authority_is_preserved(self):
out = redact_url_credentials("http://u:p@[::1]:8080/v1?key=abc")
assert out == "http://[::1]:8080/v1?key=REDACTED"
class TestNonStringInput:
@pytest.mark.parametrize(
"value", [None, "", b"https://u:p@h/v1", 12, ["https://h"]]
)
def test_returns_unknown_marker(self, value):
assert redact_url_credentials(value) == "<unknown>"
class TestFailsClosedWhenUserinfoIsNotIsolated:
"""An unencoded ``/``, ``?`` or ``#`` in a password makes ``urlsplit``
end the authority early, so ``@`` never reaches the netloc and the
password would pass through. Such input is not redacted piecemeal;
the whole value is withheld."""
@pytest.mark.parametrize(
"url",
[
"https://user:Abc/[email protected]/v1",
"https://user:Abc#[email protected]/v1",
"https://user:[email protected]/v1",
"user:Abc/[email protected]/v1",
"https://user:123/[email protected]/v1",
# An ``@`` inside the password followed by ``/``, ``?`` or
# ``#`` and another ``@``: the netloc ends at the separator,
# so it holds exactly one ``@`` and looked isolated.
"https://user:P@ss/[email protected]/v1",
"https://user:Abc@12/[email protected]/v1",
"https://user:Ab@[email protected]/v1",
"https://user:Ab@c#[email protected]/v1",
"https://user:Ab@1/[email protected]/v1",
"user:Ab@c/Secret99@host/v1",
],
)
def test_unisolated_userinfo_returns_unknown(self, url):
out = redact_url_credentials(url)
assert out == "<unknown>"
assert "secret" not in out
@pytest.mark.parametrize(
"url",
[
"http://[::1]x/v1",
"http://h.example:80a/v1",
# Parameters inside the parsed authority are not redacted
# there, so the whole value is withheld.
f"https://host;api_key={SECRET}",
f"https://host:443;api_key={SECRET}",
f"//h&sig={SECRET}",
f"https://u:p@host&token={SECRET}/v1",
f"https://host=x{SECRET}/v1",
],
)
def test_malformed_authority_returns_unknown(self, url):
assert redact_url_credentials(url) == "<unknown>"
@pytest.mark.parametrize("url", ["user:pw@", "@"])
def test_nothing_left_after_userinfo_returns_unknown(self, url):
assert redact_url_credentials(url) == "<unknown>"
@pytest.mark.parametrize(
"url",
[
"https://user:pw@host/[email protected]",
"https://host/[email protected]",
"https://host/v1/users/[email protected]",
],
)
def test_at_sign_outside_the_authority_returns_unknown(self, url):
"""Policy: any ``@`` outside the authority is withheld, including
a harmless-looking e-mail address."""
assert redact_url_credentials(url) == "<unknown>"
@pytest.mark.parametrize(
"url",
[
# Scheme-less, no ``/`` before the query or fragment: the
# leading userinfo part ends at ``?`` / ``#``, so an ``@`` in
# the query or fragment is not taken as the end of userinfo
# (which would drop the host and pass the credentials on).
"user:pw@host?x=a@b&api_key=secretKEY123",
"user:pw@host#a@b&access_token=secretKEY123",
"user:pw@host:[email protected]&api_key=secretKEY123",
"host:8080?x=a@b&api_key=secretKEY123",
"localhost:1234?note=a@b&token=secretKEY123",
"user:secretKEY123@host?x=a@b",
"user:secretKEY123@host#a@b",
"secretKEY123@host?q=r@s",
],
)
def test_schemeless_at_sign_in_query_or_fragment_returns_unknown(self, url):
assert redact_url_credentials(url) == "<unknown>"
def test_schemeless_userinfo_before_query_is_stripped(self):
assert redact_url_credentials(
"user:pw@host:8080?api_key=secretKEY123"
) == ("host:8080?api_key=REDACTED")
def test_at_sign_inside_a_password_is_still_stripped(self):
"""``@`` in the password with no later separator stays inside the
authority; the last ``@`` ends the userinfo."""
assert redact_url_credentials("https://u:p@ss@host/v1") == (
"https://host/v1"
)
def test_digit_port_and_empty_port_are_kept(self):
assert redact_url_credentials("http://h.example:99999/v1") == (
"http://h.example:99999/v1"
)
assert redact_url_credentials("https://h:/v1") == "https://h:/v1"
class TestSchemeIsLowercased:
def test_scheme_lowercased_host_case_kept(self):
out = redact_url_credentials("HTTPS://u:[email protected]:8443/v1")
assert out == "https://GW.Example.COM:8443/v1"