~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Tests for utilities/url_utils.py
Tests cover:
- URL normalization
- Scheme handling
- Private IP detection
"""
import pytest
from unittest.mock import patch
class TestNormalizeUrl:
"""Tests for normalize_url function."""
def test_normalize_url_with_http_scheme(self):
"""Test URL with http:// scheme is returned unchanged."""
from local_deep_research.utilities.url_utils import normalize_url
result = normalize_url("http://example.com")
assert result == "http://example.com"
def test_normalize_url_with_https_scheme(self):
"""Test URL with https:// scheme is returned unchanged."""
from local_deep_research.utilities.url_utils import normalize_url
result = normalize_url("https://example.com")
assert result == "https://example.com"
def test_normalize_url_with_http_scheme_and_port(self):
"""Test URL with http:// scheme and port is returned unchanged."""
from local_deep_research.utilities.url_utils import normalize_url
result = normalize_url("http://example.com:8080")
assert result == "http://example.com:8080"
def test_normalize_url_empty_raises_error(self):
"""Test that empty URL raises ValueError."""
from local_deep_research.utilities.url_utils import normalize_url
with pytest.raises(ValueError) as exc_info:
normalize_url("")
assert "empty" in str(exc_info.value).lower()
def test_normalize_url_strips_whitespace(self):
"""Test that whitespace is stripped."""
from local_deep_research.utilities.url_utils import normalize_url
result = normalize_url(" http://example.com ")
assert result == "http://example.com"
def test_normalize_url_malformed_http_colon(self):
"""Test URL with malformed http: (missing //) is fixed."""
from local_deep_research.utilities.url_utils import normalize_url
result = normalize_url("http:example.com")
assert result == "http://example.com"
def test_normalize_url_malformed_https_colon(self):
"""Test URL with malformed https: (missing //) is fixed."""
from local_deep_research.utilities.url_utils import normalize_url
result = normalize_url("https:example.com")
assert result == "https://example.com"
def test_normalize_url_localhost_gets_http(self):
"""Test that localhost gets http:// scheme."""
from local_deep_research.utilities.url_utils import normalize_url
with patch(
"local_deep_research.utilities.url_utils.is_private_ip",
return_value=True,
):
result = normalize_url("localhost:8080")
assert result == "http://localhost:8080"
def test_normalize_url_external_gets_https(self):
"""Test that external hosts get https:// scheme."""
from local_deep_research.utilities.url_utils import normalize_url
with patch(
"local_deep_research.utilities.url_utils.is_private_ip",
return_value=False,
):
result = normalize_url("example.com:443")
assert result == "https://example.com:443"
def test_normalize_url_double_slash_prefix(self):
"""Test URL starting with // has prefix removed."""
from local_deep_research.utilities.url_utils import normalize_url
with patch(
"local_deep_research.utilities.url_utils.is_private_ip",
return_value=False,
):
result = normalize_url("//example.com")
assert result == "https://example.com"
def test_normalize_url_127_0_0_1(self):
"""Test that 127.0.0.1 gets http:// scheme."""
from local_deep_research.utilities.url_utils import normalize_url
with patch(
"local_deep_research.utilities.url_utils.is_private_ip",
return_value=True,
):
result = normalize_url("127.0.0.1:11434")
assert result == "http://127.0.0.1:11434"
def test_normalize_url_preserves_path(self):
"""Test that URL path is preserved."""
from local_deep_research.utilities.url_utils import normalize_url
result = normalize_url("http://example.com/api/v1/search")
assert result == "http://example.com/api/v1/search"
def test_normalize_url_preserves_query_string(self):
"""Test that query string is preserved."""
from local_deep_research.utilities.url_utils import normalize_url
result = normalize_url("http://example.com?q=test&page=1")
assert result == "http://example.com?q=test&page=1"
class TestCanonicalUrlKey:
"""Tests for canonical_url_key — dedup key generation, NOT for display."""
def setup_method(self):
# lru_cache persists across tests; clear to keep each assertion
# independent of prior test inputs.
from local_deep_research.utilities.url_utils import canonical_url_key
canonical_url_key.cache_clear()
def test_empty_returns_empty(self):
from local_deep_research.utilities.url_utils import canonical_url_key
assert canonical_url_key("") == ""
def test_lowercases_scheme_and_host_preserves_path_case(self):
from local_deep_research.utilities.url_utils import canonical_url_key
assert (
canonical_url_key("HTTPS://EXAMPLE.COM/Foo/Bar")
== "https://example.com/Foo/Bar"
)
def test_strips_fragment(self):
from local_deep_research.utilities.url_utils import canonical_url_key
assert (
canonical_url_key("https://example.com/page#section")
== "https://example.com/page"
)
def test_trailing_slash_normalized(self):
from local_deep_research.utilities.url_utils import canonical_url_key
assert canonical_url_key("https://example.com/p") == canonical_url_key(
"https://example.com/p/"
)
# Root path '/' is preserved (not stripped to empty).
assert canonical_url_key("https://example.com/").endswith("/")
def test_strips_utm_and_common_trackers(self):
from local_deep_research.utilities.url_utils import canonical_url_key
url = (
"https://example.com/p?"
"utm_source=x&UTM_Medium=y&utm_campaign=z&"
"fbclid=a&gclid=b&msclkid=c&yclid=d&dclid=e&gad_source=f&"
"mc_eid=g&mc_cid=h&ref_src=i&igshid=j&_ga=k&_gl=l"
)
assert canonical_url_key(url) == "https://example.com/p"
def test_keeps_non_tracking_query_params(self):
from local_deep_research.utilities.url_utils import canonical_url_key
# q, ref (GitHub branch), v (YouTube id), page, id are content-bearing.
assert (
canonical_url_key("https://github.com/o/r?ref=main&utm_source=x")
== "https://github.com/o/r?ref=main"
)
assert (
canonical_url_key(
"https://www.youtube.com/watch?v=abc123&utm_source=x"
)
== "https://www.youtube.com/watch?v=abc123"
)
assert (
canonical_url_key("https://example.com/p?q=hello&page=2")
== "https://example.com/p?q=hello&page=2"
)
def test_strips_userinfo(self):
from local_deep_research.utilities.url_utils import canonical_url_key
assert (
canonical_url_key("https://user:[email protected]/p")
== "https://example.com/p"
)
# Userinfo without colon.
assert (
canonical_url_key("https://[email protected]/p")
== "https://example.com/p"
)
def test_strips_default_ports(self):
from local_deep_research.utilities.url_utils import canonical_url_key
assert (
canonical_url_key("https://example.com:443/p")
== "https://example.com/p"
)
assert (
canonical_url_key("http://example.com:80/p")
== "http://example.com/p"
)
def test_preserves_nondefault_port(self):
from local_deep_research.utilities.url_utils import canonical_url_key
assert (
canonical_url_key("https://example.com:8443/p")
== "https://example.com:8443/p"
)
def test_ipv6_host_preserved(self):
from local_deep_research.utilities.url_utils import canonical_url_key
assert (
canonical_url_key("https://[::1]:8443/page")
== "https://[::1]:8443/page"
)
def test_ipv6_host_lowercased(self):
from local_deep_research.utilities.url_utils import canonical_url_key
assert (
canonical_url_key("https://[2001:DB8::1]/source")
== "https://[2001:db8::1]/source"
)
assert canonical_url_key(
"https://[2001:DB8::1]/source"
) == canonical_url_key("https://[2001:db8::1]/source")
assert (
canonical_url_key("https://[2001:DB8::1]:8443/page")
== "https://[2001:db8::1]:8443/page"
)
# Scoped IPv6: lowercase address portion while preserving case-sensitive zone ID (RFC 6874)
assert (
canonical_url_key("https://[fe80::ABCD%25eth0]/source")
== "https://[fe80::abcd%25eth0]/source"
)
# Address-case variants with the same zone deduplicate
assert canonical_url_key(
"https://[fe80::ABCD%25eth0]/source"
) == canonical_url_key("https://[fe80::abcd%25eth0]/source")
# Differently-cased zones remain distinct
assert canonical_url_key(
"https://[fe80::1%25eth0]/source"
) != canonical_url_key("https://[fe80::1%25Eth0]/source")
# Non-default port retained with scoped IPv6
assert (
canonical_url_key("https://[fe80::ABCD%25eth0]:8443/page")
== "https://[fe80::abcd%25eth0]:8443/page"
)
def test_library_route_keys_per_document(self):
"""All views of one library document share a dedup key.
``/library/document/<id>``, its ``/pdf`` view and its
``/chunks#chunk-N`` view are one source and belong on one
bibliography line. Keying on the full route fans a document out
whenever some chunks carry ``chunk_index`` metadata and others
don't (the ``/pdf`` fallback in ``_get_document_url``).
"""
from local_deep_research.utilities.url_utils import canonical_url_key
key = "/library/document/doc1"
assert canonical_url_key("/library/document/doc1") == key
assert canonical_url_key("/library/document/doc1/pdf") == key
assert canonical_url_key("/library/document/doc1/chunks") == key
assert canonical_url_key("/library/document/doc1/chunks#chunk-0") == key
assert (
canonical_url_key("/library/document/doc1/chunks#chunk-17") == key
)
assert canonical_url_key("/library/document/doc1/chunks/") == key
# Empty query normalizes away rather than producing a second key.
assert canonical_url_key("/library/document/doc1?") == key
assert canonical_url_key("/library/document/doc1?#chunk-1") == key
# The ``/lib/`` abbreviation denotes the SAME document (the fetch
# resolver accepts either spelling), so it must key with it —
# keying it under its own prefix fans one document across two
# bibliography entries, one of which renders a dead link because
# only ``/library/document/<id>`` is a registered route.
assert canonical_url_key("/lib/document/doc1/chunks#chunk-2") == key
# Percent-encoded ids resolve to the same document too.
assert canonical_url_key("/library/document/a%2Db") == (
canonical_url_key("/library/document/a-b")
)
def test_library_route_keeps_distinct_documents_apart(self):
from local_deep_research.utilities.url_utils import canonical_url_key
assert canonical_url_key(
"/library/document/doc1/chunks#chunk-0"
) != canonical_url_key("/library/document/doc2/chunks#chunk-0")
def test_library_display_url_preserves_anchor(self):
"""The dedup key drops ``#chunk-<n>``; the DISPLAY url must not, or
a citation no longer scrolls to the chunk it cites."""
from local_deep_research.utilities.url_utils import (
library_display_url,
)
assert (
library_display_url("/library/document/doc1/chunks#chunk-7")
== "/library/document/doc1/chunks#chunk-7"
)
# The ``/lib/`` abbreviation is displayed as the real route it
# denotes — ``/lib/document/...`` is not a registered Flask route,
# so rendering it verbatim yields a dead link.
assert library_display_url(" /lib/document/d/chunks#chunk-1 ") == (
"/library/document/d/chunks#chunk-1"
)
# Non-library URLs keep rendering their canonical form instead.
assert library_display_url("https://example.com/p#frag") is None
assert library_display_url("/docs/C#/intro.md") is None
assert library_display_url("") is None
def test_absolute_library_alias_keys_as_its_relative_route(self):
"""``https://library.document/<id>`` is an alias the agent emits and
the fetch resolver accepts, handing the ORIGINAL string back to be
registered as a citation link. It must key as the route it denotes,
or one document occupies two bibliography entries."""
from local_deep_research.utilities.url_utils import canonical_url_key
key = "/library/document/abc123"
assert canonical_url_key("https://library.document/abc123") == key
assert (
canonical_url_key("https://library.document/abc123/chunks") == key
)
assert (
canonical_url_key("https://library.document/abc123/chunks#chunk-7")
== key
)
assert canonical_url_key(
"/library/document/abc123/chunks#chunk-0"
) == canonical_url_key("https://library.document/abc123/chunks#chunk-7")
def test_chunk_id_alias_still_keys_and_displays_as_library_route(self):
from local_deep_research.utilities.url_utils import (
canonical_url_key,
library_display_url,
)
alias = "https://library.document/abc123/chunks?document_chunk_id=42#chunk-3"
assert canonical_url_key(alias) == "/library/document/abc123"
assert library_display_url(alias) == (
"/library/document/abc123/chunks#chunk-3"
)
def test_absolute_library_alias_rejects_lookalikes(self):
"""Only the exact https://library.document host normalizes."""
from local_deep_research.utilities.url_utils import (
canonical_url_key,
library_display_url,
)
for bad in (
"https://library.document.evil.test/abc/chunks",
# Suffix-bypass host: a naive ``netloc.endswith(...)`` check
# accepts this. Nothing else in the suite probes it.
"https://evil-library.document/abc",
"http://library.document/abc",
"https://example.com/library/document/abc",
"https://library.document",
# Shapes the resolver refuses; treating them as library routes
# lets a crafted URL occupy a real document's entry.
"https://library.document/abc/../../../admin/api/delete-all",
"https://library.document/abc/xyz/extra",
"https://library.document/abc?evil=1",
"https://library.document/abc/chunks?document_chunk_id=0#chunk-3",
"https://library.document/abc/chunks?document_chunk_id=1#invalid",
"https://library.document/abc/pdf?document_chunk_id=1#chunk-3",
"https://library.document/https://a.test/page",
):
# Assert the invariant, not inequality with one literal: a
# route-confusion bug producing some OTHER /library/document
# path would satisfy `!= "/library/document/abc"`.
assert not canonical_url_key(bad).startswith("/library/document"), (
bad
)
assert library_display_url(bad) is None, bad
def test_library_display_url_renders_alias_as_relative_route(self):
from local_deep_research.utilities.url_utils import (
library_display_url,
)
assert (
library_display_url(
"https://library.document/abc123/chunks#chunk-7"
)
== "/library/document/abc123/chunks#chunk-7"
)
def test_library_display_url_rejects_control_characters(self):
"""The display url is rendered verbatim into the Sources block, so a
crafted url with an embedded newline could forge extra lines — a
fake ``Collection:`` tag or a whole extra numbered citation. Such a
url falls back to the canonical key, which is rebuilt from parsed
parts and cannot carry one."""
from local_deep_research.utilities.url_utils import (
library_display_url,
)
forged = (
"/library/document/doc1/chunks#chunk-0\n"
" Collection: INJECTED\n"
"[999] Fake (source nr: 999)\n"
" URL: https://evil.example/x"
)
assert library_display_url(forged) is None
assert library_display_url("/library/document/a\tb/chunks") is None
assert library_display_url("/library/document/a\x00b/chunks") is None
assert library_display_url("/library/document/a\x7fb/chunks") is None
# A legitimate url is unaffected.
assert (
library_display_url("/library/document/doc1/chunks#chunk-0")
== "/library/document/doc1/chunks#chunk-0"
)
def test_non_library_root_relative_paths_left_alone(self):
"""The canonicalization is deliberately NOT generalized to every
root-relative path. A LangChain retriever sets a result's url from
its ``source`` metadata, commonly an absolute filesystem path;
treating those as routes merges genuinely distinct sources — worse
than the chunk fan-out this canonicalization exists to fix."""
from local_deep_research.utilities.url_utils import canonical_url_key
# A '#' inside a directory name must not truncate the path, and the
# two files must not collapse onto one another.
assert canonical_url_key("/docs/C#/intro.md") == "/docs/C#/intro.md"
assert canonical_url_key("/docs/C#/intro.md") != canonical_url_key(
"/docs/C#/advanced.md"
)
# SPA-style client routes stay distinct.
assert canonical_url_key("/app#/users") != canonical_url_key(
"/app#/settings"
)
# A directory and a same-named file stay distinct.
assert canonical_url_key("/mnt/notes/") != canonical_url_key(
"/mnt/notes"
)
# urlsplit() deletes embedded tabs; the manual split must not, or
# "/a\tb" would collide with a real "/ab".
assert canonical_url_key("/a\tb") == "/a\tb"
def test_invalid_url_returned_stripped(self):
from local_deep_research.utilities.url_utils import canonical_url_key
assert canonical_url_key(" not a url ") == "not a url"
def test_mailto_returns_stripped(self):
from local_deep_research.utilities.url_utils import canonical_url_key
assert canonical_url_key("mailto:[email protected]") == "mailto:[email protected]"
def test_protocol_relative_returns_stripped(self):
from local_deep_research.utilities.url_utils import canonical_url_key
# Protocol-relative URL has no scheme — canonicalization is ambiguous,
# so we fall back to the stripped input.
assert canonical_url_key("//example.com/path") == "//example.com/path"
def test_combined_normalization(self):
from local_deep_research.utilities.url_utils import canonical_url_key
assert (
canonical_url_key(
"HTTPS://User:[email protected]:443/Path/?utm_source=x&q=1#frag"
)
== "https://example.com/Path?q=1"
)
class TestContentWebCacheKey:
"""Content-fetch identity — NOT bibliography dedup (PR #6759).
``content_web_cache_key`` shares the safe normalisations (scheme/host
case, default ports, fragments, tracking params — accepting over-merge
on sites varying content per tracking param) but preserves the
two distinctions the bibliography key collapses: a non-root trailing
slash (``/article`` vs ``/article/`` serve different bytes) and URL
userinfo (different credentials select different views).
"""
def setup_method(self):
from local_deep_research.utilities.url_utils import (
content_web_cache_key,
)
content_web_cache_key.cache_clear()
def test_trailing_slash_stays_distinct(self):
from local_deep_research.utilities.url_utils import (
canonical_url_key,
content_web_cache_key,
)
canonical_url_key.cache_clear()
# Bibliography still collapses (unchanged behaviour).
assert canonical_url_key("https://example.com/p") == canonical_url_key(
"https://example.com/p/"
)
# Content must not.
assert content_web_cache_key(
"https://example.com/p"
) != content_web_cache_key("https://example.com/p/")
assert content_web_cache_key(
"http://127.0.0.1:9/article"
) != content_web_cache_key("http://127.0.0.1:9/article/")
def test_userinfo_stays_distinct(self):
from local_deep_research.utilities.url_utils import (
canonical_url_key,
content_web_cache_key,
)
canonical_url_key.cache_clear()
assert canonical_url_key(
"https://user:[email protected]/p"
) == canonical_url_key("https://example.com/p")
assert content_web_cache_key(
"https://user:[email protected]/p"
) != content_web_cache_key("https://example.com/p")
# Different credentials are different views.
assert content_web_cache_key(
"https://[email protected]/p"
) != content_web_cache_key("https://[email protected]/p")
# Host still lowercased around preserved userinfo.
assert content_web_cache_key(
"https://User:[email protected]/p"
) == content_web_cache_key("https://User:[email protected]/p")
def test_query_and_shared_normalisations(self):
from local_deep_research.utilities.url_utils import (
content_web_cache_key,
)
# Semantically meaningful queries stay apart.
assert content_web_cache_key(
"https://example.com/p?a=1"
) != content_web_cache_key("https://example.com/p?a=2")
assert content_web_cache_key(
"https://example.com/p?foo=1"
) != content_web_cache_key("https://example.com/p")
# Tracking params still collapse (content-free, as in bibliography;
# accepts over-merge on sites that vary content per param).
assert content_web_cache_key(
"https://example.com/p?utm_source=x&foo=1"
) == content_web_cache_key("https://example.com/p?foo=1")
# Default ports name one endpoint; scheme/host case names one host;
# tracking params collapse. Fragments are preserved: client-routed SPAs
# render distinct views per fragment under JS rendering.
assert content_web_cache_key(
"https://example.com/p#section"
) != content_web_cache_key("https://example.com/p")
assert content_web_cache_key(
"https://spa.example.com/app#/route-alpha"
) != content_web_cache_key("https://spa.example.com/app#/route-beta")
assert content_web_cache_key(
"https://example.com:443/p"
) == content_web_cache_key("https://example.com/p")
assert content_web_cache_key(
"HTTPS://EXAMPLE.COM/p"
) == content_web_cache_key("https://example.com/p")
# Path case still selects distinct resources.
assert content_web_cache_key(
"https://example.com/P"
) != content_web_cache_key("https://example.com/p")
# Non-default ports stay distinct, incl. with userinfo + IPv6.
assert content_web_cache_key(
"https://example.com:8443/p"
) != content_web_cache_key("https://example.com/p")
assert content_web_cache_key(
"http://user:pass@[::1]:8080/p"
) != content_web_cache_key("http://[::1]:8080/p")
class TestLibraryRouteHardening:
"""Round-2 follow-up (#5685): the relative and absolute spellings of a
library route must enforce identical rules."""
def test_relative_route_rejects_path_traversal(self):
"""``_normalize_library_alias``'s docstring describes exactly this
attack, but only the ABSOLUTE alias was validated: the relative
path was accepted on a bare ``startswith`` prefix. The crafted
entry then keyed as the real document (inheriting its title and
citation numbers) while displaying — and linking to — ``/admin``."""
from local_deep_research.utilities.url_utils import (
canonical_url_key,
library_display_url,
)
crafted = "/library/document/7/../../../admin/wipe#chunk-1"
assert canonical_url_key(crafted) != canonical_url_key(
"/library/document/7"
)
assert library_display_url(crafted) is None
for bad in (
"/library/document/7/../8",
"/lib/document/7/../../../admin/wipe",
"/library/document/7/chunks/extra",
"/library/document/7/edit",
"/library/document/../admin",
):
assert library_display_url(bad) is None, bad
assert canonical_url_key(bad) != "/library/document/7", bad
def test_relative_route_rejects_unsafe_document_ids(self):
from local_deep_research.utilities.url_utils import (
library_display_url,
)
# Fullwidth digits pass ``str.isalnum()`` but are not ASCII, and
# would key separately from their percent-encoded form.
assert library_display_url("/library/document/7") is None
assert library_display_url("/library/document/a b") is None
assert library_display_url("/library/document/a%2Fb") is None
def test_lib_abbreviation_and_percent_encoding_share_one_key(self):
from local_deep_research.utilities.url_utils import (
canonical_url_key,
library_display_url,
)
key = "/library/document/a-b"
for spelling in (
"/library/document/a-b",
"/lib/document/a-b",
"/library/document/a%2Db",
"/lib/document/a%2Db/chunks#chunk-1",
"https://library.document/a-b/pdf",
):
assert canonical_url_key(spelling) == key, spelling
assert library_display_url(spelling).startswith(key), spelling
def test_control_char_route_does_not_merge_into_a_real_document(self):
"""A crafted route carrying a control character must not key as the
real ``/library/document/<id>`` it was prefixed with.
Truncating the doc-id segment at the first control character
discards the payload, but keys the crafted URL as document
``doc1`` — so the forged source silently joins that document's
bibliography entry and inherits its title and citation number.
Refusing the route keeps the two sources apart; the payload is
neutralised at render time by
``search_utilities._sanitize_sources_field`` instead.
"""
from local_deep_research.utilities.search_utilities import (
format_links_to_markdown,
)
from local_deep_research.utilities.url_utils import (
canonical_url_key,
library_display_url,
)
forged = (
"/library/document/doc1/chunks#chunk-0\n"
" Collection: admin-secrets\n"
"[99] Forged Source (source nr: 99)\n"
" URL: https://evil.example/pwn"
)
assert canonical_url_key(forged) != "/library/document/doc1"
assert library_display_url(forged) is None
rendered = format_links_to_markdown(
[
{"title": "Real", "url": "/library/document/doc1", "index": 1},
{"title": "Crafted", "url": forged, "index": 2},
]
)
# Two distinct sources, and the payload never starts a line: the
# forged "[99]" entry and "Collection:" tag land mid-line.
entry_lines = [
line for line in rendered.splitlines() if line.startswith("[")
]
assert len(entry_lines) == 2, entry_lines
url_lines = [
line
for line in rendered.splitlines()
if line.startswith(" URL: ")
]
assert len(url_lines) == 2, url_lines
assert not any(
line.startswith(" Collection:") for line in rendered.splitlines()
)
# A tab cannot break a line and must still render as itself, so
# "/a<TAB>b" keeps its own key rather than colliding with "/ab".
assert canonical_url_key("/a\tb") == "/a\tb"
class TestLibraryDisplayFragmentValidation:
"""``library_display_url`` re-emitted ANY fragment verbatim.
A library route carries no meaningful fragment other than a chunk
anchor, and the fetch path hands this function a string the agent
typed, so the verbatim re-emission built citation URLs whose anchor
could not name a chunk — bypassing the ``MAX_CHUNK_INDEX``/format
contract that ``preferred_chunk_display`` enforces on the same value.
"""
def test_invalid_chunk_fragment_is_dropped_not_echoed(self):
from local_deep_research.utilities.url_utils import (
library_display_url,
)
route = "/library/document/doc1/chunks"
for bad in (
"#chunk-1'\"><img src=x>",
"#chunk-" + "9" * 40, # over MAX_CHUNK_INDEX, huge digit run
"#chunk-1000001", # just over MAX_CHUNK_INDEX
"#chunk-not-a-number",
"#chunk-007", # leading zeros: not the shape the producer emits
"#chunk-٧", # non-ASCII digit int() would accept
"#section-intro",
"#",
):
assert library_display_url(route + bad) == route, bad
def test_fragment_predicate_rejects_a_trailing_newline(self):
"""``$`` matches before a single trailing newline even without
``re.MULTILINE``, so ``chunk-1\\n`` passed the shared predicate."""
from local_deep_research.utilities.url_utils import (
is_valid_chunk_fragment,
)
assert is_valid_chunk_fragment("chunk-1")
for bad in ("chunk-1\n", "chunk-1\r", "chunk-1\n\n", "chunk-1 "):
assert not is_valid_chunk_fragment(bad), repr(bad)
def test_valid_chunk_anchor_still_survives(self):
"""The drop must not take the anchor with it — that anchor is the
entire point of a chunk-targeted citation."""
from local_deep_research.utilities.url_utils import (
library_display_url,
)
for n in (0, 7, 1000000): # 0 and MAX_CHUNK_INDEX are both valid
route = f"/library/document/doc1/chunks#chunk-{n}"
assert library_display_url(route) == route
def test_display_and_store_paths_share_one_predicate(self):
"""``preferred_chunk_display`` is the stricter question asked of the
same value; neither may be the looser of the pair."""
from local_deep_research.utilities.url_utils import (
is_valid_chunk_fragment,
library_display_url,
preferred_chunk_display,
)
for frag in ("chunk-0", "chunk-9", "chunk-1000000"):
assert is_valid_chunk_fragment(frag)
for frag in ("chunk-1000001", "chunk-01", "chunk-x", "", "chunk-"):
assert not is_valid_chunk_fragment(frag)
# Whenever display keeps a fragment, the store path accepts it too.
for raw in (
"/library/document/d/chunks#chunk-3",
"/library/document/d/chunks#chunk-1000001",
"/library/document/d/chunks#bogus",
"/library/document/d/chunks",
):
display = library_display_url(raw)
assert display is not None
assert ("#" in display) == (
preferred_chunk_display(raw) is not None
)
class TestLibraryAliasPortHandling:
"""``_normalize_library_alias`` accepts ``:443`` and rejects every other
port. Both halves are real behaviour and neither had a test — removing
the port check killed nothing in the suite."""
def test_default_port_names_the_same_document(self):
from local_deep_research.utilities.url_utils import (
canonical_url_key,
library_display_url,
)
canonical_url_key.cache_clear()
assert library_display_url("https://library.document:443/abc") == (
"/library/document/abc"
)
# ...so it does not fan out into a second bibliography entry.
assert canonical_url_key(
"https://library.document:443/abc"
) == canonical_url_key("/library/document/abc")
def test_a_non_default_port_is_not_rewritten_to_an_internal_route(self):
"""The rewrite makes a string into a same-origin internal link, so
the accepted shape is deliberately narrow."""
from local_deep_research.utilities.url_utils import (
library_display_url,
)
for port in (8080, 80, 1):
assert (
library_display_url(f"https://library.document:{port}/abc")
is None
), port