Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

33.4 KB
Raw Blame
"""
Tests for utilities/url_utils.py

Tests cover:
- URL normalization
- Scheme handling
- Private IP detection
"""

import pytest
from unittest.mock import patch


class TestNormalizeUrl:
    """Tests for normalize_url function."""

    def test_normalize_url_with_http_scheme(self):
        """Test URL with http:// scheme is returned unchanged."""
        from local_deep_research.utilities.url_utils import normalize_url

        result = normalize_url("http://example.com")
        assert result == "http://example.com"

    def test_normalize_url_with_https_scheme(self):
        """Test URL with https:// scheme is returned unchanged."""
        from local_deep_research.utilities.url_utils import normalize_url

        result = normalize_url("https://example.com")
        assert result == "https://example.com"

    def test_normalize_url_with_http_scheme_and_port(self):
        """Test URL with http:// scheme and port is returned unchanged."""
        from local_deep_research.utilities.url_utils import normalize_url

        result = normalize_url("http://example.com:8080")
        assert result == "http://example.com:8080"

    def test_normalize_url_empty_raises_error(self):
        """Test that empty URL raises ValueError."""
        from local_deep_research.utilities.url_utils import normalize_url

        with pytest.raises(ValueError) as exc_info:
            normalize_url("")
        assert "empty" in str(exc_info.value).lower()

    def test_normalize_url_strips_whitespace(self):
        """Test that whitespace is stripped."""
        from local_deep_research.utilities.url_utils import normalize_url

        result = normalize_url("  http://example.com  ")
        assert result == "http://example.com"

    def test_normalize_url_malformed_http_colon(self):
        """Test URL with malformed http: (missing //) is fixed."""
        from local_deep_research.utilities.url_utils import normalize_url

        result = normalize_url("http:example.com")
        assert result == "http://example.com"

    def test_normalize_url_malformed_https_colon(self):
        """Test URL with malformed https: (missing //) is fixed."""
        from local_deep_research.utilities.url_utils import normalize_url

        result = normalize_url("https:example.com")
        assert result == "https://example.com"

    def test_normalize_url_localhost_gets_http(self):
        """Test that localhost gets http:// scheme."""
        from local_deep_research.utilities.url_utils import normalize_url

        with patch(
            "local_deep_research.utilities.url_utils.is_private_ip",
            return_value=True,
        ):
            result = normalize_url("localhost:8080")
            assert result == "http://localhost:8080"

    def test_normalize_url_external_gets_https(self):
        """Test that external hosts get https:// scheme."""
        from local_deep_research.utilities.url_utils import normalize_url

        with patch(
            "local_deep_research.utilities.url_utils.is_private_ip",
            return_value=False,
        ):
            result = normalize_url("example.com:443")
            assert result == "https://example.com:443"

    def test_normalize_url_double_slash_prefix(self):
        """Test URL starting with // has prefix removed."""
        from local_deep_research.utilities.url_utils import normalize_url

        with patch(
            "local_deep_research.utilities.url_utils.is_private_ip",
            return_value=False,
        ):
            result = normalize_url("//example.com")
            assert result == "https://example.com"

    def test_normalize_url_127_0_0_1(self):
        """Test that 127.0.0.1 gets http:// scheme."""
        from local_deep_research.utilities.url_utils import normalize_url

        with patch(
            "local_deep_research.utilities.url_utils.is_private_ip",
            return_value=True,
        ):
            result = normalize_url("127.0.0.1:11434")
            assert result == "http://127.0.0.1:11434"

    def test_normalize_url_preserves_path(self):
        """Test that URL path is preserved."""
        from local_deep_research.utilities.url_utils import normalize_url

        result = normalize_url("http://example.com/api/v1/search")
        assert result == "http://example.com/api/v1/search"

    def test_normalize_url_preserves_query_string(self):
        """Test that query string is preserved."""
        from local_deep_research.utilities.url_utils import normalize_url

        result = normalize_url("http://example.com?q=test&page=1")
        assert result == "http://example.com?q=test&page=1"


class TestCanonicalUrlKey:
    """Tests for canonical_url_key — dedup key generation, NOT for display."""

    def setup_method(self):
        # lru_cache persists across tests; clear to keep each assertion
        # independent of prior test inputs.
        from local_deep_research.utilities.url_utils import canonical_url_key

        canonical_url_key.cache_clear()

    def test_empty_returns_empty(self):
        from local_deep_research.utilities.url_utils import canonical_url_key

        assert canonical_url_key("") == ""

    def test_lowercases_scheme_and_host_preserves_path_case(self):
        from local_deep_research.utilities.url_utils import canonical_url_key

        assert (
            canonical_url_key("HTTPS://EXAMPLE.COM/Foo/Bar")
            == "https://example.com/Foo/Bar"
        )

    def test_strips_fragment(self):
        from local_deep_research.utilities.url_utils import canonical_url_key

        assert (
            canonical_url_key("https://example.com/page#section")
            == "https://example.com/page"
        )

    def test_trailing_slash_normalized(self):
        from local_deep_research.utilities.url_utils import canonical_url_key

        assert canonical_url_key("https://example.com/p") == canonical_url_key(
            "https://example.com/p/"
        )
        # Root path '/' is preserved (not stripped to empty).
        assert canonical_url_key("https://example.com/").endswith("/")

    def test_strips_utm_and_common_trackers(self):
        from local_deep_research.utilities.url_utils import canonical_url_key

        url = (
            "https://example.com/p?"
            "utm_source=x&UTM_Medium=y&utm_campaign=z&"
            "fbclid=a&gclid=b&msclkid=c&yclid=d&dclid=e&gad_source=f&"
            "mc_eid=g&mc_cid=h&ref_src=i&igshid=j&_ga=k&_gl=l"
        )
        assert canonical_url_key(url) == "https://example.com/p"

    def test_keeps_non_tracking_query_params(self):
        from local_deep_research.utilities.url_utils import canonical_url_key

        # q, ref (GitHub branch), v (YouTube id), page, id are content-bearing.
        assert (
            canonical_url_key("https://github.com/o/r?ref=main&utm_source=x")
            == "https://github.com/o/r?ref=main"
        )
        assert (
            canonical_url_key(
                "https://www.youtube.com/watch?v=abc123&utm_source=x"
            )
            == "https://www.youtube.com/watch?v=abc123"
        )
        assert (
            canonical_url_key("https://example.com/p?q=hello&page=2")
            == "https://example.com/p?q=hello&page=2"
        )

    def test_strips_userinfo(self):
        from local_deep_research.utilities.url_utils import canonical_url_key

        assert (
            canonical_url_key("https://user:[email protected]/p")
            == "https://example.com/p"
        )
        # Userinfo without colon.
        assert (
            canonical_url_key("https://[email protected]/p")
            == "https://example.com/p"
        )

    def test_strips_default_ports(self):
        from local_deep_research.utilities.url_utils import canonical_url_key

        assert (
            canonical_url_key("https://example.com:443/p")
            == "https://example.com/p"
        )
        assert (
            canonical_url_key("http://example.com:80/p")
            == "http://example.com/p"
        )

    def test_preserves_nondefault_port(self):
        from local_deep_research.utilities.url_utils import canonical_url_key

        assert (
            canonical_url_key("https://example.com:8443/p")
            == "https://example.com:8443/p"
        )

    def test_ipv6_host_preserved(self):
        from local_deep_research.utilities.url_utils import canonical_url_key

        assert (
            canonical_url_key("https://[::1]:8443/page")
            == "https://[::1]:8443/page"
        )

    def test_ipv6_host_lowercased(self):
        from local_deep_research.utilities.url_utils import canonical_url_key

        assert (
            canonical_url_key("https://[2001:DB8::1]/source")
            == "https://[2001:db8::1]/source"
        )
        assert canonical_url_key(
            "https://[2001:DB8::1]/source"
        ) == canonical_url_key("https://[2001:db8::1]/source")
        assert (
            canonical_url_key("https://[2001:DB8::1]:8443/page")
            == "https://[2001:db8::1]:8443/page"
        )

        # Scoped IPv6: lowercase address portion while preserving case-sensitive zone ID (RFC 6874)
        assert (
            canonical_url_key("https://[fe80::ABCD%25eth0]/source")
            == "https://[fe80::abcd%25eth0]/source"
        )
        # Address-case variants with the same zone deduplicate
        assert canonical_url_key(
            "https://[fe80::ABCD%25eth0]/source"
        ) == canonical_url_key("https://[fe80::abcd%25eth0]/source")
        # Differently-cased zones remain distinct
        assert canonical_url_key(
            "https://[fe80::1%25eth0]/source"
        ) != canonical_url_key("https://[fe80::1%25Eth0]/source")
        # Non-default port retained with scoped IPv6
        assert (
            canonical_url_key("https://[fe80::ABCD%25eth0]:8443/page")
            == "https://[fe80::abcd%25eth0]:8443/page"
        )

    def test_library_route_keys_per_document(self):
        """All views of one library document share a dedup key.

        ``/library/document/<id>``, its ``/pdf`` view and its
        ``/chunks#chunk-N`` view are one source and belong on one
        bibliography line. Keying on the full route fans a document out
        whenever some chunks carry ``chunk_index`` metadata and others
        don't (the ``/pdf`` fallback in ``_get_document_url``).
        """
        from local_deep_research.utilities.url_utils import canonical_url_key

        key = "/library/document/doc1"
        assert canonical_url_key("/library/document/doc1") == key
        assert canonical_url_key("/library/document/doc1/pdf") == key
        assert canonical_url_key("/library/document/doc1/chunks") == key
        assert canonical_url_key("/library/document/doc1/chunks#chunk-0") == key
        assert (
            canonical_url_key("/library/document/doc1/chunks#chunk-17") == key
        )
        assert canonical_url_key("/library/document/doc1/chunks/") == key
        # Empty query normalizes away rather than producing a second key.
        assert canonical_url_key("/library/document/doc1?") == key
        assert canonical_url_key("/library/document/doc1?#chunk-1") == key
        # The ``/lib/`` abbreviation denotes the SAME document (the fetch
        # resolver accepts either spelling), so it must key with it —
        # keying it under its own prefix fans one document across two
        # bibliography entries, one of which renders a dead link because
        # only ``/library/document/<id>`` is a registered route.
        assert canonical_url_key("/lib/document/doc1/chunks#chunk-2") == key
        # Percent-encoded ids resolve to the same document too.
        assert canonical_url_key("/library/document/a%2Db") == (
            canonical_url_key("/library/document/a-b")
        )

    def test_library_route_keeps_distinct_documents_apart(self):
        from local_deep_research.utilities.url_utils import canonical_url_key

        assert canonical_url_key(
            "/library/document/doc1/chunks#chunk-0"
        ) != canonical_url_key("/library/document/doc2/chunks#chunk-0")

    def test_library_display_url_preserves_anchor(self):
        """The dedup key drops ``#chunk-<n>``; the DISPLAY url must not, or
        a citation no longer scrolls to the chunk it cites."""
        from local_deep_research.utilities.url_utils import (
            library_display_url,
        )

        assert (
            library_display_url("/library/document/doc1/chunks#chunk-7")
            == "/library/document/doc1/chunks#chunk-7"
        )
        # The ``/lib/`` abbreviation is displayed as the real route it
        # denotes — ``/lib/document/...`` is not a registered Flask route,
        # so rendering it verbatim yields a dead link.
        assert library_display_url("  /lib/document/d/chunks#chunk-1  ") == (
            "/library/document/d/chunks#chunk-1"
        )
        # Non-library URLs keep rendering their canonical form instead.
        assert library_display_url("https://example.com/p#frag") is None
        assert library_display_url("/docs/C#/intro.md") is None
        assert library_display_url("") is None

    def test_absolute_library_alias_keys_as_its_relative_route(self):
        """``https://library.document/<id>`` is an alias the agent emits and
        the fetch resolver accepts, handing the ORIGINAL string back to be
        registered as a citation link. It must key as the route it denotes,
        or one document occupies two bibliography entries."""
        from local_deep_research.utilities.url_utils import canonical_url_key

        key = "/library/document/abc123"
        assert canonical_url_key("https://library.document/abc123") == key
        assert (
            canonical_url_key("https://library.document/abc123/chunks") == key
        )
        assert (
            canonical_url_key("https://library.document/abc123/chunks#chunk-7")
            == key
        )
        assert canonical_url_key(
            "/library/document/abc123/chunks#chunk-0"
        ) == canonical_url_key("https://library.document/abc123/chunks#chunk-7")

    def test_chunk_id_alias_still_keys_and_displays_as_library_route(self):
        from local_deep_research.utilities.url_utils import (
            canonical_url_key,
            library_display_url,
        )

        alias = "https://library.document/abc123/chunks?document_chunk_id=42#chunk-3"
        assert canonical_url_key(alias) == "/library/document/abc123"
        assert library_display_url(alias) == (
            "/library/document/abc123/chunks#chunk-3"
        )

    def test_absolute_library_alias_rejects_lookalikes(self):
        """Only the exact https://library.document host normalizes."""
        from local_deep_research.utilities.url_utils import (
            canonical_url_key,
            library_display_url,
        )

        for bad in (
            "https://library.document.evil.test/abc/chunks",
            # Suffix-bypass host: a naive ``netloc.endswith(...)`` check
            # accepts this. Nothing else in the suite probes it.
            "https://evil-library.document/abc",
            "http://library.document/abc",
            "https://example.com/library/document/abc",
            "https://library.document",
            # Shapes the resolver refuses; treating them as library routes
            # lets a crafted URL occupy a real document's entry.
            "https://library.document/abc/../../../admin/api/delete-all",
            "https://library.document/abc/xyz/extra",
            "https://library.document/abc?evil=1",
            "https://library.document/abc/chunks?document_chunk_id=0#chunk-3",
            "https://library.document/abc/chunks?document_chunk_id=1#invalid",
            "https://library.document/abc/pdf?document_chunk_id=1#chunk-3",
            "https://library.document/https://a.test/page",
        ):
            # Assert the invariant, not inequality with one literal: a
            # route-confusion bug producing some OTHER /library/document
            # path would satisfy `!= "/library/document/abc"`.
            assert not canonical_url_key(bad).startswith("/library/document"), (
                bad
            )
            assert library_display_url(bad) is None, bad

    def test_library_display_url_renders_alias_as_relative_route(self):
        from local_deep_research.utilities.url_utils import (
            library_display_url,
        )

        assert (
            library_display_url(
                "https://library.document/abc123/chunks#chunk-7"
            )
            == "/library/document/abc123/chunks#chunk-7"
        )

    def test_library_display_url_rejects_control_characters(self):
        """The display url is rendered verbatim into the Sources block, so a
        crafted url with an embedded newline could forge extra lines — a
        fake ``Collection:`` tag or a whole extra numbered citation. Such a
        url falls back to the canonical key, which is rebuilt from parsed
        parts and cannot carry one."""
        from local_deep_research.utilities.url_utils import (
            library_display_url,
        )

        forged = (
            "/library/document/doc1/chunks#chunk-0\n"
            "   Collection: INJECTED\n"
            "[999] Fake (source nr: 999)\n"
            "   URL: https://evil.example/x"
        )
        assert library_display_url(forged) is None
        assert library_display_url("/library/document/a\tb/chunks") is None
        assert library_display_url("/library/document/a\x00b/chunks") is None
        assert library_display_url("/library/document/a\x7fb/chunks") is None
        # A legitimate url is unaffected.
        assert (
            library_display_url("/library/document/doc1/chunks#chunk-0")
            == "/library/document/doc1/chunks#chunk-0"
        )

    def test_non_library_root_relative_paths_left_alone(self):
        """The canonicalization is deliberately NOT generalized to every
        root-relative path. A LangChain retriever sets a result's url from
        its ``source`` metadata, commonly an absolute filesystem path;
        treating those as routes merges genuinely distinct sources — worse
        than the chunk fan-out this canonicalization exists to fix."""
        from local_deep_research.utilities.url_utils import canonical_url_key

        # A '#' inside a directory name must not truncate the path, and the
        # two files must not collapse onto one another.
        assert canonical_url_key("/docs/C#/intro.md") == "/docs/C#/intro.md"
        assert canonical_url_key("/docs/C#/intro.md") != canonical_url_key(
            "/docs/C#/advanced.md"
        )
        # SPA-style client routes stay distinct.
        assert canonical_url_key("/app#/users") != canonical_url_key(
            "/app#/settings"
        )
        # A directory and a same-named file stay distinct.
        assert canonical_url_key("/mnt/notes/") != canonical_url_key(
            "/mnt/notes"
        )
        # urlsplit() deletes embedded tabs; the manual split must not, or
        # "/a\tb" would collide with a real "/ab".
        assert canonical_url_key("/a\tb") == "/a\tb"

    def test_invalid_url_returned_stripped(self):
        from local_deep_research.utilities.url_utils import canonical_url_key

        assert canonical_url_key("  not a url  ") == "not a url"

    def test_mailto_returns_stripped(self):
        from local_deep_research.utilities.url_utils import canonical_url_key

        assert canonical_url_key("mailto:[email protected]") == "mailto:[email protected]"

    def test_protocol_relative_returns_stripped(self):
        from local_deep_research.utilities.url_utils import canonical_url_key

        # Protocol-relative URL has no scheme — canonicalization is ambiguous,
        # so we fall back to the stripped input.
        assert canonical_url_key("//example.com/path") == "//example.com/path"

    def test_combined_normalization(self):
        from local_deep_research.utilities.url_utils import canonical_url_key

        assert (
            canonical_url_key(
                "HTTPS://User:[email protected]:443/Path/?utm_source=x&q=1#frag"
            )
            == "https://example.com/Path?q=1"
        )


class TestContentWebCacheKey:
    """Content-fetch identity — NOT bibliography dedup (PR #6759).

    ``content_web_cache_key`` shares the safe normalisations (scheme/host
    case, default ports, fragments, tracking params — accepting over-merge
    on sites varying content per tracking param) but preserves the
    two distinctions the bibliography key collapses: a non-root trailing
    slash (``/article`` vs ``/article/`` serve different bytes) and URL
    userinfo (different credentials select different views).
    """

    def setup_method(self):
        from local_deep_research.utilities.url_utils import (
            content_web_cache_key,
        )

        content_web_cache_key.cache_clear()

    def test_trailing_slash_stays_distinct(self):
        from local_deep_research.utilities.url_utils import (
            canonical_url_key,
            content_web_cache_key,
        )

        canonical_url_key.cache_clear()
        # Bibliography still collapses (unchanged behaviour).
        assert canonical_url_key("https://example.com/p") == canonical_url_key(
            "https://example.com/p/"
        )
        # Content must not.
        assert content_web_cache_key(
            "https://example.com/p"
        ) != content_web_cache_key("https://example.com/p/")
        assert content_web_cache_key(
            "http://127.0.0.1:9/article"
        ) != content_web_cache_key("http://127.0.0.1:9/article/")

    def test_userinfo_stays_distinct(self):
        from local_deep_research.utilities.url_utils import (
            canonical_url_key,
            content_web_cache_key,
        )

        canonical_url_key.cache_clear()
        assert canonical_url_key(
            "https://user:[email protected]/p"
        ) == canonical_url_key("https://example.com/p")
        assert content_web_cache_key(
            "https://user:[email protected]/p"
        ) != content_web_cache_key("https://example.com/p")
        # Different credentials are different views.
        assert content_web_cache_key(
            "https://[email protected]/p"
        ) != content_web_cache_key("https://[email protected]/p")
        # Host still lowercased around preserved userinfo.
        assert content_web_cache_key(
            "https://User:[email protected]/p"
        ) == content_web_cache_key("https://User:[email protected]/p")

    def test_query_and_shared_normalisations(self):
        from local_deep_research.utilities.url_utils import (
            content_web_cache_key,
        )

        # Semantically meaningful queries stay apart.
        assert content_web_cache_key(
            "https://example.com/p?a=1"
        ) != content_web_cache_key("https://example.com/p?a=2")
        assert content_web_cache_key(
            "https://example.com/p?foo=1"
        ) != content_web_cache_key("https://example.com/p")
        # Tracking params still collapse (content-free, as in bibliography;
        # accepts over-merge on sites that vary content per param).
        assert content_web_cache_key(
            "https://example.com/p?utm_source=x&foo=1"
        ) == content_web_cache_key("https://example.com/p?foo=1")
        # Default ports name one endpoint; scheme/host case names one host;
        # tracking params collapse. Fragments are preserved: client-routed SPAs
        # render distinct views per fragment under JS rendering.
        assert content_web_cache_key(
            "https://example.com/p#section"
        ) != content_web_cache_key("https://example.com/p")
        assert content_web_cache_key(
            "https://spa.example.com/app#/route-alpha"
        ) != content_web_cache_key("https://spa.example.com/app#/route-beta")
        assert content_web_cache_key(
            "https://example.com:443/p"
        ) == content_web_cache_key("https://example.com/p")
        assert content_web_cache_key(
            "HTTPS://EXAMPLE.COM/p"
        ) == content_web_cache_key("https://example.com/p")
        # Path case still selects distinct resources.
        assert content_web_cache_key(
            "https://example.com/P"
        ) != content_web_cache_key("https://example.com/p")
        # Non-default ports stay distinct, incl. with userinfo + IPv6.
        assert content_web_cache_key(
            "https://example.com:8443/p"
        ) != content_web_cache_key("https://example.com/p")
        assert content_web_cache_key(
            "http://user:pass@[::1]:8080/p"
        ) != content_web_cache_key("http://[::1]:8080/p")


class TestLibraryRouteHardening:
    """Round-2 follow-up (#5685): the relative and absolute spellings of a
    library route must enforce identical rules."""

    def test_relative_route_rejects_path_traversal(self):
        """``_normalize_library_alias``'s docstring describes exactly this
        attack, but only the ABSOLUTE alias was validated: the relative
        path was accepted on a bare ``startswith`` prefix. The crafted
        entry then keyed as the real document (inheriting its title and
        citation numbers) while displaying — and linking to — ``/admin``."""
        from local_deep_research.utilities.url_utils import (
            canonical_url_key,
            library_display_url,
        )

        crafted = "/library/document/7/../../../admin/wipe#chunk-1"
        assert canonical_url_key(crafted) != canonical_url_key(
            "/library/document/7"
        )
        assert library_display_url(crafted) is None
        for bad in (
            "/library/document/7/../8",
            "/lib/document/7/../../../admin/wipe",
            "/library/document/7/chunks/extra",
            "/library/document/7/edit",
            "/library/document/../admin",
        ):
            assert library_display_url(bad) is None, bad
            assert canonical_url_key(bad) != "/library/document/7", bad

    def test_relative_route_rejects_unsafe_document_ids(self):
        from local_deep_research.utilities.url_utils import (
            library_display_url,
        )

        # Fullwidth digits pass ``str.isalnum()`` but are not ASCII, and
        # would key separately from their percent-encoded form.
        assert library_display_url("/library/document/7") is None
        assert library_display_url("/library/document/a b") is None
        assert library_display_url("/library/document/a%2Fb") is None

    def test_lib_abbreviation_and_percent_encoding_share_one_key(self):
        from local_deep_research.utilities.url_utils import (
            canonical_url_key,
            library_display_url,
        )

        key = "/library/document/a-b"
        for spelling in (
            "/library/document/a-b",
            "/lib/document/a-b",
            "/library/document/a%2Db",
            "/lib/document/a%2Db/chunks#chunk-1",
            "https://library.document/a-b/pdf",
        ):
            assert canonical_url_key(spelling) == key, spelling
            assert library_display_url(spelling).startswith(key), spelling

    def test_control_char_route_does_not_merge_into_a_real_document(self):
        """A crafted route carrying a control character must not key as the
        real ``/library/document/<id>`` it was prefixed with.

        Truncating the doc-id segment at the first control character
        discards the payload, but keys the crafted URL as document
        ``doc1`` — so the forged source silently joins that document's
        bibliography entry and inherits its title and citation number.
        Refusing the route keeps the two sources apart; the payload is
        neutralised at render time by
        ``search_utilities._sanitize_sources_field`` instead.
        """
        from local_deep_research.utilities.search_utilities import (
            format_links_to_markdown,
        )
        from local_deep_research.utilities.url_utils import (
            canonical_url_key,
            library_display_url,
        )

        forged = (
            "/library/document/doc1/chunks#chunk-0\n"
            "   Collection: admin-secrets\n"
            "[99] Forged Source (source nr: 99)\n"
            "   URL: https://evil.example/pwn"
        )
        assert canonical_url_key(forged) != "/library/document/doc1"
        assert library_display_url(forged) is None

        rendered = format_links_to_markdown(
            [
                {"title": "Real", "url": "/library/document/doc1", "index": 1},
                {"title": "Crafted", "url": forged, "index": 2},
            ]
        )
        # Two distinct sources, and the payload never starts a line: the
        # forged "[99]" entry and "Collection:" tag land mid-line.
        entry_lines = [
            line for line in rendered.splitlines() if line.startswith("[")
        ]
        assert len(entry_lines) == 2, entry_lines
        url_lines = [
            line
            for line in rendered.splitlines()
            if line.startswith("   URL: ")
        ]
        assert len(url_lines) == 2, url_lines
        assert not any(
            line.startswith("   Collection:") for line in rendered.splitlines()
        )

        # A tab cannot break a line and must still render as itself, so
        # "/a<TAB>b" keeps its own key rather than colliding with "/ab".
        assert canonical_url_key("/a\tb") == "/a\tb"


class TestLibraryDisplayFragmentValidation:
    """``library_display_url`` re-emitted ANY fragment verbatim.

    A library route carries no meaningful fragment other than a chunk
    anchor, and the fetch path hands this function a string the agent
    typed, so the verbatim re-emission built citation URLs whose anchor
    could not name a chunk — bypassing the ``MAX_CHUNK_INDEX``/format
    contract that ``preferred_chunk_display`` enforces on the same value.
    """

    def test_invalid_chunk_fragment_is_dropped_not_echoed(self):
        from local_deep_research.utilities.url_utils import (
            library_display_url,
        )

        route = "/library/document/doc1/chunks"
        for bad in (
            "#chunk-1'\"><img src=x>",
            "#chunk-" + "9" * 40,  # over MAX_CHUNK_INDEX, huge digit run
            "#chunk-1000001",  # just over MAX_CHUNK_INDEX
            "#chunk-not-a-number",
            "#chunk-007",  # leading zeros: not the shape the producer emits
            "#chunk-٧",  # non-ASCII digit int() would accept
            "#section-intro",
            "#",
        ):
            assert library_display_url(route + bad) == route, bad

    def test_fragment_predicate_rejects_a_trailing_newline(self):
        """``$`` matches before a single trailing newline even without
        ``re.MULTILINE``, so ``chunk-1\\n`` passed the shared predicate."""
        from local_deep_research.utilities.url_utils import (
            is_valid_chunk_fragment,
        )

        assert is_valid_chunk_fragment("chunk-1")
        for bad in ("chunk-1\n", "chunk-1\r", "chunk-1\n\n", "chunk-1 "):
            assert not is_valid_chunk_fragment(bad), repr(bad)

    def test_valid_chunk_anchor_still_survives(self):
        """The drop must not take the anchor with it — that anchor is the
        entire point of a chunk-targeted citation."""
        from local_deep_research.utilities.url_utils import (
            library_display_url,
        )

        for n in (0, 7, 1000000):  # 0 and MAX_CHUNK_INDEX are both valid
            route = f"/library/document/doc1/chunks#chunk-{n}"
            assert library_display_url(route) == route

    def test_display_and_store_paths_share_one_predicate(self):
        """``preferred_chunk_display`` is the stricter question asked of the
        same value; neither may be the looser of the pair."""
        from local_deep_research.utilities.url_utils import (
            is_valid_chunk_fragment,
            library_display_url,
            preferred_chunk_display,
        )

        for frag in ("chunk-0", "chunk-9", "chunk-1000000"):
            assert is_valid_chunk_fragment(frag)
        for frag in ("chunk-1000001", "chunk-01", "chunk-x", "", "chunk-"):
            assert not is_valid_chunk_fragment(frag)

        # Whenever display keeps a fragment, the store path accepts it too.
        for raw in (
            "/library/document/d/chunks#chunk-3",
            "/library/document/d/chunks#chunk-1000001",
            "/library/document/d/chunks#bogus",
            "/library/document/d/chunks",
        ):
            display = library_display_url(raw)
            assert display is not None
            assert ("#" in display) == (
                preferred_chunk_display(raw) is not None
            )


class TestLibraryAliasPortHandling:
    """``_normalize_library_alias`` accepts ``:443`` and rejects every other
    port. Both halves are real behaviour and neither had a test — removing
    the port check killed nothing in the suite."""

    def test_default_port_names_the_same_document(self):
        from local_deep_research.utilities.url_utils import (
            canonical_url_key,
            library_display_url,
        )

        canonical_url_key.cache_clear()
        assert library_display_url("https://library.document:443/abc") == (
            "/library/document/abc"
        )
        # ...so it does not fan out into a second bibliography entry.
        assert canonical_url_key(
            "https://library.document:443/abc"
        ) == canonical_url_key("/library/document/abc")

    def test_a_non_default_port_is_not_rewritten_to_an_internal_route(self):
        """The rewrite makes a string into a same-origin internal link, so
        the accepted shape is deliberately narrow."""
        from local_deep_research.utilities.url_utils import (
            library_display_url,
        )

        for port in (8080, 80, 1):
            assert (
                library_display_url(f"https://library.document:{port}/abc")
                is None
            ), port