Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

22.1 KB
Raw Blame
"""
Tests for URL classifier.
"""

import pytest

from local_deep_research.content_fetcher.fetcher import ContentFetcher
from local_deep_research.content_fetcher.url_classifier import (
    URLClassifier,
    URLType,
)
from local_deep_research.utilities import arxiv as arxiv_utils


class TestArxivFamilyURL:
    @pytest.mark.parametrize(
        "url",
        (
            "https://ARXIV.ORG./not-an-arxiv-id",
            "http://export.ArXiV.OrG./any/path",
            "https://AR5IV.ORG./not-an-arxiv-id",
            "https://render.Ar5iV.OrG./any/path",
        ),
    )
    def test_http_hosts_are_owned_independently_of_path(self, url):
        # Given an HTTP(S) URL on an exact or subdomain arXiv-family host
        # When ownership is checked
        owned = arxiv_utils.is_arxiv_family_url(url)

        # Then case, DNS trailing dots, and path grammar do not affect ownership
        assert owned is True

    @pytest.mark.parametrize(
        "url",
        (
            "ftp://arxiv.org/2301.12345",
            "https://arxiv.org.example.com/not-an-arxiv-id",
            "https://ar5iv.org.example.com./not-an-arxiv-id",
            "https://example.com/path/arxiv.org/not-an-arxiv-id",
            "https:///arxiv.org/not-an-arxiv-id",
            "https://[arxiv.org",
            "https://arxiv.org:not-a-port/not-an-arxiv-id",
            "https://evil.example\\@arxiv.org/abs/2301.12345",
            "https://evil.example\\.arxiv.org/abs/2301.12345",
        ),
    )
    def test_non_http_malformed_and_lookalike_urls_are_unowned(self, url):
        # Given an unsupported, malformed, or lookalike URL
        # When ownership is checked
        owned = arxiv_utils.is_arxiv_family_url(url)

        # Then it is not claimed by the arXiv-family route
        assert owned is False

    def test_malformed_port_has_no_strict_arxiv_identifier(self):
        # Given an arXiv-looking URL with a malformed port
        url = "https://arxiv.org:not-a-port/abs/2301.12345"

        # When strict identifier extraction runs
        identifier = arxiv_utils.extract_arxiv_id(url)

        # Then malformed URL syntax is rejected
        assert identifier is None


class TestURLClassifier:
    """Test URL classification."""

    def test_classify_arxiv_abs(self):
        """Test arXiv abstract URL classification."""
        url = "https://arxiv.org/abs/2301.12345"
        assert URLClassifier.classify(url) == URLType.ARXIV

    def test_classify_arxiv_pdf(self):
        """Test arXiv PDF URL classification."""
        url = "https://arxiv.org/pdf/2301.12345.pdf"
        assert URLClassifier.classify(url) == URLType.ARXIV

    def test_classify_arxiv_html(self):
        """Test arXiv HTML URL classification."""
        url = "https://arxiv.org/html/2301.12345"
        assert URLClassifier.classify(url) == URLType.ARXIV

    def test_classify_arxiv_old_format(self):
        """Test old arXiv format URL classification."""
        url = "https://arxiv.org/abs/cond-mat/0501234"
        assert URLClassifier.classify(url) == URLType.ARXIV

    def test_classify_ar5iv(self):
        """Test ar5iv (HTML arXiv) URL classification."""
        url = "https://ar5iv.org/abs/2301.12345"
        assert URLClassifier.classify(url) == URLType.ARXIV

    def test_classify_ar5iv_direct_path(self):
        url = "https://ar5iv.org/math.AG/0601001v2"
        assert URLClassifier.classify(url) == URLType.ARXIV

    def test_classify_malformed_ar5iv_path_is_not_arxiv(self):
        url = "https://ar5iv.org/not-an-arxiv-identifier"
        assert URLClassifier.classify(url) == URLType.HTML

    def test_trailing_dot_arxiv_family_host_is_arxiv_for_a_paper_path(self):
        url = "https://AR5IV.ORG./2301.12345v2"
        assert URLClassifier.classify(url) == URLType.ARXIV

    @pytest.mark.parametrize(
        "url",
        (
            "https://arxiv.org/about",
            "https://arxiv.org/help",
            "https://arxiv.org/list/cs.AI",
            "https://arxiv.org/blog",
        ),
    )
    def test_classify_arxiv_non_paper_pages_as_html(self, url):
        assert URLClassifier.classify(url) == URLType.HTML

    @pytest.mark.parametrize(
        "url",
        (
            "https://arxiv.org/abs/2301.12345",
            "https://arxiv.org/pdf/2301.12345.pdf",
            "https://arxiv.org/html/2301.12345",
            "https://ar5iv.org/2301.12345",
            "https://ar5iv.org/abs/2301.12345",
        ),
    )
    def test_classify_paper_shaped_arxiv_urls_as_arxiv(self, url):
        # Routing follows the paper identifier, not the host: only URLs
        # that name one paper are terminal arXiv (main's merged policy).
        assert URLClassifier.classify(url) == URLType.ARXIV

    @pytest.mark.parametrize(
        "url",
        (
            "https://arxiv.org/aboutness",
            "https://ar5iv.org/about",
            "https://AR5IV.ORG./not-an-arxiv-id",
            "https://ARXIV.ORG./not-an-arxiv-id",
            "https://EXPORT.ARXIV.ORG./not-an-arxiv-id",
            "https://arxiv.org/list/cs.AI/recent",
            "https://arxiv.org/a/bengio_y_1",
            "https://info.arxiv.org/help/api/tou.html",
            "https://blog.arxiv.org/2024/01/01/something/",
            "https://arxiv.org/",
            "https://static.arxiv.org/js/x.js",
            "https://arxiv.org/abs/not-an-id",
        ),
    )
    def test_family_host_without_a_paper_identifier_is_not_arxiv(self, url):
        # Given a page on an arXiv-family host that names no paper
        # When it is classified
        # Then it is not ARXIV: ARXIV ownership is terminal on every fetch
        # entry point, and the arXiv downloader has no source for these, so
        # claiming them would turn a readable page into nothing at all
        assert URLClassifier.classify(url) == URLType.HTML

    @pytest.mark.parametrize(
        "url",
        (
            "https://arxiv.org/ftp/arxiv/papers/2301/2301.12345.pdf",
            "https://arxiv.org/ftp/cond-mat/papers/0501/0501001.pdf",
        ),
    )
    def test_legacy_ftp_pdf_layout_is_not_arxiv(self, url):
        # Given arXiv's legacy PDF-only submission layout
        # When it is classified
        # Then it is not ARXIV. classify does not drive DownloadService's
        # downloader loop -- can_handle does, and
        # test_legacy_ftp_pdf_is_left_to_the_direct_pdf_downloader covers
        # that -- but ARXIV here would make ContentFetcher and the extraction
        # pipeline terminal on a URL no arXiv source can serve. HTML rather
        # than PDF because _is_pdf_url declines every arxiv.org URL outright.
        assert URLClassifier.classify(url) == URLType.HTML

    def test_ar5iv_text_in_unrelated_path_is_not_arxiv(self):
        url = "https://example.com/ar5iv.org/not-an-arxiv-identifier"
        assert URLClassifier.classify(url) == URLType.HTML

    @pytest.mark.parametrize(
        "url",
        (
            "https://arxiv.org.example.com/2301.12345",
            "https://ar5iv.org.example.com/2301.12345",
        ),
    )
    def test_arxiv_family_lookalike_hostname_is_not_arxiv(self, url):
        assert URLClassifier.classify(url) == URLType.HTML

    def test_trailing_dot_ar5iv_lookalike_hostname_is_not_arxiv(self):
        url = "https://ar5iv.org.example.com./2301.12345"
        assert URLClassifier.classify(url) == URLType.HTML

    def test_arxiv_text_in_unrelated_path_is_not_arxiv(self):
        url = "https://example.com/arxiv.org/abs/2301.12345"
        assert URLClassifier.classify(url) == URLType.HTML

    def test_classify_pubmed(self):
        """Test PubMed URL classification."""
        url = "https://pubmed.ncbi.nlm.nih.gov/12345678"
        assert URLClassifier.classify(url) == URLType.PUBMED

    def test_classify_pubmed_old_format(self):
        """Test old PubMed URL format."""
        url = "https://www.ncbi.nlm.nih.gov/pubmed/12345678"
        assert URLClassifier.classify(url) == URLType.PUBMED

    def test_classify_pmc(self):
        """Test PubMed Central URL classification."""
        url = "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC1234567"
        assert URLClassifier.classify(url) == URLType.PMC

    def test_classify_europe_pmc(self):
        """Test Europe PMC URL classification."""
        url = "https://europepmc.org/article/PMC/1234567"
        assert URLClassifier.classify(url) == URLType.PMC

    def test_classify_semantic_scholar(self):
        """Test Semantic Scholar URL classification."""
        # Semantic Scholar uses 40-char hex paper IDs - this is a fake test ID, not a secret
        url = "https://www.semanticscholar.org/paper/Attention-Is-All-You-Need-Vaswani-Shazeer/a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2"  # DevSkim: ignore DS173237
        assert URLClassifier.classify(url) == URLType.SEMANTIC_SCHOLAR

    def test_classify_semantic_scholar_api(self):
        """Test Semantic Scholar API URL classification."""
        url = "https://api.semanticscholar.org/v1/paper/abcd1234"
        assert URLClassifier.classify(url) == URLType.SEMANTIC_SCHOLAR

    def test_classify_biorxiv(self):
        """Test bioRxiv URL classification."""
        url = "https://www.biorxiv.org/content/10.1101/2021.01.01.12345v1"
        assert URLClassifier.classify(url) == URLType.BIORXIV

    def test_classify_medrxiv(self):
        """Test medRxiv URL classification."""
        url = "https://www.medrxiv.org/content/10.1101/2021.01.01.12345v1"
        assert URLClassifier.classify(url) == URLType.MEDRXIV

    def test_classify_doi(self):
        """Test DOI URL classification."""
        url = "https://doi.org/10.1234/example.2021.01"
        assert URLClassifier.classify(url) == URLType.DOI

    def test_classify_dx_doi(self):
        """Test dx.doi.org URL classification."""
        url = "https://dx.doi.org/10.1234/example"
        assert URLClassifier.classify(url) == URLType.DOI

    def test_classify_direct_pdf(self):
        """Test direct PDF URL classification."""
        url = "https://example.com/papers/document.pdf"
        assert URLClassifier.classify(url) == URLType.PDF

    def test_classify_html_generic(self):
        """Test generic HTML URL classification."""
        url = "https://www.example.com/article/123"
        assert URLClassifier.classify(url) == URLType.HTML

    def test_classify_wikipedia(self):
        """Test Wikipedia URL (should be HTML)."""
        url = "https://en.wikipedia.org/wiki/Machine_learning"
        assert URLClassifier.classify(url) == URLType.HTML

    def test_classify_github(self):
        """Test GitHub URL (should be HTML)."""
        url = "https://github.com/anthropics/claude-code"
        assert URLClassifier.classify(url) == URLType.HTML

    def test_classify_news_site(self):
        """Test news site URL (should be HTML)."""
        url = "https://news.ycombinator.com/item?id=12345"
        assert URLClassifier.classify(url) == URLType.HTML


class TestAcademicSourceHostAnchoring:
    """The academic sources are matched on the URL's host and path (#6835).

    Searching for them anywhere in the URL string also found them in another
    site's path or query and inside look-alike hosts. Such a URL was then
    routed to the source's downloader and cited under its tag.
    """

    @pytest.mark.parametrize(
        "url",
        (
            "https://evil.example/doi.org/10.1234/x",
            "https://evil.example/?x=doi.org/10.1234/x",
            "https://evil.example/redirect?u=pubmed.ncbi.nlm.nih.gov/12345",
            "https://evil.example/ncbi.nlm.nih.gov/pubmed/12345",
            "https://evil.example/ncbi.nlm.nih.gov/pmc/articles/PMC123",
            "https://evil.example/europepmc.org/article/PMC/123",
            "https://evil.example/semanticscholar.org/paper/abc",
            "https://evil.example/api.semanticscholar.org/graph/v1",
            "https://evil.example/biorxiv.org/content/10.1101/x",
            "https://evil.example/medrxiv.org/content/10.1101/x",
        ),
    )
    def test_a_source_domain_in_another_sites_path_or_query_is_html(self, url):
        assert URLClassifier.classify(url) == URLType.HTML

    @pytest.mark.parametrize(
        "url",
        (
            "https://fakedoi.org/10.1234/x",
            "https://notbiorxiv.org/content/10.1101/x",
            "https://evilmedrxiv.org/content/10.1101/x",
            "https://mysemanticscholar.org/paper/abc",
            "https://fakeeuropepmc.org/article/PMC/123",
            "https://pubmed.ncbi.nlm.nih.gov.evil.example/12345",
        ),
    )
    def test_a_lookalike_host_is_html(self, url):
        assert URLClassifier.classify(url) == URLType.HTML

    @pytest.mark.parametrize(
        "url,url_type",
        (
            ("https://pubmed.ncbi.nlm.nih.gov:443/12345", URLType.PUBMED),
            ("https://DOI.ORG/10.1234/x", URLType.DOI),
            ("https://doi.org./10.1234/x", URLType.DOI),
            ("doi.org/10.1234/x", URLType.DOI),
            ("https://europepmc.org/articles/PMC123", URLType.PMC),
            ("https://ncbi.nlm.nih.gov/pubmed/12345", URLType.PUBMED),
            ("https:/doi.org/10.1234/x", URLType.DOI),
            ("https:doi.org/10.1234/x", URLType.DOI),
        ),
    )
    def test_the_real_host_keeps_its_type_however_it_is_written(
        self, url, url_type
    ):
        assert URLClassifier.classify(url) == url_type

    def test_the_path_pattern_is_a_prefix(self):
        # A bioRxiv collection page, not an article: /content/ appears only
        # further down its path.
        url = "https://connect.biorxiv.org/relate/content/181"
        assert URLClassifier.classify(url) == URLType.HTML

    @pytest.mark.parametrize(
        "url",
        (
            "https://evil.example\\@doi.org/10.1234/x",
            "https://evil.example\\.doi.org/10.1234/x",
            "https://evil.example\\@pubmed.ncbi.nlm.nih.gov/12345",
            "https://evil.example\\@ncbi.nlm.nih.gov/pmc/articles/PMC123",
            "https://evil.example\\@europepmc.org/article/PMC/123",
            "https://evil.example\\@semanticscholar.org/paper/abc",
            "https://evil.example\\@biorxiv.org/content/10.1101/x",
            "https://evil.example\\@medrxiv.org/content/10.1101/x",
            "https://evil.example\\@arxiv.org/abs/2301.12345",
            "https://evil.example\\.arxiv.org/abs/2301.12345",
        ),
    )
    def test_a_backslash_in_the_authority_earns_no_source(self, url):
        # urllib reads the host after the backslash; browsers and requests
        # end the authority at it and connect to evil.example.
        assert URLClassifier.classify(url) == URLType.HTML

    @pytest.mark.parametrize(
        "url",
        (
            "https://pubmed.ncbi.nlm.nih.gov:443.evil.example/12345",
            "https://doi.org:99999/10.1234/x",
        ),
    )
    def test_a_port_that_is_not_a_port_earns_no_source(self, url):
        assert URLClassifier.classify(url) == URLType.HTML

    def test_an_authority_urllib_cannot_parse_is_html(self):
        assert URLClassifier.classify("[evil.example/x") == URLType.HTML

    @pytest.mark.parametrize(
        "url",
        (
            "https://evil.example/ncbi.nlm.nih.gov/pmc/articles/PMC123",
            "https://evil.example\\@ncbi.nlm.nih.gov/pmc/articles/PMC123",
        ),
    )
    def test_the_fetcher_does_not_route_an_impostor_to_a_source(self, url):
        # Routed as PMC, this link's content was the real article PMC123:
        # PubMedDownloader reads the PMC id from anywhere in the URL and asks
        # Europe PMC for that article's text.
        info = ContentFetcher().get_url_info(url)

        assert info["url_type"] == "html"
        assert info["extracted_id"] is None


class TestURLClassifierExtractID:
    """Test ID extraction from URLs."""

    def test_extract_arxiv_id_new_format(self):
        """Test arXiv ID extraction (new format)."""
        url = "https://arxiv.org/abs/2301.12345"
        assert URLClassifier.extract_id(url) == "2301.12345"

    def test_extract_arxiv_id_with_version(self):
        """Test arXiv ID extraction with version."""
        url = "https://arxiv.org/abs/2301.12345v2"
        assert URLClassifier.extract_id(url) == "2301.12345v2"

    def test_extract_arxiv_id_old_format(self):
        """Test arXiv ID extraction (old format)."""
        url = "https://arxiv.org/abs/cond-mat/0501234"
        assert URLClassifier.extract_id(url) == "cond-mat/0501234"

    def test_extract_arxiv_dotted_legacy_id_from_pdf(self):
        url = "https://export.arxiv.org/pdf/math.AG/0601001v3.pdf?q=1#page=2"
        assert URLClassifier.extract_id(url) == "math.AG/0601001v3"

    def test_extract_arxiv_id_rejects_unrelated_host(self):
        url = "https://example.com/arxiv.org/abs/2301.12345v2"
        assert URLClassifier.extract_id(url, URLType.ARXIV) is None

    def test_extract_pubmed_id(self):
        """Test PubMed ID extraction."""
        url = "https://pubmed.ncbi.nlm.nih.gov/12345678/"
        assert URLClassifier.extract_id(url) == "12345678"

    def test_extract_pmc_id(self):
        """Test PMC ID extraction."""
        url = "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC1234567/"
        assert URLClassifier.extract_id(url) == "PMC1234567"

    def test_extract_semantic_scholar_id(self):
        """Test Semantic Scholar ID extraction."""
        # Semantic Scholar uses 40-char hex paper IDs - this is a fake test ID, not a secret
        url = "https://www.semanticscholar.org/paper/Title/a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2"  # DevSkim: ignore DS173237
        extracted = URLClassifier.extract_id(url)
        assert extracted == "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2"

    def test_extract_doi(self):
        """Test DOI extraction."""
        url = "https://doi.org/10.1234/example.2021"
        assert URLClassifier.extract_id(url) == "10.1234/example.2021"

    def test_extract_id_unknown_url(self):
        """Test ID extraction from unknown URL type."""
        url = "https://example.com/article/123"
        assert URLClassifier.extract_id(url) is None


class TestURLClassifierSourceName:
    """Test human-readable source names."""

    def test_source_name_arxiv(self):
        """Test arXiv source name."""
        assert URLClassifier.get_source_name(URLType.ARXIV) == "arXiv"

    def test_source_name_pubmed(self):
        """Test PubMed source name."""
        assert URLClassifier.get_source_name(URLType.PUBMED) == "PubMed"

    def test_source_name_pmc(self):
        """Test PMC source name."""
        assert URLClassifier.get_source_name(URLType.PMC) == "PubMed Central"

    def test_source_name_semantic_scholar(self):
        """Test Semantic Scholar source name."""
        assert (
            URLClassifier.get_source_name(URLType.SEMANTIC_SCHOLAR)
            == "Semantic Scholar"
        )

    def test_source_name_html(self):
        """Test HTML source name."""
        assert URLClassifier.get_source_name(URLType.HTML) == "Web Page"

    def test_source_name_pdf(self):
        """Test PDF source name."""
        assert URLClassifier.get_source_name(URLType.PDF) == "PDF"


class TestURLClassifierEdgeCases:
    """Test edge cases and malformed URLs."""

    def test_classify_empty_url(self):
        """Test empty URL defaults to HTML."""
        assert URLClassifier.classify("") == URLType.HTML

    def test_classify_invalid_url(self):
        """Test invalid URL defaults to HTML."""
        assert URLClassifier.classify("not-a-valid-url") == URLType.HTML

    def test_classify_case_insensitive(self):
        """Test URL classification is case-insensitive."""
        url = "https://ARXIV.ORG/abs/2301.12345"
        assert URLClassifier.classify(url) == URLType.ARXIV

    def test_classify_with_query_params(self):
        """Test URL with query parameters."""
        url = "https://arxiv.org/abs/2301.12345?ref=homepage"
        assert URLClassifier.classify(url) == URLType.ARXIV

    def test_classify_with_fragment(self):
        """Test URL with fragment."""
        url = "https://pubmed.ncbi.nlm.nih.gov/12345678#abstract"
        assert URLClassifier.classify(url) == URLType.PUBMED


class TestURLClassifierSecurity:
    """Test security-related URL classification."""

    def test_javascript_url_rejected(self):
        """Test that javascript: URLs are classified as INVALID."""
        url = "javascript:alert('XSS')"
        assert URLClassifier.classify(url) == URLType.INVALID

    def test_data_url_rejected(self):
        """Test that data: URLs are classified as INVALID."""
        url = "data:text/html,<script>alert('XSS')</script>"
        assert URLClassifier.classify(url) == URLType.INVALID

    def test_file_url_rejected(self):
        """Test that file: URLs are classified as INVALID."""
        url = "file:///etc/passwd"
        assert URLClassifier.classify(url) == URLType.INVALID

    def test_vbscript_url_rejected(self):
        """Test that vbscript: URLs are classified as INVALID."""
        url = "vbscript:msgbox('XSS')"
        assert URLClassifier.classify(url) == URLType.INVALID

    def test_about_url_rejected(self):
        """Test that about: URLs are classified as INVALID."""
        url = "about:blank"
        assert URLClassifier.classify(url) == URLType.INVALID

    def test_ftp_url_rejected(self):
        """Test that ftp: URLs are classified as INVALID."""
        url = "ftp://ftp.example.com/file.txt"
        assert URLClassifier.classify(url) == URLType.INVALID

    def test_non_http_ar5iv_url_is_rejected(self):
        url = "ftp://ar5iv.org/2301.12345"
        assert URLClassifier.classify(url) == URLType.INVALID

    def test_http_url_accepted(self):
        """Test that http: URLs are accepted."""
        url = "http://example.com/page"
        assert URLClassifier.classify(url) == URLType.HTML

    def test_https_url_accepted(self):
        """Test that https: URLs are accepted."""
        url = "https://example.com/page"
        assert URLClassifier.classify(url) == URLType.HTML

    def test_invalid_source_name(self):
        """Test INVALID URL source name."""
        assert URLClassifier.get_source_name(URLType.INVALID) == "Invalid URL"

    def test_javascript_url_case_insensitive(self):
        """Test that JavaScript: URLs (mixed case) are rejected."""
        url = "JavaScript:alert('XSS')"
        assert URLClassifier.classify(url) == URLType.INVALID

    def test_whitespace_trimmed(self):
        """Test that URLs with whitespace are trimmed."""
        url = "  https://example.com/page  "
        assert URLClassifier.classify(url) == URLType.HTML