~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Tests for URL classifier.
"""
import pytest
from local_deep_research.content_fetcher.fetcher import ContentFetcher
from local_deep_research.content_fetcher.url_classifier import (
URLClassifier,
URLType,
)
from local_deep_research.utilities import arxiv as arxiv_utils
class TestArxivFamilyURL:
@pytest.mark.parametrize(
"url",
(
"https://ARXIV.ORG./not-an-arxiv-id",
"http://export.ArXiV.OrG./any/path",
"https://AR5IV.ORG./not-an-arxiv-id",
"https://render.Ar5iV.OrG./any/path",
),
)
def test_http_hosts_are_owned_independently_of_path(self, url):
# Given an HTTP(S) URL on an exact or subdomain arXiv-family host
# When ownership is checked
owned = arxiv_utils.is_arxiv_family_url(url)
# Then case, DNS trailing dots, and path grammar do not affect ownership
assert owned is True
@pytest.mark.parametrize(
"url",
(
"ftp://arxiv.org/2301.12345",
"https://arxiv.org.example.com/not-an-arxiv-id",
"https://ar5iv.org.example.com./not-an-arxiv-id",
"https://example.com/path/arxiv.org/not-an-arxiv-id",
"https:///arxiv.org/not-an-arxiv-id",
"https://[arxiv.org",
"https://arxiv.org:not-a-port/not-an-arxiv-id",
"https://evil.example\\@arxiv.org/abs/2301.12345",
"https://evil.example\\.arxiv.org/abs/2301.12345",
),
)
def test_non_http_malformed_and_lookalike_urls_are_unowned(self, url):
# Given an unsupported, malformed, or lookalike URL
# When ownership is checked
owned = arxiv_utils.is_arxiv_family_url(url)
# Then it is not claimed by the arXiv-family route
assert owned is False
def test_malformed_port_has_no_strict_arxiv_identifier(self):
# Given an arXiv-looking URL with a malformed port
url = "https://arxiv.org:not-a-port/abs/2301.12345"
# When strict identifier extraction runs
identifier = arxiv_utils.extract_arxiv_id(url)
# Then malformed URL syntax is rejected
assert identifier is None
class TestURLClassifier:
"""Test URL classification."""
def test_classify_arxiv_abs(self):
"""Test arXiv abstract URL classification."""
url = "https://arxiv.org/abs/2301.12345"
assert URLClassifier.classify(url) == URLType.ARXIV
def test_classify_arxiv_pdf(self):
"""Test arXiv PDF URL classification."""
url = "https://arxiv.org/pdf/2301.12345.pdf"
assert URLClassifier.classify(url) == URLType.ARXIV
def test_classify_arxiv_html(self):
"""Test arXiv HTML URL classification."""
url = "https://arxiv.org/html/2301.12345"
assert URLClassifier.classify(url) == URLType.ARXIV
def test_classify_arxiv_old_format(self):
"""Test old arXiv format URL classification."""
url = "https://arxiv.org/abs/cond-mat/0501234"
assert URLClassifier.classify(url) == URLType.ARXIV
def test_classify_ar5iv(self):
"""Test ar5iv (HTML arXiv) URL classification."""
url = "https://ar5iv.org/abs/2301.12345"
assert URLClassifier.classify(url) == URLType.ARXIV
def test_classify_ar5iv_direct_path(self):
url = "https://ar5iv.org/math.AG/0601001v2"
assert URLClassifier.classify(url) == URLType.ARXIV
def test_classify_malformed_ar5iv_path_is_not_arxiv(self):
url = "https://ar5iv.org/not-an-arxiv-identifier"
assert URLClassifier.classify(url) == URLType.HTML
def test_trailing_dot_arxiv_family_host_is_arxiv_for_a_paper_path(self):
url = "https://AR5IV.ORG./2301.12345v2"
assert URLClassifier.classify(url) == URLType.ARXIV
@pytest.mark.parametrize(
"url",
(
"https://arxiv.org/about",
"https://arxiv.org/help",
"https://arxiv.org/list/cs.AI",
"https://arxiv.org/blog",
),
)
def test_classify_arxiv_non_paper_pages_as_html(self, url):
assert URLClassifier.classify(url) == URLType.HTML
@pytest.mark.parametrize(
"url",
(
"https://arxiv.org/abs/2301.12345",
"https://arxiv.org/pdf/2301.12345.pdf",
"https://arxiv.org/html/2301.12345",
"https://ar5iv.org/2301.12345",
"https://ar5iv.org/abs/2301.12345",
),
)
def test_classify_paper_shaped_arxiv_urls_as_arxiv(self, url):
# Routing follows the paper identifier, not the host: only URLs
# that name one paper are terminal arXiv (main's merged policy).
assert URLClassifier.classify(url) == URLType.ARXIV
@pytest.mark.parametrize(
"url",
(
"https://arxiv.org/aboutness",
"https://ar5iv.org/about",
"https://AR5IV.ORG./not-an-arxiv-id",
"https://ARXIV.ORG./not-an-arxiv-id",
"https://EXPORT.ARXIV.ORG./not-an-arxiv-id",
"https://arxiv.org/list/cs.AI/recent",
"https://arxiv.org/a/bengio_y_1",
"https://info.arxiv.org/help/api/tou.html",
"https://blog.arxiv.org/2024/01/01/something/",
"https://arxiv.org/",
"https://static.arxiv.org/js/x.js",
"https://arxiv.org/abs/not-an-id",
),
)
def test_family_host_without_a_paper_identifier_is_not_arxiv(self, url):
# Given a page on an arXiv-family host that names no paper
# When it is classified
# Then it is not ARXIV: ARXIV ownership is terminal on every fetch
# entry point, and the arXiv downloader has no source for these, so
# claiming them would turn a readable page into nothing at all
assert URLClassifier.classify(url) == URLType.HTML
@pytest.mark.parametrize(
"url",
(
"https://arxiv.org/ftp/arxiv/papers/2301/2301.12345.pdf",
"https://arxiv.org/ftp/cond-mat/papers/0501/0501001.pdf",
),
)
def test_legacy_ftp_pdf_layout_is_not_arxiv(self, url):
# Given arXiv's legacy PDF-only submission layout
# When it is classified
# Then it is not ARXIV. classify does not drive DownloadService's
# downloader loop -- can_handle does, and
# test_legacy_ftp_pdf_is_left_to_the_direct_pdf_downloader covers
# that -- but ARXIV here would make ContentFetcher and the extraction
# pipeline terminal on a URL no arXiv source can serve. HTML rather
# than PDF because _is_pdf_url declines every arxiv.org URL outright.
assert URLClassifier.classify(url) == URLType.HTML
def test_ar5iv_text_in_unrelated_path_is_not_arxiv(self):
url = "https://example.com/ar5iv.org/not-an-arxiv-identifier"
assert URLClassifier.classify(url) == URLType.HTML
@pytest.mark.parametrize(
"url",
(
"https://arxiv.org.example.com/2301.12345",
"https://ar5iv.org.example.com/2301.12345",
),
)
def test_arxiv_family_lookalike_hostname_is_not_arxiv(self, url):
assert URLClassifier.classify(url) == URLType.HTML
def test_trailing_dot_ar5iv_lookalike_hostname_is_not_arxiv(self):
url = "https://ar5iv.org.example.com./2301.12345"
assert URLClassifier.classify(url) == URLType.HTML
def test_arxiv_text_in_unrelated_path_is_not_arxiv(self):
url = "https://example.com/arxiv.org/abs/2301.12345"
assert URLClassifier.classify(url) == URLType.HTML
def test_classify_pubmed(self):
"""Test PubMed URL classification."""
url = "https://pubmed.ncbi.nlm.nih.gov/12345678"
assert URLClassifier.classify(url) == URLType.PUBMED
def test_classify_pubmed_old_format(self):
"""Test old PubMed URL format."""
url = "https://www.ncbi.nlm.nih.gov/pubmed/12345678"
assert URLClassifier.classify(url) == URLType.PUBMED
def test_classify_pmc(self):
"""Test PubMed Central URL classification."""
url = "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC1234567"
assert URLClassifier.classify(url) == URLType.PMC
def test_classify_europe_pmc(self):
"""Test Europe PMC URL classification."""
url = "https://europepmc.org/article/PMC/1234567"
assert URLClassifier.classify(url) == URLType.PMC
def test_classify_semantic_scholar(self):
"""Test Semantic Scholar URL classification."""
# Semantic Scholar uses 40-char hex paper IDs - this is a fake test ID, not a secret
url = "https://www.semanticscholar.org/paper/Attention-Is-All-You-Need-Vaswani-Shazeer/a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2" # DevSkim: ignore DS173237
assert URLClassifier.classify(url) == URLType.SEMANTIC_SCHOLAR
def test_classify_semantic_scholar_api(self):
"""Test Semantic Scholar API URL classification."""
url = "https://api.semanticscholar.org/v1/paper/abcd1234"
assert URLClassifier.classify(url) == URLType.SEMANTIC_SCHOLAR
def test_classify_biorxiv(self):
"""Test bioRxiv URL classification."""
url = "https://www.biorxiv.org/content/10.1101/2021.01.01.12345v1"
assert URLClassifier.classify(url) == URLType.BIORXIV
def test_classify_medrxiv(self):
"""Test medRxiv URL classification."""
url = "https://www.medrxiv.org/content/10.1101/2021.01.01.12345v1"
assert URLClassifier.classify(url) == URLType.MEDRXIV
def test_classify_doi(self):
"""Test DOI URL classification."""
url = "https://doi.org/10.1234/example.2021.01"
assert URLClassifier.classify(url) == URLType.DOI
def test_classify_dx_doi(self):
"""Test dx.doi.org URL classification."""
url = "https://dx.doi.org/10.1234/example"
assert URLClassifier.classify(url) == URLType.DOI
def test_classify_direct_pdf(self):
"""Test direct PDF URL classification."""
url = "https://example.com/papers/document.pdf"
assert URLClassifier.classify(url) == URLType.PDF
def test_classify_html_generic(self):
"""Test generic HTML URL classification."""
url = "https://www.example.com/article/123"
assert URLClassifier.classify(url) == URLType.HTML
def test_classify_wikipedia(self):
"""Test Wikipedia URL (should be HTML)."""
url = "https://en.wikipedia.org/wiki/Machine_learning"
assert URLClassifier.classify(url) == URLType.HTML
def test_classify_github(self):
"""Test GitHub URL (should be HTML)."""
url = "https://github.com/anthropics/claude-code"
assert URLClassifier.classify(url) == URLType.HTML
def test_classify_news_site(self):
"""Test news site URL (should be HTML)."""
url = "https://news.ycombinator.com/item?id=12345"
assert URLClassifier.classify(url) == URLType.HTML
class TestAcademicSourceHostAnchoring:
"""The academic sources are matched on the URL's host and path (#6835).
Searching for them anywhere in the URL string also found them in another
site's path or query and inside look-alike hosts. Such a URL was then
routed to the source's downloader and cited under its tag.
"""
@pytest.mark.parametrize(
"url",
(
"https://evil.example/doi.org/10.1234/x",
"https://evil.example/?x=doi.org/10.1234/x",
"https://evil.example/redirect?u=pubmed.ncbi.nlm.nih.gov/12345",
"https://evil.example/ncbi.nlm.nih.gov/pubmed/12345",
"https://evil.example/ncbi.nlm.nih.gov/pmc/articles/PMC123",
"https://evil.example/europepmc.org/article/PMC/123",
"https://evil.example/semanticscholar.org/paper/abc",
"https://evil.example/api.semanticscholar.org/graph/v1",
"https://evil.example/biorxiv.org/content/10.1101/x",
"https://evil.example/medrxiv.org/content/10.1101/x",
),
)
def test_a_source_domain_in_another_sites_path_or_query_is_html(self, url):
assert URLClassifier.classify(url) == URLType.HTML
@pytest.mark.parametrize(
"url",
(
"https://fakedoi.org/10.1234/x",
"https://notbiorxiv.org/content/10.1101/x",
"https://evilmedrxiv.org/content/10.1101/x",
"https://mysemanticscholar.org/paper/abc",
"https://fakeeuropepmc.org/article/PMC/123",
"https://pubmed.ncbi.nlm.nih.gov.evil.example/12345",
),
)
def test_a_lookalike_host_is_html(self, url):
assert URLClassifier.classify(url) == URLType.HTML
@pytest.mark.parametrize(
"url,url_type",
(
("https://pubmed.ncbi.nlm.nih.gov:443/12345", URLType.PUBMED),
("https://DOI.ORG/10.1234/x", URLType.DOI),
("https://doi.org./10.1234/x", URLType.DOI),
("doi.org/10.1234/x", URLType.DOI),
("https://europepmc.org/articles/PMC123", URLType.PMC),
("https://ncbi.nlm.nih.gov/pubmed/12345", URLType.PUBMED),
("https:/doi.org/10.1234/x", URLType.DOI),
("https:doi.org/10.1234/x", URLType.DOI),
),
)
def test_the_real_host_keeps_its_type_however_it_is_written(
self, url, url_type
):
assert URLClassifier.classify(url) == url_type
def test_the_path_pattern_is_a_prefix(self):
# A bioRxiv collection page, not an article: /content/ appears only
# further down its path.
url = "https://connect.biorxiv.org/relate/content/181"
assert URLClassifier.classify(url) == URLType.HTML
@pytest.mark.parametrize(
"url",
(
"https://evil.example\\@doi.org/10.1234/x",
"https://evil.example\\.doi.org/10.1234/x",
"https://evil.example\\@pubmed.ncbi.nlm.nih.gov/12345",
"https://evil.example\\@ncbi.nlm.nih.gov/pmc/articles/PMC123",
"https://evil.example\\@europepmc.org/article/PMC/123",
"https://evil.example\\@semanticscholar.org/paper/abc",
"https://evil.example\\@biorxiv.org/content/10.1101/x",
"https://evil.example\\@medrxiv.org/content/10.1101/x",
"https://evil.example\\@arxiv.org/abs/2301.12345",
"https://evil.example\\.arxiv.org/abs/2301.12345",
),
)
def test_a_backslash_in_the_authority_earns_no_source(self, url):
# urllib reads the host after the backslash; browsers and requests
# end the authority at it and connect to evil.example.
assert URLClassifier.classify(url) == URLType.HTML
@pytest.mark.parametrize(
"url",
(
"https://pubmed.ncbi.nlm.nih.gov:443.evil.example/12345",
"https://doi.org:99999/10.1234/x",
),
)
def test_a_port_that_is_not_a_port_earns_no_source(self, url):
assert URLClassifier.classify(url) == URLType.HTML
def test_an_authority_urllib_cannot_parse_is_html(self):
assert URLClassifier.classify("[evil.example/x") == URLType.HTML
@pytest.mark.parametrize(
"url",
(
"https://evil.example/ncbi.nlm.nih.gov/pmc/articles/PMC123",
"https://evil.example\\@ncbi.nlm.nih.gov/pmc/articles/PMC123",
),
)
def test_the_fetcher_does_not_route_an_impostor_to_a_source(self, url):
# Routed as PMC, this link's content was the real article PMC123:
# PubMedDownloader reads the PMC id from anywhere in the URL and asks
# Europe PMC for that article's text.
info = ContentFetcher().get_url_info(url)
assert info["url_type"] == "html"
assert info["extracted_id"] is None
class TestURLClassifierExtractID:
"""Test ID extraction from URLs."""
def test_extract_arxiv_id_new_format(self):
"""Test arXiv ID extraction (new format)."""
url = "https://arxiv.org/abs/2301.12345"
assert URLClassifier.extract_id(url) == "2301.12345"
def test_extract_arxiv_id_with_version(self):
"""Test arXiv ID extraction with version."""
url = "https://arxiv.org/abs/2301.12345v2"
assert URLClassifier.extract_id(url) == "2301.12345v2"
def test_extract_arxiv_id_old_format(self):
"""Test arXiv ID extraction (old format)."""
url = "https://arxiv.org/abs/cond-mat/0501234"
assert URLClassifier.extract_id(url) == "cond-mat/0501234"
def test_extract_arxiv_dotted_legacy_id_from_pdf(self):
url = "https://export.arxiv.org/pdf/math.AG/0601001v3.pdf?q=1#page=2"
assert URLClassifier.extract_id(url) == "math.AG/0601001v3"
def test_extract_arxiv_id_rejects_unrelated_host(self):
url = "https://example.com/arxiv.org/abs/2301.12345v2"
assert URLClassifier.extract_id(url, URLType.ARXIV) is None
def test_extract_pubmed_id(self):
"""Test PubMed ID extraction."""
url = "https://pubmed.ncbi.nlm.nih.gov/12345678/"
assert URLClassifier.extract_id(url) == "12345678"
def test_extract_pmc_id(self):
"""Test PMC ID extraction."""
url = "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC1234567/"
assert URLClassifier.extract_id(url) == "PMC1234567"
def test_extract_semantic_scholar_id(self):
"""Test Semantic Scholar ID extraction."""
# Semantic Scholar uses 40-char hex paper IDs - this is a fake test ID, not a secret
url = "https://www.semanticscholar.org/paper/Title/a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2" # DevSkim: ignore DS173237
extracted = URLClassifier.extract_id(url)
assert extracted == "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2"
def test_extract_doi(self):
"""Test DOI extraction."""
url = "https://doi.org/10.1234/example.2021"
assert URLClassifier.extract_id(url) == "10.1234/example.2021"
def test_extract_id_unknown_url(self):
"""Test ID extraction from unknown URL type."""
url = "https://example.com/article/123"
assert URLClassifier.extract_id(url) is None
class TestURLClassifierSourceName:
"""Test human-readable source names."""
def test_source_name_arxiv(self):
"""Test arXiv source name."""
assert URLClassifier.get_source_name(URLType.ARXIV) == "arXiv"
def test_source_name_pubmed(self):
"""Test PubMed source name."""
assert URLClassifier.get_source_name(URLType.PUBMED) == "PubMed"
def test_source_name_pmc(self):
"""Test PMC source name."""
assert URLClassifier.get_source_name(URLType.PMC) == "PubMed Central"
def test_source_name_semantic_scholar(self):
"""Test Semantic Scholar source name."""
assert (
URLClassifier.get_source_name(URLType.SEMANTIC_SCHOLAR)
== "Semantic Scholar"
)
def test_source_name_html(self):
"""Test HTML source name."""
assert URLClassifier.get_source_name(URLType.HTML) == "Web Page"
def test_source_name_pdf(self):
"""Test PDF source name."""
assert URLClassifier.get_source_name(URLType.PDF) == "PDF"
class TestURLClassifierEdgeCases:
"""Test edge cases and malformed URLs."""
def test_classify_empty_url(self):
"""Test empty URL defaults to HTML."""
assert URLClassifier.classify("") == URLType.HTML
def test_classify_invalid_url(self):
"""Test invalid URL defaults to HTML."""
assert URLClassifier.classify("not-a-valid-url") == URLType.HTML
def test_classify_case_insensitive(self):
"""Test URL classification is case-insensitive."""
url = "https://ARXIV.ORG/abs/2301.12345"
assert URLClassifier.classify(url) == URLType.ARXIV
def test_classify_with_query_params(self):
"""Test URL with query parameters."""
url = "https://arxiv.org/abs/2301.12345?ref=homepage"
assert URLClassifier.classify(url) == URLType.ARXIV
def test_classify_with_fragment(self):
"""Test URL with fragment."""
url = "https://pubmed.ncbi.nlm.nih.gov/12345678#abstract"
assert URLClassifier.classify(url) == URLType.PUBMED
class TestURLClassifierSecurity:
"""Test security-related URL classification."""
def test_javascript_url_rejected(self):
"""Test that javascript: URLs are classified as INVALID."""
url = "javascript:alert('XSS')"
assert URLClassifier.classify(url) == URLType.INVALID
def test_data_url_rejected(self):
"""Test that data: URLs are classified as INVALID."""
url = "data:text/html,<script>alert('XSS')</script>"
assert URLClassifier.classify(url) == URLType.INVALID
def test_file_url_rejected(self):
"""Test that file: URLs are classified as INVALID."""
url = "file:///etc/passwd"
assert URLClassifier.classify(url) == URLType.INVALID
def test_vbscript_url_rejected(self):
"""Test that vbscript: URLs are classified as INVALID."""
url = "vbscript:msgbox('XSS')"
assert URLClassifier.classify(url) == URLType.INVALID
def test_about_url_rejected(self):
"""Test that about: URLs are classified as INVALID."""
url = "about:blank"
assert URLClassifier.classify(url) == URLType.INVALID
def test_ftp_url_rejected(self):
"""Test that ftp: URLs are classified as INVALID."""
url = "ftp://ftp.example.com/file.txt"
assert URLClassifier.classify(url) == URLType.INVALID
def test_non_http_ar5iv_url_is_rejected(self):
url = "ftp://ar5iv.org/2301.12345"
assert URLClassifier.classify(url) == URLType.INVALID
def test_http_url_accepted(self):
"""Test that http: URLs are accepted."""
url = "http://example.com/page"
assert URLClassifier.classify(url) == URLType.HTML
def test_https_url_accepted(self):
"""Test that https: URLs are accepted."""
url = "https://example.com/page"
assert URLClassifier.classify(url) == URLType.HTML
def test_invalid_source_name(self):
"""Test INVALID URL source name."""
assert URLClassifier.get_source_name(URLType.INVALID) == "Invalid URL"
def test_javascript_url_case_insensitive(self):
"""Test that JavaScript: URLs (mixed case) are rejected."""
url = "JavaScript:alert('XSS')"
assert URLClassifier.classify(url) == URLType.INVALID
def test_whitespace_trimmed(self):
"""Test that URLs with whitespace are trimmed."""
url = " https://example.com/page "
assert URLClassifier.classify(url) == URLType.HTML