Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

test: remove shadow test file test_arxiv_downloader_extended.py (#4094)

LearningCircuitef1cd8776de0
diff --git a/tests/research_library/downloaders/test_arxiv_downloader_extended.py b/tests/research_library/downloaders/test_arxiv_downloader_extended.py
deleted file mode 100644
index 119f183e3b..0000000000
--- a/tests/research_library/downloaders/test_arxiv_downloader_extended.py
+++ /dev/null
@@ -1,479 +0,0 @@
-"""
-Extended tests for ArxivDownloader - arXiv paper downloading.
-
-Tests cover:
-- URL handling and validation
-- arXiv ID extraction
-- PDF downloading
-- Text/abstract fetching
-- API integration
-- Error handling and edge cases
-"""
-
-import re
-
-
-class TestURLHandling:
- """Tests for URL handling and validation."""
-
- def test_can_handle_arxiv_org(self):
- """Should handle arxiv.org URLs."""
- url = "https://arxiv.org/abs/2301.12345"
-
- from urllib.parse import urlparse
-
- hostname = urlparse(url).hostname
- can_handle = hostname == "arxiv.org" or hostname.endswith(".arxiv.org")
-
- assert can_handle is True
-
- def test_can_handle_subdomain(self):
- """Should handle arXiv subdomains."""
- url = "https://export.arxiv.org/api/query"
-
- from urllib.parse import urlparse
-
- hostname = urlparse(url).hostname
- can_handle = hostname and hostname.endswith(".arxiv.org")
-
- assert can_handle is True
-
- def test_cannot_handle_other_domains(self):
- """Should not handle non-arXiv URLs."""
- url = "https://example.com/paper.pdf"
-
- from urllib.parse import urlparse
-
- hostname = urlparse(url).hostname
- can_handle = (
- hostname == "arxiv.org" or hostname.endswith(".arxiv.org")
- if hostname
- else False
- )
-
- assert can_handle is False
-
- def test_handles_invalid_url_gracefully(self):
- """Should handle invalid URLs gracefully."""
- url = "not a valid url"
-
- try:
- from urllib.parse import urlparse
-
- hostname = urlparse(url).hostname
- can_handle = bool(
- hostname
- and (hostname == "arxiv.org" or hostname.endswith(".arxiv.org"))
- )
- except Exception:
- can_handle = False
-
- assert can_handle is False
-
-
-class TestArxivIDExtraction:
- """Tests for arXiv ID extraction from URLs."""
-
- def test_extract_new_format_id(self):
- """Should extract new format arXiv ID (YYMM.NNNNN)."""
- url = "https://arxiv.org/abs/2301.12345"
- pattern = r"arxiv\.org/abs/(\d+\.\d+)(?:v\d+)?"
-
- match = re.search(pattern, url)
- arxiv_id = match.group(1) if match else None
-
- assert arxiv_id == "2301.12345"
-
- def test_extract_new_format_with_version(self):
- """Should extract ID ignoring version suffix."""
- url = "https://arxiv.org/abs/2301.12345v2"
- pattern = r"arxiv\.org/abs/(\d+\.\d+)(?:v\d+)?"
-
- match = re.search(pattern, url)
- arxiv_id = match.group(1) if match else None
-
- assert arxiv_id == "2301.12345"
-
- def test_extract_from_pdf_url(self):
- """Should extract ID from PDF URL."""
- url = "https://arxiv.org/pdf/2301.12345.pdf"
- pattern = r"arxiv\.org/pdf/(\d+\.\d+)(?:v\d+)?"
-
- match = re.search(pattern, url)
- arxiv_id = match.group(1) if match else None
-
- assert arxiv_id == "2301.12345"
-
- def test_extract_old_format_id(self):
- """Should extract old format arXiv ID (category/NNNNNNN)."""
- url = "https://arxiv.org/abs/cond-mat/0501234"
- pattern = r"arxiv\.org/abs/([a-z-]+/\d+)(?:v\d+)?"
-
- match = re.search(pattern, url)
- arxiv_id = match.group(1) if match else None
-
- assert arxiv_id == "cond-mat/0501234"
-
- def test_extract_returns_none_for_invalid(self):
- """Should return None for invalid URLs."""
- url = "https://example.com/paper"
- patterns = [
- r"arxiv\.org/abs/(\d+\.\d+)(?:v\d+)?",
- r"arxiv\.org/pdf/(\d+\.\d+)(?:v\d+)?",
- ]
-
- arxiv_id = None
- for pattern in patterns:
- match = re.search(pattern, url)
- if match:
- arxiv_id = match.group(1)
- break
-
- assert arxiv_id is None
-
-
-class TestPDFURLConstruction:
- """Tests for PDF URL construction."""
-
- def test_construct_pdf_url_new_format(self):
- """Should construct PDF URL from new format ID."""
- arxiv_id = "2301.12345"
- pdf_url = f"https://arxiv.org/pdf/{arxiv_id}.pdf"
-
- assert pdf_url == "https://arxiv.org/pdf/2301.12345.pdf"
-
- def test_construct_pdf_url_old_format(self):
- """Should construct PDF URL from old format ID."""
- arxiv_id = "cond-mat/0501234"
- pdf_url = f"https://arxiv.org/pdf/{arxiv_id}.pdf"
-
- assert pdf_url == "https://arxiv.org/pdf/cond-mat/0501234.pdf"
-
-
-class TestAPIURLConstruction:
- """Tests for arXiv API URL construction."""
-
- def test_api_url_new_format(self):
- """Should construct API URL for new format ID."""
- arxiv_id = "2301.12345"
- clean_id = arxiv_id.replace("/", "")
- api_url = f"https://export.arxiv.org/api/query?id_list={clean_id}"
-
- assert (
- api_url == "https://export.arxiv.org/api/query?id_list=2301.12345"
- )
-
- def test_api_url_old_format(self):
- """Should clean old format ID for API."""
- arxiv_id = "cond-mat/0501234"
- clean_id = arxiv_id.replace("/", "")
-
- assert clean_id == "cond-mat0501234"
-
-
-class TestContentTypeHandling:
- """Tests for content type handling."""
-
- def test_content_type_pdf(self):
- """Should handle PDF content type."""
- content_type = "PDF"
- is_pdf = content_type == "PDF"
-
- assert is_pdf is True
-
- def test_content_type_text(self):
- """Should handle TEXT content type."""
- content_type = "TEXT"
- is_text = content_type == "TEXT"
-
- assert is_text is True
-
- def test_default_content_type_pdf(self):
- """Default content type should be PDF."""
- default = "PDF"
- assert default == "PDF"
-
-
-class TestDownloadResult:
- """Tests for download result structure."""
-
- def test_success_result_structure(self):
- """Success result should have content and is_success."""
- result = {
- "content": b"PDF content here",
- "is_success": True,
- }
-
- assert result["is_success"] is True
- assert result["content"] is not None
-
- def test_failure_result_structure(self):
- """Failure result should have skip_reason."""
- result = {
- "skip_reason": "Failed to download PDF",
- "is_success": False,
- }
-
- assert "skip_reason" in result
- assert result["is_success"] is False
-
- def test_invalid_url_skip_reason(self):
- """Invalid URL should have descriptive skip reason."""
- result = {
- "skip_reason": "Invalid arXiv URL - could not extract article ID",
- }
-
- assert "Invalid arXiv URL" in result["skip_reason"]
-
-
-class TestHTTPHeaders:
- """Tests for HTTP header configuration."""
-
- def test_user_agent_header(self):
- """Should include User-Agent header."""
- headers = {
- "User-Agent": "LocalDeepResearch/1.0",
- "Accept": "application/pdf",
- }
-
- assert "User-Agent" in headers
-
- def test_accept_pdf_header(self):
- """Should accept PDF content type."""
- headers = {
- "Accept": "application/pdf,application/octet-stream,*/*",
- }
-
- assert "application/pdf" in headers["Accept"]
-
- def test_connection_keep_alive(self):
- """Should use keep-alive connection."""
- headers = {
- "Connection": "keep-alive",
- }
-
- assert headers["Connection"] == "keep-alive"
-
-
-class TestTextExtraction:
- """Tests for text extraction from arXiv."""
-
- def test_full_text_includes_metadata(self):
- """Full text should include metadata when available."""
- metadata = "Title: Test Paper\nAuthors: John Doe"
- extracted_text = "Full paper text here..."
-
- full_text = f"{metadata}\n\n{'=' * 80}\nFULL PAPER TEXT\n{'=' * 80}\n\n{extracted_text}"
-
- assert "Title: Test Paper" in full_text
- assert "FULL PAPER TEXT" in full_text
- assert extracted_text in full_text
-
- def test_text_without_metadata(self):
- """Should return just extracted text if no metadata."""
- extracted_text = "Full paper text here..."
- metadata = None
-
- if metadata:
- full_text = f"{metadata}\n\n{extracted_text}"
- else:
- full_text = extracted_text
-
- assert full_text == extracted_text
-
- def test_text_encoding_utf8(self):
- """Text should be encoded as UTF-8."""
- text = "Test with unicode: café résumé"
- encoded = text.encode("utf-8", errors="ignore")
-
- assert isinstance(encoded, bytes)
-
-
-class TestAPIResponseParsing:
- """Tests for arXiv API response parsing."""
-
- def test_extract_title_from_xml(self):
- """Should extract title from API response."""
- # Simulated extraction
- title_text = " A Sample Paper Title "
- clean_title = f"Title: {title_text.strip()}"
-
- assert clean_title == "Title: A Sample Paper Title"
-
- def test_extract_authors_from_xml(self):
- """Should extract authors from API response."""
- author_names = ["John Doe", "Jane Smith"]
- authors_text = f"Authors: {', '.join(author_names)}"
-
- assert authors_text == "Authors: John Doe, Jane Smith"
-
- def test_extract_abstract_from_xml(self):
- """Should extract abstract from API response."""
- abstract_text = "This paper presents..."
- formatted = f"\nAbstract:\n{abstract_text.strip()}"
-
- assert "Abstract:" in formatted
- assert abstract_text in formatted
-
- def test_extract_categories_from_xml(self):
- """Should extract categories from API response."""
- categories = ["cs.AI", "cs.LG", "stat.ML"]
- categories_text = f"\nCategories: {', '.join(categories)}"
-
- assert "cs.AI" in categories_text
- assert "stat.ML" in categories_text
-
- def test_combine_metadata_parts(self):
- """Should combine all metadata parts."""
- text_parts = [
- "Title: Test Paper",
- "Authors: John Doe",
- "\nAbstract:\nTest abstract",
- "\nCategories: cs.AI",
- ]
-
- combined = "\n".join(text_parts)
-
- assert "Title:" in combined
- assert "Authors:" in combined
- assert "Abstract:" in combined
- assert "Categories:" in combined
-
-
-class TestErrorHandling:
- """Tests for error handling."""
-
- def test_handle_extraction_failure(self):
- """Should handle ID extraction failure."""
- url = "invalid-url"
- arxiv_id = None # Simulated extraction failure
-
- if not arxiv_id:
- error_message = f"Could not extract arXiv ID from {url}"
- else:
- error_message = None
-
- assert error_message is not None
- assert "invalid-url" in error_message
-
- def test_handle_download_failure(self):
- """Should handle download failure."""
- arxiv_id = "2301.12345"
- pdf_content = None # Simulated download failure
-
- if not pdf_content:
- skip_reason = f"Failed to download PDF for arXiv:{arxiv_id}"
- else:
- skip_reason = None
-
- assert skip_reason is not None
- assert arxiv_id in skip_reason
-
- def test_handle_text_extraction_failure(self):
- """Should handle text extraction failure."""
- arxiv_id = "2301.12345"
- extracted_text = None
-
- if not extracted_text:
- skip_reason = f"Could not retrieve full text for arXiv:{arxiv_id}"
- else:
- skip_reason = None
-
- assert skip_reason is not None
-
- def test_handle_api_failure(self):
- """Should handle API fetch failure gracefully."""
- # Simulated API failure
- try:
- raise Exception("API timeout")
- except Exception:
- metadata = None
-
- assert metadata is None
-
-
-class TestURLPatterns:
- """Tests for various arXiv URL patterns."""
-
- def test_abs_url_pattern(self):
- """Should match abstract page URL."""
- url = "https://arxiv.org/abs/2301.12345"
- pattern = r"arxiv\.org/abs/"
-
- assert re.search(pattern, url) is not None
-
- def test_pdf_url_pattern(self):
- """Should match PDF URL."""
- url = "https://arxiv.org/pdf/2301.12345.pdf"
- pattern = r"arxiv\.org/pdf/"
-
- assert re.search(pattern, url) is not None
-
- def test_versioned_url_pattern(self):
- """Should match versioned URL."""
- url = "https://arxiv.org/abs/2301.12345v3"
- pattern = r"arxiv\.org/abs/\d+\.\d+v\d+"
-
- assert re.search(pattern, url) is not None
-
- def test_old_category_pattern(self):
- """Should match old category format."""
- url = "https://arxiv.org/abs/hep-th/9901001"
- pattern = r"arxiv\.org/abs/[a-z-]+/\d+"
-
- assert re.search(pattern, url) is not None
-
-
-class TestEdgeCases:
- """Tests for edge cases."""
-
- def test_empty_url(self):
- """Should handle empty URL."""
- url = ""
-
- try:
- from urllib.parse import urlparse
-
- hostname = urlparse(url).hostname
- can_handle = hostname is not None
- except Exception:
- can_handle = False
-
- assert can_handle is False
-
- def test_none_url(self):
- """Should handle None URL."""
- url = None
-
- try:
- if url is None:
- raise ValueError("URL is None")
- can_handle = True
- except Exception:
- can_handle = False
-
- assert can_handle is False
-
- def test_url_with_special_characters(self):
- """Should handle URLs with special characters."""
- url = "https://arxiv.org/abs/2301.12345?format=pdf"
- pattern = r"arxiv\.org/abs/(\d+\.\d+)"
-
- match = re.search(pattern, url)
- arxiv_id = match.group(1) if match else None
-
- assert arxiv_id == "2301.12345"
-
- def test_http_vs_https(self):
- """Should handle both HTTP and HTTPS."""
- urls = [
- "http://arxiv.org/abs/2301.12345",
- "https://arxiv.org/abs/2301.12345",
- ]
-
- for url in urls:
- from urllib.parse import urlparse
-
- hostname = urlparse(url).hostname
- can_handle = hostname == "arxiv.org"
- assert can_handle is True