~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
test: remove shadow test file test_arxiv_downloader_extended.py (#4094)
LearningCircuitef1cd8776de0
diff --git a/tests/research_library/downloaders/test_arxiv_downloader_extended.py b/tests/research_library/downloaders/test_arxiv_downloader_extended.pydeleted file mode 100644index 119f183e3b..0000000000--- a/tests/research_library/downloaders/test_arxiv_downloader_extended.py+++ /dev/null@@ -1,479 +0,0 @@-"""-Extended tests for ArxivDownloader - arXiv paper downloading.--Tests cover:-- URL handling and validation-- arXiv ID extraction-- PDF downloading-- Text/abstract fetching-- API integration-- Error handling and edge cases-"""--import re---class TestURLHandling:- """Tests for URL handling and validation."""-- def test_can_handle_arxiv_org(self):- """Should handle arxiv.org URLs."""- url = "https://arxiv.org/abs/2301.12345"-- from urllib.parse import urlparse-- hostname = urlparse(url).hostname- can_handle = hostname == "arxiv.org" or hostname.endswith(".arxiv.org")-- assert can_handle is True-- def test_can_handle_subdomain(self):- """Should handle arXiv subdomains."""- url = "https://export.arxiv.org/api/query"-- from urllib.parse import urlparse-- hostname = urlparse(url).hostname- can_handle = hostname and hostname.endswith(".arxiv.org")-- assert can_handle is True-- def test_cannot_handle_other_domains(self):- """Should not handle non-arXiv URLs."""- url = "https://example.com/paper.pdf"-- from urllib.parse import urlparse-- hostname = urlparse(url).hostname- can_handle = (- hostname == "arxiv.org" or hostname.endswith(".arxiv.org")- if hostname- else False- )-- assert can_handle is False-- def test_handles_invalid_url_gracefully(self):- """Should handle invalid URLs gracefully."""- url = "not a valid url"-- try:- from urllib.parse import urlparse-- hostname = urlparse(url).hostname- can_handle = bool(- hostname- and (hostname == "arxiv.org" or hostname.endswith(".arxiv.org"))- )- except Exception:- can_handle = False-- assert can_handle is False---class TestArxivIDExtraction:- """Tests for arXiv ID extraction from URLs."""-- def test_extract_new_format_id(self):- """Should extract new format arXiv ID (YYMM.NNNNN)."""- url = "https://arxiv.org/abs/2301.12345"- pattern = r"arxiv\.org/abs/(\d+\.\d+)(?:v\d+)?"-- match = re.search(pattern, url)- arxiv_id = match.group(1) if match else None-- assert arxiv_id == "2301.12345"-- def test_extract_new_format_with_version(self):- """Should extract ID ignoring version suffix."""- url = "https://arxiv.org/abs/2301.12345v2"- pattern = r"arxiv\.org/abs/(\d+\.\d+)(?:v\d+)?"-- match = re.search(pattern, url)- arxiv_id = match.group(1) if match else None-- assert arxiv_id == "2301.12345"-- def test_extract_from_pdf_url(self):- """Should extract ID from PDF URL."""- url = "https://arxiv.org/pdf/2301.12345.pdf"- pattern = r"arxiv\.org/pdf/(\d+\.\d+)(?:v\d+)?"-- match = re.search(pattern, url)- arxiv_id = match.group(1) if match else None-- assert arxiv_id == "2301.12345"-- def test_extract_old_format_id(self):- """Should extract old format arXiv ID (category/NNNNNNN)."""- url = "https://arxiv.org/abs/cond-mat/0501234"- pattern = r"arxiv\.org/abs/([a-z-]+/\d+)(?:v\d+)?"-- match = re.search(pattern, url)- arxiv_id = match.group(1) if match else None-- assert arxiv_id == "cond-mat/0501234"-- def test_extract_returns_none_for_invalid(self):- """Should return None for invalid URLs."""- url = "https://example.com/paper"- patterns = [- r"arxiv\.org/abs/(\d+\.\d+)(?:v\d+)?",- r"arxiv\.org/pdf/(\d+\.\d+)(?:v\d+)?",- ]-- arxiv_id = None- for pattern in patterns:- match = re.search(pattern, url)- if match:- arxiv_id = match.group(1)- break-- assert arxiv_id is None---class TestPDFURLConstruction:- """Tests for PDF URL construction."""-- def test_construct_pdf_url_new_format(self):- """Should construct PDF URL from new format ID."""- arxiv_id = "2301.12345"- pdf_url = f"https://arxiv.org/pdf/{arxiv_id}.pdf"-- assert pdf_url == "https://arxiv.org/pdf/2301.12345.pdf"-- def test_construct_pdf_url_old_format(self):- """Should construct PDF URL from old format ID."""- arxiv_id = "cond-mat/0501234"- pdf_url = f"https://arxiv.org/pdf/{arxiv_id}.pdf"-- assert pdf_url == "https://arxiv.org/pdf/cond-mat/0501234.pdf"---class TestAPIURLConstruction:- """Tests for arXiv API URL construction."""-- def test_api_url_new_format(self):- """Should construct API URL for new format ID."""- arxiv_id = "2301.12345"- clean_id = arxiv_id.replace("/", "")- api_url = f"https://export.arxiv.org/api/query?id_list={clean_id}"-- assert (- api_url == "https://export.arxiv.org/api/query?id_list=2301.12345"- )-- def test_api_url_old_format(self):- """Should clean old format ID for API."""- arxiv_id = "cond-mat/0501234"- clean_id = arxiv_id.replace("/", "")-- assert clean_id == "cond-mat0501234"---class TestContentTypeHandling:- """Tests for content type handling."""-- def test_content_type_pdf(self):- """Should handle PDF content type."""- content_type = "PDF"- is_pdf = content_type == "PDF"-- assert is_pdf is True-- def test_content_type_text(self):- """Should handle TEXT content type."""- content_type = "TEXT"- is_text = content_type == "TEXT"-- assert is_text is True-- def test_default_content_type_pdf(self):- """Default content type should be PDF."""- default = "PDF"- assert default == "PDF"---class TestDownloadResult:- """Tests for download result structure."""-- def test_success_result_structure(self):- """Success result should have content and is_success."""- result = {- "content": b"PDF content here",- "is_success": True,- }-- assert result["is_success"] is True- assert result["content"] is not None-- def test_failure_result_structure(self):- """Failure result should have skip_reason."""- result = {- "skip_reason": "Failed to download PDF",- "is_success": False,- }-- assert "skip_reason" in result- assert result["is_success"] is False-- def test_invalid_url_skip_reason(self):- """Invalid URL should have descriptive skip reason."""- result = {- "skip_reason": "Invalid arXiv URL - could not extract article ID",- }-- assert "Invalid arXiv URL" in result["skip_reason"]---class TestHTTPHeaders:- """Tests for HTTP header configuration."""-- def test_user_agent_header(self):- """Should include User-Agent header."""- headers = {- "User-Agent": "LocalDeepResearch/1.0",- "Accept": "application/pdf",- }-- assert "User-Agent" in headers-- def test_accept_pdf_header(self):- """Should accept PDF content type."""- headers = {- "Accept": "application/pdf,application/octet-stream,*/*",- }-- assert "application/pdf" in headers["Accept"]-- def test_connection_keep_alive(self):- """Should use keep-alive connection."""- headers = {- "Connection": "keep-alive",- }-- assert headers["Connection"] == "keep-alive"---class TestTextExtraction:- """Tests for text extraction from arXiv."""-- def test_full_text_includes_metadata(self):- """Full text should include metadata when available."""- metadata = "Title: Test Paper\nAuthors: John Doe"- extracted_text = "Full paper text here..."-- full_text = f"{metadata}\n\n{'=' * 80}\nFULL PAPER TEXT\n{'=' * 80}\n\n{extracted_text}"-- assert "Title: Test Paper" in full_text- assert "FULL PAPER TEXT" in full_text- assert extracted_text in full_text-- def test_text_without_metadata(self):- """Should return just extracted text if no metadata."""- extracted_text = "Full paper text here..."- metadata = None-- if metadata:- full_text = f"{metadata}\n\n{extracted_text}"- else:- full_text = extracted_text-- assert full_text == extracted_text-- def test_text_encoding_utf8(self):- """Text should be encoded as UTF-8."""- text = "Test with unicode: café résumé"- encoded = text.encode("utf-8", errors="ignore")-- assert isinstance(encoded, bytes)---class TestAPIResponseParsing:- """Tests for arXiv API response parsing."""-- def test_extract_title_from_xml(self):- """Should extract title from API response."""- # Simulated extraction- title_text = " A Sample Paper Title "- clean_title = f"Title: {title_text.strip()}"-- assert clean_title == "Title: A Sample Paper Title"-- def test_extract_authors_from_xml(self):- """Should extract authors from API response."""- author_names = ["John Doe", "Jane Smith"]- authors_text = f"Authors: {', '.join(author_names)}"-- assert authors_text == "Authors: John Doe, Jane Smith"-- def test_extract_abstract_from_xml(self):- """Should extract abstract from API response."""- abstract_text = "This paper presents..."- formatted = f"\nAbstract:\n{abstract_text.strip()}"-- assert "Abstract:" in formatted- assert abstract_text in formatted-- def test_extract_categories_from_xml(self):- """Should extract categories from API response."""- categories = ["cs.AI", "cs.LG", "stat.ML"]- categories_text = f"\nCategories: {', '.join(categories)}"-- assert "cs.AI" in categories_text- assert "stat.ML" in categories_text-- def test_combine_metadata_parts(self):- """Should combine all metadata parts."""- text_parts = [- "Title: Test Paper",- "Authors: John Doe",- "\nAbstract:\nTest abstract",- "\nCategories: cs.AI",- ]-- combined = "\n".join(text_parts)-- assert "Title:" in combined- assert "Authors:" in combined- assert "Abstract:" in combined- assert "Categories:" in combined---class TestErrorHandling:- """Tests for error handling."""-- def test_handle_extraction_failure(self):- """Should handle ID extraction failure."""- url = "invalid-url"- arxiv_id = None # Simulated extraction failure-- if not arxiv_id:- error_message = f"Could not extract arXiv ID from {url}"- else:- error_message = None-- assert error_message is not None- assert "invalid-url" in error_message-- def test_handle_download_failure(self):- """Should handle download failure."""- arxiv_id = "2301.12345"- pdf_content = None # Simulated download failure-- if not pdf_content:- skip_reason = f"Failed to download PDF for arXiv:{arxiv_id}"- else:- skip_reason = None-- assert skip_reason is not None- assert arxiv_id in skip_reason-- def test_handle_text_extraction_failure(self):- """Should handle text extraction failure."""- arxiv_id = "2301.12345"- extracted_text = None-- if not extracted_text:- skip_reason = f"Could not retrieve full text for arXiv:{arxiv_id}"- else:- skip_reason = None-- assert skip_reason is not None-- def test_handle_api_failure(self):- """Should handle API fetch failure gracefully."""- # Simulated API failure- try:- raise Exception("API timeout")- except Exception:- metadata = None-- assert metadata is None---class TestURLPatterns:- """Tests for various arXiv URL patterns."""-- def test_abs_url_pattern(self):- """Should match abstract page URL."""- url = "https://arxiv.org/abs/2301.12345"- pattern = r"arxiv\.org/abs/"-- assert re.search(pattern, url) is not None-- def test_pdf_url_pattern(self):- """Should match PDF URL."""- url = "https://arxiv.org/pdf/2301.12345.pdf"- pattern = r"arxiv\.org/pdf/"-- assert re.search(pattern, url) is not None-- def test_versioned_url_pattern(self):- """Should match versioned URL."""- url = "https://arxiv.org/abs/2301.12345v3"- pattern = r"arxiv\.org/abs/\d+\.\d+v\d+"-- assert re.search(pattern, url) is not None-- def test_old_category_pattern(self):- """Should match old category format."""- url = "https://arxiv.org/abs/hep-th/9901001"- pattern = r"arxiv\.org/abs/[a-z-]+/\d+"-- assert re.search(pattern, url) is not None---class TestEdgeCases:- """Tests for edge cases."""-- def test_empty_url(self):- """Should handle empty URL."""- url = ""-- try:- from urllib.parse import urlparse-- hostname = urlparse(url).hostname- can_handle = hostname is not None- except Exception:- can_handle = False-- assert can_handle is False-- def test_none_url(self):- """Should handle None URL."""- url = None-- try:- if url is None:- raise ValueError("URL is None")- can_handle = True- except Exception:- can_handle = False-- assert can_handle is False-- def test_url_with_special_characters(self):- """Should handle URLs with special characters."""- url = "https://arxiv.org/abs/2301.12345?format=pdf"- pattern = r"arxiv\.org/abs/(\d+\.\d+)"-- match = re.search(pattern, url)- arxiv_id = match.group(1) if match else None-- assert arxiv_id == "2301.12345"-- def test_http_vs_https(self):- """Should handle both HTTP and HTTPS."""- urls = [- "http://arxiv.org/abs/2301.12345",- "https://arxiv.org/abs/2301.12345",- ]-- for url in urls:- from urllib.parse import urlparse-- hostname = urlparse(url).hostname- can_handle = hostname == "arxiv.org"- assert can_handle is True