Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

2.4 KB
Raw Blame
import sys
from pathlib import Path

PROJECT_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(PROJECT_ROOT / "src"))

from local_deep_research.research_library.downloaders.arxiv import (  # noqa: E402
    ArxivDownloader,
)
from local_deep_research.utilities.arxiv_api import (  # noqa: E402
    ArxivQueryRequest,
    fetch_arxiv_results,
)

MAX_RESULTS = 50
PAPER_DIR = Path("./../local_search_files/research_papers")
WIKI_DIR = Path("./../local_search_files/wiki_sample")


def download_papers(papers, paper_dir, downloader) -> list[str]:
    """Download each paper's PDF through the application transport.

    ``ArxivDownloader.download`` returns ``None`` instead of raising
    whenever no PDF arrives, for example when arXiv has none to serve, the
    response is not a PDF, or the request failed (an HTTP error, rate
    limiting, a timeout or connection error, or another error while
    requesting or reading the response; the downloader logs each). Those
    papers are returned as failures. This function catches nothing itself,
    so an error the downloader does raise, or a failure writing a file,
    propagates.
    """
    paper_dir.mkdir(parents=True, exist_ok=True)
    failures: list[str] = []
    for i, paper in enumerate(papers):
        pdf = downloader.download(paper.entry_id)
        if not pdf:
            failures.append(paper.title)
            print(f"No PDF downloaded for {paper.title} ({paper.entry_id})")
            continue
        (paper_dir / f"paper_{i}.pdf").write_bytes(pdf)
        print(f"Downloaded {i + 1} / {len(papers)}: {paper.title}")
    return failures


def main() -> int:
    from datasets import load_dataset

    papers = fetch_arxiv_results(
        ArxivQueryRequest(query="machine learning", max_results=MAX_RESULTS)
    )
    with ArxivDownloader() as downloader:
        failures = download_papers(papers, PAPER_DIR, downloader)

    wiki = load_dataset(
        "wikipedia",
        "20220301.en",
        split=f"train[:{MAX_RESULTS}]",
        trust_remote_code=True,
    )

    WIKI_DIR.mkdir(parents=True, exist_ok=True)
    for i, article in enumerate(wiki):
        (WIKI_DIR / f"article_{i}.txt").write_text(
            article["title"] + "\n\n" + article["text"]
        )

    if failures:
        print(f"{len(failures)} paper(s) could not be downloaded")
        return 1
    return 0


if __name__ == "__main__":
    sys.exit(main())