~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
import sys
from pathlib import Path
PROJECT_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(PROJECT_ROOT / "src"))
from local_deep_research.research_library.downloaders.arxiv import ( # noqa: E402
ArxivDownloader,
)
from local_deep_research.utilities.arxiv_api import ( # noqa: E402
ArxivQueryRequest,
fetch_arxiv_results,
)
MAX_RESULTS = 50
PAPER_DIR = Path("./../local_search_files/research_papers")
WIKI_DIR = Path("./../local_search_files/wiki_sample")
def download_papers(papers, paper_dir, downloader) -> list[str]:
"""Download each paper's PDF through the application transport.
``ArxivDownloader.download`` returns ``None`` instead of raising
whenever no PDF arrives, for example when arXiv has none to serve, the
response is not a PDF, or the request failed (an HTTP error, rate
limiting, a timeout or connection error, or another error while
requesting or reading the response; the downloader logs each). Those
papers are returned as failures. This function catches nothing itself,
so an error the downloader does raise, or a failure writing a file,
propagates.
"""
paper_dir.mkdir(parents=True, exist_ok=True)
failures: list[str] = []
for i, paper in enumerate(papers):
pdf = downloader.download(paper.entry_id)
if not pdf:
failures.append(paper.title)
print(f"No PDF downloaded for {paper.title} ({paper.entry_id})")
continue
(paper_dir / f"paper_{i}.pdf").write_bytes(pdf)
print(f"Downloaded {i + 1} / {len(papers)}: {paper.title}")
return failures
def main() -> int:
from datasets import load_dataset
papers = fetch_arxiv_results(
ArxivQueryRequest(query="machine learning", max_results=MAX_RESULTS)
)
with ArxivDownloader() as downloader:
failures = download_papers(papers, PAPER_DIR, downloader)
wiki = load_dataset(
"wikipedia",
"20220301.en",
split=f"train[:{MAX_RESULTS}]",
trust_remote_code=True,
)
WIKI_DIR.mkdir(parents=True, exist_ok=True)
for i, article in enumerate(wiki):
(WIKI_DIR / f"article_{i}.txt").write_text(
article["title"] + "\n\n" + article["text"]
)
if failures:
print(f"{len(failures)} paper(s) could not be downloaded")
return 1
return 0
if __name__ == "__main__":
sys.exit(main())