~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
#!/usr/bin/env python
"""
SimpleQA Benchmark Runner for Local Deep Research.
This script provides a convenient way to run the SimpleQA benchmark.
Usage:
# Install dependencies with PDM
cd /path/to/local-deep-research
pdm install
# Run the script with PDM
pdm run python examples/benchmarks/run_simpleqa.py --help
"""
import argparse
import sys
from pathlib import Path
# Import the benchmark functionality
from local_deep_research.benchmarks.benchmark_functions import evaluate_simpleqa
def main():
"""Run the SimpleQA benchmark with the specified parameters."""
parser = argparse.ArgumentParser(description="Run SimpleQA benchmark")
parser.add_argument(
"--examples", type=int, default=10, help="Number of examples to run"
)
parser.add_argument(
"--iterations", type=int, default=3, help="Number of search iterations"
)
parser.add_argument(
"--questions", type=int, default=3, help="Questions per iteration"
)
parser.add_argument(
"--search-tool", type=str, default="searxng", help="Search tool to use"
)
parser.add_argument(
"--output-dir",
type=str,
default=str(Path("examples") / "benchmarks" / "results" / "simpleqa"),
help="Output directory",
)
parser.add_argument(
"--no-eval", action="store_true", help="Skip evaluation"
)
# Optional evaluation parameters
parser.add_argument(
"--human-eval", action="store_true", help="Use human evaluation"
)
parser.add_argument(
"--eval-model", type=str, help="Model to use for evaluation"
)
parser.add_argument(
"--eval-provider", type=str, help="Provider to use for evaluation"
)
# Add model configuration options
parser.add_argument(
"--search-model", type=str, help="Model to use for the search system"
)
parser.add_argument(
"--search-provider",
type=str,
help="Provider to use for the search system",
)
parser.add_argument(
"--endpoint-url",
type=str,
help="Endpoint URL for OpenRouter or other API services",
)
parser.add_argument(
"--search-strategy",
type=str,
default="source_based",
choices=[
"source_based",
"standard",
"rapid",
"parallel",
"iterdrag",
"modular",
],
help="Search strategy to use (default: source_based)",
)
parser.add_argument("--api-key", type=str, help="API key for LLM provider")
args = parser.parse_args()
print(f"Starting SimpleQA benchmark with {args.examples} examples...")
# Run the benchmark
results = evaluate_simpleqa(
num_examples=args.examples,
search_iterations=args.iterations,
questions_per_iteration=args.questions,
search_tool=args.search_tool,
human_evaluation=args.human_eval,
evaluation_model=args.eval_model,
evaluation_provider=args.eval_provider,
output_dir=args.output_dir,
)
# Print summary
print("\nSimpleQA Benchmark Results:")
print(f" Accuracy: {results.get('accuracy', 0):.3f}")
print(f" Total examples: {results.get('total_examples', 0)}")
print(f" Report saved to: {results.get('report_path', '')}")
return 0
if __name__ == "__main__":
sys.exit(main())