~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
#!/usr/bin/env python
"""
Run SimpleQA and BrowseComp benchmarks in parallel with 300 examples each.
This script demonstrates running multiple benchmarks in parallel with a large number of examples.
Usage:
# Install dependencies with PDM
cd /path/to/local-deep-research
pdm install
# Run the script with PDM
pdm run python examples/optimization/run_parallel_benchmark.py
"""
import argparse
import concurrent.futures
import os
import sys
import time
from datetime import datetime, UTC
from pathlib import Path
from loguru import logger
# Add the src directory to the Python path
project_root = str(Path(__file__).parent.parent.parent.resolve())
sys.path.insert(0, str(Path(project_root) / "src"))
def run_simpleqa_benchmark(
num_examples,
output_dir,
model=None,
provider=None,
endpoint_url=None,
api_key=None,
):
"""Run SimpleQA benchmark with specified number of examples."""
from local_deep_research.benchmarks.benchmark_functions import (
evaluate_simpleqa,
)
logger.info(f"Starting SimpleQA benchmark with {num_examples} examples")
start_time = time.time()
# Run the benchmark
results = evaluate_simpleqa(
num_examples=num_examples,
search_iterations=2,
questions_per_iteration=3,
search_strategy="source_based",
search_tool="searxng",
search_model=model,
search_provider=provider,
endpoint_url=endpoint_url,
output_dir=str(Path(output_dir) / "simpleqa"),
evaluation_provider="ANTHROPIC",
evaluation_model="claude-3-7-sonnet-20250219",
)
duration = time.time() - start_time
logger.info(f"SimpleQA benchmark completed in {duration:.1f} seconds")
if results and isinstance(results, dict):
logger.info(f"SimpleQA accuracy: {results.get('accuracy', 'N/A')}")
return results
def run_browsecomp_benchmark(
num_examples,
output_dir,
model=None,
provider=None,
endpoint_url=None,
api_key=None,
):
"""Run BrowseComp benchmark with specified number of examples."""
from local_deep_research.benchmarks.benchmark_functions import (
evaluate_browsecomp,
)
logger.info(f"Starting BrowseComp benchmark with {num_examples} examples")
start_time = time.time()
# Run the benchmark
results = evaluate_browsecomp(
num_examples=num_examples,
search_iterations=3,
questions_per_iteration=3,
search_strategy="source_based",
search_tool="searxng",
search_model=model,
search_provider=provider,
endpoint_url=endpoint_url,
output_dir=str(Path(output_dir) / "browsecomp"),
evaluation_provider="ANTHROPIC",
evaluation_model="claude-3-7-sonnet-20250219",
)
duration = time.time() - start_time
logger.info(f"BrowseComp benchmark completed in {duration:.1f} seconds")
if results and isinstance(results, dict):
logger.info(f"BrowseComp accuracy: {results.get('accuracy', 'N/A')}")
return results
def setup_llm_environment(
model=None, provider=None, endpoint_url=None, api_key=None
):
"""Set up environment variables for LLM configuration."""
if model:
os.environ["LDR_LLM__MODEL"] = model
logger.info(f"Using LLM model: {model}")
if provider:
os.environ["LDR_LLM__PROVIDER"] = provider
logger.info(f"Using LLM provider: {provider}")
if endpoint_url:
os.environ["OPENAI_ENDPOINT_URL"] = endpoint_url
os.environ["LDR_LLM__OPENAI_ENDPOINT_URL"] = endpoint_url
logger.info(f"Using endpoint URL: {endpoint_url}")
if api_key:
# Set the appropriate environment variable based on provider
if provider == "openai":
os.environ["OPENAI_API_KEY"] = api_key
os.environ["LDR_LLM__OPENAI_API_KEY"] = api_key
elif provider == "openai_endpoint":
os.environ["OPENAI_ENDPOINT_API_KEY"] = api_key
os.environ["LDR_LLM__OPENAI_ENDPOINT_API_KEY"] = api_key
elif provider == "anthropic":
os.environ["ANTHROPIC_API_KEY"] = api_key
os.environ["LDR_LLM__ANTHROPIC_API_KEY"] = api_key
logger.info("API key configured")
def main():
parser = argparse.ArgumentParser(
description="Run SimpleQA and BrowseComp benchmarks in parallel"
)
parser.add_argument(
"--examples",
type=int,
default=300,
help="Number of examples for each benchmark (default: 300)",
)
# LLM configuration options
parser.add_argument(
"--model",
help="Model name for the LLM (e.g., 'claude-3-sonnet-20240229')",
)
parser.add_argument(
"--provider",
help="Provider for the LLM (e.g., 'anthropic', 'openai', 'openai_endpoint')",
)
parser.add_argument(
"--endpoint-url",
help="Custom endpoint URL (e.g., 'https://openrouter.ai/api/v1')",
)
parser.add_argument("--api-key", help="API key for the LLM provider")
args = parser.parse_args()
# Create timestamp for unique output directory
timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S")
output_dir = str(
Path(project_root)
/ "benchmark_results"
/ f"parallel_benchmark_{timestamp}"
)
Path(output_dir).mkdir(parents=True, exist_ok=True)
# Display start information
print(f"Starting parallel benchmarks with {args.examples} examples each")
print(f"Results will be saved to: {output_dir}")
# Set up LLM environment if specified
setup_llm_environment(
model=args.model,
provider=args.provider,
endpoint_url=args.endpoint_url,
api_key=args.api_key,
)
# Start time for total execution
total_start_time = time.time()
# Run benchmarks in parallel using ThreadPoolExecutor
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
# Submit both benchmark jobs
simpleqa_future = executor.submit(
run_simpleqa_benchmark,
args.examples,
output_dir,
args.model,
args.provider,
args.endpoint_url,
args.api_key,
)
browsecomp_future = executor.submit(
run_browsecomp_benchmark,
args.examples,
output_dir,
args.model,
args.provider,
args.endpoint_url,
args.api_key,
)
# Get results from both futures
try:
simpleqa_results = simpleqa_future.result()
print("SimpleQA benchmark completed successfully")
except Exception:
logger.exception("Error in SimpleQA benchmark")
simpleqa_results = None
try:
browsecomp_results = browsecomp_future.result()
print("BrowseComp benchmark completed successfully")
except Exception:
logger.exception("Error in BrowseComp benchmark")
browsecomp_results = None
# Calculate total time
total_duration = time.time() - total_start_time
# Print summary
print("\n" + "=" * 50)
print(" PARALLEL BENCHMARK SUMMARY ")
print("=" * 50)
print(f"Total duration: {total_duration:.1f} seconds")
print(f"Examples per benchmark: {args.examples}")
if simpleqa_results and isinstance(simpleqa_results, dict):
print(f"SimpleQA accuracy: {simpleqa_results.get('accuracy', 'N/A')}")
else:
print("SimpleQA: Failed or no results")
if browsecomp_results and isinstance(browsecomp_results, dict):
print(
f"BrowseComp accuracy: {browsecomp_results.get('accuracy', 'N/A')}"
)
else:
print("BrowseComp: Failed or no results")
print(f"Results saved to: {output_dir}")
print("=" * 50)
# Save summary to JSON file
try:
import json
summary = {
"timestamp": timestamp,
"examples_per_benchmark": args.examples,
"total_duration": total_duration,
"simpleqa": {
"accuracy": (
simpleqa_results.get("accuracy")
if simpleqa_results
else None
),
"completed": simpleqa_results is not None,
},
"browsecomp": {
"accuracy": (
browsecomp_results.get("accuracy")
if browsecomp_results
else None
),
"completed": browsecomp_results is not None,
},
"model": args.model,
"provider": args.provider,
}
with open(
Path(output_dir) / "parallel_benchmark_summary.json",
"w",
encoding="utf-8",
) as f:
json.dump(summary, f, indent=2)
except Exception:
logger.exception("Error saving summary")
return 0
if __name__ == "__main__":
sys.exit(main())