~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Benchmark runners for Local Deep Research.
This module provides the main functions for running benchmarks using LDR.
"""
import json
from loguru import logger
import os
from pathlib import Path
import time
from typing import Any, Callable, Dict, Optional
from ..api import quick_summary
from .datasets import DEFAULT_DATASET_URLS, load_dataset
from .datasets.base import DatasetRegistry
from .graders import (
check_grader_endpoint_key,
extract_answer_from_response,
grade_results,
)
from .metrics import calculate_metrics, generate_report
from .templates import BROWSECOMP_QUERY_TEMPLATE
def format_query(question: str, dataset_type: str = "simpleqa") -> str:
"""
Format query based on dataset type.
Args:
question: Original question
dataset_type: Type of dataset
Returns:
Formatted query for LDR
"""
if dataset_type.lower() == "browsecomp":
# BrowseComp requires specific formatting
return BROWSECOMP_QUERY_TEMPLATE.format(question=question)
# Simple format for SimpleQA
return question
class BenchmarkOutputDirectoryError(ValueError):
"""No benchmark output directory below the data directory is usable."""
def _below_data_directory(value: str, data_dir: Path) -> Optional[Path]:
"""Resolve ``value`` and return it if it is strictly below ``data_dir``.
Returns None for a value that resolves to ``data_dir`` itself, outside
it, or cannot be resolved.
"""
from ..security.path_validator import resolve_allow_missing
try:
path = Path(os.path.expandvars(str(value))).expanduser()
if not path.is_absolute():
path = data_dir / path
resolved = resolve_allow_missing(path)
except (OSError, RuntimeError, ValueError):
return None
if resolved != data_dir and resolved.is_relative_to(data_dir):
return resolved
return None
def resolve_output_dir(output_dir: Optional[str], *, fallback: str) -> Path:
"""Return the benchmark output directory, resolved below the data dir.
``$VAR`` and ``~`` are expanded, and a relative value is taken relative
to the data directory (``LDR_DATA_DIR`` or the platform default), not
the working directory. ``None``, ``""`` and ``"."`` select ``fallback``.
Any other value that resolves outside the data directory, to the data
directory itself, or cannot be resolved, is replaced by ``fallback``
(relative to the data directory) and a warning is logged.
Raises:
BenchmarkOutputDirectoryError: ``fallback`` is not usable either, for
example because ``<data dir>/<fallback>`` is a symlink to
another disk. Benchmark files are never written into the data
directory root.
ValueError, OSError: ``LDR_DATA_DIR`` is invalid or the data
directory cannot be resolved.
Symlinks are followed with ``resolve_allow_missing`` before the
comparison, so a link that leaves the data directory, a symlink loop and
a chain whose resolved form outgrows PATH_MAX are not accepted. The
check is a snapshot: it does not protect against a concurrent change to
the filesystem.
"""
# Lazy imports: path_validator imports config.paths at module level.
from ..config.paths import get_data_directory
from ..security.path_validator import resolve_allow_missing
data_dir = resolve_allow_missing(get_data_directory())
if output_dir is not None and Path(str(output_dir)) != Path("."):
resolved = _below_data_directory(output_dir, data_dir)
if resolved is not None:
return resolved
logger.warning(
"Benchmark output directory is not below the data directory or "
f"cannot be resolved; using the default {fallback!r} there"
)
resolved = _below_data_directory(fallback, data_dir)
if resolved is not None:
return resolved
raise BenchmarkOutputDirectoryError(
"No benchmark output directory below the data directory is usable: "
f"neither the requested directory nor the default {fallback!r} "
"resolves to a directory inside it. Choose an output directory "
"inside the data directory, set LDR_DATA_DIR to a data directory "
"that contains the location you want, or remove the symlink that "
f"points {fallback!r} outside the data directory."
)
def run_benchmark(
dataset_type: str,
dataset_path: Optional[str] = None,
num_examples: Optional[int] = None,
output_dir: str = "benchmark_results",
run_evaluation: bool = True,
evaluation_config: Optional[Dict[str, Any]] = None,
search_config: Optional[Dict[str, Any]] = None,
human_evaluation: bool = False,
progress_callback: Optional[Callable[[str, int, Dict], None]] = None,
seed: int = 42,
settings_snapshot: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
"""
Run a benchmark on the specified dataset.
Args:
dataset_type: Type of dataset ("simpleqa" or "browsecomp")
dataset_path: Optional custom dataset path
num_examples: Number of examples to use
output_dir: Directory to save results. It is resolved below the
data directory by :func:`resolve_output_dir`: a relative value
is relative to the data directory, and a value outside it (or
the data directory itself) is replaced by ``benchmark_results``
there. Raises ``BenchmarkOutputDirectoryError`` before any
work when ``benchmark_results`` is not usable either.
run_evaluation: Whether to evaluate results
evaluation_config: Custom LLM config for evaluation
search_config: Custom search parameters
human_evaluation: Whether to use human evaluation
progress_callback: Optional callback for progress updates
seed: Random seed for reproducibility
Returns:
Dictionary with benchmark results and metrics. The returned file
paths are inside the resolved output directory.
"""
# Refuse up front when the grader would have no API key for its
# endpoint, before any research budget is spent. Raises
# LLMNotConfiguredError naming benchmark.evaluation.api_key.
if run_evaluation and not human_evaluation:
check_grader_endpoint_key(evaluation_config, settings_snapshot)
output_dir = str(
resolve_output_dir(output_dir, fallback="benchmark_results")
)
# Ensure output directory exists
os.makedirs(output_dir, exist_ok=True)
# Default search configuration
if not search_config:
search_config = {
"iterations": 3,
"questions_per_iteration": 3,
"search_tool": "searxng",
}
# Load dataset using the class-based approach
try:
# Create the dataset instance from registry
dataset_instance = DatasetRegistry.create_dataset(
dataset_id=dataset_type.lower(),
dataset_path=dataset_path,
num_examples=num_examples,
seed=seed,
)
# Load the examples
dataset = dataset_instance.load()
logger.info(
f"Loaded {len(dataset)} examples using dataset class {type(dataset_instance).__name__}"
)
except Exception:
# Fallback to legacy function if there's any issue
logger.warning(
"Error using dataset class. Falling back to legacy function."
)
# This is the local benchmark registry factory, not a Hugging Face download API.
dataset = load_dataset( # nosec B615
dataset_type=dataset_type,
dataset_path=dataset_path,
num_examples=num_examples,
seed=seed,
)
# Set up output files
timestamp = time.strftime("%Y%m%d_%H%M%S")
results_file = str(
Path(output_dir) / f"{dataset_type}_{timestamp}_results.jsonl"
)
evaluation_file = str(
Path(output_dir) / f"{dataset_type}_{timestamp}_evaluation.jsonl"
)
report_file = str(
Path(output_dir) / f"{dataset_type}_{timestamp}_report.md"
)
# Make sure output files don't exist
for file in [results_file, evaluation_file, report_file]:
file_path = Path(file)
if file_path.exists():
# nosemgrep: semgrep.rules.path-traversal-risk, reason: CLI/Python benchmark output paths are selected by the invoking program; web benchmarks use the separate database runner
file_path.unlink()
# Progress tracking
total_examples = len(dataset)
if progress_callback:
progress_callback(
"Starting benchmark",
0,
{
"status": "started",
"dataset_type": dataset_type,
"total_examples": total_examples,
},
)
# Process each example
results = []
for i, example in enumerate(dataset):
# Extract question and answer in a way that uses the dataset class when available
if "dataset_instance" in locals() and isinstance(
dataset_instance,
DatasetRegistry.get_dataset_class(dataset_type.lower()),
):
# Use the dataset class methods to extract question and answer
question = dataset_instance.get_question(example)
correct_answer = dataset_instance.get_answer(example)
logger.debug(
"Using dataset class methods to extract question and answer"
)
else:
# Fallback to the legacy approach
if dataset_type.lower() == "simpleqa":
question = example.get("problem", "")
correct_answer = example.get("answer", "")
else: # browsecomp
question = example.get("problem", "")
# For BrowseComp, the answer should be in "correct_answer" after decryption
correct_answer = example.get("correct_answer", "")
if not correct_answer and "answer" in example:
# Fallback to "answer" field if "correct_answer" is not available
correct_answer = example.get("answer", "")
# Update progress
if progress_callback:
progress_callback(
f"Processing example {i + 1}/{total_examples}",
int(i / total_examples * 50),
{
"status": "processing",
"current": i + 1,
"total": total_examples,
"question": (
question[:50] + "..."
if len(question) > 50
else question
),
},
)
logger.info(f"Processing {i + 1}/{total_examples}: {question[:50]}...")
try:
# Format query based on dataset type
formatted_query = format_query(question, dataset_type)
# Time the search
start_time = time.time()
# Get response from LDR. Pass settings_snapshot through so
# the egress policy PEP fires on the benchmark's inner
# search-engine / LLM construction calls. Without it, the
# PEP's ``if settings_snapshot is not None`` guard skips and
# benchmarks silently bypass the user's saved scope.
search_result = quick_summary(
query=formatted_query,
iterations=search_config.get("iterations", 3),
questions_per_iteration=search_config.get(
"questions_per_iteration", 3
),
search_tool=search_config.get("search_tool", "searxng"),
settings_snapshot=settings_snapshot,
)
end_time = time.time()
processing_time = end_time - start_time
# Extract response and search info
response = search_result.get("summary", "")
# Extract structured information
extracted = extract_answer_from_response(response, dataset_type)
# Format result
result = {
"id": example.get("id", f"example_{i}"),
"problem": question,
"correct_answer": correct_answer,
"response": response,
"extracted_answer": extracted["extracted_answer"],
"confidence": extracted["confidence"],
"processing_time": processing_time,
"sources": search_result.get("sources", []),
"search_config": search_config,
}
# Add to results list
results.append(result)
# Write result to file
# nosemgrep: semgrep.rules.path-traversal-risk, reason: CLI/Python benchmark result file generated under its caller-selected output root
with open(results_file, "a", encoding="utf-8") as f:
f.write(json.dumps(result) + "\n")
# Update progress
if progress_callback:
progress_callback(
f"Completed example {i + 1}/{total_examples}",
int((i + 0.5) / total_examples * 50),
{
"status": "completed_example",
"current": i + 1,
"total": total_examples,
"result": result,
},
)
except Exception as e:
logger.exception(f"Error processing example {i + 1}")
# Create error result
error_result = {
"id": example.get("id", f"example_{i}"),
"problem": question,
"correct_answer": correct_answer,
"error": str(e),
"processing_time": (
time.time() - start_time if "start_time" in locals() else 0
),
}
# Add to results list
results.append(error_result)
# Write error result to file
# nosemgrep: semgrep.rules.path-traversal-risk, reason: Same generated benchmark result file used by the success path above
with open(results_file, "a", encoding="utf-8") as f:
f.write(json.dumps(error_result) + "\n")
# Update progress
if progress_callback:
progress_callback(
f"Error processing example {i + 1}/{total_examples}",
int((i + 0.5) / total_examples * 50),
{
"status": "error",
"current": i + 1,
"total": total_examples,
"error": str(e),
"result": error_result,
},
)
logger.info(f"Completed processing {total_examples} examples")
# Run evaluation if requested
if run_evaluation:
if progress_callback:
progress_callback(
"Starting evaluation",
50,
{"status": "evaluating", "results_file": results_file},
)
if human_evaluation:
from .graders import human_evaluation as evaluate
logger.info("Running human evaluation...")
evaluation_results = evaluate(
results_file=results_file,
output_file=evaluation_file,
interactive=True,
)
else:
logger.info("Running automated evaluation...")
try:
evaluation_results = grade_results(
results_file=results_file,
output_file=evaluation_file,
dataset_type=dataset_type,
evaluation_config=evaluation_config,
settings_snapshot=settings_snapshot,
progress_callback=lambda current, total, meta: (
progress_callback(
f"Evaluating {current + 1}/{total}",
50 + int((current + 0.5) / total * 40),
{**meta, "status": "evaluating"},
)
if progress_callback
else None
),
)
except Exception as e:
logger.exception("Automated evaluation failed")
if progress_callback:
progress_callback(
"Automated evaluation failed. Falling back to human evaluation.",
60,
{"status": "evaluation_fallback", "error": str(e)},
)
# Ask if user wants to fall back to human evaluation
fallback_to_human = False
print("\nAutomated evaluation failed with error:", str(e))
response = input(
"Do you want to fall back to human evaluation? (y/n): "
)
fallback_to_human = response.strip().lower() == "y"
if fallback_to_human:
logger.info("Falling back to human evaluation...")
from .graders import human_evaluation as evaluate
evaluation_results = evaluate(
results_file=results_file,
output_file=evaluation_file,
interactive=True,
)
else:
from ..security.file_write_verifier import (
write_file_verified,
)
logger.info("Skipping evaluation due to error.")
# Create an empty evaluation file to prevent issues
write_file_verified(
evaluation_file,
"",
"benchmark.allow_file_output",
context="empty evaluation placeholder",
)
return {
"status": "evaluation_error",
"dataset_type": dataset_type,
"results_path": results_file,
"evaluation_error": str(e),
"total_examples": total_examples,
}
# Calculate metrics
if progress_callback:
progress_callback(
"Calculating metrics", 90, {"status": "calculating_metrics"}
)
metrics = calculate_metrics(evaluation_file)
# Generate report
if progress_callback:
progress_callback(
"Generating report", 95, {"status": "generating_report"}
)
dataset_name = dataset_type.capitalize()
report_path = generate_report(
metrics=metrics,
results_file=evaluation_file,
output_file=report_file,
dataset_name=dataset_name,
config_info={
"Dataset": dataset_path
or DEFAULT_DATASET_URLS.get(dataset_type, "Unknown"),
"Examples": total_examples,
"Iterations": search_config.get("iterations", 3),
"Questions per iteration": search_config.get(
"questions_per_iteration", 3
),
"Search tool": search_config.get("search_tool", "searxng"),
"Evaluation method": "Human"
if human_evaluation
else "Automated",
},
)
# Mark as complete
if progress_callback:
progress_callback(
"Benchmark complete",
100,
{
"status": "complete",
"metrics": metrics,
"report_path": report_path,
},
)
return {
"status": "complete",
"dataset_type": dataset_type,
"results_path": results_file,
"evaluation_path": evaluation_file,
"report_path": report_path,
"metrics": metrics,
"total_examples": total_examples,
"accuracy": metrics.get("accuracy", 0),
}
# No evaluation, just return results
if progress_callback:
progress_callback(
"Benchmark complete (no evaluation)",
100,
{"status": "complete_no_eval", "results_path": results_file},
)
return {
"status": "complete_no_eval",
"dataset_type": dataset_type,
"results_path": results_file,
"total_examples": total_examples,
}
def run_simpleqa_benchmark(num_examples: int = 100, **kwargs) -> Dict[str, Any]:
"""
Run SimpleQA benchmark with default settings.
Args:
num_examples: Number of examples to process
**kwargs: Additional arguments to pass to run_benchmark
Returns:
Dictionary with benchmark results
"""
return run_benchmark(
dataset_type="simpleqa", num_examples=num_examples, **kwargs
)
def run_browsecomp_benchmark(
num_examples: int = 100, **kwargs
) -> Dict[str, Any]:
"""
Run BrowseComp benchmark with default settings.
Args:
num_examples: Number of examples to process
**kwargs: Additional arguments to pass to run_benchmark
Returns:
Dictionary with benchmark results
"""
return run_benchmark(
dataset_type="browsecomp", num_examples=num_examples, **kwargs
)
def run_xbench_deepsearch_benchmark(
num_examples: int = 100, **kwargs
) -> Dict[str, Any]:
"""
Run xbench-DeepSearch benchmark with default settings.
Args:
num_examples: Number of examples to process
**kwargs: Additional arguments to pass to run_benchmark
Returns:
Dictionary with benchmark results
"""
return run_benchmark(
dataset_type="xbench_deepsearch", num_examples=num_examples, **kwargs
)