~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Command-line interface for benchmarking.
This module provides CLI commands for running benchmarks.
"""
import argparse
# import logging - replaced with loguru
import sys
from loguru import logger
from ...config.paths import get_data_directory
from .. import (
get_available_datasets,
run_browsecomp_benchmark,
run_simpleqa_benchmark,
)
def setup_benchmark_parser(subparsers):
"""
Set up the benchmark CLI commands.
Args:
subparsers: argparse subparsers object to add commands to
"""
# Common benchmark arguments
benchmark_parent = argparse.ArgumentParser(add_help=False)
benchmark_parent.add_argument(
"--examples",
type=int,
default=100,
help="Number of examples to run (default: 100)",
)
benchmark_parent.add_argument(
"--iterations",
type=int,
default=3,
help="Number of search iterations (default: 3)",
)
benchmark_parent.add_argument(
"--questions",
type=int,
default=3,
help="Questions per iteration (default: 3)",
)
benchmark_parent.add_argument(
"--search-tool",
type=str,
default="searxng",
help="Search tool to use (default: searxng)",
)
benchmark_parent.add_argument(
"--output-dir",
type=str,
default=str(get_data_directory() / "benchmark_results"),
help="Directory to save results (default: user data directory/benchmark_results)",
)
benchmark_parent.add_argument(
"--human-eval",
action="store_true",
help="Use human evaluation instead of automated",
)
benchmark_parent.add_argument(
"--eval-model", type=str, help="Model to use for evaluation"
)
benchmark_parent.add_argument(
"--eval-provider", type=str, help="Provider to use for evaluation"
)
benchmark_parent.add_argument(
"--custom-dataset", type=str, help="Path to custom dataset"
)
benchmark_parent.add_argument(
"--no-eval", action="store_true", help="Skip evaluation phase"
)
# Add model configuration options for the search system
benchmark_parent.add_argument(
"--search-model", type=str, help="Model to use for the search system"
)
benchmark_parent.add_argument(
"--search-provider",
type=str,
help="Provider to use for the search system",
)
benchmark_parent.add_argument(
"--endpoint-url",
type=str,
help="Endpoint URL for OpenRouter or other API services",
)
benchmark_parent.add_argument(
"--search-strategy",
type=str,
default="source_based",
choices=["source_based", "standard", "rapid", "parallel", "iterdrag"],
help="Search strategy to use (default: source_based)",
)
# SimpleQA benchmark command
simpleqa_parser = subparsers.add_parser(
"simpleqa", parents=[benchmark_parent], help="Run SimpleQA benchmark"
)
simpleqa_parser.set_defaults(func=run_simpleqa_cli)
# BrowseComp benchmark command
browsecomp_parser = subparsers.add_parser(
"browsecomp",
parents=[benchmark_parent],
help="Run BrowseComp benchmark",
)
browsecomp_parser.set_defaults(func=run_browsecomp_cli)
# List available benchmarks command
list_parser = subparsers.add_parser(
"list", help="List available benchmarks"
)
list_parser.set_defaults(func=list_benchmarks_cli)
# Compare configurations command
compare_parser = subparsers.add_parser(
"compare", help="Compare multiple search configurations"
)
compare_parser.add_argument(
"--dataset",
type=str,
default="simpleqa",
choices=["simpleqa", "browsecomp"],
help="Dataset to use for comparison",
)
compare_parser.add_argument(
"--examples",
type=int,
default=20,
help="Number of examples for each configuration (default: 20)",
)
compare_parser.add_argument(
"--output-dir",
type=str,
default=str(get_data_directory() / "benchmark_results" / "comparison"),
help="Directory to save comparison results",
)
compare_parser.set_defaults(func=compare_configs_cli)
def run_simpleqa_cli(args):
"""
CLI handler for SimpleQA benchmark.
Args:
args: Parsed command-line arguments
"""
# Set up search configuration
search_config = {
"iterations": args.iterations,
"questions_per_iteration": args.questions,
"search_tool": args.search_tool,
}
# Add model configuration if provided
if hasattr(args, "search_model") and args.search_model:
search_config["model_name"] = args.search_model
if hasattr(args, "search_provider") and args.search_provider:
search_config["provider"] = args.search_provider
if hasattr(args, "endpoint_url") and args.endpoint_url:
search_config["openai_endpoint_url"] = args.endpoint_url
if hasattr(args, "search_strategy") and args.search_strategy:
search_config["search_strategy"] = args.search_strategy
# Set up evaluation configuration if needed
evaluation_config = None
if args.eval_model or args.eval_provider:
evaluation_config = {}
if args.eval_model:
evaluation_config["model_name"] = args.eval_model
if args.eval_provider:
evaluation_config["provider"] = args.eval_provider
# Run the benchmark
result = run_simpleqa_benchmark(
num_examples=args.examples,
dataset_path=args.custom_dataset,
output_dir=args.output_dir,
search_config=search_config,
evaluation_config=evaluation_config,
human_evaluation=args.human_eval,
run_evaluation=not args.no_eval,
)
# Print results summary
if "metrics" in result:
print("\nSimpleQA Benchmark Results:")
print(f" Accuracy: {result['metrics'].get('accuracy', 0):.3f}")
print(f" Total examples: {result['total_examples']}")
print(f" Correct answers: {result['metrics'].get('correct', 0)}")
print(
f" Average time: {result['metrics'].get('average_processing_time', 0):.2f}s"
)
print(f"\nReport saved to: {result.get('report_path', 'N/A')}")
else:
print("\nSimpleQA Benchmark Completed (no evaluation)")
print(f" Total examples: {result['total_examples']}")
print(f" Results saved to: {result.get('results_path', 'N/A')}")
def run_browsecomp_cli(args):
"""
CLI handler for BrowseComp benchmark.
Args:
args: Parsed command-line arguments
"""
# Set up search configuration
search_config = {
"iterations": args.iterations,
"questions_per_iteration": args.questions,
"search_tool": args.search_tool,
}
# Add model configuration if provided
if hasattr(args, "search_model") and args.search_model:
search_config["model_name"] = args.search_model
if hasattr(args, "search_provider") and args.search_provider:
search_config["provider"] = args.search_provider
if hasattr(args, "endpoint_url") and args.endpoint_url:
search_config["openai_endpoint_url"] = args.endpoint_url
if hasattr(args, "search_strategy") and args.search_strategy:
search_config["search_strategy"] = args.search_strategy
# Set up evaluation configuration if needed
evaluation_config = None
if args.eval_model or args.eval_provider:
evaluation_config = {}
if args.eval_model:
evaluation_config["model_name"] = args.eval_model
if args.eval_provider:
evaluation_config["provider"] = args.eval_provider
# Run the benchmark
result = run_browsecomp_benchmark(
num_examples=args.examples,
dataset_path=args.custom_dataset,
output_dir=args.output_dir,
search_config=search_config,
evaluation_config=evaluation_config,
human_evaluation=args.human_eval,
run_evaluation=not args.no_eval,
)
# Print results summary
if "metrics" in result:
print("\nBrowseComp Benchmark Results:")
print(f" Accuracy: {result['metrics'].get('accuracy', 0):.3f}")
print(f" Total examples: {result['total_examples']}")
print(f" Correct answers: {result['metrics'].get('correct', 0)}")
print(
f" Average time: {result['metrics'].get('average_processing_time', 0):.2f}s"
)
print(f"\nReport saved to: {result.get('report_path', 'N/A')}")
else:
print("\nBrowseComp Benchmark Completed (no evaluation)")
print(f" Total examples: {result['total_examples']}")
print(f" Results saved to: {result.get('results_path', 'N/A')}")
def list_benchmarks_cli(args):
"""
CLI handler for listing available benchmarks.
Args:
args: Parsed command-line arguments
"""
datasets = get_available_datasets()
print("\nAvailable Benchmarks:")
for dataset in datasets:
print(f" {dataset['id']}: {dataset['name']}")
print(f" {dataset['description']}")
print(f" URL: {dataset['url']}")
print()
def compare_configs_cli(args):
"""
CLI handler for comparing multiple configurations.
Args:
args: Parsed command-line arguments
"""
# Import the compare configurations function
from ...api.benchmark_functions import compare_configurations
# Run the comparison
result = compare_configurations(
dataset_type=args.dataset,
num_examples=args.examples,
output_dir=args.output_dir,
)
# Print results summary
print("\nConfiguration Comparison Results:")
print(f" Dataset: {args.dataset}")
print(f" Configurations tested: {result['configurations_tested']}")
print(f" Report saved to: {result['report_path']}")
# Print brief comparison table
print("\nResults Summary:")
print("Configuration | Accuracy | Avg. Time")
print("--------------- | -------- | ---------")
for res in result["results"]:
name = res["configuration_name"]
acc = res.get("metrics", {}).get("accuracy", 0)
time = res.get("metrics", {}).get("average_processing_time", 0)
print(f"{name:15} | {acc:.3f} | {time:.2f}s")
def main():
"""
Main entry point for benchmark CLI.
"""
parser = argparse.ArgumentParser(
description="Local Deep Research Benchmarking Tool",
prog="ldr-benchmark",
)
# Set up logging
parser.add_argument(
"--verbose", action="store_true", help="Enable verbose logging"
)
# Create subparsers
subparsers = parser.add_subparsers(
dest="command", help="Command to run", required=True
)
# Set up commands
setup_benchmark_parser(subparsers)
# Parse arguments
args = parser.parse_args()
# Set up logging with loguru
# diagnose=False: loguru defaults to True, which renders repr() of every
# local in every traceback frame on exception. The benchmarks code path
# has many logger.exception() call sites whose frame locals hold LLM
# api_key, search-engine secrets, and Authorization headers; leaving the
# default on would write them to stderr on any failure. Companion to
# #4185 / #4384 (config_logger) and #4394 (MCP); the benchmark CLI also
# bypasses config_logger, so the LDR_LOGURU_DIAGNOSE gate does not reach
# it. Pinned off here unconditionally — the CLI has --verbose for level
# but no operator workflow that needs frame-local repr dumps.
# For the same reason, install config_logger's credential-redaction
# patcher explicitly, or credentials in messages, bound extras and
# exception text would reach stderr unscrubbed. Imported here, not at
# module top, to keep log_utils' web/database imports off `--help`.
#
# local_deep_research/__init__.py disables the package's own loguru
# namespace, so until it is re-enabled every record logged from inside
# the package is dropped before the patcher or any sink sees it. Enable
# it as config_logger and the MCP server do -- but only after removing
# loguru's default stderr handler, which keeps loguru's diagnose=True
# (frame-local repr dumps) and would otherwise print those records a
# second time, unredacted locals included.
from ...utilities.log_utils import install_log_redaction
logger.remove()
install_log_redaction()
if args.verbose:
logger.add(sys.stderr, level="DEBUG", diagnose=False)
else:
logger.add(sys.stderr, level="INFO", diagnose=False)
logger.enable("local_deep_research")
# Run command
if hasattr(args, "func"):
args.func(args)
else:
parser.print_help()
if __name__ == "__main__":
main()