Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

12.6 KB
Raw Blame
"""
Command-line interface for benchmarking.

This module provides CLI commands for running benchmarks.
"""

import argparse

# import logging - replaced with loguru
import sys
from loguru import logger

from ...config.paths import get_data_directory
from .. import (
    get_available_datasets,
    run_browsecomp_benchmark,
    run_simpleqa_benchmark,
)


def setup_benchmark_parser(subparsers):
    """
    Set up the benchmark CLI commands.

    Args:
        subparsers: argparse subparsers object to add commands to
    """
    # Common benchmark arguments
    benchmark_parent = argparse.ArgumentParser(add_help=False)
    benchmark_parent.add_argument(
        "--examples",
        type=int,
        default=100,
        help="Number of examples to run (default: 100)",
    )
    benchmark_parent.add_argument(
        "--iterations",
        type=int,
        default=3,
        help="Number of search iterations (default: 3)",
    )
    benchmark_parent.add_argument(
        "--questions",
        type=int,
        default=3,
        help="Questions per iteration (default: 3)",
    )
    benchmark_parent.add_argument(
        "--search-tool",
        type=str,
        default="searxng",
        help="Search tool to use (default: searxng)",
    )
    benchmark_parent.add_argument(
        "--output-dir",
        type=str,
        default=str(get_data_directory() / "benchmark_results"),
        help="Directory to save results (default: user data directory/benchmark_results)",
    )
    benchmark_parent.add_argument(
        "--human-eval",
        action="store_true",
        help="Use human evaluation instead of automated",
    )
    benchmark_parent.add_argument(
        "--eval-model", type=str, help="Model to use for evaluation"
    )
    benchmark_parent.add_argument(
        "--eval-provider", type=str, help="Provider to use for evaluation"
    )
    benchmark_parent.add_argument(
        "--custom-dataset", type=str, help="Path to custom dataset"
    )
    benchmark_parent.add_argument(
        "--no-eval", action="store_true", help="Skip evaluation phase"
    )

    # Add model configuration options for the search system
    benchmark_parent.add_argument(
        "--search-model", type=str, help="Model to use for the search system"
    )
    benchmark_parent.add_argument(
        "--search-provider",
        type=str,
        help="Provider to use for the search system",
    )
    benchmark_parent.add_argument(
        "--endpoint-url",
        type=str,
        help="Endpoint URL for OpenRouter or other API services",
    )
    benchmark_parent.add_argument(
        "--search-strategy",
        type=str,
        default="source_based",
        choices=["source_based", "standard", "rapid", "parallel", "iterdrag"],
        help="Search strategy to use (default: source_based)",
    )

    # SimpleQA benchmark command
    simpleqa_parser = subparsers.add_parser(
        "simpleqa", parents=[benchmark_parent], help="Run SimpleQA benchmark"
    )
    simpleqa_parser.set_defaults(func=run_simpleqa_cli)

    # BrowseComp benchmark command
    browsecomp_parser = subparsers.add_parser(
        "browsecomp",
        parents=[benchmark_parent],
        help="Run BrowseComp benchmark",
    )
    browsecomp_parser.set_defaults(func=run_browsecomp_cli)

    # List available benchmarks command
    list_parser = subparsers.add_parser(
        "list", help="List available benchmarks"
    )
    list_parser.set_defaults(func=list_benchmarks_cli)

    # Compare configurations command
    compare_parser = subparsers.add_parser(
        "compare", help="Compare multiple search configurations"
    )
    compare_parser.add_argument(
        "--dataset",
        type=str,
        default="simpleqa",
        choices=["simpleqa", "browsecomp"],
        help="Dataset to use for comparison",
    )
    compare_parser.add_argument(
        "--examples",
        type=int,
        default=20,
        help="Number of examples for each configuration (default: 20)",
    )
    compare_parser.add_argument(
        "--output-dir",
        type=str,
        default=str(get_data_directory() / "benchmark_results" / "comparison"),
        help="Directory to save comparison results",
    )
    compare_parser.set_defaults(func=compare_configs_cli)


def run_simpleqa_cli(args):
    """
    CLI handler for SimpleQA benchmark.

    Args:
        args: Parsed command-line arguments
    """
    # Set up search configuration
    search_config = {
        "iterations": args.iterations,
        "questions_per_iteration": args.questions,
        "search_tool": args.search_tool,
    }

    # Add model configuration if provided
    if hasattr(args, "search_model") and args.search_model:
        search_config["model_name"] = args.search_model
    if hasattr(args, "search_provider") and args.search_provider:
        search_config["provider"] = args.search_provider
    if hasattr(args, "endpoint_url") and args.endpoint_url:
        search_config["openai_endpoint_url"] = args.endpoint_url
    if hasattr(args, "search_strategy") and args.search_strategy:
        search_config["search_strategy"] = args.search_strategy

    # Set up evaluation configuration if needed
    evaluation_config = None
    if args.eval_model or args.eval_provider:
        evaluation_config = {}
        if args.eval_model:
            evaluation_config["model_name"] = args.eval_model
        if args.eval_provider:
            evaluation_config["provider"] = args.eval_provider

    # Run the benchmark
    result = run_simpleqa_benchmark(
        num_examples=args.examples,
        dataset_path=args.custom_dataset,
        output_dir=args.output_dir,
        search_config=search_config,
        evaluation_config=evaluation_config,
        human_evaluation=args.human_eval,
        run_evaluation=not args.no_eval,
    )

    # Print results summary
    if "metrics" in result:
        print("\nSimpleQA Benchmark Results:")
        print(f"  Accuracy: {result['metrics'].get('accuracy', 0):.3f}")
        print(f"  Total examples: {result['total_examples']}")
        print(f"  Correct answers: {result['metrics'].get('correct', 0)}")
        print(
            f"  Average time: {result['metrics'].get('average_processing_time', 0):.2f}s"
        )
        print(f"\nReport saved to: {result.get('report_path', 'N/A')}")
    else:
        print("\nSimpleQA Benchmark Completed (no evaluation)")
        print(f"  Total examples: {result['total_examples']}")
        print(f"  Results saved to: {result.get('results_path', 'N/A')}")


def run_browsecomp_cli(args):
    """
    CLI handler for BrowseComp benchmark.

    Args:
        args: Parsed command-line arguments
    """
    # Set up search configuration
    search_config = {
        "iterations": args.iterations,
        "questions_per_iteration": args.questions,
        "search_tool": args.search_tool,
    }

    # Add model configuration if provided
    if hasattr(args, "search_model") and args.search_model:
        search_config["model_name"] = args.search_model
    if hasattr(args, "search_provider") and args.search_provider:
        search_config["provider"] = args.search_provider
    if hasattr(args, "endpoint_url") and args.endpoint_url:
        search_config["openai_endpoint_url"] = args.endpoint_url
    if hasattr(args, "search_strategy") and args.search_strategy:
        search_config["search_strategy"] = args.search_strategy

    # Set up evaluation configuration if needed
    evaluation_config = None
    if args.eval_model or args.eval_provider:
        evaluation_config = {}
        if args.eval_model:
            evaluation_config["model_name"] = args.eval_model
        if args.eval_provider:
            evaluation_config["provider"] = args.eval_provider

    # Run the benchmark
    result = run_browsecomp_benchmark(
        num_examples=args.examples,
        dataset_path=args.custom_dataset,
        output_dir=args.output_dir,
        search_config=search_config,
        evaluation_config=evaluation_config,
        human_evaluation=args.human_eval,
        run_evaluation=not args.no_eval,
    )

    # Print results summary
    if "metrics" in result:
        print("\nBrowseComp Benchmark Results:")
        print(f"  Accuracy: {result['metrics'].get('accuracy', 0):.3f}")
        print(f"  Total examples: {result['total_examples']}")
        print(f"  Correct answers: {result['metrics'].get('correct', 0)}")
        print(
            f"  Average time: {result['metrics'].get('average_processing_time', 0):.2f}s"
        )
        print(f"\nReport saved to: {result.get('report_path', 'N/A')}")
    else:
        print("\nBrowseComp Benchmark Completed (no evaluation)")
        print(f"  Total examples: {result['total_examples']}")
        print(f"  Results saved to: {result.get('results_path', 'N/A')}")


def list_benchmarks_cli(args):
    """
    CLI handler for listing available benchmarks.

    Args:
        args: Parsed command-line arguments
    """
    datasets = get_available_datasets()

    print("\nAvailable Benchmarks:")
    for dataset in datasets:
        print(f"  {dataset['id']}: {dataset['name']}")
        print(f"    {dataset['description']}")
        print(f"    URL: {dataset['url']}")
        print()


def compare_configs_cli(args):
    """
    CLI handler for comparing multiple configurations.

    Args:
        args: Parsed command-line arguments
    """
    # Import the compare configurations function
    from ...api.benchmark_functions import compare_configurations

    # Run the comparison
    result = compare_configurations(
        dataset_type=args.dataset,
        num_examples=args.examples,
        output_dir=args.output_dir,
    )

    # Print results summary
    print("\nConfiguration Comparison Results:")
    print(f"  Dataset: {args.dataset}")
    print(f"  Configurations tested: {result['configurations_tested']}")
    print(f"  Report saved to: {result['report_path']}")

    # Print brief comparison table
    print("\nResults Summary:")
    print("Configuration   | Accuracy | Avg. Time")
    print("--------------- | -------- | ---------")
    for res in result["results"]:
        name = res["configuration_name"]
        acc = res.get("metrics", {}).get("accuracy", 0)
        time = res.get("metrics", {}).get("average_processing_time", 0)
        print(f"{name:15} | {acc:.3f}    | {time:.2f}s")


def main():
    """
    Main entry point for benchmark CLI.
    """
    parser = argparse.ArgumentParser(
        description="Local Deep Research Benchmarking Tool",
        prog="ldr-benchmark",
    )

    # Set up logging
    parser.add_argument(
        "--verbose", action="store_true", help="Enable verbose logging"
    )

    # Create subparsers
    subparsers = parser.add_subparsers(
        dest="command", help="Command to run", required=True
    )

    # Set up commands
    setup_benchmark_parser(subparsers)

    # Parse arguments
    args = parser.parse_args()

    # Set up logging with loguru
    # diagnose=False: loguru defaults to True, which renders repr() of every
    # local in every traceback frame on exception. The benchmarks code path
    # has many logger.exception() call sites whose frame locals hold LLM
    # api_key, search-engine secrets, and Authorization headers; leaving the
    # default on would write them to stderr on any failure. Companion to
    # #4185 / #4384 (config_logger) and #4394 (MCP); the benchmark CLI also
    # bypasses config_logger, so the LDR_LOGURU_DIAGNOSE gate does not reach
    # it. Pinned off here unconditionally — the CLI has --verbose for level
    # but no operator workflow that needs frame-local repr dumps.
    # For the same reason, install config_logger's credential-redaction
    # patcher explicitly, or credentials in messages, bound extras and
    # exception text would reach stderr unscrubbed. Imported here, not at
    # module top, to keep log_utils' web/database imports off `--help`.
    #
    # local_deep_research/__init__.py disables the package's own loguru
    # namespace, so until it is re-enabled every record logged from inside
    # the package is dropped before the patcher or any sink sees it. Enable
    # it as config_logger and the MCP server do -- but only after removing
    # loguru's default stderr handler, which keeps loguru's diagnose=True
    # (frame-local repr dumps) and would otherwise print those records a
    # second time, unredacted locals included.
    from ...utilities.log_utils import install_log_redaction

    logger.remove()
    install_log_redaction()
    if args.verbose:
        logger.add(sys.stderr, level="DEBUG", diagnose=False)
    else:
        logger.add(sys.stderr, level="INFO", diagnose=False)
    logger.enable("local_deep_research")

    # Run command
    if hasattr(args, "func"):
        args.func(args)
    else:
        parser.print_help()


if __name__ == "__main__":
    main()