Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

45.4 KB
Raw Blame
"""
Optuna-based parameter optimizer for Local Deep Research.

This module provides the core optimization functionality using Optuna
to find optimal parameters for the research system, balancing quality
and performance metrics.
"""

import os
from pathlib import Path
import time
from datetime import datetime, UTC
from functools import partial
from typing import Any, Callable, Dict, List, Optional, Tuple

import joblib
import numpy as np
import optuna

# (matplotlib / optuna.visualization are imported lazily — see
# _ensure_plotting_loaded below and #4431.)

from local_deep_research.benchmarks.efficiency.speed_profiler import (
    SpeedProfiler,
)
from local_deep_research.security import sanitize_data
from loguru import logger

from local_deep_research.benchmarks.evaluators import (
    CompositeBenchmarkEvaluator,
)

# Import benchmark evaluator components

# Visualization libraries (matplotlib + optuna.visualization) are imported
# LAZILY — see _ensure_plotting_loaded below and #4431. find_spec only
# probes availability; it does NOT execute/import the module, so importing
# this file (and therefore `local_deep_research.benchmarks`) never pays
# matplotlib's ~60s cold-import cost.
import importlib.util

PLOTTING_AVAILABLE = importlib.util.find_spec("matplotlib") is not None
if not PLOTTING_AVAILABLE:
    logger.warning("Matplotlib not available, visualization will be limited")


# Module-level placeholders so tests can @patch these names and so the
# loader below can fill them in place. None until first real visualization.
plt = None
Line2D = None
plot_contour = None
plot_optimization_history = None
plot_param_importances = None
plot_slice = None


def _ensure_plotting_loaded():
    """Import matplotlib + optuna.visualization into module globals on first
    use.

    Deferred from module load on purpose: matplotlib's import is heavy and,
    under the 2-core CI runner's GIL/CPU starvation, stretched to ~60s while
    holding the import lock. Because benchmarks/__init__.py imports this
    module, a module-level matplotlib import froze the whole werkzeug request
    pipeline the first time any request touched `local_deep_research.benchmarks`
    — the flaky UI-shard navigation timeouts (#4431). The
    _create_*_visualizations methods call this after their PLOTTING_AVAILABLE
    guard; visualization only happens in benchmark-optimization runs, never
    on a request path. Early-returns if already loaded (or a test has patched
    plt) so it never clobbers mocks.
    """
    global plt, Line2D
    global plot_contour, plot_optimization_history
    global plot_param_importances, plot_slice
    if plt is not None:
        return
    import matplotlib.pyplot as plt
    from matplotlib.lines import Line2D
    from optuna.visualization import (
        plot_contour,
        plot_optimization_history,
        plot_param_importances,
        plot_slice,
    )


class OptunaOptimizer:
    """
    Optimize parameters for Local Deep Research using Optuna.

    This class provides functionality to:
    1. Define search spaces for parameter optimization
    2. Evaluate parameter combinations using objective functions
    3. Find optimal parameters via Optuna
    4. Visualize and analyze optimization results
    """

    def __init__(
        self,
        base_query: str,
        output_dir: str = "optimization_results",
        model_name: Optional[str] = None,
        provider: Optional[str] = None,
        search_tool: Optional[str] = None,
        temperature: float = 0.7,
        n_trials: int = 30,
        timeout: Optional[int] = None,
        n_jobs: int = 1,
        study_name: Optional[str] = None,
        optimization_metrics: Optional[List[str]] = None,
        metric_weights: Optional[Dict[str, float]] = None,
        progress_callback: Optional[Callable[[int, int, Dict], None]] = None,
        benchmark_weights: Optional[Dict[str, float]] = None,
    ):
        """
        Initialize the optimizer.

        Args:
            base_query: The research query to use for all experiments
            output_dir: Directory to save optimization results. It is used
                as given, not resolved below the data directory; benchmark
                files that ``run_benchmark`` writes for a trial follow
                ``runners.resolve_output_dir`` instead.
            model_name: Name of the LLM model to use
            provider: LLM provider
            search_tool: Search engine to use
            temperature: LLM temperature
            n_trials: Number of parameter combinations to try
            timeout: Maximum seconds to run optimization (None for no limit)
            n_jobs: Number of parallel jobs for optimization
            study_name: Name of the Optuna study
            optimization_metrics: List of metrics to optimize (default: ["quality", "speed"])
            metric_weights: Dictionary of weights for each metric (e.g., {"quality": 0.6, "speed": 0.4})
            progress_callback: Optional callback for progress updates
            benchmark_weights: Dictionary mapping benchmark types to weights
                (e.g., {"simpleqa": 0.6, "browsecomp": 0.4})
                If None, only SimpleQA is used with weight 1.0
        """
        self.base_query = base_query
        self.output_dir = output_dir
        self.model_name = model_name
        self.provider = provider
        self.search_tool = search_tool
        self.temperature = temperature
        self.n_trials = n_trials
        self.timeout = timeout
        self.n_jobs = n_jobs
        self.optimization_metrics = optimization_metrics or ["quality", "speed"]
        self.metric_weights = metric_weights or {"quality": 0.6, "speed": 0.4}
        self.progress_callback = progress_callback

        # Initialize benchmark evaluator with weights
        self.benchmark_weights = benchmark_weights or {"simpleqa": 1.0}
        self.benchmark_evaluator = CompositeBenchmarkEvaluator(
            self.benchmark_weights
        )

        # Normalize weights to sum to 1.0
        total_weight = sum(self.metric_weights.values())
        if total_weight > 0:
            self.metric_weights = {
                k: v / total_weight for k, v in self.metric_weights.items()
            }

        # Generate a unique study name if not provided
        timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S")
        self.study_name = study_name or f"ldr_opt_{timestamp}"

        # Create output directory
        os.makedirs(output_dir, exist_ok=True)

        # Store the trial history for analysis
        self.trials_history: List[Dict[str, Any]] = []

        # Storage for the best parameters and study
        self.best_params: Optional[Dict[str, Any]] = None
        self.study: Optional[optuna.Study] = None

    def optimize(
        self, param_space: Optional[Dict[str, Any]] = None
    ) -> Tuple[Dict[str, Any], float]:
        """
        Run the optimization process using Optuna.

        Args:
            param_space: Dictionary defining parameter search spaces
                         (if None, use default spaces)

        Returns:
            Tuple containing (best_parameters, best_score)
        """
        param_space = param_space or self._get_default_param_space()

        # Create a study object
        storage_name = f"sqlite:///{self.output_dir}/{self.study_name}.db"
        self.study = optuna.create_study(
            study_name=self.study_name,
            storage=storage_name,
            load_if_exists=True,
            direction="maximize",
            sampler=optuna.samplers.TPESampler(seed=42),
        )

        # Create partial function with param_space
        objective = partial(self._objective, param_space=param_space)

        # Log optimization start
        logger.info(
            f"Starting optimization with {self.n_trials} trials, {self.n_jobs} parallel jobs"
        )
        logger.info(f"Parameter space: {param_space}")
        logger.info(f"Metric weights: {self.metric_weights}")
        logger.info(f"Benchmark weights: {self.benchmark_weights}")

        # Initialize progress tracking
        if self.progress_callback:
            self.progress_callback(
                0,
                self.n_trials,
                {
                    "status": "starting",
                    "stage": "initialization",
                    "trials_completed": 0,
                    "total_trials": self.n_trials,
                },
            )

        try:
            # Run optimization
            self.study.optimize(
                objective,
                n_trials=self.n_trials,
                timeout=self.timeout,
                n_jobs=self.n_jobs,
                callbacks=[self._optimization_callback],
                show_progress_bar=True,
            )

            # Store best parameters
            if self.study is None:
                raise RuntimeError("Study was not created")
            _completed_study = self.study
            self.best_params = _completed_study.best_params

            # Save the results
            self._save_results()

            # Create visualizations
            self._create_visualizations()

            if self.best_params is None:
                raise RuntimeError("No best parameters found")
            logger.info(
                f"Optimization complete. Best parameters: {self.best_params}"
            )
            logger.info(f"Best value: {_completed_study.best_value}")

            # Report completion
            if self.progress_callback:
                self.progress_callback(
                    self.n_trials,
                    self.n_trials,
                    {
                        "status": "completed",
                        "stage": "finished",
                        "trials_completed": len(_completed_study.trials),
                        "total_trials": self.n_trials,
                        "best_params": self.best_params,
                        "best_value": _completed_study.best_value,
                    },
                )

            return self.best_params, _completed_study.best_value

        except KeyboardInterrupt:
            logger.info("Optimization interrupted by user")
            # Still save what we have
            self._save_results()
            self._create_visualizations()

            if self.study is None:
                raise RuntimeError("Study was not created")
            _interrupted_study = self.study
            # Report interruption
            if self.progress_callback:
                self.progress_callback(
                    len(_interrupted_study.trials),
                    self.n_trials,
                    {
                        "status": "interrupted",
                        "stage": "interrupted",
                        "trials_completed": len(_interrupted_study.trials),
                        "total_trials": self.n_trials,
                        "best_params": _interrupted_study.best_params,
                        "best_value": _interrupted_study.best_value,
                    },
                )

            return _interrupted_study.best_params, _interrupted_study.best_value

    def _get_default_param_space(self) -> Dict[str, Any]:
        """
        Get default parameter search space.

        Returns:
            Dictionary defining the default parameter search spaces
        """
        return {
            "iterations": {
                "type": "int",
                "low": 1,
                "high": 5,
                "step": 1,
            },
            "questions_per_iteration": {
                "type": "int",
                "low": 1,
                "high": 5,
                "step": 1,
            },
            "search_strategy": {
                "type": "categorical",
                "choices": [
                    "source-based",
                    "focused-iteration",
                    "focused-iteration-standard",
                    "topic-organization",
                ],
            },
            "max_results": {
                "type": "int",
                "low": 10,
                "high": 100,
                "step": 10,
            },
        }

    def _objective(
        self, trial: optuna.Trial, param_space: Dict[str, Any]
    ) -> float:
        """
        Objective function for Optuna optimization.

        Args:
            trial: Optuna trial object
            param_space: Dictionary defining parameter search spaces

        Returns:
            Score to maximize
        """
        # Generate parameters for this trial
        params: Dict[str, Any] = {}
        for param_name, param_config in param_space.items():
            param_type = param_config["type"]

            if param_type == "int":
                params[param_name] = trial.suggest_int(
                    param_name,
                    param_config["low"],
                    param_config["high"],
                    step=param_config.get("step", 1),
                )
            elif param_type == "float":
                params[param_name] = trial.suggest_float(
                    param_name,
                    param_config["low"],
                    param_config["high"],
                    step=param_config.get("step"),
                    log=param_config.get("log", False),
                )
            elif param_type == "categorical":
                params[param_name] = trial.suggest_categorical(
                    param_name, param_config["choices"]
                )

        # Log the trial parameters
        logger.info(f"Trial {trial.number}: {params}")

        # Update progress callback if available
        if self.progress_callback:
            self.progress_callback(
                trial.number,
                self.n_trials,
                {
                    "status": "running",
                    "stage": "trial_started",
                    "trial_number": trial.number,
                    "params": params,
                    "trials_completed": trial.number,
                    "total_trials": self.n_trials,
                },
            )

        # Run an experiment with these parameters
        try:
            start_time = time.time()
            result = self._run_experiment(params)
            duration = time.time() - start_time

            # Store details about the trial
            trial_info = {
                "trial_number": trial.number,
                "params": params,
                "result": result,
                "score": result.get("score", 0),
                "duration": duration,
                "timestamp": datetime.now(UTC).isoformat(),
            }
            self.trials_history.append(trial_info)

            # Update callback with results
            if self.progress_callback:
                self.progress_callback(
                    trial.number,
                    self.n_trials,
                    {
                        "status": "completed",
                        "stage": "trial_completed",
                        "trial_number": trial.number,
                        "params": params,
                        "score": result.get("score", 0),
                        "trials_completed": trial.number + 1,
                        "total_trials": self.n_trials,
                    },
                )

            logger.info(
                f"Trial {trial.number} completed: {params}, score: {result['score']:.4f}"
            )

            return float(result["score"])
        except Exception as e:
            logger.exception(f"Error in trial {trial.number}")

            # Update callback with error
            if self.progress_callback:
                self.progress_callback(
                    trial.number,
                    self.n_trials,
                    {
                        "status": "error",
                        "stage": "trial_error",
                        "trial_number": trial.number,
                        "params": params,
                        "error": str(e),
                        "trials_completed": trial.number,
                        "total_trials": self.n_trials,
                    },
                )

            return float("-inf")  # Return a very low score for failed trials

    def _run_experiment(self, params: Dict[str, Any]) -> Dict[str, Any]:
        """
        Run a single experiment with the given parameters.

        Args:
            params: Dictionary of parameters to test

        Returns:
            Results dictionary with metrics and score
        """
        # Extract parameters
        iterations = params.get("iterations", 2)
        questions_per_iteration = params.get("questions_per_iteration", 2)
        search_strategy = params.get("search_strategy", "source-based")
        max_results = params.get("max_results", 50)

        # Initialize profiling tools
        speed_profiler = SpeedProfiler()

        # Start profiling
        speed_profiler.start()

        try:
            # Create system configuration
            system_config = {
                "iterations": iterations,
                "questions_per_iteration": questions_per_iteration,
                "search_strategy": search_strategy,
                "search_tool": self.search_tool,
                "max_results": max_results,
                "model_name": self.model_name,
                "provider": self.provider,
            }

            # Evaluate quality using composite benchmark evaluator
            # Use a small number of examples for efficiency
            benchmark_dir = str(Path(self.output_dir) / "benchmark_temp")
            quality_results = self.benchmark_evaluator.evaluate(
                system_config=system_config,
                num_examples=5,  # Small number for optimization efficiency
                output_dir=benchmark_dir,
            )

            # Stop timing
            speed_profiler.stop()
            timing_results = speed_profiler.get_summary()

            # Extract key metrics
            quality_score = quality_results.get("quality_score", 0.0)
            benchmark_results = quality_results.get("benchmark_results", {})

            # Speed score: convert duration to a 0-1 score where faster is better
            # Using a reasonable threshold (e.g., 180 seconds for 5 examples)
            # Below this threshold: high score, above it: declining score
            total_duration = timing_results.get("total_duration", 180)
            speed_score = max(0.0, min(1.0, 1.0 - (total_duration - 60) / 180))

            # Calculate combined score based on weights
            combined_score = (
                self.metric_weights.get("quality", 0.6) * quality_score
                + self.metric_weights.get("speed", 0.4) * speed_score
            )

            # Return streamlined results
            return {
                "quality_score": quality_score,
                "benchmark_results": benchmark_results,
                "speed_score": speed_score,
                "total_duration": total_duration,
                "score": combined_score,
                "success": True,
            }

        except Exception as e:
            # Stop profiling on error
            speed_profiler.stop()

            # Log error
            logger.exception("Error in experiment")

            # Return error information
            return {"error": str(e), "score": 0.0, "success": False}

    def _optimization_callback(self, study: optuna.Study, trial: optuna.Trial):
        """
        Callback for the Optuna optimization process.

        Args:
            study: Optuna study object
            trial: Current trial
        """
        # Save intermediate results periodically
        if trial.number % 10 == 0 and trial.number > 0:
            self._save_results()
            self._create_quick_visualizations()

    def _save_results(self):
        """Save the optimization results to disk."""
        # Create a timestamp for filenames
        timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S")

        # Save trial history
        from ...security.file_write_verifier import write_json_verified

        history_file = str(
            Path(self.output_dir) / f"{self.study_name}_history.json"
        )

        # Convert numpy values to native Python types for JSON serialization
        clean_history: List[Dict[str, Any]] = []
        for trial in self.trials_history:
            clean_trial: Dict[str, Any] = {}
            for k, v in trial.items():
                if isinstance(v, dict):
                    clean_trial[k] = {
                        dk: (float(dv) if isinstance(dv, np.number) else dv)
                        for dk, dv in v.items()
                    }
                elif isinstance(v, np.number):
                    clean_trial[k] = float(v)
                else:
                    clean_trial[k] = v
            clean_history.append(clean_trial)

        # Sanitize sensitive data before writing to disk
        sanitized_history = sanitize_data(clean_history)

        write_json_verified(
            history_file,
            sanitized_history,
            "benchmark.allow_file_output",
            context="optimization history",
        )

        # Save current best parameters
        if (
            self.study
            and hasattr(self.study, "best_params")
            and self.study.best_params
        ):
            best_params_file = str(
                Path(self.output_dir) / f"{self.study_name}_best_params.json"
            )

            best_params_data = {
                "best_params": self.study.best_params,
                "best_value": float(self.study.best_value),
                "n_trials": len(self.study.trials),
                "timestamp": timestamp,
                "base_query": self.base_query,
                "model_name": self.model_name,
                "provider": self.provider,
                "search_tool": self.search_tool,
                "metric_weights": self.metric_weights,
                "benchmark_weights": self.benchmark_weights,
            }

            # Sanitize sensitive data before writing to disk
            sanitized_best_params = sanitize_data(best_params_data)

            write_json_verified(
                best_params_file,
                sanitized_best_params,
                "benchmark.allow_file_output",
                context="optimization best params",
            )

        # Save the Optuna study
        if self.study:
            study_file = str(
                Path(self.output_dir) / f"{self.study_name}_study.pkl"
            )
            joblib.dump(self.study, study_file)

        logger.info(f"Results saved to {self.output_dir}")

    def _create_visualizations(self):
        """Create and save comprehensive visualizations of the optimization results."""
        if not PLOTTING_AVAILABLE:
            logger.warning(
                "Matplotlib not available, skipping visualization creation"
            )
            return
        _ensure_plotting_loaded()

        if not self.study or len(self.study.trials) < 2:
            logger.warning("Not enough trials to create visualizations")
            return

        # Create directory for visualizations
        _viz_dir_path = Path(self.output_dir) / "visualizations"
        # nosemgrep: semgrep.rules.path-traversal-risk, reason: CLI/Python optimizer output directory with a fixed visualizations child
        _viz_dir_path.mkdir(parents=True, exist_ok=True)
        viz_dir = str(_viz_dir_path)

        # Create Optuna visualizations
        self._create_optuna_visualizations(viz_dir)

        # Create custom visualizations
        self._create_custom_visualizations(viz_dir)

        logger.info(f"Visualizations saved to {viz_dir}")

    def _create_quick_visualizations(self):
        """Create a smaller set of visualizations for intermediate progress."""
        if (
            not PLOTTING_AVAILABLE
            or not self.study
            or len(self.study.trials) < 2
        ):
            return
        _ensure_plotting_loaded()

        # Create directory for visualizations
        _quick_viz_dir_path = Path(self.output_dir) / "visualizations"
        # nosemgrep: semgrep.rules.path-traversal-risk, reason: CLI/Python optimizer output directory with a fixed visualizations child
        _quick_viz_dir_path.mkdir(parents=True, exist_ok=True)
        viz_dir = str(_quick_viz_dir_path)

        # Create optimization history only (faster than full visualization)
        try:
            fig = plot_optimization_history(self.study)
            fig.write_image(
                str(
                    Path(viz_dir)
                    / f"{self.study_name}_optimization_history_current.png"
                )
            )
        except Exception:
            logger.exception("Error creating optimization history plot")

    def _create_optuna_visualizations(self, viz_dir: str):
        """
        Create and save Optuna's built-in visualizations.

        Args:
            viz_dir: Directory to save visualizations
        """
        if not self.study or not PLOTTING_AVAILABLE:
            return
        _ensure_plotting_loaded()
        study = self.study
        timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S")

        # 1. Optimization history
        try:
            fig = plot_optimization_history(study)
            fig.write_image(
                str(
                    Path(viz_dir)
                    / f"{self.study_name}_optimization_history_{timestamp}.png"
                )
            )
        except Exception:
            logger.exception("Error creating optimization history plot")

        # 2. Parameter importances
        try:
            fig = plot_param_importances(study)
            fig.write_image(
                str(
                    Path(viz_dir)
                    / f"{self.study_name}_param_importances_{timestamp}.png"
                )
            )
        except Exception:
            logger.exception("Error creating parameter importances plot")

        # 3. Slice plot for each parameter
        try:
            for param_name in study.best_params.keys():
                fig = plot_slice(study, [param_name])
                fig.write_image(
                    str(
                        Path(viz_dir)
                        / f"{self.study_name}_slice_{param_name}_{timestamp}.png"
                    )
                )
        except Exception:
            logger.exception("Error creating slice plots")

        # 4. Contour plots for important parameter pairs
        try:
            # Get all parameter names
            param_names = list(study.best_params.keys())

            # Create contour plots for each pair
            for i in range(len(param_names)):
                for j in range(i + 1, len(param_names)):
                    try:
                        fig = plot_contour(
                            study, params=[param_names[i], param_names[j]]
                        )
                        fig.write_image(
                            str(
                                Path(viz_dir)
                                / f"{self.study_name}_contour_{param_names[i]}_{param_names[j]}_{timestamp}.png"
                            )
                        )
                    except Exception:
                        logger.warning(
                            f"Error creating contour plot for {param_names[i]} vs {param_names[j]}"
                        )
        except Exception:
            logger.exception("Error creating contour plots")

    def _create_custom_visualizations(self, viz_dir: str):
        """
        Create custom visualizations based on trial history.

        Args:
            viz_dir: Directory to save visualizations
        """
        if not self.trials_history:
            return

        timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S")

        # Create quality vs speed plot
        self._create_quality_vs_speed_plot(viz_dir, timestamp)

        # Create parameter evolution plots
        self._create_parameter_evolution_plots(viz_dir, timestamp)

        # Create trial duration vs score plot
        self._create_duration_vs_score_plot(viz_dir, timestamp)

    def _create_quality_vs_speed_plot(self, viz_dir: str, timestamp: str):
        """Create a plot showing quality vs. speed trade-off."""
        if not self.trials_history:
            return

        # Extract data from successful trials
        successful_trials = [
            t
            for t in self.trials_history
            if t.get("result", {}).get("success", False)
        ]

        if not successful_trials:
            logger.warning("No successful trials for visualization")
            return

        try:
            plt.figure(figsize=(10, 8))

            # Extract metrics
            quality_scores = []
            speed_scores = []
            labels = []
            iterations_values = []
            questions_values = []

            for trial in successful_trials:
                result = trial["result"]
                quality = result.get("quality_score", 0)
                speed = result.get("speed_score", 0)
                iterations = trial["params"].get("iterations", 0)
                questions = trial["params"].get("questions_per_iteration", 0)

                quality_scores.append(quality)
                speed_scores.append(speed)
                labels.append(f"Trial {trial['trial_number']}")
                iterations_values.append(iterations)
                questions_values.append(questions)

            # Create scatter plot with size based on iterations*questions
            sizes = [
                i * q * 5
                for i, q in zip(
                    iterations_values, questions_values, strict=False
                )
            ]
            scatter = plt.scatter(
                quality_scores,
                speed_scores,
                s=sizes,
                alpha=0.7,
                c=range(len(quality_scores)),
                cmap="viridis",
            )

            # Highlight best trial
            best_trial = max(
                successful_trials,
                key=lambda x: x.get("result", {}).get("score", 0),
            )
            best_quality = best_trial["result"].get("quality_score", 0)
            best_speed = best_trial["result"].get("speed_score", 0)
            best_iter = best_trial["params"].get("iterations", 0)
            best_questions = best_trial["params"].get(
                "questions_per_iteration", 0
            )

            plt.scatter(
                [best_quality],
                [best_speed],
                s=200,
                facecolors="none",
                edgecolors="red",
                linewidth=2,
                label=f"Best: {best_iter}×{best_questions}",
            )

            # Add annotations for key points
            for i, (q, s, label) in enumerate(
                zip(quality_scores, speed_scores, labels, strict=False)
            ):
                if i % max(1, len(quality_scores) // 5) == 0:  # Label ~5 points
                    plt.annotate(
                        f"{iterations_values[i]}×{questions_values[i]}",
                        (q, s),
                        xytext=(5, 5),
                        textcoords="offset points",
                    )

            # Add colorbar and labels
            cbar = plt.colorbar(scatter)
            cbar.set_label("Trial Progression")

            # Add benchmark weight information
            weights_str = ", ".join(
                [f"{k}:{v:.1f}" for k, v in self.benchmark_weights.items()]
            )
            plt.title(
                f"Quality vs. Speed Trade-off\nBenchmark Weights: {weights_str}"
            )
            plt.xlabel("Quality Score (Benchmark Accuracy)")
            plt.ylabel("Speed Score")
            plt.grid(True, linestyle="--", alpha=0.7)

            # Add legend explaining size
            legend_elements = [
                Line2D(
                    [0],
                    [0],
                    marker="o",
                    color="w",
                    markerfacecolor="gray",
                    markersize=np.sqrt(n * 5 / np.pi),
                    label=f"{n} Total Questions",
                )
                for n in [5, 10, 15, 20, 25]
            ]
            plt.legend(handles=legend_elements, title="Workload")

            # Save the figure
            plt.tight_layout()
            plt.savefig(
                str(
                    Path(viz_dir)
                    / f"{self.study_name}_quality_vs_speed_{timestamp}.png"
                )
            )
            plt.close()
        except Exception:
            logger.exception("Error creating quality vs speed plot")

    def _create_parameter_evolution_plots(self, viz_dir: str, timestamp: str):
        """Create plots showing how parameter values evolve over trials."""
        try:
            successful_trials = [
                t
                for t in self.trials_history
                if t.get("result", {}).get("success", False)
            ]

            if not successful_trials or len(successful_trials) < 5:
                return

            # Get key parameters
            main_params = list(successful_trials[0]["params"].keys())

            # For each parameter, plot its values over trials
            for param_name in main_params:
                plt.figure(figsize=(12, 6))

                trial_numbers = []
                param_values = []
                scores = []

                for trial in self.trials_history:
                    if "params" in trial and param_name in trial["params"]:
                        trial_numbers.append(trial["trial_number"])
                        param_values.append(trial["params"][param_name])
                        scores.append(trial.get("score", 0))

                # Create evolution plot
                scatter = plt.scatter(
                    trial_numbers,
                    param_values,
                    c=scores,
                    cmap="plasma",
                    alpha=0.8,
                    s=80,
                )

                # Add best trial marker
                best_trial_idx = scores.index(max(scores))
                plt.scatter(
                    [trial_numbers[best_trial_idx]],
                    [param_values[best_trial_idx]],
                    s=150,
                    facecolors="none",
                    edgecolors="red",
                    linewidth=2,
                    label=f"Best Value: {param_values[best_trial_idx]}",
                )

                # Add colorbar
                cbar = plt.colorbar(scatter)
                cbar.set_label("Score")

                # Set chart properties
                plt.title(f"Evolution of {param_name} Values")
                plt.xlabel("Trial Number")
                plt.ylabel(param_name)
                plt.grid(True, linestyle="--", alpha=0.7)
                plt.legend()

                # For categorical parameters, adjust y-axis
                if isinstance(param_values[0], str):
                    unique_values = sorted(set(param_values))
                    plt.yticks(range(len(unique_values)), unique_values)

                # Save the figure
                plt.tight_layout()
                plt.savefig(
                    str(
                        Path(viz_dir)
                        / f"{self.study_name}_param_evolution_{param_name}_{timestamp}.png"
                    )
                )
                plt.close()
        except Exception:
            logger.exception("Error creating parameter evolution plots")

    def _create_duration_vs_score_plot(self, viz_dir: str, timestamp: str):
        """Create a plot showing trial duration vs score."""
        try:
            plt.figure(figsize=(10, 6))

            successful_trials = [
                t
                for t in self.trials_history
                if t.get("result", {}).get("success", False)
            ]

            if not successful_trials:
                return

            trial_durations = []
            trial_scores = []
            trial_iterations = []
            trial_questions = []

            for trial in successful_trials:
                duration = trial.get("duration", 0)
                score = trial.get("score", 0)
                iterations = trial.get("params", {}).get("iterations", 1)
                questions = trial.get("params", {}).get(
                    "questions_per_iteration", 1
                )

                trial_durations.append(duration)
                trial_scores.append(score)
                trial_iterations.append(iterations)
                trial_questions.append(questions)

            # Total questions per trial
            total_questions = [
                i * q
                for i, q in zip(trial_iterations, trial_questions, strict=False)
            ]

            # Create scatter plot with size based on total questions
            plt.scatter(
                trial_durations,
                trial_scores,
                s=[
                    q * 5 for q in total_questions
                ],  # Size based on total questions
                alpha=0.7,
                c=range(len(trial_durations)),
                cmap="viridis",
            )

            # Add labels
            plt.xlabel("Trial Duration (seconds)")
            plt.ylabel("Score")
            plt.title("Trial Duration vs. Score")
            plt.grid(True, linestyle="--", alpha=0.7)

            # Add trial number annotations for selected points
            for i, (d, s) in enumerate(
                zip(trial_durations, trial_scores, strict=False)
            ):
                if (
                    i % max(1, len(trial_durations) // 5) == 0
                ):  # Annotate ~5 points
                    plt.annotate(
                        f"{trial_iterations[i]}×{trial_questions[i]}",
                        (d, s),
                        xytext=(5, 5),
                        textcoords="offset points",
                    )

            # Save the figure
            plt.tight_layout()
            plt.savefig(
                str(
                    Path(viz_dir)
                    / f"{self.study_name}_duration_vs_score_{timestamp}.png"
                )
            )
            plt.close()
        except Exception:
            logger.exception("Error creating duration vs score plot")


def optimize_parameters(
    query: str,
    param_space: Optional[Dict[str, Any]] = None,
    output_dir: str = str(Path("data") / "optimization_results"),
    model_name: Optional[str] = None,
    provider: Optional[str] = None,
    search_tool: Optional[str] = None,
    temperature: float = 0.7,
    n_trials: int = 30,
    timeout: Optional[int] = None,
    n_jobs: int = 1,
    study_name: Optional[str] = None,
    optimization_metrics: Optional[List[str]] = None,
    metric_weights: Optional[Dict[str, float]] = None,
    progress_callback: Optional[Callable[[int, int, Dict], None]] = None,
    benchmark_weights: Optional[Dict[str, float]] = None,
) -> Tuple[Dict[str, Any], float]:
    """
    Optimize parameters for Local Deep Research.

    Args:
        query: The research query to use for all experiments
        param_space: Dictionary defining parameter search spaces (optional)
        output_dir: Directory to save optimization results
        model_name: Name of the LLM model to use
        provider: LLM provider
        search_tool: Search engine to use
        temperature: LLM temperature
        n_trials: Number of parameter combinations to try
        timeout: Maximum seconds to run optimization (None for no limit)
        n_jobs: Number of parallel jobs for optimization
        study_name: Name of the Optuna study
        optimization_metrics: List of metrics to optimize (default: ["quality", "speed"])
        metric_weights: Dictionary of weights for each metric (e.g., {"quality": 0.6, "speed": 0.4})
        progress_callback: Optional callback for progress updates
        benchmark_weights: Dictionary mapping benchmark types to weights
            (e.g., {"simpleqa": 0.6, "browsecomp": 0.4})
            If None, only SimpleQA is used with weight 1.0

    Returns:
        Tuple of (best_parameters, best_score)
    """
    # Create optimizer
    optimizer = OptunaOptimizer(
        base_query=query,
        output_dir=output_dir,
        model_name=model_name,
        provider=provider,
        search_tool=search_tool,
        temperature=temperature,
        n_trials=n_trials,
        timeout=timeout,
        n_jobs=n_jobs,
        study_name=study_name,
        optimization_metrics=optimization_metrics,
        metric_weights=metric_weights,
        progress_callback=progress_callback,
        benchmark_weights=benchmark_weights,
    )

    # Run optimization
    return optimizer.optimize(param_space)


def optimize_for_speed(
    query: str,
    n_trials: int = 20,
    output_dir: str = str(Path("data") / "optimization_results"),
    model_name: Optional[str] = None,
    provider: Optional[str] = None,
    search_tool: Optional[str] = None,
    progress_callback: Optional[Callable[[int, int, Dict], None]] = None,
    benchmark_weights: Optional[Dict[str, float]] = None,
) -> Tuple[Dict[str, Any], float]:
    """
    Optimize parameters with a focus on speed performance.

    Args:
        query: The research query to use for all experiments
        n_trials: Number of parameter combinations to try
        output_dir: Directory to save optimization results
        model_name: Name of the LLM model to use
        provider: LLM provider
        search_tool: Search engine to use
        progress_callback: Optional callback for progress updates
        benchmark_weights: Dictionary mapping benchmark types to weights
            (e.g., {"simpleqa": 0.6, "browsecomp": 0.4})
            If None, only SimpleQA is used with weight 1.0

    Returns:
        Tuple of (best_parameters, best_score)
    """
    # Focus on speed with reduced parameter space
    param_space = {
        "iterations": {
            "type": "int",
            "low": 1,
            "high": 3,
            "step": 1,
        },
        "questions_per_iteration": {
            "type": "int",
            "low": 1,
            "high": 3,
            "step": 1,
        },
        "search_strategy": {
            "type": "categorical",
            "choices": ["source-based", "focused-iteration"],
        },
    }

    # Speed-focused weights
    metric_weights = {"speed": 0.8, "quality": 0.2}

    return optimize_parameters(
        query=query,
        param_space=param_space,
        output_dir=output_dir,
        model_name=model_name,
        provider=provider,
        search_tool=search_tool,
        n_trials=n_trials,
        metric_weights=metric_weights,
        optimization_metrics=["speed", "quality"],
        progress_callback=progress_callback,
        benchmark_weights=benchmark_weights,
    )


def optimize_for_quality(
    query: str,
    n_trials: int = 30,
    output_dir: str = str(Path("data") / "optimization_results"),
    model_name: Optional[str] = None,
    provider: Optional[str] = None,
    search_tool: Optional[str] = None,
    progress_callback: Optional[Callable[[int, int, Dict], None]] = None,
    benchmark_weights: Optional[Dict[str, float]] = None,
) -> Tuple[Dict[str, Any], float]:
    """
    Optimize parameters with a focus on result quality.

    Args:
        query: The research query to use for all experiments
        n_trials: Number of parameter combinations to try
        output_dir: Directory to save optimization results
        model_name: Name of the LLM model to use
        provider: LLM provider
        search_tool: Search engine to use
        progress_callback: Optional callback for progress updates
        benchmark_weights: Dictionary mapping benchmark types to weights
            (e.g., {"simpleqa": 0.6, "browsecomp": 0.4})
            If None, only SimpleQA is used with weight 1.0

    Returns:
        Tuple of (best_parameters, best_score)
    """
    # Quality-focused weights
    metric_weights = {"quality": 0.9, "speed": 0.1}

    return optimize_parameters(
        query=query,
        output_dir=output_dir,
        model_name=model_name,
        provider=provider,
        search_tool=search_tool,
        n_trials=n_trials,
        metric_weights=metric_weights,
        optimization_metrics=["quality", "speed"],
        progress_callback=progress_callback,
        benchmark_weights=benchmark_weights,
    )


def optimize_for_efficiency(
    query: str,
    n_trials: int = 25,
    output_dir: str = str(Path("data") / "optimization_results"),
    model_name: Optional[str] = None,
    provider: Optional[str] = None,
    search_tool: Optional[str] = None,
    progress_callback: Optional[Callable[[int, int, Dict], None]] = None,
    benchmark_weights: Optional[Dict[str, float]] = None,
) -> Tuple[Dict[str, Any], float]:
    """
    Optimize parameters with a focus on resource efficiency.

    Args:
        query: The research query to use for all experiments
        n_trials: Number of parameter combinations to try
        output_dir: Directory to save optimization results
        model_name: Name of the LLM model to use
        provider: LLM provider
        search_tool: Search engine to use
        progress_callback: Optional callback for progress updates
        benchmark_weights: Dictionary mapping benchmark types to weights
            (e.g., {"simpleqa": 0.6, "browsecomp": 0.4})
            If None, only SimpleQA is used with weight 1.0

    Returns:
        Tuple of (best_parameters, best_score)
    """
    # Balance of quality, speed and resource usage
    metric_weights = {"quality": 0.4, "speed": 0.3, "resource": 0.3}

    return optimize_parameters(
        query=query,
        output_dir=output_dir,
        model_name=model_name,
        provider=provider,
        search_tool=search_tool,
        n_trials=n_trials,
        metric_weights=metric_weights,
        optimization_metrics=["quality", "speed", "resource"],
        progress_callback=progress_callback,
        benchmark_weights=benchmark_weights,
    )