Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

81.1 KB
Raw Blame
"""Benchmark service for handling web-based benchmark execution."""

import hashlib
import json
import threading
import time
from contextlib import nullcontext
from datetime import UTC, datetime
from enum import Enum
from typing import Any, Dict, List, Optional, Tuple

from loguru import logger
from sqlalchemy.exc import SQLAlchemyError

from ...api.research_functions import quick_summary
from ...exceptions import ResearchTerminatedException
from ...settings.exceptions import SettingsValueError
from ...settings.manager import SnapshotSettingsContext
from ...web.services.research_service import _global_research_semaphore
from ...database.models.benchmark import (
    BenchmarkResult,
    BenchmarkRun,
    BenchmarkStatus,
    DatasetType,
)
from ..datasets import load_dataset
from ..graders import extract_answer_from_response, grade_single_result
from ..runners import format_query
from ...database.thread_local_session import thread_cleanup
from ...utilities.request_context import request_user

# Generic failure message surfaced to API clients in place of the raw benchmark
# exception. ``benchmark_run.error_message`` is set from ``str(e)`` of the
# benchmark run (LLM calls, search engines, grading — the same stack as
# research) and returned by get_benchmark_status() via
# GET /benchmark/api/status/<id>. The raw text can carry server-level LLM
# endpoints / hosts / filesystem paths (settings_snapshot includes LDR_* env
# overrides), so it must not reach the client (CWE-209). Full detail stays in
# the logger.exception calls at the failure sites.
_GENERIC_BENCHMARK_ERROR = (
    "Benchmark run failed due to an internal error. "
    "Check the server logs for details."
)
_GENERIC_BENCHMARK_RESEARCH_ERROR = (
    "Research failed. Check the server logs for details."
)
_RESULT_PERSISTENCE_ERROR = {
    "code": "database_write_failed",
    "message": (
        "Benchmark results could not be saved. Check server logs and "
        "database storage, then retry."
    ),
}


class SocketIOService:
    """Adapter over socketio_asgi's module-level emit functions.

    Keeps the Flask-era ``SocketIOService`` interface (which
    BenchmarkService and its tests are written against) while delegating
    to the FastAPI-native emit path. An earlier port made this a no-op
    stub, silently dropping every benchmark progress / completion /
    rate-limit WebSocket event.

    ``owner`` is keyword-only and REQUIRED here, where the upstream
    ``SocketIOService.emit_to_subscribers`` takes an optional
    ``username=None``. Same fix, stricter contract: a benchmark id is a
    per-user integer, so an omitted owner silently files the event under
    the wrong subscriber set. Making it required turns that into a
    TypeError at the call site instead of a misrouted event.
    """

    def emit_to_subscribers(
        self, event_base, research_id, data, *, owner, enable_logging=True
    ):
        from ...web.services.socketio_asgi import emit_to_subscribers

        return emit_to_subscribers(
            event_base,
            research_id,
            data,
            owner=owner,
            enable_logging=enable_logging,
        )

    def emit_to_room(self, event, data, room=None):
        from ...web.services.socketio_asgi import emit_socket_event

        return emit_socket_event(event, data, room=room)


def _required_persistence_fields(
    result: Any,
) -> tuple[Any, str, DatasetType, Any, Any]:
    """Extract required fields or raise a data-shape error."""
    if not isinstance(result, dict):
        raise TypeError("result must be a mapping")
    query_hash = result["query_hash"]
    if not isinstance(query_hash, str) or not query_hash:
        raise ValueError("query_hash must be a non-empty string")
    return (
        result["example_id"],
        query_hash,
        DatasetType(result["dataset_type"]),
        result["question"],
        result["correct_answer"],
    )


class BenchmarkTaskStatus(Enum):
    """Status values for benchmark tasks in the queue tracker."""

    QUEUED = "queued"
    PROCESSING = "processing"
    COMPLETED = "completed"
    FAILED = "failed"
    CANCELLED = "cancelled"


class BenchmarkQueueTracker:
    """Simple in-memory tracker for benchmark queue status.

    This replaces the removed memory_queue functionality for benchmarks.
    Since benchmarks are temporary and don't need persistence,
    this simple in-memory solution is sufficient.

    Thread-safe for concurrent access from multiple benchmark threads.
    """

    def __init__(self):
        self.tasks = {}
        self._lock = threading.Lock()

    def add_task(
        self, task_id: str, username: str, task_type: str = "benchmark"
    ):
        """Add a new task to tracking.

        Also performs opportunistic cleanup of old completed tasks.
        """
        # Cleanup old tasks before adding new one (outside lock for better performance)
        self.cleanup_completed_tasks()

        with self._lock:
            self.tasks[task_id] = {
                "username": username,
                "task_type": task_type,
                "status": BenchmarkTaskStatus.QUEUED.value,
                "created_at": datetime.now(UTC),
            }

    def update_task_status(self, task_id: str, status: BenchmarkTaskStatus):
        """Update the status of a task."""
        with self._lock:
            if task_id in self.tasks:
                self.tasks[task_id]["status"] = status.value
                self.tasks[task_id]["updated_at"] = datetime.now(UTC)
            else:
                logger.warning(
                    f"Attempted to update status for non-existent task: {task_id}"
                )

    def get_task_status(self, task_id: str) -> Optional[Dict]:
        """Get the current status of a task."""
        with self._lock:
            return self.tasks.get(task_id)

    def remove_task(self, task_id: str):
        """Remove a task from tracking."""
        with self._lock:
            self.tasks.pop(task_id, None)

    def cleanup_completed_tasks(self, max_age_seconds: int = 3600):
        """Remove completed tasks older than max_age_seconds.

        Args:
            max_age_seconds: Maximum age in seconds for completed tasks (default 1 hour)
        """
        with self._lock:
            now = datetime.now(UTC)
            to_remove = []
            for task_id, task_data in self.tasks.items():
                # Only cleanup completed, failed, or cancelled tasks
                if task_data["status"] in [
                    BenchmarkTaskStatus.COMPLETED.value,
                    BenchmarkTaskStatus.FAILED.value,
                    BenchmarkTaskStatus.CANCELLED.value,
                ]:
                    # Check if task has updated_at timestamp
                    updated_at = task_data.get(
                        "updated_at", task_data.get("created_at")
                    )
                    if updated_at:
                        age = (now - updated_at).total_seconds()
                        if age > max_age_seconds:
                            to_remove.append(task_id)

            for task_id in to_remove:
                self.tasks.pop(task_id, None)
                logger.debug(f"Cleaned up old task: {task_id}")

            if to_remove:
                logger.info(f"Cleaned up {len(to_remove)} old benchmark tasks")


class BenchmarkService:
    """Service for managing benchmark runs through the web interface."""

    def __init__(self, socket_service=None):
        # Keyed by (username, BenchmarkRun.id). BenchmarkRun.id is a PER-USER
        # autoincrement, so two users can each own run id 1; keying by the
        # owner as well prevents one user's start_benchmark from clobbering
        # another's in-memory run (and the cross-user reads that enabled).
        self.active_runs: Dict[Tuple[str, int], Dict] = {}
        self.socket_service = socket_service or SocketIOService()
        # Track rate limiting per user benchmark run.
        self.rate_limit_detected: Dict[Tuple[str, int], bool] = {}
        self.queue_tracker = BenchmarkQueueTracker()  # Initialize queue tracker
        # Serializes benchmark-result persistence. _sync_results_to_database
        # runs on the worker thread while sync_pending_results runs on the
        # request thread; without this they can both INSERT the same
        # (benchmark_run_id, query_hash) row and trip the uix_run_query
        # unique constraint, which rolls back the whole pending batch.
        self._results_sync_lock = threading.Lock()

    def generate_config_hash(self, search_config: Dict[str, Any]) -> str:
        """Generate a hash for search configuration compatibility checking."""
        relevant_params = {
            "iterations": search_config.get("iterations"),
            "questions_per_iteration": search_config.get(
                "questions_per_iteration"
            ),
            "search_tool": search_config.get("search_tool"),
            "search_strategy": search_config.get("search_strategy"),
            "model_name": search_config.get("model_name"),
            "provider": search_config.get("provider"),
        }
        # Remove None values
        relevant_params = {
            k: v for k, v in relevant_params.items() if v is not None
        }
        config_str = json.dumps(relevant_params, sort_keys=True)
        return hashlib.md5(  # DevSkim: ignore DS126858
            config_str.encode(), usedforsecurity=False
        ).hexdigest()[:8]

    def generate_query_hash(
        self, question: str, dataset_type: str, example_id: str
    ) -> str:
        """Generate a hash for a query to enable deduplication.

        The example id is part of the key so two distinct examples that share
        the same question text (and dataset) get distinct hashes and each
        persist a row, instead of colliding on the ``uix_run_query`` unique
        constraint and dropping one. Re-hashing the same example is stable, so
        the sync dedup still collapses a genuine re-sync of that example.
        """
        query_content = (
            f"{example_id}|{question.strip()}|{dataset_type.lower()}"
        )
        return hashlib.md5(  # DevSkim: ignore DS126858
            query_content.encode(), usedforsecurity=False
        ).hexdigest()

    def create_benchmark_run(
        self,
        run_name: Optional[str],
        search_config: Dict[str, Any],
        evaluation_config: Dict[str, Any],
        datasets_config: Dict[str, Dict],
        username: Optional[str] = None,
        user_password: Optional[str] = None,
    ) -> int:
        """Create a new benchmark run in the database."""
        from ...database.session_context import get_user_db_session

        with get_user_db_session(username, user_password) as session:
            try:
                config_hash = self.generate_config_hash(search_config)

                # Calculate total examples
                total_examples = sum(
                    config.get("count", 0)
                    for config in datasets_config.values()
                )

                benchmark_run = BenchmarkRun(
                    run_name=run_name,
                    config_hash=config_hash,
                    query_hash_list=[],  # Will be populated as we process
                    search_config=search_config,
                    evaluation_config=evaluation_config,
                    datasets_config=datasets_config,
                    total_examples=total_examples,
                    status=BenchmarkStatus.PENDING,
                )

                session.add(benchmark_run)
                session.commit()

                logger.info(
                    f"Created benchmark run {benchmark_run.id} with config hash {config_hash}"
                )
                return int(benchmark_run.id)

            except Exception:
                session.rollback()
                logger.exception("Error creating benchmark run")
                raise

    def start_benchmark(
        self,
        benchmark_run_id: int,
        username: Optional[str] = None,
        user_password: Optional[str] = None,
    ) -> bool:
        """Start a benchmark run in a background thread."""
        from ...database.session_context import get_user_db_session

        try:
            # Get all data from the database in the main thread
            # This avoids database access from the background thread
            with get_user_db_session(username, user_password) as session:
                # Get benchmark run details
                benchmark_run = (
                    session.query(BenchmarkRun)
                    .filter(BenchmarkRun.id == benchmark_run_id)
                    .first()
                )
                if not benchmark_run:
                    raise ValueError(  # noqa: TRY301 — caught by except, sets FAILED status in DB
                        f"Benchmark run {benchmark_run_id} not found"
                    )

                # Create settings snapshot for thread safety + provenance.
                # The settings snapshot is BEST-EFFORT provenance, not a
                # critical path: a recoverable read failure must not break
                # the user's ability to start a benchmark. We catch only the
                # failures that can legitimately escape get_all_settings:
                # SQLAlchemyError (transient DB issues) and LookupError /
                # ValueError (a stale enum-typed settings row, e.g. a legacy
                # 'CHAT' value that fails enum coercion). Those degrade to an
                # empty snapshot — ldr_version is still recorded. Anything
                # else (a genuine programming error) is deliberately NOT
                # swallowed here: it propagates to the outer handler, which
                # logs a full traceback and marks the run FAILED — the right
                # outcome, since a settings subsystem that breaks unexpectedly
                # would also corrupt the benchmark's own config reads.
                from local_deep_research.settings import SettingsManager

                settings_manager = SettingsManager(session)
                try:
                    settings_snapshot = settings_manager.get_all_settings()
                except SettingsValueError:
                    # Invalid configuration cannot run with an empty snapshot.
                    raise
                except (SQLAlchemyError, LookupError, ValueError):
                    # Use logger.exception so the traceback goes to stderr
                    # without splatting `repr(exc)` into the message body —
                    # exception details may include user data.
                    logger.exception(
                        "Failed to capture settings snapshot for benchmark "
                        f"{benchmark_run_id}; running with empty snapshot."
                    )
                    settings_snapshot = {}

                # Get user password for metrics tracking in background thread
                from ...utilities.request_context import (
                    get_current_session_id,
                )
                from ...database.session_passwords import session_password_store

                _user_password = None
                session_id = get_current_session_id()
                if session_id and username:
                    _user_password = (
                        session_password_store.get_session_password(
                            username, session_id
                        )
                    )
                    if not _user_password:
                        logger.warning(
                            f"No password found for user {username} in current session"
                        )

                # Extract all data we need
                benchmark_data = {
                    "benchmark_run_id": benchmark_run_id,
                    "username": username or "benchmark_user",
                    "user_password": _user_password,  # Add password for metrics tracking
                    "config_hash": benchmark_run.config_hash,
                    "datasets_config": benchmark_run.datasets_config,
                    "search_config": benchmark_run.search_config,
                    "evaluation_config": benchmark_run.evaluation_config,
                    # The grader later calls get_llm using only this snapshot.
                    # Scope it to the authenticated owner so ADAPTIVE can see
                    # their private retriever primary and refuse cloud egress.
                    "settings_snapshot": (
                        {**settings_snapshot, "_username": username}
                        if username
                        else settings_snapshot
                    ),
                }

                # Update status in database. Persist provenance (ldr_version
                # + redacted settings_snapshot) BEFORE the commit so the
                # YAML download can later reproduce the run; the snapshot
                # already exists in memory from line 306, we just store the
                # redacted form so secrets don't sit in the DB even though
                # SQLCipher protects at rest.
                from local_deep_research import __version__
                from local_deep_research.security.data_sanitizer import (
                    DataSanitizer,
                )

                benchmark_run.ldr_version = __version__
                benchmark_run.settings_snapshot = (
                    DataSanitizer.redact_settings_snapshot(settings_snapshot)
                )
                benchmark_run.status = BenchmarkStatus.IN_PROGRESS
                benchmark_run.start_time = datetime.now(UTC)
                session.commit()

            # Store data in memory for the thread. Key by (username, id) so a
            # run another user owns under the same per-user id can't overwrite
            # this entry (see _run_key).
            run_key = self._run_key(username, benchmark_run_id)
            self.active_runs[run_key] = {
                "data": benchmark_data,
                "start_time": datetime.now(UTC),
                "status": "running",
                "results": [],
            }

            # Start background thread. Pass the owner so the thread rebuilds the
            # same composite key for its own active_runs accesses.
            thread = threading.Thread(
                target=self._run_benchmark_thread,
                args=(username, benchmark_run_id),
                daemon=True,
            )
            # Publish the handle before the worker can finish and remove its
            # active_runs entry during result synchronization.
            self.active_runs[run_key]["thread"] = thread
            thread.start()

            logger.info(f"Started benchmark run {benchmark_run_id}")
            return True

        except Exception as e:
            logger.exception(f"Error starting benchmark {benchmark_run_id}")
            # If we populated active_runs before the spawn failed, drop the
            # stale entry — it has no thread and would mislead subsequent
            # cancel_benchmark / get_run_status calls. Recompute the key here
            # (the local run_key may not exist yet if we failed before it).
            self.active_runs.pop(
                self._run_key(username, benchmark_run_id), None
            )
            # Update status using user database
            with get_user_db_session(username, user_password) as session:
                benchmark_run = (
                    session.query(BenchmarkRun)
                    .filter(BenchmarkRun.id == benchmark_run_id)
                    .first()
                )
                if benchmark_run:
                    benchmark_run.status = BenchmarkStatus.FAILED
                    benchmark_run.error_message = str(e)
                    session.commit()
            return False

    @thread_cleanup
    def _run_benchmark_thread(
        self, username: Optional[str], benchmark_run_id: int
    ):
        """Main benchmark execution thread."""
        # A plain threading.Thread does not inherit the request context
        # (contextvars), so quick_summary's @no_db_settings wrapper — which
        # resolves get_settings_manager() before the wrapped function runs —
        # finds no user here and trips the background-thread DB guard:
        # every task failed with "Research error" and 0% accuracy. Push the
        # owner for the whole run, the same way scheduler jobs do
        # (scheduler/background.py). A missing owner pushes nothing (as in
        # scheduler/background.py) rather than a "benchmark_user" fallback,
        # which could resolve an unrelated account of that name.
        user_ctx = request_user(username) if username else nullcontext()
        with user_ctx:
            self._run_benchmark_thread_impl(username, benchmark_run_id)

    def _run_benchmark_thread_impl(
        self, username: Optional[str], benchmark_run_id: int
    ):
        # IMPORTANT: This runs in a background thread, so we cannot access the user database
        # Using in-memory queue tracker for benchmark status tracking

        task_id = None
        # Rebuild the same composite key start_benchmark stored this run under.
        run_key = self._run_key(username, benchmark_run_id)

        # Get the benchmark data that was passed to us
        # We need to retrieve this from the service database or from memory
        benchmark_data = self.active_runs.get(run_key, {}).get("data")

        try:
            if not benchmark_data:
                raise ValueError(  # noqa: TRY301
                    f"Benchmark data for run {benchmark_run_id} not found"
                )
            # Set up settings context for thread-local access
            settings_snapshot = benchmark_data.get("settings_snapshot", {})
            # ``or`` (not ``.get(..., default)``) so an explicit None still
            # falls back — symmetric with _run_key's normalization.
            username = benchmark_data.get("username") or "benchmark_user"

            # Create a settings context that threads can use
            settings_context = SnapshotSettingsContext(
                settings_snapshot,
                username=username,
                missing_key_log_level="WARNING",
            )

            # Set the context in thread-local storage
            from ...config.thread_settings import set_settings_context

            set_settings_context(settings_context)

            # Extract all the data we need
            datasets_config = benchmark_data["datasets_config"]
            search_config = benchmark_data["search_config"]
            evaluation_config = benchmark_data["evaluation_config"]

            # Create task queue
            task_queue = self._create_task_queue(
                datasets_config,
                benchmark_run_id,
            )

            # Calculate totals
            total_examples = len(task_queue)
            completed_examples = 0

            # Initialize task tracking
            task_id = f"benchmark_{benchmark_run_id}_{int(datetime.now(UTC).timestamp())}"
            username = benchmark_data.get("username") or "benchmark_user"
            self.queue_tracker.add_task(task_id, username, "benchmark")
            self.queue_tracker.update_task_status(
                task_id, BenchmarkTaskStatus.PROCESSING
            )

            # Track progress in memory
            progress_info = {
                "total_examples": total_examples,
                "completed_examples": completed_examples,
                "failed_examples": 0,
                "start_time": datetime.now(UTC),
            }

            # Process tasks
            logger.info(
                f"Benchmark {benchmark_run_id} starting to process {len(task_queue)} tasks"
            )
            for i, task in enumerate(task_queue):
                # Check if benchmark has been cancelled
                if (
                    run_key in self.active_runs
                    and self.active_runs[run_key].get("status") == "cancelled"
                ):
                    logger.info(
                        f"Benchmark {benchmark_run_id} was cancelled, stopping processing"
                    )
                    break

                logger.info(
                    f"Benchmark {benchmark_run_id} processing task {i + 1}/{len(task_queue)}"
                )
                try:
                    # Add username and password to task for metrics tracking.
                    # ``or`` keeps the fallback symmetric with _run_key: a
                    # missing/None owner still maps to "benchmark_user" so
                    # the publish below hits the same entry cancel reads.
                    task["username"] = benchmark_data.get("username") or (
                        "benchmark_user"
                    )
                    task["user_password"] = benchmark_data.get("user_password")

                    # Acquire the global research semaphore so benchmark
                    # tasks count against the server-wide concurrency limit
                    _global_research_semaphore.acquire()
                    try:
                        # Re-check after the semaphore wait: Cancel may have
                        # landed while blocked, when there was no tracking ID
                        # to flag. Starting the task now would publish a fresh
                        # ID the finished handshake cannot see and run to
                        # completion after Cancel.
                        if (
                            run_key in self.active_runs
                            and self.active_runs[run_key].get("status")
                            == "cancelled"
                        ):
                            logger.info(
                                f"Benchmark {benchmark_run_id} was cancelled "
                                f"while waiting for capacity, skipping task "
                                f"{i + 1}/{len(task_queue)}"
                            )
                            break
                        # Process single task
                        result = self._process_benchmark_task(
                            task,
                            search_config,
                            evaluation_config,
                        )
                    except ResearchTerminatedException:
                        # Cancel arrived mid-task: the in-flight LLM call
                        # aborted via its termination flag (published as
                        # current_tracking_id, see _process_benchmark_task).
                        # Stop the run without counting the aborted task as
                        # failed — the was_cancelled check below still marks
                        # the run CANCELLED.
                        logger.info(
                            f"Benchmark {benchmark_run_id} task {i + 1} "
                            f"aborted mid-task on cancel; stopping run"
                        )
                        break
                    finally:
                        _global_research_semaphore.release()
                        # Drop the finished task's research scope and flag
                        # so one task's cancellation state can never leak
                        # into the next task's LLM calls.
                        finished_entry = self.active_runs.get(run_key)
                        finished_tid = (
                            finished_entry.pop("current_tracking_id", None)
                            if finished_entry is not None
                            else None
                        )
                        if finished_tid:
                            try:
                                from ...web.research_state import (
                                    clear_termination_flag,
                                )

                                clear_termination_flag(finished_tid)
                            except Exception:
                                logger.debug(
                                    "Failed to clear benchmark task flag "
                                    "(non-critical)"
                                )
                        # Close the DB sessions this task opened through
                        # db_utils' per-thread cache (quick_summary's ambient
                        # settings manager). The run's thread lives for every
                        # task, so without this they would accumulate until
                        # @thread_cleanup at thread exit; the next task
                        # re-opens one lazily on a cache miss.
                        try:
                            from ...utilities.db_utils import (
                                cleanup_cached_user_sessions_current_thread,
                            )

                            cleanup_cached_user_sessions_current_thread()
                        except Exception:
                            logger.debug(
                                "Failed to clean up benchmark task DB "
                                "sessions (non-critical)"
                            )

                    # Store result in memory for now (will be saved later)
                    if "results" not in self.active_runs[run_key]:
                        self.active_runs[run_key]["results"] = []
                    self.active_runs[run_key]["results"].append(result)

                    # Update progress
                    progress_info["completed_examples"] += 1

                    logger.info(
                        f"Benchmark {benchmark_run_id} task {i + 1}/{len(task_queue)} completed successfully. "
                        f"Progress: {progress_info['completed_examples']}/{progress_info['total_examples']} total examples"
                    )

                    # Send real-time update
                    self._send_progress_update(
                        benchmark_run_id,
                        progress_info["completed_examples"],
                        progress_info["total_examples"],
                        username=username,
                    )

                except Exception as e:
                    logger.exception(f"Error processing task {i}")
                    progress_info["failed_examples"] += 1
                    logger.info(
                        f"Benchmark {benchmark_run_id} task {i + 1}/{len(task_queue)} failed. "
                        f"Total failed: {progress_info['failed_examples']}"
                    )

                    # Check if this is a rate limiting error
                    error_str = str(e).lower()
                    if (
                        "403" in error_str
                        or "rate limit" in error_str
                        or "forbidden" in error_str
                    ):
                        self.rate_limit_detected[run_key] = True
                        # Send rate limit warning via WebSocket. Pass the
                        # run's owner so the socket service files this under
                        # the right per-user subscriber set -- benchmark_run_id
                        # is a per-user integer, so two users can share the
                        # same id (see SocketIOService.__subscription_key).
                        self.socket_service.emit_to_subscribers(
                            "research_progress",
                            benchmark_run_id,
                            {
                                "rate_limit_detected": True,
                                "message": "SearXNG rate limiting detected",
                            },
                            owner=username,
                        )

            # Mark as completed in memory tracker
            progress_info["end_time"] = datetime.now(UTC)

            # Check if benchmark was cancelled
            was_cancelled = (
                run_key in self.active_runs
                and self.active_runs[run_key].get("status") == "cancelled"
            )

            if was_cancelled:
                status = BenchmarkStatus.CANCELLED
                message = "Benchmark cancelled by user"
                if task_id:
                    self.queue_tracker.update_task_status(
                        task_id, BenchmarkTaskStatus.CANCELLED
                    )
            else:
                status = BenchmarkStatus.COMPLETED
                message = "Benchmark completed successfully"
                if task_id:
                    self.queue_tracker.update_task_status(
                        task_id, BenchmarkTaskStatus.COMPLETED
                    )

            # Store completion info for later database update
            self.active_runs[run_key]["completion_info"] = {
                "status": status,
                "end_time": progress_info["end_time"],
                "completed_examples": progress_info["completed_examples"],
                "failed_examples": progress_info["failed_examples"],
            }

            # Send completion notification. Pass the owner so this reaches
            # only their subscriber set (see rate-limit emit above for why).
            self.socket_service.emit_to_subscribers(
                "research_progress",
                benchmark_run_id,
                {
                    "status": "cancelled" if was_cancelled else "completed",
                    "message": message,
                    "progress": (
                        progress_info["completed_examples"]
                        / progress_info["total_examples"]
                        * 100
                    )
                    if progress_info["total_examples"] > 0
                    else 0,
                    "benchmark_run_id": benchmark_run_id,
                },
                owner=username,
            )

        except Exception as e:
            logger.exception(f"Benchmark run {benchmark_run_id} failed")
            # Update task status if we have a task_id
            if task_id:
                self.queue_tracker.update_task_status(
                    task_id, BenchmarkTaskStatus.FAILED
                )
            # Store failure info for later database update
            if run_key in self.active_runs:
                self.active_runs[run_key]["completion_info"] = {
                    "status": BenchmarkStatus.FAILED,
                    "error_message": str(e),
                }
        finally:
            # Clean up active run tracking
            if run_key in self.active_runs:
                # Mark that thread is done but keep data for database update
                self.active_runs[run_key]["thread_complete"] = True

                # Try to save results to database immediately if possible
                self._sync_results_to_database(username, benchmark_run_id)

    def _create_task_queue(
        self,
        datasets_config: Dict,
        benchmark_run_id: int,
    ) -> List[Dict]:
        """Create list of tasks to process.

        Each dataset config may carry a "seed" for reproducible sampling:
        the same seed and count select the same questions on every run,
        keeping accuracy comparable across runs. Without a seed the sample
        is random each time.
        """
        tasks: List[Dict[str, Any]] = []

        for dataset_name, config in datasets_config.items():
            if config.get("count", 0) > 0:
                # This is the local benchmark registry factory, not a Hugging Face download API.
                dataset = load_dataset(  # nosec B615
                    dataset_type=dataset_name,
                    num_examples=config["count"],
                    seed=config.get("seed"),
                )

                for i, example in enumerate(dataset):
                    # All registered datasets expose "problem"/"answer"
                    question = example.get("problem", "")
                    correct_answer = example.get("answer", "")
                    example_id = example.get("id", f"example_{i}")

                    # Generate query hash (keyed on the example so distinct
                    # examples with identical question text don't collide)
                    query_hash = self.generate_query_hash(
                        question, dataset_name, example_id
                    )

                    tasks.append(
                        {
                            "benchmark_run_id": benchmark_run_id,
                            "example_id": example_id,
                            "dataset_type": dataset_name,
                            "question": question,
                            "correct_answer": correct_answer,
                            "query_hash": query_hash,
                            "task_index": len(tasks),
                        }
                    )

        return tasks

    def _process_benchmark_task(
        self, task: Dict, search_config: Dict, evaluation_config: Dict
    ) -> Dict:
        """Process a single benchmark task."""
        try:
            logger.info(
                f"Starting benchmark task {task['task_index'] + 1}: "
                f"example_id={task['example_id']}, dataset={task['dataset_type']}, "
                f"question_preview='{task['question'][:100]}...'"
            )

            # Get settings context from thread-local storage
            from ...config.thread_settings import get_settings_context

            settings_context = get_settings_context()

            # Generate a unique tracking ID for this benchmark task
            import uuid

            tracking_id = str(uuid.uuid4())
            logger.info(
                f"Task {task['example_id']} assigned tracking_id: {tracking_id}"
            )

            # Publish the in-flight research scope so cancel_benchmark can
            # abort this task's LLM calls mid-generation (not just between
            # tasks). quick_summary binds this id to its LLM (explicit
            # research_id) and thread context, and the grading phase below
            # re-arms the same scope — the termination flag then aborts the
            # current call within one streamed chunk. Cleared by the worker
            # loop after every task, so ids never outlive their task.
            # The key must match the worker loop's run_key: task["username"]
            # is set from benchmark_data (with the same "benchmark_user"
            # fallback) before this call, so web runs always hit. A direct
            # caller that bypasses the loop with a mismatched username
            # misses here instead of publishing to another user's same-id
            # run (BenchmarkRun.id is per-user) — log so the trap is visible.
            tracker_entry = self.active_runs.get(
                self._run_key(task.get("username"), task["benchmark_run_id"])
            )
            if tracker_entry is None:
                logger.warning(
                    f"Benchmark {task['benchmark_run_id']} task "
                    f"{task['example_id']} has no live run entry for "
                    f"owner {task.get('username')!r}; skipping scope publish "
                    f"(cancel will only stop between tasks)"
                )
            else:
                tracker_entry["current_tracking_id"] = tracking_id
                # Cooperative handoff: Cancel may have landed between the
                # worker's status checks and this publish, seeing no ID to
                # flag. If the run is already cancelled, arm the flag for the
                # just-published ID and abort before any LLM call starts, so
                # neither ordering loses the stop request.
                if tracker_entry.get("status") == "cancelled":
                    try:
                        from ...web.research_state import (
                            set_termination_flag,
                        )

                        set_termination_flag(tracking_id)
                    except Exception:
                        logger.debug(
                            "Failed to set benchmark termination flag "
                            "(non-critical)"
                        )
                    raise ResearchTerminatedException(  # noqa: TRY301
                        "Benchmark cancelled before task started"
                    )

            # Format query
            formatted_query = format_query(
                task["question"], task["dataset_type"]
            )
            logger.info(
                f"Task {task['example_id']} formatted query: '{formatted_query[:150]}...'"
            )

            # Run research with progress callback for WebSocket updates
            start_time = time.time()
            logger.info(f"Task {task['example_id']} starting research phase...")

            def benchmark_progress_callback(
                status: str, progress: int, data: dict
            ):
                """Progress callback to emit detailed research progress via WebSocket"""
                try:
                    timestamp = datetime.now(UTC).isoformat()

                    # Create research-compatible log entry
                    log_entry = {
                        "time": timestamp,
                        "message": f"Example {task['example_id']}: {status}",
                        "progress": progress,
                        "metadata": {
                            "phase": data.get("phase", "benchmark_processing"),
                            "type": data.get("type", "info"),
                            "example_id": task["example_id"],
                            "benchmark_run_id": task["benchmark_run_id"],
                            **data,  # Include all other data
                        },
                    }

                    # Determine log type based on status/message content
                    if (
                        "complete" in status.lower()
                        or "finished" in status.lower()
                    ):
                        log_entry["metadata"]["type"] = "milestone"
                    elif (
                        "error" in status.lower() or "failed" in status.lower()
                    ):
                        log_entry["metadata"]["type"] = "error"
                    elif (
                        "starting" in status.lower()
                        or "begin" in status.lower()
                    ):
                        log_entry["metadata"]["type"] = "milestone"

                    # Create progress data in research format
                    progress_data = {
                        "progress": progress,
                        "message": status,
                        "status": "in_progress",
                        "log_entry": log_entry,
                        "progress_log": json.dumps(
                            [log_entry]
                        ),  # Array format expected by socket.js
                    }

                    # Emit using research_progress format that the UI expects.
                    # Pass the owner (set on task before this callback's
                    # enclosing call, see _run_benchmark_thread) so this
                    # reaches only their subscriber set -- benchmark_run_id
                    # is a per-user integer and can collide across users.
                    self.socket_service.emit_to_subscribers(
                        "research_progress",
                        task["benchmark_run_id"],
                        progress_data,
                        owner=task.get("username"),
                    )

                except Exception:
                    logger.exception("Error sending benchmark progress update")

            # Get user password from task data
            user_password = task.get("user_password")

            search_result = quick_summary(
                query=formatted_query,
                research_id=tracking_id,  # Pass the tracking ID
                iterations=search_config.get("iterations", 8),
                questions_per_iteration=search_config.get(
                    "questions_per_iteration", 5
                ),
                search_tool=search_config.get("search_tool", "searxng"),
                search_strategy=search_config.get(
                    "search_strategy", "focused_iteration"
                ),
                progress_callback=benchmark_progress_callback,
                model_name=search_config.get("model_name"),
                provider=search_config.get("provider"),
                temperature=search_config.get("temperature", 0.7),
                openai_endpoint_url=search_config.get("openai_endpoint_url"),
                settings_snapshot=settings_context.snapshot,  # Pass settings snapshot for thread safety
                username=task.get("username"),  # Pass username
                user_password=user_password,  # Pass password for metrics tracking
                # The web benchmark runs against the user's encrypted DB
                # (it has username/password and wants search metrics
                # persisted). quick_summary's default is programmatic_mode=True
                # for true library callers; override here so the engine's
                # metrics path stays active.
                programmatic_mode=False,
            )
            processing_time = time.time() - start_time
            logger.info(
                f"Task {task['example_id']} research completed in {processing_time:.2f}s, "
                f"model={search_config.get('model_name')}, provider={search_config.get('provider')}"
            )

            # Extract answer
            response = search_result.get("summary", "")
            findings = search_result.get("findings")
            if isinstance(findings, list) and findings:
                final_finding = findings[-1]
                if (
                    isinstance(final_finding, dict)
                    and final_finding.get("phase") == "Error"
                    and final_finding.get("content") == response
                ):
                    return {
                        **task,
                        "research_error": _GENERIC_BENCHMARK_RESEARCH_ERROR,
                        "completed_at": datetime.now(UTC),
                    }
            logger.info(
                f"Task {task['example_id']} response length: {len(response)} chars"
            )

            extracted_data = extract_answer_from_response(
                response, task["dataset_type"]
            )
            extracted_answer = (
                extracted_data.get("extracted_answer", "")
                if isinstance(extracted_data, dict)
                else str(extracted_data)
            )
            logger.info(
                f"Task {task['example_id']} extracted answer: '{extracted_answer[:100]}...'"
            )

            # Extract sources - handle both direct sources and all_links_of_system
            sources = search_result.get("sources", [])
            if not sources and "all_links_of_system" in search_result:
                sources = search_result.get("all_links_of_system", [])

            # Log for debugging
            logger.debug(f"Search result keys: {list(search_result.keys())}")
            logger.debug(f"Sources found: {len(sources)} items")

            # Prepare result
            result = {
                **task,
                "response": response,
                "extracted_answer": extracted_answer,
                "confidence": str(
                    extracted_data.get("confidence", "100")
                    if isinstance(extracted_data, dict)
                    else "100"
                ),
                "processing_time": processing_time,
                "sources": json.dumps(sources),  # Convert to JSON string
                "completed_at": datetime.now(UTC),
                "research_id": tracking_id,  # Store the UUID in the research_id field
            }

            # Evaluate result - requires proper grading model
            try:
                logger.info(f"Task {task['example_id']} starting evaluation...")
                eval_start_time = time.time()

                # Always attempt evaluation, regardless of provider
                # Modern local models like Ollama are capable of grading
                # Try to evaluate with proper model
                result_data = {
                    "id": task["example_id"],
                    "problem": task["question"],
                    "correct_answer": task["correct_answer"],
                    "response": response,
                    "extracted_answer": extracted_answer,
                }

                # Re-arm the task's research scope around grading:
                # quick_summary cleared its own context on return, and the
                # grader LLM resolves cancellation from the thread context
                # (graders call get_llm without a research_id). The same
                # termination flag therefore aborts a long grading call
                # mid-stream too.
                from ...utilities.thread_context import (
                    clear_search_context,
                    set_search_context,
                )

                set_search_context(
                    {
                        "research_id": tracking_id,
                        "research_query": task["question"],
                        "username": task.get("username"),
                    }
                )
                try:
                    eval_result = grade_single_result(
                        result_data,
                        task["dataset_type"],
                        evaluation_config,
                        settings_context.snapshot,
                    )
                finally:
                    clear_search_context()
                eval_time = time.time() - eval_start_time
                logger.info(
                    f"Task {task['example_id']} evaluation completed in {eval_time:.2f}s"
                )
                if eval_result and not eval_result.get("grading_error"):
                    result.update(
                        {
                            "is_correct": eval_result.get("is_correct", False),
                            "graded_confidence": eval_result.get(
                                "graded_confidence", "0"
                            ),
                            "grader_response": eval_result.get(
                                "grader_response", ""
                            ),
                        }
                    )
                else:
                    error_msg = (
                        eval_result.get(
                            "grading_error", "Unknown evaluation error"
                        )
                        if eval_result
                        else "No evaluation results returned"
                    )
                    result.update(
                        {
                            "is_correct": None,
                            "graded_confidence": "0",
                            "grader_response": f"Evaluation failed: {error_msg}",
                            "evaluation_error": error_msg,
                        }
                    )

            except Exception as e:
                logger.exception("Evaluation error")
                result.update(
                    {
                        "is_correct": None,
                        "graded_confidence": "0",
                        "grader_response": f"Evaluation failed: {e!s}",
                        "evaluation_error": str(e),
                    }
                )

            return result

        except Exception as e:
            logger.exception("Research error")
            return {
                **task,
                "research_error": str(e),
                "completed_at": datetime.now(UTC),
            }

    def _persist_unsaved_results(
        self, session, benchmark_run_id: int, run_data: dict
    ) -> list[int]:
        """Stage INSERTs for this run's not-yet-persisted results; return the
        result indices that were staged.

        Idempotent and rollback-safe. A result is skipped when its index is
        already known-saved (``saved_indices``), its ``query_hash`` is already
        committed for this run, or it repeats a ``query_hash`` staged earlier
        in this same batch. Malformed result entries are also skipped so one
        bad payload cannot make every later valid result roll back (#4861).
        Since ``query_hash`` covers ``example_id`` as well as the question text
        (#5080), a repeat no longer means "a dataset legitimately repeats a
        question": it is the same result entry reaching this method twice, so
        the skip is logged at WARNING rather than passed over in silence
        (#4860).
        Dedup correctness rests on the DB-backed ``seen_hashes``, which
        survives a rollback (the next sync re-reads it), NOT on any in-memory
        flag.

        This method deliberately does NOT mark ``saved_indices``: the caller
        must do that ONLY after its commit succeeds. Marking before the commit
        would, on a commit failure (disk full, lock timeout, ...), leave rows
        flagged saved that were actually rolled back — silently dropped
        forever. Combined with ``self._results_sync_lock`` held by the caller
        ACROSS the commit, this also stops the worker thread and the request
        thread from both inserting the same ``(benchmark_run_id, query_hash)``
        row and tripping the ``uix_run_query`` unique constraint.
        """
        results = run_data.get("results", [])
        # Read-only here — marking indices saved is the caller's job, and only
        # after the commit lands (see the method docstring).
        saved_indices = run_data.get("saved_indices", set())

        # Every query_hash already committed for this run, fetched once. This
        # is the rollback-safe source of truth for dedup; it is extended below
        # so a question repeated within this batch is staged only once.
        seen_hashes = {
            row[0]
            for row in session.query(BenchmarkResult.query_hash)
            .filter(BenchmarkResult.benchmark_run_id == benchmark_run_id)
            .all()
        }

        staged_indices = []
        for idx, result in enumerate(results):
            if idx in saved_indices:
                continue
            try:
                (
                    example_id,
                    query_hash,
                    dataset_type,
                    question,
                    correct_answer,
                ) = _required_persistence_fields(result)
            except (KeyError, TypeError, ValueError) as exc:
                example_id = (
                    result.get("example_id")
                    if isinstance(result, dict)
                    else None
                )
                task_index = (
                    result.get("task_index")
                    if isinstance(result, dict)
                    else None
                )
                logger.warning(
                    f"Benchmark run {benchmark_run_id}: skipping malformed "
                    f"result index {idx} (example_id={example_id!r}, "
                    f"task_index={task_index!r}): {type(exc).__name__}"
                )
                continue
            if query_hash in seen_hashes:
                logger.warning(
                    f"Benchmark run {benchmark_run_id}: skipping result index "
                    f"{idx} (example_id={example_id!r}), its "
                    f"query_hash is already persisted or staged for this run"
                )
                continue
            session.add(
                BenchmarkResult(
                    benchmark_run_id=benchmark_run_id,
                    example_id=example_id,
                    query_hash=query_hash,
                    dataset_type=dataset_type,
                    research_id=result.get("research_id"),
                    question=question,
                    correct_answer=correct_answer,
                    response=result.get("response"),
                    extracted_answer=result.get("extracted_answer"),
                    confidence=result.get("confidence"),
                    processing_time=result.get("processing_time"),
                    sources=result.get("sources"),
                    is_correct=result.get("is_correct"),
                    graded_confidence=result.get("graded_confidence"),
                    grader_response=result.get("grader_response"),
                    completed_at=result.get("completed_at"),
                    research_error=result.get("research_error"),
                    evaluation_error=result.get("evaluation_error"),
                    task_index=result.get("task_index"),
                )
            )
            seen_hashes.add(query_hash)
            staged_indices.append(idx)

        return staged_indices

    @staticmethod
    def _run_key(
        username: Optional[str], benchmark_run_id: int
    ) -> Tuple[str, int]:
        """Composite ``active_runs`` key: ``(username, BenchmarkRun.id)``.

        ``active_runs`` is a process-global map, but ``BenchmarkRun.id`` is a
        PER-USER autoincrement integer -- so two different users can each own a
        run with the same id (e.g. both have id == 1). Keying by the owner too
        means a request can only ever reach the caller's own in-flight run:
        one user's ``start_benchmark`` cannot clobber another's entry, and no
        user-facing lookup can name another user's run. ``username`` is
        normalized the same way ``start_benchmark`` records it so the key a run
        is stored under matches the key every later lookup computes.
        """
        return (username or "benchmark_user", benchmark_run_id)

    @staticmethod
    def _run_owner(run_data: Optional[Dict]) -> Optional[str]:
        """Username that started the given in-memory run entry, if any."""
        if not run_data:
            return None
        return run_data.get("data", {}).get("username")

    def sync_pending_results(
        self, benchmark_run_id: int, username: Optional[str] = None
    ):
        """Sync any pending results to database. Can be called from main thread."""
        # Read outside _results_sync_lock, so the worker thread's
        # ``del self.active_runs[run_key]`` (in _sync_results_to_database,
        # also unlocked) can land between the membership check and the subscript.
        # A plain ``active_runs[key]`` would then raise KeyError, which escapes
        # before the try below and surfaces as a spurious 500 on /api/results.
        # Fetch once so the check and the read see the same state. See #4859.
        # active_runs is keyed by (username, id) (see _run_key), so this lookup
        # can only ever return the caller's own run -- a caller naming another
        # user's per-user id simply misses. That keeps one user's results (and,
        # via the run's stored password, their credentials) unreachable here.
        run_data = self.active_runs.get(
            self._run_key(username, benchmark_run_id)
        )
        if run_data is None:
            return 0

        # For an anonymous/internal caller (username None) fall back to the
        # owner recorded on the run for the DB session below.
        if not username:
            username = self._run_owner(run_data)
        user_password = run_data.get("data", {}).get("user_password")

        saved_count = 0
        session = None
        from ...database.session_context import (
            get_user_db_session,
            safe_rollback,
        )

        try:
            # Serialize with the worker thread's _sync_results_to_database so
            # the two can't insert the same (benchmark_run_id, query_hash) row.
            with self._results_sync_lock:
                with get_user_db_session(username, user_password) as session:
                    staged = self._persist_unsaved_results(
                        session, benchmark_run_id, run_data
                    )
                    if staged:
                        session.commit()
                        # Mark saved ONLY after the commit succeeds, so a
                        # failed commit leaves these for the next sync to retry
                        # instead of dropping them silently.
                        run_data.setdefault("saved_indices", set()).update(
                            staged
                        )
                        saved_count = len(staged)
                        logger.info(
                            f"Saved {saved_count} new results for benchmark "
                            f"{benchmark_run_id}"
                        )
                    # Reaching the end of the transaction proves persistence
                    # is healthy again. Clear a prior transient failure even
                    # when another thread already saved the pending rows.
                    run_data.pop("result_persistence_failed", None)
        except Exception:
            logger.exception(
                f"Error syncing pending results for benchmark {benchmark_run_id}"
            )
            # Thread-local sessions are reused, so a failed flush/commit must be
            # rolled back explicitly or the next use raises PendingRollbackError.
            if session is not None:
                safe_rollback(session, "sync_pending_results")
            run_data["result_persistence_failed"] = True

        return saved_count

    def get_result_persistence_error(
        self, benchmark_run_id: int, username: Optional[str] = None
    ) -> Optional[Dict[str, str]]:
        """Return a safe client-facing result persistence error, if active."""
        # Keyed by (username, id) (see _run_key): a caller can only ever read
        # the state of their own run, never one that belongs to someone else.
        run_data = self.active_runs.get(
            self._run_key(username, benchmark_run_id)
        )
        if not run_data or not run_data.get("result_persistence_failed"):
            return None
        # Return a copy so callers cannot mutate the shared constant.
        return dict(_RESULT_PERSISTENCE_ERROR)

    def _sync_results_to_database(
        self, username: Optional[str], benchmark_run_id: int
    ):
        """Sync benchmark results from memory to database after thread completes."""
        run_key = self._run_key(username, benchmark_run_id)
        if run_key not in self.active_runs:
            return

        run_data = self.active_runs[run_key]
        if not run_data.get("thread_complete"):
            return

        username = run_data.get("data", {}).get("username")
        user_password = run_data.get("data", {}).get("user_password")
        session = None
        from ...database.session_context import (
            get_user_db_session,
            safe_rollback,
        )

        try:
            # Serialize with sync_pending_results (request thread) so the two
            # can't insert the same (benchmark_run_id, query_hash) row.
            with (
                self._results_sync_lock,
                get_user_db_session(username, user_password) as session,
            ):
                # Update benchmark run status
                benchmark_run = (
                    session.query(BenchmarkRun)
                    .filter(BenchmarkRun.id == benchmark_run_id)
                    .first()
                )

                if benchmark_run and "completion_info" in run_data:
                    info = run_data["completion_info"]
                    benchmark_run.status = info["status"]
                    benchmark_run.end_time = info.get(
                        "end_time", datetime.now(UTC)
                    )
                    benchmark_run.completed_examples = info.get(
                        "completed_examples", 0
                    )
                    benchmark_run.failed_examples = info.get(
                        "failed_examples", 0
                    )
                    benchmark_run.error_message = info.get("error_message")

                    # Stage any results not yet persisted (idempotent; safe
                    # against a concurrent sync_pending_results on the request
                    # thread — see _persist_unsaved_results).
                    staged = self._persist_unsaved_results(
                        session, benchmark_run_id, run_data
                    )

                    # Calculate final accuracy
                    if benchmark_run.status == BenchmarkStatus.COMPLETED:
                        correct_results = [
                            r
                            for r in run_data.get("results", [])
                            if r.get("is_correct")
                        ]
                        evaluated_results = [
                            r
                            for r in run_data.get("results", [])
                            if r.get("is_correct") is not None
                        ]

                        if evaluated_results:
                            benchmark_run.overall_accuracy = (
                                len(correct_results) / len(evaluated_results)
                            ) * 100

                            # Calculate processing rate
                            total_time = sum(
                                r.get("processing_time", 0)
                                for r in evaluated_results
                            )
                            if total_time > 0:
                                benchmark_run.processing_rate = len(
                                    evaluated_results
                                ) / (total_time / 60)

                    session.commit()
                    # Mark saved only after the commit lands (see
                    # _persist_unsaved_results) so a failed commit retries.
                    if staged:
                        run_data.setdefault("saved_indices", set()).update(
                            staged
                        )
                    run_data.pop("result_persistence_failed", None)
                    logger.info(
                        f"Successfully synced results for benchmark {benchmark_run_id}"
                    )

            # Clean up memory
            del self.active_runs[run_key]

        except Exception:
            logger.exception("Error syncing benchmark results to database")
            # Thread-local session is reused — roll back a failed flush/commit
            # so the next use doesn't raise PendingRollbackError.
            if session is not None:
                safe_rollback(session, "_sync_results_to_database")
            run_data["result_persistence_failed"] = True

    def _send_progress_update(
        self,
        benchmark_run_id: int,
        completed: int,
        total: int,
        username: Optional[str] = None,
    ):
        """Send real-time progress update via websocket.

        ``username`` is the run's owner and is passed through to
        ``emit_to_subscribers`` so the event reaches only that user's
        socket subscriber set -- ``benchmark_run_id`` is a per-user integer
        and can collide across users (see
        ``SocketIOService.__subscription_key``).
        """
        try:
            percentage = (completed / total * 100) if total > 0 else 0

            # Create log entry for milestone progress
            log_entry = {
                "time": datetime.now(UTC).isoformat(),
                "message": f"Completed {completed}/{total} examples ({percentage:.1f}%)",
                "progress": percentage,
                "metadata": {
                    "phase": "benchmark_progress",
                    "type": "milestone",
                    "completed": completed,
                    "total": total,
                    "benchmark_run_id": benchmark_run_id,
                },
            }

            progress_data = {
                "status": "in_progress",
                "message": f"Processing examples: {completed}/{total}",
                "progress": percentage,
                "completed": completed,
                "total": total,
                "benchmark_run_id": benchmark_run_id,
                "log_entry": log_entry,
                "progress_log": json.dumps([log_entry]),
            }

            self.socket_service.emit_to_subscribers(
                "research_progress",
                benchmark_run_id,
                progress_data,
                owner=username,
            )

        except Exception:
            logger.exception("Error sending progress update")

    def _calculate_final_accuracy(
        self, benchmark_run_id: int, username: Optional[str] = None
    ):
        """Calculate and save final accuracy metrics."""
        from ...database.session_context import get_user_db_session

        with get_user_db_session(username) as session:
            try:
                # Get all results for this run
                results = (
                    session.query(BenchmarkResult)
                    .filter(
                        BenchmarkResult.benchmark_run_id == benchmark_run_id
                    )
                    .filter(BenchmarkResult.is_correct.isnot(None))
                    .all()
                )

                if results:
                    correct_count = sum(1 for r in results if r.is_correct)
                    overall_accuracy = (correct_count / len(results)) * 100

                    # Calculate processing rate
                    total_time = sum(r.processing_time or 0 for r in results)
                    processing_rate = (
                        (len(results) / (total_time / 60))
                        if total_time > 0
                        else 0
                    )

                    # Update benchmark run
                    benchmark_run = (
                        session.query(BenchmarkRun)
                        .filter(BenchmarkRun.id == benchmark_run_id)
                        .first()
                    )
                    if benchmark_run:
                        benchmark_run.overall_accuracy = overall_accuracy
                        benchmark_run.processing_rate = processing_rate
                        session.commit()

            except Exception:
                logger.exception("Error calculating final accuracy")

    def update_benchmark_status(
        self,
        benchmark_run_id: int,
        status: BenchmarkStatus,
        error_message: Optional[str] = None,
        username: Optional[str] = None,
    ):
        """Update benchmark run status."""
        from ...database.session_context import get_user_db_session

        with get_user_db_session(username) as session:
            try:
                benchmark_run = (
                    session.query(BenchmarkRun)
                    .filter(BenchmarkRun.id == benchmark_run_id)
                    .first()
                )
                if benchmark_run:
                    benchmark_run.status = status
                    benchmark_run.updated_at = datetime.now(UTC)

                    if error_message:
                        benchmark_run.error_message = error_message

                    if (
                        status == BenchmarkStatus.IN_PROGRESS
                        and not benchmark_run.start_time
                    ):
                        benchmark_run.start_time = datetime.now(UTC)
                    elif (
                        status
                        in [BenchmarkStatus.COMPLETED, BenchmarkStatus.FAILED]
                        and not benchmark_run.end_time
                    ):
                        benchmark_run.end_time = datetime.now(UTC)

                    session.commit()

            except Exception:
                session.rollback()
                logger.exception("Error updating benchmark status")

    def get_benchmark_status(
        self, benchmark_run_id: int, username: str = None
    ) -> Optional[Dict]:
        """Get current status of a benchmark run."""
        from ...database.session_context import get_user_db_session

        with get_user_db_session(username) as session:
            try:
                benchmark_run = (
                    session.query(BenchmarkRun)
                    .filter(BenchmarkRun.id == benchmark_run_id)
                    .first()
                )
                if not benchmark_run:
                    return None

                # Calculate running accuracy from this run's evaluated results
                results = (
                    session.query(BenchmarkResult)
                    .filter(
                        BenchmarkResult.benchmark_run_id == benchmark_run_id
                    )
                    .filter(BenchmarkResult.is_correct.isnot(None))
                    .all()
                )

                running_accuracy = None
                # Dynamic per-dataset accuracy tracking
                dataset_accuracies = {}

                if results:
                    # Overall running accuracy
                    correct_count = sum(1 for r in results if r.is_correct)
                    running_accuracy = (correct_count / len(results)) * 100

                    # Calculate accuracy for each dataset type dynamically
                    from collections import defaultdict

                    dataset_results = defaultdict(list)

                    # Group results by dataset type
                    for r in results:
                        dataset_results[r.dataset_type.value].append(r)

                    # Calculate accuracy for each dataset
                    for (
                        dataset_type,
                        dataset_result_list,
                    ) in dataset_results.items():
                        if dataset_result_list:
                            correct = sum(
                                1 for r in dataset_result_list if r.is_correct
                            )
                            accuracy = (
                                correct / len(dataset_result_list)
                            ) * 100
                            # Store with _accuracy suffix for consistency
                            dataset_accuracies[f"{dataset_type}_accuracy"] = (
                                accuracy
                            )

                # Calculate time estimates and reliability metrics
                estimated_time_remaining = None
                total_elapsed_time = None
                avg_time_per_example = None
                accuracy_confidence = None

                # Get ALL results for timing calculation (including those pending evaluation)
                all_results_for_timing = (
                    session.query(BenchmarkResult)
                    .filter(
                        BenchmarkResult.benchmark_run_id == benchmark_run_id
                    )
                    .all()
                )

                if benchmark_run.start_time and all_results_for_timing:
                    # Calculate elapsed time
                    current_time = datetime.now(UTC)
                    total_elapsed_time = (
                        current_time - benchmark_run.start_time
                    ).total_seconds()

                    # Calculate average processing time per example using actual count
                    avg_time_per_example = total_elapsed_time / len(
                        all_results_for_timing
                    )

                    logger.info(
                        f"Time calculation - elapsed: {total_elapsed_time:.2f}s, "
                        f"results_count: {len(all_results_for_timing)}, "
                        f"avg_per_example: {avg_time_per_example:.2f}s"
                    )

                    # Estimate remaining time
                    remaining_examples = benchmark_run.total_examples - len(
                        all_results_for_timing
                    )
                    if remaining_examples > 0:
                        estimated_time_remaining = (
                            avg_time_per_example * remaining_examples
                        )
                        logger.info(
                            f"Time estimation - total: {benchmark_run.total_examples}, "
                            f"completed: {len(all_results_for_timing)}, remaining: {remaining_examples}, "
                            f"avg_time: {avg_time_per_example:.2f}s, "
                            f"estimated_remaining: {estimated_time_remaining:.2f}s"
                        )

                # Calculate accuracy confidence interval (Wilson score, 95%)
                if results and len(results) >= 3:
                    from local_deep_research.benchmarks.metrics.statistics import (
                        wilson_score_interval,
                    )

                    n = len(results)
                    correct = sum(1 for r in results if r.is_correct)
                    ci = wilson_score_interval(correct, n)
                    accuracy_confidence = {
                        "lower_bound": ci["lower"] * 100,
                        "upper_bound": ci["upper"] * 100,
                        "margin_of_error": ci["margin_of_error"] * 100,
                        "sample_size": n,
                    }

                status_data = {
                    "id": benchmark_run.id,
                    "run_name": benchmark_run.run_name,
                    "status": benchmark_run.status.value,
                    "completed_examples": len(
                        all_results_for_timing
                    ),  # Use actual count from DB
                    "total_examples": benchmark_run.total_examples,
                    "failed_examples": benchmark_run.failed_examples,
                    "overall_accuracy": benchmark_run.overall_accuracy
                    or running_accuracy,  # Use running accuracy if final not calculated
                    "running_accuracy": running_accuracy,  # Current running accuracy
                    "processing_rate": benchmark_run.processing_rate,
                    "estimated_time_remaining": estimated_time_remaining,  # seconds
                    "total_elapsed_time": total_elapsed_time,  # seconds
                    "avg_time_per_example": avg_time_per_example,  # seconds
                    "accuracy_confidence": accuracy_confidence,  # confidence interval
                    "created_at": benchmark_run.created_at.isoformat()
                    if benchmark_run.created_at
                    else None,
                    "start_time": benchmark_run.start_time.isoformat()
                    if benchmark_run.start_time
                    else None,
                    "end_time": benchmark_run.end_time.isoformat()
                    if benchmark_run.end_time
                    else None,
                    # Genericized at this boundary (CWE-209) — see
                    # _GENERIC_BENCHMARK_ERROR. Returns None when there is no
                    # failure so non-error states are unaffected.
                    "error_message": (
                        _GENERIC_BENCHMARK_ERROR
                        if benchmark_run.error_message
                        else None
                    ),
                    # Add all per-dataset accuracies dynamically
                    **dataset_accuracies,
                }

                logger.info(
                    f"Benchmark {benchmark_run_id} status - completed: {benchmark_run.completed_examples}, "
                    f"running_acc: {running_accuracy}, dataset_accuracies: {dataset_accuracies}, "
                    f"avg_time: {avg_time_per_example}"
                )

                return status_data

            except Exception:
                logger.exception("Error getting benchmark status")
                return None

    def cancel_benchmark(
        self, benchmark_run_id: int, username: Optional[str] = None
    ) -> bool:
        """Cancel a running benchmark."""
        try:
            # active_runs is keyed by (username, id) (see _run_key), so this
            # lookup only ever finds the requester's own run -- one user can no
            # longer set another user's in-flight run to "cancelled" and stop
            # it (cross-user DoS). The DB update below is already scoped to the
            # caller's own database.
            run = self.active_runs.get(
                self._run_key(username, benchmark_run_id)
            )
            if run is not None:
                run["status"] = "cancelled"
                # Abort the in-flight task's LLM calls mid-generation, not
                # just between tasks (see _process_benchmark_task): the
                # worker publishes each task's research scope here, and the
                # LLM wrapper aborts on the flag within one streamed chunk.
                # Scoped to the caller's own run via _run_key (above), and
                # the id is an unguessable per-task UUID besides.
                tracking_id = run.get("current_tracking_id")
                if tracking_id:
                    try:
                        from ...web.research_state import (
                            clear_termination_flag,
                            set_termination_flag,
                        )

                        set_termination_flag(tracking_id)
                        # The worker pops the id then clears its flag in a
                        # per-task finally. If it finished between our read
                        # and set, we just created a stale process-global
                        # flag for a fresh UUID that will never be reused
                        # (cosmetic leak). Re-check and reclaim it.
                        if run.get("current_tracking_id") != tracking_id:
                            clear_termination_flag(tracking_id)
                    except Exception:
                        logger.debug(
                            "Failed to set benchmark termination flag "
                            "(non-critical)"
                        )

            self.update_benchmark_status(
                benchmark_run_id, BenchmarkStatus.CANCELLED, username=username
            )
            logger.info(f"Cancelled benchmark run {benchmark_run_id}")
            return True

        except Exception:
            logger.exception(f"Error cancelling benchmark {benchmark_run_id}")
            return False


# Global service instance
benchmark_service = BenchmarkService()