Skip to content

Personal AI, On Personal Devices

Python
10.2 KB
Raw Blame
"""Native TerminalBench V2.1 backend.

Uses Harness for Docker-based execution and scoring.
"""

from __future__ import annotations

import inspect
import logging
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple

from openjarvis.evals.core.backend import InferenceBackend
from openjarvis.evals.core.types import RunSummary

LOGGER = logging.getLogger(__name__)

try:
    from terminal_bench import BenchmarkResults, Harness

    _HAS_TB = True
except ImportError:
    _HAS_TB = False

# terminal-bench FailureMode values that are definitionally infrastructure
# failures (the harness broke before/while driving the agent), never a
# judgment on the model's answer. NOTE: clean trials leave failure_mode
# "unset" in terminal-bench 0.2.18 — both on success AND on genuine
# unresolved misses — so failure_mode alone can NOT be used to detect
# harness errors (it would misflag every real model miss).
_INFRA_FAILURE_MODES = frozenset({"agent_installation_failed", "unknown_agent_error"})

# Harness kwargs that older terminal-bench versions may not support.
_TIMEOUT_KWARGS = ("global_agent_timeout_sec", "global_timeout_multiplier")


def summarize_benchmark_results(
    results: Any,
    *,
    model: str,
    benchmark: str = "terminalbench-native",
) -> Tuple[RunSummary, List[Dict[str, str]]]:
    """Convert terminal-bench ``BenchmarkResults`` into a ``RunSummary``.

    Trials are classified into three buckets:

    - resolved: ``is_resolved`` is True -> counted correct.
    - model miss: unresolved, but the model was actually contacted ->
      counted in the accuracy denominator.
    - harness/infra failure: excluded from the accuracy denominator and
      reported in ``RunSummary.errors`` plus the returned failure list.

    Zero-model-contact signal choice: terminal-bench 0.2.18 leaves
    ``failure_mode`` UNSET both on clean success and on genuine unresolved
    misses, so failure_mode cannot distinguish "the model tried and failed"
    from "the agent never called the model". Token usage can: this backend
    always runs terminus-2, which reports real LiteLLM usage, so an
    unresolved trial with zero/missing input+output tokens means no model
    request ever completed — an infrastructure failure (in-container setup
    hang/death, tmux failure), not a model miss. CAVEAT: terminal-bench
    "installed agents" (openhands, claude-code, ...) hardcode 0 tokens even
    on success; if this backend ever honors ``agent_name`` for installed
    agents, this heuristic must be gated on the agent type.
    """
    trials = list(getattr(results, "results", None) or [])

    harness_failures: List[Dict[str, str]] = []
    scored = 0
    correct = 0

    for tr in trials:
        task_id = getattr(tr, "task_id", None) or getattr(tr, "trial_name", "unknown")
        is_resolved = getattr(tr, "is_resolved", None) is True
        fm = getattr(tr, "failure_mode", None)
        fm_value = str(getattr(fm, "value", fm) or "unset").lower()
        tokens = (getattr(tr, "total_input_tokens", None) or 0) + (
            getattr(tr, "total_output_tokens", None) or 0
        )

        zero_model_contact = not is_resolved and tokens == 0
        infra_failure_mode = fm_value in _INFRA_FAILURE_MODES

        if zero_model_contact or infra_failure_mode:
            harness_failures.append(
                {
                    "task_id": str(task_id),
                    "failure_mode": fm_value,
                    "reason": (
                        "zero_model_requests" if zero_model_contact else fm_value
                    ),
                }
            )
            continue

        scored += 1
        if is_resolved:
            correct += 1

    return (
        RunSummary(
            benchmark=benchmark,
            category="agentic",
            backend="terminalbench-native",
            model=model,
            total_samples=len(trials),
            scored_samples=scored,
            correct=correct,
            accuracy=correct / scored if scored else 0.0,
            errors=len(harness_failures),
            mean_latency_seconds=0.0,
            total_cost_usd=0.0,
        ),
        harness_failures,
    )


class TerminalBenchNativeBackend(InferenceBackend):
    """Runs terminal-bench tasks natively via Harness with Docker execution.

    Uses terminal-bench's own agent + LiteLLM to call the model,
    Docker containers for task execution, and built-in test scripts
    for scoring. This gives real agentic evaluation, not text-only.
    """

    backend_id = "terminalbench-native"

    def __init__(
        self,
        model: str = "openai/default",
        api_base: str = "http://localhost:8000/v1",
        temperature: float = 0.2,
        agent_name: str = "naive",
        output_dir: str = "results/terminalbench/",
        max_samples: Optional[int] = None,
        dataset_name: str = "terminal-bench-core",
        dataset_version: str = "0.1.1",
        system_prompt: str = "",
        max_tokens: int = 16384,
        n_concurrent: int = 4,
        global_agent_timeout_sec: Optional[float] = 1800.0,
        global_timeout_multiplier: Optional[float] = None,
    ) -> None:
        """Args of note:

        global_agent_timeout_sec: Hard wall-clock bound for each trial's
            agent phase. terminal-bench runs installed-agent SETUP inside
            this same budget with an infinite tmux timeout, so this bounds
            SETUP+RUN together (a setup-only timeout needs an upstream
            terminal-bench change). When set, it REPLACES each task's own
            ``max_agent_timeout_sec``. Set ``None`` or ``0`` to fall back
            to per-task budgets.
        global_timeout_multiplier: Scales per-task budgets when
            ``global_agent_timeout_sec`` is not set. ``None`` keeps
            terminal-bench's default (1.0).
        """
        if not _HAS_TB:
            raise ImportError("terminal-bench is required: pip install terminal-bench")

        self._model = model
        self._api_base = api_base
        self._temperature = temperature
        self._agent_name = agent_name
        self._output_dir = Path(output_dir)
        self._max_samples = max_samples
        self._dataset_name = dataset_name
        self._dataset_version = dataset_version
        self._system_prompt = system_prompt
        self._max_tokens = max_tokens
        self._n_concurrent = n_concurrent
        self._global_agent_timeout_sec = global_agent_timeout_sec
        self._global_timeout_multiplier = global_timeout_multiplier
        self._results: Optional[BenchmarkResults] = None

    def run_harness(self, run_id: str) -> BenchmarkResults:
        """Run the full terminal-bench harness and return results."""
        output_path = self._output_dir / run_id
        output_path.mkdir(parents=True, exist_ok=True)

        harness_kwargs: Dict[str, Any] = {
            "output_path": output_path,
            "run_id": run_id,
            "dataset_name": self._dataset_name,
            "dataset_version": self._dataset_version,
            "model_name": self._model,
            "n_concurrent_trials": self._n_concurrent,
            "cleanup": True,
        }

        # Use terminus-2 agent which accepts model_name + api_base as
        # serializable strings (avoids Pydantic serialization issues with
        # LLM objects in the harness lock file).
        from terminal_bench.agents.agent_name import AgentName

        harness_kwargs["agent_name"] = AgentName("terminus-2")
        harness_kwargs["agent_kwargs"] = {
            "model_name": self._model,
            "api_base": self._api_base,
            "temperature": self._temperature,
        }

        if self._max_samples is not None:
            harness_kwargs["n_tasks"] = self._max_samples

        # Bound each trial's agent phase. Without this, an in-container
        # installed-agent SETUP hang runs with an infinite tmux timeout,
        # bounded only by whatever budget the task happens to declare.
        if self._global_agent_timeout_sec:
            harness_kwargs["global_agent_timeout_sec"] = float(
                self._global_agent_timeout_sec
            )
        if self._global_timeout_multiplier is not None:
            harness_kwargs["global_timeout_multiplier"] = float(
                self._global_timeout_multiplier
            )

        self._check_timeout_kwargs_supported(harness_kwargs)

        harness = Harness(**harness_kwargs)
        self._results = harness.run()
        return self._results

    @staticmethod
    def _check_timeout_kwargs_supported(harness_kwargs: Dict[str, Any]) -> None:
        """Fail loudly if this terminal-bench build lacks the timeout kwargs.

        terminal-bench is an undeclared, unpinned dependency, so installs may
        predate the global timeout kwargs (added by 0.2.x). Passing an
        unknown kwarg raises an opaque TypeError; dropping it silently would
        re-create the unbounded-setup hang. Detect and explain instead.
        """
        try:
            params = inspect.signature(Harness.__init__).parameters
        except (TypeError, ValueError):
            return
        if any(p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values()):
            return
        unsupported = [
            key
            for key in _TIMEOUT_KWARGS
            if key in harness_kwargs and key not in params
        ]
        if unsupported:
            raise RuntimeError(
                "The installed terminal-bench does not support "
                f"{', '.join(unsupported)} (requires terminal-bench >= "
                "0.2.18). Upgrade terminal-bench, or disable the bound by "
                "setting global_agent_timeout_sec = 0 in the eval config "
                "[run] section."
            )

    def generate(
        self,
        prompt: str,
        *,
        model: str,
        system: str = "",
        temperature: float = 0.0,
        max_tokens: int = 2048,
    ) -> str:
        return ""

    def generate_full(
        self,
        prompt: str,
        *,
        model: str,
        system: str = "",
        temperature: float = 0.0,
        max_tokens: int = 2048,
    ) -> Dict[str, Any]:
        return {"content": "", "usage": {}, "model": model, "latency_seconds": 0.0}

    def close(self) -> None:
        pass


__all__ = ["TerminalBenchNativeBackend", "summarize_benchmark_results"]