Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

51.0 KB
Raw Blame
"""soup eval — evaluation platform with benchmarks, custom evals, LLM judge, and more."""

from __future__ import annotations

import json
from pathlib import Path
from typing import Optional

import typer
from rich.console import Console
from rich.panel import Panel
from rich.table import Table

from soup_cli.utils.exit_codes import (
    EXIT_GATE_FAILED,
    EXIT_OK,
    EXIT_RUNTIME_ERROR,
    EXIT_USAGE_ERROR,
    GateCommand,
)
from soup_cli.utils.terminal import for_terminal

console = Console()

app = typer.Typer(
    name="eval",
    help="Evaluate models: benchmarks, custom evals, LLM judge, leaderboard.",
    no_args_is_help=True,
    rich_markup_mode="rich",
)


def _reject_lm_eval_injection(value: str, field: str) -> None:
    """Block ``,`` / ``=`` in a value interpolated into lm-eval ``model_args``.

    lm-eval parses ``model_args`` as comma-separated ``key=value`` pairs, so a
    crafted ``base_model_name_or_path`` in an adapter's own
    ``adapter_config.json`` (e.g. ``attacker/m,trust_remote_code=True``) would
    otherwise smuggle ``trust_remote_code=True`` and enable RCE on load.
    Mirrors the ``commands/ship.py`` guard (v0.71.25).
    """
    if "," in value or "=" in value:
        console.print(
            f"[red]Refusing to evaluate:[/] {field} contains ',' or '=' "
            f"({for_terminal(repr(value))}); these are lm-eval model_args delimiters and could "
            "smuggle trust_remote_code=True (arbitrary code execution)."
        )
        raise typer.Exit(1)


# ─── soup eval benchmark ───


@app.command()
def benchmark(
    model: str = typer.Option(
        ..., "--model", "-m",
        help="Path to model or LoRA adapter directory",
    ),
    benchmarks: str = typer.Option(
        "mmlu", "--benchmarks", "-b",
        help="Comma-separated benchmark names (mmlu, gsm8k, hellaswag, etc.)",
    ),
    num_fewshot: Optional[int] = typer.Option(
        None, "--fewshot", "-f",
        help="Number of few-shot examples (benchmark default if not set)",
    ),
    batch_size: int = typer.Option(
        8, "--batch-size",
        help="Batch size for evaluation",
    ),
    run_id: Optional[str] = typer.Option(
        None, "--run-id",
        help="Link results to an existing training run",
    ),
    device: Optional[str] = typer.Option(
        None, "--device",
        help="Device: cuda, mps, cpu. Auto-detected if not set.",
    ),
    trust_remote_code: bool = typer.Option(
        False,
        "--trust-remote-code",
        help=(
            "Allow loading models that ship custom Python via auto_map. "
            "Default deny (v0.36.0). Only enable if you trust the source."
        ),
    ),
):
    """Evaluate on standard benchmarks (wraps lm-evaluation-harness)."""
    model_path = Path(model)
    if not model_path.exists():
        console.print(f"[red]Model path not found: {for_terminal(model_path)}[/]")
        raise typer.Exit(1)

    # Both the user-supplied path and the adapter's declared base model are
    # interpolated into lm-eval ``model_args`` below; reject the ','/'=' that
    # would smuggle extra kwargs (e.g. trust_remote_code=True).
    _reject_lm_eval_injection(str(model_path), "--model path")

    # Check for LoRA adapter and resolve base model
    adapter_config = model_path / "adapter_config.json"
    model_arg = str(model_path)
    if adapter_config.exists():
        adapter_info = json.loads(adapter_config.read_text())
        base_model = adapter_info.get("base_model_name_or_path", "")
        if base_model:
            _reject_lm_eval_injection(
                base_model, "base_model_name_or_path (from adapter_config.json)"
            )
            model_arg = f"pretrained={base_model},peft={model_path}"
            console.print(
                f"[dim]LoRA adapter detected. Base model: {for_terminal(base_model)}[/]"
            )
        else:
            model_arg = f"pretrained={model_path}"
    else:
        model_arg = f"pretrained={model_path}"

    if trust_remote_code:
        # Appended only from the resolved --trust-remote-code flag, never
        # from adapter_config.json: _reject_lm_eval_injection above already
        # refused a base_model_name_or_path smuggling its own ','/'=' pair.
        model_arg = f"{model_arg},trust_remote_code=True"

    benchmark_list = [b.strip() for b in benchmarks.split(",")]
    console.print(f"[dim]Evaluating on: {', '.join(for_terminal(b) for b in benchmark_list)}[/]")

    # Lazy import lm_eval
    try:
        import lm_eval  # noqa: F401
    except ImportError:
        console.print(
            "[red]lm-eval not installed.[/]\n"
            "Install with: [bold]pip install \"soup-cli\\[eval]\"[/]"
        )
        raise typer.Exit(1)

    # Detect device
    if not device:
        from soup_cli.utils.gpu import detect_device
        device, _ = detect_device()

    console.print("[dim]Running evaluation (this may take a while)...[/]")
    results = _run_lm_eval(
        model_arg=model_arg,
        tasks=benchmark_list,
        num_fewshot=num_fewshot,
        batch_size=batch_size,
        device=device,
    )

    _display_benchmark_results(results, benchmark_list)
    saved = _save_benchmark_results(results, str(model_path), benchmark_list, run_id)
    if not saved:
        console.print("[red]Nothing measured; nothing saved to experiment tracker.[/]")
        raise typer.Exit(1)
    console.print(
        f"\n[green]Results saved to experiment tracker: {saved} of {len(benchmark_list)}.[/]"
    )


# ─── soup eval aider ───


@app.command()
def aider(
    model: str = typer.Option(
        ..., "--model", "-m",
        help="Aider model identifier (for example, openai/gpt-4.1)",
    ),
    output: str = typer.Option(
        ..., "--output", "-o",
        help="Contained directory for upstream and Soup result files",
    ),
    exercises_dir: str = typer.Option(
        "tmp.benchmarks/polyglot-benchmark", "--exercises-dir",
        help="Prepared Aider-AI/polyglot-benchmark checkout",
    ),
    image: str = typer.Option(
        "aider-benchmark", "--image",
        help="Locally built upstream Aider benchmark image",
    ),
    threads: int = typer.Option(
        1, "--threads", "-t", min=1, max=64,
        help="Number of benchmark exercises to run concurrently",
    ),
    num_tests: int = typer.Option(
        -1, "--num-tests", "-n",
        help="Number of exercises to run (-1 runs all exercises)",
    ),
    timeout: int = typer.Option(
        86_400, "--timeout", min=60, max=86_400,
        help="Maximum benchmark runtime in seconds",
    ),
    run_id: Optional[str] = typer.Option(
        None, "--run-id",
        help="Save the aggregate score under an existing Soup run ID",
    ),
    allow_host_services: bool = typer.Option(
        False, "--allow-host-services",
        help="Let the benchmark container reach services on the host",
    ),
):
    """Run Aider's Polyglot coding benchmark in its official Docker image."""
    from rich.markup import escape

    from soup_cli.eval.aider_polyglot import (
        AiderEvalError,
        build_docker_command,
        parse_aider_results,
        preflight_docker,
        prepare_output_dir,
        validate_exercises_dir,
        write_soup_result,
    )
    from soup_cli.utils.paths import is_under_cwd

    tracker = None
    if run_id:
        from soup_cli.experiment.tracker import ExperimentTracker

        tracker = ExperimentTracker()
        if tracker.get_run(run_id) is None:
            console.print(f"[red]Run not found: {escape(run_id)}[/]")
            tracker.close()
            raise typer.Exit(1)

    try:
        output_dir = prepare_output_dir(output)
        corpus_dir = validate_exercises_dir(exercises_dir)
        corpus_outside_cwd = not is_under_cwd(corpus_dir)
        docker = preflight_docker(image)
        command = build_docker_command(
            docker=docker,
            image=image,
            model=model,
            exercises_dir=corpus_dir,
            output_dir=output_dir,
            threads=threads,
            num_tests=num_tests,
            allow_host_services=allow_host_services,
        )
    except AiderEvalError as exc:
        if tracker is not None:
            tracker.close()
        console.print(f"[red]Aider Polyglot preflight failed:[/] {escape(str(exc))}")
        raise typer.Exit(1) from exc

    if corpus_outside_cwd:
        console.print(
            "[yellow]Warning:[/] --exercises-dir resolves outside the current "
            "working directory; Docker will still mount it read-only."
        )

    console.print(
        Panel(
            f"[bold]Aider Polyglot[/]\n"
            f"Model: {escape(model)}\n"
            f"Output: {escape(str(output_dir))}\n"
            f"Image: {escape(image)}\n"
            f"Exercises: {escape(str(corpus_dir))}",
            title="Evaluation",
            border_style="blue",
        )
    )
    console.print(
        "[yellow]The benchmark executes untrusted model-generated code inside Docker.[/]"
    )

    import subprocess

    try:
        process = subprocess.run(  # noqa: S603 -- fixed argv, no shell
            command,
            check=False,
            shell=False,
            timeout=timeout,
        )
    except subprocess.TimeoutExpired as exc:
        if tracker is not None:
            tracker.close()
        console.print(f"[red]Aider benchmark timed out after {timeout} seconds.[/]")
        raise typer.Exit(1) from exc
    except OSError as exc:
        if tracker is not None:
            tracker.close()
        console.print(
            f"[red]Could not start the Aider benchmark:[/] {type(exc).__name__}"
        )
        raise typer.Exit(1) from exc

    if process.returncode != 0:
        if tracker is not None:
            tracker.close()
        console.print(
            f"[red]Aider benchmark exited with status {process.returncode}.[/] "
            f"Partial files remain in {escape(str(output_dir))}."
        )
        raise typer.Exit(1)

    try:
        row = parse_aider_results(output_dir, model=model)
        result_path = write_soup_result(output_dir, row)
    except AiderEvalError as exc:
        if tracker is not None:
            tracker.close()
        console.print(f"[red]Could not aggregate Aider results:[/] {escape(str(exc))}")
        raise typer.Exit(1) from exc

    if tracker is not None:
        tracker.save_eval_result(
            model_path=model,
            benchmark=row["task"],
            score=row["score"],
            details={"errors": row["errors"], **row["details"]},
            run_id=run_id,
        )
        tracker.close()
        console.print(f"[green]Saved result under run {escape(run_id)}.[/]")

    completed = row["details"]["completed_tests"]
    passed = row["details"]["passed_tests"]
    console.print(
        f"[green]Aider Polyglot complete:[/] {passed}/{completed} passed "
        f"(score {row['score']:.4f})"
    )
    console.print(f"[green]Soup result written to:[/] {escape(str(result_path))}")


# ─── soup eval custom ───


@app.command()
def custom(
    tasks: str = typer.Option(
        ..., "--tasks", "-t",
        help="Path to eval tasks JSONL file",
    ),
    model: str = typer.Option(
        ..., "--model", "-m",
        help="Path to model directory",
    ),
    run_id: Optional[str] = typer.Option(
        None, "--run-id",
        help="Link results to an existing training run",
    ),
    attach_to_registry: Optional[str] = typer.Option(
        None, "--attach-to-registry",
        help="Attach the eval JSON to a registry entry as kind=eval_results",
    ),
    output: Optional[str] = typer.Option(
        None, "--output", "-o",
        help=(
            "Path to write the eval JSON output. Honored independently of "
            "--attach-to-registry (v0.40.1 / G10)."
        ),
    ),
    trust_remote_code: bool = typer.Option(
        False,
        "--trust-remote-code",
        help=(
            "Allow loading models that ship custom Python via auto_map. "
            "Default deny (v0.36.0). Only enable if you trust the source."
        ),
    ),
):
    """Run custom evaluation tasks from a JSONL file."""
    from soup_cli.eval.custom import load_eval_tasks

    tasks_path = Path(tasks)
    model_path = Path(model)

    if not model_path.exists():
        console.print(f"[red]Model path not found: {for_terminal(model_path)}[/]")
        raise typer.Exit(1)

    # Load and validate tasks
    try:
        eval_tasks = load_eval_tasks(tasks_path)
    except (FileNotFoundError, ValueError) as exc:
        console.print(f"[red]{exc}[/]")
        raise typer.Exit(1)

    console.print(
        f"[dim]Loaded {len(eval_tasks)} eval tasks from {for_terminal(tasks_path)}[/]"
    )

    # Run evaluation with progress
    from rich.progress import (
        BarColumn,
        Progress,
        SpinnerColumn,
        TaskProgressColumn,
        TextColumn,
    )

    progress = Progress(
        SpinnerColumn(),
        TextColumn("[progress.description]{task.description}"),
        BarColumn(),
        TaskProgressColumn(),
        console=console,
    )

    results_list = []
    from soup_cli.eval.custom import _create_default_generator, score_task

    console.print("[dim]Loading model...[/]")
    generate_fn = _create_default_generator(
        str(model_path), trust_remote_code=trust_remote_code,
    )

    with progress:
        task_bar = progress.add_task(
            "Evaluating...", total=len(eval_tasks),
        )
        # v0.40.1 Part D / G10 — keep the CLI ``--output`` path separate
        # from the per-task model response (was previously shadowed by the
        # loop variable, masking ``-o`` whenever attach-to-registry was off).
        for eval_task in eval_tasks:
            response = generate_fn(eval_task.prompt)
            result = score_task(eval_task, response)
            results_list.append(result)
            progress.advance(task_bar)

    from soup_cli.eval.custom import EvalResults

    eval_results = EvalResults(results=results_list)
    eval_results.compute()

    # Display results
    _display_custom_results(eval_results)

    # Save to tracker
    _save_custom_results(eval_results, str(model_path), run_id)
    console.print("\n[green]Results saved to experiment tracker.[/]")

    # v0.40.1 Part D / G10 — write `--output` JSON regardless of registry
    # attach. Both flags compose; either alone is sufficient.
    payload = None
    json_path = None
    if output or attach_to_registry:
        from soup_cli.registry.attach import attach_artifact, write_eval_json

        payload = {
            "model": str(model_path),
            "tasks": str(tasks_path),
            "total": eval_results.total,
            "correct": eval_results.correct,
            "accuracy": eval_results.accuracy,
            "category_scores": eval_results.category_scores,
        }
        write_target = output or "eval_results.json"
        try:
            json_path = write_eval_json(write_target, payload=payload)
        except (ValueError, FileNotFoundError) as exc:
            console.print(f"[red]Failed to write eval JSON:[/] {exc}")
            raise typer.Exit(1) from exc
        if output:
            console.print(f"[green]Eval JSON written to:[/] {json_path}")

    if attach_to_registry:
        try:
            attach_artifact(
                attach_to_registry, path=str(json_path), kind="eval_results",
            )
        except (ValueError, FileNotFoundError) as exc:
            console.print(f"[red]Registry attach failed:[/] {exc}")
            raise typer.Exit(1) from exc
        console.print(
            f"[green]Attached eval results to registry entry "
            f"'{attach_to_registry}' as eval_results.[/]"
        )


# ─── soup eval judge ───


@app.command()
def judge(
    target: str = typer.Option(
        ..., "--target",
        help="Path to JSONL with prompt+response pairs to evaluate",
    ),
    judge_model: str = typer.Option(
        "gpt-4o-mini", "--model", "-m",
        help="Judge model name",
    ),
    provider: str = typer.Option(
        "openai", "--provider",
        help="Judge provider: openai, server, ollama",
    ),
    rubric: Optional[str] = typer.Option(
        None, "--rubric", "-r",
        help="Path to rubric YAML file",
    ),
    api_base: Optional[str] = typer.Option(
        None, "--api-base",
        help="API base URL for judge model",
    ),
    run_id: Optional[str] = typer.Option(
        None, "--run-id",
        help="Link results to an existing training run",
    ),
):
    """Evaluate model outputs using LLM-as-a-judge."""
    from soup_cli.eval.judge import (
        JudgeDownError,
        JudgeEvaluator,
        JudgeUnavailableError,
        load_rubric,
        validate_judge_api_base,
    )

    # Validate API base
    try:
        validate_judge_api_base(api_base)
    except ValueError as exc:
        console.print(f"[red]{exc}[/]")
        raise typer.Exit(1)

    # Load target data
    target_path = Path(target)
    if not target_path.exists():
        console.print(f"[red]Target file not found: {target_path}[/]")
        raise typer.Exit(1)

    items: list[dict] = []
    with open(target_path, encoding="utf-8") as fh:
        for line_num, line in enumerate(fh, 1):
            line = line.strip()
            if not line:
                continue
            try:
                row = json.loads(line)
            except json.JSONDecodeError as exc:
                console.print(
                    f"[red]Invalid JSON on line {line_num}: {exc}[/]"
                )
                raise typer.Exit(1)
            if "prompt" not in row or "response" not in row:
                console.print(
                    f"[red]Line {line_num}: missing 'prompt' or 'response'[/]"
                )
                raise typer.Exit(1)
            items.append(row)

    # Load rubric
    rubric_data = None
    if rubric:
        try:
            rubric_data = load_rubric(Path(rubric))
        except (FileNotFoundError, ValueError) as exc:
            # The message quotes the path it was given.
            console.print(f"[red]{for_terminal(str(exc))}[/]")
            raise typer.Exit(1)

    console.print(
        f"[dim]Judging {len(items)} responses with {judge_model} "
        f"(provider: {provider})...[/]"
    )

    # Run judge evaluation
    try:
        evaluator = JudgeEvaluator(
            rubric=rubric_data,
            provider=provider,
            model=judge_model,
            api_base=api_base,
        )
    except ValueError as exc:
        console.print(f"[red]{exc}[/]")
        raise typer.Exit(1)

    from rich.progress import (
        BarColumn,
        Progress,
        SpinnerColumn,
        TaskProgressColumn,
        TextColumn,
    )

    from soup_cli.eval.judge import JudgeResults

    judge_scores = []
    skipped_count = 0
    down = None
    progress = Progress(
        SpinnerColumn(),
        TextColumn("[progress.description]{task.description}"),
        BarColumn(),
        TaskProgressColumn(),
        console=console,
    )
    with progress:
        task_bar = progress.add_task("Judging...", total=len(items))
        for item in items:
            try:
                score = evaluator.evaluate(
                    prompt=item["prompt"],
                    response=item["response"],
                    category=item.get("category", "default"),
                )
                judge_scores.append(score)
            except JudgeDownError as exc:
                # #1522: it has stopped answering, so the rest would fail alike.
                down = exc.for_rows(len(items) - len(judge_scores), len(items), "items")
                break
            except (ValueError, OSError, KeyError, JudgeUnavailableError) as exc:
                skipped_count += 1
                console.print(
                    f"[yellow]Warning: judge failed for prompt: {for_terminal(exc)}[/]"
                )
            progress.advance(task_bar)

    if down is not None:
        console.print(f"[red]Judging stopped: {for_terminal(down)}[/]")
        if judge_scores:
            # Not saved to the tracker: a mean over the rows before the outage
            # is not this benchmark's score.
            results = JudgeResults(scores=judge_scores)
            results.compute()
            _display_judge_results(results)
        else:
            console.print("[red]All items failed. Check judge configuration.[/]")
        raise typer.Exit(1)

    if skipped_count > 0:
        console.print(
            f"[yellow]{skipped_count}/{len(items)} items skipped "
            f"due to judge errors.[/]"
        )

    if not judge_scores:
        console.print("[red]All items failed. Check judge configuration.[/]")
        raise typer.Exit(1)

    results = JudgeResults(scores=judge_scores)
    results.compute()

    _display_judge_results(results)

    # Save to tracker
    if run_id:
        from soup_cli.experiment.tracker import ExperimentTracker
        tracker = ExperimentTracker()
        tracker.save_eval_result(
            model_path=target,
            benchmark=f"judge:{judge_model}",
            score=results.overall_score,
            details={
                "criteria_averages": results.criteria_averages,
                "category_scores": results.category_scores,
                "total_evaluated": len(results.scores),
            },
            run_id=run_id,
        )
        console.print("[green]Results saved to experiment tracker.[/]")


# ─── soup eval auto ───


@app.command()
def auto(
    config: str = typer.Option(
        "soup.yaml", "--config", "-c",
        help="Path to soup.yaml config file",
    ),
    benchmarks: Optional[str] = typer.Option(
        None, "--benchmarks", "-b",
        help="Comma-separated benchmarks (overrides config)",
    ),
    custom_tasks: Optional[str] = typer.Option(
        None, "--tasks", "-t",
        help="Path to custom eval JSONL (overrides config)",
    ),
    trust_remote_code: bool = typer.Option(
        False,
        "--trust-remote-code",
        help=(
            "Allow loading models that ship custom Python via auto_map. "
            "Default deny (v0.36.0). Only enable if you trust the source."
        ),
    ),
):
    """Run automatic evaluation using config from soup.yaml."""
    from soup_cli.config.loader import load_config

    config_path = Path(config)
    if not config_path.exists():
        console.print(f"[red]Config not found: {config_path}[/]")
        raise typer.Exit(1)

    soup_config = load_config(str(config_path))
    output_dir = Path(soup_config.output)

    if not output_dir.exists():
        console.print(f"[red]Output directory not found: {output_dir}[/]")
        console.print("[dim]Run soup train first to create a model.[/]")
        raise typer.Exit(1)

    eval_config = getattr(soup_config, "eval", None)

    # Determine benchmarks
    bench_list = []
    if benchmarks:
        bench_list = [b.strip() for b in benchmarks.split(",")]
    elif eval_config and hasattr(eval_config, "benchmarks"):
        bench_list = eval_config.benchmarks or []

    # Determine custom tasks
    tasks_file = custom_tasks
    if not tasks_file and eval_config and hasattr(eval_config, "custom_tasks"):
        tasks_file = eval_config.custom_tasks

    if not bench_list and not tasks_file:
        console.print(
            "[yellow]No benchmarks or custom tasks specified.[/]\n"
            "Use --benchmarks or --tasks, or add eval config to soup.yaml."
        )
        raise typer.Exit(1)

    console.print(Panel(
        f"[bold]Auto-Eval[/]\n"
        f"Model: {output_dir}\n"
        f"Benchmarks: {', '.join(bench_list) if bench_list else 'none'}\n"
        f"Custom tasks: {tasks_file or 'none'}",
        title="Evaluation",
        border_style="blue",
    ))

    # #1808: a leg that could not run used to be swallowed, and "Auto-eval
    # complete." exited 0 even when it was the only leg. Count what ran.
    legs_ran = 0

    # Run standard benchmarks if specified
    if bench_list:
        console.print("\n[bold]Standard Benchmarks[/]")
        try:
            benchmark(
                model=str(output_dir),
                benchmarks=",".join(bench_list),
                num_fewshot=None,
                batch_size=8,
                run_id=None,
                device=None,
                trust_remote_code=trust_remote_code,
            )
        except (typer.Exit, SystemExit):
            # benchmark() signals failure with typer.Exit (a RuntimeError, NOT
            # SystemExit), so the old `except SystemExit` never caught it and a
            # benchmark failure aborted the whole auto-eval instead of falling
            # through to the custom eval below.
            console.print("[yellow]Benchmark eval skipped (see above).[/]")
        else:
            legs_ran += 1

    # Run custom eval if specified
    if tasks_file:
        console.print("\n[bold]Custom Evaluation[/]")
        try:
            # #752 — every typer parameter must be passed explicitly. Typer
            # fills them only when typer invokes the command; called as a
            # function, an unpassed parameter keeps its OptionInfo default,
            # which is truthy, so the --output/--attach-to-registry block ran
            # and died on write_eval_json(OptionInfo).
            custom(
                tasks=tasks_file,
                model=str(output_dir),
                run_id=None,
                attach_to_registry=None,
                output=None,
                trust_remote_code=trust_remote_code,
            )
        except (typer.Exit, SystemExit):
            # The same pair as the benchmark leg (#1808): custom() also fails
            # with typer.Exit, which `except SystemExit` alone never caught, so
            # the exit code depended on which leg had failed.
            console.print("[yellow]Custom eval skipped (see above).[/]")
        else:
            legs_ran += 1

    if legs_ran == 0:
        console.print(
            "\n[red]Auto-eval failed: nothing was evaluated.[/] Every leg was skipped "
            "(see above); fix the cause and run again."
        )
        raise typer.Exit(1)

    console.print("\n[green]Auto-eval complete.[/]")


# ─── soup eval compare ───


@app.command()
def compare(
    run1: str = typer.Argument(..., help="First run ID"),
    run2: str = typer.Argument(..., help="Second run ID"),
):
    """Compare eval results between two training runs."""
    from soup_cli.eval.leaderboard import compare_runs
    from soup_cli.experiment.tracker import ExperimentTracker

    tracker = ExperimentTracker()

    # Verify runs exist
    for rid in [run1, run2]:
        run_data = tracker.get_run(rid)
        if run_data is None:
            console.print(f"[red]Run not found: {rid}[/]")
            raise typer.Exit(1)

    comparison = compare_runs(tracker, run1, run2)

    if not comparison["comparisons"]:
        console.print("[yellow]No eval results found for these runs.[/]")
        console.print("[dim]Run soup eval benchmark --run-id ... first.[/]")
        raise typer.Exit(1)

    # Display comparison table
    table = Table(title=f"Eval Comparison: {run1} vs {run2}")
    table.add_column("Benchmark", style="bold")
    table.add_column("Run 1", justify="right")
    table.add_column("Run 2", justify="right")
    table.add_column("Delta", justify="right")

    for comp in comparison["comparisons"]:
        score_a = (
            f"{comp['run_1_score']:.4f}" if comp["run_1_score"] is not None
            else "-"
        )
        score_b = (
            f"{comp['run_2_score']:.4f}" if comp["run_2_score"] is not None
            else "-"
        )
        if comp["delta"] is not None:
            delta_val = comp["delta"]
            if delta_val > 0.01:
                delta_str = f"[green]+{delta_val:.4f}[/]"
            elif delta_val < -0.01:
                delta_str = f"[red]{delta_val:.4f}[/]"
            else:
                delta_str = f"{delta_val:.4f}"
        else:
            delta_str = "-"

        table.add_row(comp["benchmark"], score_a, score_b, delta_str)

    console.print(table)

    if comparison["has_regressions"]:
        console.print(
            f"\n[red bold]Regressions detected:[/] "
            f"{', '.join(comparison['regressions'])}"
        )


# ─── soup eval leaderboard ───


@app.command()
def leaderboard(
    sort_by: Optional[str] = typer.Option(
        None, "--sort-by", "-s",
        help="Sort by specific benchmark (default: average)",
    ),
    fmt: str = typer.Option(
        "table", "--format", "-f",
        help="Output format: table, json, csv",
    ),
):
    """Show local leaderboard across all evaluated models."""
    from soup_cli.eval.leaderboard import (
        build_leaderboard_from_tracker,
        export_leaderboard,
    )
    from soup_cli.experiment.tracker import ExperimentTracker

    tracker = ExperimentTracker()
    lb = build_leaderboard_from_tracker(tracker)

    if not lb.entries:
        console.print("[yellow]No eval results found.[/]")
        console.print(
            "[dim]Run soup eval benchmark or soup eval custom first.[/]"
        )
        raise typer.Exit(1)

    if fmt in ("json", "csv"):
        output = export_leaderboard(lb, fmt=fmt)
        # Machine-readable JSON/CSV: raw stdout, not console.print — Rich folds
        # rows/lines at the console width and parses ``[...]`` as markup when
        # piped, corrupting the output (issue #1468).
        typer.echo(output)
        return

    # Table format
    sorted_models = lb.get_sorted_models(sort_by=sort_by)

    # Collect all benchmarks
    all_benchmarks: set[str] = set()
    for _, scores, _ in sorted_models:
        all_benchmarks.update(scores.keys())
    benchmarks = sorted(all_benchmarks)

    table = Table(title="Eval Leaderboard")
    table.add_column("#", style="dim", width=3)
    table.add_column("Model", style="bold")
    for bench in benchmarks:
        table.add_column(bench, justify="right")
    table.add_column("Avg", justify="right", style="green")

    for rank, (model_name, scores, avg) in enumerate(sorted_models, 1):
        row = [str(rank), _short_model_name(model_name)]
        for bench in benchmarks:
            if bench in scores:
                row.append(f"{scores[bench]:.4f}")
            else:
                row.append("-")
        row.append(f"{avg:.4f}")
        table.add_row(*row)

    console.print(table)


# ─── soup eval human ───


@app.command()
def human(
    prompts_file: str = typer.Option(
        ..., "--input", "-i",
        help="Path to JSONL file with evaluation prompts",
    ),
    model_a: str = typer.Option(
        ..., "--model-a", "-a",
        help="Path to first model",
    ),
    model_b: str = typer.Option(
        ..., "--model-b", "-b",
        help="Path to second model",
    ),
    output: str = typer.Option(
        "human_eval_results.json", "--output", "-o",
        help="Output file for results",
    ),
):
    """Run human A/B evaluation between two models."""
    from soup_cli.eval.human import (
        HumanEvalResults,
        HumanJudgment,
        load_prompts,
        save_results,
    )

    # Load prompts
    try:
        prompts = load_prompts(Path(prompts_file))
    except (FileNotFoundError, ValueError) as exc:
        console.print(f"[red]{exc}[/]")
        raise typer.Exit(1)

    # Verify models exist
    for mpath in [model_a, model_b]:
        if not Path(mpath).exists():
            console.print(f"[red]Model not found: {for_terminal(mpath)}[/]")
            raise typer.Exit(1)

    console.print(Panel(
        f"[bold]Human Evaluation[/]\n"
        f"Prompts: {len(prompts)}\n"
        f"Model A: {for_terminal(model_a)}\n"
        f"Model B: {for_terminal(model_b)}",
        title="A/B Comparison",
        border_style="blue",
    ))

    # Generate responses
    console.print("[dim]Loading models and generating responses...[/]")
    from soup_cli.eval.custom import _create_default_generator

    gen_a = _create_default_generator(model_a)
    gen_b = _create_default_generator(model_b)

    results = HumanEvalResults()

    for idx, prompt_data in enumerate(prompts):
        prompt_text = prompt_data["prompt"]
        resp_a = gen_a(prompt_text)
        resp_b = gen_b(prompt_text)

        # Display for human judgment
        console.print(f"\n[bold]--- Prompt {idx + 1}/{len(prompts)} ---[/]")
        console.print(Panel(for_terminal(prompt_text), title="Prompt", border_style="dim"))
        console.print(Panel(for_terminal(resp_a), title="[blue]Response A[/]"))
        console.print(Panel(for_terminal(resp_b), title="[magenta]Response B[/]"))

        # Get human choice
        while True:
            choice = console.input(
                "[bold]Winner? (a/b/tie/q to quit): [/]"
            ).strip().lower()
            if choice in ("a", "b", "tie", "q"):
                break
            console.print("[yellow]Enter a, b, tie, or q[/]")

        if choice == "q":
            console.print("[dim]Session ended early.[/]")
            break

        results.judgments.append(HumanJudgment(
            prompt=prompt_text,
            response_a=resp_a,
            response_b=resp_b,
            model_a=model_a,
            model_b=model_b,
            winner=choice,
        ))

    results.compute_ratings()

    # Display Elo ratings
    _display_elo_ratings(results)

    # Save results
    save_results(results, Path(output))
    console.print(f"\n[green]Results saved to {for_terminal(output)}[/]")


# ─── Helper functions ───


def _run_lm_eval(
    model_arg: str,
    tasks: list[str],
    num_fewshot: Optional[int],
    batch_size: int,
    device: str,
) -> dict:
    """Run lm-evaluation-harness and return results dict."""
    import lm_eval

    results = lm_eval.simple_evaluate(
        model="hf",
        model_args=model_arg,
        tasks=tasks,
        num_fewshot=num_fewshot,
        batch_size=batch_size,
        device=device,
    )
    return results


def _benchmark_metric(bench_data: dict) -> Optional[tuple[str, int | float]]:
    """Use the same measured-score selection for the table and tracker."""
    preferred = ["acc,none", "acc_norm,none", "exact_match,none", "em,none"]
    for key in [*preferred, *(key for key in bench_data if key not in preferred)]:
        value = bench_data.get(key)
        if (isinstance(value, (int, float)) and not isinstance(value, bool)
                and not key.startswith("alias")):
            return key.split(",")[0], value
    return None


def _display_benchmark_results(results: dict, benchmarks: list[str]) -> None:
    """Display benchmark results as a rich table."""
    table = Table(title="Evaluation Results")
    table.add_column("Benchmark", style="bold")
    table.add_column("Metric", style="dim")
    table.add_column("Score", justify="right", style="green")

    task_results = results.get("results", {})
    for bench in benchmarks:
        bench_data = task_results.get(bench, {})
        if bench not in task_results:
            table.add_row(for_terminal(bench), "-", "[red]not found[/]")
            continue
        metric = _benchmark_metric(bench_data)
        if metric is not None:
            metric_name, score = metric
            table.add_row(for_terminal(bench), for_terminal(metric_name), f"{score:.4f}")
        else:
            table.add_row(for_terminal(bench), "-", "[yellow]no numeric result[/]")

    console.print(table)


def _save_benchmark_results(
    results: dict,
    model_path: str,
    benchmarks: list[str],
    run_id: Optional[str],
) -> int:
    """Save measured requested results, reporting skipped names and returning the count."""
    from soup_cli.experiment.tracker import ExperimentTracker

    tracker = ExperimentTracker()
    task_results = results.get("results", {})
    saved = 0
    try:
        for bench in benchmarks:
            bench_data = task_results.get(bench, {})
            metric = _benchmark_metric(bench_data)
            if metric is None:
                reason = "not found" if bench not in task_results else "no numeric result"
                console.print(f"[yellow]Skipped {for_terminal(bench)}: {reason}[/]")
                continue
            _, score = metric
            tracker.save_eval_result(
                model_path=model_path,
                benchmark=bench,
                score=score,
                details=bench_data,
                run_id=run_id,
            )
            saved += 1
    finally:
        tracker.close()
    skipped = len(benchmarks) - saved
    if skipped:
        console.print(f"[yellow]Skipped {skipped} of {len(benchmarks)} requested benchmarks.[/]")
    return saved


def _display_custom_results(eval_results: object) -> None:
    """Display custom eval results as a rich table."""
    table = Table(title="Custom Eval Results")
    table.add_column("Category", style="bold")
    table.add_column("Correct", justify="right")
    table.add_column("Total", justify="right")
    table.add_column("Accuracy", justify="right", style="green")

    for cat, cat_data in eval_results.category_scores.items():
        table.add_row(
            cat,
            str(cat_data["correct"]),
            str(cat_data["total"]),
            f"{cat_data['accuracy']:.2%}",
        )

    # Overall
    table.add_section()
    table.add_row(
        "[bold]Overall[/]",
        str(eval_results.correct),
        str(eval_results.total),
        f"[bold green]{eval_results.accuracy:.2%}[/]",
    )

    console.print(table)


def _save_custom_results(
    eval_results: object,
    model_path: str,
    run_id: Optional[str],
) -> None:
    """Save custom eval results to the experiment tracker."""
    from soup_cli.eval.custom import TASK_NAME
    from soup_cli.experiment.tracker import ExperimentTracker

    tracker = ExperimentTracker()
    tracker.save_eval_result(
        model_path=model_path,
        benchmark=TASK_NAME,
        score=eval_results.accuracy,
        details={
            "total": eval_results.total,
            "correct": eval_results.correct,
            "category_scores": eval_results.category_scores,
        },
        run_id=run_id,
    )


def _display_judge_results(results: object) -> None:
    """Display judge evaluation results."""
    # Criteria averages
    table = Table(title="Judge Evaluation Results")
    table.add_column("Criterion", style="bold")
    table.add_column("Average Score", justify="right", style="green")

    for crit, avg in results.criteria_averages.items():
        table.add_row(crit, f"{avg:.2f}")

    table.add_section()
    table.add_row(
        "[bold]Overall[/]",
        f"[bold green]{results.overall_score:.2f}[/]",
    )

    console.print(table)

    # Category breakdown
    if len(results.category_scores) > 1:
        cat_table = Table(title="Scores by Category")
        cat_table.add_column("Category", style="bold")
        cat_table.add_column("Average", justify="right", style="green")

        for cat, score in results.category_scores.items():
            cat_table.add_row(cat, f"{score:.2f}")

        console.print(cat_table)


def _display_elo_ratings(results: object) -> None:
    """Display Elo ratings from human evaluation."""
    table = Table(title="Elo Ratings")
    table.add_column("Model", style="bold")
    table.add_column("Rating", justify="right", style="green")
    table.add_column("W", justify="right")
    table.add_column("L", justify="right")
    table.add_column("T", justify="right")

    for name, rating in sorted(
        results.ratings.items(),
        key=lambda item: item[1].rating,
        reverse=True,
    ):
        table.add_row(
            _short_model_name(name),
            f"{rating.rating:.0f}",
            str(rating.wins),
            str(rating.losses),
            str(rating.ties),
        )

    console.print(table)


def _short_model_name(path: str) -> str:
    """Shorten model path for display."""
    parts = Path(path).parts
    if len(parts) > 2:
        return for_terminal("/".join(parts[-2:]))
    return for_terminal(path)


# ─── soup eval gate (v0.26.0 Part B) ───


@app.command(name="gate", cls=GateCommand)
def gate_cmd(
    suite: str = typer.Option(
        ..., "--suite", "-s",
        help="Path to eval suite YAML (see evals/gate.yaml example)",
    ),
    baseline: Optional[str] = typer.Option(
        None, "--baseline", "-b",
        help="Baseline: registry://<id> or path to stamped/flat score JSON",
    ),
    regression_threshold: float = typer.Option(
        0.05, "--regression-threshold",
        help="Max absolute drop vs baseline before regression fires (0.0-1.0)",
    ),
    model: Optional[str] = typer.Option(
        None, "--model", "-m",
        help="Model path or HF id to evaluate (required for live scoring)",
    ),
    write_baseline: Optional[str] = typer.Option(
        None,
        "--write-baseline",
        help=(
            "Write this run's task scores as a stamped baseline JSON "
            "({'scores', 'provenance'}) consumable by --baseline (#404)."
        ),
    ),
) -> None:
    """Run an eval-gate suite standalone (post-hoc verdict)."""
    if not 0.0 <= regression_threshold <= 1.0:
        console.print(
            "[red]--regression-threshold must be between 0.0 and 1.0[/]"
        )
        raise typer.Exit(EXIT_USAGE_ERROR)

    if write_baseline and not model:
        console.print("[red]--write-baseline requires --model[/]")
        raise typer.Exit(EXIT_USAGE_ERROR)

    from soup_cli.eval.gate import (
        load_suite,
        resolve_baseline,
        run_gate,
        write_baseline_file,
    )

    try:
        eval_suite = load_suite(suite)
    except (FileNotFoundError, ValueError) as exc:
        console.print(f"[red]Cannot load suite:[/] {for_terminal(str(exc))}")
        raise typer.Exit(EXIT_USAGE_ERROR) from exc

    try:
        baseline_scores = resolve_baseline(
            baseline,
            warn=lambda msg: console.print(f"[yellow]Warning:[/] {for_terminal(msg)}"),
        )
    except (FileNotFoundError, ValueError) as exc:
        console.print(f"[red]Cannot resolve baseline:[/] {exc}")
        raise typer.Exit(EXIT_USAGE_ERROR) from exc

    # When --model is provided, build a transformers-backed generator.
    # Otherwise fall back to an empty-string stub for smoke runs.
    # (--write-baseline already refused the stub path above.)
    if model is None:
        console.print(
            "[yellow]No --model given; using stub generator "
            "(empty output per prompt) for a smoke run.[/]"
        )

        def _stub_generate(_: str) -> str:
            return ""

        generate_fn = _stub_generate
    else:
        from soup_cli.eval.quant_check import make_model_generator

        try:
            generate_fn = make_model_generator(model)
        except (OSError, ValueError, ImportError) as exc:
            console.print(
                f"[red]Failed to load --model '{model}':[/] {exc}"
            )
            raise typer.Exit(1) from exc

    result = run_gate(
        eval_suite, generate_fn=generate_fn, baseline=baseline_scores,
        regression_threshold=regression_threshold,
    )

    _print_gate_result(result)

    if write_baseline:
        scores = {
            row.name: float(row.score)
            for row in result.task_results
            if row.score is not None
        }
        try:
            written = write_baseline_file(write_baseline, scores)
        except (OSError, ValueError, TypeError) as exc:
            console.print(
                f"[red]Cannot write --write-baseline:[/] {exc}"
            )
            raise typer.Exit(EXIT_RUNTIME_ERROR) from exc
        console.print(f"[green]Wrote stamped baseline[/] {written}")

    raise typer.Exit(EXIT_OK if result.passed else EXIT_GATE_FAILED)


# ─── soup eval quant-check (v0.26.0 Part D) ───


@app.command(name="quant-check", cls=GateCommand)
def quant_check_cmd(
    before: str = typer.Option(
        ..., "--before",
        help="Before-quantization model path or registry://<id>",
    ),
    after: str = typer.Option(
        ..., "--after",
        help="After-quantization model path or registry://<id>",
    ),
    tasks: str = typer.Option(
        ..., "--tasks",
        help="JSONL eval tasks file (see 'soup eval custom')",
    ),
    fmt: str = typer.Option(
        "table", "--format",
        help="Output format: table | json | markdown",
    ),
    allow_stub: bool = typer.Option(
        False, "--allow-stub",
        help="Allow deterministic stub generators if live model loading fails.",
    ),
) -> None:
    """Compare accuracy before vs after quantization on the same eval suite.

    Runs the same JSONL eval tasks through both models sequentially (memory
    safe) and renders a per-task delta with OK / MINOR / MAJOR verdicts.
    Exits with code 0 on OK / MINOR, code 2 on MAJOR, code 1 on model loading
    or runtime errors, and code 3 on usage or input validation errors.
    """
    from soup_cli.eval.quant_check import (
        ensure_format,
        is_under_cwd,
        render,
        resolve_model_ref,
        run_quant_check,
        stub_generator,
    )

    try:
        ensure_format(fmt)
    except ValueError as exc:
        console.print(f"[red]{exc}[/]")
        raise typer.Exit(EXIT_USAGE_ERROR) from exc

    resolved: dict[str, Optional[str]] = {}
    for label, ref in (("--before", before), ("--after", after)):
        try:
            resolved[label] = resolve_model_ref(ref)
        except ValueError as exc:  # AmbiguousRefError: a registry prefix or name:tag with twins
            console.print(f"[red]Cannot resolve {label}:[/] {for_terminal(str(exc))}")
            raise typer.Exit(EXIT_USAGE_ERROR) from exc
    resolved_before, resolved_after = resolved["--before"], resolved["--after"]
    if resolved_before is None:
        console.print(f"[red]Cannot resolve --before: {before}[/]")
        raise typer.Exit(EXIT_USAGE_ERROR)
    if resolved_after is None:
        console.print(f"[red]Cannot resolve --after: {after}[/]")
        raise typer.Exit(EXIT_USAGE_ERROR)

    for label, path_str in (("--before", resolved_before),
                            ("--after", resolved_after),
                            ("--tasks", tasks)):
        path_obj = Path(path_str)
        if not is_under_cwd(path_obj):
            console.print(
                f"[red]{label} '{path_str}' is outside cwd - refusing[/]"
            )
            raise typer.Exit(EXIT_USAGE_ERROR)
        if not path_obj.exists() and label == "--tasks":
            console.print(f"[red]{label} not found: {path_str}[/]")
            raise typer.Exit(EXIT_USAGE_ERROR)

    for label, path_str in (("--before", resolved_before), ("--after", resolved_after)):
        path_obj = Path(path_str)
        if path_obj.is_file() and path_obj.suffix.lower() == ".gguf":
            console.print(
                f"[red]{label} specifies a standalone GGUF file '{path_str}', "
                "which is not supported for live eval quant-check.\n"
                "Supported: directories containing safetensors / HuggingFace "
                "model files or registry:// refs.[/]"
            )
            raise typer.Exit(EXIT_USAGE_ERROR)

    before_path = Path(resolved_before)
    after_path = Path(resolved_after)
    if not before_path.exists():
        console.print(f"[red]--before not found: {resolved_before}[/]")
        raise typer.Exit(EXIT_USAGE_ERROR)
    if not after_path.exists():
        console.print(f"[red]--after not found: {resolved_after}[/]")
        raise typer.Exit(EXIT_USAGE_ERROR)

    # Live model scoring: build transformers-backed generators per side.
    # Deterministic stubs are used only if --allow-stub is explicitly passed.
    from soup_cli.eval.quant_check import make_model_generator

    is_stub = False

    try:
        before_gen = make_model_generator(resolved_before)
    except (OSError, ValueError, ImportError) as exc:
        if not allow_stub:
            console.print(
                f"[red]Failed to load --before model ({resolved_before}): {exc} "
                "(pass --allow-stub to score with deterministic stubs instead)[/]"
            )
            raise typer.Exit(EXIT_RUNTIME_ERROR) from exc
        if fmt != "json":
            console.print(
                f"[yellow]Failed to load --before model ({exc}); using deterministic stub.[/]"
            )
        before_gen = stub_generator("before")
        is_stub = True

    try:
        after_gen = make_model_generator(resolved_after)
    except (OSError, ValueError, ImportError) as exc:
        if not allow_stub:
            console.print(
                f"[red]Failed to load --after model ({resolved_after}): {exc} "
                "(pass --allow-stub to score with deterministic stubs instead)[/]"
            )
            raise typer.Exit(EXIT_RUNTIME_ERROR) from exc
        if fmt != "json":
            console.print(
                f"[yellow]Failed to load --after model ({exc}); using deterministic stub.[/]"
            )
        after_gen = stub_generator("after")
        is_stub = True

    result = run_quant_check(
        before_gen=before_gen,
        after_gen=after_gen,
        tasks_file=tasks,
        stub=is_stub,
    )
    rendered = render(result, fmt=fmt)
    if fmt == "table":
        console.print(rendered)
    else:
        # Plain text (markdown / json) — skip Rich markup interpretation so
        # pipe chars in markdown don't render as Rich tags, and disable
        # highlighting so JSON output is not polluted with ANSI escape codes.
        console.print(rendered, markup=False, highlight=False)

    has_major = any(r.verdict == "MAJOR" for r in result.rows)
    raise typer.Exit(EXIT_GATE_FAILED if has_major else EXIT_OK)


def _print_gate_result(result) -> None:
    """Render a GateResult as a Rich table + pass/fail panel."""
    table = Table(title="Eval gate")
    table.add_column("Task", style="cyan")
    table.add_column("Score", justify="right")
    table.add_column("Threshold", justify="right")
    table.add_column("Baseline", justify="right")
    table.add_column("Delta", justify="right")
    table.add_column("Verdict")
    for row in result.task_results:
        score_text = (
            f"{row.score:.3f}" if row.score is not None
            else "[red]ERROR[/]"
        )
        verdict = (
            "[green]PASS[/]" if row.passed
            else (f"[red]FAIL ({row.error})[/]" if row.error else "[red]FAIL[/]")
        )
        table.add_row(
            row.name,
            score_text,
            f"{row.threshold:.3f}",
            f"{row.baseline:.3f}" if row.baseline is not None else "-",
            f"{row.delta:+.3f}" if row.delta is not None else "-",
            verdict,
        )
    console.print(table)
    verdict = "[green]GATE PASSED[/]" if result.passed else "[red]GATE FAILED[/]"
    if result.regression:
        verdict += " [yellow](regression vs baseline)[/]"
    console.print(Panel(verdict, border_style="green" if result.passed else "red"))


# Register v0.55.0 subcommands (eval design / discover / lock / coverage / gate-install)
from soup_cli.commands._eval_v0550 import register as _register_v0550  # noqa: E402

_register_v0550(app, console)

# Register v0.61.0 subcommands (eval unlearning)
from soup_cli.commands._eval_v0610 import register as _register_v0610  # noqa: E402

_register_v0610(app, console)

# Register v0.65.0 subcommands (eval behavior / capability / checklist / irt-subset)
from soup_cli.commands._eval_v0650 import register as _register_v0650  # noqa: E402

_register_v0650(app, console)

# Register v0.71.10 subcommand (eval citation — #202)
from soup_cli.commands._eval_v07110 import register as _register_v07110  # noqa: E402

_register_v07110(app, console)