Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""soup eval — evaluation platform with benchmarks, custom evals, LLM judge, and more."""
from __future__ import annotations
import json
from pathlib import Path
from typing import Optional
import typer
from rich.console import Console
from rich.panel import Panel
from rich.table import Table
from soup_cli.utils.exit_codes import (
EXIT_GATE_FAILED,
EXIT_OK,
EXIT_RUNTIME_ERROR,
EXIT_USAGE_ERROR,
GateCommand,
)
from soup_cli.utils.terminal import for_terminal
console = Console()
app = typer.Typer(
name="eval",
help="Evaluate models: benchmarks, custom evals, LLM judge, leaderboard.",
no_args_is_help=True,
rich_markup_mode="rich",
)
def _reject_lm_eval_injection(value: str, field: str) -> None:
"""Block ``,`` / ``=`` in a value interpolated into lm-eval ``model_args``.
lm-eval parses ``model_args`` as comma-separated ``key=value`` pairs, so a
crafted ``base_model_name_or_path`` in an adapter's own
``adapter_config.json`` (e.g. ``attacker/m,trust_remote_code=True``) would
otherwise smuggle ``trust_remote_code=True`` and enable RCE on load.
Mirrors the ``commands/ship.py`` guard (v0.71.25).
"""
if "," in value or "=" in value:
console.print(
f"[red]Refusing to evaluate:[/] {field} contains ',' or '=' "
f"({for_terminal(repr(value))}); these are lm-eval model_args delimiters and could "
"smuggle trust_remote_code=True (arbitrary code execution)."
)
raise typer.Exit(1)
# ─── soup eval benchmark ───
@app.command()
def benchmark(
model: str = typer.Option(
..., "--model", "-m",
help="Path to model or LoRA adapter directory",
),
benchmarks: str = typer.Option(
"mmlu", "--benchmarks", "-b",
help="Comma-separated benchmark names (mmlu, gsm8k, hellaswag, etc.)",
),
num_fewshot: Optional[int] = typer.Option(
None, "--fewshot", "-f",
help="Number of few-shot examples (benchmark default if not set)",
),
batch_size: int = typer.Option(
8, "--batch-size",
help="Batch size for evaluation",
),
run_id: Optional[str] = typer.Option(
None, "--run-id",
help="Link results to an existing training run",
),
device: Optional[str] = typer.Option(
None, "--device",
help="Device: cuda, mps, cpu. Auto-detected if not set.",
),
trust_remote_code: bool = typer.Option(
False,
"--trust-remote-code",
help=(
"Allow loading models that ship custom Python via auto_map. "
"Default deny (v0.36.0). Only enable if you trust the source."
),
),
):
"""Evaluate on standard benchmarks (wraps lm-evaluation-harness)."""
model_path = Path(model)
if not model_path.exists():
console.print(f"[red]Model path not found: {for_terminal(model_path)}[/]")
raise typer.Exit(1)
# Both the user-supplied path and the adapter's declared base model are
# interpolated into lm-eval ``model_args`` below; reject the ','/'=' that
# would smuggle extra kwargs (e.g. trust_remote_code=True).
_reject_lm_eval_injection(str(model_path), "--model path")
# Check for LoRA adapter and resolve base model
adapter_config = model_path / "adapter_config.json"
model_arg = str(model_path)
if adapter_config.exists():
adapter_info = json.loads(adapter_config.read_text())
base_model = adapter_info.get("base_model_name_or_path", "")
if base_model:
_reject_lm_eval_injection(
base_model, "base_model_name_or_path (from adapter_config.json)"
)
model_arg = f"pretrained={base_model},peft={model_path}"
console.print(
f"[dim]LoRA adapter detected. Base model: {for_terminal(base_model)}[/]"
)
else:
model_arg = f"pretrained={model_path}"
else:
model_arg = f"pretrained={model_path}"
if trust_remote_code:
# Appended only from the resolved --trust-remote-code flag, never
# from adapter_config.json: _reject_lm_eval_injection above already
# refused a base_model_name_or_path smuggling its own ','/'=' pair.
model_arg = f"{model_arg},trust_remote_code=True"
benchmark_list = [b.strip() for b in benchmarks.split(",")]
console.print(f"[dim]Evaluating on: {', '.join(for_terminal(b) for b in benchmark_list)}[/]")
# Lazy import lm_eval
try:
import lm_eval # noqa: F401
except ImportError:
console.print(
"[red]lm-eval not installed.[/]\n"
"Install with: [bold]pip install \"soup-cli\\[eval]\"[/]"
)
raise typer.Exit(1)
# Detect device
if not device:
from soup_cli.utils.gpu import detect_device
device, _ = detect_device()
console.print("[dim]Running evaluation (this may take a while)...[/]")
results = _run_lm_eval(
model_arg=model_arg,
tasks=benchmark_list,
num_fewshot=num_fewshot,
batch_size=batch_size,
device=device,
)
_display_benchmark_results(results, benchmark_list)
saved = _save_benchmark_results(results, str(model_path), benchmark_list, run_id)
if not saved:
console.print("[red]Nothing measured; nothing saved to experiment tracker.[/]")
raise typer.Exit(1)
console.print(
f"\n[green]Results saved to experiment tracker: {saved} of {len(benchmark_list)}.[/]"
)
# ─── soup eval aider ───
@app.command()
def aider(
model: str = typer.Option(
..., "--model", "-m",
help="Aider model identifier (for example, openai/gpt-4.1)",
),
output: str = typer.Option(
..., "--output", "-o",
help="Contained directory for upstream and Soup result files",
),
exercises_dir: str = typer.Option(
"tmp.benchmarks/polyglot-benchmark", "--exercises-dir",
help="Prepared Aider-AI/polyglot-benchmark checkout",
),
image: str = typer.Option(
"aider-benchmark", "--image",
help="Locally built upstream Aider benchmark image",
),
threads: int = typer.Option(
1, "--threads", "-t", min=1, max=64,
help="Number of benchmark exercises to run concurrently",
),
num_tests: int = typer.Option(
-1, "--num-tests", "-n",
help="Number of exercises to run (-1 runs all exercises)",
),
timeout: int = typer.Option(
86_400, "--timeout", min=60, max=86_400,
help="Maximum benchmark runtime in seconds",
),
run_id: Optional[str] = typer.Option(
None, "--run-id",
help="Save the aggregate score under an existing Soup run ID",
),
allow_host_services: bool = typer.Option(
False, "--allow-host-services",
help="Let the benchmark container reach services on the host",
),
):
"""Run Aider's Polyglot coding benchmark in its official Docker image."""
from rich.markup import escape
from soup_cli.eval.aider_polyglot import (
AiderEvalError,
build_docker_command,
parse_aider_results,
preflight_docker,
prepare_output_dir,
validate_exercises_dir,
write_soup_result,
)
from soup_cli.utils.paths import is_under_cwd
tracker = None
if run_id:
from soup_cli.experiment.tracker import ExperimentTracker
tracker = ExperimentTracker()
if tracker.get_run(run_id) is None:
console.print(f"[red]Run not found: {escape(run_id)}[/]")
tracker.close()
raise typer.Exit(1)
try:
output_dir = prepare_output_dir(output)
corpus_dir = validate_exercises_dir(exercises_dir)
corpus_outside_cwd = not is_under_cwd(corpus_dir)
docker = preflight_docker(image)
command = build_docker_command(
docker=docker,
image=image,
model=model,
exercises_dir=corpus_dir,
output_dir=output_dir,
threads=threads,
num_tests=num_tests,
allow_host_services=allow_host_services,
)
except AiderEvalError as exc:
if tracker is not None:
tracker.close()
console.print(f"[red]Aider Polyglot preflight failed:[/] {escape(str(exc))}")
raise typer.Exit(1) from exc
if corpus_outside_cwd:
console.print(
"[yellow]Warning:[/] --exercises-dir resolves outside the current "
"working directory; Docker will still mount it read-only."
)
console.print(
Panel(
f"[bold]Aider Polyglot[/]\n"
f"Model: {escape(model)}\n"
f"Output: {escape(str(output_dir))}\n"
f"Image: {escape(image)}\n"
f"Exercises: {escape(str(corpus_dir))}",
title="Evaluation",
border_style="blue",
)
)
console.print(
"[yellow]The benchmark executes untrusted model-generated code inside Docker.[/]"
)
import subprocess
try:
process = subprocess.run( # noqa: S603 -- fixed argv, no shell
command,
check=False,
shell=False,
timeout=timeout,
)
except subprocess.TimeoutExpired as exc:
if tracker is not None:
tracker.close()
console.print(f"[red]Aider benchmark timed out after {timeout} seconds.[/]")
raise typer.Exit(1) from exc
except OSError as exc:
if tracker is not None:
tracker.close()
console.print(
f"[red]Could not start the Aider benchmark:[/] {type(exc).__name__}"
)
raise typer.Exit(1) from exc
if process.returncode != 0:
if tracker is not None:
tracker.close()
console.print(
f"[red]Aider benchmark exited with status {process.returncode}.[/] "
f"Partial files remain in {escape(str(output_dir))}."
)
raise typer.Exit(1)
try:
row = parse_aider_results(output_dir, model=model)
result_path = write_soup_result(output_dir, row)
except AiderEvalError as exc:
if tracker is not None:
tracker.close()
console.print(f"[red]Could not aggregate Aider results:[/] {escape(str(exc))}")
raise typer.Exit(1) from exc
if tracker is not None:
tracker.save_eval_result(
model_path=model,
benchmark=row["task"],
score=row["score"],
details={"errors": row["errors"], **row["details"]},
run_id=run_id,
)
tracker.close()
console.print(f"[green]Saved result under run {escape(run_id)}.[/]")
completed = row["details"]["completed_tests"]
passed = row["details"]["passed_tests"]
console.print(
f"[green]Aider Polyglot complete:[/] {passed}/{completed} passed "
f"(score {row['score']:.4f})"
)
console.print(f"[green]Soup result written to:[/] {escape(str(result_path))}")
# ─── soup eval custom ───
@app.command()
def custom(
tasks: str = typer.Option(
..., "--tasks", "-t",
help="Path to eval tasks JSONL file",
),
model: str = typer.Option(
..., "--model", "-m",
help="Path to model directory",
),
run_id: Optional[str] = typer.Option(
None, "--run-id",
help="Link results to an existing training run",
),
attach_to_registry: Optional[str] = typer.Option(
None, "--attach-to-registry",
help="Attach the eval JSON to a registry entry as kind=eval_results",
),
output: Optional[str] = typer.Option(
None, "--output", "-o",
help=(
"Path to write the eval JSON output. Honored independently of "
"--attach-to-registry (v0.40.1 / G10)."
),
),
trust_remote_code: bool = typer.Option(
False,
"--trust-remote-code",
help=(
"Allow loading models that ship custom Python via auto_map. "
"Default deny (v0.36.0). Only enable if you trust the source."
),
),
):
"""Run custom evaluation tasks from a JSONL file."""
from soup_cli.eval.custom import load_eval_tasks
tasks_path = Path(tasks)
model_path = Path(model)
if not model_path.exists():
console.print(f"[red]Model path not found: {for_terminal(model_path)}[/]")
raise typer.Exit(1)
# Load and validate tasks
try:
eval_tasks = load_eval_tasks(tasks_path)
except (FileNotFoundError, ValueError) as exc:
console.print(f"[red]{exc}[/]")
raise typer.Exit(1)
console.print(
f"[dim]Loaded {len(eval_tasks)} eval tasks from {for_terminal(tasks_path)}[/]"
)
# Run evaluation with progress
from rich.progress import (
BarColumn,
Progress,
SpinnerColumn,
TaskProgressColumn,
TextColumn,
)
progress = Progress(
SpinnerColumn(),
TextColumn("[progress.description]{task.description}"),
BarColumn(),
TaskProgressColumn(),
console=console,
)
results_list = []
from soup_cli.eval.custom import _create_default_generator, score_task
console.print("[dim]Loading model...[/]")
generate_fn = _create_default_generator(
str(model_path), trust_remote_code=trust_remote_code,
)
with progress:
task_bar = progress.add_task(
"Evaluating...", total=len(eval_tasks),
)
# v0.40.1 Part D / G10 — keep the CLI ``--output`` path separate
# from the per-task model response (was previously shadowed by the
# loop variable, masking ``-o`` whenever attach-to-registry was off).
for eval_task in eval_tasks:
response = generate_fn(eval_task.prompt)
result = score_task(eval_task, response)
results_list.append(result)
progress.advance(task_bar)
from soup_cli.eval.custom import EvalResults
eval_results = EvalResults(results=results_list)
eval_results.compute()
# Display results
_display_custom_results(eval_results)
# Save to tracker
_save_custom_results(eval_results, str(model_path), run_id)
console.print("\n[green]Results saved to experiment tracker.[/]")
# v0.40.1 Part D / G10 — write `--output` JSON regardless of registry
# attach. Both flags compose; either alone is sufficient.
payload = None
json_path = None
if output or attach_to_registry:
from soup_cli.registry.attach import attach_artifact, write_eval_json
payload = {
"model": str(model_path),
"tasks": str(tasks_path),
"total": eval_results.total,
"correct": eval_results.correct,
"accuracy": eval_results.accuracy,
"category_scores": eval_results.category_scores,
}
write_target = output or "eval_results.json"
try:
json_path = write_eval_json(write_target, payload=payload)
except (ValueError, FileNotFoundError) as exc:
console.print(f"[red]Failed to write eval JSON:[/] {exc}")
raise typer.Exit(1) from exc
if output:
console.print(f"[green]Eval JSON written to:[/] {json_path}")
if attach_to_registry:
try:
attach_artifact(
attach_to_registry, path=str(json_path), kind="eval_results",
)
except (ValueError, FileNotFoundError) as exc:
console.print(f"[red]Registry attach failed:[/] {exc}")
raise typer.Exit(1) from exc
console.print(
f"[green]Attached eval results to registry entry "
f"'{attach_to_registry}' as eval_results.[/]"
)
# ─── soup eval judge ───
@app.command()
def judge(
target: str = typer.Option(
..., "--target",
help="Path to JSONL with prompt+response pairs to evaluate",
),
judge_model: str = typer.Option(
"gpt-4o-mini", "--model", "-m",
help="Judge model name",
),
provider: str = typer.Option(
"openai", "--provider",
help="Judge provider: openai, server, ollama",
),
rubric: Optional[str] = typer.Option(
None, "--rubric", "-r",
help="Path to rubric YAML file",
),
api_base: Optional[str] = typer.Option(
None, "--api-base",
help="API base URL for judge model",
),
run_id: Optional[str] = typer.Option(
None, "--run-id",
help="Link results to an existing training run",
),
):
"""Evaluate model outputs using LLM-as-a-judge."""
from soup_cli.eval.judge import (
JudgeDownError,
JudgeEvaluator,
JudgeUnavailableError,
load_rubric,
validate_judge_api_base,
)
# Validate API base
try:
validate_judge_api_base(api_base)
except ValueError as exc:
console.print(f"[red]{exc}[/]")
raise typer.Exit(1)
# Load target data
target_path = Path(target)
if not target_path.exists():
console.print(f"[red]Target file not found: {target_path}[/]")
raise typer.Exit(1)
items: list[dict] = []
with open(target_path, encoding="utf-8") as fh:
for line_num, line in enumerate(fh, 1):
line = line.strip()
if not line:
continue
try:
row = json.loads(line)
except json.JSONDecodeError as exc:
console.print(
f"[red]Invalid JSON on line {line_num}: {exc}[/]"
)
raise typer.Exit(1)
if "prompt" not in row or "response" not in row:
console.print(
f"[red]Line {line_num}: missing 'prompt' or 'response'[/]"
)
raise typer.Exit(1)
items.append(row)
# Load rubric
rubric_data = None
if rubric:
try:
rubric_data = load_rubric(Path(rubric))
except (FileNotFoundError, ValueError) as exc:
# The message quotes the path it was given.
console.print(f"[red]{for_terminal(str(exc))}[/]")
raise typer.Exit(1)
console.print(
f"[dim]Judging {len(items)} responses with {judge_model} "
f"(provider: {provider})...[/]"
)
# Run judge evaluation
try:
evaluator = JudgeEvaluator(
rubric=rubric_data,
provider=provider,
model=judge_model,
api_base=api_base,
)
except ValueError as exc:
console.print(f"[red]{exc}[/]")
raise typer.Exit(1)
from rich.progress import (
BarColumn,
Progress,
SpinnerColumn,
TaskProgressColumn,
TextColumn,
)
from soup_cli.eval.judge import JudgeResults
judge_scores = []
skipped_count = 0
down = None
progress = Progress(
SpinnerColumn(),
TextColumn("[progress.description]{task.description}"),
BarColumn(),
TaskProgressColumn(),
console=console,
)
with progress:
task_bar = progress.add_task("Judging...", total=len(items))
for item in items:
try:
score = evaluator.evaluate(
prompt=item["prompt"],
response=item["response"],
category=item.get("category", "default"),
)
judge_scores.append(score)
except JudgeDownError as exc:
# #1522: it has stopped answering, so the rest would fail alike.
down = exc.for_rows(len(items) - len(judge_scores), len(items), "items")
break
except (ValueError, OSError, KeyError, JudgeUnavailableError) as exc:
skipped_count += 1
console.print(
f"[yellow]Warning: judge failed for prompt: {for_terminal(exc)}[/]"
)
progress.advance(task_bar)
if down is not None:
console.print(f"[red]Judging stopped: {for_terminal(down)}[/]")
if judge_scores:
# Not saved to the tracker: a mean over the rows before the outage
# is not this benchmark's score.
results = JudgeResults(scores=judge_scores)
results.compute()
_display_judge_results(results)
else:
console.print("[red]All items failed. Check judge configuration.[/]")
raise typer.Exit(1)
if skipped_count > 0:
console.print(
f"[yellow]{skipped_count}/{len(items)} items skipped "
f"due to judge errors.[/]"
)
if not judge_scores:
console.print("[red]All items failed. Check judge configuration.[/]")
raise typer.Exit(1)
results = JudgeResults(scores=judge_scores)
results.compute()
_display_judge_results(results)
# Save to tracker
if run_id:
from soup_cli.experiment.tracker import ExperimentTracker
tracker = ExperimentTracker()
tracker.save_eval_result(
model_path=target,
benchmark=f"judge:{judge_model}",
score=results.overall_score,
details={
"criteria_averages": results.criteria_averages,
"category_scores": results.category_scores,
"total_evaluated": len(results.scores),
},
run_id=run_id,
)
console.print("[green]Results saved to experiment tracker.[/]")
# ─── soup eval auto ───
@app.command()
def auto(
config: str = typer.Option(
"soup.yaml", "--config", "-c",
help="Path to soup.yaml config file",
),
benchmarks: Optional[str] = typer.Option(
None, "--benchmarks", "-b",
help="Comma-separated benchmarks (overrides config)",
),
custom_tasks: Optional[str] = typer.Option(
None, "--tasks", "-t",
help="Path to custom eval JSONL (overrides config)",
),
trust_remote_code: bool = typer.Option(
False,
"--trust-remote-code",
help=(
"Allow loading models that ship custom Python via auto_map. "
"Default deny (v0.36.0). Only enable if you trust the source."
),
),
):
"""Run automatic evaluation using config from soup.yaml."""
from soup_cli.config.loader import load_config
config_path = Path(config)
if not config_path.exists():
console.print(f"[red]Config not found: {config_path}[/]")
raise typer.Exit(1)
soup_config = load_config(str(config_path))
output_dir = Path(soup_config.output)
if not output_dir.exists():
console.print(f"[red]Output directory not found: {output_dir}[/]")
console.print("[dim]Run soup train first to create a model.[/]")
raise typer.Exit(1)
eval_config = getattr(soup_config, "eval", None)
# Determine benchmarks
bench_list = []
if benchmarks:
bench_list = [b.strip() for b in benchmarks.split(",")]
elif eval_config and hasattr(eval_config, "benchmarks"):
bench_list = eval_config.benchmarks or []
# Determine custom tasks
tasks_file = custom_tasks
if not tasks_file and eval_config and hasattr(eval_config, "custom_tasks"):
tasks_file = eval_config.custom_tasks
if not bench_list and not tasks_file:
console.print(
"[yellow]No benchmarks or custom tasks specified.[/]\n"
"Use --benchmarks or --tasks, or add eval config to soup.yaml."
)
raise typer.Exit(1)
console.print(Panel(
f"[bold]Auto-Eval[/]\n"
f"Model: {output_dir}\n"
f"Benchmarks: {', '.join(bench_list) if bench_list else 'none'}\n"
f"Custom tasks: {tasks_file or 'none'}",
title="Evaluation",
border_style="blue",
))
# #1808: a leg that could not run used to be swallowed, and "Auto-eval
# complete." exited 0 even when it was the only leg. Count what ran.
legs_ran = 0
# Run standard benchmarks if specified
if bench_list:
console.print("\n[bold]Standard Benchmarks[/]")
try:
benchmark(
model=str(output_dir),
benchmarks=",".join(bench_list),
num_fewshot=None,
batch_size=8,
run_id=None,
device=None,
trust_remote_code=trust_remote_code,
)
except (typer.Exit, SystemExit):
# benchmark() signals failure with typer.Exit (a RuntimeError, NOT
# SystemExit), so the old `except SystemExit` never caught it and a
# benchmark failure aborted the whole auto-eval instead of falling
# through to the custom eval below.
console.print("[yellow]Benchmark eval skipped (see above).[/]")
else:
legs_ran += 1
# Run custom eval if specified
if tasks_file:
console.print("\n[bold]Custom Evaluation[/]")
try:
# #752 — every typer parameter must be passed explicitly. Typer
# fills them only when typer invokes the command; called as a
# function, an unpassed parameter keeps its OptionInfo default,
# which is truthy, so the --output/--attach-to-registry block ran
# and died on write_eval_json(OptionInfo).
custom(
tasks=tasks_file,
model=str(output_dir),
run_id=None,
attach_to_registry=None,
output=None,
trust_remote_code=trust_remote_code,
)
except (typer.Exit, SystemExit):
# The same pair as the benchmark leg (#1808): custom() also fails
# with typer.Exit, which `except SystemExit` alone never caught, so
# the exit code depended on which leg had failed.
console.print("[yellow]Custom eval skipped (see above).[/]")
else:
legs_ran += 1
if legs_ran == 0:
console.print(
"\n[red]Auto-eval failed: nothing was evaluated.[/] Every leg was skipped "
"(see above); fix the cause and run again."
)
raise typer.Exit(1)
console.print("\n[green]Auto-eval complete.[/]")
# ─── soup eval compare ───
@app.command()
def compare(
run1: str = typer.Argument(..., help="First run ID"),
run2: str = typer.Argument(..., help="Second run ID"),
):
"""Compare eval results between two training runs."""
from soup_cli.eval.leaderboard import compare_runs
from soup_cli.experiment.tracker import ExperimentTracker
tracker = ExperimentTracker()
# Verify runs exist
for rid in [run1, run2]:
run_data = tracker.get_run(rid)
if run_data is None:
console.print(f"[red]Run not found: {rid}[/]")
raise typer.Exit(1)
comparison = compare_runs(tracker, run1, run2)
if not comparison["comparisons"]:
console.print("[yellow]No eval results found for these runs.[/]")
console.print("[dim]Run soup eval benchmark --run-id ... first.[/]")
raise typer.Exit(1)
# Display comparison table
table = Table(title=f"Eval Comparison: {run1} vs {run2}")
table.add_column("Benchmark", style="bold")
table.add_column("Run 1", justify="right")
table.add_column("Run 2", justify="right")
table.add_column("Delta", justify="right")
for comp in comparison["comparisons"]:
score_a = (
f"{comp['run_1_score']:.4f}" if comp["run_1_score"] is not None
else "-"
)
score_b = (
f"{comp['run_2_score']:.4f}" if comp["run_2_score"] is not None
else "-"
)
if comp["delta"] is not None:
delta_val = comp["delta"]
if delta_val > 0.01:
delta_str = f"[green]+{delta_val:.4f}[/]"
elif delta_val < -0.01:
delta_str = f"[red]{delta_val:.4f}[/]"
else:
delta_str = f"{delta_val:.4f}"
else:
delta_str = "-"
table.add_row(comp["benchmark"], score_a, score_b, delta_str)
console.print(table)
if comparison["has_regressions"]:
console.print(
f"\n[red bold]Regressions detected:[/] "
f"{', '.join(comparison['regressions'])}"
)
# ─── soup eval leaderboard ───
@app.command()
def leaderboard(
sort_by: Optional[str] = typer.Option(
None, "--sort-by", "-s",
help="Sort by specific benchmark (default: average)",
),
fmt: str = typer.Option(
"table", "--format", "-f",
help="Output format: table, json, csv",
),
):
"""Show local leaderboard across all evaluated models."""
from soup_cli.eval.leaderboard import (
build_leaderboard_from_tracker,
export_leaderboard,
)
from soup_cli.experiment.tracker import ExperimentTracker
tracker = ExperimentTracker()
lb = build_leaderboard_from_tracker(tracker)
if not lb.entries:
console.print("[yellow]No eval results found.[/]")
console.print(
"[dim]Run soup eval benchmark or soup eval custom first.[/]"
)
raise typer.Exit(1)
if fmt in ("json", "csv"):
output = export_leaderboard(lb, fmt=fmt)
# Machine-readable JSON/CSV: raw stdout, not console.print — Rich folds
# rows/lines at the console width and parses ``[...]`` as markup when
# piped, corrupting the output (issue #1468).
typer.echo(output)
return
# Table format
sorted_models = lb.get_sorted_models(sort_by=sort_by)
# Collect all benchmarks
all_benchmarks: set[str] = set()
for _, scores, _ in sorted_models:
all_benchmarks.update(scores.keys())
benchmarks = sorted(all_benchmarks)
table = Table(title="Eval Leaderboard")
table.add_column("#", style="dim", width=3)
table.add_column("Model", style="bold")
for bench in benchmarks:
table.add_column(bench, justify="right")
table.add_column("Avg", justify="right", style="green")
for rank, (model_name, scores, avg) in enumerate(sorted_models, 1):
row = [str(rank), _short_model_name(model_name)]
for bench in benchmarks:
if bench in scores:
row.append(f"{scores[bench]:.4f}")
else:
row.append("-")
row.append(f"{avg:.4f}")
table.add_row(*row)
console.print(table)
# ─── soup eval human ───
@app.command()
def human(
prompts_file: str = typer.Option(
..., "--input", "-i",
help="Path to JSONL file with evaluation prompts",
),
model_a: str = typer.Option(
..., "--model-a", "-a",
help="Path to first model",
),
model_b: str = typer.Option(
..., "--model-b", "-b",
help="Path to second model",
),
output: str = typer.Option(
"human_eval_results.json", "--output", "-o",
help="Output file for results",
),
):
"""Run human A/B evaluation between two models."""
from soup_cli.eval.human import (
HumanEvalResults,
HumanJudgment,
load_prompts,
save_results,
)
# Load prompts
try:
prompts = load_prompts(Path(prompts_file))
except (FileNotFoundError, ValueError) as exc:
console.print(f"[red]{exc}[/]")
raise typer.Exit(1)
# Verify models exist
for mpath in [model_a, model_b]:
if not Path(mpath).exists():
console.print(f"[red]Model not found: {for_terminal(mpath)}[/]")
raise typer.Exit(1)
console.print(Panel(
f"[bold]Human Evaluation[/]\n"
f"Prompts: {len(prompts)}\n"
f"Model A: {for_terminal(model_a)}\n"
f"Model B: {for_terminal(model_b)}",
title="A/B Comparison",
border_style="blue",
))
# Generate responses
console.print("[dim]Loading models and generating responses...[/]")
from soup_cli.eval.custom import _create_default_generator
gen_a = _create_default_generator(model_a)
gen_b = _create_default_generator(model_b)
results = HumanEvalResults()
for idx, prompt_data in enumerate(prompts):
prompt_text = prompt_data["prompt"]
resp_a = gen_a(prompt_text)
resp_b = gen_b(prompt_text)
# Display for human judgment
console.print(f"\n[bold]--- Prompt {idx + 1}/{len(prompts)} ---[/]")
console.print(Panel(for_terminal(prompt_text), title="Prompt", border_style="dim"))
console.print(Panel(for_terminal(resp_a), title="[blue]Response A[/]"))
console.print(Panel(for_terminal(resp_b), title="[magenta]Response B[/]"))
# Get human choice
while True:
choice = console.input(
"[bold]Winner? (a/b/tie/q to quit): [/]"
).strip().lower()
if choice in ("a", "b", "tie", "q"):
break
console.print("[yellow]Enter a, b, tie, or q[/]")
if choice == "q":
console.print("[dim]Session ended early.[/]")
break
results.judgments.append(HumanJudgment(
prompt=prompt_text,
response_a=resp_a,
response_b=resp_b,
model_a=model_a,
model_b=model_b,
winner=choice,
))
results.compute_ratings()
# Display Elo ratings
_display_elo_ratings(results)
# Save results
save_results(results, Path(output))
console.print(f"\n[green]Results saved to {for_terminal(output)}[/]")
# ─── Helper functions ───
def _run_lm_eval(
model_arg: str,
tasks: list[str],
num_fewshot: Optional[int],
batch_size: int,
device: str,
) -> dict:
"""Run lm-evaluation-harness and return results dict."""
import lm_eval
results = lm_eval.simple_evaluate(
model="hf",
model_args=model_arg,
tasks=tasks,
num_fewshot=num_fewshot,
batch_size=batch_size,
device=device,
)
return results
def _benchmark_metric(bench_data: dict) -> Optional[tuple[str, int | float]]:
"""Use the same measured-score selection for the table and tracker."""
preferred = ["acc,none", "acc_norm,none", "exact_match,none", "em,none"]
for key in [*preferred, *(key for key in bench_data if key not in preferred)]:
value = bench_data.get(key)
if (isinstance(value, (int, float)) and not isinstance(value, bool)
and not key.startswith("alias")):
return key.split(",")[0], value
return None
def _display_benchmark_results(results: dict, benchmarks: list[str]) -> None:
"""Display benchmark results as a rich table."""
table = Table(title="Evaluation Results")
table.add_column("Benchmark", style="bold")
table.add_column("Metric", style="dim")
table.add_column("Score", justify="right", style="green")
task_results = results.get("results", {})
for bench in benchmarks:
bench_data = task_results.get(bench, {})
if bench not in task_results:
table.add_row(for_terminal(bench), "-", "[red]not found[/]")
continue
metric = _benchmark_metric(bench_data)
if metric is not None:
metric_name, score = metric
table.add_row(for_terminal(bench), for_terminal(metric_name), f"{score:.4f}")
else:
table.add_row(for_terminal(bench), "-", "[yellow]no numeric result[/]")
console.print(table)
def _save_benchmark_results(
results: dict,
model_path: str,
benchmarks: list[str],
run_id: Optional[str],
) -> int:
"""Save measured requested results, reporting skipped names and returning the count."""
from soup_cli.experiment.tracker import ExperimentTracker
tracker = ExperimentTracker()
task_results = results.get("results", {})
saved = 0
try:
for bench in benchmarks:
bench_data = task_results.get(bench, {})
metric = _benchmark_metric(bench_data)
if metric is None:
reason = "not found" if bench not in task_results else "no numeric result"
console.print(f"[yellow]Skipped {for_terminal(bench)}: {reason}[/]")
continue
_, score = metric
tracker.save_eval_result(
model_path=model_path,
benchmark=bench,
score=score,
details=bench_data,
run_id=run_id,
)
saved += 1
finally:
tracker.close()
skipped = len(benchmarks) - saved
if skipped:
console.print(f"[yellow]Skipped {skipped} of {len(benchmarks)} requested benchmarks.[/]")
return saved
def _display_custom_results(eval_results: object) -> None:
"""Display custom eval results as a rich table."""
table = Table(title="Custom Eval Results")
table.add_column("Category", style="bold")
table.add_column("Correct", justify="right")
table.add_column("Total", justify="right")
table.add_column("Accuracy", justify="right", style="green")
for cat, cat_data in eval_results.category_scores.items():
table.add_row(
cat,
str(cat_data["correct"]),
str(cat_data["total"]),
f"{cat_data['accuracy']:.2%}",
)
# Overall
table.add_section()
table.add_row(
"[bold]Overall[/]",
str(eval_results.correct),
str(eval_results.total),
f"[bold green]{eval_results.accuracy:.2%}[/]",
)
console.print(table)
def _save_custom_results(
eval_results: object,
model_path: str,
run_id: Optional[str],
) -> None:
"""Save custom eval results to the experiment tracker."""
from soup_cli.eval.custom import TASK_NAME
from soup_cli.experiment.tracker import ExperimentTracker
tracker = ExperimentTracker()
tracker.save_eval_result(
model_path=model_path,
benchmark=TASK_NAME,
score=eval_results.accuracy,
details={
"total": eval_results.total,
"correct": eval_results.correct,
"category_scores": eval_results.category_scores,
},
run_id=run_id,
)
def _display_judge_results(results: object) -> None:
"""Display judge evaluation results."""
# Criteria averages
table = Table(title="Judge Evaluation Results")
table.add_column("Criterion", style="bold")
table.add_column("Average Score", justify="right", style="green")
for crit, avg in results.criteria_averages.items():
table.add_row(crit, f"{avg:.2f}")
table.add_section()
table.add_row(
"[bold]Overall[/]",
f"[bold green]{results.overall_score:.2f}[/]",
)
console.print(table)
# Category breakdown
if len(results.category_scores) > 1:
cat_table = Table(title="Scores by Category")
cat_table.add_column("Category", style="bold")
cat_table.add_column("Average", justify="right", style="green")
for cat, score in results.category_scores.items():
cat_table.add_row(cat, f"{score:.2f}")
console.print(cat_table)
def _display_elo_ratings(results: object) -> None:
"""Display Elo ratings from human evaluation."""
table = Table(title="Elo Ratings")
table.add_column("Model", style="bold")
table.add_column("Rating", justify="right", style="green")
table.add_column("W", justify="right")
table.add_column("L", justify="right")
table.add_column("T", justify="right")
for name, rating in sorted(
results.ratings.items(),
key=lambda item: item[1].rating,
reverse=True,
):
table.add_row(
_short_model_name(name),
f"{rating.rating:.0f}",
str(rating.wins),
str(rating.losses),
str(rating.ties),
)
console.print(table)
def _short_model_name(path: str) -> str:
"""Shorten model path for display."""
parts = Path(path).parts
if len(parts) > 2:
return for_terminal("/".join(parts[-2:]))
return for_terminal(path)
# ─── soup eval gate (v0.26.0 Part B) ───
@app.command(name="gate", cls=GateCommand)
def gate_cmd(
suite: str = typer.Option(
..., "--suite", "-s",
help="Path to eval suite YAML (see evals/gate.yaml example)",
),
baseline: Optional[str] = typer.Option(
None, "--baseline", "-b",
help="Baseline: registry://<id> or path to stamped/flat score JSON",
),
regression_threshold: float = typer.Option(
0.05, "--regression-threshold",
help="Max absolute drop vs baseline before regression fires (0.0-1.0)",
),
model: Optional[str] = typer.Option(
None, "--model", "-m",
help="Model path or HF id to evaluate (required for live scoring)",
),
write_baseline: Optional[str] = typer.Option(
None,
"--write-baseline",
help=(
"Write this run's task scores as a stamped baseline JSON "
"({'scores', 'provenance'}) consumable by --baseline (#404)."
),
),
) -> None:
"""Run an eval-gate suite standalone (post-hoc verdict)."""
if not 0.0 <= regression_threshold <= 1.0:
console.print(
"[red]--regression-threshold must be between 0.0 and 1.0[/]"
)
raise typer.Exit(EXIT_USAGE_ERROR)
if write_baseline and not model:
console.print("[red]--write-baseline requires --model[/]")
raise typer.Exit(EXIT_USAGE_ERROR)
from soup_cli.eval.gate import (
load_suite,
resolve_baseline,
run_gate,
write_baseline_file,
)
try:
eval_suite = load_suite(suite)
except (FileNotFoundError, ValueError) as exc:
console.print(f"[red]Cannot load suite:[/] {for_terminal(str(exc))}")
raise typer.Exit(EXIT_USAGE_ERROR) from exc
try:
baseline_scores = resolve_baseline(
baseline,
warn=lambda msg: console.print(f"[yellow]Warning:[/] {for_terminal(msg)}"),
)
except (FileNotFoundError, ValueError) as exc:
console.print(f"[red]Cannot resolve baseline:[/] {exc}")
raise typer.Exit(EXIT_USAGE_ERROR) from exc
# When --model is provided, build a transformers-backed generator.
# Otherwise fall back to an empty-string stub for smoke runs.
# (--write-baseline already refused the stub path above.)
if model is None:
console.print(
"[yellow]No --model given; using stub generator "
"(empty output per prompt) for a smoke run.[/]"
)
def _stub_generate(_: str) -> str:
return ""
generate_fn = _stub_generate
else:
from soup_cli.eval.quant_check import make_model_generator
try:
generate_fn = make_model_generator(model)
except (OSError, ValueError, ImportError) as exc:
console.print(
f"[red]Failed to load --model '{model}':[/] {exc}"
)
raise typer.Exit(1) from exc
result = run_gate(
eval_suite, generate_fn=generate_fn, baseline=baseline_scores,
regression_threshold=regression_threshold,
)
_print_gate_result(result)
if write_baseline:
scores = {
row.name: float(row.score)
for row in result.task_results
if row.score is not None
}
try:
written = write_baseline_file(write_baseline, scores)
except (OSError, ValueError, TypeError) as exc:
console.print(
f"[red]Cannot write --write-baseline:[/] {exc}"
)
raise typer.Exit(EXIT_RUNTIME_ERROR) from exc
console.print(f"[green]Wrote stamped baseline[/] {written}")
raise typer.Exit(EXIT_OK if result.passed else EXIT_GATE_FAILED)
# ─── soup eval quant-check (v0.26.0 Part D) ───
@app.command(name="quant-check", cls=GateCommand)
def quant_check_cmd(
before: str = typer.Option(
..., "--before",
help="Before-quantization model path or registry://<id>",
),
after: str = typer.Option(
..., "--after",
help="After-quantization model path or registry://<id>",
),
tasks: str = typer.Option(
..., "--tasks",
help="JSONL eval tasks file (see 'soup eval custom')",
),
fmt: str = typer.Option(
"table", "--format",
help="Output format: table | json | markdown",
),
allow_stub: bool = typer.Option(
False, "--allow-stub",
help="Allow deterministic stub generators if live model loading fails.",
),
) -> None:
"""Compare accuracy before vs after quantization on the same eval suite.
Runs the same JSONL eval tasks through both models sequentially (memory
safe) and renders a per-task delta with OK / MINOR / MAJOR verdicts.
Exits with code 0 on OK / MINOR, code 2 on MAJOR, code 1 on model loading
or runtime errors, and code 3 on usage or input validation errors.
"""
from soup_cli.eval.quant_check import (
ensure_format,
is_under_cwd,
render,
resolve_model_ref,
run_quant_check,
stub_generator,
)
try:
ensure_format(fmt)
except ValueError as exc:
console.print(f"[red]{exc}[/]")
raise typer.Exit(EXIT_USAGE_ERROR) from exc
resolved: dict[str, Optional[str]] = {}
for label, ref in (("--before", before), ("--after", after)):
try:
resolved[label] = resolve_model_ref(ref)
except ValueError as exc: # AmbiguousRefError: a registry prefix or name:tag with twins
console.print(f"[red]Cannot resolve {label}:[/] {for_terminal(str(exc))}")
raise typer.Exit(EXIT_USAGE_ERROR) from exc
resolved_before, resolved_after = resolved["--before"], resolved["--after"]
if resolved_before is None:
console.print(f"[red]Cannot resolve --before: {before}[/]")
raise typer.Exit(EXIT_USAGE_ERROR)
if resolved_after is None:
console.print(f"[red]Cannot resolve --after: {after}[/]")
raise typer.Exit(EXIT_USAGE_ERROR)
for label, path_str in (("--before", resolved_before),
("--after", resolved_after),
("--tasks", tasks)):
path_obj = Path(path_str)
if not is_under_cwd(path_obj):
console.print(
f"[red]{label} '{path_str}' is outside cwd - refusing[/]"
)
raise typer.Exit(EXIT_USAGE_ERROR)
if not path_obj.exists() and label == "--tasks":
console.print(f"[red]{label} not found: {path_str}[/]")
raise typer.Exit(EXIT_USAGE_ERROR)
for label, path_str in (("--before", resolved_before), ("--after", resolved_after)):
path_obj = Path(path_str)
if path_obj.is_file() and path_obj.suffix.lower() == ".gguf":
console.print(
f"[red]{label} specifies a standalone GGUF file '{path_str}', "
"which is not supported for live eval quant-check.\n"
"Supported: directories containing safetensors / HuggingFace "
"model files or registry:// refs.[/]"
)
raise typer.Exit(EXIT_USAGE_ERROR)
before_path = Path(resolved_before)
after_path = Path(resolved_after)
if not before_path.exists():
console.print(f"[red]--before not found: {resolved_before}[/]")
raise typer.Exit(EXIT_USAGE_ERROR)
if not after_path.exists():
console.print(f"[red]--after not found: {resolved_after}[/]")
raise typer.Exit(EXIT_USAGE_ERROR)
# Live model scoring: build transformers-backed generators per side.
# Deterministic stubs are used only if --allow-stub is explicitly passed.
from soup_cli.eval.quant_check import make_model_generator
is_stub = False
try:
before_gen = make_model_generator(resolved_before)
except (OSError, ValueError, ImportError) as exc:
if not allow_stub:
console.print(
f"[red]Failed to load --before model ({resolved_before}): {exc} "
"(pass --allow-stub to score with deterministic stubs instead)[/]"
)
raise typer.Exit(EXIT_RUNTIME_ERROR) from exc
if fmt != "json":
console.print(
f"[yellow]Failed to load --before model ({exc}); using deterministic stub.[/]"
)
before_gen = stub_generator("before")
is_stub = True
try:
after_gen = make_model_generator(resolved_after)
except (OSError, ValueError, ImportError) as exc:
if not allow_stub:
console.print(
f"[red]Failed to load --after model ({resolved_after}): {exc} "
"(pass --allow-stub to score with deterministic stubs instead)[/]"
)
raise typer.Exit(EXIT_RUNTIME_ERROR) from exc
if fmt != "json":
console.print(
f"[yellow]Failed to load --after model ({exc}); using deterministic stub.[/]"
)
after_gen = stub_generator("after")
is_stub = True
result = run_quant_check(
before_gen=before_gen,
after_gen=after_gen,
tasks_file=tasks,
stub=is_stub,
)
rendered = render(result, fmt=fmt)
if fmt == "table":
console.print(rendered)
else:
# Plain text (markdown / json) — skip Rich markup interpretation so
# pipe chars in markdown don't render as Rich tags, and disable
# highlighting so JSON output is not polluted with ANSI escape codes.
console.print(rendered, markup=False, highlight=False)
has_major = any(r.verdict == "MAJOR" for r in result.rows)
raise typer.Exit(EXIT_GATE_FAILED if has_major else EXIT_OK)
def _print_gate_result(result) -> None:
"""Render a GateResult as a Rich table + pass/fail panel."""
table = Table(title="Eval gate")
table.add_column("Task", style="cyan")
table.add_column("Score", justify="right")
table.add_column("Threshold", justify="right")
table.add_column("Baseline", justify="right")
table.add_column("Delta", justify="right")
table.add_column("Verdict")
for row in result.task_results:
score_text = (
f"{row.score:.3f}" if row.score is not None
else "[red]ERROR[/]"
)
verdict = (
"[green]PASS[/]" if row.passed
else (f"[red]FAIL ({row.error})[/]" if row.error else "[red]FAIL[/]")
)
table.add_row(
row.name,
score_text,
f"{row.threshold:.3f}",
f"{row.baseline:.3f}" if row.baseline is not None else "-",
f"{row.delta:+.3f}" if row.delta is not None else "-",
verdict,
)
console.print(table)
verdict = "[green]GATE PASSED[/]" if result.passed else "[red]GATE FAILED[/]"
if result.regression:
verdict += " [yellow](regression vs baseline)[/]"
console.print(Panel(verdict, border_style="green" if result.passed else "red"))
# Register v0.55.0 subcommands (eval design / discover / lock / coverage / gate-install)
from soup_cli.commands._eval_v0550 import register as _register_v0550 # noqa: E402
_register_v0550(app, console)
# Register v0.61.0 subcommands (eval unlearning)
from soup_cli.commands._eval_v0610 import register as _register_v0610 # noqa: E402
_register_v0610(app, console)
# Register v0.65.0 subcommands (eval behavior / capability / checklist / irt-subset)
from soup_cli.commands._eval_v0650 import register as _register_v0650 # noqa: E402
_register_v0650(app, console)
# Register v0.71.10 subcommand (eval citation — #202)
from soup_cli.commands._eval_v07110 import register as _register_v07110 # noqa: E402
_register_v07110(app, console)