Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""soup bench -- measure inference speed (``infer``) or a training step (``train``)."""
import json
import time
from pathlib import Path
from typing import Optional
import typer
from rich.console import Console
from rich.panel import Panel
from rich.table import Table
from typer.core import TyperGroup
console = Console()
class _BenchGroup(TyperGroup):
"""``soup bench <model>`` predates the group, so anything that is not a
subcommand routes to ``infer`` -- a bare model path, or an option given
first (``soup bench --p50 ./output``). Only ``--help`` stays the group's."""
def parse_args(self, ctx, args):
if args and args[0] not in self.commands and args[0] not in ctx.help_option_names:
args = ["infer", *args]
return super().parse_args(ctx, args)
app = typer.Typer(
cls=_BenchGroup,
no_args_is_help=True,
help="Benchmark a model: inference speed (infer, the default) or training (train).",
)
@app.command(name="infer")
def bench(
model: str = typer.Argument(
...,
help="Path to model (LoRA adapter or full model) to benchmark",
),
base: Optional[str] = typer.Option(
None,
"--base",
"-b",
help="Base model for LoRA adapter (auto-detected if not set)",
),
max_tokens: int = typer.Option(
128,
"--max-tokens",
help="Maximum tokens to generate per prompt",
),
num_prompts: int = typer.Option(
3,
"--num-prompts",
"-n",
help="Number of prompts to run for averaging (ignored when --prompts-file is set)",
),
prompts_file: Optional[str] = typer.Option(
None,
"--prompts-file",
help="Path to custom prompts file (.txt or .jsonl)",
),
p50: bool = typer.Option(
False,
"--p50",
help="Print p50 (median) per-prompt latency. v0.53.9 #26.",
),
p95: bool = typer.Option(
False,
"--p95",
help="Print p95 per-prompt latency. v0.53.9 #26.",
),
backend: str = typer.Option(
"auto",
"--backend",
help=(
"Inference backend hint: auto (default) | transformers | mlx. "
"v0.53.9 #28."
),
),
cuda_graphs: bool = typer.Option(
False,
"--cuda-graphs",
help="Experimental CUDA graph decode for resident, unquantized Qwen2/Llama models.",
),
) -> None:
"""Run an inference benchmark (speed and memory) on a loaded model."""
from soup_cli.commands.infer import _generate, _load_model, _resolve_model_source
from soup_cli.utils.gpu import detect_device
# Resolve local-path-or-HF-id (#N7).
try:
model_kind, model_ref = _resolve_model_source(model)
except FileNotFoundError as exc:
console.print(
f"[red]{exc}[/]\n"
"[dim]If you meant a HuggingFace repo, use the form "
"'owner/repo-name' (no leading './').[/]"
)
raise typer.Exit(1) from exc
if model_kind == "hf":
console.print(
f"[dim]Local path not found; treating {model_ref!r} as a HF repo id.[/]"
)
model_target = model_ref
# v0.53.9 #28 — backend auto-detect.
from soup_cli.utils.backend_detect import SUPPORTED_BACKENDS, detect_backend
backend_lower = (backend or "auto").strip().lower()
if backend_lower == "auto":
backend_resolved = detect_backend(model_target)
console.print(
f"[dim]Backend auto-detected:[/] [bold]{backend_resolved}[/]"
)
else:
if backend_lower not in SUPPORTED_BACKENDS:
console.print(
f"[red]Unknown --backend:[/] {backend} "
f"(expected: auto | {' | '.join(sorted(SUPPORTED_BACKENDS))})"
)
raise typer.Exit(2)
backend_resolved = backend_lower
if cuda_graphs is True and backend_resolved != "transformers":
raise typer.BadParameter(
"--cuda-graphs requires --backend transformers. Omit --cuda-graphs to bench this "
"backend"
)
if cuda_graphs is True:
from soup_cli.commands.infer import _with_omit_hint
from soup_cli.utils.cuda_graphs import check_cuda_graphs_preflight
from soup_cli.utils.terminal import for_terminal
try:
check_cuda_graphs_preflight(
device=None,
model_target=model_ref if model_kind == "local" else None,
base=base,
)
except RuntimeError as exc:
console.print(f"[red]{for_terminal(_with_omit_hint(str(exc)))}[/]")
raise typer.Exit(1) from exc
device, _ = detect_device()
if device == "cpu":
console.print(
"[yellow]Warning:[/] Running on CPU. Inference speed is typically "
"10-100x slower than GPU -- results will not reflect production TPS."
)
if prompts_file:
import json
import os as _os
from soup_cli.utils.paths import is_under_cwd
if not is_under_cwd(prompts_file):
console.print(
"[red]Security Error:[/] Prompts file must stay under the "
"current working directory."
)
raise typer.Exit(1)
# v0.53.9 review fix M2 — reject symlinked prompts file on the
# RAW path BEFORE realpath resolution (mirrors v0.53.7 #106 policy).
import stat as _stat
try:
_st = _os.lstat(prompts_file)
except OSError:
console.print(f"[red]Prompts file not found:[/] {prompts_file}")
raise typer.Exit(1)
if _stat.S_ISLNK(_st.st_mode):
console.print(
"[red]Prompts file must not be a symlink.[/]"
)
raise typer.Exit(1)
p_path = Path(_os.path.realpath(prompts_file))
if not p_path.is_file():
console.print(f"[red]Prompts file not found:[/] {p_path}")
raise typer.Exit(1)
prompts = []
try:
if p_path.suffix == ".jsonl":
with open(p_path, encoding="utf-8-sig") as fh:
for line in fh:
line = line.strip()
if line:
data = json.loads(line)
if "prompt" in data:
prompts.append(data["prompt"])
else:
with open(p_path, encoding="utf-8-sig") as fh:
for line in fh:
line = line.strip()
if line:
prompts.append(line)
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
console.print(f"[red]Failed to read prompts file:[/] {exc}")
raise typer.Exit(1) from exc
if not prompts:
console.print("[red]No prompts found in file.[/]")
raise typer.Exit(1)
# When --prompts-file is provided, run all prompts in the file; --num-prompts is ignored.
else:
prompts = [
"Explain the theory of relativity briefly.",
"Write a short Python function to calculate fibonacci numbers.",
"What are the main consequences of the Industrial Revolution?",
"Compose a poem about a wandering space traveler.",
"Describe how a database index works under the hood.",
]
actual_num_prompts = len(prompts) if prompts_file else num_prompts
console.print(
Panel(
f"Model: [bold]{model_target}[/]\n"
f"Device: [bold]{device}[/]\n"
f"Prompts: [bold]{actual_num_prompts}[/]\n"
f"Tokens/P: [bold]{max_tokens}[/]",
title="Benchmarking Configuration",
)
)
console.print("[dim]Loading model to measure resource usage...[/]")
# Reset peak stats before load -- "Max VRAM" reflects total footprint
# (model load + inference), i.e. what users need for deployment planning.
import torch
if torch.cuda.is_available():
torch.cuda.reset_peak_memory_stats()
start_load = time.time()
try:
model_obj, tokenizer = _load_model(
model_target, base, device, is_local=(model_kind == "local")
)
except typer.Exit:
# typer.Exit subclasses RuntimeError, so the broad except below would
# swallow an already-reported CLI exit and mis-print "Failed to load
# model: 1". Let it propagate unchanged.
raise
except (OSError, ImportError, RuntimeError, ValueError) as exc:
console.print(f"[red]Failed to load model:[/] {exc}")
raise typer.Exit(1) from exc
if cuda_graphs is True:
from soup_cli.utils.cuda_graphs import cuda_graph_generation_kwargs
try:
cuda_graph_generation_kwargs(model_obj)
except RuntimeError as exc:
from soup_cli.commands.infer import _with_omit_hint
from soup_cli.utils.terminal import for_terminal
console.print(f"[red]{for_terminal(_with_omit_hint(str(exc)))}[/]")
raise typer.Exit(1) from exc
load_time = time.time() - start_load
console.print(f"[green]Model loaded in {load_time:.2f}s.[/]\n")
test_prompts = (prompts * (actual_num_prompts // len(prompts) + 1))[:actual_num_prompts]
# Warmup run: first inference includes CUDA kernel JIT compilation,
# which would skew the average. Discarded from timing.
console.print("[dim]Warmup run (discarded from timing)...[/]")
if cuda_graphs is True:
from soup_cli.commands.infer import _warm_cuda_graphs
# Capture on the longest prompt at the timed length, so no timed request
# changes the static cache's shape and recompiles inside the measurement.
_warm_cuda_graphs(model_obj, tokenizer, list(dict.fromkeys(test_prompts)), max_tokens)
else:
warmup_messages = [{"role": "user", "content": test_prompts[0]}]
_generate(
model_obj, tokenizer, warmup_messages,
max_tokens=min(max_tokens, 32), temperature=0.0,
)
total_tokens = 0
total_latency = 0.0
per_prompt_latencies: list[float] = []
console.print(f"[bold]Running {len(test_prompts)} test inferences...[/]")
for i, prompt_text in enumerate(test_prompts):
messages = [{"role": "user", "content": prompt_text}]
start_time = time.time()
try:
_, token_count = _generate(
model_obj, tokenizer, messages,
max_tokens=max_tokens, temperature=0.0,
**({"cuda_graphs": True} if cuda_graphs is True else {}),
)
except Exception as exc:
if cuda_graphs is not True:
raise
from soup_cli.commands.infer import _cuda_graph_failure
raise _cuda_graph_failure(exc) from exc
latency = time.time() - start_time
total_tokens += token_count
total_latency += latency
per_prompt_latencies.append(latency)
console.print(f" [dim]Prompt {i + 1}: {token_count} tokens in {latency:.2f}s[/]")
avg_tps = total_tokens / total_latency if total_latency > 0 else 0
peak_vram_gb = 0.0
if torch.cuda.is_available():
peak_vram_gb = torch.cuda.max_memory_allocated() / (1024**3)
table = Table(title="Inference Benchmark Results")
table.add_column("Backend", style="cyan")
table.add_column("TPS (Avg)", style="green", justify="right")
table.add_column("Latency (Total)", style="yellow", justify="right")
table.add_column("Max VRAM", style="magenta", justify="right")
vram_str = f"{peak_vram_gb:.2f} GB" if torch.cuda.is_available() else "N/A"
table.add_row(
backend_resolved.capitalize(),
f"{avg_tps:.2f}",
f"{total_latency:.2f}s",
vram_str,
)
console.print()
console.print(table)
# v0.53.9 #26 — tail-latency percentiles.
if (p50 or p95) and per_prompt_latencies:
from soup_cli.utils.tail_latency import summarise_latency
summary = summarise_latency(per_prompt_latencies)
pct_table = Table(title="Per-prompt latency")
pct_table.add_column("Statistic", style="cyan")
pct_table.add_column("Latency (s)", style="green", justify="right")
pct_table.add_row("count", str(summary.count))
if summary.mean is not None:
pct_table.add_row("mean", f"{summary.mean:.3f}")
if p50 and summary.p50 is not None:
pct_table.add_row("p50", f"{summary.p50:.3f}")
if p95 and summary.p95 is not None:
pct_table.add_row("p95", f"{summary.p95:.3f}")
# p99 is a natural superset of p95 — only print when p95 was requested.
if p95 and summary.p99 is not None:
pct_table.add_row("p99", f"{summary.p99:.3f}")
console.print()
console.print(pct_table)
@app.command(name="train")
def train(
config: str = typer.Option("soup.yaml", "--config", "-c", help="Config to train"),
steps: int = typer.Option(20, "--steps", min=1, help="Optimizer steps to run"),
warmup: int = typer.Option(
3, "--warmup", min=0, help="Leading steps measured but left out of the timing"
),
output: str = typer.Option(
"bench-train.json", "--output", "-o", help="Where to write the JSON report"
),
) -> None:
"""Train for a fixed number of steps and report timing, memory and tokens --
failing when the model was not actually training."""
# The contract and why each check exists: #836.
from rich.markup import escape
from soup_cli.utils.paths import atomic_write_text, check_report_output_path
from soup_cli.utils.terminal import for_terminal
# The destination is validated first, before the config is read and before any
# model is built: a path that will be refused must fail in milliseconds, not after
# the whole run has finished and its measurements are about to be lost.
try:
check_report_output_path(output, "--output")
except (TypeError, ValueError) as exc:
console.print(f"[red]Invalid output:[/] {for_terminal(exc)}")
raise typer.Exit(2) from exc
from soup_cli.bench.train_run import check_steps_warmup
err = check_steps_warmup(steps, warmup)
if err is not None:
console.print(f"[red]{err}[/]")
raise typer.Exit(1)
from soup_cli.bench.train_run import bench_scope_error, run_bench_train
from soup_cli.config.loader import load_config
from soup_cli.data.loader import load_dataset
from soup_cli.utils.gpu import detect_device
if not Path(config).is_file():
console.print(f"[red]Config not found:[/] {escape(config)}")
raise typer.Exit(1)
try:
cfg = load_config(Path(config))
except Exception as exc: # noqa: BLE001 -- surfaced to the user as-is
console.print(f"[red]Config invalid:[/] {escape(str(exc))}")
raise typer.Exit(1) from exc
refused = bench_scope_error(cfg)
if refused:
console.print(f"[red]Cannot bench this config:[/] {escape(refused)}")
raise typer.Exit(1)
device, _ = detect_device(backend=cfg.backend)
try:
report = run_bench_train(
cfg, steps=steps, warmup=warmup, device=device,
load_dataset=lambda c: load_dataset(c.data),
)
except ValueError as exc:
console.print(f"[red]{escape(str(exc))}[/]")
raise typer.Exit(1) from exc
try:
atomic_write_text(
json.dumps(report, indent=2, default=str),
output,
prefix=".bench.",
suffix=".json.tmp",
field="--output",
)
except (OSError, ValueError) as exc:
console.print(f"[red]Cannot write the report:[/] {for_terminal(exc)}")
raise typer.Exit(1) from exc
timing, tokens = report["timing"], report["tokens"]
tok_s = report["throughput"]["useful_tokens_per_second"]
console.print(
f"{report['steps_measured']} steps ({timing['warmup_steps_discarded']} warm-up "
f"discarded): median {timing['median_seconds']:.4f}s, "
f"p95 {timing['p95_seconds']:.4f}s, "
f"{tokens['useful']} supervised of {tokens['total']} tokens"
+ (f", {tok_s:.1f} supervised tok/s" if tok_s else "")
)
console.print(f"[dim]Report:[/] {escape(output)}")
if not report["valid"]:
for failure in report["failures"]:
console.print(f"[red]FAILED {failure['check']}:[/] {escape(failure['message'])}")
console.print("[red]The report is written but NOT valid: this run was not training.[/]")
raise typer.Exit(1)
console.print("[green]Valid:[/] the model trained while it was measured.")