Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

16.4 KB
Raw Blame
"""soup bench -- measure inference speed (``infer``) or a training step (``train``)."""

import json
import time
from pathlib import Path
from typing import Optional

import typer
from rich.console import Console
from rich.panel import Panel
from rich.table import Table
from typer.core import TyperGroup

console = Console()


class _BenchGroup(TyperGroup):
    """``soup bench <model>`` predates the group, so anything that is not a
    subcommand routes to ``infer`` -- a bare model path, or an option given
    first (``soup bench --p50 ./output``). Only ``--help`` stays the group's."""

    def parse_args(self, ctx, args):
        if args and args[0] not in self.commands and args[0] not in ctx.help_option_names:
            args = ["infer", *args]
        return super().parse_args(ctx, args)


app = typer.Typer(
    cls=_BenchGroup,
    no_args_is_help=True,
    help="Benchmark a model: inference speed (infer, the default) or training (train).",
)


@app.command(name="infer")
def bench(
    model: str = typer.Argument(
        ...,
        help="Path to model (LoRA adapter or full model) to benchmark",
    ),
    base: Optional[str] = typer.Option(
        None,
        "--base",
        "-b",
        help="Base model for LoRA adapter (auto-detected if not set)",
    ),
    max_tokens: int = typer.Option(
        128,
        "--max-tokens",
        help="Maximum tokens to generate per prompt",
    ),
    num_prompts: int = typer.Option(
        3,
        "--num-prompts",
        "-n",
        help="Number of prompts to run for averaging (ignored when --prompts-file is set)",
    ),
    prompts_file: Optional[str] = typer.Option(
        None,
        "--prompts-file",
        help="Path to custom prompts file (.txt or .jsonl)",
    ),
    p50: bool = typer.Option(
        False,
        "--p50",
        help="Print p50 (median) per-prompt latency. v0.53.9 #26.",
    ),
    p95: bool = typer.Option(
        False,
        "--p95",
        help="Print p95 per-prompt latency. v0.53.9 #26.",
    ),
    backend: str = typer.Option(
        "auto",
        "--backend",
        help=(
            "Inference backend hint: auto (default) | transformers | mlx. "
            "v0.53.9 #28."
        ),
    ),
    cuda_graphs: bool = typer.Option(
        False,
        "--cuda-graphs",
        help="Experimental CUDA graph decode for resident, unquantized Qwen2/Llama models.",
    ),
) -> None:
    """Run an inference benchmark (speed and memory) on a loaded model."""
    from soup_cli.commands.infer import _generate, _load_model, _resolve_model_source
    from soup_cli.utils.gpu import detect_device

    # Resolve local-path-or-HF-id (#N7).
    try:
        model_kind, model_ref = _resolve_model_source(model)
    except FileNotFoundError as exc:
        console.print(
            f"[red]{exc}[/]\n"
            "[dim]If you meant a HuggingFace repo, use the form "
            "'owner/repo-name' (no leading './').[/]"
        )
        raise typer.Exit(1) from exc
    if model_kind == "hf":
        console.print(
            f"[dim]Local path not found; treating {model_ref!r} as a HF repo id.[/]"
        )
    model_target = model_ref

    # v0.53.9 #28 — backend auto-detect.
    from soup_cli.utils.backend_detect import SUPPORTED_BACKENDS, detect_backend

    backend_lower = (backend or "auto").strip().lower()
    if backend_lower == "auto":
        backend_resolved = detect_backend(model_target)
        console.print(
            f"[dim]Backend auto-detected:[/] [bold]{backend_resolved}[/]"
        )
    else:
        if backend_lower not in SUPPORTED_BACKENDS:
            console.print(
                f"[red]Unknown --backend:[/] {backend} "
                f"(expected: auto | {' | '.join(sorted(SUPPORTED_BACKENDS))})"
            )
            raise typer.Exit(2)
        backend_resolved = backend_lower

    if cuda_graphs is True and backend_resolved != "transformers":
        raise typer.BadParameter(
            "--cuda-graphs requires --backend transformers. Omit --cuda-graphs to bench this "
            "backend"
        )

    if cuda_graphs is True:
        from soup_cli.commands.infer import _with_omit_hint
        from soup_cli.utils.cuda_graphs import check_cuda_graphs_preflight
        from soup_cli.utils.terminal import for_terminal

        try:
            check_cuda_graphs_preflight(
                device=None,
                model_target=model_ref if model_kind == "local" else None,
                base=base,
            )
        except RuntimeError as exc:
            console.print(f"[red]{for_terminal(_with_omit_hint(str(exc)))}[/]")
            raise typer.Exit(1) from exc

    device, _ = detect_device()

    if device == "cpu":
        console.print(
            "[yellow]Warning:[/] Running on CPU. Inference speed is typically "
            "10-100x slower than GPU -- results will not reflect production TPS."
        )

    if prompts_file:
        import json
        import os as _os

        from soup_cli.utils.paths import is_under_cwd

        if not is_under_cwd(prompts_file):
            console.print(
                "[red]Security Error:[/] Prompts file must stay under the "
                "current working directory."
            )
            raise typer.Exit(1)
        # v0.53.9 review fix M2 — reject symlinked prompts file on the
        # RAW path BEFORE realpath resolution (mirrors v0.53.7 #106 policy).
        import stat as _stat

        try:
            _st = _os.lstat(prompts_file)
        except OSError:
            console.print(f"[red]Prompts file not found:[/] {prompts_file}")
            raise typer.Exit(1)
        if _stat.S_ISLNK(_st.st_mode):
            console.print(
                "[red]Prompts file must not be a symlink.[/]"
            )
            raise typer.Exit(1)
        p_path = Path(_os.path.realpath(prompts_file))

        if not p_path.is_file():
            console.print(f"[red]Prompts file not found:[/] {p_path}")
            raise typer.Exit(1)

        prompts = []
        try:
            if p_path.suffix == ".jsonl":
                with open(p_path, encoding="utf-8-sig") as fh:
                    for line in fh:
                        line = line.strip()
                        if line:
                            data = json.loads(line)
                            if "prompt" in data:
                                prompts.append(data["prompt"])
            else:
                with open(p_path, encoding="utf-8-sig") as fh:
                    for line in fh:
                        line = line.strip()
                        if line:
                            prompts.append(line)
        except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
            console.print(f"[red]Failed to read prompts file:[/] {exc}")
            raise typer.Exit(1) from exc

        if not prompts:
            console.print("[red]No prompts found in file.[/]")
            raise typer.Exit(1)

        # When --prompts-file is provided, run all prompts in the file; --num-prompts is ignored.
    else:
        prompts = [
            "Explain the theory of relativity briefly.",
            "Write a short Python function to calculate fibonacci numbers.",
            "What are the main consequences of the Industrial Revolution?",
            "Compose a poem about a wandering space traveler.",
            "Describe how a database index works under the hood.",
        ]

    actual_num_prompts = len(prompts) if prompts_file else num_prompts

    console.print(
        Panel(
            f"Model:    [bold]{model_target}[/]\n"
            f"Device:   [bold]{device}[/]\n"
            f"Prompts:  [bold]{actual_num_prompts}[/]\n"
            f"Tokens/P: [bold]{max_tokens}[/]",
            title="Benchmarking Configuration",
        )
    )

    console.print("[dim]Loading model to measure resource usage...[/]")

    # Reset peak stats before load -- "Max VRAM" reflects total footprint
    # (model load + inference), i.e. what users need for deployment planning.
    import torch

    if torch.cuda.is_available():
        torch.cuda.reset_peak_memory_stats()

    start_load = time.time()
    try:
        model_obj, tokenizer = _load_model(
            model_target, base, device, is_local=(model_kind == "local")
        )
    except typer.Exit:
        # typer.Exit subclasses RuntimeError, so the broad except below would
        # swallow an already-reported CLI exit and mis-print "Failed to load
        # model: 1". Let it propagate unchanged.
        raise
    except (OSError, ImportError, RuntimeError, ValueError) as exc:
        console.print(f"[red]Failed to load model:[/] {exc}")
        raise typer.Exit(1) from exc

    if cuda_graphs is True:
        from soup_cli.utils.cuda_graphs import cuda_graph_generation_kwargs

        try:
            cuda_graph_generation_kwargs(model_obj)
        except RuntimeError as exc:
            from soup_cli.commands.infer import _with_omit_hint
            from soup_cli.utils.terminal import for_terminal

            console.print(f"[red]{for_terminal(_with_omit_hint(str(exc)))}[/]")
            raise typer.Exit(1) from exc

    load_time = time.time() - start_load
    console.print(f"[green]Model loaded in {load_time:.2f}s.[/]\n")

    test_prompts = (prompts * (actual_num_prompts // len(prompts) + 1))[:actual_num_prompts]

    # Warmup run: first inference includes CUDA kernel JIT compilation,
    # which would skew the average. Discarded from timing.
    console.print("[dim]Warmup run (discarded from timing)...[/]")
    if cuda_graphs is True:
        from soup_cli.commands.infer import _warm_cuda_graphs

        # Capture on the longest prompt at the timed length, so no timed request
        # changes the static cache's shape and recompiles inside the measurement.
        _warm_cuda_graphs(model_obj, tokenizer, list(dict.fromkeys(test_prompts)), max_tokens)
    else:
        warmup_messages = [{"role": "user", "content": test_prompts[0]}]
        _generate(
            model_obj, tokenizer, warmup_messages,
            max_tokens=min(max_tokens, 32), temperature=0.0,
        )

    total_tokens = 0
    total_latency = 0.0
    per_prompt_latencies: list[float] = []

    console.print(f"[bold]Running {len(test_prompts)} test inferences...[/]")

    for i, prompt_text in enumerate(test_prompts):
        messages = [{"role": "user", "content": prompt_text}]
        start_time = time.time()

        try:
            _, token_count = _generate(
                model_obj, tokenizer, messages,
                max_tokens=max_tokens, temperature=0.0,
                **({"cuda_graphs": True} if cuda_graphs is True else {}),
            )
        except Exception as exc:
            if cuda_graphs is not True:
                raise
            from soup_cli.commands.infer import _cuda_graph_failure

            raise _cuda_graph_failure(exc) from exc

        latency = time.time() - start_time
        total_tokens += token_count
        total_latency += latency
        per_prompt_latencies.append(latency)
        console.print(f"  [dim]Prompt {i + 1}: {token_count} tokens in {latency:.2f}s[/]")

    avg_tps = total_tokens / total_latency if total_latency > 0 else 0

    peak_vram_gb = 0.0
    if torch.cuda.is_available():
        peak_vram_gb = torch.cuda.max_memory_allocated() / (1024**3)

    table = Table(title="Inference Benchmark Results")
    table.add_column("Backend", style="cyan")
    table.add_column("TPS (Avg)", style="green", justify="right")
    table.add_column("Latency (Total)", style="yellow", justify="right")
    table.add_column("Max VRAM", style="magenta", justify="right")

    vram_str = f"{peak_vram_gb:.2f} GB" if torch.cuda.is_available() else "N/A"
    table.add_row(
        backend_resolved.capitalize(),
        f"{avg_tps:.2f}",
        f"{total_latency:.2f}s",
        vram_str,
    )

    console.print()
    console.print(table)

    # v0.53.9 #26 — tail-latency percentiles.
    if (p50 or p95) and per_prompt_latencies:
        from soup_cli.utils.tail_latency import summarise_latency

        summary = summarise_latency(per_prompt_latencies)
        pct_table = Table(title="Per-prompt latency")
        pct_table.add_column("Statistic", style="cyan")
        pct_table.add_column("Latency (s)", style="green", justify="right")
        pct_table.add_row("count", str(summary.count))
        if summary.mean is not None:
            pct_table.add_row("mean", f"{summary.mean:.3f}")
        if p50 and summary.p50 is not None:
            pct_table.add_row("p50", f"{summary.p50:.3f}")
        if p95 and summary.p95 is not None:
            pct_table.add_row("p95", f"{summary.p95:.3f}")
        # p99 is a natural superset of p95 — only print when p95 was requested.
        if p95 and summary.p99 is not None:
            pct_table.add_row("p99", f"{summary.p99:.3f}")
        console.print()
        console.print(pct_table)


@app.command(name="train")
def train(
    config: str = typer.Option("soup.yaml", "--config", "-c", help="Config to train"),
    steps: int = typer.Option(20, "--steps", min=1, help="Optimizer steps to run"),
    warmup: int = typer.Option(
        3, "--warmup", min=0, help="Leading steps measured but left out of the timing"
    ),
    output: str = typer.Option(
        "bench-train.json", "--output", "-o", help="Where to write the JSON report"
    ),
) -> None:
    """Train for a fixed number of steps and report timing, memory and tokens --
    failing when the model was not actually training."""
    # The contract and why each check exists: #836.
    from rich.markup import escape

    from soup_cli.utils.paths import atomic_write_text, check_report_output_path
    from soup_cli.utils.terminal import for_terminal

    # The destination is validated first, before the config is read and before any
    # model is built: a path that will be refused must fail in milliseconds, not after
    # the whole run has finished and its measurements are about to be lost.
    try:
        check_report_output_path(output, "--output")
    except (TypeError, ValueError) as exc:
        console.print(f"[red]Invalid output:[/] {for_terminal(exc)}")
        raise typer.Exit(2) from exc

    from soup_cli.bench.train_run import check_steps_warmup

    err = check_steps_warmup(steps, warmup)
    if err is not None:
        console.print(f"[red]{err}[/]")
        raise typer.Exit(1)

    from soup_cli.bench.train_run import bench_scope_error, run_bench_train
    from soup_cli.config.loader import load_config
    from soup_cli.data.loader import load_dataset
    from soup_cli.utils.gpu import detect_device

    if not Path(config).is_file():
        console.print(f"[red]Config not found:[/] {escape(config)}")
        raise typer.Exit(1)
    try:
        cfg = load_config(Path(config))
    except Exception as exc:  # noqa: BLE001 -- surfaced to the user as-is
        console.print(f"[red]Config invalid:[/] {escape(str(exc))}")
        raise typer.Exit(1) from exc
    refused = bench_scope_error(cfg)
    if refused:
        console.print(f"[red]Cannot bench this config:[/] {escape(refused)}")
        raise typer.Exit(1)

    device, _ = detect_device(backend=cfg.backend)
    try:
        report = run_bench_train(
            cfg, steps=steps, warmup=warmup, device=device,
            load_dataset=lambda c: load_dataset(c.data),
        )
    except ValueError as exc:
        console.print(f"[red]{escape(str(exc))}[/]")
        raise typer.Exit(1) from exc

    try:
        atomic_write_text(
            json.dumps(report, indent=2, default=str),
            output,
            prefix=".bench.",
            suffix=".json.tmp",
            field="--output",
        )
    except (OSError, ValueError) as exc:
        console.print(f"[red]Cannot write the report:[/] {for_terminal(exc)}")
        raise typer.Exit(1) from exc
    timing, tokens = report["timing"], report["tokens"]
    tok_s = report["throughput"]["useful_tokens_per_second"]
    console.print(
        f"{report['steps_measured']} steps ({timing['warmup_steps_discarded']} warm-up "
        f"discarded): median {timing['median_seconds']:.4f}s, "
        f"p95 {timing['p95_seconds']:.4f}s, "
        f"{tokens['useful']} supervised of {tokens['total']} tokens"
        + (f", {tok_s:.1f} supervised tok/s" if tok_s else "")
    )
    console.print(f"[dim]Report:[/] {escape(output)}")
    if not report["valid"]:
        for failure in report["failures"]:
            console.print(f"[red]FAILED {failure['check']}:[/] {escape(failure['message'])}")
        console.print("[red]The report is written but NOT valid: this run was not training.[/]")
        raise typer.Exit(1)
    console.print("[green]Valid:[/] the model trained while it was measured.")