Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""soup infer — batch inference on a list of prompts."""
from __future__ import annotations
import json
import time
from pathlib import Path
from typing import Callable, Optional
import typer
from rich.console import Console
from rich.panel import Panel
from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
from soup_cli.utils.terminal import for_terminal
console = Console()
def _is_path_like(value: str) -> bool:
"""Heuristic: looks like a filesystem path rather than a HF repo id.
HF repo ids are ``owner/name`` with no leading dot/slash and no Windows
drive letter; anything else (``./foo``, ``/abs/path``, ``C:\\...``) is
treated as a path so we surface a meaningful FileNotFoundError instead
of attempting an HF download.
"""
if not value:
return True
if value.startswith((".", "/", "\\", "~")):
return True
# Windows drive letter, e.g. "C:\..." or "C:/..."
if len(value) >= 2 and value[1] == ":":
return True
return False
def _resolve_model_source(model: str) -> tuple[str, str]:
"""Return ``("local", path)`` or ``("hf", repo_id)`` for ``--model``.
Falls through to HF when the local path doesn't exist *and* the value
looks like a HF repo id (no leading ``./`` etc.). Raises
:class:`FileNotFoundError` when the value looks like a path but doesn't
exist locally — distinguishes "your file is missing" from "your HF id
is wrong" so the error message is actionable.
"""
candidate = Path(model)
if candidate.exists():
return "local", str(candidate)
if _is_path_like(model):
raise FileNotFoundError(f"Model path not found: {model}")
# Looks like a HF repo id — let transformers handle the download.
return "hf", model
def infer(
model: str = typer.Option(
...,
"--model",
"-m",
help="Path to model (LoRA adapter or full model)",
),
input_file: str = typer.Option(
...,
"--input",
"-i",
help="Path to input JSONL file (each line: {\"prompt\": \"...\"})",
),
output_file: str = typer.Option(
...,
"--output",
"-o",
help="Path to output JSONL file for results",
),
base: Optional[str] = typer.Option(
None,
"--base",
"-b",
help="Base model for LoRA adapter (auto-detected if not set)",
),
max_tokens: int = typer.Option(
256,
"--max-tokens",
min=1,
max=16384,
help="Maximum tokens to generate per response (1-16384)",
),
temperature: float = typer.Option(
0.7,
"--temperature",
"-t",
help="Sampling temperature (0 = greedy)",
),
batch_size: int = typer.Option(
1,
"--batch-size",
min=1,
help=(
"Number of causal-LM prompts to generate together "
"(default: 1; batching is opt-in; fp16/bf16 batching "
"may not be bit-exact with batch size 1)"
),
),
device: Optional[str] = typer.Option(
None,
"--device",
help="Device: cuda, mps, cpu. Auto-detected if not set.",
),
task: str = typer.Option(
"text",
"--task",
help=(
"Inference task: 'text' (default, chat generation) or 'asr' "
"(Whisper transcription; input rows are {\"audio\": path[, "
"\"text\": reference]})."
),
),
asr_language: Optional[str] = typer.Option(
None,
"--asr-language",
help="ASR decode language (overrides the training sidecar). --task asr only.",
),
asr_task: Optional[str] = typer.Option(
None,
"--asr-task",
help="ASR decode objective: transcribe | translate. --task asr only.",
),
audio_dir: Optional[str] = typer.Option(
None,
"--audio-dir",
help=(
"Base directory audio paths in --input must stay under (--task asr; "
"defaults to the --input file's directory). Traversal / UNC paths "
"are rejected."
),
),
trust_remote_code: bool = typer.Option(
False,
"--trust-remote-code",
help=(
"Allow loading models that ship custom Python via auto_map. "
"Default deny (v0.36.0). Only enable if you trust the source."
),
),
hub: str = typer.Option(
"hf",
"--hub",
help=(
"Source hub for the base model: hf (default) / modelscope / "
"modelers. Non-HF hubs require the matching SDK (v0.53.10 #152)."
),
),
cuda_graphs: bool = typer.Option(
False,
"--cuda-graphs",
help="Experimental CUDA graph decode for resident, unquantized Qwen2/Llama models.",
),
):
"""Run batch inference on a JSONL file of prompts."""
# v0.53.10 #152 — pre-fetch base from a non-HF hub before any resolution.
if hub and hub != "hf":
from soup_cli.utils.hubs import apply_hub_to_cli_model
try:
model, base = apply_hub_to_cli_model(model, base, hub, console=console)
except (TypeError, ValueError) as exc:
console.print(f"[red]{exc}[/]")
raise typer.Exit(code=2) from exc
except ImportError as exc:
console.print(f"[red]{exc}[/]")
raise typer.Exit(code=1) from exc
from soup_cli.utils.paths import is_under_cwd
# Validate input file
input_path = Path(input_file)
if not input_path.exists():
console.print(f"[red]Input file not found: {input_path}[/]")
raise typer.Exit(1)
# v0.71.32 — ASR (Whisper) transcription branch. Diverts before the chat
# model-resolution path; _infer_asr owns its own Whisper load + output.
if task == "asr":
if cuda_graphs is True:
raise typer.BadParameter(
"--cuda-graphs supports text generation only. Omit --cuda-graphs for --task asr"
)
# Validate --asr-task up front: a typo would otherwise be passed to
# whisper.generate(task=...) and fail INSIDE every row (100k confusing
# per-row skips instead of one upfront rejection).
if asr_task is not None and asr_task not in ("transcribe", "translate"):
console.print(
f"[red]--asr-task must be 'transcribe' or 'translate', "
f"got {asr_task!r}.[/]"
)
raise typer.Exit(2)
_infer_asr(
model=model,
base=base,
input_file=input_file,
device=device,
output_file=output_file,
max_tokens=max_tokens,
trust_remote_code=trust_remote_code,
asr_language=asr_language,
asr_task=asr_task,
audio_dir=audio_dir,
)
return
if cuda_graphs is True and batch_size > 1:
raise typer.BadParameter(
"--cuda-graphs records a batch of 1 and cannot be combined with "
"--batch-size above 1. Omit --cuda-graphs, or use --batch-size 1"
)
if task != "text":
console.print(f"[red]Unknown --task {task!r}; expected 'text' or 'asr'.[/]")
raise typer.Exit(2)
# Resolve model: local path or HF repo id (auto-fallback, #N7).
try:
model_kind, model_ref = _resolve_model_source(model)
except FileNotFoundError as exc:
console.print(
f"[red]{exc}[/]\n"
"[dim]If you meant a HuggingFace repo, use the form "
"'owner/repo-name' (no leading './').[/]"
)
raise typer.Exit(1) from exc
if model_kind == "hf":
console.print(
f"[dim]Local path not found; treating {model_ref!r} as a HF repo id.[/]"
)
# Read prompts
prompts = _read_prompts(input_path)
if not prompts:
console.print("[red]No prompts found in input file.[/]")
console.print("[dim]Expected JSONL with {\"prompt\": \"...\"} or plain text lines.[/]")
raise typer.Exit(1)
if cuda_graphs is True:
from soup_cli.utils.cuda_graphs import check_cuda_graphs_preflight
try:
check_cuda_graphs_preflight(
device=device,
model_target=model_ref if model_kind == "local" else None,
base=base,
)
except RuntimeError as exc:
console.print(f"[red]{for_terminal(_with_omit_hint(str(exc)))}[/]")
raise typer.Exit(1) from exc
# Detect device
if not device:
from soup_cli.utils.gpu import detect_device
device, _ = detect_device()
console.print(
Panel(
f"Model: [bold]{model_ref}[/]\n"
f"Input: [bold]{input_path}[/] ({len(prompts)} prompts)\n"
f"Output: [bold]{output_file}[/]\n"
f"Device: [bold]{device}[/]\n"
f"Tokens: [bold]{max_tokens}[/]\n"
f"Temp: [bold]{temperature}[/]",
title="Batch Inference",
)
)
# Load model — gate trust_remote_code via the v0.36.0 helper.
console.print("[dim]Loading model...[/]")
model_obj, tokenizer = _load_model(
model_ref,
base,
device,
trust_remote_code,
is_local=(model_kind == "local"),
)
if cuda_graphs is True:
from soup_cli.utils.cuda_graphs import cuda_graph_generation_kwargs
try:
cuda_graph_generation_kwargs(model_obj)
except RuntimeError as exc:
console.print(f"[red]{for_terminal(_with_omit_hint(str(exc)))}[/]")
raise typer.Exit(1) from exc
console.print("[green]Model loaded.[/]\n")
# Output path containment — defence-in-depth (project policy v0.20.0+).
# Checked late, after model+inputs validate, so that pre-existing tests
# asserting on "model not found" / "no prompts" errors keep working when
# they pass an out-of-cwd `tmp_path`.
if not is_under_cwd(output_file):
console.print(
"[red]--output must stay under the current working directory.[/]"
)
raise typer.Exit(1)
if cuda_graphs is True:
_warm_cuda_graphs(model_obj, tokenizer, prompts, max_tokens)
# Run inference — stream results to disk as they are generated
output_path = Path(output_file)
total_tokens = 0
num_results = 0
start_time = time.time()
with (
open(output_path, "w", encoding="utf-8") as out_f,
Progress(
SpinnerColumn(),
TextColumn("[progress.description]{task.description}"),
BarColumn(),
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
TimeElapsedColumn(),
console=console,
) as progress,
):
progress_task = progress.add_task("Generating...", total=len(prompts))
if batch_size == 1:
prompt_batches = [[prompt] for prompt in prompts]
else:
prompt_batches = [
prompts[start : start + batch_size]
for start in range(0, len(prompts), batch_size)
]
for prompt_batch in prompt_batches:
if batch_size == 1:
messages = [{"role": "user", "content": prompt_batch[0]}]
try:
generated = [
_generate(
model_obj,
tokenizer,
messages,
max_tokens=max_tokens,
temperature=temperature,
**({"cuda_graphs": True} if cuda_graphs is True else {}),
)
]
except Exception as exc:
if cuda_graphs is not True:
raise
raise _cuda_graph_failure(exc) from exc
else:
generated = _generate_batch(
model_obj,
tokenizer,
prompt_batch,
max_tokens=max_tokens,
temperature=temperature,
)
for prompt_text, (response, token_count) in zip(
prompt_batch, generated, strict=True
):
result = {
"prompt": prompt_text,
"response": response,
"tokens_generated": token_count,
}
out_f.write(json.dumps(result, ensure_ascii=False) + "\n")
out_f.flush()
total_tokens += token_count
num_results += 1
progress.update(progress_task, advance=1)
elapsed = time.time() - start_time
tokens_per_sec = total_tokens / elapsed if elapsed > 0 else 0
console.print(
Panel(
f"Prompts: [bold]{num_results}[/]\n"
f"Total tokens: [bold]{total_tokens}[/]\n"
f"Duration: [bold]{elapsed:.1f}s[/]\n"
f"Throughput: [bold]{tokens_per_sec:.1f} tok/s[/]\n"
f"Output: [bold]{output_path}[/]",
title="[bold green]Inference Complete![/]",
)
)
# v0.71.32 — test seam: when set to ``callable(audio_path) -> str`` it replaces
# the real Whisper transcriber, so the ASR path is unit-testable without a
# model download.
_ASR_TRANSCRIBER_OVERRIDE = None
# Cap on ASR batch rows — each row is an audio decode + a full generate(), far
# costlier than a text prompt, so an unbounded --input is a resource-exhaustion
# vector (mirrors the project's 10k custom-eval / 1e6 HF-download caps).
_MAX_ASR_ROWS: int = 100_000
def _read_asr_rows(path: Path) -> list[dict]:
"""Read ASR rows ``{"audio": path[, "text": reference]}`` from JSONL.
Rows without a non-empty string ``audio`` are dropped (a warning is
printed once). ``text``, when present, is the reference transcript used for
WER reporting. Capped at ``_MAX_ASR_ROWS``.
"""
rows: list[dict] = []
dropped = 0
capped = False
with open(path, encoding="utf-8") as f:
for raw_line in f:
line = raw_line.strip()
if not line:
continue
try:
obj = json.loads(line)
except json.JSONDecodeError:
dropped += 1
continue
audio = obj.get("audio") if isinstance(obj, dict) else None
if not isinstance(audio, str) or not audio.strip():
dropped += 1
continue
rows.append(obj)
if len(rows) >= _MAX_ASR_ROWS:
capped = True
break
if dropped:
console.print(
f"[yellow]Skipped {dropped} row(s) with no 'audio' path.[/]"
)
if capped:
console.print(
f"[yellow]Capped at {_MAX_ASR_ROWS} rows; remaining input ignored.[/]"
)
return rows
def _resolve_asr_audio(audio: str, base_dir: Path) -> str:
"""Resolve a row's audio path against ``base_dir`` with containment.
Rejects UNC / network / device paths (the same check the training loader
runs) and anything that resolves outside
``base_dir`` (realpath + commonpath) — the infer path is fed JSONL the
operator may not have authored (the training path already enforces this
via ``_validate_audio_files``). Raises ``ValueError`` on rejection.
"""
from soup_cli.utils.paths import is_network_or_device_path, is_under
if "\x00" in audio:
raise ValueError("audio path must not contain null bytes")
# UNC (\\host\share) / network (//host) paths trigger outbound SMB on
# Windows — reject before any filesystem touch.
if is_network_or_device_path(audio):
raise ValueError("audio path must not be a UNC / network / device path")
candidate = Path(audio)
if not candidate.is_absolute():
candidate = base_dir / candidate
if not is_under(candidate, base_dir):
raise ValueError(
f"audio path {Path(audio).name!r} must stay under the audio dir"
)
return str(candidate)
def _resolve_asr_gen_prefix(
asr_language: Optional[str], asr_task: Optional[str], sidecar: dict
) -> dict:
"""Decode-prefix precedence: explicit CLI flags > training sidecar.
Returns the ``language``/``task`` kwargs to pass to ``whisper.generate``
(omitting either when unresolved so the model default applies).
"""
gen_kwargs: dict = {}
language = asr_language or sidecar.get("language")
task = asr_task or sidecar.get("task")
if language:
gen_kwargs["language"] = language
if task:
gen_kwargs["task"] = task
return gen_kwargs
def _build_asr_transcriber(
model: str,
base: Optional[str],
device: Optional[str],
max_tokens: int,
trust_remote_code: bool,
asr_language: Optional[str] = None,
asr_task: Optional[str] = None,
) -> Callable[[str], str]:
"""Build a ``transcribe(audio_path) -> str`` closure over a Whisper model.
Handles both a full fine-tuned Whisper directory and a PEFT/LoRA adapter
dir (base resolved from ``--base`` or the adapter's
``base_model_name_or_path``). The decode language/task come from (in order)
the explicit CLI flags, then the training ``asr_generation.json`` sidecar,
then the model default.
"""
from soup_cli.trainer.asr import _require_whisper_base, read_asr_sidecar
from soup_cli.utils.tts_codec import load_audio_mono
if not device:
from soup_cli.utils.gpu import detect_device
device, _ = detect_device()
from soup_cli.utils.trust_remote import resolve_trust_remote_code
# Detect a PEFT/LoRA adapter dir; resolve its base for the weight load.
model_path = Path(model)
adapter_cfg = model_path / "adapter_config.json"
is_adapter = adapter_cfg.exists()
base_ref = base
if is_adapter and not base_ref:
try:
with open(adapter_cfg, encoding="utf-8") as fh:
base_ref = json.load(fh).get("base_model_name_or_path")
except (json.JSONDecodeError, OSError):
base_ref = None
if not base_ref:
console.print(
f"[red]Cannot detect base model for adapter {model_path}; "
"pass --base.[/]"
)
raise typer.Exit(1)
# The base (adapter case) or the model itself carries the Whisper weights
# AND the processor / arch identity.
weights_ref = base_ref if is_adapter else model
from soup_cli.utils.trust_remote import model_requires_trust_remote_code
requires = model_requires_trust_remote_code(weights_ref) or False
resolved_trust = resolve_trust_remote_code(
weights_ref, requested=trust_remote_code, console=console,
requires_remote_code=requires,
)
# Arch guard: reject a non-Whisper base before the (large) weight load.
_require_whisper_base(weights_ref, resolved_trust)
from transformers import WhisperForConditionalGeneration, WhisperProcessor
console.print(f"[dim]Loading Whisper model: {model}[/]")
# Prefer the fine-tuned dir's processor (carries any resized vocab); fall
# back to the base for an adapter-only dir.
processor = WhisperProcessor.from_pretrained(
model if not is_adapter else weights_ref, trust_remote_code=resolved_trust
)
whisper = WhisperForConditionalGeneration.from_pretrained(
weights_ref, trust_remote_code=resolved_trust
)
if is_adapter:
from peft import PeftModel
whisper = PeftModel.from_pretrained(whisper, model)
whisper.to(device)
whisper.eval()
# Resolve decode prefix: explicit flags > training sidecar > model default.
sidecar = read_asr_sidecar(model)
gen_kwargs = _resolve_asr_gen_prefix(asr_language, asr_task, sidecar)
gen_kwargs["max_new_tokens"] = max_tokens
def transcribe(audio_path: str) -> str:
import torch
wave = load_audio_mono(audio_path, target_sr=16000)
features = processor.feature_extractor(
wave, sampling_rate=16000, return_tensors="pt"
).input_features.to(device)
with torch.no_grad():
generated = whisper.generate(features, **gen_kwargs)
return processor.batch_decode(generated, skip_special_tokens=True)[0].strip()
return transcribe
def _infer_asr(
*,
model: str,
base: Optional[str],
input_file: str,
device: Optional[str],
output_file: str,
max_tokens: int,
trust_remote_code: bool,
asr_language: Optional[str] = None,
asr_task: Optional[str] = None,
audio_dir: Optional[str] = None,
) -> None:
"""Transcribe an ASR JSONL input and (optionally) report WER/CER."""
from soup_cli.utils.asr_metrics import cer, corpus_wer, wer
from soup_cli.utils.paths import (
atomic_write_text,
enforce_under_cwd_and_no_symlink,
is_under_cwd,
)
# Output containment + no-symlink (TOCTOU). Atomic write at the end.
try:
enforce_under_cwd_and_no_symlink(output_file, "--output")
except (TypeError, ValueError) as exc:
console.print(f"[red]{exc}[/]")
raise typer.Exit(1) from exc
# Audio containment base: --audio-dir (must be under cwd) or the input's dir.
if audio_dir:
if not is_under_cwd(audio_dir):
console.print("[red]--audio-dir must stay under the cwd.[/]")
raise typer.Exit(1)
base_dir = Path(audio_dir)
else:
base_dir = Path(input_file).resolve().parent
rows = _read_asr_rows(Path(input_file))
if not rows:
console.print("[red]No ASR rows found (need {\"audio\": path} JSONL).[/]")
raise typer.Exit(1)
if _ASR_TRANSCRIBER_OVERRIDE is not None:
transcribe = _ASR_TRANSCRIBER_OVERRIDE
else:
try:
transcribe = _build_asr_transcriber(
model, base, device, max_tokens, trust_remote_code,
asr_language=asr_language, asr_task=asr_task,
)
except ImportError as exc:
console.print(f"[red]{exc}[/]")
raise typer.Exit(1) from exc
except ValueError as exc: # arch guard / bad base
console.print(f"[red]{exc}[/]")
raise typer.Exit(2) from exc
refs: list[str] = []
hyps: list[str] = []
out_lines: list[str] = []
skipped = 0
for row in rows:
audio = row["audio"]
try:
resolved = _resolve_asr_audio(audio, base_dir)
hyp = transcribe(resolved)
except (ValueError, OSError, ImportError) as exc:
skipped += 1
# Quote the dataset-derived filename first, then escape and
# control-strip it (repr() after the escape would double the
# escape's backslash and make a tag live again); the exception,
# whose message embeds that filename, is escaped too.
name = for_terminal(repr(Path(str(audio)).name))
console.print(
f"[yellow]Skipped {name}: {for_terminal(str(exc))}[/]"
)
continue
rec = {"audio": audio, "transcription": hyp}
ref = row.get("text")
if isinstance(ref, str):
# WER/CER can raise ValueError (the _MAX_RAW_CHARS DoS guard) on an
# oversized reference. Keep it INSIDE a try so one hostile row is
# skipped-unscored, not an uncaught crash that loses every already
# transcribed row's output (transcription cost is already paid).
try:
row_wer = wer(ref, hyp)
row_cer = cer(ref, hyp)
except ValueError as exc:
name = for_terminal(repr(Path(str(audio)).name))
console.print(
f"[yellow]Metric skipped for {name}: "
f"{for_terminal(str(exc))}[/]"
)
else:
rec["reference"] = ref
rec["wer"] = row_wer
rec["cer"] = row_cer
refs.append(ref)
hyps.append(hyp)
out_lines.append(json.dumps(rec, ensure_ascii=False))
# All rows failed to transcribe — do not write an empty file and claim
# success; a scripted pipeline (soup ship, CI) must see a non-zero exit.
if not out_lines:
console.print(
f"[red]No clips transcribed ({skipped} skipped). "
"Check --audio-dir and the input audio paths.[/]"
)
raise typer.Exit(2)
atomic_write_text("\n".join(out_lines) + "\n", output_file, field="--output")
summary = f"Transcribed [bold]{len(out_lines)}[/] clip(s) -> {output_file}"
if skipped:
summary += f" ([yellow]{skipped} skipped[/])"
if refs:
summary += f"\nCorpus WER: [bold]{corpus_wer(refs, hyps):.3f}[/]"
console.print(Panel(summary, title="[bold green]ASR Complete![/]"))
def _read_prompts(path: Path) -> list[str]:
"""Read prompts from a JSONL or plain text file."""
prompts = []
with open(path, encoding="utf-8") as f:
for raw_line in f:
line = raw_line.strip()
if not line:
continue
# Try JSONL
try:
obj = json.loads(line)
if isinstance(obj, dict) and "prompt" in obj:
prompts.append(obj["prompt"])
continue
except json.JSONDecodeError:
pass
# Plain text
prompts.append(line)
return prompts
def _load_model(
model_path: str,
base_model: Optional[str],
device: str,
trust_remote_code: bool = False,
is_local: Optional[bool] = None,
) -> tuple:
"""Load a model and tokenizer (reuses diff.py pattern)."""
from transformers import AutoModelForCausalLM, AutoTokenizer
from soup_cli.utils.gpu import resolve_inference_device_map_and_dtype
from soup_cli.utils.trust_remote import (
model_requires_trust_remote_code,
resolve_trust_remote_code,
)
device_map, torch_dtype = resolve_inference_device_map_and_dtype(device)
if is_local is None:
try:
is_local = Path(model_path).exists()
except OSError:
is_local = False
is_adapter = False
if is_local:
path = Path(model_path)
adapter_config_path = path / "adapter_config.json"
is_adapter = adapter_config_path.exists()
if is_adapter and not base_model:
try:
with open(adapter_config_path, encoding="utf-8") as f:
config = json.load(f)
base_model = config.get("base_model_name_or_path")
except (json.JSONDecodeError, OSError):
pass
if is_adapter and not base_model:
console.print(
f"[red]Cannot detect base model for {path}. Use --base.[/]"
)
raise typer.Exit(1)
probe_target = base_model or model_path
requires = model_requires_trust_remote_code(model_path) or False
trc = resolve_trust_remote_code(
probe_target,
requested=trust_remote_code,
console=console,
requires_remote_code=requires,
)
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=trc)
if tokenizer.pad_token is None:
tokenizer.pad_token = tokenizer.eos_token
if is_adapter:
from peft import PeftModel
base_obj = AutoModelForCausalLM.from_pretrained(
base_model,
trust_remote_code=trc,
device_map=device_map,
torch_dtype=torch_dtype,
)
model_obj = PeftModel.from_pretrained(base_obj, model_path)
else:
model_obj = AutoModelForCausalLM.from_pretrained(
model_path,
trust_remote_code=trc,
device_map=device_map,
torch_dtype=torch_dtype,
)
model_obj.eval()
return model_obj, tokenizer
def _generate(
model, tokenizer, messages, max_tokens=256, temperature=0.7,
cuda_graphs: bool = False, min_tokens: int | None = None,
) -> tuple[str, int]:
"""Generate a response from the model. Returns (text, token_count)."""
import torch
from soup_cli.utils.vllm import encode_chat_prompt
inputs = encode_chat_prompt(
messages, tokenizer, fallback_on_error=False, return_tensors="pt"
)
input_ids = inputs["input_ids"].to(model.device)
attention_mask = inputs["attention_mask"].to(model.device)
with torch.no_grad():
gen_kwargs = {
"input_ids": input_ids,
"attention_mask": attention_mask,
"max_new_tokens": max_tokens,
"do_sample": temperature > 0,
"pad_token_id": tokenizer.pad_token_id,
}
if temperature > 0:
gen_kwargs["temperature"] = temperature
gen_kwargs["top_p"] = 0.9
if min_tokens:
gen_kwargs["min_new_tokens"] = min_tokens
if cuda_graphs:
from soup_cli.utils.cuda_graphs import cuda_graph_generation_kwargs
gen_kwargs.update(cuda_graph_generation_kwargs(model))
outputs = model.generate(**gen_kwargs)
new_tokens = outputs[0][input_ids.shape[1]:]
token_count = new_tokens.shape[0]
response_text = tokenizer.decode(new_tokens, skip_special_tokens=True).strip()
return response_text, token_count
def _generate_batch(
model, tokenizer, prompts, max_tokens=256, temperature=0.7,
) -> list[tuple[str, int]]:
"""Generate responses for a causal-LM prompt batch in input order."""
from soup_cli.utils.vllm import _render_chat_prompt, encode_rendered_prompt
rendered = []
templated = None
for prompt in prompts:
text, row_templated = _render_chat_prompt(
[{"role": "user", "content": prompt}],
tokenizer,
fallback_on_error=False,
)
if templated is None:
templated = row_templated
elif templated != row_templated:
raise ValueError("mixed chat-template rendering modes in one batch")
rendered.append(text)
original_padding_side = tokenizer.padding_side
tokenizer.padding_side = "left"
try:
inputs = encode_rendered_prompt(
tokenizer,
rendered,
templated=bool(templated),
padding=True,
return_tensors="pt",
)
input_ids = inputs["input_ids"].to(model.device)
attention_mask = inputs["attention_mask"].to(model.device)
input_width = input_ids.shape[1]
import torch
with torch.no_grad():
gen_kwargs = {
"input_ids": input_ids,
"attention_mask": attention_mask,
"max_new_tokens": max_tokens,
"do_sample": temperature > 0,
"pad_token_id": tokenizer.pad_token_id,
}
if temperature > 0:
gen_kwargs["temperature"] = temperature
gen_kwargs["top_p"] = 0.9
outputs = model.generate(**gen_kwargs)
results = []
eos_token_ids = getattr(tokenizer, "eos_token_id", None)
if eos_token_ids is None:
eos_token_ids = set()
elif isinstance(eos_token_ids, (list, tuple, set)):
eos_token_ids = set(eos_token_ids)
else:
eos_token_ids = {int(eos_token_ids)}
for output in outputs:
new_tokens = output[input_width:]
token_count = new_tokens.shape[0]
pad_token_id = tokenizer.pad_token_id
for index, token in enumerate(new_tokens):
token_id = int(token)
if token_id in eos_token_ids:
token_count = index + 1
break
if token_id == pad_token_id:
token_count = index
break
actual_tokens = new_tokens[:token_count]
results.append(
(
tokenizer.decode(
actual_tokens, skip_special_tokens=True
).strip(),
token_count,
)
)
return results
finally:
tokenizer.padding_side = original_padding_side
def _count_prompt_tokens(tokenizer, prompt_text: str) -> int:
"""Tokens in the chat-templated prompt: the length the static cache must hold."""
from soup_cli.utils.vllm import encode_chat_prompt
inputs = encode_chat_prompt(
[{"role": "user", "content": prompt_text}], tokenizer,
fallback_on_error=False, return_tensors="pt",
)
return int(inputs["input_ids"].shape[1])
def _longest_prompt(tokenizer, prompts: list[str]) -> str:
return max(prompts, key=lambda text: _count_prompt_tokens(tokenizer, text))
def _with_omit_hint(message: str) -> str:
"""Every --cuda-graphs refusal ends by naming the way out."""
if "omit --cuda-graphs" in message.lower():
return message
return f"{message.rstrip('. ')}. Omit --cuda-graphs."
def _cuda_graph_failure(exc: BaseException) -> typer.Exit:
console.print(
f"[red]Generation failed with --cuda-graphs: {for_terminal(str(exc))}. "
"Omit --cuda-graphs to use normal generation.[/]"
)
return typer.Exit(1)
# Graph trees warm up on a compiled function's first call, record on the second and
# replay from the third; decode forwards are new tokens minus one (prefill is eager).
_CUDA_GRAPH_WARMUP_TOKENS = 4
def _warm_cuda_graphs(model, tokenizer, prompts: list[str], max_tokens: int) -> None:
"""Capture once, on the longest prompt, before any output is written.
Transformers sizes a static cache as max(this request, every earlier one), so
warming on the longest prompt means no later request changes the cache shape and
recompiles. Greedy, so it consumes none of the sampling RNG the real rows use, and
held to a few tokens past EOS so the graph is recorded even on a short answer.
"""
console.print("[dim]Compiling CUDA graph decode (a one-time warm-up)...[/]")
messages = [{"role": "user", "content": _longest_prompt(tokenizer, prompts)}]
try:
_generate(
model,
tokenizer,
messages,
max_tokens=max_tokens,
temperature=0.0,
cuda_graphs=True,
min_tokens=min(max_tokens, _CUDA_GRAPH_WARMUP_TOKENS),
)
except Exception as exc:
raise _cuda_graph_failure(exc) from exc