Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""soup train — the main training command."""
from __future__ import annotations
import contextlib
import json
import logging
import os
import re
import sys
from pathlib import Path
from typing import TYPE_CHECKING, Annotated, Any
import typer
from rich.console import Console
from rich.markup import escape as markup_escape
from rich.panel import Panel
from soup_cli.config.loader import load_config
from soup_cli.data.loader import (
PREFERENCE_PAIR_TASKS,
data_config_for_task,
load_dataset,
task_preserves_source_columns,
)
from soup_cli.monitoring.display import TrainingDisplay
from soup_cli.trainer.classifier import CLASSIFICATION_TASKS
from soup_cli.utils.gpu import (
configure_cuda_alloc_hint,
configure_mps_async_load,
detect_device,
get_gpu_info,
resolve_quantization,
)
if TYPE_CHECKING: # pragma: no cover - type hints only, no runtime import
from soup_cli.config.schema import SoupConfig
from soup_cli.utils.energy import EnergyMeasurement
logger = logging.getLogger(__name__)
console = Console()
# Optimizers the analytical hardware-fit predictor understands (mirror of
# hardware_fit._VALID_OPTIMIZERS); an unknown optimizer maps to the
# highest-state default so the estimate stays conservative.
_HW_FIT_OPTIMIZERS = frozenset({
"adamw_torch", "adamw_torch_fused", "adafactor", "sgd",
"adamw_bnb_8bit", "paged_adamw_8bit", "lion_8bit",
"lomo", "adalomo", "schedule_free_adamw",
})
# training.quantization -> the hardware-fit predictor's name for it. A value
# that is not here (``hqq:*``) has no memory model, and the pre-flight says it
# skipped the run (#1652).
_HW_FIT_QUANT = {
"none": "none",
"4bit": "4bit",
"8bit": "8bit",
# Stay packed on the card.
"gptq": "gptq",
"awq": "awq",
"aqlm": "aqlm",
"eetq": "eetq",
# Dequantized on load, so priced (and gated) like "none" (#1631, #1652).
"mxfp4": "mxfp4",
"fp8": "fp8",
}
# The formats above that train in bf16, and how the gate's panel names them.
_HW_FIT_DEQUANTIZED_ON_LOAD = {"mxfp4": "MXFP4", "fp8": "FP8"}
_UNWIRED_TRAINING_TUNABLES = (
# Group B tunables (forgetting_eval_steps, forgetting_benchmark, forgetting_stop,
# checkpoint_eval_steps, checkpoint_eval_metric, checkpoint_eval_tasks,
# checkpoint_keep_top, convergence_window, convergence_rel_tol) moved to
# config/staged_fields.py (#808), and early_stop_patience moved in #761:
# the loader warns about them with the refusal date, so they are not listed here.
"forgetting_threshold",
)
def _stdin_isatty() -> bool:
"""True when standard input is an interactive terminal."""
return sys.stdin.isatty()
def _nondefault_unwired_training_settings(training_config) -> list[str]:
"""Return staged training settings whose value differs from the schema default."""
fields = type(training_config).model_fields
enabled_flags = [
name
for name, enabled in (
("forgetting_detection", training_config.forgetting_detection),
("checkpoint_intelligence", training_config.checkpoint_intelligence),
("early_stop_on_regression", training_config.early_stop_on_regression),
("convergence_detection", training_config.convergence_detection),
)
if enabled
]
changed_tunables = [
name
for name in _UNWIRED_TRAINING_TUNABLES
if getattr(training_config, name) != fields[name].default
]
return enabled_flags + changed_tunables
UNSUPPORTED_RESUME_TASKS: frozenset[str] = frozenset({"unlearn"})
def _format_training_complete_loss(result: dict) -> str:
"""Render only a loss comparison that the trainer actually measured."""
summary_kind = result.get("loss_summary_kind")
if summary_kind == "unavailable":
return "Loss: [bold]unavailable[/]"
loss_key = result.get("loss_key")
label = f" [dim]({loss_key})[/]" if isinstance(loss_key, str) else ""
if summary_kind in {"mean", "single"} or (
summary_kind is None and result["initial_loss"] == result["final_loss"]
):
return f"Loss: [bold]{result['final_loss']:.4f}[/]{label}"
return f"Loss: [bold]{result['initial_loss']:.4f} -> {result['final_loss']:.4f}[/]{label}"
def _format_duration_display(result: dict) -> str:
"""Duration line for the completion panel (#1529).
Every trainer wrapper returns both a pre-formatted ``duration`` string and
the raw ``duration_secs`` — except unlearn, which until #1529 returned only
the seconds, so the panel's ``result['duration']`` raised ``KeyError``
AFTER the adapter was saved and the run was otherwise complete. Read the
string when present and fall back to formatting the seconds (``unknown``
when both are missing — a measured ``0m`` stays ``0m``, but an absent
measurement should not read as a zero) so no wrapper can lose a finished
run's summary.
"""
duration = result.get("duration")
if duration:
return duration
duration_secs = result.get("duration_secs")
if duration_secs is None:
return "unknown"
hours = int(duration_secs // 3600)
minutes = int((duration_secs % 3600) // 60)
return f"{hours}h {minutes}m" if hours > 0 else f"{minutes}m"
def _run_auto_eval_after_training(
eval_config,
output_dir: str,
run_id: str,
*,
trust_remote_code: bool = False,
) -> None:
"""Run configured evaluation once after the trained model is saved.
``trust_remote_code`` is the run's own ``--trust-remote-code`` flag (CLI only;
there is no soup.yaml field). It reaches only the loads of the trained output
and its base: ``benchmark`` and ``custom``.
"""
if not eval_config or not getattr(eval_config, "auto_eval", False):
return
if not output_dir or not _should_run_diagnose_gate_on_rank():
return
console.print("\n[bold blue]Running auto-eval...[/]")
benchmarks = getattr(eval_config, "benchmarks", None) or []
custom_tasks = getattr(eval_config, "custom_tasks", None)
if benchmarks:
try:
from soup_cli.commands.eval import benchmark
benchmark(
model=output_dir,
benchmarks=",".join(benchmarks),
num_fewshot=None,
batch_size=8,
run_id=run_id,
device=None,
trust_remote_code=trust_remote_code,
)
except typer.Exit:
logger.debug("Auto-eval benchmark skipped", exc_info=True)
console.print(
"[yellow]Auto-eval benchmark skipped (see the message above)[/]"
)
except Exception as exc:
logger.exception("Auto-eval benchmark failed")
console.print(
f"[yellow]Auto-eval benchmark failed: {markup_escape(str(exc))}[/]"
)
if custom_tasks:
try:
from soup_cli.commands.eval import custom
custom(
tasks=custom_tasks,
model=output_dir,
run_id=run_id,
attach_to_registry=None,
output=None,
trust_remote_code=trust_remote_code,
)
except typer.Exit:
logger.debug("Auto-eval custom skipped", exc_info=True)
console.print(
"[yellow]Auto-eval custom skipped (see the message above)[/]"
)
except Exception as exc:
logger.exception("Auto-eval custom failed")
console.print(
f"[yellow]Auto-eval custom failed: {markup_escape(str(exc))}[/]"
)
def _train_sample_count(dcfg, dataset) -> int:
"""Rows training will actually consume (#1054).
For ``format: pre_tokenized`` the rows come from the Arrow cache at
``data.tokenized_path`` (loaded later, in the trainer), not from
``load_dataset`` — which sees the ORIGINAL ``data.train`` file and drops
every row for want of an ``input_ids`` column, printing "0 train samples"
for a run that then trains on the whole cache. Read the count preprocess
recorded instead, falling back to the loader's view if it is unavailable or
not a plausible count (a negative ``row_count`` is a corrupt or hand-edited
metadata.json, and printing "Loaded: -3 train samples" helps nobody).
"""
rows = len(dataset.get("train", []))
if dcfg.format != "pre_tokenized" or not dcfg.tokenized_path:
return rows
try:
with open(
os.path.join(dcfg.tokenized_path, "metadata.json"), encoding="utf-8"
) as f:
count = json.load(f).get("row_count")
except (OSError, ValueError):
return rows
valid = isinstance(count, int) and not isinstance(count, bool) and count >= 0
return count if valid else rows
def _validate_classification_dataset_if_applicable(cfg: Any, dataset: dict) -> None:
"""Validate sequence classification rows upfront if task is in CLASSIFICATION_TASKS."""
if cfg.task in CLASSIFICATION_TASKS:
from rich.markup import escape
from soup_cli.trainer.classifier import validate_classification_dataset
try:
validate_classification_dataset(cfg, dataset)
except (ValueError, TypeError) as exc:
console.print(
f"[red]Error validating {cfg.task} dataset:[/] {escape(str(exc))}"
)
raise typer.Exit(1) from exc
def _validate_csm_dataset_if_applicable(cfg: Any, dataset: dict) -> None:
if cfg.task != "tts" or cfg.training.tts_family != "sesame_csm":
return
from rich.markup import escape
from soup_cli.utils.csm import validate_csm_dataset
try:
validate_csm_dataset(dataset)
except ValueError as exc:
console.print(f"[red]Error validating CSM dataset:[/] {escape(str(exc))}")
raise typer.Exit(1) from exc
def _validate_preference_dataset_if_applicable(cfg: Any, dataset: dict) -> None:
"""Reject a lossy format choice before constructing any preference trainer."""
if cfg.task not in PREFERENCE_PAIR_TASKS:
return
from soup_cli.data.loader import last_load_outcome
required = {"prompt", "chosen", "rejected"}
outcome = last_load_outcome()
fmt = outcome.fmt or cfg.data.format
for split in ("train", "val"):
rows = dataset.get(split, [])
for index, row in enumerate(rows, 1):
if isinstance(row, dict) and required.issubset(row):
continue
console.print(
f"[red]Invalid {cfg.task} dataset:[/] {split} row {index} loaded as "
f"format {markup_escape(repr(fmt))} lacks prompt, chosen or rejected. "
"Use data.format: dpo with preference-pair rows."
)
raise typer.Exit(1)
if not dataset.get("train") and outcome.first_drop is not None:
console.print(
f"[red]No training rows:[/] preference data loaded as format "
f"{markup_escape(repr(fmt))} must provide prompt, chosen and rejected. "
"Use data.format: dpo with preference-pair rows."
)
raise typer.Exit(1)
def _refuse_empty_train(dcfg, dataset) -> None:
"""Stop a run whose data loaded zero training rows (#1217).
The loader drops rows its format cannot convert (one bad line must not
abort a load), so a format mismatch loads nothing and still returns
cleanly. Before this, ``--dry-run`` then printed "Data OK: 0 train
samples" and "Config valid. Ready to train!", and the real run loaded the
model before failing on an empty dataset. Names the format and the first
drop reason, which is what the user has to change.
"""
from rich.markup import escape
from soup_cli.data.loader import last_load_outcome
if _train_sample_count(dcfg, dataset) > 0:
return
outcome = last_load_outcome()
fmt = outcome.fmt or dcfg.format
message = (
f"[red]No training rows:[/] {escape(str(dcfg.train))} loaded 0 train "
f"samples as format {escape(repr(fmt))}."
)
if outcome.first_drop is not None:
drop_fmt, index, reason, _source = outcome.first_drop
message += (
f"\nEvery row that failed to convert was dropped. First: row "
f"{index} (as {escape(repr(drop_fmt))}): {escape(reason)}."
"\nFix those rows, or set data.format to the format they are in."
)
else:
message += (
"\nNo row failed to convert: the data is empty, every row went to "
"validation (data.val_split), or the rows were skipped for missing "
"media files."
)
console.print(message)
raise typer.Exit(1)
def _refuse_over_judge_budget(cfg, dataset) -> None:
"""Refuse a ``judge_rank`` run that could exceed its judge-call budget (#2047).
The rows are known only once the dataset is loaded, and the trainer loads
the model before it reads them, so this runs between the two.
"""
from rich.markup import escape
from soup_cli.trainer.grpo_judge import check_judge_rank_budget
try:
check_judge_rank_budget(cfg, dataset)
except ValueError as exc:
console.print(f"[red]Judge budget:[/] {escape(str(exc))}")
raise typer.Exit(1) from exc
def _build_hardware_fit_input(cfg):
"""Best-effort ``HardwareFitInput`` from a ``SoupConfig``.
Returns ``None`` when the run is not statically predictable (batch_size
``"auto"``, unknown model size, unsupported quant, out-of-range dims), in
which case the caller skips the gate rather than guess.
"""
from soup_cli.utils.gpu import model_size_from_name
from soup_cli.utils.hardware_fit import HardwareFitInput, resolve_peft_mode
tcfg = cfg.training
bs = getattr(tcfg, "batch_size", None)
if not isinstance(bs, int) or isinstance(bs, bool):
return None # "auto" resolves later — can't predict yet
params_b = model_size_from_name(getattr(cfg, "base", "") or "")
if not isinstance(params_b, (int, float)) or params_b <= 0:
return None
seq_len = getattr(cfg.data, "max_length", None)
if not isinstance(seq_len, int) or isinstance(seq_len, bool):
return None
quant = _HW_FIT_QUANT.get(str(getattr(tcfg, "quantization", "none") or "none"))
if quant is None:
return None
peft = resolve_peft_mode(cfg, quant=quant)
optimizer = str(getattr(tcfg, "optimizer", "adamw_torch") or "adamw_torch")
if optimizer not in _HW_FIT_OPTIMIZERS:
optimizer = "adamw_torch"
gc = bool(getattr(tcfg, "gradient_checkpointing", False))
try:
return HardwareFitInput(
params_b=float(params_b),
seq_len=int(seq_len),
batch_size=int(bs),
optimizer=optimizer,
quant=quant,
peft=peft,
gradient_checkpointing=gc,
)
except (ValueError, TypeError):
return None # dims out of the predictor's supported range
def _hardware_fit_preflight(cfg, gpu_info, *, allow_oom_attempt: bool) -> None:
"""Refuse (or warn) before launch when the predicted peak VRAM won't fit.
Skips silently on CPU / when VRAM is unknown / when the run isn't
statically predictable, so CI and small runs are unaffected. A quantization
the predictor has no figure for is skipped with one line (#1652). Honors
the documented ``--allow-oom-attempt`` opt-out.
"""
# v0.72.0 — layer streaming bounds peak VRAM by ONE decoder layer, so the
# resident prediction (full weights + optimizer + grads on the card) is the
# wrong model entirely: it refuses exactly the runs streaming exists to
# enable. The streaming path runs its own pre-flight instead (RAM-tier fit
# + the plan panel in _setup_streaming_transformers).
if getattr(cfg.training, "stream_layers", False):
return
# MLX uses Apple unified memory and its own runtime allocator, so the
# CUDA-shaped analytical VRAM predictor is skipped. On a non-Apple host,
# ``backend: mlx`` still skips harmlessly: ``resolve_trainer`` fails on
# the ``mlx_lm`` import before training starts, so there is no silent
# hazard from bypassing the gate.
if getattr(cfg, "backend", None) == "mlx":
return
total_bytes = 0
try:
total_bytes = int(gpu_info.get("memory_total_bytes", 0) or 0)
except (AttributeError, TypeError, ValueError):
return
if total_bytes <= 0:
return # no CUDA VRAM to predict against
inp = _build_hardware_fit_input(cfg)
if inp is None:
quantization = str(getattr(cfg.training, "quantization", "none") or "none")
if quantization not in _HW_FIT_QUANT:
# #1652: silence here read as "fits".
console.print(
"[yellow]Pre-flight skipped:[/] no memory model for "
f"quantization: {quantization}"
)
return
from soup_cli.utils.hardware_fit import VRAM_SAFETY_MARGIN, decide_hardware_fit
report = decide_hardware_fit(inp, available_vram_gb=total_bytes / 1e9)
if report.ok:
return
b = report.breakdown
# #1631: a "quantized" run priced like an unquantized one needs a word why.
dequantized = _HW_FIT_DEQUANTIZED_ON_LOAD.get(inp.quant)
note = (
f"An {dequantized} base is dequantized on load, so this estimate prices "
"the bf16 model.\n"
if dequantized
else ""
)
tail = (
"[yellow]--allow-oom-attempt set: launching anyway.[/]"
if allow_oom_attempt
else "Reduce batch_size / max_length, enable gradient_checkpointing or "
"quantization, or pass [bold]--allow-oom-attempt[/] to try anyway."
)
console.print(
Panel(
f"Predicted peak VRAM [bold]{report.peak_vram_gb:.1f} GB[/] "
f"(+{int(VRAM_SAFETY_MARGIN * 100)}% margin = "
f"{report.required_with_margin_gb:.1f} GB) exceeds "
f"{report.available_vram_gb:.1f} GB available.\n"
f"weights {b.weights_gb:.1f} | optim {b.optimizer_gb:.1f} | "
f"grads {b.gradients_gb:.1f} | activations {b.activations_gb:.1f} "
f"| overhead {b.overhead_gb:.1f} GB\n" + note + "\n" + tail,
title=(
"[yellow]Hardware-fit warning[/]"
if allow_oom_attempt
else "[bold red]Hardware-fit gate[/]"
),
border_style="yellow" if allow_oom_attempt else "red",
)
)
if not allow_oom_attempt:
raise typer.Exit(1)
def _loads_on_unsloth(cfg) -> bool:
"""#1736 — would this config still load with ``backend: unsloth``?
The tip that suggests the backend used to look at the backend alone, so it
was also printed for configs the schema refuses on unsloth (a task with no
unsloth setup, ``freeze_layers``, ``moe_lora``...). The schema is asked
instead of a list kept here: the dumped config is validated again with the
backend changed. Only that pass's ``ValidationError`` means no; anything
else is a bug and must surface.
"""
from pydantic import ValidationError
from soup_cli.config.schema import SoupConfig
try:
SoupConfig.model_validate({**cfg.model_dump(), "backend": "unsloth"})
except ValidationError:
return False
return True
def _unsloth_tip_applies(cfg) -> bool:
"""Whether to suggest ``backend: unsloth``: only to a run on transformers,
with unsloth installed, whose config would load with it (#1736). The
installed check comes before the second validation, so a machine without
unsloth validates nothing twice."""
if cfg.backend != "transformers":
return False
from soup_cli.utils.unsloth import is_unsloth_available
return is_unsloth_available() and _loads_on_unsloth(cfg)
def _apply_replay_overrides(cfg, *, replay, replay_ratio, replay_seed=None):
"""Apply the ``--replay*`` flags, then RE-VALIDATE.
Re-validation is the point: a CLI override must clear the same
cross-validators as YAML, or ``--replay`` on ``task='dpo'`` would slip
past ``_validate_replay_compat``. Rebuilding the model (rather than
mutating in place) is what re-runs them, and leaves the caller's config
untouched.
"""
if replay is None and replay_ratio is None and replay_seed is None:
return cfg
payload = cfg.model_dump()
if replay is not None:
payload["data"]["replay"] = replay
if replay_ratio is not None:
payload["data"]["replay_ratio"] = replay_ratio
if replay_seed is not None:
payload["data"]["replay_seed"] = replay_seed
return type(cfg)(**payload)
def _describe_exception_for_tracker(exc: BaseException) -> str:
"""Format ``exc`` for ``ExperimentTracker.fail_run``'s ``error=`` column.
A handled setup failure (a bad ``--tracker`` value, a hub download
error, a hub-cache path-containment refusal) exits via
``raise typer.Exit(...)``, sometimes chained with ``from exc`` and
sometimes not. ``typer.Exit`` carries no message of its own, so
formatting it directly writes ``"Exit: "`` to the row and loses the
reason a human needs. Unwrapping to ``__cause__`` recovers it for the
chained sites; recording ``exit_code`` covers the one site that
raises bare, so all three read as something more useful than
``"Exit: "`` (#764/#767 review).
The unwrap is gated to ``typer.Exit`` specifically — this helper also
runs on ordinary training failures (``raise X from Y`` deep inside a
library), where the outer exception X is the operator-facing reason
and the inner cause Y is just the mechanism. Unwrapping unconditionally
would discard X and keep only Y, which is less informative than before
this fix existed — the exact regression the same review caught.
"""
if isinstance(exc, typer.Exit):
if exc.__cause__ is not None:
cause = exc.__cause__
return f"{type(cause).__name__}: {cause} (exit code {exc.exit_code})"
return f"Exit(code={exc.exit_code})"
return f"{type(exc).__name__}: {exc}"
def train(
config: str = typer.Option(
"soup.yaml",
"--config",
"-c",
help="Path to soup.yaml config file",
),
name: str = typer.Option(
None,
"--name",
"-n",
help="Experiment name (auto-generated if not set)",
),
dry_run: bool = typer.Option(
False,
"--dry-run",
help="Validate config and data without training",
),
resume: str = typer.Option(
None,
"--resume",
"-r",
help="Resume from checkpoint: path to checkpoint dir ('auto' for latest); "
"on the MLX backend, a path to a .safetensors adapter file instead. "
"Not supported when training.relora_steps is configured",
),
wandb: bool = typer.Option(
False,
"--wandb",
help="Enable Weights & Biases logging",
),
tensorboard: bool = typer.Option(
False,
"--tensorboard",
help="Enable TensorBoard logging (logs to output_dir/runs/)",
),
tracker: str = typer.Option(
None,
"--tracker",
help=(
"Experiment tracker: mlflow / swanlab / trackio (v0.43.0). "
"Mutually exclusive with --wandb / --tensorboard."
),
),
deepspeed: str = typer.Option(
None,
"--deepspeed",
help=(
"Enable DeepSpeed: zero2, zero3, zero2_offload, zero3_offload "
"(stage 3 + CPU parameter offload), zero++ (ZeRO++), "
"or path to config JSON"
),
),
fsdp: str = typer.Option(
None,
"--fsdp",
help="Enable FSDP2: full_shard, shard_grad, or full_offload",
),
gpus: str = typer.Option(
None,
"--gpus",
help="GPUs per node for distributed training ('auto' or integer)",
),
nodes: Annotated[int, typer.Option(
"--nodes", help="Number of machines; each must use the same --gpus count",
)] = 1,
node_rank: Annotated[int | None, typer.Option(
"--node-rank", help="This machine's rank, from 0 to nodes minus 1 (default 0)",
)] = None,
master_addr: Annotated[str | None, typer.Option(
"--master-addr",
help="Rank 0 hostname or IP reachable by all nodes; required with --nodes > 1",
)] = None,
master_port: Annotated[int | None, typer.Option(
"--master-port", help="Shared coordinator port for multiple nodes (default 29500)",
)] = None,
no_reexec: bool = typer.Option(
False,
"--no-reexec",
help=(
"Print the distributed launch command instead of launching it"
),
),
gate: str = typer.Option(
None,
"--gate",
help=(
"Enable eval-gated training with a suite file "
"(shortcut for training.eval_gate.enabled=true + suite=<path>)"
),
),
push_as: str = typer.Option(
None,
"--push-as",
help=(
"Auto-push each save_steps checkpoint to HF Hub as "
"'checkpoint-<step>' branch of the given repo (e.g. user/my-model)"
),
),
hf_resume: bool = typer.Option(
False,
"--hf-resume",
help=(
"Download the latest checkpoint branch from the --push-as repo "
"and resume from it. Requires --push-as; not supported when "
"training.relora_steps is configured."
),
),
find_lr: bool = typer.Option(
False,
"--find-lr",
help=(
"LR range finder (v0.32.0): run a short geometric LR sweep, write "
"a JSON report with the recommended LR, then exit without training."
),
),
find_lr_start: float = typer.Option(
1e-7,
"--find-lr-start",
help="LR range finder: starting LR (default 1e-7)",
),
find_lr_end: float = typer.Option(
1e-1,
"--find-lr-end",
help="LR range finder: ending LR (default 1e-1)",
),
find_lr_steps: int = typer.Option(
100,
"--find-lr-steps",
help="LR range finder: number of sweep steps (default 100)",
),
find_lr_output: str = typer.Option(
"lr_finder.json",
"--find-lr-output",
help="LR range finder: JSON report path (default ./lr_finder.json)",
),
yes: bool = typer.Option(
False,
"--yes",
"-y",
help=(
"Skip the start-training confirmation. Required when stdin is "
"not a TTY (scripts, CI, pipes)."
),
),
trust_remote_code: bool = typer.Option(
False,
"--trust-remote-code",
help=(
"Allow loading models that ship custom Python via auto_map. "
"Default deny (v0.36.0). Only enable if you trust the source."
),
),
echo_trap_tokenizer_aware: bool = typer.Option(
False,
"--echo-trap-tokenizer-aware",
help=(
"Use tokenizer-id n-grams for echo-trap scoring. Requires "
"training.echo_trap_enabled=true on grpo/ppo."
),
),
reward_hack_detector: str = typer.Option(
None,
"--reward-hack-detector",
help=(
"Reward-hacking detector for GRPO/PPO: info_rm | rm_ensemble. "
"Overrides training.reward_hack_detector. (v0.71.26)"
),
),
reward_hack_halt: bool = typer.Option(
False,
"--reward-hack-halt",
help=(
"Auto-halt training on a HACK verdict. Requires "
"--reward-hack-detector (or training.reward_hack_detector). (v0.71.26)"
),
),
replay: str = typer.Option(
None, "--replay",
help=(
"Old dataset to interleave as continual-learning rehearsal, so "
"training on the new task does not erase the previous one. "
"sft/pretrain only; incompatible with packing/multipack. "
"Overrides data.replay. (v0.71.36)"
),
),
replay_ratio: float = typer.Option(
None, "--replay-ratio",
help=(
"Fraction of the FINAL mixed train set that is replay rows "
"(default 0.1). Overrides data.replay_ratio. (v0.71.36)"
),
),
replay_seed: int = typer.Option(
None, "--replay-seed",
help=(
"Seed for the replay sample + interleave. Overrides "
"data.replay_seed. (v0.71.36)"
),
),
reward_hack_mitigation: str = typer.Option(
None,
"--reward-hack-mitigation",
help=(
"Closed-loop reward-hacking mitigation mode: off | log_only | "
"kl_control | pid_lagrangian. Requires training.reward_hack_detector "
"on grpo/ppo. Overrides training.reward_hack_mitigation. (v0.71.26)"
),
),
minillm_on_policy: bool = typer.Option(
False,
"--minillm-on-policy",
help=(
"Use the TRUE on-policy MiniLLM teacher-mixed rollout (v0.71.18 "
"#257) instead of the offline distribution blend. Requires "
"training.minillm_enabled=true on task='distill'."
),
),
profile_run: bool = typer.Option(
False,
"--profile",
help=(
"Record a torch.profiler trace (Chrome trace JSON) during early "
"training steps. Output: <output>/profiles/<run_id>.trace.json"
),
),
allow_oom_attempt: bool = typer.Option(
False,
"--allow-oom-attempt",
help=(
"Bypass the analytical hardware-fit VRAM gate and launch even when "
"the run is predicted to run out of GPU memory (opt-out)."
),
),
diagnose_gate: str = typer.Option(
None,
"--diagnose-gate",
help=(
"After training, run `soup diagnose` against the supplied evidence "
"JSON (or scratch evidence). Refuses to mark the run successful "
"if any failure mode returns MAJOR (exit 2) or NOT_RUN (exit 3)."
),
),
annex_xi: str = typer.Option(
None,
"--annex-xi",
help=(
"After training, render an EU AI Act Annex XI/XII auto-doc to the "
"given output path (cwd-contained). Markdown body now; PDF in v0.59.1."
),
),
repro_receipt: str = typer.Option(
None,
"--repro-receipt",
help=(
"After training, write an SR 11-7-style reproducibility receipt "
"(seeds + kernel versions + GPU + OS) to the given path. v0.59.0."
),
),
capture_activations: str = typer.Option(
None,
"--capture-activations",
help=(
"After training, capture residual-stream activations at the named "
"decoder layer (e.g. model.layers.5) on --capture-prompts and write "
"them to <output>/activations/activations.json for soup probe "
"sae-diff / sleeper. v0.71.8 #219."
),
),
capture_prompts: str = typer.Option(
None,
"--capture-prompts",
help=(
"JSONL (or .txt) of prompts to run for --capture-activations "
"(one prompt per line; 'prompt'/'text' field or raw text)."
),
),
track_energy: bool = typer.Option(
False,
"--track-energy",
help=(
"Measure the training window's energy + CO2 via codecarbon "
"(offline; requires `pip install soup-cli\\[carbon]`). Feeds the "
"kWh / CO2 into --annex-xi. v0.71.3."
),
),
energy_country: str = typer.Option(
"USA",
"--energy-country",
help=(
"ISO 3166-1 alpha-3 country code for the CO2 grid-intensity "
"estimate used by --track-energy (default USA)."
),
),
energy_out: str = typer.Option(
None,
"--energy-out",
help=(
"Write the --track-energy measurement to this JSON file (cwd-"
"contained) so `soup bom emit --energy <path>` can consume it. "
"v0.71.15."
),
),
cloud: str = typer.Option(
None,
"--cloud",
help=(
"Train on a cloud GPU instead of locally (v0.71.18 #16). Supported: "
"modal, lambda, runpod. Renders a cloud app stub from the config "
"(plan-only); use --cloud-submit to submit live."
),
),
gpu: str = typer.Option(
"a100",
"--gpu",
help=(
"Cloud GPU type for --cloud (t4 / l4 / a10 / a10g / a100 / a100-80gb / "
"l40s / h100 / a6000). Default a100. Provider-specific allowlists apply."
),
),
cloud_submit: bool = typer.Option(
False,
"--cloud-submit",
help=(
"With --cloud, submit the rendered run live via the cloud's SDK or API "
"(gated on respective provider token/API key). Default is plan-only "
"(render + print the command). Lambda requires a registered SSH key."
),
),
):
"""Start training from a soup.yaml config."""
from soup_cli.utils.launcher import validate_multi_node_options
try:
validate_multi_node_options(nodes, node_rank, master_addr, master_port)
if nodes > 1:
if not gpus:
raise ValueError("--nodes > 1 requires --gpus (GPUs per node)")
if cloud or find_lr:
raise ValueError("--nodes > 1 cannot be combined with --cloud or --find-lr")
except ValueError as exc:
message = str(exc)
for field, flag in (
("num_machines", "--nodes"), ("machine_rank", "--node-rank"),
("main_process_ip", "--master-addr"), ("main_process_port", "--master-port"),
):
message = message.replace(field, flag)
console.print(f"[red]Invalid distributed launch:[/] {markup_escape(message)}")
raise typer.Exit(1) from exc
from soup_cli.utils.terminal import for_terminal
config_path = Path(config)
if not config_path.exists():
console.print(f"[red]Config not found: {for_terminal(config_path)}[/]")
console.print("Run [bold]soup init[/] to create one.")
raise typer.Exit(1)
# --- LR range finder fast path ---
if find_lr:
from soup_cli.utils.lr_finder import (
LrSweepUnavailableError,
SweepTooShortError,
compute_lr_schedule,
save_lr_finder_report,
)
try:
schedule = compute_lr_schedule(
start_lr=find_lr_start,
end_lr=find_lr_end,
num_steps=find_lr_steps,
)
except ValueError as exc:
console.print(f"[red]Invalid --find-lr range:[/] {exc}")
raise typer.Exit(1) from exc
# v0.33.0 #56: live LR-sweep training loop. #1203: when it cannot run, the
# command refuses rather than writing a curve nobody measured.
# #1189: the sweep returns the LRs it actually ran, which can be fewer than
# --find-lr-steps (a short dataset, or a loss that went non-finite).
try:
lrs, losses_for_report = _run_live_lr_sweep(
config_path, schedule, trust_remote_code=trust_remote_code,
)
except (SweepTooShortError, LrSweepUnavailableError) as exc:
# The refusal can quote the config path and text from the file.
console.print(f"[red]{for_terminal(str(exc))}[/]")
raise typer.Exit(1) from exc
try:
save_lr_finder_report(lrs, losses_for_report, find_lr_output)
except ValueError as exc:
console.print(f"[red]Invalid --find-lr-output:[/] {exc}")
raise typer.Exit(1) from exc
console.print(f"[green]LR finder report written to:[/] {find_lr_output}")
raise typer.Exit(0)
# Load & validate config
console.print(f"[dim]Loading config from {for_terminal(config_path)}...[/]")
cfg = load_config(
config_path,
training_overrides=(
{"minillm_on_policy": True} if minillm_on_policy else None
),
)
# --- v0.71.36 replay passthrough ---
try:
cfg = _apply_replay_overrides(
cfg,
replay=replay,
replay_ratio=replay_ratio,
replay_seed=replay_seed,
)
except Exception as exc: # noqa: BLE001 — pydantic ValidationError et al.
console.print(f"[red]{markup_escape(str(exc))}[/]")
raise typer.Exit(code=2) from exc
if cfg.training.relora_steps is not None and (resume is not None or hf_resume):
console.print(
"[red]ReLoRA runs cannot resume from checkpoints.[/] "
"Remove training.relora_steps or start a fresh run."
)
raise typer.Exit(1)
# An unregistered data.chat_template name raises KeyError in the trainer,
# after the model has loaded. Check it before anything is downloaded.
from soup_cli.data.chat_templates import resolve_chat_template
try:
resolve_chat_template(cfg.data.chat_template)
except KeyError as exc:
console.print(f"[red]Invalid data.chat_template:[/] {for_terminal(exc.args[0])}")
raise typer.Exit(1) from exc
# v0.72.3 — --resume / --hf-resume now work with layer streaming. v0.72.0-.2
# refused them because a streamed model's `named_parameters()` carry an
# `.inner.` segment that `load_state_dict` narrows away, so PEFT matched
# NOTHING and silently continued with a freshly initialised adapter (measured:
# 0 of 12 tensors, and a resumed loss curve byte-identical to a from-scratch
# one). `StreamedDecoderLayer` now redirects canonical keys at load time,
# mirroring the v0.72.1 save-side delegation.
# --- RA-DIT generator-stage auto-link (v0.71.10 #200) ---
# When a generator stage has no retriever model set, splice in the latest
# RA-DIT retriever output from the Registry. A manual value always wins.
if getattr(cfg.training, "ra_dit_stage", None) == "generator":
from soup_cli.utils.ra_dit_run import autolink_generator_retriever
advisory = autolink_generator_retriever(cfg)
if advisory:
# `advisory` embeds a Registry-derived `output` path — escape it
# before printing into the Rich-markup console (security MEDIUM).
console.print(f"[yellow]RA-DIT:[/] {markup_escape(advisory)}")
# --- Echo-trap tokenizer-aware shortcut ---
if echo_trap_tokenizer_aware:
if not cfg.training.echo_trap_enabled:
console.print(
"[red]--echo-trap-tokenizer-aware requires "
"training.echo_trap_enabled=true[/]"
)
raise typer.Exit(1)
cfg.training.echo_trap_tokenizer_aware = True
console.print("[green]Echo-trap tokenizer-aware scoring enabled[/]")
# --- Reward-hack detector / halt shortcut (v0.71.26) ---
if reward_hack_detector is not None:
if reward_hack_detector not in ("info_rm", "rm_ensemble"):
console.print(
"[red]--reward-hack-detector must be info_rm or rm_ensemble[/]"
)
raise typer.Exit(1)
cfg.training.reward_hack_detector = reward_hack_detector
console.print(f"[green]Reward-hack detector:[/] {reward_hack_detector}")
if reward_hack_halt:
if cfg.training.reward_hack_detector is None:
console.print(
"[red]--reward-hack-halt requires --reward-hack-detector "
"(or training.reward_hack_detector)[/]"
)
raise typer.Exit(1)
cfg.training.reward_hack_halt = True
console.print("[green]Reward-hack auto-halt enabled[/]")
# --- Reward-hack mitigation shortcut (v0.71.26) ---
if reward_hack_mitigation is not None:
valid_modes = ("off", "log_only", "kl_control", "pid_lagrangian")
if reward_hack_mitigation not in valid_modes:
console.print(
"[red]--reward-hack-mitigation must be one of "
f"{', '.join(valid_modes)}[/]"
)
raise typer.Exit(1)
if (
reward_hack_mitigation != "off"
and cfg.training.reward_hack_detector is None
):
console.print(
"[red]--reward-hack-mitigation requires "
"training.reward_hack_detector to be set (the signal source)[/]"
)
raise typer.Exit(1)
cfg.training.reward_hack_mitigation = reward_hack_mitigation
console.print(
f"[green]Reward-hack mitigation:[/] {reward_hack_mitigation}"
)
# --- MiniLLM on-policy rollout shortcut (v0.71.18 #257) ---
# Applied before SoupConfig validation via load_config(training_overrides=),
# so --minillm-on-policy can still select student-only sampling when the
# YAML leaves mix at 0 (#692 / #977). The assignment below is therefore a
# no-op when the flag was set; it stays so a later reader sees the flag
# take effect on cfg.training.
if minillm_on_policy:
if not cfg.training.minillm_enabled:
console.print(
"[red]--minillm-on-policy requires "
"training.minillm_enabled=true (task='distill')[/]"
)
raise typer.Exit(1)
cfg.training.minillm_on_policy = True
console.print("[green]MiniLLM on-policy rollout enabled[/]")
# --- Cloud GPU training (v0.71.18 #16, v0.71.22 #264) ---
if cloud:
from soup_cli import __version__ as _soup_version
cloud = cloud.lower()
if cloud == "runpod":
from soup_cli.cloud import runpod as cloud_mod
elif cloud == "modal":
from soup_cli.cloud import modal as cloud_mod
elif cloud == "lambda":
from soup_cli.cloud import lambda_labs as cloud_mod
else:
console.print(
f"[red]Invalid --cloud:[/] {markup_escape(cloud)}. "
"Supported: modal, lambda, runpod."
)
raise typer.Exit(2)
try:
cloud_mod.validate_cloud(cloud)
cloud_mod.validate_gpu(gpu)
except ValueError as exc:
console.print(
f"[red]Invalid --cloud / --gpu:[/] {markup_escape(str(exc))}"
)
raise typer.Exit(2) from exc
try:
# We call the generic-shaped plan function dynamically
plan_func = getattr(cloud_mod, f"plan_{cloud}_run", None)
if plan_func is None:
raise ValueError(f"cloud backend {cloud!r} has no plan function")
plan = plan_func(
str(config_path),
gpu=gpu,
output_dir=cfg.output,
soup_version=_soup_version,
)
stub_realpath = cloud_mod.write_stub(plan)
except (ValueError, TypeError) as exc:
console.print(f"[red]Cloud plan failed:[/] {markup_escape(str(exc))}")
raise typer.Exit(2) from exc
console.print(
Panel(
f"Cloud: [bold]{markup_escape(cloud)}[/]\n"
f"GPU: [bold]{markup_escape(plan.gpu)}[/]\n"
f"Stub: [bold]{markup_escape(os.path.relpath(stub_realpath))}[/]\n"
f"Output: [bold]{markup_escape(plan.output_dir)}[/]\n\n"
f"[bold]Run:[/] {markup_escape(plan.run_command)}",
title=f"[bold green]soup train --cloud {markup_escape(cloud)}[/]",
)
)
if cloud_submit:
try:
submit_func = getattr(cloud_mod, f"submit_{cloud}_run", None)
if submit_func is None:
raise RuntimeError(f"cloud backend {cloud!r} has no submit function")
rc = submit_func(plan)
except RuntimeError as exc:
console.print(
f"[yellow]{cloud.title()} submit unavailable:[/] "
f"{markup_escape(str(exc))}"
)
raise typer.Exit(1) from exc
raise typer.Exit(rc)
console.print(
f"[yellow]Note:[/] plan-only. Authenticate with {cloud}, then run the "
"command above (or re-run with --cloud-submit)."
)
raise typer.Exit(0)
# --- --push-as / --hf-resume validation ---
if push_as:
from soup_cli.utils.hf import validate_repo_id
try:
validate_repo_id(push_as)
except ValueError as exc:
console.print(f"[red]Invalid --push-as repo id:[/] {exc}")
raise typer.Exit(1) from exc
if hf_resume and not push_as:
console.print("[red]--hf-resume requires --push-as <repo>[/]")
raise typer.Exit(1)
if hf_resume and cfg.task in UNSUPPORTED_RESUME_TASKS:
console.print(f"[red]--hf-resume is not supported for task {cfg.task!r}[/]")
raise typer.Exit(1)
# --- Eval-gate shortcut: --gate <path> sets training.eval_gate ---
if gate:
from soup_cli.config.schema import EvalGateConfig
from soup_cli.eval.gate import load_suite
try:
# Validate the suite path up-front (path containment + parse).
load_suite(gate)
except (FileNotFoundError, ValueError) as exc:
console.print(f"[red]Invalid --gate suite: {for_terminal(str(exc))}[/]")
raise typer.Exit(1) from exc
cfg.training.eval_gate = EvalGateConfig(enabled=True, suite=gate)
console.print(f"[green]Eval gate enabled[/] with suite: {gate}")
# Honesty guard: these staged flags (plus forgetting_threshold) are accepted
# but not enforced mid-training in this build. Their other tuning knobs are
# reported by the loader with refusal dates, so only active flags and
# non-default forgetting_threshold are reported here.
_unwired_gates = _nondefault_unwired_training_settings(cfg.training)
if _unwired_gates:
console.print(
"[yellow]Note:[/] "
+ ", ".join(_unwired_gates)
+ " are set but not enforced during training in this build. "
"Use [bold]--gate <suite.yaml>[/] for a live eval gate, or run "
"[bold]soup eval[/] / [bold]soup diagnose[/] after training."
)
# --- Resolve resume checkpoint (fail fast before heavy operations) ---
resume_from = _resolve_resume_or_exit(resume, cfg)
# --- HF auto-resume: pull latest checkpoint branch into output dir ---
if hf_resume and push_as and resume_from is None:
from soup_cli.monitoring.hf_push import prepare_hf_resume
from soup_cli.utils.hf import resolve_endpoint, resolve_token
try:
hf_endpoint = resolve_endpoint()
except ValueError as exc:
console.print(f"[red]--hf-resume: {exc}[/]")
raise typer.Exit(1) from exc
hf_token = resolve_token()
if hf_token is None:
console.print(
"[yellow]--hf-resume: no HF token available; skipping auto-resume[/]"
)
else:
local_ckpt = prepare_hf_resume(
repo_id=push_as,
output_dir=cfg.output,
token=hf_token,
endpoint=hf_endpoint,
)
if local_ckpt:
resume_from = local_ckpt
console.print(f"[green]Resumed from HF:[/] {local_ckpt}")
else:
console.print(
"[yellow]--hf-resume: no checkpoint branch found; starting fresh[/]"
)
# --- Validate logging flags ---
if wandb and tensorboard:
console.print(
"[red]Cannot use --wandb and --tensorboard together. Pick one.[/]"
)
raise typer.Exit(1)
# --- TensorBoard setup ---
if tensorboard:
try:
import tensorboard # noqa: F401
console.print("[green]TensorBoard logging enabled[/]")
except ImportError:
console.print(
"[red]TensorBoard not installed.[/]\n"
"Run: [bold]pip install tensorboard[/]"
)
raise typer.Exit(1)
# --- W&B setup (fail fast if wandb not installed) ---
if wandb:
try:
import wandb as _wandb # noqa: F401
console.print("[green]W&B logging enabled[/]")
except ImportError:
console.print(
"[red]wandb not installed.[/]\n"
"Run: [bold]pip install \"soup-cli\\[wandb]\"[/]"
)
raise typer.Exit(1)
except Exception as wandb_err:
console.print(
f"[red]wandb import error:[/] {wandb_err}\n"
"Try: [bold]pip install 'wandb>=0.15.0,<0.18.0'[/]"
)
raise typer.Exit(1)
# --- DeepSpeed setup ---
ds_config_path = None
if deepspeed:
ds_config_path = _resolve_deepspeed(deepspeed)
if ds_config_path:
console.print(f"[green]DeepSpeed enabled:[/] {deepspeed}")
# --- FSDP2 setup ---
fsdp_kwargs = None
if fsdp:
from soup_cli.utils.fsdp import FSDP_CONFIGS, get_fsdp_training_args
if fsdp not in FSDP_CONFIGS:
console.print(
f"[red]Invalid FSDP preset: {fsdp}[/]\n"
f"Options: {', '.join(FSDP_CONFIGS.keys())}"
)
raise typer.Exit(1)
fsdp_kwargs = get_fsdp_training_args(fsdp)
console.print(f"[green]FSDP2 enabled:[/] {fsdp}")
# #350 — BNB's default uint8 quant storage is not merely slow under FSDP:
# FSDP cannot flatten it. Resolve storage to the exact BNB compute dtype
# before any trainer builds its BitsAndBytesConfig. This updates the
# effective config shared by every wrapper and the reproducibility receipt.
from soup_cli.utils.quant_menu import resolve_fsdp_qlora_quant_storage
original_quant_storage = cfg.training.bnb_4bit_quant_storage
resolved_training = resolve_fsdp_qlora_quant_storage(
cfg.training,
fsdp=bool(fsdp),
)
if resolved_training is not cfg.training:
cfg = cfg.model_copy(update={"training": resolved_training})
action = "selected" if original_quant_storage is None else "overrode"
console.print(
f"[green]FSDP QLoRA:[/] {action} bnb_4bit_quant_storage="
f"{resolved_training.bnb_4bit_quant_storage} to match compute dtype"
)
# --- v0.38.0 Quant Menu × multi-GPU compatibility check ---
from soup_cli.utils.quant_menu import check_quant_distributed_compat
quant_problems = check_quant_distributed_compat(
quantization=cfg.training.quantization,
deepspeed=deepspeed,
fsdp=bool(fsdp),
bnb_4bit_quant_storage=cfg.training.bnb_4bit_quant_storage,
)
if quant_problems:
hard = [p for p in quant_problems if not p.lower().startswith("warning")]
warn = [p for p in quant_problems if p.lower().startswith("warning")]
for problem in hard:
console.print(f"[red]Quant compat:[/] {problem}")
for problem in warn:
console.print(f"[yellow]{problem}[/]")
if hard:
raise typer.Exit(1)
# --- Multi-GPU topology + --gpus resolution ---
num_gpus = None
if gpus:
from soup_cli.utils.topology import detect_topology, resolve_num_gpus
try:
num_gpus = resolve_num_gpus(gpus)
except ValueError as exc:
console.print(f"[red]Invalid --gpus:[/] {exc}")
raise typer.Exit(1) from exc
topo = detect_topology()
if num_gpus is not None and num_gpus < 1:
if nodes > 1:
console.print("[red]Multi-node training requires at least one GPU per node.[/]")
raise typer.Exit(1)
# --gpus auto on CPU / no-CUDA box — explicit, not silent.
console.print(
"[yellow]--gpus auto detected 0 GPUs; continuing as a "
"single-process CPU run.[/]"
)
elif num_gpus is not None and num_gpus * nodes > 1:
from soup_cli.utils.launcher import (
build_accelerate_argv,
build_train_reexec_argv,
collect_reexec_passthrough,
format_advice,
hint_argv_from_reexec,
is_in_distributed,
run_launcher,
)
num_processes = num_gpus * nodes
if not is_in_distributed():
# v0.33.0 #37 — auto-reexec under accelerate launch unless
# --no-reexec was passed. On POSIX the accelerate process
# replaces this one (os.execvp): no leftover PID tree, stdio
# passes through unchanged. Windows cannot do that, so there
# this process waits for the launcher (see run_launcher).
# #372 — one argv builder for both the re-exec and the printed
# hint, so they cannot drift. collect_reexec_passthrough is the
# only list of "flags the user typed" that survive a launch.
script_args = build_train_reexec_argv(
config,
collect_reexec_passthrough(
name=name,
fsdp=fsdp,
deepspeed=deepspeed,
resume=resume,
wandb=wandb,
tensorboard=tensorboard,
echo_trap_tokenizer_aware=echo_trap_tokenizer_aware,
reward_hack_detector=reward_hack_detector,
reward_hack_halt=reward_hack_halt,
reward_hack_mitigation=reward_hack_mitigation,
gate=gate,
push_as=push_as,
hf_resume=hf_resume,
trust_remote_code=trust_remote_code,
tracker=tracker,
diagnose_gate=diagnose_gate,
annex_xi=annex_xi,
repro_receipt=repro_receipt,
profile_run=profile_run,
allow_oom_attempt=allow_oom_attempt,
track_energy=track_energy,
energy_country=energy_country,
energy_out=energy_out,
yes=yes,
minillm_on_policy=minillm_on_policy,
capture_activations=capture_activations,
capture_prompts=capture_prompts,
replay=replay,
replay_ratio=replay_ratio,
replay_seed=replay_seed,
),
)
if nodes > 1:
console.print(
"[dim]Multi-node NCCL default: NCCL_IB_DISABLE=0. "
"Set NCCL_IB_DISABLE=1 for TCP-only networks. "
"Existing environment settings are preserved.[/]"
)
if (dry_run and nodes > 1) or (no_reexec and not dry_run):
# Multi-node advice is the executable module-form argv,
# exactly as passed to execvp, including the child guard.
hint_args = script_args if nodes > 1 else hint_argv_from_reexec(script_args)
console.print(
Panel(
markup_escape(format_advice(
num_processes, hint_args, num_machines=nodes,
machine_rank=node_rank, main_process_ip=master_addr,
main_process_port=master_port,
)),
title="[yellow]Multi-GPU launch required[/]",
)
)
console.print(
f"[dim]Detected topology: {topo['gpu_count']} GPUs, "
f"{topo['interconnect']}[/]"
)
if not dry_run:
raise typer.Exit(1)
if dry_run:
console.print(
f"[dim]--dry-run: skipping accelerate re-exec "
f"({num_processes} GPUs, {topo['interconnect']}).[/]"
)
else:
argv = build_accelerate_argv(
num_processes=num_processes, script_args=script_args,
num_machines=nodes, machine_rank=node_rank,
main_process_ip=master_addr, main_process_port=master_port,
)
if nodes > 1:
from soup_cli.utils.topology import suggest_nccl_env
for key, val in suggest_nccl_env(
gpu_count=num_gpus, interconnect=topo["interconnect"],
num_machines=nodes,
).items():
os.environ.setdefault(key, val)
console.print(
f"[green]Auto-reexec under accelerate "
f"({num_processes} GPUs, {topo['interconnect']})[/]"
)
console.print(f"[dim]argv: {markup_escape(' '.join(argv))}[/]")
# The launcher takes over; the child carries --no-reexec. On
# POSIX run_launcher execs and never returns. On Windows,
# where exec would return 0 at once and leave the launcher
# running, it waits and returns the launcher's exit code.
try:
launcher_code = run_launcher(argv)
except OSError as exc:
console.print(
f"[red]accelerate launch failed:[/] {markup_escape(str(exc))}\n"
"Use [bold]--no-reexec[/] to print the launch command."
)
raise typer.Exit(1) from exc
raise typer.Exit(launcher_code)
elif not dry_run:
# Already a launched rank — announce + apply NCCL hints. (The
# dry_run branch above intentionally does neither.)
console.print(
f"[green]Distributed run detected[/] "
f"({num_processes} procs, {topo['interconnect']} interconnect)"
)
# Apply NCCL env hints. All current keys (``NCCL_P2P_DISABLE`` /
# ``NCCL_IB_DISABLE`` / ``NCCL_NVLS_ENABLE``) are rank-idempotent
# string literals so it is safe to run on every rank. If a
# rank-sensitive key is ever added to ``suggest_nccl_env``, this
# loop must be gated to ``LOCAL_RANK == 0``. ``setdefault`` keeps
# user / launcher overrides winning over our suggestions.
from soup_cli.utils.topology import suggest_nccl_env
for key, val in suggest_nccl_env(
gpu_count=num_gpus, interconnect=topo["interconnect"], num_machines=nodes,
).items():
os.environ.setdefault(key, val)
# Before detect_device, which creates the CUDA context the allocator reads it at (#2149).
configure_cuda_alloc_hint(cfg.training.stream_layers)
# Detect hardware with backend awareness
device, device_name = detect_device(backend=cfg.backend)
gpu_info = get_gpu_info(backend=cfg.backend)
configure_mps_async_load(device)
# Quantization guard: explicit decision per #423. See resolve_quantization()
# docstring for the full rationale — MLX 4-bit uses pre-quantized mlx-community
# weights (not bitsandbytes NF4), CPU cannot run bitsandbytes at all.
resolved_quant, quant_warning = resolve_quantization(
device=device,
backend=cfg.backend,
quantization=cfg.training.quantization,
)
if quant_warning:
console.print(f"[yellow]{quant_warning}[/]")
cfg.training.quantization = resolved_quant
# Hardware-fit preflight: refuse (unless --allow-oom-attempt) when the
# analytical VRAM predictor says the run won't fit. Skips silently on CPU
# or when the config isn't statically predictable (e.g. batch_size='auto');
# says so when the quantization has no memory model (hqq:*).
_hardware_fit_preflight(cfg, gpu_info, allow_oom_attempt=allow_oom_attempt)
backend_label = cfg.backend
if cfg.backend == "unsloth":
backend_label = "unsloth [green](fast mode)[/]"
quant_label = cfg.training.quantization
if cfg.training.quantization_aware == "quest":
quant_label = "mixed W4/A4+A16 (QuEST fake quant)"
elif cfg.training.quantization_aware:
quant_label += " + QAT"
# v0.53.2 review-fix: classifier-family tasks train a sequence-classification
# head, not a causal-LM LoRA — render "head" instead of LoRA r/alpha.
if cfg.task in CLASSIFICATION_TASKS:
# v0.71.12 #146 — render BOTH the head line AND a LoRA line when the
# opt-in classifier LoRA path is active.
head_line = (
f"Head: [bold]num_labels={cfg.training.num_labels}, "
f"kind={cfg.training.classifier_kind}[/]"
)
if getattr(cfg.training, "classifier_lora", False) and cfg.training.lora.r > 0:
peft_line = (
head_line
+ f"\nLoRA: [bold]r={cfg.training.lora.r}, "
+ f"alpha={cfg.training.lora.alpha} (SEQ_CLS)[/]"
)
else:
peft_line = head_line
else:
peft_line = (
f"LoRA: [bold]r={cfg.training.lora.r}, "
f"alpha={cfg.training.lora.alpha}[/]"
)
# #353 review-fix: a wrong seed is invisible, which is how `training.seed`
# reaching one wrapper out of nineteen survived releases. Report the value
# the run actually trains at, and say when it is the unset default, so
# "did my seed take?" is answered by looking rather than by reading source.
from soup_cli.utils.seeding import resolve_training_seed
seed_label = str(resolve_training_seed(cfg.training))
if cfg.training.seed is None:
seed_label += " [dim](unset default)[/]"
if cfg.training.data_seed is not None:
seed_label += f", data_seed={cfg.training.data_seed}"
console.print(
Panel(
f"Device: [bold]{device_name}[/]\n"
f"Memory: [bold]{gpu_info['memory_total']}[/]\n"
f"Model: [bold]{cfg.base}[/]\n"
f"Task: [bold]{cfg.task}[/]\n"
f"Backend: [bold]{backend_label}[/]\n"
f"{peft_line}\n"
f"Quant: [bold]{quant_label}[/]\n"
f"Seed: [bold]{seed_label}[/]",
title="Training Setup",
)
)
# Validate GaLore configuration
if cfg.training.use_galore:
from soup_cli.utils.galore import validate_galore_config
galore_errors = validate_galore_config(
cfg.training.use_galore, cfg.training.quantization, cfg.backend,
)
for err in galore_errors:
console.print(f"[red]GaLore error:[/] {for_terminal(err)}")
if galore_errors:
raise typer.Exit(1)
# Validate QAT configuration
if (
cfg.training.quantization_aware is True
or cfg.training.quantization_aware == "fp8"
):
from soup_cli.utils.qat import validate_qat_config
qat_errors = validate_qat_config(
cfg.training.quantization, cfg.backend, cfg.modality,
quantization_aware=cfg.training.quantization_aware,
fp8_recipe=cfg.training.fp8_recipe,
check_card=not dry_run,
)
for err in qat_errors:
console.print(f"[red]QAT error:[/] {for_terminal(err)}")
if dry_run and cfg.training.quantization_aware == "fp8" and cfg.backend != "unsloth":
# #1154 review: a dry run validates the config, and FP8 configs are
# routinely written on a laptop for a remote card, so the local card
# is a note here -- printed before any error exit, so a dry run that
# also lacks torchao still shows the whole picture. The real run
# still stops on the card. (Unsloth: its refusal is the answer.)
from soup_cli.utils.fp8 import fp8_training_supported
card_ok, card_reason = fp8_training_supported(cfg.training.fp8_recipe)
if not card_ok:
console.print(
"[yellow]Note:[/] this machine could not run it: "
f"{markup_escape(card_reason)}"
)
if qat_errors:
raise typer.Exit(1)
# Validate FSDP configuration
if fsdp:
from soup_cli.utils.fsdp import validate_fsdp_config
fsdp_errors = validate_fsdp_config(
fsdp_preset=fsdp,
deepspeed_config=ds_config_path,
backend=cfg.backend,
device=device,
)
for err in fsdp_errors:
console.print(f"[red]FSDP error:[/] {for_terminal(err)}")
if fsdp_errors:
raise typer.Exit(1)
# Validate FSDP2 + torch.compile (v0.27.0 Part D)
if cfg.training.use_fsdp2_compile:
from soup_cli.utils.fsdp import validate_fsdp2_compile_config
compile_errors = validate_fsdp2_compile_config(
use_compile=cfg.training.use_fsdp2_compile,
fsdp_preset=fsdp,
backend=cfg.backend,
device=device,
deepspeed_config=ds_config_path,
)
for err in compile_errors:
console.print(f"[red]FSDP2 + torch.compile error:[/] {for_terminal(err)}")
if compile_errors:
raise typer.Exit(1)
# Validate pipeline parallelism (v0.27.0 Part F)
if cfg.training.parallelism == "pipeline":
from soup_cli.utils.pipeline import validate_pipeline_config
pp_errors = validate_pipeline_config(
parallelism=cfg.training.parallelism,
pipeline_stages=cfg.training.pipeline_stages,
device=device,
gpu_count=gpu_info.get("gpu_count", 0),
)
for err in pp_errors:
console.print(f"[red]Pipeline parallel error:[/] {for_terminal(err)}")
if pp_errors:
raise typer.Exit(1)
console.print(
Panel(
(
f"Pipeline parallelism is configured "
f"({cfg.training.pipeline_stages} stages) but live "
f"execution wiring ships in v0.27.1. Your config is "
f"validated and the trainer will run in data-parallel "
f"mode for now."
),
title="[yellow]Pipeline parallelism (deferred execution)[/]",
border_style="yellow",
)
)
# Validate Liger Kernel configuration
if cfg.training.use_liger:
from soup_cli.utils.liger import validate_liger_config
liger_errors = validate_liger_config(
cfg.training.use_liger, cfg.backend, device,
)
for err in liger_errors:
console.print(f"[red]Liger error:[/] {for_terminal(err)}")
if liger_errors:
raise typer.Exit(1)
# Validate FlashAttention configuration
if cfg.training.use_flash_attn:
from soup_cli.utils.flash_attn import validate_flash_attn_config
fa_errors = validate_flash_attn_config(
cfg.training.use_flash_attn, cfg.backend, device,
)
for err in fa_errors:
console.print(f"[red]FlashAttention error:[/] {for_terminal(err)}")
if fa_errors:
raise typer.Exit(1)
# Validate Ring FlashAttention configuration
if cfg.training.use_ring_attention:
from soup_cli.utils.ring_attention import validate_ring_attention_config
ring_errors = validate_ring_attention_config(
cfg.training.use_ring_attention, device, cfg.data.max_length,
)
for err in ring_errors:
console.print(f"[red]Ring Attention error:[/] {for_terminal(err)}")
if ring_errors:
raise typer.Exit(1)
# Validate long-context configuration
if cfg.training.rope_scaling_type:
from soup_cli.utils.long_context import validate_long_context_config
ctx_errors = validate_long_context_config(
cfg.data.max_length,
cfg.training.rope_scaling_type,
cfg.training.gradient_checkpointing,
)
for err in ctx_errors:
console.print(f"[yellow]Long-context warning:[/] {err}")
# Suggest unsloth if available but not being used, and only to a config the
# loader would take with it (#1736).
if _unsloth_tip_applies(cfg):
console.print(
"[dim]Tip: unsloth is installed. Add [bold]backend: unsloth[/dim]"
"[dim] to soup.yaml for 2-5x faster training.[/]"
)
# #1613: cheap stripe roots validation ahead of confirmation, --dry-run,
# dataset loading, and run creation.
from soup_cli.utils.stripe_roots import STRIPE_DIRS_ENV
if getattr(getattr(cfg, "training", None), "stream_layers", False) and os.environ.get(
STRIPE_DIRS_ENV
):
from soup_cli.utils.errors import format_friendly_error
from soup_cli.utils.layer_shard import resolve_cache_root
from soup_cli.utils.stripe_roots import StripeRootError, validate_early_stripe_roots
try:
validate_early_stripe_roots(resolve_cache_root())
except StripeRootError as exc:
format_friendly_error(exc)
raise typer.Exit(1) from exc
if not dry_run and not yes:
# #1913: on a non-TTY stdin the confirm can never be answered — a pipe
# hangs until its writer closes (EOF aborts via typer) and an
# unattended service waits forever. Fail with the escape hatch named.
from soup_cli.utils.exit_codes import EXIT_USAGE_ERROR
if not _stdin_isatty():
console.print(
"[red]stdin is not a TTY, so the start-training confirmation "
"cannot be answered.[/] Pass --yes to start without the "
"confirmation."
)
raise typer.Exit(EXIT_USAGE_ERROR)
if not typer.confirm("Start training?", default=True):
console.print("[yellow]Cancelled.[/]")
raise typer.Exit()
# #1223: a split nothing evaluates is not withheld -- generation tasks train
# on every row unless training.eval_steps asks for evaluation.
from soup_cli.utils.eval_schedule import loader_data_config, validation_notice
run_data_config = loader_data_config(cfg)
val_notice = validation_notice(cfg)
if dry_run:
console.print("[yellow]Dry run - validating data...[/]")
if val_notice:
console.print(f"[yellow]Note:[/] {val_notice}")
dataset = load_dataset(
data_config_for_task(run_data_config, cfg.task),
preserve_source_columns=task_preserves_source_columns(cfg.task),
)
_validate_preference_dataset_if_applicable(cfg, dataset)
_refuse_empty_train(cfg.data, dataset)
_validate_classification_dataset_if_applicable(cfg, dataset)
_validate_csm_dataset_if_applicable(cfg, dataset)
_refuse_over_judge_budget(cfg, dataset)
console.print(
f"[green]Data OK:[/] {_train_sample_count(cfg.data, dataset)} train samples"
)
if "val" in dataset:
console.print(f"[green]Val:[/] {len(dataset['val'])} samples")
console.print("[green]Config valid. Ready to train![/]")
raise typer.Exit()
# Capture the source before loading/transformation, never at verdict/registration time.
from soup_cli.utils.training_identity import capture_training_data_hash
training_data_sha256 = capture_training_data_hash(cfg)
# Load data
console.print("[dim]Loading dataset...[/]")
if val_notice:
console.print(f"[yellow]Note:[/] {val_notice}")
dataset = load_dataset(
data_config_for_task(run_data_config, cfg.task),
preserve_source_columns=task_preserves_source_columns(cfg.task),
)
_validate_preference_dataset_if_applicable(cfg, dataset)
_refuse_empty_train(cfg.data, dataset)
_validate_classification_dataset_if_applicable(cfg, dataset)
_validate_csm_dataset_if_applicable(cfg, dataset)
_refuse_over_judge_budget(cfg, dataset)
console.print(
f"[green]Loaded:[/] {_train_sample_count(cfg.data, dataset)} train samples"
)
# Capture the --tracker CLI value BEFORE the local ExperimentTracker
# shadows it (v0.43.0 review fix — name-collision regression).
tracker_backend = tracker
# Start experiment tracking
from soup_cli.experiment.tracker import ExperimentTracker
tracker = ExperimentTracker()
run_base = cfg.base
experiment_name = cfg.experiment_name or name
run_id = tracker.start_run(
config_dict=cfg.model_dump(),
device=device,
device_name=device_name,
gpu_info=gpu_info,
experiment_name=experiment_name,
run_id=os.environ.get("SOUP_MCP_RUN_ID") or None,
)
# #764: without a pid, a SIGKILL'd or Ctrl+C'd run has no way for
# _reconcile_orphaned_run to ever notice the process is gone — the
# rescue path #401 built for MCP-spawned runs was otherwise inert here.
tracker.mark_running(run_id, pid=os.getpid())
tracker.record_training_data_hash(run_id, training_data_sha256)
console.print(f"[dim]Run ID: {run_id}[/]")
try:
# Build trainer based on task type
from soup_cli.utils.trackers import resolve_report_to
try:
report_to = resolve_report_to(
wandb=wandb, tensorboard=tensorboard, tracker=tracker_backend
)
except ValueError as exc:
from rich.markup import escape as _esc
console.print(f"[red]{_esc(str(exc))}[/]")
raise typer.Exit(code=2) from exc
console.print("[dim]Setting up model + trainer...[/]")
# v0.53.8 #130 — pre-fetch model from non-HF hub into a local cache and
# rewrite cfg.base to point at the local snapshot. The trainer wrappers
# still use transformers.from_pretrained, which reads HF Hub by default;
# by snapshotting first we keep every wrapper unchanged.
hub_name = getattr(cfg.training, "hub", "hf") or "hf"
if hub_name != "hf":
import re as _re
from rich.markup import escape as _markup_escape
from soup_cli.utils.hubs import download_repo
from soup_cli.utils.paths import is_under_cwd
# Sanitise cache subdir name — strip every path-separator and
# `..` segment so a crafted ``base: ../../etc`` cannot escape the
# cache root (Windows ``\\`` and POSIX ``/`` both blocked).
safe_slug = _re.sub(r"[^A-Za-z0-9._-]+", "__", cfg.base).strip("._-") or "model"
cache_dir = (Path.cwd() / ".soup_hub_cache" / safe_slug).resolve()
if not is_under_cwd(str(cache_dir)):
console.print(
"[red]Resolved hub cache dir escaped the current working "
"directory; refusing to download.[/]"
)
raise typer.Exit(code=1)
try:
# Idempotency: if the cache dir already has a config.json, skip
# the re-download (modelscope/openmind-hub also short-circuit on
# match but having an explicit probe lets us print a clear hint).
existing_cfg = cache_dir / "config.json"
if existing_cfg.is_file():
local_path = str(cache_dir)
console.print(
f"[dim]Using cached snapshot at {local_path}[/]"
)
else:
local_path = download_repo(
hub_name,
cfg.base,
local_dir=str(cache_dir),
)
console.print(
f"[dim]Fetched {cfg.base} from hub={hub_name} → "
f"{local_path}[/]"
)
# model_copy replaces base without validation and leaves the
# original config untouched. local_path comes from download_repo
# under the cache directory checked above, so no revalidation is needed.
cfg = cfg.model_copy(update={"base": local_path})
except ImportError as exc:
console.print(f"[red]{_markup_escape(str(exc))}[/]")
raise typer.Exit(code=1) from exc
# v0.53.8 #89 — friendly missing-dep advisory for `--tracker <name>`.
if report_to and report_to not in ("none", "wandb", "tensorboard"):
from soup_cli.utils.trackers import tracker_missing_dep_message
msg = tracker_missing_dep_message(report_to)
if msg:
console.print(f"[yellow]{msg}[/]")
trainer_kwargs = {
"device": device,
"report_to": report_to,
"deepspeed_config": ds_config_path,
"fsdp_config": fsdp_kwargs,
}
# v0.40.4 #63 — every transformer-backend trainer now threads
# --trust-remote-code through the wrapper (closes the v0.36.0 Part B gap).
trainer_kwargs = dict(trainer_kwargs, trust_remote_code=trust_remote_code)
# #1213 — one dispatch, shared with `soup sweep`: the two commands used
# to keep separate chains and sweep's copy fell ten tasks and the MLX
# route behind. See soup_cli/trainer/dispatch.py.
from soup_cli.trainer.dispatch import build_trainer
from soup_cli.utils.run_provenance import (
BaseRevision,
base_revision_scope,
confirm_distributed_base_revision,
synchronize_base_revision,
)
base_identity = BaseRevision(
run_base,
binding_supported=(cfg.backend == "transformers" and hub_name == "hf"),
)
synchronize_base_revision(base_identity, cpu=device == "cpu")
with base_revision_scope(base_identity):
trainer_wrapper = build_trainer(cfg, **trainer_kwargs)
trainer_wrapper.setup(dataset)
confirm_distributed_base_revision(
base_identity, model=getattr(trainer_wrapper, "model", None),
)
tracker.record_base_revision(
run_id, revision=base_identity.revision, source=base_identity.source,
)
if base_identity.source == "hub_commit":
console.print(f"[dim]Base revision: {base_identity.revision} (bound Hub commit)[/]")
elif base_identity.source == "cache_path":
console.print("[dim]Base revision: cache-path metadata; local bytes are unverified[/]")
else:
console.print("[dim]Base revision: unknown (existing loading behavior preserved)[/]")
# #350 — PEFT promotes newly-created adapters to fp32. FSDP cannot flatten
# those beside bf16/float16 BNB storage, so align every trainable floating
# tensor after setup creates PEFT modules and before train() wraps the model.
aligned_fsdp_params = 0
if fsdp and cfg.training.quantization == "4bit":
from soup_cli.utils.gpu import get_compute_dtype
from soup_cli.utils.mixed_precision import align_trainable_dtype_for_fsdp_qlora
aligned_fsdp_params = align_trainable_dtype_for_fsdp_qlora(
getattr(trainer_wrapper, "model", None),
fsdp=True,
quantization=cfg.training.quantization,
compute_dtype=get_compute_dtype(),
)
if aligned_fsdp_params:
console.print(
f"[green]FSDP QLoRA:[/] aligned {aligned_fsdp_params} trainable "
"parameter tensor(s) to the compute dtype"
)
# --- HF auto-push callback (Part B of v0.29.0) ---
if push_as:
from soup_cli.monitoring.hf_push import build_push_callback
push_cb = build_push_callback(
repo_id=push_as,
output_dir=cfg.output,
private=False,
)
if push_cb is None:
console.print(
"[yellow]--push-as: no HF token available; skipping auto-push[/]"
)
else:
hf_trainer = getattr(trainer_wrapper, "trainer", None)
if hf_trainer is not None and hasattr(hf_trainer, "add_callback"):
hf_trainer.add_callback(push_cb)
console.print(
f"[green]HF auto-push enabled[/] -> {push_as} "
"(one branch per save_steps)"
)
else:
console.print(
"[yellow]--push-as: trainer does not expose add_callback; "
"auto-push disabled for this run[/]"
)
# Train with live display and experiment tracking
display = TrainingDisplay(cfg, device_name=device_name)
console.print("[bold green]Training started![/]\n")
profiler_ctx = contextlib.nullcontext()
if profile_run:
from soup_cli.utils.profiling import profile_training
profiler_ctx = profile_training(output_dir=Path(cfg.output), run_id=run_id)
console.print(
"[cyan]--profile:[/] writing torch.profiler trace to "
f"{cfg.output}/profiles/{run_id}.trace.json (early-steps window)"
)
# v0.71.3 #180 — optional codecarbon energy/CO2 measurement around the
# training window. Lazy-built; a graceful no-op when codecarbon is absent.
energy_ctx = contextlib.nullcontext()
energy_tracker = None
if track_energy:
try:
from soup_cli.utils.energy import EnergyTracker
energy_tracker = EnergyTracker(country_iso_code=energy_country)
energy_ctx = energy_tracker
except ValueError as exc:
console.print(f"[yellow]--track-energy disabled:[/] {exc}")
energy_tracker = None
energy_ctx = contextlib.nullcontext()
except Exception as exc:
tracker.fail_run(run_id, error=_describe_exception_for_tracker(exc))
raise
try:
with profiler_ctx as profiler, energy_ctx, base_revision_scope(base_identity):
if profile_run and profiler is not None:
from soup_cli.utils.profiling import attach_profile_callback
if not attach_profile_callback(trainer_wrapper, profiler):
console.print(
"[yellow]--profile:[/] this trainer has no step hook, "
"so no trace will be written"
)
result = trainer_wrapper.train(
display=display, tracker=tracker, run_id=run_id,
resume_from_checkpoint=resume_from,
)
from soup_cli.utils.run_provenance import write_run_provenance
confirm_distributed_base_revision(
base_identity, model=getattr(trainer_wrapper, "model", None),
)
tracker.record_base_revision(
run_id, revision=base_identity.revision, source=base_identity.source,
)
# Only the saving rank writes portable metadata beside the final artifact.
if _should_run_diagnose_gate_on_rank():
metadata = base_identity.metadata(run_id)
recorded_run = tracker.get_run(run_id)
if isinstance(recorded_run, dict):
metadata["training_data_sha256"] = recorded_run.get("training_data_sha256")
write_run_provenance(result["output_dir"], metadata)
# Save completion to tracker
tracker.finish_run(
run_id=run_id,
initial_loss=result["initial_loss"],
final_loss=result["final_loss"],
total_steps=result["total_steps"],
duration_secs=result["duration_secs"],
output_dir=result["output_dir"],
)
_run_auto_eval_after_training(
cfg.eval,
result["output_dir"],
run_id,
trust_remote_code=trust_remote_code,
)
except Exception as exc:
tracker.fail_run(run_id, error=_describe_exception_for_tracker(exc))
# v0.34.0 Part D — write a .crash bundle next to the run for triage.
try:
from soup_cli.utils.crash import build_crash_bundle, write_crash_bundle
metrics = tracker.get_metrics(run_id)
bundle = build_crash_bundle(
error=exc,
config=cfg.model_dump() if hasattr(cfg, "model_dump") else None,
metrics=metrics,
run_id=run_id,
output_dir=getattr(cfg, "output", None),
)
crash_path = write_crash_bundle(bundle)
console.print(
f"[yellow]Crash bundle written:[/] {crash_path}\n"
"[dim]Attach this file when reporting the failure.[/]"
)
except Exception as crash_err:
# Never let the crash reporter mask the original error, but tell
# the user the bundle is missing so they don't hunt for it.
console.print(
f"[dim]Could not write crash bundle: {crash_err}[/]"
)
raise
# Report
merge_hint = (
"ReLoRA output is already dense; no soup merge is needed"
if cfg.training.relora_steps is not None
else f"[bold]Merge LoRA:[/] soup merge --adapter {result['output_dir']}"
)
console.print(
Panel(
f"{_format_training_complete_loss(result)}\n"
f"Duration: [bold]{_format_duration_display(result)}[/]\n"
f"Output: [bold]{result['output_dir']}[/]\n"
f"Run ID: [bold]{run_id}[/]\n\n"
f"Quick test: [bold]soup chat --model {result['output_dir']}[/]\n"
f"Push to HF: [bold]soup push --model {result['output_dir']}[/]\n"
f"{merge_hint}\n"
f"Export GGUF: [bold]soup export --model {result['output_dir']}[/]\n"
f"Run details: [bold]soup runs show {run_id}[/]",
title=("[bold yellow]Training Stopped — Loss watchdog[/]"
if result.get("stop_reason") == "loss_watchdog"
else "[bold green]Training Complete![/]"),
)
)
# --- v0.56.0 --diagnose-gate: post-training failure-mode check ---
if diagnose_gate and _should_run_diagnose_gate_on_rank():
_run_diagnose_gate_or_exit(
diagnose_gate, run_id, cfg.base, result["output_dir"]
)
# --- v0.71.3 #180 --track-energy: print the measured energy/CO2 -------
energy_measurement = (
energy_tracker.measurement if energy_tracker is not None else None
)
if track_energy:
if energy_measurement is not None:
console.print(
f"[cyan]--track-energy:[/] {energy_measurement.energy_kwh:.4f} kWh"
f" / {energy_measurement.co2_kg:.4f} kg CO2"
f" (grid {energy_measurement.grid_intensity_g_per_kwh:.0f} g/kWh,"
f" PUE {energy_measurement.pue})"
)
else:
console.print(
"[yellow]--track-energy:[/] no reading "
"(install `pip install soup-cli\\[carbon]`)"
)
# --- v0.71.15 #244 --energy-out: persist for `soup bom emit --energy` -
if energy_out and _should_run_diagnose_gate_on_rank():
if energy_measurement is not None:
try:
_write_energy_json(energy_out, energy_measurement)
console.print(
"[cyan]--energy-out:[/] wrote "
f"{os.path.basename(energy_out)} "
"(feed to `soup bom emit --energy`)"
)
except (OSError, ValueError) as exc:
console.print(
f"[yellow]--energy-out skipped:[/] {type(exc).__name__}: {exc}"
)
else:
console.print(
"[yellow]--energy-out skipped:[/] no energy reading "
"(set --track-energy + install `pip install soup-cli\\[carbon]`)"
)
# --- v0.59.0 --annex-xi: Annex XI/XII auto-doc -----------------------
if annex_xi and _should_run_diagnose_gate_on_rank():
try:
_write_annex_xi(annex_xi, run_id, cfg, energy=energy_measurement)
except typer.Exit:
raise
except (OSError, ValueError) as exc:
console.print(
f"[yellow]--annex-xi skipped:[/] {type(exc).__name__}: {exc}"
)
# --- v0.59.0 --repro-receipt: SR 11-7 receipt ------------------------
if repro_receipt and _should_run_diagnose_gate_on_rank():
try:
_write_repro_receipt(repro_receipt, run_id, cfg)
except typer.Exit:
raise
except (OSError, ValueError) as exc:
console.print(
f"[yellow]--repro-receipt skipped:[/] {type(exc).__name__}: {exc}"
)
# --- v0.71.8 #219 --capture-activations: SAE-diff-ready snapshot ------
if capture_activations and _should_run_diagnose_gate_on_rank():
try:
written = _capture_activations(
capture_activations,
capture_prompts,
cfg.base,
result["output_dir"],
trust_remote_code=trust_remote_code,
)
console.print(f"[green]--capture-activations[/] -> {written}")
except (OSError, ValueError, RuntimeError, ImportError) as exc:
console.print(
f"[yellow]--capture-activations skipped:[/] "
f"{type(exc).__name__}: {exc}"
)
def _write_annex_xi(out_path: str, run_id: str, cfg, *, energy=None) -> None:
"""Render an Annex XI doc using values from the resolved soup.yaml.
v0.71.3 #180: the optional ``energy`` measurement populates the kWh / CO2
fields. v0.71.3 #184: the top crawled domains are auto-extracted from the
training JSONL. v0.71.3 #181: a ``.pdf`` output path renders a PDF.
"""
from datetime import datetime, timezone
from soup_cli import __version__
from soup_cli.utils.annex_xi import (
AnnexXIData,
load_top_domains_from_jsonl,
write_annex_doc,
)
modality = getattr(cfg, "modality", "text") or "text"
# #1446: without --track-energy nothing was measured; say so instead of 0.000.
energy_kwh = float(getattr(energy, "energy_kwh", 0.0)) if energy is not None else None
co2_kg = float(getattr(energy, "co2_kg", 0.0)) if energy is not None else None
raw_train = getattr(cfg.data, "train", "") or ""
# #443 — pass the raw str|list through so top-domain extraction
# aggregates across every interleaved dataset, instead of stringifying
# a list into a nonexistent path (pre-fix: `str(getattr(...) or "")`
# applied `or ""` to the getattr result BEFORE str(), so a non-empty
# list became its own Python repr string, e.g. "['a.jsonl', 'b.jsonl']"
# — neither a valid path nor useful doc text).
train_display = ", ".join(raw_train) if isinstance(raw_train, list) else str(raw_train)
# #184 — best-effort extract the top crawled domains from the training data.
top_domains = load_top_domains_from_jsonl(raw_train)
fmt = "pdf" if out_path.lower().endswith(".pdf") else "markdown"
data = AnnexXIData(
model_name=str(getattr(cfg, "output", run_id) or run_id),
base_model=str(cfg.base),
task=str(cfg.task),
dataset_summary=train_display,
modalities=(modality,),
train_compute_flops=None, # #1446: Soup does not measure FLOPs; never claim 0
train_energy_kwh=energy_kwh,
train_co2_kg=co2_kg,
top_domains=top_domains,
soup_version=__version__,
run_id=run_id,
created_at=datetime.now(tz=timezone.utc).isoformat(),
)
written = write_annex_doc(data, "xi", out_path, fmt=fmt)
console.print(f"[green]--annex-xi[/] -> {written}")
def _write_repro_receipt(out_path: str, run_id: str, cfg) -> None:
"""Render an SR 11-7 receipt from the resolved soup.yaml."""
from soup_cli.utils.repro_receipt import build_repro_receipt, write_repro_receipt
seeds: dict[str, int] = {}
seed = getattr(cfg.training, "seed", None)
if isinstance(seed, int) and not isinstance(seed, bool):
seeds["torch"] = seed
seeds["numpy"] = seed
seeds["python"] = seed
receipt = build_repro_receipt(seeds=seeds, run_id=run_id)
written = write_repro_receipt(receipt, out_path)
console.print(f"[green]--repro-receipt[/] -> {written}")
def _load_capture_prompts(canonical_path: str) -> list[str]:
"""Read prompts for --capture-activations (JSONL objects, JSON strings, or raw lines)."""
import json
flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
fd = os.open(canonical_path, flags)
# File-size cap on the SAME fd (no TOCTOU) bounds a pathological
# newline-free multi-GB line before the `for line in handle` read.
if os.fstat(fd).st_size > 64 * 1024 * 1024:
os.close(fd)
raise ValueError("--capture-prompts file exceeds 64 MiB")
prompts: list[str] = []
with os.fdopen(fd, encoding="utf-8") as handle:
for line in handle:
line = line.strip()
if not line:
continue
try:
obj = json.loads(line)
except json.JSONDecodeError:
prompts.append(line) # raw-text line
else:
if isinstance(obj, str) and obj:
prompts.append(obj)
elif isinstance(obj, dict):
for key in ("prompt", "text", "instruction", "input", "question"):
val = obj.get(key)
if isinstance(val, str) and val:
prompts.append(val)
break
if len(prompts) >= 256:
break
return prompts
def _capture_activations(
layer: str,
prompts_path: str,
base: str,
output_dir: str,
*,
trust_remote_code: bool = False,
) -> str:
"""Capture residual-stream activations from the trained model (v0.71.8 #219).
Loads the trained model at ``output_dir`` (or ``base`` + adapter when the
output is a LoRA adapter), runs ``prompts_path`` through a forward hook on
``layer``, and writes the per-token activations to
``<output_dir>/activations/activations.json`` in the
``{"activations": [[...]], "layer", "num_tokens", "hidden_dim"}`` shape that
``soup probe sae-diff`` / ``sleeper`` consume directly.
"""
import json
from soup_cli.utils import live_eval
from soup_cli.utils.paths import enforce_under_cwd_and_no_symlink
if not prompts_path:
raise ValueError("--capture-activations requires --capture-prompts <jsonl>")
if not isinstance(layer, str) or not layer.strip():
raise ValueError("--capture-activations layer must be a non-empty string")
canonical = enforce_under_cwd_and_no_symlink(prompts_path, "--capture-prompts")
prompts = _load_capture_prompts(canonical)
if not prompts:
raise ValueError("--capture-prompts has no usable prompts")
# When the output is a LoRA adapter dir, load base + adapter; else load the
# full fine-tuned model from output_dir directly.
is_adapter = os.path.isfile(os.path.join(output_dir, "adapter_config.json"))
has_full = os.path.isfile(os.path.join(output_dir, "config.json"))
if not is_adapter and not has_full:
raise ValueError(
f"output dir {os.path.basename(output_dir)} has neither "
"adapter_config.json nor config.json — cannot capture activations"
)
adapter = output_dir if is_adapter else None
model_id = base if adapter else output_dir
model, tokenizer, dev = live_eval.load_model_and_tokenizer(
model_id, adapter=adapter, trust_remote_code=trust_remote_code
)
acts = live_eval.extract_layer_activations(
model, tokenizer, prompts, layer=layer, device=dev, pool="none"
)
acts_dir = os.path.join(output_dir, "activations")
# Refuse a pre-placed `activations` symlink so the write cannot be
# redirected outside output_dir (defence-in-depth — output_dir is config-
# derived/trusted but the symlink check is cheap).
if os.path.islink(acts_dir):
raise ValueError("activations subdir is a symlink — refusing to write")
os.makedirs(acts_dir, exist_ok=True)
out_path = os.path.join(acts_dir, "activations.json")
payload = {
"layer": layer,
"num_tokens": int(acts.shape[0]),
"hidden_dim": int(acts.shape[1]),
"activations": acts.tolist(),
}
with open(out_path, "w", encoding="utf-8") as handle:
json.dump(payload, handle)
return out_path
def _write_energy_json(path: str, measurement: "EnergyMeasurement") -> None:
"""Persist an ``EnergyMeasurement`` as JSON for ``soup bom emit --energy``.
Writes exactly the five fields ``EnergyMeasurement(**parsed)`` expects so
the producer (``soup train --track-energy --energy-out``) and the consumer
(``soup bom emit --energy`` — v0.71.3 #256) round-trip cleanly. Atomic +
cwd-contained + symlink-rejected via the shared helper (v0.71.15 #244).
"""
import json
from dataclasses import asdict
from soup_cli.utils.paths import atomic_write_text
payload = json.dumps(asdict(measurement), indent=2)
atomic_write_text(payload, path, field="--energy-out")
def _should_run_diagnose_gate_on_rank() -> bool:
"""Return True only for the single chief worker in a distributed launch.
The diagnose gate (and the --annex-xi / --repro-receipt / capture hooks)
write one report and, on a shared filesystem, read one shared output dir
-- so they should fire exactly once per *cluster*, not once per node.
Resolution (v0.71.15 #170 — fixes the v0.56.0 limitation where a shared-FS
multi-node run fired the gate once per node):
* If RANK is set (multi-node / torchrun launch) -> gate only on the
global chief, RANK == 0. On a single-node torchrun this is equivalent
to LOCAL_RANK == 0 (both 0 only for the one chief process).
* Else fall back to LOCAL_RANK == 0 (plain single-node accelerate launch
that doesn't export RANK) so a one-box multi-GPU run still gates once.
Defaults to True (run gate) on any parse error: a malformed env var is
safer to over-run than to silently skip.
"""
rank = os.environ.get("RANK")
if rank: # non-empty -> multi-node / torchrun sets a global RANK
try:
return int(rank) == 0
except ValueError:
return True
try:
return int(os.environ.get("LOCAL_RANK", "0")) == 0
except ValueError:
return True
def _run_diagnose_gate(
evidence_path: str, run_id: str, base: str, adapter: str
) -> None:
"""Post-training failure-mode gate (v0.56.0).
Loads a JSON ``evidence`` file with optional per-mode scores and
refuses to mark the run successful if any mode comes back MAJOR (exit 2)
or NOT_RUN (exit 3, #1435). Missing modes fall back to a neutral OK score so partial evidence
still produces a useful report card. The train command only calls
this helper on the chief worker (RANK==0 in a multi-node launch, else
LOCAL_RANK==0) so distributed runs execute the gate once per cluster,
not once per worker (v0.71.15 #170).
"""
import json
from soup_cli.utils.diagnose.report import FAILURE_MODES, FailureScore
from soup_cli.utils.diagnose.runner import build_report
from soup_cli.utils.paths import enforce_under_cwd_and_no_symlink
from soup_cli.utils.terminal import for_terminal
enforce_under_cwd_and_no_symlink(evidence_path, "--diagnose-gate evidence")
# 16 MiB cap on evidence JSON (security review HIGH — symmetric with
# `commands/diagnose._MAX_EVIDENCE_BYTES`; prevents `/dev/zero` /
# multi-GB symlink-pointed OOM at json.load time).
if os.path.getsize(evidence_path) > 16 * 1024 * 1024:
raise ValueError(
"--diagnose-gate evidence exceeds 16 MiB"
)
with open(evidence_path, encoding="utf-8") as handle:
payload = json.load(handle)
if not isinstance(payload, dict):
raise ValueError("evidence file must contain a JSON object")
raw_scores = payload.get("scores") or {}
if not isinstance(raw_scores, dict):
raise ValueError("evidence.scores must be an object")
from soup_cli.utils.diagnose.report import classify_score, evidence_default_score
from soup_cli.utils.exit_codes import EXIT_USAGE_ERROR
scores: dict = {}
for mode in FAILURE_MODES:
entry = raw_scores.get(mode)
if entry is None:
continue
if not isinstance(entry, dict):
raise ValueError(f"scores.{mode} must be an object")
score = entry.get("score", evidence_default_score(entry))
if isinstance(score, bool) or not isinstance(score, (int, float)):
raise ValueError(f"scores.{mode}.score must be a number")
verdict = entry.get("verdict") or classify_score(score)
scores[mode] = FailureScore(
mode=mode,
score=float(score),
verdict=verdict,
evidence=str(entry.get("evidence", "supplied by --diagnose-gate")),
)
report = build_report(
run_id=run_id, base=base, adapter=adapter, scores=scores
)
if report.overall == "MAJOR":
console.print(
"[red]--diagnose-gate: MAJOR regression in one or more modes.[/]"
)
for mode in FAILURE_MODES:
sc = report.scores[mode]
if sc.verdict == "MAJOR":
console.print(f" [red]MAJOR[/] {mode}: {for_terminal(sc.evidence)}")
raise typer.Exit(2)
if report.overall == "NOT_RUN":
# An unmeasured mode is not a pass (#1435); same exit as `soup diagnose`.
console.print(
"[yellow]--diagnose-gate: one or more modes did not run (NOT_RUN); "
"the run is not verified.[/]"
)
for mode in FAILURE_MODES:
sc = report.scores[mode]
if sc.verdict == "NOT_RUN":
console.print(f" [yellow]NOT_RUN[/] {mode}: {for_terminal(sc.evidence)}")
raise typer.Exit(EXIT_USAGE_ERROR)
console.print(
f"[green]--diagnose-gate: {report.overall}[/] across "
f"{len(FAILURE_MODES)} modes."
)
def _run_diagnose_gate_or_exit(
evidence_path: str, run_id: str, base: str, adapter: str
) -> None:
"""Run the gate; an unreadable or refused evidence file is reported and exits 1."""
from soup_cli.utils.terminal import for_terminal
try:
_run_diagnose_gate(evidence_path, run_id, base, adapter)
except typer.Exit:
raise
except (OSError, ValueError) as exc:
console.print(
f"[red]--diagnose-gate failed:[/] {type(exc).__name__}: {for_terminal(exc)}"
)
raise typer.Exit(1) from exc
def _resolve_deepspeed(deepspeed: str) -> str:
"""Resolve DeepSpeed config: named preset or path to JSON file."""
import soup_cli.utils.deepspeed as ds
# Named preset
if deepspeed in ds.CONFIGS:
return ds.write_deepspeed_config(deepspeed)
# Path to config file. Resolved the same way a preset is (#359): a config
# that needs no run-dependent rewrite comes back by this very path, and one
# that copied the ZeRO++ placeholders is repaired into a temp copy with the
# change printed. The user's file is never modified.
ds_path = Path(deepspeed)
if ds_path.exists() and ds_path.suffix == ".json":
return ds.resolve_user_deepspeed_file(str(ds_path))
console.print(
f"[red]Invalid DeepSpeed config: {deepspeed}[/]\n"
f"Options: {', '.join(ds.CONFIGS.keys())} or path to JSON file."
)
raise typer.Exit(1)
_MLX_CHECKPOINT_RE = re.compile(r"^(\d+)_adapters\.safetensors$")
def _highest_numbered_mlx_checkpoint(paths) -> Path | None:
"""Pick the highest step-numbered ``NNNNNNN_adapters.safetensors`` file.
``max()`` over the parsed step number: not a sort over whatever order
the filesystem's ``iterdir()`` happened to enumerate (that order isn't
guaranteed), and not a comparison of the filenames as strings (that
would only agree with numeric order if every step number in a run
happened to be zero-padded to the same width, which nothing here
enforces). Keying on the parsed int makes both mistakes fail instead
of coincidentally still returning the right file.
"""
numbered: list[tuple[int, Path]] = []
for path in paths:
if not path.is_file():
continue
match = _MLX_CHECKPOINT_RE.match(path.name)
if match:
numbered.append((int(match.group(1)), path))
if not numbered:
return None
return max(numbered, key=lambda pair: pair[0])[1]
def _resolve_mlx_checkpoint(resume: str, output_dir: str) -> str | None:
"""MLX-style checkpoint resolution (#634).
mlx-lm's tuner saves step-numbered adapter snapshots
(``NNNNNNN_adapters.safetensors``) plus a final ``adapters.safetensors``
with no step prefix — files, not the ``checkpoint-N`` directories the
transformers/unsloth backends write. "auto" picks the highest-numbered
snapshot if one exists, else the final file. A direct ``resume`` value
must point at one of these files directly.
Takes no ``experiment_name``, unlike the transformers/unsloth resolver
below: ``mlx_sft.py``'s ``output_dir`` is always ``Path(cfg.output)``,
flat, never nested under it the way ``trainer/sft.py`` nests under
``output_dir / cfg.experiment_name``. Nesting here would look inside a
directory MLX never writes to, and "auto" would report no checkpoint
found on every run that sets ``experiment_name`` — the exact symptom
#634 reported, reintroduced for a config the original fix didn't cover.
"""
if resume.lower() == "auto":
base = Path(output_dir)
if not base.is_dir():
return None
highest = _highest_numbered_mlx_checkpoint(base.iterdir())
if highest is not None:
return str(highest)
final = base / "adapters.safetensors"
if final.is_file():
return str(final)
return None
# Direct path to a specific adapter file
checkpoint_path = Path(resume)
if checkpoint_path.exists() and checkpoint_path.is_file():
return str(checkpoint_path)
return None
def _resolve_checkpoint(
resume: str,
output_dir: str,
experiment_name: str | None = None,
*,
backend: str = "transformers",
) -> str | None:
"""Resolve the checkpoint path from --resume argument.
If resume == "auto", find the latest checkpoint in the output directory.
Otherwise, treat it as a direct path to a checkpoint directory.
``backend="mlx"`` dispatches to :func:`_resolve_mlx_checkpoint`, since MLX
writes step-numbered adapter files rather than ``checkpoint-N``
directories (#634) — the shape below never matches an MLX run's output.
"""
if backend == "mlx":
return _resolve_mlx_checkpoint(resume, output_dir)
if resume.lower() == "auto":
base = Path(output_dir)
if experiment_name:
base = base / experiment_name
if not base.exists():
return None
checkpoints = sorted(
[d for d in base.iterdir() if d.is_dir() and d.name.startswith("checkpoint-")],
key=lambda d: int(d.name.split("-")[-1]) if d.name.split("-")[-1].isdigit() else 0,
)
if checkpoints:
return str(checkpoints[-1])
return None
# Direct path
checkpoint_path = Path(resume)
if checkpoint_path.exists() and checkpoint_path.is_dir():
return str(checkpoint_path)
return None
def _resolve_resume_or_exit(resume: str, cfg: "SoupConfig") -> str | None:
"""Resolve ``--resume`` against ``cfg``, printing status and exiting on
failure. Extracted out of ``train()`` (#634 review) so the
``backend=cfg.backend`` wiring — the entire seam the MLX checkpoint fix
depends on — is directly testable without invoking the rest of the
command, which needs real hardware/model loading this can be exercised
without. Returns ``None`` only when ``resume`` itself is falsy; a
``resume`` value that fails to resolve exits rather than returning.
"""
if not resume:
return None
if cfg.task in UNSUPPORTED_RESUME_TASKS:
console.print(f"[red]--resume is not supported for task {cfg.task!r}[/]")
raise typer.Exit(1)
resume_from = _resolve_checkpoint(resume, cfg.output, cfg.experiment_name, backend=cfg.backend)
if resume_from:
console.print(f"[green]Resuming from:[/] {resume_from}")
else:
console.print("[red]No checkpoint found to resume from.[/]")
raise typer.Exit(1)
return resume_from
def _run_live_lr_sweep(
config_path: str, schedule: list[float], trust_remote_code: bool = False,
) -> tuple[list[float], list[float]]:
"""v0.33.0 #56 — run the in-process LR sweep over the config's own model.
Returns ``(lrs, losses)`` of equal length.
#1203: every reason the sweep cannot run is a refusal.
:class:`LrSweepUnavailableError` covers a config that will not load and a
model or dataset that cannot be swept; :class:`SweepTooShortError` covers
fewer than ``MIN_NUM_STEPS`` rows, and a loss that is non-finite from the
first step or turns non-finite before 4 steps have run.
This used to answer all of them with a synthetic curve shaped by the LR
schedule alone, so a base model that does not exist exited 0 with a
``recommended_lr`` that described no model and no data.
"""
from soup_cli.utils.lr_finder import LrSweepUnavailableError, SweepTooShortError
try:
# The refusal below quotes the reason, so the loader hands it over
# instead of printing its own line and exiting.
cfg = load_config(config_path, raise_unreadable=True)
except Exception as exc: # noqa: BLE001 — one refusal line, not a traceback
raise LrSweepUnavailableError(
f"--find-lr: config load failed ({exc}); no LR report was written."
) from exc
try:
return _live_lr_sweep_from_config(
cfg, schedule, trust_remote_code=trust_remote_code,
)
except SweepTooShortError:
raise
except Exception as exc: # noqa: BLE001 — the cause is the useful half
raise LrSweepUnavailableError(
f"--find-lr: live sweep unavailable ({exc}); no LR report was written."
) from exc
def _lr_finder_dataset_path(train) -> str:
"""#443 — LR-finder samples one representative dataset; it already
bypasses load_dataset()/_finalize() for a lightweight sweep, so full
interleave fidelity is out of this issue's scope. Falls back to the
first dataset rather than crashing on a list. Extracted as its own
function so it can be exercised directly by
tests/test_issue443_interleave_wiring.py's enumerating test.
"""
return train[0] if isinstance(train, list) else train
def _live_lr_sweep_from_config(
cfg, schedule: list[float], trust_remote_code: bool = False,
) -> tuple[list[float], list[float]]:
"""Build a tiny in-process loop: load model + tokenizer + a slice of
the train dataset, then call :func:`run_lr_sweep`. Returns the
``(lrs, losses)`` the sweep actually ran."""
# v0.40.1 Part C / G12 — fix broken `load_local` import that previously
# made the live sweep fail every time. The actual exported symbol
# is ``load_raw_data`` (path-only loader) — we use that.
from pathlib import Path as _Path
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
from soup_cli.data.loader import load_raw_data
from soup_cli.utils.lr_finder import (
MIN_NUM_STEPS,
SweepTooShortError,
compute_lr_schedule,
run_lr_sweep,
)
# #1189: one row is one sweep step, so size the sweep before the model loads.
lr_finder_train_path = _lr_finder_dataset_path(cfg.data.train)
rows = list(load_raw_data(_Path(lr_finder_train_path)))[: len(schedule)]
if len(rows) < MIN_NUM_STEPS:
row_count = f"{len(rows)} row" + ("" if len(rows) == 1 else "s")
raise SweepTooShortError(
f"--find-lr: the training set has {row_count}, but the sweep needs at "
f"least {MIN_NUM_STEPS} (one row per step; --find-lr-steps {len(schedule)})"
)
if len(rows) < len(schedule):
# Rebuilt over the same range: keeping the first rows' worth of the original
# schedule would never reach the high LRs where training diverges.
console.print(
f"[yellow]--find-lr: the training set has {len(rows)} rows, fewer than "
f"--find-lr-steps {len(schedule)}; sweeping {len(rows)} steps over the "
f"same {schedule[0]:.3g} to {schedule[-1]:.3g} range.[/]"
)
schedule = compute_lr_schedule(schedule[0], schedule[-1], len(rows))
device = "cuda" if torch.cuda.is_available() else "cpu"
tokenizer = AutoTokenizer.from_pretrained(
cfg.base, trust_remote_code=trust_remote_code,
)
if tokenizer.pad_token is None:
tokenizer.pad_token = tokenizer.eos_token
model = AutoModelForCausalLM.from_pretrained(
cfg.base, trust_remote_code=trust_remote_code,
).to(device)
model.train()
def _tokenize(row):
text = row.get("text") or row.get("prompt") or ""
if not text and "messages" in row:
text = " ".join(m.get("content", "") for m in row["messages"])
enc = tokenizer(
text or " ", return_tensors="pt", truncation=True,
max_length=min(cfg.data.max_length or 256, 256),
padding="max_length",
)
enc["labels"] = enc["input_ids"].clone()
return {k: v.squeeze(0) for k, v in enc.items()}
def _batched_loader():
for row in rows:
tok = _tokenize(row)
yield {k: v.unsqueeze(0) for k, v in tok.items()}
losses = run_lr_sweep(
model=model,
dataloader=_batched_loader(),
schedule=schedule,
optimizer_factory=lambda params: torch.optim.AdamW(params, lr=schedule[0]),
device=device,
)
# With a row for every step, a short result means the loss went non-finite.
if len(losses) < len(schedule):
diverged_lr = schedule[len(losses)]
if not losses:
# No update has run yet, so the learning rate cannot be the cause.
raise SweepTooShortError(
f"--find-lr: the loss was non-finite on the first step (lr {diverged_lr:.3g}), "
"before any update; check the training data, dtype and model rather "
"than the LR range"
)
if len(losses) < MIN_NUM_STEPS:
steps_run = f"{len(losses)} step" + ("" if len(losses) == 1 else "s")
raise SweepTooShortError(
f"--find-lr: the loss became non-finite at lr {diverged_lr:.3g} after "
f"{steps_run}, too few for a recommendation (need "
f"{MIN_NUM_STEPS}); lower --find-lr-start / --find-lr-end"
)
console.print(
f"[yellow]--find-lr: the loss became non-finite at lr {diverged_lr:.3g}; "
f"the report covers the {len(losses)} steps before it.[/]"
)
return schedule[: len(losses)], losses